neil/silverbullet
1
0
Fork 0
Code Activity

Materialized query parser redo

Browse Source
This commit is contained in:
Zef Hemel
2022-04-11 20:34:09 +02:00
parent d649009dd2
commit b3c3302970
31 changed files with 1182 additions and 667 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugs/core/core.plug.yaml
+27 -7
View File
@@ -1,3 +1,23 @@
syntax:
HashTag:
firstCharacters:
- "#"
regex: "#[A-Za-z\\.]+"
styles:
color: blue
AtMention:
firstCharacters:
- "@"
regex: "@[A-Za-z\\.]+"
styles:
color: blue
URL:
firstCharacters:
- "h"
regex: "https?:\\/\\/[-a-zA-Z0-9@:%._\\+~#=]{1,256}([-a-zA-Z0-9()@:%_\\+.~#?&=\\/]*)"
styles:
color: "#0330cb"
textDecoration: underline
functions:
clearPageIndex:
path: "./page.ts:clearPageIndex"
@@ -55,13 +75,13 @@ functions:
path: "./dates.ts:indexDates"
events:
- page:index
updateMaterializedQueriesOnPage:
path: ./materialized_queries.ts:updateMaterializedQueriesOnPage
updateMaterializedQueriesCommand:
path: ./materialized_queries.ts:updateMaterializedQueriesCommand
parseServerCommand:
path: ./page.ts:parseServerPageCommand
command:
name: "Materialized Queries: Update"
parseCommand:
name: "Debug: Parse Document on Server"
parsePage:
path: ./page.ts:parsePage
parseCommand:
path: ./page.ts:parsePageCommand
command:
name: Parse Document
name: "Debug: Parse Document"
plugs/core/dates.ts
+7 -3
View File
@@ -1,7 +1,7 @@
import { insertAtCursor } from "plugos-silverbullet-syscall/editor";
import { IndexEvent } from "../../webapp/app_event";
import { batchSet } from "plugos-silverbullet-syscall";
import { whiteOutQueries } from "./materialized_queries";
import { whiteOutQueries } from "../query/materialized_queries";
const dateMatchRegex = /(\d{4}\-\d{2}\-\d{2})/g;
@@ -23,12 +23,16 @@ export async function indexDates({ name, text }: IndexEvent) {
await batchSet(name, dates);
}
export function niceDate(d: Date): string {
return new Date().toISOString().split("T")[0];
}
export async function insertToday() {
await insertAtCursor(new Date().toISOString().split("T")[0]);
await insertAtCursor(niceDate(new Date()));
}
export async function insertTomorrow() {
let d = new Date();
d.setDate(d.getDate() + 1);
await insertAtCursor(d.toISOString().split("T")[0]);
await insertAtCursor(niceDate(d));
}
plugs/core/item.ts
+5 -5
View File
@@ -1,9 +1,9 @@
import { IndexEvent } from "../../webapp/app_event";
import { whiteOutQueries } from "./materialized_queries";
import { whiteOutQueries } from "../query/materialized_queries";
import { batchSet } from "plugos-silverbullet-syscall/index";
import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
import { collectNodesMatching, MarkdownTree, renderMarkdown } from "../lib/tree";
import { collectNodesMatching, ParseTree, renderToText } from "../../common/tree";
type Item = {
item: string;
@@ -23,16 +23,16 @@ export async function indexItems({ name, text }: IndexEvent) {
if (!n.children) {
return;
}
let textNodes: MarkdownTree[] = [];
let textNodes: ParseTree[] = [];
let nested: string | undefined;
for (let child of n.children!.slice(1)) {
if (child.type === "OrderedList" || child.type === "BulletList") {
nested = renderMarkdown(child);
nested = renderToText(child);
break;
}
textNodes.push(child);
}
let item = textNodes.map(renderMarkdown).join("").trim();
let item = textNodes.map(renderToText).join("").trim();
let value: Item = {
item,
};
plugs/core/materialized_queries.ts
-133
View File
@@ -1,133 +0,0 @@
import { flashNotification, getCurrentPage, reloadPage, save } from "plugos-silverbullet-syscall/editor";
import { listPages, readPage, writePage } from "plugos-silverbullet-syscall/space";
import { invokeFunction } from "plugos-silverbullet-syscall/system";
import { scanPrefixGlobal } from "plugos-silverbullet-syscall";
export const queryRegex =
/(<!--\s*#query\s+(?<table>\w+)\s*(filter\s+["'“”‘’](?<filter>[^"'“”‘’]+)["'“”‘’])?\s*\s*(order by\s+(?<orderBy>\w+)(?<orderDesc>\s+desc)?)?(group by\s+(?<groupBy>\w+))?\s*(limit\s+(?<limit>\d+))?\s*-->)(.+?)(<!--\s*#end\s*-->)/gs;
export function whiteOutQueries(text: string): string {
return text.replaceAll(queryRegex, (match) =>
new Array(match.length + 1).join(" ")
);
}
async function replaceAsync(
str: string,
regex: RegExp,
asyncFn: (match: string, ...args: any[]) => Promise<string>
) {
const promises: Promise<string>[] = [];
str.replace(regex, (match: string, ...args: any[]): string => {
const promise = asyncFn(match, ...args);
promises.push(promise);
return "";
});
const data = await Promise.all(promises);
return str.replace(regex, () => data.shift()!);
}
export async function updateMaterializedQueriesCommand() {
const currentPage = await getCurrentPage();
await save();
await invokeFunction(
"server",
"updateMaterializedQueriesOnPage",
currentPage
);
await reloadPage();
await flashNotification("Updated materialized queries");
}
// Called from client, running on server
export async function updateMaterializedQueriesOnPage(pageName: string) {
let { text } = await readPage(pageName);
text = await replaceAsync(text, queryRegex, async (match, ...args) => {
let { table, filter, groupBy, limit, orderBy, orderDesc } =
args[args.length - 1];
const startQuery = args[0];
const endQuery = args[args.length - 4];
let results = [];
switch (table) {
case "page":
let pages = await listPages();
if (orderBy) {
pages = pages.sort((a: any, b: any) => {
if (a[orderBy] === b[orderBy]) {
return 0;
}
if (a[orderBy] < b[orderBy]) {
return !!orderDesc ? 1 : -1;
} else {
return !!orderDesc ? -1 : 1;
}
});
}
let matchCount = 0;
for (let pageMeta of pages) {
if (!filter || (filter && pageMeta.name.includes(filter))) {
matchCount++;
results.push(`* [[${pageMeta.name}]]`);
if (limit && matchCount === +limit) {
break;
}
}
}
return `${startQuery}\n${results.join("\n")}\n${endQuery}`;
case "task":
for (let {
key,
page,
value: { task, complete, nested },
} of await scanPrefixGlobal("task:")) {
let [, pos] = key.split(":");
if (!filter || (filter && task.includes(filter))) {
results.push(
`* [${complete ? "x" : " "}] [[${page}@${pos}]] ${task}` +
(nested ? "\n " + nested : "")
);
}
}
return `${startQuery}\n${results.sort().join("\n")}\n${endQuery}`;
case "link":
let uniqueLinks = new Set<string>();
console.log("Here!!");
for (let { key, page, value: name } of await scanPrefixGlobal(
`pl:${pageName}:`
)) {
console.log("Here!!");
let [, pos] = key.split(":");
if (!filter || (filter && name.includes(filter))) {
uniqueLinks.add(name);
}
}
console.log("Here!!");
for (const uniqueResult of uniqueLinks) {
results.push(`* [[${uniqueResult}]]`);
}
console.log("Here!!");
return `${startQuery}\n${results.sort().join("\n")}\n${endQuery}`;
case "item":
for (let {
key,
page,
value: { item, nested },
} of await scanPrefixGlobal("it:")) {
let [, pos] = key.split(":");
if (!filter || (filter && item.includes(filter))) {
results.push(
`* [[${page}@${pos}]] ${item}` + (nested ? "\n " + nested : "")
);
}
}
return `${startQuery}\n${results.sort().join("\n")}\n${endQuery}`;
default:
return match;
}
});
// console.log("New text", text);
await writePage(pageName, text);
}
plugs/core/navigate.ts
+7 -15
View File
@@ -1,13 +1,11 @@
import { ClickEvent } from "../../webapp/app_event";
import { updateMaterializedQueriesCommand } from "./materialized_queries";
import { getCursor, getText, navigate as navigateTo, openUrl } from "plugos-silverbullet-syscall/editor";
import { taskToggleAtPos } from "../tasks/task";
import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
import { MarkdownTree, nodeAtPos } from "../lib/tree";
import { nodeAtPos, ParseTree } from "../../common/tree";
const materializedQueryPrefix = /<!--\s*#query\s+/;
async function actionClickOrActionEnter(mdTree: MarkdownTree | null) {
async function actionClickOrActionEnter(mdTree: ParseTree | null) {
if (!mdTree) {
return;
}
@@ -24,17 +22,9 @@ async function actionClickOrActionEnter(mdTree: MarkdownTree | null) {
case "URL":
await openUrl(mdTree.children![0].text!);
break;
case "CommentBlock":
if (mdTree.children![0].text!.match(materializedQueryPrefix)) {
await updateMaterializedQueriesCommand();
}
break;
case "Link":
await openUrl(mdTree.children![4].children![0].text!);
break;
case "TaskMarker":
await taskToggleAtPos(mdTree.from! + 1);
break;
}
}
@@ -45,9 +35,11 @@ export async function linkNavigate() {
}
export async function clickNavigate(event: ClickEvent) {
// Navigate by default, don't navigate when Ctrl or Cmd is held
if (event.ctrlKey || event.metaKey) {
let mdTree = await parseMarkdown(await getText());
let newNode = nodeAtPos(mdTree, event.pos);
await actionClickOrActionEnter(newNode);
return;
}
let mdTree = await parseMarkdown(await getText());
let newNode = nodeAtPos(mdTree, event.pos);
await actionClickOrActionEnter(newNode);
}
plugs/core/page.ts
+15 -7
View File
@@ -21,10 +21,10 @@ import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
import {
addParentPointers,
collectNodesMatching,
MarkdownTree,
renderMarkdown,
ParseTree,
renderToText,
replaceNodesMatching
} from "../lib/tree";
} from "../../common/tree";
export async function indexLinks({ name, text }: IndexEvent) {
let backLinks: { key: string; value: string }[] = [];
@@ -93,7 +93,7 @@ export async function renamePage() {
}
let mdTree = await parseMarkdown(text);
addParentPointers(mdTree);
replaceNodesMatching(mdTree, (n): MarkdownTree | undefined | null => {
replaceNodesMatching(mdTree, (n): ParseTree | undefined | null => {
if (n.type === "WikiLinkPage") {
let pageName = n.children![0].text!;
if (pageName === oldName) {
@@ -110,7 +110,7 @@ export async function renamePage() {
return;
});
// let newText = text.replaceAll(`[[${oldName}]]`, `[[${newName}]]`);
let newText = renderMarkdown(mdTree);
let newText = renderToText(mdTree);
if (text !== newText) {
console.log("Changes made, saving...");
await writePage(pageToUpdate, newText);
@@ -179,6 +179,14 @@ export async function clearPageIndex(page: string) {
await clearPageIndexForPage(page);
}
export async function parsePage() {
console.log(await parseMarkdown(await getText()));
export async function parseServerPageCommand() {
console.log(await invokeFunction("server", "parsePage", await getText()));
}
export async function parsePageCommand() {
parsePage(await getText());
}
export async function parsePage(text: string) {
console.log("AST", JSON.stringify(await parseMarkdown(text), null, 2));
}