Work on materialized queries

plugs/core/core.plug.yaml

@@ -9,6 +9,10 @@ functions:
     path: "./page.ts:indexLinks"
     events:
       - page:index
+  indexItems:
+    path: "./item.ts:indexItems"
+    events:
+      - page:index
   deletePage:
     path: "./page.ts:deletePage"
     command:
@@ -50,3 +54,10 @@ functions:
     events:
       - plug:load
     env: server
+  updateMaterializedQueriesOnPage:
+    path: ./materialized_queries.ts:updateMaterializedQueriesOnPage
+    env: server
+  updateMaterializedQueriesCommand:
+    path: ./materialized_queries.ts:updateMaterializedQueriesCommand
+    command:
+      name: "Materialized Queries: Update"

plugs/core/item.ts

@@ -0,0 +1,39 @@
+import { IndexEvent } from "../../webapp/app_event";
+import { whiteOutQueries } from "./materialized_queries";
+import { syscall } from "../lib/syscall";
+
+type Item = {
+  item: string;
+  children?: string[];
+};
+
+const pageRefRe = /\[\[[^\]]+@\d+\]\]/;
+const itemFullRe =
+  /(?<prefix>[\t ]*)[\-\*]\s*([^\n]+)(\n\k<prefix>\s+[\-\*][^\n]+)*/g;
+
+export async function indexItems({ name, text }: IndexEvent) {
+  let items: { key: string; value: Item }[] = [];
+  text = whiteOutQueries(text);
+  for (let match of text.matchAll(itemFullRe)) {
+    let entire = match[0];
+    let item = match[2];
+    if (item.match(pageRefRe)) {
+      continue;
+    }
+    let pos = match.index!;
+    let lines = entire.split("\n");
+
+    let value: Item = {
+      item,
+    };
+    if (lines.length > 1) {
+      value.children = lines.slice(1);
+    }
+    items.push({
+      key: `it:${pos}`,
+      value,
+    });
+  }
+  console.log("Found", items.length, "item(s)");
+  await syscall("indexer.batchSet", name, items);
+}

plugs/core/materialized_queries.ts

@@ -0,0 +1,83 @@
+import { syscall } from "../lib/syscall";
+
+export const queryRegex =
+  /(<!--\s*#query\s+(?<table>\w+)\s*(filter\s+["'“”‘’](?<filter>[^"'“”‘’]+)["'“”‘’])?\s*(group by\s+(?<groupBy>\w+))?\s*-->)(.+?)(<!--\s*#end\s*-->)/gs;
+
+export function whiteOutQueries(text: string): string {
+  return text.replaceAll(queryRegex, (match) =>
+    new Array(match.length + 1).join(" ")
+  );
+}
+
+async function replaceAsync(
+  str: string,
+  regex: RegExp,
+  asyncFn: (match: string, ...args: any[]) => Promise<string>
+) {
+  const promises: Promise<string>[] = [];
+  str.replace(regex, (match: string, ...args: any[]): string => {
+    const promise = asyncFn(match, ...args);
+    promises.push(promise);
+    return "";
+  });
+  const data = await Promise.all(promises);
+  return str.replace(regex, () => data.shift()!);
+}
+
+export async function updateMaterializedQueriesCommand() {
+  await syscall(
+    "system.invokeFunctionOnServer",
+    "updateMaterializedQueriesOnPage",
+    await syscall("editor.getCurrentPage")
+  );
+  syscall("editor.flashNotification", "Updated materialized queries");
+}
+
+// Called from client, running on server
+export async function updateMaterializedQueriesOnPage(pageName: string) {
+  let { text } = await syscall("space.readPage", pageName);
+  text = await replaceAsync(text, queryRegex, async (match, ...args) => {
+    let { table, filter, groupBy } = args[args.length - 1];
+    const startQuery = args[0];
+    const endQuery = args[args.length - 4];
+    let results = [];
+    switch (table) {
+      case "task":
+        for (let {
+          key,
+          page,
+          value: { task, complete, children },
+        } of await syscall("indexer.scanPrefixGlobal", "task:")) {
+          let [, pos] = key.split(":");
+          if (!filter || (filter && task.includes(filter))) {
+            results.push(
+              `* [${complete ? "x" : " "}] [[${page}@${pos}]] ${task}`
+            );
+            if (children) {
+              results.push(children.join("\n"));
+            }
+          }
+        }
+        return `${startQuery}\n${results.join("\n")}\n${endQuery}`;
+      case "item":
+        for (let {
+          key,
+          page,
+          value: { item, children },
+        } of await syscall("indexer.scanPrefixGlobal", "it:")) {
+          let [, pos] = key.split(":");
+          if (!filter || (filter && item.includes(filter))) {
+            results.push(`* [[${page}@${pos}]] ${item}`);
+            if (children) {
+              results.push(children.join("\n"));
+            }
+          }
+        }
+        return `${startQuery}\n${results.join("\n")}\n${endQuery}`;
+      default:
+        return match;
+    }
+  });
+  // console.log("New text", text);
+  await syscall("space.writePage", pageName, text);
+}

plugs/core/navigate.ts

@@ -1,5 +1,8 @@
 import { ClickEvent } from "../../webapp/app_event";
 import { syscall } from "../lib/syscall";
+import { updateMaterializedQueriesCommand } from "./materialized_queries";
+
+const materializedQueryPrefix = /<!--\s*#query\s+/;
 
 async function navigate(syntaxNode: any) {
   if (!syntaxNode) {
@@ -18,6 +21,11 @@ async function navigate(syntaxNode: any) {
     case "URL":
       await syscall("editor.openUrl", syntaxNode.text);
       break;
+    case "CommentBlock":
+      if (syntaxNode.text.match(materializedQueryPrefix)) {
+        await updateMaterializedQueriesCommand();
+      }
+      break;
     case "Link":
       // Markdown link: [bla](URLHERE) needs extraction
       let match = /\[[^\\]+\]\(([^\)]+)\)/.exec(syntaxNode.text);