Work on materialized queries

plugos/bin/plugos-server.ts

@@ -11,9 +11,9 @@ import {EndpointHook, EndpointHookT} from "../hooks/endpoint";
 import {safeRun} from "../util";
 import knex from "knex";
 import {
-  ensureTable,
+    ensureTable,
-  storeReadSyscalls,
+    storeReadSyscalls,
-  storeWriteSyscalls,
+    storeWriteSyscalls,
 } from "../syscalls/store.knex_node";
 import {fetchSyscalls} from "../syscalls/fetch.node";
 import {EventHook, EventHookT} from "../hooks/event";
@@ -21,13 +21,13 @@ import {eventSyscalls} from "../syscalls/event";
 
 let args = yargs(hideBin(process.argv))
     .option("port", {
-      type: "number",
+        type: "number",
-      default: 1337,
+        default: 1337,
     })
     .parse();
 
 if (!args._.length) {
-  console.error("Usage: plugos-server <path-to-plugs>");
+    console.error("Usage: plugos-server <path-to-plugs>");
   process.exit(1);
 }
 

plugs/core/core.plug.yaml

@@ -9,6 +9,10 @@ functions:
     path: "./page.ts:indexLinks"
     events:
       - page:index
+  indexItems:
+    path: "./item.ts:indexItems"
+    events:
+      - page:index
   deletePage:
     path: "./page.ts:deletePage"
     command:
@@ -50,3 +54,10 @@ functions:
     events:
       - plug:load
     env: server
+  updateMaterializedQueriesOnPage:
+    path: ./materialized_queries.ts:updateMaterializedQueriesOnPage
+    env: server
+  updateMaterializedQueriesCommand:
+    path: ./materialized_queries.ts:updateMaterializedQueriesCommand
+    command:
+      name: "Materialized Queries: Update"

plugs/core/item.ts

@@ -0,0 +1,39 @@
+import { IndexEvent } from "../../webapp/app_event";
+import { whiteOutQueries } from "./materialized_queries";
+import { syscall } from "../lib/syscall";
+
+type Item = {
+  item: string;
+  children?: string[];
+};
+
+const pageRefRe = /\[\[[^\]]+@\d+\]\]/;
+const itemFullRe =
+  /(?<prefix>[\t ]*)[\-\*]\s*([^\n]+)(\n\k<prefix>\s+[\-\*][^\n]+)*/g;
+
+export async function indexItems({ name, text }: IndexEvent) {
+  let items: { key: string; value: Item }[] = [];
+  text = whiteOutQueries(text);
+  for (let match of text.matchAll(itemFullRe)) {
+    let entire = match[0];
+    let item = match[2];
+    if (item.match(pageRefRe)) {
+      continue;
+    }
+    let pos = match.index!;
+    let lines = entire.split("\n");
+
+    let value: Item = {
+      item,
+    };
+    if (lines.length > 1) {
+      value.children = lines.slice(1);
+    }
+    items.push({
+      key: `it:${pos}`,
+      value,
+    });
+  }
+  console.log("Found", items.length, "item(s)");
+  await syscall("indexer.batchSet", name, items);
+}

plugs/core/materialized_queries.ts

@@ -0,0 +1,83 @@
+import { syscall } from "../lib/syscall";
+
+export const queryRegex =
+  /(<!--\s*#query\s+(?<table>\w+)\s*(filter\s+["'“”‘’](?<filter>[^"'“”‘’]+)["'“”‘’])?\s*(group by\s+(?<groupBy>\w+))?\s*-->)(.+?)(<!--\s*#end\s*-->)/gs;
+
+export function whiteOutQueries(text: string): string {
+  return text.replaceAll(queryRegex, (match) =>
+    new Array(match.length + 1).join(" ")
+  );
+}
+
+async function replaceAsync(
+  str: string,
+  regex: RegExp,
+  asyncFn: (match: string, ...args: any[]) => Promise<string>
+) {
+  const promises: Promise<string>[] = [];
+  str.replace(regex, (match: string, ...args: any[]): string => {
+    const promise = asyncFn(match, ...args);
+    promises.push(promise);
+    return "";
+  });
+  const data = await Promise.all(promises);
+  return str.replace(regex, () => data.shift()!);
+}
+
+export async function updateMaterializedQueriesCommand() {
+  await syscall(
+    "system.invokeFunctionOnServer",
+    "updateMaterializedQueriesOnPage",
+    await syscall("editor.getCurrentPage")
+  );
+  syscall("editor.flashNotification", "Updated materialized queries");
+}
+
+// Called from client, running on server
+export async function updateMaterializedQueriesOnPage(pageName: string) {
+  let { text } = await syscall("space.readPage", pageName);
+  text = await replaceAsync(text, queryRegex, async (match, ...args) => {
+    let { table, filter, groupBy } = args[args.length - 1];
+    const startQuery = args[0];
+    const endQuery = args[args.length - 4];
+    let results = [];
+    switch (table) {
+      case "task":
+        for (let {
+          key,
+          page,
+          value: { task, complete, children },
+        } of await syscall("indexer.scanPrefixGlobal", "task:")) {
+          let [, pos] = key.split(":");
+          if (!filter || (filter && task.includes(filter))) {
+            results.push(
+              `* [${complete ? "x" : " "}] [[${page}@${pos}]] ${task}`
+            );
+            if (children) {
+              results.push(children.join("\n"));
+            }
+          }
+        }
+        return `${startQuery}\n${results.join("\n")}\n${endQuery}`;
+      case "item":
+        for (let {
+          key,
+          page,
+          value: { item, children },
+        } of await syscall("indexer.scanPrefixGlobal", "it:")) {
+          let [, pos] = key.split(":");
+          if (!filter || (filter && item.includes(filter))) {
+            results.push(`* [[${page}@${pos}]] ${item}`);
+            if (children) {
+              results.push(children.join("\n"));
+            }
+          }
+        }
+        return `${startQuery}\n${results.join("\n")}\n${endQuery}`;
+      default:
+        return match;
+    }
+  });
+  // console.log("New text", text);
+  await syscall("space.writePage", pageName, text);
+}

plugs/core/navigate.ts

@@ -1,5 +1,8 @@
 import { ClickEvent } from "../../webapp/app_event";
 import { syscall } from "../lib/syscall";
+import { updateMaterializedQueriesCommand } from "./materialized_queries";
+
+const materializedQueryPrefix = /<!--\s*#query\s+/;
 
 async function navigate(syntaxNode: any) {
   if (!syntaxNode) {
@@ -18,6 +21,11 @@ async function navigate(syntaxNode: any) {
     case "URL":
       await syscall("editor.openUrl", syntaxNode.text);
       break;
+    case "CommentBlock":
+      if (syntaxNode.text.match(materializedQueryPrefix)) {
+        await updateMaterializedQueriesCommand();
+      }
+      break;
     case "Link":
       // Markdown link: [bla](URLHERE) needs extraction
       let match = /\[[^\\]+\]\(([^\)]+)\)/.exec(syntaxNode.text);

plugs/markdown/yarn.lock

@@ -0,0 +1,94 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"@types/commonmark@^0.27.5":
+  version "0.27.5"
+  resolved "https://registry.yarnpkg.com/@types/commonmark/-/commonmark-0.27.5.tgz#008f2e8fb845c906146aa97510d66953d916aed2"
+  integrity sha512-vIqgmHyLsc8Or3EWLz6QkhI8/v61FNeH0yxRupA7VqSbA2eFMoHHJAhZSHudplAV89wqg1CKSmShE016ziRXuw==
+
+"@types/linkify-it@*":
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/@types/linkify-it/-/linkify-it-3.0.2.tgz#fd2cd2edbaa7eaac7e7f3c1748b52a19143846c9"
+  integrity sha512-HZQYqbiFVWufzCwexrvh694SOim8z2d+xJl5UNamcvQFejLY/2YUtzXHYi3cHdI7PMlS8ejH2slRAOJQ32aNbA==
+
+"@types/markdown-it@^12.2.3":
+  version "12.2.3"
+  resolved "https://registry.yarnpkg.com/@types/markdown-it/-/markdown-it-12.2.3.tgz#0d6f6e5e413f8daaa26522904597be3d6cd93b51"
+  integrity sha512-GKMHFfv3458yYy+v/N8gjufHO6MSZKCOXpZc5GXIWWy8uldwfmPn98vp81gZ5f9SVw8YYBctgfJ22a2d7AOMeQ==
+  dependencies:
+    "@types/linkify-it" "*"
+    "@types/mdurl" "*"
+
+"@types/mdurl@*":
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/@types/mdurl/-/mdurl-1.0.2.tgz#e2ce9d83a613bacf284c7be7d491945e39e1f8e9"
+  integrity sha512-eC4U9MlIcu2q0KQmXszyn5Akca/0jrQmwDRgpAMJai7qBWq4amIQhZyNau4VYGtCeALvW1/NtjzJJ567aZxfKA==
+
+argparse@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38"
+  integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==
+
+commonmark@^0.30.0:
+  version "0.30.0"
+  resolved "https://registry.yarnpkg.com/commonmark/-/commonmark-0.30.0.tgz#38811dc7bbf0f59d277ae09054d4d73a332f2e45"
+  integrity sha512-j1yoUo4gxPND1JWV9xj5ELih0yMv1iCWDG6eEQIPLSWLxzCXiFoyS7kvB+WwU+tZMf4snwJMMtaubV0laFpiBA==
+  dependencies:
+    entities "~2.0"
+    mdurl "~1.0.1"
+    minimist ">=1.2.2"
+    string.prototype.repeat "^0.2.0"
+
+entities@~2.0:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/entities/-/entities-2.0.3.tgz#5c487e5742ab93c15abb5da22759b8590ec03b7f"
+  integrity sha512-MyoZ0jgnLvB2X3Lg5HqpFmn1kybDiIfEQmKzTb5apr51Rb+T3KdmMiqa70T+bhGnyv7bQ6WMj2QMHpGMmlrUYQ==
+
+entities@~2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/entities/-/entities-2.1.0.tgz#992d3129cf7df6870b96c57858c249a120f8b8b5"
+  integrity sha512-hCx1oky9PFrJ611mf0ifBLBRW8lUUVRlFolb5gWRfIELabBlbp9xZvrqZLZAs+NxFnbfQoeGd8wDkygjg7U85w==
+
+linkify-it@^3.0.1:
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/linkify-it/-/linkify-it-3.0.3.tgz#a98baf44ce45a550efb4d49c769d07524cc2fa2e"
+  integrity sha512-ynTsyrFSdE5oZ/O9GEf00kPngmOfVwazR5GKDq6EYfhlpFug3J2zybX56a2PRRpc9P+FuSoGNAwjlbDs9jJBPQ==
+  dependencies:
+    uc.micro "^1.0.1"
+
+markdown-it-task-lists@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/markdown-it-task-lists/-/markdown-it-task-lists-2.1.1.tgz#f68f4d2ac2bad5a2c373ba93081a1a6848417088"
+  integrity sha512-TxFAc76Jnhb2OUu+n3yz9RMu4CwGfaT788br6HhEDlvWfdeJcLUsxk1Hgw2yJio0OXsxv7pyIPmvECY7bMbluA==
+
+markdown-it@^12.3.2:
+  version "12.3.2"
+  resolved "https://registry.yarnpkg.com/markdown-it/-/markdown-it-12.3.2.tgz#bf92ac92283fe983fe4de8ff8abfb5ad72cd0c90"
+  integrity sha512-TchMembfxfNVpHkbtriWltGWc+m3xszaRD0CZup7GFFhzIgQqxIfn3eGj1yZpfuflzPvfkt611B2Q/Bsk1YnGg==
+  dependencies:
+    argparse "^2.0.1"
+    entities "~2.1.0"
+    linkify-it "^3.0.1"
+    mdurl "^1.0.1"
+    uc.micro "^1.0.5"
+
+mdurl@^1.0.1, mdurl@~1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/mdurl/-/mdurl-1.0.1.tgz#fe85b2ec75a59037f2adfec100fd6c601761152e"
+  integrity sha1-/oWy7HWlkDfyrf7BAP1sYBdhFS4=
+
+minimist@>=1.2.2:
+  version "1.2.6"
+  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44"
+  integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==
+
+string.prototype.repeat@^0.2.0:
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/string.prototype.repeat/-/string.prototype.repeat-0.2.0.tgz#aba36de08dcee6a5a337d49b2ea1da1b28fc0ecf"
+  integrity sha1-q6Nt4I3O5qWjN9SbLqHaGyj8Ds8=
+
+uc.micro@^1.0.1, uc.micro@^1.0.5:
+  version "1.0.6"
+  resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-1.0.6.tgz#9c411a802a409a91fc6cf74081baba34b24499ac"
+  integrity sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==

plugs/tasks/task.ts

@@ -2,11 +2,20 @@ import type { ClickEvent } from "../../webapp/app_event";
 import { IndexEvent } from "../../webapp/app_event";
 import { syscall } from "../lib/syscall";
 
+import { whiteOutQueries } from "../core/materialized_queries";
+
 const allTasksPageName = "ALL TASKS";
 const taskRe = /[\-\*]\s*\[([ Xx])\]\s*(.*)/g;
+const taskFullRe =
+  /(?<prefix>[\t ]*)[\-\*]\s*\[([ Xx])\]\s*([^\n]+)(\n\k<prefix>\s+[\-\*][^\n]+)*/g;
 const extractPageLink = /[\-\*]\s*\[[ Xx]\]\s\[\[([^\]]+)@(\d+)\]\]\s*(.*)/;
 
-type Task = { task: string; complete: boolean; pos?: number };
+type Task = {
+  task: string;
+  complete: boolean;
+  pos?: number;
+  children?: string[];
+};
 
 export async function indexTasks({ name, text }: IndexEvent) {
   if (name === allTasksPageName) {
@@ -15,16 +24,24 @@ export async function indexTasks({ name, text }: IndexEvent) {
 
   console.log("Indexing tasks");
   let tasks: { key: string; value: Task }[] = [];
-  for (let match of text.matchAll(taskRe)) {
+  text = whiteOutQueries(text);
-    let complete = match[1] !== " ";
+  for (let match of text.matchAll(taskFullRe)) {
-    let task = match[2];
+    let entire = match[0];
+    let complete = match[2] !== " ";
+    let task = match[3];
     let pos = match.index!;
+    let lines = entire.split("\n");
+
+    let value: Task = {
+      task,
+      complete,
+    };
+    if (lines.length > 1) {
+      value.children = lines.slice(1);
+    }
     tasks.push({
       key: `task:${pos}`,
-      value: {
+      value,
-        task,
-        complete,
-      },
     });
   }
   console.log("Found", tasks.length, "task(s)");
@@ -37,7 +54,7 @@ export async function updateTaskPage() {
   for (let {
     key,
     page,
-    value: { task, complete, pos },
+    value: { task, complete },
   } of allTasks) {
     if (complete) {
       continue;

server/server.ts

@@ -21,8 +21,8 @@ let args = yargs(hideBin(process.argv))
     .parse();
 
 if (!args._.length) {
-    console.error("Usage: silverbullet <path-to-pages>");
+  console.error("Usage: silverbullet <path-to-pages>");
-    process.exit(1);
+  process.exit(1);
 }
 
 const pagesPath = args._[0] as string;

webapp/styles/editor.scss

@@ -211,4 +211,8 @@
   .line-comment {
     background-color: rgba(255, 255, 0, 0.5);
   }
+
+  .comment {
+    color: gray;
+  }
 }