Changed all indexes to use (pre) parsed trees.

common/spaces/evented_space_primitives.ts

@@ -45,9 +45,9 @@ export class EventedSpacePrimitives implements SpacePrimitives {
       this.eventHook
         .dispatchEvent("page:saved", pageName)
         .then(() => {
-          return this.eventHook.dispatchEvent("page:index", {
+          return this.eventHook.dispatchEvent("page:index_text", {
             name: pageName,
-            text: text,
+            text,
           });
         })
         .catch((e) => {

common/tree.ts

@@ -13,6 +13,10 @@ export function addParentPointers(tree: ParseTree) {
     return;
   }
   for (let child of tree.children) {
+    if (child.parent) {
+      // Already added parent pointers before
+      return;
+    }
     child.parent = tree;
     addParentPointers(child);
   }

plugos-syscall/event.ts

@@ -6,13 +6,18 @@ export async function dispatch(
   timeout?: number
 ): Promise<any[]> {
   return new Promise((resolve, reject) => {
-    let timeOut = setTimeout(() => {
+    let timeouter: any = -1;
+    if (timeout) {
+      timeouter = setTimeout(() => {
         console.log("Timeout!");
         reject("timeout");
       }, timeout);
+    }
     syscall("event.dispatch", eventName, data)
       .then((r) => {
-        clearTimeout(timeOut);
+        if (timeouter !== -1) {
+          clearTimeout(timeouter);
+        }
         resolve(r);
       })
       .catch(reject);

plugs/core/core.plug.yaml

@@ -29,6 +29,10 @@ functions:
     path: ./page.ts:pageQueryProvider
     events:
       - query:page
+  parseIndexTextRepublish:
+    path: "./page.ts:parseIndexTextRepublish"
+    events:
+      - page:index_text
   indexLinks:
     path: "./page.ts:indexLinks"
     events:

plugs/core/dates.ts

@@ -1,30 +1,9 @@
 import { insertAtCursor } from "plugos-silverbullet-syscall/editor";
-import { IndexEvent } from "../../webapp/app_event";
-import { batchSet } from "plugos-silverbullet-syscall";
-import { whiteOutQueries } from "../query/util";
 
 const dateMatchRegex = /(\d{4}\-\d{2}\-\d{2})/g;
 
-// Index key space:
-// d:[date]:page@pos
-
-export async function indexDates({ name, text }: IndexEvent) {
-  let dates: { key: string; value: boolean }[] = [];
-  text = whiteOutQueries(text);
-  console.log("Now date indexing", name);
-  for (let match of text.matchAll(dateMatchRegex)) {
-    // console.log("Date match", match[0]);
-    dates.push({
-      key: `d:${match[0]}:${name}@${match.index}`,
-      value: true,
-    });
-  }
-  console.log("Found", dates.length, "dates");
-  await batchSet(name, dates);
-}
-
 export function niceDate(d: Date): string {
-  return new Date().toISOString().split("T")[0];
+  return d.toISOString().split("T")[0];
 }
 
 export async function insertToday() {

plugs/core/item.ts

@@ -1,9 +1,8 @@
-import { IndexEvent } from "../../webapp/app_event";
+import { IndexTreeEvent } from "../../webapp/app_event";
 
 import { batchSet, scanPrefixGlobal } from "plugos-silverbullet-syscall/index";
-import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
 import { collectNodesOfType, ParseTree, renderToText } from "../../common/tree";
-import { whiteOutQueries } from "../query/util";
+import { removeQueries } from "../query/util";
 import { applyQuery, QueryProviderEvent } from "../query/engine";
 
 export type Item = {
@@ -14,14 +13,13 @@ export type Item = {
   pos?: number;
 };
 
-export async function indexItems({ name, text }: IndexEvent) {
+export async function indexItems({ name, tree }: IndexTreeEvent) {
   let items: { key: string; value: Item }[] = [];
-  text = whiteOutQueries(text);
+  removeQueries(tree);
 
   console.log("Indexing items", name);
-  let mdTree = await parseMarkdown(text);
 
-  let coll = collectNodesOfType(mdTree, "ListItem");
+  let coll = collectNodesOfType(tree, "ListItem");
 
   coll.forEach((n) => {
     if (!n.children) {

plugs/core/page.ts

@@ -1,4 +1,4 @@
-import { IndexEvent } from "../../webapp/app_event";
+import { IndexEvent, IndexTreeEvent } from "../../webapp/app_event";
 import {
   batchSet,
   clearPageIndex as clearPageIndexSyscall,
@@ -28,13 +28,11 @@ import {
 import { applyQuery, QueryProviderEvent } from "../query/engine";
 import { PageMeta } from "../../common/types";
 
-export async function indexLinks({ name, text }: IndexEvent) {
+export async function indexLinks({ name, tree }: IndexTreeEvent) {
   let backLinks: { key: string; value: string }[] = [];
   // [[Style Links]]
   console.log("Now indexing", name);
-  let mdTree = await parseMarkdown(text);
+  collectNodesMatching(tree, (n) => n.type === "WikiLinkPage").forEach((n) => {
-  collectNodesMatching(mdTree, (n) => n.type === "WikiLinkPage").forEach(
-    (n) => {
     let toPage = n.children![0].text!;
     if (toPage.includes("@")) {
       toPage = toPage.split("@")[0];
@@ -43,8 +41,7 @@ export async function indexLinks({ name, text }: IndexEvent) {
       key: `pl:${toPage}:${n.from}`,
       value: name,
     });
-    }
+  });
-  );
   console.log("Found", backLinks.length, "wiki link(s)");
   await batchSet(name, backLinks);
 }
@@ -193,10 +190,11 @@ export async function reindexSpace() {
   let pages = await listPages();
   for (let { name } of pages) {
     console.log("Indexing", name);
-    const pageObj = await readPage(name);
+    const { text } = await readPage(name);
+    let parsed = await parseMarkdown(text);
     await dispatch("page:index", {
       name,
-      text: pageObj.text,
+      tree: parsed,
     });
   }
 }
@@ -206,6 +204,13 @@ export async function clearPageIndex(page: string) {
   await clearPageIndexForPage(page);
 }
 
+export async function parseIndexTextRepublish({ name, text }: IndexEvent) {
+  await dispatch("page:index", {
+    name,
+    tree: await parseMarkdown(text),
+  });
+}
+
 export async function parseServerPageCommand() {
   console.log(await invokeFunction("server", "parsePage", await getText()));
 }

plugs/core/template.ts

@@ -4,7 +4,6 @@ import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
 import { extractMeta } from "../query/data";
 import { renderToText } from "../../common/tree";
 import { niceDate } from "./dates";
-import { dispatch } from "plugos-syscall/event";
 import { invokeFunction } from "plugos-silverbullet-syscall/system";
 
 const pageTemplatePrefix = `template/page/`;
@@ -60,17 +59,17 @@ export async function replaceTemplateVarsCommand() {
 
 export function replaceTemplateVars(s: string, pageName: string): string {
   return s.replaceAll(/\{\{([^\}]+)\}\}/g, (match, v) => {
-    if (v === "today") {
+    switch (v) {
+      case "today":
         return niceDate(new Date());
-    }
+      case "yesterday":
-    if (v.startsWith("placeholder:")) {
+        let yesterday = new Date();
-      // Dispatch event, to be replaced in the file async later
+        yesterday.setDate(yesterday.getDate() - 1);
-      dispatch(v, {
+        return niceDate(yesterday);
-        pageName: pageName,
+      case "lastWeek":
-        placeholder: v,
+        let lastWeek = new Date();
-      }).catch((e) => {
+        lastWeek.setDate(lastWeek.getDate() - 7);
-        console.error("Failed to dispatch placeholder event", e);
+        return niceDate(lastWeek);
-      });
     }
     return match;
   });

plugs/markdown/preview.ts

@@ -15,6 +15,24 @@ body {
   padding-right: 20px;
 }
 
+table {
+  width: 100%;
+  border-spacing: 0;
+}
+
+thead tr {
+    background-color: #333;
+    color: #eee;
+}
+
+th, td {
+    padding: 8px;
+}
+
+tbody tr:nth-of-type(even) {
+    background-color: #f3f3f3;
+}
+
 a[href] {
   text-decoration: none;
 }

plugs/mattermost/client.ts

@@ -1,11 +1,12 @@
 import { readPage } from "plugos-silverbullet-syscall/space";
 import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
 import { extractMeta } from "../query/data";
-import { UserProfile } from "@hmhealey/types/lib/users";
 import { json } from "plugos-syscall/fetch";
-import { Post } from "@hmhealey/types/lib/posts";
+import type { UserProfile } from "@mattermost/types/lib/users";
-import { Channel } from "@hmhealey/types/lib/channels";
+import type { Post } from "@mattermost/types/lib/posts";
-import { Team } from "@hmhealey/types/lib/teams";
+import type { Channel } from "@mattermost/types/lib/channels";
+import type { Team } from "@mattermost/types/lib/teams";
+import { niceDate } from "../core/dates";
 
 type MattermostConfig = {
   url: string;
@@ -19,6 +20,13 @@ async function getConfig(): Promise<MattermostConfig> {
   return pageMeta as MattermostConfig;
 }
 
+type AugmentedPost = Post & {
+  // Dates we can use to filter
+  createdAt: string;
+  updatedAt: string;
+  editedAt: string;
+};
+
 export class MattermostClient {
   userCache = new Map<string, UserProfile>();
   channelCache = new Map<string, Channel>();
@@ -77,7 +85,10 @@ export class MattermostClient {
     return team!;
   }
 
-  async getFlaggedPosts(userId: string, perPage: number = 10): Promise<Post[]> {
+  async getFlaggedPosts(
+    userId: string,
+    perPage: number = 10
+  ): Promise<AugmentedPost[]> {
     let postCollection = await json(
       `${this.url}/api/v4/users/${userId}/posts/flagged?per_page=${perPage}`,
       {
@@ -86,10 +97,24 @@ export class MattermostClient {
         },
       }
     );
-    let posts: Post[] = [];
+    let posts: AugmentedPost[] = [];
     for (let order of postCollection.order) {
-      posts.push(postCollection.posts[order]);
+      let post = postCollection.posts[order];
+      augmentPost(post);
+      posts.push(post);
     }
     return posts;
   }
 }
+
+function augmentPost(post: AugmentedPost) {
+  if (post.create_at) {
+    post.createdAt = niceDate(new Date(post.create_at));
+  }
+  if (post.update_at) {
+    post.updatedAt = niceDate(new Date(post.update_at));
+  }
+  if (post.edit_at) {
+    post.editedAt = niceDate(new Date(post.edit_at));
+  }
+}

plugs/mattermost/mattermost.ts

@@ -20,17 +20,16 @@ export async function savedPostsQueryProvider({
   let savedPostsMd = [];
   savedPosts = applyQuery(query, savedPosts);
   for (let savedPost of savedPosts) {
-    // savedPost.
     let channel = await client.getChannel(savedPost.channel_id);
     let team = await client.getTeam(channel.team_id);
     savedPostsMd.push(
-      `@${
+      `@${(await client.getUser(savedPost.user_id)).username} [${
-        (await client.getUser(savedPost.user_id)).username
+        savedPost.createdAt
-      } [link](${mattermostDesktopUrlForPost(
+      }](${mattermostDesktopUrlForPost(
         client.url,
         team.name,
         savedPost.id
-      )}):\n> ${savedPost.message.replaceAll(/\n/g, "\n> ")}`
+      )}):\n> ${savedPost.message.substring(0, 1000).replaceAll(/\n/g, "\n> ")}`
     );
   }
   return savedPostsMd.join("\n\n");

plugs/package-lock.json

@@ -8,10 +8,10 @@
       "name": "plugs",
       "version": "1.0.0",
       "dependencies": {
-        "@hmhealey/types": "^6.6.0-4",
         "@jest/globals": "^27.5.1",
         "@lezer/generator": "^0.15.4",
         "@lezer/lr": "^0.15.8",
+        "@mattermost/types": "^6.7.0-0",
         "@types/yaml": "^1.9.7",
         "plugos-silverbullet-syscall": "file:../plugos-silverbullet-syscall",
         "plugos-syscall": "file:../plugos-syscall",
@@ -120,19 +120,6 @@
         "node": ">=4"
       }
     },
-    "node_modules/@hmhealey/types": {
-      "version": "6.6.0-4",
-      "resolved": "https://registry.npmjs.org/@hmhealey/types/-/types-6.6.0-4.tgz",
-      "integrity": "sha512-71IxVaXhrUesmLnvQQh4RtUqqhmVL+ejci4qo4R6rTWTdY77BniRtBx269uAz34wzTlAgITysN8x7MBTdt/XBg==",
-      "peerDependencies": {
-        "typescript": "^4.3"
-      },
-      "peerDependenciesMeta": {
-        "typescript": {
-          "optional": true
-        }
-      }
-    },
     "node_modules/@jest/environment": {
       "version": "27.5.1",
       "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-27.5.1.tgz",
@@ -216,6 +203,19 @@
         "@lezer/common": "^0.15.0"
       }
     },
+    "node_modules/@mattermost/types": {
+      "version": "6.7.0-0",
+      "resolved": "https://registry.npmjs.org/@mattermost/types/-/types-6.7.0-0.tgz",
+      "integrity": "sha512-mT8wJwWEp20KPo9D12y7bW7EdUHO7VhUHxr3gH8nPGapWooGcl0Ra0H3u1iCjPpqPWvp7LiodcneU0IysunYKQ==",
+      "peerDependencies": {
+        "typescript": "^4.3"
+      },
+      "peerDependenciesMeta": {
+        "typescript": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/@sinonjs/commons": {
       "version": "1.8.3",
       "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz",
@@ -697,12 +697,6 @@
         }
       }
     },
-    "@hmhealey/types": {
-      "version": "6.6.0-4",
-      "resolved": "https://registry.npmjs.org/@hmhealey/types/-/types-6.6.0-4.tgz",
-      "integrity": "sha512-71IxVaXhrUesmLnvQQh4RtUqqhmVL+ejci4qo4R6rTWTdY77BniRtBx269uAz34wzTlAgITysN8x7MBTdt/XBg==",
-      "requires": {}
-    },
     "@jest/environment": {
       "version": "27.5.1",
       "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-27.5.1.tgz",
@@ -771,6 +765,12 @@
         "@lezer/common": "^0.15.0"
       }
     },
+    "@mattermost/types": {
+      "version": "6.7.0-0",
+      "resolved": "https://registry.npmjs.org/@mattermost/types/-/types-6.7.0-0.tgz",
+      "integrity": "sha512-mT8wJwWEp20KPo9D12y7bW7EdUHO7VhUHxr3gH8nPGapWooGcl0Ra0H3u1iCjPpqPWvp7LiodcneU0IysunYKQ==",
+      "requires": {}
+    },
     "@sinonjs/commons": {
       "version": "1.8.3",
       "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz",

plugs/package.json

@@ -5,7 +5,7 @@
     "generate": "lezer-generator query/query.grammar -o query/parse-query.js"
   },
   "dependencies": {
-    "@hmhealey/types": "^6.6.0-4",
+    "@mattermost/types": "^6.7.0-0",
     "@jest/globals": "^27.5.1",
     "@lezer/generator": "^0.15.4",
     "@lezer/lr": "^0.15.8",

plugs/query/data.ts

@@ -1,23 +1,17 @@
 // Index key space:
 // data:page@pos
 
-import { IndexEvent } from "../../webapp/app_event";
+import { IndexTreeEvent } from "../../webapp/app_event";
 import { batchSet, scanPrefixGlobal } from "plugos-silverbullet-syscall";
-import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
 import { collectNodesOfType, findNodeOfType, ParseTree, replaceNodesMatching } from "../../common/tree";
 import YAML, { parse as parseYaml, parseAllDocuments } from "yaml";
-import { whiteOutQueries } from "./util";
 import type { QueryProviderEvent } from "./engine";
 import { applyQuery } from "./engine";
 
-export async function indexData({ name, text }: IndexEvent) {
+export async function indexData({ name, tree }: IndexTreeEvent) {
-  text = whiteOutQueries(text);
-  // console.log("Now data indexing", name);
-  let mdTree = await parseMarkdown(text);
-
   let dataObjects: { key: string; value: Object }[] = [];
 
-  collectNodesOfType(mdTree, "FencedCode").forEach((t) => {
+  collectNodesOfType(tree, "FencedCode").forEach((t) => {
     let codeInfoNode = findNodeOfType(t, "CodeInfo");
     if (!codeInfoNode) {
       return;

plugs/query/materialized_queries.ts

@@ -1,30 +1,18 @@
-import { flashNotification, getCurrentPage, reloadPage, save } from "plugos-silverbullet-syscall/editor";
+import { flashNotification, getCurrentPage, getText, reloadPage, save } from "plugos-silverbullet-syscall/editor";
 
 import { readPage, writePage } from "plugos-silverbullet-syscall/space";
 import { invokeFunction } from "plugos-silverbullet-syscall/system";
 import { parseQuery } from "./engine";
 import { replaceTemplateVars } from "../core/template";
-import { queryRegex } from "./util";
+import { queryRegex, removeQueries } from "./util";
 import { dispatch } from "plugos-syscall/event";
-
+import { replaceAsync } from "../lib/util";
-async function replaceAsync(
+import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
-  str: string,
-  regex: RegExp,
-  asyncFn: (match: string, ...args: any[]) => Promise<string>
-) {
-  const promises: Promise<string>[] = [];
-  str.replace(regex, (match: string, ...args: any[]): string => {
-    const promise = asyncFn(match, ...args);
-    promises.push(promise);
-    return "";
-  });
-  const data = await Promise.all(promises);
-  return str.replace(regex, () => data.shift()!);
-}
 
 export async function updateMaterializedQueriesCommand() {
   const currentPage = await getCurrentPage();
   await save();
+  await flashNotification("Updating materialized queries...");
   await invokeFunction(
     "server",
     "updateMaterializedQueriesOnPage",
@@ -34,6 +22,12 @@ export async function updateMaterializedQueriesCommand() {
   await flashNotification("Updated materialized queries");
 }
 
+export async function whiteOutQueriesCommand() {
+  const text = await getText();
+  const parsed = await parseMarkdown(text);
+  console.log(removeQueries(parsed));
+}
+
 // Called from client, running on server
 export async function updateMaterializedQueriesOnPage(pageName: string) {
   let { text } = await readPage(pageName);
@@ -49,7 +43,7 @@ export async function updateMaterializedQueriesOnPage(pageName: string) {
       let results = await dispatch(
         `query:${parsedQuery.table}`,
         { query: parsedQuery, pageName: pageName },
-        5000
+        10 * 1000
       );
       if (results.length === 0) {
         return `${startQuery}\n${endQuery}`;

plugs/query/parse-query.js

@@ -3,15 +3,19 @@ import { LRParser } from "@lezer/lr";
 
 export const parser = LRParser.deserialize({
   version: 13,
-  states: "$[OVQPOOO[QQO'#C^QOQPOOOjQPO'#C`OoQQO'#CjOtQPO'#ClOOQO'#Cm'#CmOyQQO,58xO!XQPO'#CcO!sQQO'#CaOOQO'#Ca'#CaOOQO,58z,58zO#UQPO,59UOOQO,59W,59WOOQO-E6k-E6kO#ZQQO,58}OjQPO,58|O#oQQO1G.pOOQO'#Cg'#CgOOQO'#Ci'#CiOOQO'#Cd'#CdOOQO1G.i1G.iOOQO1G.h1G.hOOQO'#Ck'#CkOOQO7+$[7+$[",
+  states:
-  stateData: "$T~OdOS~ORPO~OeROrSOvTObQX~ORWO~Os[O~OX]O~OeROrSOvTObQa~Of_Oj_Ok_Ol_Om_On_Oo_Op_O~Oq`ObTXeTXrTXvTX~ORaO~OXdOYdO[dOgbOhbOicO~OtgOugOb^ie^ir^iv^i~O",
+    "$[OVQPOOO[QQO'#C^QOQPOOOjQPO'#C`OoQQO'#CjOtQPO'#ClOOQO'#Cm'#CmOyQQO,58xO!XQPO'#CcO!sQQO'#CaOOQO'#Ca'#CaOOQO,58z,58zO#UQPO,59UOOQO,59W,59WOOQO-E6k-E6kO#ZQQO,58}OjQPO,58|O#oQQO1G.pOOQO'#Cg'#CgOOQO'#Ci'#CiOOQO'#Cd'#CdOOQO1G.i1G.iOOQO1G.h1G.hOOQO'#Ck'#CkOOQO7+$[7+$[",
+  stateData:
+    "$T~OdOS~ORPO~OeROrSOvTObQX~ORWO~Os[O~OX]O~OeROrSOvTObQa~Of_Oj_Ok_Ol_Om_On_Oo_Op_O~Oq`ObTXeTXrTXvTX~ORaO~OXdOYdO[dOgbOhbOicO~OtgOugOb^ie^ir^iv^i~O",
   goto: "!VbPPcPfjmpvPPyPyf|f!PRQOTUPVRZRRYRQXRRf`Re_Rd_RhaQVPR^V",
-  nodeNames: "⚠ Program Query Name WhereClause LogicalExpr AndExpr FilterExpr Value Number String Bool Regex Null OrderClause Order LimitClause",
+  nodeNames:
+    "⚠ Program Query Name WhereClause LogicalExpr AndExpr FilterExpr Value Number String Bool Regex Null OrderClause Order LimitClause",
   maxTerm: 38,
   skippedNodes: [0],
   repeatNodeCount: 1,
-  tokenData: ":W~RvX^#ipq#iqr$^rs$q}!O%]!P!Q%n!Q![&e!^!_&m!_!`&z!`!a'X!c!}%]#R#S%]#T#U'f#U#V){#V#W%]#W#X*w#X#Y%]#Y#Z,s#Z#`%]#`#a/T#a#b%]#b#c1h#c#d3d#d#h%]#h#i5w#i#k%]#k#l7s#l#o%]#y#z#i$f$g#i#BY#BZ#i$IS$I_#i$Ip$Iq$q$Iq$Ir$q$I|$JO#i$JT$JU#i$KV$KW#i&FU&FV#i~#nYd~X^#ipq#i#y#z#i$f$g#i#BY#BZ#i$IS$I_#i$I|$JO#i$JT$JU#i$KV$KW#i&FU&FV#i~$aP!_!`$d~$iPl~#r#s$l~$qOp~~$tUOr$qrs%Ws$Ip$q$Ip$Iq%W$Iq$Ir%W$Ir~$q~%]OY~P%bSRP}!O%]!c!}%]#R#S%]#T#o%]~%sV[~OY%nZ]%n^!P%n!P!Q&Y!Q#O%n#O#P&_#P~%n~&_O[~~&bPO~%n~&jPX~!Q![&e~&rPf~!_!`&u~&zOj~~'PPk~#r#s'S~'XOo~~'^Pn~!_!`'a~'fOm~R'kWRP}!O%]!c!}%]#R#S%]#T#b%]#b#c(T#c#g%]#g#h)P#h#o%]R(YURP}!O%]!c!}%]#R#S%]#T#W%]#W#X(l#X#o%]R(sSqQRP}!O%]!c!}%]#R#S%]#T#o%]R)UURP}!O%]!c!}%]#R#S%]#T#V%]#V#W)h#W#o%]R)oSuQRP}!O%]!c!}%]#R#S%]#T#o%]R*QURP}!O%]!c!}%]#R#S%]#T#m%]#m#n*d#n#o%]R*kSsQRP}!O%]!c!}%]#R#S%]#T#o%]R*|URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y+`#Y#o%]R+eURP}!O%]!c!}%]#R#S%]#T#g%]#g#h+w#h#o%]R+|URP}!O%]!c!}%]#R#S%]#T#V%]#V#W,`#W#o%]R,gStQRP}!O%]!c!}%]#R#S%]#T#o%]R,xTRP}!O%]!c!}%]#R#S%]#T#U-X#U#o%]R-^URP}!O%]!c!}%]#R#S%]#T#`%]#`#a-p#a#o%]R-uURP}!O%]!c!}%]#R#S%]#T#g%]#g#h.X#h#o%]R.^URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y.p#Y#o%]R.wShQRP}!O%]!c!}%]#R#S%]#T#o%]R/YURP}!O%]!c!}%]#R#S%]#T#]%]#]#^/l#^#o%]R/qURP}!O%]!c!}%]#R#S%]#T#a%]#a#b0T#b#o%]R0YURP}!O%]!c!}%]#R#S%]#T#]%]#]#^0l#^#o%]R0qURP}!O%]!c!}%]#R#S%]#T#h%]#h#i1T#i#o%]R1[SvQRP}!O%]!c!}%]#R#S%]#T#o%]R1mURP}!O%]!c!}%]#R#S%]#T#i%]#i#j2P#j#o%]R2UURP}!O%]!c!}%]#R#S%]#T#`%]#`#a2h#a#o%]R2mURP}!O%]!c!}%]#R#S%]#T#`%]#`#a3P#a#o%]R3WSiQRP}!O%]!c!}%]#R#S%]#T#o%]R3iURP}!O%]!c!}%]#R#S%]#T#f%]#f#g3{#g#o%]R4QURP}!O%]!c!}%]#R#S%]#T#W%]#W#X4d#X#o%]R4iURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y4{#Y#o%]R5QURP}!O%]!c!}%]#R#S%]#T#f%]#f#g5d#g#o%]R5kSrQRP}!O%]!c!}%]#R#S%]#T#o%]R5|URP}!O%]!c!}%]#R#S%]#T#f%]#f#g6`#g#o%]R6eURP}!O%]!c!}%]#R#S%]#T#i%]#i#j6w#j#o%]R6|URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y7`#Y#o%]R7gSgQRP}!O%]!c!}%]#R#S%]#T#o%]R7xURP}!O%]!c!}%]#R#S%]#T#[%]#[#]8[#]#o%]R8aURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y8s#Y#o%]R8xURP}!O%]!c!}%]#R#S%]#T#f%]#f#g9[#g#o%]R9aURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y9s#Y#o%]R9zSeQRP}!O%]!c!}%]#R#S%]#T#o%]",
+  tokenData:
+    ":W~RvX^#ipq#iqr$^rs$q}!O%]!P!Q%n!Q![&e!^!_&m!_!`&z!`!a'X!c!}%]#R#S%]#T#U'f#U#V){#V#W%]#W#X*w#X#Y%]#Y#Z,s#Z#`%]#`#a/T#a#b%]#b#c1h#c#d3d#d#h%]#h#i5w#i#k%]#k#l7s#l#o%]#y#z#i$f$g#i#BY#BZ#i$IS$I_#i$Ip$Iq$q$Iq$Ir$q$I|$JO#i$JT$JU#i$KV$KW#i&FU&FV#i~#nYd~X^#ipq#i#y#z#i$f$g#i#BY#BZ#i$IS$I_#i$I|$JO#i$JT$JU#i$KV$KW#i&FU&FV#i~$aP!_!`$d~$iPl~#r#s$l~$qOp~~$tUOr$qrs%Ws$Ip$q$Ip$Iq%W$Iq$Ir%W$Ir~$q~%]OY~P%bSRP}!O%]!c!}%]#R#S%]#T#o%]~%sV[~OY%nZ]%n^!P%n!P!Q&Y!Q#O%n#O#P&_#P~%n~&_O[~~&bPO~%n~&jPX~!Q![&e~&rPf~!_!`&u~&zOj~~'PPk~#r#s'S~'XOo~~'^Pn~!_!`'a~'fOm~R'kWRP}!O%]!c!}%]#R#S%]#T#b%]#b#c(T#c#g%]#g#h)P#h#o%]R(YURP}!O%]!c!}%]#R#S%]#T#W%]#W#X(l#X#o%]R(sSqQRP}!O%]!c!}%]#R#S%]#T#o%]R)UURP}!O%]!c!}%]#R#S%]#T#V%]#V#W)h#W#o%]R)oSuQRP}!O%]!c!}%]#R#S%]#T#o%]R*QURP}!O%]!c!}%]#R#S%]#T#m%]#m#n*d#n#o%]R*kSsQRP}!O%]!c!}%]#R#S%]#T#o%]R*|URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y+`#Y#o%]R+eURP}!O%]!c!}%]#R#S%]#T#g%]#g#h+w#h#o%]R+|URP}!O%]!c!}%]#R#S%]#T#V%]#V#W,`#W#o%]R,gStQRP}!O%]!c!}%]#R#S%]#T#o%]R,xTRP}!O%]!c!}%]#R#S%]#T#U-X#U#o%]R-^URP}!O%]!c!}%]#R#S%]#T#`%]#`#a-p#a#o%]R-uURP}!O%]!c!}%]#R#S%]#T#g%]#g#h.X#h#o%]R.^URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y.p#Y#o%]R.wShQRP}!O%]!c!}%]#R#S%]#T#o%]R/YURP}!O%]!c!}%]#R#S%]#T#]%]#]#^/l#^#o%]R/qURP}!O%]!c!}%]#R#S%]#T#a%]#a#b0T#b#o%]R0YURP}!O%]!c!}%]#R#S%]#T#]%]#]#^0l#^#o%]R0qURP}!O%]!c!}%]#R#S%]#T#h%]#h#i1T#i#o%]R1[SvQRP}!O%]!c!}%]#R#S%]#T#o%]R1mURP}!O%]!c!}%]#R#S%]#T#i%]#i#j2P#j#o%]R2UURP}!O%]!c!}%]#R#S%]#T#`%]#`#a2h#a#o%]R2mURP}!O%]!c!}%]#R#S%]#T#`%]#`#a3P#a#o%]R3WSiQRP}!O%]!c!}%]#R#S%]#T#o%]R3iURP}!O%]!c!}%]#R#S%]#T#f%]#f#g3{#g#o%]R4QURP}!O%]!c!}%]#R#S%]#T#W%]#W#X4d#X#o%]R4iURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y4{#Y#o%]R5QURP}!O%]!c!}%]#R#S%]#T#f%]#f#g5d#g#o%]R5kSrQRP}!O%]!c!}%]#R#S%]#T#o%]R5|URP}!O%]!c!}%]#R#S%]#T#f%]#f#g6`#g#o%]R6eURP}!O%]!c!}%]#R#S%]#T#i%]#i#j6w#j#o%]R6|URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y7`#Y#o%]R7gSgQRP}!O%]!c!}%]#R#S%]#T#o%]R7xURP}!O%]!c!}%]#R#S%]#T#[%]#[#]8[#]#o%]R8aURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y8s#Y#o%]R8xURP}!O%]!c!}%]#R#S%]#T#f%]#f#g9[#g#o%]R9aURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y9s#Y#o%]R9zSeQRP}!O%]!c!}%]#R#S%]#T#o%]",
   tokenizers: [0, 1],
-  topRules: {"Program":[0,1]},
+  topRules: { Program: [0, 1] },
-  tokenPrec: 0
+  tokenPrec: 0,
-})
+});

plugs/query/query.plug.yaml

@@ -6,6 +6,10 @@ functions:
     command:
       name: "Materialized Queries: Update"
       key: "Alt-q"
+  whiteOutQueriesCommand:
+    path: ./materialized_queries.ts:whiteOutQueriesCommand
+    command:
+      name: "Debug: Whiteout Queries"
   indexData:
     path: ./data.ts:indexData
     events:

plugs/query/util.ts

@@ -1,8 +1,45 @@
+import { addParentPointers, collectNodesMatching, ParseTree, renderToText } from "../../common/tree";
+
 export const queryRegex =
   /(<!--\s*#query\s+(.+?)-->)(.+?)(<!--\s*#end\s*-->)/gs;
 
-export function whiteOutQueries(text: string): string {
+export const queryStartRegex = /<!--\s*#query\s+(.+?)-->/s;
-  return text.replaceAll(queryRegex, (match) =>
+
-    new Array(match.length + 1).join(" ")
+export const queryEndRegex = /<!--\s*#end\s*-->/s;
-  );
+
+// export function whiteOutQueries(text: string): string {
+//   return text.replaceAll(queryRegex, (match) =>
+//     new Array(match.length + 1).join(" ")
+//   );
+// }
+
+export function removeQueries(pt: ParseTree) {
+  addParentPointers(pt);
+  collectNodesMatching(pt, (t) => {
+    if (t.type !== "CommentBlock") {
+      return false;
+    }
+    let text = t.children![0].text!;
+    if (!queryStartRegex.exec(text)) {
+      return false;
+    }
+    let parentChildren = t.parent!.children!;
+    let index = parentChildren.indexOf(t);
+    let nodesToReplace: ParseTree[] = [];
+    for (let i = index + 1; i < parentChildren.length; i++) {
+      let n = parentChildren[i];
+      if (n.type === "CommentBlock") {
+        let text = n.children![0].text!;
+        if (queryEndRegex.exec(text)) {
+          break;
+        }
+      }
+      nodesToReplace.push(n);
+    }
+    let renderedText = nodesToReplace.map(renderToText).join("");
+    parentChildren.splice(index + 1, nodesToReplace.length, {
+      text: new Array(renderedText.length + 1).join(" "),
+    });
+    return true;
+  });
 }

plugs/tasks/task.ts

@@ -1,4 +1,4 @@
-import type { ClickEvent, IndexEvent } from "../../webapp/app_event";
+import type { ClickEvent, IndexTreeEvent } from "../../webapp/app_event";
 
 import { batchSet, scanPrefixGlobal } from "plugos-silverbullet-syscall/index";
 import { readPage, writePage } from "plugos-silverbullet-syscall/space";
@@ -11,7 +11,7 @@ import {
   nodeAtPos,
   renderToText
 } from "../../common/tree";
-import { whiteOutQueries } from "../query/util";
+import { removeQueries } from "../query/util";
 import { applyQuery, QueryProviderEvent } from "../query/engine";
 
 export type Task = {
@@ -24,13 +24,11 @@ export type Task = {
   page?: string;
 };
 
-export async function indexTasks({ name, text }: IndexEvent) {
+export async function indexTasks({ name, tree }: IndexTreeEvent) {
   // console.log("Indexing tasks");
   let tasks: { key: string; value: Task }[] = [];
-  text = whiteOutQueries(text);
+  removeQueries(tree);
-  let mdTree = await parseMarkdown(text);
+  collectNodesOfType(tree, "Task").forEach((n) => {
-  addParentPointers(mdTree);
-  collectNodesOfType(mdTree, "Task").forEach((n) => {
     let task = n.children!.slice(1).map(renderToText).join("").trim();
     let complete = n.children![0].children![0].text! !== "[ ]";
     let value: Task = {

plugs/yarn.lock

@@ -23,11 +23,6 @@
     "chalk" "^2.0.0"
     "js-tokens" "^4.0.0"
 
-"@hmhealey/types@^6.6.0-4":
-  "integrity" "sha512-71IxVaXhrUesmLnvQQh4RtUqqhmVL+ejci4qo4R6rTWTdY77BniRtBx269uAz34wzTlAgITysN8x7MBTdt/XBg=="
-  "resolved" "https://registry.npmjs.org/@hmhealey/types/-/types-6.6.0-4.tgz"
-  "version" "6.6.0-4"
-
 "@jest/environment@^27.5.1":
   "integrity" "sha512-/WQjhPJe3/ghaol/4Bq480JKXV/Rfw8nQdN7f41fM8VDHLcxKXou6QyXAh3EFr9/bVG3x74z1NWDkP87EiY8gA=="
   "resolved" "https://registry.npmjs.org/@jest/environment/-/environment-27.5.1.tgz"
@@ -90,6 +85,11 @@
   dependencies:
     "@lezer/common" "^0.15.0"
 
+"@mattermost/types@^6.7.0-0":
+  "integrity" "sha512-mT8wJwWEp20KPo9D12y7bW7EdUHO7VhUHxr3gH8nPGapWooGcl0Ra0H3u1iCjPpqPWvp7LiodcneU0IysunYKQ=="
+  "resolved" "https://registry.npmjs.org/@mattermost/types/-/types-6.7.0-0.tgz"
+  "version" "6.7.0-0"
+
 "@sinonjs/commons@^1.7.0":
   "integrity" "sha512-xkNcLAn/wZaX14RPlwizcKicDk9G3F8m2nU3L7Ukm5zBgTwiT0wsoFAHx9Jq56fJA1z/7uKGtCRu16sOUCLIHQ=="
   "resolved" "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz"

webapp/app_event.ts

@@ -1,3 +1,5 @@
+import type { ParseTree } from "../common/tree";
+
 export type AppEvent = "page:click" | "page:complete";
 
 export type ClickEvent = {
@@ -12,3 +14,8 @@ export type IndexEvent = {
   name: string;
   text: string;
 };
+
+export type IndexTreeEvent = {
+  name: string;
+  tree: ParseTree;
+};

webapp/editor.tsx

@@ -300,7 +300,6 @@ export class Editor {
         closeBrackets(),
         autocompletion({
           override: [
-            // this.completerHook.plugCompleter.bind(this.completerHook),
             this.completer.bind(this),
             this.slashCommandHook.slashCommandCompleter.bind(
               this.slashCommandHook
@@ -320,6 +319,7 @@ export class Editor {
           { selector: "Comment", class: "line-comment" },
           { selector: "BulletList", class: "line-ul" },
           { selector: "OrderedList", class: "line-ol" },
+          { selector: "TableHeader", class: "line-tbl-header" },
         ]),
         keymap.of([
           ...smartQuoteKeymap,
@@ -433,6 +433,9 @@ export class Editor {
       editorView.setState(
         this.createEditorState(this.currentPage, editorView.state.sliceDoc())
       );
+      if (editorView.contentDOM) {
+        editorView.contentDOM.spellcheck = true;
+      }
     }
   }
 
@@ -503,6 +506,9 @@ export class Editor {
     let editorState = this.createEditorState(pageName, doc.text);
     let pageState = this.openPages.get(pageName);
     editorView.setState(editorState);
+    if (editorView.contentDOM) {
+      editorView.contentDOM.spellcheck = true;
+    }
     if (!pageState) {
       pageState = new PageState(0, editorState.selection);
       this.openPages.set(pageName, pageState!);

webapp/markdown/markdown.ts

@@ -5,17 +5,10 @@ import {
   Language,
   languageDataProp,
   LanguageDescription,
-  ParseContext,
+  ParseContext
 } from "@codemirror/language";
 import { styleTags, tags as t } from "@codemirror/highlight";
-import {
+import { Emoji, GFM, MarkdownParser, parser as baseParser, Subscript, Superscript } from "@lezer/markdown";
-  Emoji,
-  GFM,
-  MarkdownParser,
-  parser as baseParser,
-  Subscript,
-  Superscript,
-} from "@lezer/markdown";
 
 const data = defineLanguageFacet({ block: { open: "<!--", close: "-->" } });
 
@@ -37,8 +30,6 @@ export const commonmark = baseParser.configure({
       "StrongEmphasis/...": t.strong,
       "Link/... Image/...": t.link,
       "OrderedList/... BulletList/...": t.list,
-
-      // "CodeBlock/... FencedCode/...": t.blockComment,
       "InlineCode CodeText": t.monospace,
       URL: t.url,
       "HeaderMark HardBreak QuoteMark ListMark LinkMark EmphasisMark CodeMark":

webapp/parser.ts

@@ -1,5 +1,5 @@
-import { styleTags } from "@codemirror/highlight";
+import { styleTags, tags as t } from "@codemirror/highlight";
-import { BlockContext, LeafBlock, LeafBlockParser, MarkdownConfig, parseCode, TaskList } from "@lezer/markdown";
+import { BlockContext, LeafBlock, LeafBlockParser, MarkdownConfig, parseCode, Table, TaskList } from "@lezer/markdown";
 import { commonmark, getCodeParser, mkLang } from "./markdown/markdown";
 import * as ct from "./customtags";
 import { Language, LanguageDescription, LanguageSupport } from "@codemirror/language";
@@ -73,6 +73,7 @@ export default function buildMarkdown(mdExtensions: MDExt[]): Language {
       WikiLink,
       TaskList,
       Comment,
+      Table,
       ...mdExtensions.map(mdExtensionSyntaxConfig),
       parseCode({
         codeParser: getCodeParser([
@@ -96,6 +97,10 @@ export default function buildMarkdown(mdExtensions: MDExt[]): Language {
             Task: ct.TaskTag,
             TaskMarker: ct.TaskMarkerTag,
             Comment: ct.CommentTag,
+            "TableDelimiter SubscriptMark SuperscriptMark StrikethroughMark":
+              t.processingInstruction,
+            "TableHeader/...": t.heading,
+            TableCell: t.content,
           }),
           ...mdExtensions.map((mdExt) =>
             styleTags(mdExtensionStyleTags(mdExt))

webapp/style.ts

@@ -36,6 +36,8 @@ export default function highlightStyles(mdExtension: MDExt[]) {
     { tag: t.variableName, class: "variableName" },
     { tag: t.comment, class: "comment" },
     { tag: t.invalid, class: "invalid" },
+    { tag: t.processingInstruction, class: "meta" },
+    // { tag: t.content, class: "tbl-content" },
     { tag: t.punctuation, class: "punctuation" },
     ...mdExtension.map((mdExt) => {
       return { tag: mdExt.tag, ...mdExt.styles };

webapp/styles/editor.scss

@@ -58,12 +58,35 @@
   }
 
   .line-code {
-    background-color: #efefef;
+    background-color: rgba(72, 72, 72, 0.1);
-    margin-left: 30px;
+
+    .code {
+      background-color: transparent;
+    }
+  }
+
+  .line-tbl-header {
+    font-weight: bold;
+
+    .meta {
+      font-weight: normal;
+    }
+  }
+
+  .struct {
+    color: darkred;
+  }
+
+  .code {
+    background-color: rgba(72, 72, 72, 0.1);
   }
 
   .line-fenced-code {
     background-color: rgba(72, 72, 72, 0.1);
+
+    .code {
+      background-color: transparent;
+    }
   }
 
   .meta {
@@ -117,10 +140,6 @@
 
 
 
-  .code {
-    background-color: #efefef;
-  }
-
   // Indentation of follow-up lines
   @mixin lineOverflow($baseIndent) {
     text-indent: -1 * ($baseIndent + 2ch);