Rewrote all plugs using MarkdownTree

plugs/core/item.ts

@@ -1,40 +1,49 @@
-import { IndexEvent } from "../../webapp/app_event";
-import { whiteOutQueries } from "./materialized_queries";
+import {IndexEvent} from "../../webapp/app_event";
+import {whiteOutQueries} from "./materialized_queries";
 
-import { batchSet } from "plugos-silverbullet-syscall/index";
+import {batchSet} from "plugos-silverbullet-syscall/index";
+import {parseMarkdown} from "plugos-silverbullet-syscall/markdown";
+import {collectNodesMatching, MarkdownTree, render} from "../lib/tree";
 
 type Item = {
   item: string;
-  children?: string[];
+  nested?: string;
 };
 
-const pageRefRe = /\[\[[^\]]+@\d+\]\]/;
-const itemFullRe =
-  /(?<prefix>[\t ]*)[\-\*]\s*([^\n]+)(\n\k<prefix>\s+[\-\*][^\n]+)*/g;
-
 export async function indexItems({ name, text }: IndexEvent) {
   let items: { key: string; value: Item }[] = [];
   text = whiteOutQueries(text);
-  for (let match of text.matchAll(itemFullRe)) {
-    let entire = match[0];
-    let item = match[2];
-    if (item.match(pageRefRe)) {
-      continue;
-    }
-    let pos = match.index!;
-    let lines = entire.split("\n");
 
+  console.log("Indexing items", name);
+  let mdTree = await parseMarkdown(text);
+
+  let coll = collectNodesMatching(mdTree, (n) => n.type === "ListItem");
+
+  coll.forEach((n) => {
+    if (!n.children) {
+      return;
+    }
+    let textNodes: MarkdownTree[] = [];
+    let nested: string | undefined;
+    for (let child of n.children!.slice(1)) {
+      if (child.type === "OrderedList" || child.type === "BulletList") {
+        nested = render(child);
+        break;
+      }
+      textNodes.push(child);
+    }
+    let item = textNodes.map(render).join("").trim();
     let value: Item = {
       item,
     };
-    if (lines.length > 1) {
-      value.children = lines.slice(1);
+    if (nested) {
+      value.nested = nested;
     }
     items.push({
-      key: `it:${pos}`,
+      key: `it:${n.from}`,
       value,
     });
-  }
+  });
   console.log("Found", items.length, "item(s)");
   await batchSet(name, items);
 }

plugs/core/materialized_queries.ts

@@ -49,13 +49,13 @@ export async function updateMaterializedQueriesOnPage(pageName: string) {
         for (let {
           key,
           page,
-          value: { task, complete, children },
+          value: { task, complete, nested },
         } of await scanPrefixGlobal("task:")) {
           let [, pos] = key.split(":");
           if (!filter || (filter && task.includes(filter))) {
             results.push(
               `* [${complete ? "x" : " "}] [[${page}@${pos}]] ${task}` +
-                (children ? "\n" + children.join("\n") : "")
+                (nested ? "\n  " + nested : "")
             );
           }
         }
@@ -78,13 +78,12 @@ export async function updateMaterializedQueriesOnPage(pageName: string) {
         for (let {
           key,
           page,
-          value: { item, children },
+          value: { item, nested },
         } of await scanPrefixGlobal("it:")) {
           let [, pos] = key.split(":");
           if (!filter || (filter && item.includes(filter))) {
             results.push(
-              `* [[${page}@${pos}]] ${item}` +
-                (children ? "\n" + children.join("\n") : "")
+              `* [[${page}@${pos}]] ${item}` + (nested ? "\n  " + nested : "")
             );
           }
         }

plugs/core/navigate.ts

@@ -2,7 +2,8 @@ import {ClickEvent} from "../../webapp/app_event";
 import {updateMaterializedQueriesCommand} from "./materialized_queries";
 import {getCursor, getText, navigate as navigateTo, openUrl,} from "plugos-silverbullet-syscall/editor";
 import {taskToggleAtPos} from "../tasks/task";
-import {nodeAtPos, parse} from "plugos-silverbullet-syscall/markdown";
+import {parseMarkdown} from "plugos-silverbullet-syscall/markdown";
+import {nodeAtPos} from "../lib/tree";
 import type {MarkdownTree} from "../../common/tree";
 
 const materializedQueryPrefix = /<!--\s*#query\s+/;
@@ -39,16 +40,15 @@ async function actionClickOrActionEnter(mdTree: MarkdownTree | null) {
 }
 
 export async function linkNavigate() {
-  let mdTree = await parse(await getText());
+  let mdTree = await parseMarkdown(await getText());
   let newNode = await nodeAtPos(mdTree, await getCursor());
   await actionClickOrActionEnter(newNode);
 }
 
 export async function clickNavigate(event: ClickEvent) {
   if (event.ctrlKey || event.metaKey) {
-    let mdTree = await parse(await getText());
-    let newNode = await nodeAtPos(mdTree, event.pos);
-    console.log("New node", newNode);
+    let mdTree = await parseMarkdown(await getText());
+    let newNode = nodeAtPos(mdTree, event.pos);
     await actionClickOrActionEnter(newNode);
   }
 }

plugs/lib/tree.test.ts

@@ -0,0 +1,52 @@
+import {expect, test} from "@jest/globals";
+import {parse} from "../../common/tree";
+import {addParentPointers, collectNodesMatching, findParentMatching, nodeAtPos, render,} from "./tree";
+
+const mdTest1 = `
+# Heading
+## Sub _heading_ cool
+
+Hello, this is some **bold** text and *italic*. And [a link](http://zef.me).
+
+%% My comment here
+%% And second line
+
+And an @mention
+
+http://zef.plus
+
+- This is a list [[PageLink]]
+- With another item
+- TODOs:
+  - [ ] A task that's not yet done
+  - [x] Hello
+- And a _third_ one [[Wiki Page]] yo
+`;
+
+const mdTest2 = `
+Hello
+
+* Item 1
+*
+
+Sup`;
+
+test("Run a Node sandbox", async () => {
+  let mdTree = parse(mdTest1);
+  addParentPointers(mdTree);
+  // console.log(JSON.stringify(mdTree, null, 2));
+  let wikiLink = nodeAtPos(mdTree, mdTest1.indexOf("Wiki Page"))!;
+  expect(wikiLink.type).toBe("WikiLink");
+  expect(
+    findParentMatching(wikiLink, (n) => n.type === "BulletList")
+  ).toBeDefined();
+
+  let allTodos = collectNodesMatching(mdTree, (n) => n.type === "Task");
+  expect(allTodos.length).toBe(2);
+
+  // Render back into markdown should be equivalent
+  expect(render(mdTree)).toBe(mdTest1);
+
+  let mdTree2 = parse(mdTest2);
+  console.log(JSON.stringify(mdTree2, null, 2));
+});

plugs/lib/tree.ts

@@ -0,0 +1,101 @@
+export type MarkdownTree = {
+  type?: string; // undefined === text node
+  from: number;
+  to: number;
+  text?: string;
+  children?: MarkdownTree[];
+  parent?: MarkdownTree;
+};
+
+export function addParentPointers(mdTree: MarkdownTree) {
+  if (!mdTree.children) {
+    return;
+  }
+  for (let child of mdTree.children) {
+    child.parent = mdTree;
+    addParentPointers(child);
+  }
+}
+
+export function removeParentPointers(mdTree: MarkdownTree) {
+  delete mdTree.parent;
+  if (!mdTree.children) {
+    return;
+  }
+  for (let child of mdTree.children) {
+    removeParentPointers(child);
+  }
+}
+
+export function findParentMatching(
+  mdTree: MarkdownTree,
+  matchFn: (mdTree: MarkdownTree) => boolean
+): MarkdownTree | null {
+  let node = mdTree.parent;
+  while (node) {
+    if (matchFn(node)) {
+      return node;
+    }
+    node = node.parent;
+  }
+  return null;
+}
+
+export function collectNodesMatching(
+  mdTree: MarkdownTree,
+  matchFn: (mdTree: MarkdownTree) => boolean
+): MarkdownTree[] {
+  if (matchFn(mdTree)) {
+    return [mdTree];
+  }
+  let results: MarkdownTree[] = [];
+  if (mdTree.children) {
+    for (let child of mdTree.children) {
+      results = [...results, ...collectNodesMatching(child, matchFn)];
+    }
+  }
+  return results;
+}
+
+export function findNodeMatching(
+  mdTree: MarkdownTree,
+  matchFn: (mdTree: MarkdownTree) => boolean
+): MarkdownTree | null {
+  return collectNodesMatching(mdTree, matchFn)[0];
+}
+
+// Finds non-text node at position
+export function nodeAtPos(
+  mdTree: MarkdownTree,
+  pos: number
+): MarkdownTree | null {
+  if (pos < mdTree.from || pos > mdTree.to) {
+    return null;
+  }
+  if (!mdTree.children) {
+    return mdTree;
+  }
+  for (let child of mdTree.children) {
+    let n = nodeAtPos(child, pos);
+    if (n && n.text !== undefined) {
+      // Got a text node, let's return its parent
+      return mdTree;
+    } else if (n) {
+      // Got it
+      return n;
+    }
+  }
+  return null;
+}
+
+// Turn MarkdownTree back into regular markdown text
+export function render(mdTree: MarkdownTree): string {
+  let pieces: string[] = [];
+  if (mdTree.text !== undefined) {
+    return mdTree.text;
+  }
+  for (let child of mdTree.children!) {
+    pieces.push(render(child));
+  }
+  return pieces.join("");
+}

plugs/package.json

@@ -2,6 +2,7 @@
   "name": "plugs",
   "version": "1.0.0",
   "dependencies": {
+    "@jest/globals": "^27.5.1",
     "plugos-silverbullet-syscall": "file:../plugos-silverbullet-syscall",
     "plugos-syscall": "file:../plugos-syscall"
   }

plugs/tasks/task.ts

@@ -1,50 +1,45 @@
-import type { ClickEvent } from "../../webapp/app_event";
-import { IndexEvent } from "../../webapp/app_event";
+import type {ClickEvent} from "../../webapp/app_event";
+import {IndexEvent} from "../../webapp/app_event";
 
-import { whiteOutQueries } from "../core/materialized_queries";
-import { batchSet } from "plugos-silverbullet-syscall/index";
-import { readPage, writePage } from "plugos-silverbullet-syscall/space";
-import {
-  dispatch,
-  getLineUnderCursor,
-  getSyntaxNodeAtPos,
-} from "plugos-silverbullet-syscall/editor";
-
-const taskFullRe =
-  /(?<prefix>[\t ]*)[\-\*]\s*\[([ Xx])\]\s*([^\n]+)(\n\k<prefix>\s+[\-\*][^\n]+)*/g;
-
-const extractPageLink = /[\-\*]\s*\[[ Xx]\]\s\[\[([^\]]+)@(\d+)\]\]\s*(.*)/;
+import {whiteOutQueries} from "../core/materialized_queries";
+import {batchSet} from "plugos-silverbullet-syscall/index";
+import {readPage, writePage} from "plugos-silverbullet-syscall/space";
+import {parseMarkdown} from "plugos-silverbullet-syscall/markdown";
+import {dispatch, getText,} from "plugos-silverbullet-syscall/editor";
+import {addParentPointers, collectNodesMatching, nodeAtPos, render,} from "../lib/tree";
 
 type Task = {
   task: string;
   complete: boolean;
   pos?: number;
-  children?: string[];
+  nested?: string;
 };
 
 export async function indexTasks({ name, text }: IndexEvent) {
   console.log("Indexing tasks");
   let tasks: { key: string; value: Task }[] = [];
   text = whiteOutQueries(text);
-  for (let match of text.matchAll(taskFullRe)) {
-    let entire = match[0];
-    let complete = match[2] !== " ";
-    let task = match[3];
-    let pos = match.index!;
-    let lines = entire.split("\n");
+  let mdTree = await parseMarkdown(text);
+  addParentPointers(mdTree);
+  collectNodesMatching(mdTree, (n) => n.type === "Task").forEach((n) => {
+    let task = n.children!.slice(1).map(render).join("").trim();
+    let complete = n.children![0].children![0].text! !== "[ ]";
 
     let value: Task = {
       task,
       complete,
     };
-    if (lines.length > 1) {
-      value.children = lines.slice(1);
+    let taskIndex = n.parent!.children!.indexOf(n);
+    let nestedItems = n.parent!.children!.slice(taskIndex + 1);
+    if (nestedItems.length > 0) {
+      value.nested = nestedItems.map(render).join("").trim();
     }
     tasks.push({
-      key: `task:${pos}`,
+      key: `task:${n.from}`,
       value,
     });
-  }
+  });
+
   console.log("Found", tasks.length, "task(s)");
   await batchSet(name, tasks);
 }
@@ -54,41 +49,55 @@ export async function taskToggle(event: ClickEvent) {
 }
 
 export async function taskToggleAtPos(pos: number) {
-  let syntaxNode = await getSyntaxNodeAtPos(pos);
-  if (syntaxNode && syntaxNode.name === "TaskMarker") {
+  let text = await getText();
+  let mdTree = await parseMarkdown(text);
+  addParentPointers(mdTree);
+
+  let node = nodeAtPos(mdTree, pos);
+  if (node && node.type === "TaskMarker") {
     let changeTo = "[x]";
-    if (syntaxNode.text === "[x]" || syntaxNode.text === "[X]") {
+    if (node.children![0].text === "[x]" || node.children![0].text === "[X]") {
       changeTo = "[ ]";
     }
     await dispatch({
       changes: {
-        from: syntaxNode.from,
-        to: syntaxNode.to,
+        from: node.from,
+        to: node.to,
         insert: changeTo,
       },
       selection: {
         anchor: pos,
       },
     });
-    // In case there's a page reference with @ position in the task, let's propagate this change back to that page
-    // Example: * [ ] [[MyPage@123]] My task
-    let line = await getLineUnderCursor();
-    let match = line.match(extractPageLink);
-    if (match) {
-      console.log("Found a remote task reference, updating other page");
-      let [, page, posS] = match;
-      let pos = +posS;
-      let pageData = await readPage(page);
-      let text = pageData.text;
 
-      // Apply the toggle
-      text =
-        text.substring(0, pos) +
-        text
-          .substring(pos)
-          .replace(/^(\s*[\-\*]\s*)\[[ xX]\]/, "$1" + changeTo);
+    let parentWikiLinks = collectNodesMatching(
+      node.parent!,
+      (n) => n.type === "WikiLinkPage"
+    );
+    for (let wikiLink of parentWikiLinks) {
+      let ref = wikiLink.children![0].text!;
+      if (ref.includes("@")) {
+        let [page, pos] = ref.split("@");
+        let pageData = await readPage(page);
+        let text = pageData.text;
 
-      await writePage(page, text);
+        let referenceMdTree = await parseMarkdown(text);
+        // Adding +1 to immediately hit the task marker
+        let taskMarkerNode = nodeAtPos(referenceMdTree, +pos + 1);
+
+        if (!taskMarkerNode || taskMarkerNode.type !== "TaskMarker") {
+          console.error(
+            "Reference not a task marker, out of date?",
+            taskMarkerNode
+          );
+          return;
+        }
+        taskMarkerNode.children![0].text = changeTo;
+        console.log("This will be the new marker", render(taskMarkerNode));
+        text = render(referenceMdTree);
+        console.log("Updated reference paged text", text);
+        await writePage(page, text);
+      }
     }
   }
 }