Major directive refactor (#195)

Fixes #188 #144 #76: major refactor of directive parsing, rendering, styling

plugs/collab/collab.ts

@@ -63,7 +63,7 @@ export async function shareCommand() {
   if (!serverUrl) {
     return;
   }
-  const roomId = nanoid();
+  const roomId = nanoid().replaceAll("_", "-");
   await editor.save();
   const text = await editor.getText();
   const tree = await markdown.parseMarkdown(text);

plugs/directive/command.ts

@@ -1,7 +1,11 @@
 import { editor, markdown, system } from "$sb/silverbullet-syscall/mod.ts";
-import { nodeAtPos } from "$sb/lib/tree.ts";
-import { replaceAsync } from "$sb/lib/util.ts";
-import { directiveRegex, renderDirectives } from "./directives.ts";
+import {
+  ParseTree,
+  removeParentPointers,
+  renderToText,
+  traverseTree,
+} from "$sb/lib/tree.ts";
+import { renderDirectives } from "./directives.ts";
 import { extractFrontmatter } from "$sb/lib/frontmatter.ts";
 
 export async function updateDirectivesOnPageCommand(arg: any) {
@@ -33,38 +37,47 @@ export async function updateDirectivesOnPageCommand(arg: any) {
   }
 
   // Collect all directives and their body replacements
-  const replacements: { fullMatch: string; text?: string }[] = [];
+  const replacements: { fullMatch: string; textPromise: Promise<string> }[] =
+    [];
+
+  // Convenience array to wait for all promises to resolve
+  const allPromises: Promise<string>[] = [];
+
+  removeParentPointers(tree);
+
+  traverseTree(tree, (tree) => {
+    if (tree.type !== "Directive") {
+      return false;
+    }
+    const fullMatch = text.substring(tree.from!, tree.to!);
+    try {
+      const promise = system.invokeFunction(
+        "server",
+        "serverRenderDirective",
+        pageName,
+        tree,
+      );
+      replacements.push({
+        textPromise: promise,
+        fullMatch,
+      });
+      allPromises.push(promise);
+    } catch (e: any) {
+      replacements.push({
+        fullMatch,
+        textPromise: Promise.resolve(
+          `${renderToText(tree.children![0])}\n**ERROR:** ${e.message}\n${
+            renderToText(tree.children![tree.children!.length - 1])
+          }`,
+        ),
+      });
+    }
+    return true;
+  });
+
+  // Wait for all to have processed
+  await Promise.all(allPromises);
 
-  await replaceAsync(
-    text,
-    directiveRegex,
-    async (fullMatch, startInst, _type, _arg, _body, endInst, index) => {
-      const replacement: { fullMatch: string; text?: string } = { fullMatch };
-      // Pushing to the replacement array
-      const currentNode = nodeAtPos(tree, index + 1);
-      if (currentNode?.type !== "CommentBlock") {
-        // If not a comment block, it's likely a code block, ignore
-        // console.log("Not comment block, ignoring", fullMatch);
-        return fullMatch;
-      }
-      replacements.push(replacement);
-      try {
-        const replacementText = await system.invokeFunction(
-          "server",
-          "serverRenderDirective",
-          pageName,
-          fullMatch,
-        );
-        replacement.text = replacementText;
-        // Return value is ignored, we're using the replacements array
-        return fullMatch;
-      } catch (e: any) {
-        replacement.text = `${startInst}\n**ERROR:** ${e.message}\n${endInst}`;
-        // Return value is ignored, we're using the replacements array
-        return fullMatch;
-      }
-    },
-  );
   // Iterate again and replace the bodies. Iterating again (not using previous positions)
   // because text may have changed in the mean time (directive processing may take some time)
   // Hypothetically in the mean time directives in text may have been changed/swapped, in which
@@ -77,6 +90,10 @@ export async function updateDirectivesOnPageCommand(arg: any) {
 
     // This may happen if the query itself, or the user is editing inside the directive block (WHY!?)
     if (index === -1) {
+      console.warn(
+        "Text I got",
+        text,
+      );
       console.warn(
         "Could not find directive in text, skipping",
         replacement.fullMatch,
@@ -84,7 +101,8 @@ export async function updateDirectivesOnPageCommand(arg: any) {
       continue;
     }
     const from = index, to = index + replacement.fullMatch.length;
-    if (text.substring(from, to) === replacement.text) {
+    const newText = await replacement.textPromise;
+    if (text.substring(from, to) === newText) {
       // No change, skip
       continue;
     }
@@ -92,21 +110,67 @@ export async function updateDirectivesOnPageCommand(arg: any) {
       changes: {
         from,
         to,
-        insert: replacement.text,
+        insert: newText,
       },
     });
   }
 }
 
-export function serverPing() {
-  return "pong";
-}
-
 // Called from client, running on server
 // The text passed here is going to be a single directive block (not a full page)
 export function serverRenderDirective(
   pageName: string,
-  text: string,
+  tree: ParseTree,
 ): Promise<string> {
-  return renderDirectives(pageName, text);
+  return renderDirectives(pageName, tree);
+}
+
+// Pure server driven implementation of directive updating
+export async function serverUpdateDirectives(
+  pageName: string,
+  text: string,
+) {
+  const tree = await markdown.parseMarkdown(text);
+  // Collect all directives and their body replacements
+  const replacements: { fullMatch: string; textPromise: Promise<string> }[] =
+    [];
+
+  const allPromises: Promise<string>[] = [];
+
+  traverseTree(tree, (tree) => {
+    if (tree.type !== "Directive") {
+      return false;
+    }
+    const fullMatch = text.substring(tree.from!, tree.to!);
+    try {
+      const promise = renderDirectives(
+        pageName,
+        tree,
+      );
+      replacements.push({
+        textPromise: promise,
+        fullMatch,
+      });
+      allPromises.push(promise);
+    } catch (e: any) {
+      replacements.push({
+        fullMatch,
+        textPromise: Promise.resolve(
+          `${renderToText(tree.children![0])}\n**ERROR:** ${e.message}\n${
+            renderToText(tree.children![tree.children!.length - 1])
+          }`,
+        ),
+      });
+    }
+    return true;
+  });
+
+  // Wait for all to have processed
+  await Promise.all(allPromises);
+
+  // Iterate again and replace the bodies.
+  for (const replacement of replacements) {
+    text = text.replace(replacement.fullMatch, await replacement.textPromise);
+  }
+  return text;
 }

plugs/directive/directives.ts

@@ -1,4 +1,4 @@
-import { nodeAtPos, ParseTree } from "$sb/lib/tree.ts";
+import { nodeAtPos, ParseTree, renderToText } from "$sb/lib/tree.ts";
 import { replaceAsync } from "$sb/lib/util.ts";
 import { markdown } from "$sb/silverbullet-syscall/mod.ts";
 
@@ -9,56 +9,72 @@ import {
   templateDirectiveRenderer,
 } from "./template_directive.ts";
 
+export const directiveStartRegex =
+  /<!--\s*#(use|use-verbose|include|eval|query)\s+(.*?)-->/i;
+
 export const directiveRegex =
   /(<!--\s*#(use|use-verbose|include|eval|query)\s+(.*?)-->)(.+?)(<!--\s*\/\2\s*-->)/gs;
-
 /**
  * Looks for directives in the text dispatches them based on name
  */
-export function directiveDispatcher(
+export async function directiveDispatcher(
   pageName: string,
-  text: string,
-  tree: ParseTree,
+  directiveTree: ParseTree,
   directiveRenderers: Record<
     string,
-    (directive: string, pageName: string, arg: string) => Promise<string>
+    (
+      directive: string,
+      pageName: string,
+      arg: string | ParseTree,
+    ) => Promise<string>
   >,
 ): Promise<string> {
-  return replaceAsync(
-    text,
-    directiveRegex,
-    async (fullMatch, startInst, type, arg, _body, endInst, index) => {
-      const currentNode = nodeAtPos(tree, index + 1);
-      // console.log("Node type", currentNode?.type);
-      if (currentNode?.type !== "CommentBlock") {
-        // If not a comment block, it's likely a code block, ignore
-        // console.log("Not comment block, ingoring", fullMatch);
-        return fullMatch;
-      }
+  const directiveStart = directiveTree.children![0]; // <!-- #directive -->
+  const directiveEnd = directiveTree.children![2]; // <!-- /directive -->
+
+  const directiveStartText = renderToText(directiveStart).trim();
+  const directiveEndText = renderToText(directiveEnd).trim();
+
+  if (directiveStart.children!.length === 1) {
+    // Everything not #query
+    const match = directiveStartRegex.exec(directiveStart.children![0].text!);
+    if (!match) {
+      throw Error("No match");
+    }
+
+    let [_fullMatch, type, arg] = match;
+    try {
       arg = arg.trim();
-      try {
-        const newBody = await directiveRenderers[type](type, pageName, arg);
-        return `${startInst}\n${newBody.trim()}\n${endInst}`;
-      } catch (e: any) {
-        return `${startInst}\n**ERROR:** ${e.message}\n${endInst}`;
-      }
-    },
-  );
+      const newBody = await directiveRenderers[type](type, pageName, arg);
+      const result =
+        `${directiveStartText}\n${newBody.trim()}\n${directiveEndText}`;
+      return result;
+    } catch (e: any) {
+      return `${directiveStartText}\n**ERROR:** ${e.message}\n${directiveEndText}`;
+    }
+  } else {
+    // #query
+    const newBody = await directiveRenderers["query"](
+      "query",
+      pageName,
+      directiveStart.children![1], // The query ParseTree
+    );
+    const result =
+      `${directiveStartText}\n${newBody.trim()}\n${directiveEndText}`;
+    return result;
+  }
 }
 
 export async function renderDirectives(
   pageName: string,
-  text: string,
+  directiveTree: ParseTree,
 ): Promise<string> {
-  const tree = await markdown.parseMarkdown(text);
-
-  text = await directiveDispatcher(pageName, text, tree, {
+  const replacementText = await directiveDispatcher(pageName, directiveTree, {
     use: templateDirectiveRenderer,
-    "use-verbose": templateDirectiveRenderer,
-    "include": templateDirectiveRenderer,
+    include: templateDirectiveRenderer,
     query: queryDirectiveRenderer,
     eval: evalDirectiveRenderer,
   });
 
-  return await cleanTemplateInstantiations(text);
+  return cleanTemplateInstantiations(replacementText);
 }

plugs/directive/eval_directive.ts

@@ -1,6 +1,7 @@
 // This is some shocking stuff. My profession would kill me for this.
 
 import * as YAML from "yaml";
+import { ParseTree } from "../../plug-api/lib/tree.ts";
 import { jsonToMDTable, renderTemplate } from "./util.ts";
 
 // Enables plugName.functionName(arg1, arg2) syntax in JS expressions
@@ -20,8 +21,11 @@ const expressionRegex = /(.+?)(\s+render\s+\[\[([^\]]+)\]\])?$/;
 export async function evalDirectiveRenderer(
   _directive: string,
   _pageName: string,
-  expression: string,
+  expression: string | ParseTree,
 ): Promise<string> {
+  if (typeof expression !== "string") {
+    throw new Error("Expected a string");
+  }
   console.log("Got JS expression", expression);
   const match = expressionRegex.exec(expression);
   if (!match) {

plugs/directive/parse-query.js

@@ -1,16 +0,0 @@
-// This file was generated by lezer-generator. You probably shouldn't edit it.
-import {LRParser} from "@lezer/lr"
-export const parser = LRParser.deserialize({
-  version: 14,
-  states: "&fOVQPOOOmQQO'#C^QOQPOOOtQPO'#C`OyQQO'#CkO!OQPO'#CmO!TQPO'#CnO!YQPO'#CoOOQO'#Cq'#CqO!bQQO,58xO!iQQO'#CcO#WQQO'#CaOOQO'#Ca'#CaOOQO,58z,58zO#oQPO,59VOOQO,59X,59XO#tQQO'#DaOOQO,59Y,59YOOQO,59Z,59ZOOQO-E6o-E6oO$]QQO,58}OtQPO,58|O$tQQO1G.qO%`QPO'#CsO%eQQO,59{OOQO'#Cg'#CgOOQO'#Ci'#CiO$]QQO'#CjOOQO'#Cd'#CdOOQO1G.i1G.iOOQO1G.h1G.hOOQO'#Cl'#ClOOQO7+$]7+$]OOQO,59_,59_OOQO-E6q-E6qO%|QPO'#C}O&UQPO,59UO$]QQO'#CrO&ZQPO,59iOOQO1G.p1G.pOOQO,59^,59^OOQO-E6p-E6p",
-  stateData: "&c~OjOS~ORPO~OkRO}SO!RTO!SUO!UVO~OhQX~P[ORYO~O!O^O~OX_O~OR`O~OYbOdbO~OhQa~P[OldOtdOudOvdOwdOxdOydOzdO{dO~O|eOhTXkTX}TX!RTX!STX!UTX~ORfO~OrgOh!TXk!TX}!TX!R!TX!S!TX!U!TX~OXlOYlO[lOmiOniOojOpkO~O!PoO!QoOh_ik_i}_i!R_i!S_i!U_i~ORqO~OrgOh!Tak!Ta}!Ta!R!Ta!S!Ta!U!Ta~OruOsqX~OswO~OruOsqa~O",
-  goto: "#e!UPP!VP!Y!^!a!d!jPP!sP!s!s!Y!x!Y!Y!YP!{#R#XPPPPPPPPP#_PPPPPPPPPPPPPPPPP#bRQOTWPXR]RR[RQZRRneQmdQskRxuVldkuRpfQXPRcXQvsRyvQh`RrhRtkRaU",
-  nodeNames: "⚠ Program Query Name WhereClause LogicalExpr AndExpr FilterExpr Value Number String Bool Regex Null List OrderClause Order LimitClause SelectClause RenderClause PageRef",
-  maxTerm: 52,
-  skippedNodes: [0],
-  repeatNodeCount: 3,
-  tokenData: "B[~R}X^$Opq$Oqr$srs%W|}%r}!O%w!P!Q&Y!Q!['P!^!_'X!_!`'f!`!a's!c!}%w!}#O(Q#P#Q(q#R#S%w#T#U(v#U#V+]#V#W%w#W#X,X#X#Y%w#Y#Z.T#Z#]%w#]#^0e#^#`%w#`#a1a#a#b%w#b#c3t#c#d5p#d#f%w#f#g8T#g#h;P#h#i={#i#k%w#k#l?w#l#o%w#y#z$O$f$g$O#BY#BZ$O$IS$I_$O$Ip$Iq%W$Iq$Ir%W$I|$JO$O$JT$JU$O$KV$KW$O&FU&FV$O~$TYj~X^$Opq$O#y#z$O$f$g$O#BY#BZ$O$IS$I_$O$I|$JO$O$JT$JU$O$KV$KW$O&FU&FV$O~$vP!_!`$y~%OPv~#r#s%R~%WOz~~%ZUOr%Wrs%ms$Ip%W$Ip$Iq%m$Iq$Ir%m$Ir~%W~%rOY~~%wOr~P%|SRP}!O%w!c!}%w#R#S%w#T#o%w~&_V[~OY&YZ]&Y^!P&Y!P!Q&t!Q#O&Y#O#P&y#P~&Y~&yO[~~&|PO~&Y~'UPX~!Q!['P~'^Pl~!_!`'a~'fOt~~'kPu~#r#s'n~'sOy~~'xPx~!_!`'{~(QOw~R(VPpQ!}#O(YP(]RO#P(Y#P#Q(f#Q~(YP(iP#P#Q(lP(qOdP~(vOs~R({WRP}!O%w!c!}%w#R#S%w#T#b%w#b#c)e#c#g%w#g#h*a#h#o%wR)jURP}!O%w!c!}%w#R#S%w#T#W%w#W#X)|#X#o%wR*TS|QRP}!O%w!c!}%w#R#S%w#T#o%wR*fURP}!O%w!c!}%w#R#S%w#T#V%w#V#W*x#W#o%wR+PS!QQRP}!O%w!c!}%w#R#S%w#T#o%wR+bURP}!O%w!c!}%w#R#S%w#T#m%w#m#n+t#n#o%wR+{S!OQRP}!O%w!c!}%w#R#S%w#T#o%wR,^URP}!O%w!c!}%w#R#S%w#T#X%w#X#Y,p#Y#o%wR,uURP}!O%w!c!}%w#R#S%w#T#g%w#g#h-X#h#o%wR-^URP}!O%w!c!}%w#R#S%w#T#V%w#V#W-p#W#o%wR-wS!PQRP}!O%w!c!}%w#R#S%w#T#o%wR.YTRP}!O%w!c!}%w#R#S%w#T#U.i#U#o%wR.nURP}!O%w!c!}%w#R#S%w#T#`%w#`#a/Q#a#o%wR/VURP}!O%w!c!}%w#R#S%w#T#g%w#g#h/i#h#o%wR/nURP}!O%w!c!}%w#R#S%w#T#X%w#X#Y0Q#Y#o%wR0XSnQRP}!O%w!c!}%w#R#S%w#T#o%wR0jURP}!O%w!c!}%w#R#S%w#T#b%w#b#c0|#c#o%wR1TS{QRP}!O%w!c!}%w#R#S%w#T#o%wR1fURP}!O%w!c!}%w#R#S%w#T#]%w#]#^1x#^#o%wR1}URP}!O%w!c!}%w#R#S%w#T#a%w#a#b2a#b#o%wR2fURP}!O%w!c!}%w#R#S%w#T#]%w#]#^2x#^#o%wR2}URP}!O%w!c!}%w#R#S%w#T#h%w#h#i3a#i#o%wR3hS!RQRP}!O%w!c!}%w#R#S%w#T#o%wR3yURP}!O%w!c!}%w#R#S%w#T#i%w#i#j4]#j#o%wR4bURP}!O%w!c!}%w#R#S%w#T#`%w#`#a4t#a#o%wR4yURP}!O%w!c!}%w#R#S%w#T#`%w#`#a5]#a#o%wR5dSoQRP}!O%w!c!}%w#R#S%w#T#o%wR5uURP}!O%w!c!}%w#R#S%w#T#f%w#f#g6X#g#o%wR6^URP}!O%w!c!}%w#R#S%w#T#W%w#W#X6p#X#o%wR6uURP}!O%w!c!}%w#R#S%w#T#X%w#X#Y7X#Y#o%wR7^URP}!O%w!c!}%w#R#S%w#T#f%w#f#g7p#g#o%wR7wS}QRP}!O%w!c!}%w#R#S%w#T#o%wR8YURP}!O%w!c!}%w#R#S%w#T#X%w#X#Y8l#Y#o%wR8qURP}!O%w!c!}%w#R#S%w#T#b%w#b#c9T#c#o%wR9YURP}!O%w!c!}%w#R#S%w#T#W%w#W#X9l#X#o%wR9qURP}!O%w!c!}%w#R#S%w#T#X%w#X#Y:T#Y#o%wR:YURP}!O%w!c!}%w#R#S%w#T#f%w#f#g:l#g#o%wR:sS!UQRP}!O%w!c!}%w#R#S%w#T#o%wR;UURP}!O%w!c!}%w#R#S%w#T#X%w#X#Y;h#Y#o%wR;mURP}!O%w!c!}%w#R#S%w#T#`%w#`#a<P#a#o%wR<UURP}!O%w!c!}%w#R#S%w#T#X%w#X#Y<h#Y#o%wR<mURP}!O%w!c!}%w#R#S%w#T#V%w#V#W=P#W#o%wR=UURP}!O%w!c!}%w#R#S%w#T#h%w#h#i=h#i#o%wR=oS!SQRP}!O%w!c!}%w#R#S%w#T#o%wR>QURP}!O%w!c!}%w#R#S%w#T#f%w#f#g>d#g#o%wR>iURP}!O%w!c!}%w#R#S%w#T#i%w#i#j>{#j#o%wR?QURP}!O%w!c!}%w#R#S%w#T#X%w#X#Y?d#Y#o%wR?kSmQRP}!O%w!c!}%w#R#S%w#T#o%wR?|URP}!O%w!c!}%w#R#S%w#T#[%w#[#]@`#]#o%wR@eURP}!O%w!c!}%w#R#S%w#T#X%w#X#Y@w#Y#o%wR@|URP}!O%w!c!}%w#R#S%w#T#f%w#f#gA`#g#o%wRAeURP}!O%w!c!}%w#R#S%w#T#X%w#X#YAw#Y#o%wRBOSkQRP}!O%w!c!}%w#R#S%w#T#o%w",
-  tokenizers: [0, 1],
-  topRules: {"Program":[0,1]},
-  tokenPrec: 0
-})

plugs/directive/parse-query.terms.js

@@ -1,22 +0,0 @@
-// This file was generated by lezer-generator. You probably shouldn't edit it.
-export const
-  Program = 1,
-  Query = 2,
-  Name = 3,
-  WhereClause = 4,
-  LogicalExpr = 5,
-  AndExpr = 6,
-  FilterExpr = 7,
-  Value = 8,
-  Number = 9,
-  String = 10,
-  Bool = 11,
-  Regex = 12,
-  Null = 13,
-  List = 14,
-  OrderClause = 15,
-  Order = 16,
-  LimitClause = 17,
-  SelectClause = 18,
-  RenderClause = 19,
-  PageRef = 20

plugs/directive/parser.ts

@@ -4,16 +4,14 @@ import {
   ParseTree,
   replaceNodesMatching,
 } from "$sb/lib/tree.ts";
-import { lezerToParseTree } from "../../common/parse_tree.ts";
 
 // @ts-ignore auto generated
-import { parser } from "./parse-query.js";
 import { ParsedQuery, QueryFilter } from "$sb/lib/query.ts";
 
-export function parseQuery(query: string): ParsedQuery {
-  const n = lezerToParseTree(query, parser.parse(query).topNode);
+export function parseQuery(queryTree: ParseTree): ParsedQuery {
+  // const n = lezerToParseTree(query, parser.parse(query).topNode);
   // Clean the tree a bit
-  replaceNodesMatching(n, (n) => {
+  replaceNodesMatching(queryTree, (n) => {
     if (!n.type) {
       const trimmed = n.text!.trim();
       if (!trimmed) {
@@ -24,7 +22,7 @@ export function parseQuery(query: string): ParsedQuery {
   });
 
   // console.log("Parsed", JSON.stringify(n, null, 2));
-  const queryNode = n.children![0];
+  const queryNode = queryTree.children![0];
   const parsedQuery: ParsedQuery = {
     table: queryNode.children![0].children![0].text!,
     filter: [],
@@ -33,7 +31,7 @@ export function parseQuery(query: string): ParsedQuery {
   if (orderByNode) {
     const nameNode = findNodeOfType(orderByNode, "Name");
     parsedQuery.orderBy = nameNode!.children![0].text!;
-    const orderNode = findNodeOfType(orderByNode, "Order");
+    const orderNode = findNodeOfType(orderByNode, "OrderDirection");
     parsedQuery.orderDesc = orderNode
       ? orderNode.children![0].text! === "desc"
       : false;

plugs/directive/query.grammar

@@ -1,65 +0,0 @@
-@precedence { logic @left }
-
-@top Program { Query }
-
-Query {
-    Name ( WhereClause | OrderClause  | LimitClause | SelectClause | RenderClause )*
-}
-
-commaSep<content> { content ("," content)* }
-
-WhereClause { "where" LogicalExpr }
-OrderClause { "order" "by" Name Order? }
-LimitClause { "limit" Number }
-SelectClause { "select" commaSep<Name> }
-RenderClause { "render" (PageRef | String) }
-
-Order {
-  "desc" | "asc"
-}
-
-Value { Number | String | Bool | Regex | Null | List }
-
-LogicalExpr { AndExpr | FilterExpr }
-
-AndExpr { FilterExpr !logic "and" FilterExpr }
-
-FilterExpr {
-  Name "<" Value
-| Name "<=" Value
-| Name "=" Value
-| Name "!=" Value
-| Name ">=" Value
-| Name ">" Value
-| Name "=~" Value
-| Name "!=~" Value
-| Name "in" Value
-}
-
-List { "[" commaSep<Value> "]" }
-
-@skip { space }
-
-
-
-Bool {
-  "true" | "false"
-}
-
-Null {
-  "null"
-}
-
-@tokens {
-  space { std.whitespace+ }
-  Name { (std.asciiLetter | "-" | "_")+ }
-  String {
-    ("\"" | "“" | "”") ![\"”“]* ("\"" | "“" | "”")
-  }
-  PageRef {
-    "[" "[" ![\]]* "]" "]"
-  }
-  Regex { "/" ( ![/\\\n\r] | "\\" _ )* "/"? }
-
-  Number { std.digit+ }
-}

plugs/directive/query.test.ts

@@ -1,6 +1,22 @@
 import { assertEquals } from "../../test_deps.ts";
 import { applyQuery } from "$sb/lib/query.ts";
-import { parseQuery } from "./parser.ts";
+
+import wikiMarkdownLang from "../../common/markdown_parser/parser.ts";
+import { parse } from "../../common/markdown_parser/parse_tree.ts";
+import { parseQuery as parseQueryQuery } from "./parser.ts";
+import { findNodeOfType, renderToText } from "../../plug-api/lib/tree.ts";
+
+function parseQuery(query: string) {
+  const lang = wikiMarkdownLang([]);
+  const mdTree = parse(
+    lang,
+    `<!-- #query ${query} -->
+  
+  <!-- /query -->`,
+  );
+  const programNode = findNodeOfType(mdTree, "Program")!;
+  return parseQueryQuery(programNode);
+}
 
 Deno.test("Test parser", () => {
   const parsedBasicQuery = parseQuery(`page`);
@@ -154,3 +170,14 @@ Deno.test("Test applyQuery with multi value", () => {
     ],
   );
 });
+
+const testQuery = `<!-- #query source where a = 1 and b = "2" and c = "3" -->
+
+<!-- /query -->`;
+
+Deno.test("Query parsing and serialization", () => {
+  const lang = wikiMarkdownLang([]);
+  const mdTree = parse(lang, testQuery);
+  // console.log(JSON.stringify(mdTree, null, 2));
+  assertEquals(renderToText(mdTree), testQuery);
+});

plugs/directive/query_directive.ts

@@ -4,15 +4,21 @@ import { replaceTemplateVars } from "../core/template.ts";
 import { renderTemplate } from "./util.ts";
 import { parseQuery } from "./parser.ts";
 import { jsonToMDTable } from "./util.ts";
+import { ParseTree } from "../../plug-api/lib/tree.ts";
 
 export async function queryDirectiveRenderer(
   _directive: string,
   pageName: string,
-  query: string,
+  query: string | ParseTree,
 ): Promise<string> {
-  const parsedQuery = parseQuery(replaceTemplateVars(query, pageName));
+  if (typeof query === "string") {
+    throw new Error("Argument must be a ParseTree");
+  }
+  const parsedQuery = parseQuery(
+    JSON.parse(replaceTemplateVars(JSON.stringify(query), pageName)),
+  );
 
-  console.log("Parsed query", parsedQuery);
+  // console.log("Parsed query", parsedQuery);
   // Let's dispatch an event and see what happens
   const results = await events.dispatchEvent(
     `query:${parsedQuery.table}`,

plugs/directive/template_directive.ts

@@ -1,20 +1,24 @@
 import { queryRegex } from "$sb/lib/query.ts";
-import { renderToText } from "$sb/lib/tree.ts";
+import { ParseTree, renderToText } from "$sb/lib/tree.ts";
 import { replaceAsync } from "$sb/lib/util.ts";
 import { markdown, space } from "$sb/silverbullet-syscall/mod.ts";
 import Handlebars from "handlebars";
 
 import { replaceTemplateVars } from "../core/template.ts";
 import { extractFrontmatter } from "$sb/lib/frontmatter.ts";
-import { directiveRegex, renderDirectives } from "./directives.ts";
+import { directiveRegex } from "./directives.ts";
+import { serverUpdateDirectives } from "./command.ts";
 
 const templateRegex = /\[\[([^\]]+)\]\]\s*(.*)\s*/;
 
 export async function templateDirectiveRenderer(
   directive: string,
   pageName: string,
-  arg: string,
+  arg: string | ParseTree,
 ): Promise<string> {
+  if (typeof arg !== "string") {
+    throw new Error("Template directives must be a string");
+  }
   const match = arg.match(templateRegex);
   if (!match) {
     throw new Error(`Invalid template directive: ${arg}`);
@@ -42,7 +46,7 @@ export async function templateDirectiveRenderer(
   }
   let newBody = templateText;
   // if it's a template injection (not a literal "include")
-  if (directive === "use" || directive === "use-verbose") {
+  if (directive === "use") {
     const tree = await markdown.parseMarkdown(templateText);
     extractFrontmatter(tree, ["$disableDirectives"]);
     templateText = renderToText(tree);
@@ -53,7 +57,7 @@ export async function templateDirectiveRenderer(
     newBody = templateFn(parsedArgs);
 
     // Recursively render directives
-    newBody = await renderDirectives(pageName, newBody);
+    newBody = await serverUpdateDirectives(pageName, newBody);
   }
   return newBody.trim();
 }

plugs/global.plug.yaml

@@ -2,4 +2,3 @@ name: global
 dependencies:
   "https://esm.sh/handlebars": "https://esm.sh/handlebars@4.7.7"
   "https://deno.land/std/encoding/yaml.ts": "https://deno.land/std@0.165.0/encoding/yaml.ts"
-  "https://esm.sh/@lezer/lr": "https://esm.sh/@lezer/lr@1.2.3"

plugs/markdown/markdown_render.test.ts

@@ -1,6 +1,5 @@
-import buildMarkdown from "../../common/parser.ts";
-import { parse } from "../../common/parse_tree.ts";
-import { renderHtml } from "./html_render.ts";
+import buildMarkdown from "../../common/markdown_parser/parser.ts";
+import { parse } from "../../common/markdown_parser/parse_tree.ts";
 import { System } from "../../plugos/system.ts";
 
 import corePlug from "../../dist_bundle/_plug/core.plug.json" assert {
@@ -10,9 +9,9 @@ import tasksPlug from "../../dist_bundle/_plug/tasks.plug.json" assert {
   type: "json",
 };
 import { createSandbox } from "../../plugos/environments/deno_sandbox.ts";
-import { loadMarkdownExtensions } from "../../common/markdown_ext.ts";
+import { loadMarkdownExtensions } from "../../common/markdown_parser/markdown_ext.ts";
 import { renderMarkdownToHtml } from "./markdown_render.ts";
-import { assertEquals } from "https://deno.land/std@0.165.0/testing/asserts.ts";
+import { assertEquals } from "../../test_deps.ts";
 
 Deno.test("Markdown render", async () => {
   const system = new System<any>("server");

plugs/markdown/markdown_render.ts

@@ -362,6 +362,10 @@ function render(
         body: cleanTags(mapRender(newChildren)),
       };
     }
+    case "Directive": {
+      const body = findNodeOfType(t, "DirectiveBody")!;
+      return posPreservingRender(body.children![0], options);
+    }
     // Text
     case undefined:
       return t.text!;

plugs/tasks/task.ts

@@ -40,9 +40,9 @@ function getDeadline(deadlineNode: ParseTree): string {
 }
 
 export async function indexTasks({ name, tree }: IndexTreeEvent) {
-  // console.log("Indexing tasks");
   const tasks: { key: string; value: Task }[] = [];
   removeQueries(tree);
+  addParentPointers(tree);
   collectNodesOfType(tree, "Task").forEach((n) => {
     const complete = n.children![0].children![0].text! !== "[ ]";
     const task: Task = {
@@ -78,7 +78,6 @@ export async function indexTasks({ name, tree }: IndexTreeEvent) {
       key: `task:${n.from}`,
       value: task,
     });
-    // console.log("Task", task);
   });
 
   // console.log("Found", tasks.length, "task(s)");
@@ -132,7 +131,6 @@ async function toggleTaskMarker(node: ParseTree, moveToPos: number) {
       }
       taskMarkerNode.children![0].text = changeTo;
       text = renderToText(referenceMdTree);
-      console.log("Updated reference paged text", text);
       await space.writePage(page, text);
     }
   }