Complete redo of content indexing and querying (#517)

Complete redo of data store
Introduces live queries and live templates

plugs/directive/command.ts

@@ -1,5 +1,8 @@
 import { editor, markdown, mq, space, sync } from "$sb/syscalls.ts";
 import {
+  addParentPointers,
+  findParentMatching,
+  nodeAtPos,
   ParseTree,
   removeParentPointers,
   renderToText,
@@ -7,9 +10,8 @@ import {
 } from "$sb/lib/tree.ts";
 import { renderDirectives } from "./directives.ts";
 import { extractFrontmatter } from "$sb/lib/frontmatter.ts";
-import type { PageMeta } from "../../web/types.ts";
 import { isFederationPath } from "$sb/lib/resolve.ts";
-import { MQMessage } from "$sb/types.ts";
+import { MQMessage, PageMeta } from "$sb/types.ts";
 import { sleep } from "$sb/lib/async.ts";
 
 const directiveUpdateQueueName = "directiveUpdateQueue";
@@ -200,3 +202,30 @@ export async function updateDirectives(
   }
   return text;
 }
+
+export async function convertToLiveQuery() {
+  const text = await editor.getText();
+  const pos = await editor.getCursor();
+  const tree = await markdown.parseMarkdown(text);
+  addParentPointers(tree);
+  const currentNode = nodeAtPos(tree, pos);
+  const directive = findParentMatching(
+    currentNode!,
+    (node) => node.type === "Directive",
+  );
+  if (!directive) {
+    await editor.flashNotification(
+      "No directive found at cursor position",
+      "error",
+    );
+    return;
+  }
+  const queryText = renderToText(directive!.children![0].children![1]);
+  await editor.dispatch({
+    changes: {
+      from: directive.from,
+      to: directive.to,
+      insert: "```query\n" + queryText + "\n```",
+    },
+  });
+}

plugs/directive/complete.ts

@@ -1,11 +1,11 @@
 import { events } from "$sb/syscalls.ts";
 import { CompleteEvent } from "$sb/app_event.ts";
 import { buildHandebarOptions } from "./util.ts";
-import type { PageMeta } from "../../web/types.ts";
 import type {
   AttributeCompleteEvent,
   AttributeCompletion,
 } from "../index/attributes.ts";
+import { PageMeta } from "$sb/types.ts";
 
 export async function queryComplete(completeEvent: CompleteEvent) {
   const querySourceMatch = /#query\s+([\w\-_]*)$/.exec(
@@ -14,18 +14,31 @@ export async function queryComplete(completeEvent: CompleteEvent) {
   if (querySourceMatch) {
     const allEvents = await events.listEvents();
 
+    const completionOptions = allEvents
+      .filter((eventName) =>
+        eventName.startsWith("query:") && !eventName.includes("*")
+      )
+      .map((source) => ({
+        label: source.substring("query:".length),
+      }));
+
+    const allObjectTypes: string[] = (await events.dispatchEvent("query_", {}))
+      .flat();
+
+    for (const type of allObjectTypes) {
+      completionOptions.push({
+        label: type,
+      });
+    }
+
     return {
       from: completeEvent.pos - querySourceMatch[1].length,
-      options: allEvents
-        .filter((eventName) => eventName.startsWith("query:"))
-        .map((source) => ({
-          label: source.substring("query:".length),
-        })),
+      options: completionOptions,
     };
   }
 
   if (completeEvent.parentNodes.includes("DirectiveStart")) {
-    const querySourceMatch = /#query\s+([\w\-_]+)/.exec(
+    const querySourceMatch = /#query\s+([\w\-_\/]+)/.exec(
       completeEvent.linePrefix,
     );
     const whereMatch =
@@ -69,9 +82,9 @@ export async function templateVariableComplete(completeEvent: CompleteEvent) {
   );
 
   const completions = (await events.dispatchEvent(
-    `attribute:complete:*`,
+    `attribute:complete:_`,
     {
-      source: "*",
+      source: "",
       prefix: match[1],
     } as AttributeCompleteEvent,
   )).flat() as AttributeCompletion[];
@@ -92,7 +105,7 @@ export function attributeCompletionsToCMCompletion(
   return completions.map(
     (completion) => ({
       label: completion.name,
-      detail: `${completion.type} (${completion.source})`,
+      detail: `${completion.attributeType} (${completion.source})`,
       type: "attribute",
     }),
   );

plugs/directive/data.ts

@@ -1,66 +0,0 @@
-// Index key space:
-// data:page@pos
-
-import type { IndexTreeEvent, QueryProviderEvent } from "$sb/app_event.ts";
-import { index, YAML } from "$sb/syscalls.ts";
-import { collectNodesOfType, findNodeOfType } from "$sb/lib/tree.ts";
-import { applyQuery, removeQueries } from "$sb/lib/query.ts";
-
-export async function indexData({ name, tree }: IndexTreeEvent) {
-  const dataObjects: { key: string; value: any }[] = [];
-
-  removeQueries(tree);
-
-  await Promise.all(
-    collectNodesOfType(tree, "FencedCode").map(async (t) => {
-      const codeInfoNode = findNodeOfType(t, "CodeInfo");
-      if (!codeInfoNode) {
-        return;
-      }
-      if (codeInfoNode.children![0].text !== "data") {
-        return;
-      }
-      const codeTextNode = findNodeOfType(t, "CodeText");
-      if (!codeTextNode) {
-        // Honestly, this shouldn't happen
-        return;
-      }
-      const codeText = codeTextNode.children![0].text!;
-      try {
-        const docs = codeText.split("---");
-        // We support multiple YAML documents in one block
-        for (let i = 0; i < docs.length; i++) {
-          const doc = await YAML.parse(docs[i]);
-          if (!doc) {
-            continue;
-          }
-          dataObjects.push({
-            key: `data:${name}@${t.from! + i}`,
-            value: doc,
-          });
-        }
-        // console.log("Parsed data", parsedData);
-      } catch (e) {
-        console.error("Could not parse data", codeText, "error:", e);
-        return;
-      }
-    }),
-  );
-  // console.log("Found", dataObjects.length, "data objects");
-  await index.batchSet(name, dataObjects);
-}
-
-export async function queryProvider({
-  query,
-}: QueryProviderEvent): Promise<any[]> {
-  const allData: any[] = [];
-  for (const { key, page, value } of await index.queryPrefix("data:")) {
-    const [, pos] = key.split("@");
-    allData.push({
-      ...value,
-      page: page,
-      pos: +pos,
-    });
-  }
-  return applyQuery(query, allData);
-}

plugs/directive/directive.plug.yaml

@@ -20,14 +20,6 @@ functions:
     mqSubscriptions:
     - queue: directiveUpdateQueue
       batchSize: 3
-  indexData:
-    path: ./data.ts:indexData
-    events:
-      - page:index
-  dataQueryProvider:
-    path: ./data.ts:queryProvider
-    events:
-      - query:data
   queryComplete:
     path: ./complete.ts:queryComplete
     events:
@@ -37,43 +29,13 @@ functions:
     events:
       - editor:complete
 
+  # Conversion
+  convertToLiveQuery:
+    path: command.ts:convertToLiveQuery
+    command:
+      name: "Directive: Convert Query to Live Query"
+
   # Templates
-  insertQuery:
-    redirect: template.insertTemplateText
-    slashCommand:
-      name: query
-      description: Insert a query
-      value: |
-        <!-- #query |^| -->
-
-        <!-- /query -->
-  insertInclude:
-    redirect: template.insertTemplateText
-    slashCommand:
-      name: include
-      description: Include another page
-      value: |
-        <!-- #include [[|^|]] -->
-
-        <!-- /include -->
-  insertUseTemplate:
-    redirect: template.insertTemplateText
-    slashCommand:
-      name: use
-      description: Use a template
-      value: |
-        <!-- #use [[|^|]] {} -->
-
-        <!-- /use -->
-  insertUseVerboseTemplate:
-    redirect: template.insertTemplateText
-    slashCommand:
-      name: use-verbose
-      description: Use a template (verbose mode)
-      value: |
-        <!-- #use-verbose [[|^|]] {} -->
-
-        <!-- /use-verbose -->
   insertEvalTemplate:
     redirect: template.insertTemplateText
     slashCommand:

plugs/directive/directives.ts

@@ -1,5 +1,11 @@
-import { ParseTree, renderToText } from "$sb/lib/tree.ts";
-import { PageMeta } from "../../web/types.ts";
+import {
+  addParentPointers,
+  findParentMatching,
+  ParseTree,
+  renderToText,
+} from "$sb/lib/tree.ts";
+import { PageMeta } from "$sb/types.ts";
+import { editor, markdown } from "$sb/syscalls.ts";
 
 import { evalDirectiveRenderer } from "./eval_directive.ts";
 import { queryDirectiveRenderer } from "./query_directive.ts";
@@ -53,8 +59,29 @@ export async function directiveDispatcher(
   const directiveStartText = renderToText(directiveStart).trim();
   const directiveEndText = renderToText(directiveEnd).trim();
 
-  if (directiveStart.children!.length === 1) {
-    // Everything not #query
+  const firstPart = directiveStart.children![0].text!;
+  if (firstPart?.includes("#query")) {
+    // #query
+    const newBody = await directiveRenderers["query"](
+      "query",
+      pageMeta,
+      directiveStart.children![1].children![0], // The query ParseTree
+    );
+    const result =
+      `${directiveStartText}\n${newBody.trim()}\n${directiveEndText}`;
+    return result;
+  } else if (firstPart?.includes("#eval")) {
+    console.log("Eval stuff", directiveStart.children![1].children![0]);
+    const newBody = await directiveRenderers["eval"](
+      "eval",
+      pageMeta,
+      directiveStart.children![1].children![0],
+    );
+    const result =
+      `${directiveStartText}\n${newBody.trim()}\n${directiveEndText}`;
+    return result;
+  } else {
+    // Everything not #query and #eval
     const match = directiveStartRegex.exec(directiveStart.children![0].text!);
     if (!match) {
       throw Error("No match");
@@ -70,16 +97,6 @@ export async function directiveDispatcher(
     } catch (e: any) {
       return `${directiveStartText}\n**ERROR:** ${e.message}\n${directiveEndText}`;
     }
-  } else {
-    // #query
-    const newBody = await directiveRenderers["query"](
-      "query",
-      pageMeta,
-      directiveStart.children![1], // The query ParseTree
-    );
-    const result =
-      `${directiveStartText}\n${newBody.trim()}\n${directiveEndText}`;
-    return result;
   }
 }
 

plugs/directive/eval_directive.ts

@@ -1,23 +1,9 @@
-// This is some shocking stuff. My profession would kill me for this.
-
-import { YAML } from "$sb/syscalls.ts";
-import { ParseTree } from "$sb/lib/tree.ts";
-import { jsonToMDTable, renderTemplate } from "./util.ts";
-import type { PageMeta } from "../../web/types.ts";
+import { ParseTree, parseTreeToAST } from "$sb/lib/tree.ts";
 import { replaceTemplateVars } from "../template/template.ts";
-
-// Enables plugName.functionName(arg1, arg2) syntax in JS expressions
-function translateJs(js: string): string {
-  return js.replaceAll(
-    /(\w+\.\w+)\s*\(/g,
-    'await invokeFunction("$1", ',
-  );
-}
-
-// Syntaxes to support:
-// - random JS expression
-// - random JS expression render [[some/template]]
-const expressionRegex = /(.+?)(\s+render\s+\[\[([^\]]+)\]\])?$/;
+import { PageMeta } from "$sb/types.ts";
+import { expressionToKvQueryExpression } from "$sb/lib/parse-query.ts";
+import { evalQueryExpression } from "$sb/lib/query.ts";
+import { builtinFunctions } from "$sb/lib/builtin_query_functions.ts";
 
 // This is rather scary and fragile stuff, but it works.
 export async function evalDirectiveRenderer(
@@ -25,42 +11,19 @@ export async function evalDirectiveRenderer(
   pageMeta: PageMeta,
   expression: string | ParseTree,
 ): Promise<string> {
-  if (typeof expression !== "string") {
-    throw new Error("Expected a string");
-  }
-  // console.log("Got JS expression", expression);
-  const match = expressionRegex.exec(expression);
-  if (!match) {
-    throw new Error(`Invalid eval directive: ${expression}`);
-  }
-  let template = "";
-  if (match[3]) {
-    // This is the template reference
-    expression = match[1];
-    template = match[3];
-  }
   try {
-    // Why the weird "eval" call? https://esbuild.github.io/content-types/#direct-eval
-    const result = await (0, eval)(
-      `(async () => { 
-        function invokeFunction(name, ...args) {
-          return syscall("system.invokeFunction", name, ...args);
-        }
-        return ${replaceTemplateVars(translateJs(expression), pageMeta)};
-      })()`,
+    const result = evalQueryExpression(
+      expressionToKvQueryExpression(parseTreeToAST(
+        JSON.parse(
+          await replaceTemplateVars(JSON.stringify(expression), pageMeta),
+        ),
+      )),
+      {},
+      builtinFunctions,
     );
-    if (template) {
-      return await renderTemplate(pageMeta, template, result);
-    }
-    if (typeof result === "string") {
-      return result;
-    } else if (typeof result === "number") {
-      return "" + result;
-    } else if (Array.isArray(result)) {
-      return jsonToMDTable(result);
-    }
-    return await YAML.stringify(result);
+
+    return Promise.resolve("" + result);
   } catch (e: any) {
-    return `**ERROR:** ${e.message}`;
+    return Promise.resolve(`**ERROR:** ${e.message}`);
   }
 }

plugs/directive/handlebar_helpers.ts

@@ -1,40 +0,0 @@
-import { niceDate } from "$sb/lib/dates.ts";
-
-export function handlebarHelpers(_pageName: string) {
-  return {
-    json: (v: any) => JSON.stringify(v),
-    niceDate: (ts: any) => niceDate(new Date(ts)),
-    escapeRegexp: (ts: any) => {
-      return ts.replace(/[-\/\\^$*+?.()|[\]{}]/g, "\\$&");
-    },
-    replaceRegexp: (s: string, regexp: string, replacement: string) => {
-      return s.replace(new RegExp(regexp, "g"), replacement);
-    },
-    prefixLines: (v: string, prefix: string) =>
-      v.split("\n").map((l) => prefix + l).join("\n"),
-    substring: (s: string, from: number, to: number, elipsis = "") =>
-      s.length > to - from ? s.substring(from, to) + elipsis : s,
-
-    today: () => niceDate(new Date()),
-    tomorrow: () => {
-      const tomorrow = new Date();
-      tomorrow.setDate(tomorrow.getDate() + 1);
-      return niceDate(tomorrow);
-    },
-    yesterday: () => {
-      const yesterday = new Date();
-      yesterday.setDate(yesterday.getDate() - 1);
-      return niceDate(yesterday);
-    },
-    lastWeek: () => {
-      const lastWeek = new Date();
-      lastWeek.setDate(lastWeek.getDate() - 7);
-      return niceDate(lastWeek);
-    },
-    nextWeek: () => {
-      const nextWeek = new Date();
-      nextWeek.setDate(nextWeek.getDate() + 7);
-      return niceDate(nextWeek);
-    },
-  };
-}

plugs/directive/parser.ts

@@ -1,122 +0,0 @@
-import {
-  collectNodesOfType,
-  findNodeOfType,
-  ParseTree,
-  replaceNodesMatching,
-} from "$sb/lib/tree.ts";
-
-// @ts-ignore auto generated
-import { ParsedQuery, QueryFilter } from "$sb/lib/query.ts";
-
-export function parseQuery(queryTree: ParseTree): ParsedQuery {
-  // const n = lezerToParseTree(query, parser.parse(query).topNode);
-  // Clean the tree a bit
-  replaceNodesMatching(queryTree, (n) => {
-    if (!n.type) {
-      const trimmed = n.text!.trim();
-      if (!trimmed) {
-        return null;
-      }
-      n.text = trimmed;
-    }
-  });
-
-  // console.log("Parsed", JSON.stringify(n, null, 2));
-  const queryNode = queryTree.children![0];
-  const parsedQuery: ParsedQuery = {
-    table: queryNode.children![0].children![0].text!,
-    filter: [],
-    ordering: [],
-  };
-
-  const orderByNodes = collectNodesOfType(queryNode, "OrderClause");
-  for (const orderByNode of orderByNodes) {
-    const nameNode = findNodeOfType(orderByNode, "Name");
-    const orderBy = nameNode!.children![0].text!;
-    const orderNode = findNodeOfType(orderByNode, "OrderDirection");
-    const orderDesc = orderNode
-      ? orderNode.children![0].text! === "desc"
-      : false;
-    parsedQuery.ordering.push({ orderBy, orderDesc });
-  }
-  /**
-   * @deprecated due to PR #387
-   * We'll take the first ordering and send that as the deprecated
-   * fields orderBy and orderDesc. This way it will be backward
-   * Plugs using the old ParsedQuery.
-   * Remove this block completely when ParsedQuery no longer have
-   * those two fields
-   */
-  if (parsedQuery.ordering.length > 0) {
-    parsedQuery.orderBy = parsedQuery.ordering[0].orderBy;
-    parsedQuery.orderDesc = parsedQuery.ordering[0].orderDesc;
-  }
-  /** @end-deprecation due to PR #387  */
-
-  const limitNode = findNodeOfType(queryNode, "LimitClause");
-  if (limitNode) {
-    const nameNode = findNodeOfType(limitNode, "Number");
-    parsedQuery.limit = valueNodeToVal(nameNode!);
-  }
-
-  const filterNodes = collectNodesOfType(queryNode, "FilterExpr");
-  for (const filterNode of filterNodes) {
-    let val: any = undefined;
-    const valNode = filterNode.children![2].children![0];
-    val = valueNodeToVal(valNode);
-    const f: QueryFilter = {
-      prop: filterNode.children![0].children![0].text!,
-      op: filterNode.children![1].text!,
-      value: val,
-    };
-    parsedQuery.filter.push(f);
-  }
-  const selectNode = findNodeOfType(queryNode, "SelectClause");
-  if (selectNode) {
-    parsedQuery.select = [];
-    collectNodesOfType(selectNode, "Name").forEach((t) => {
-      parsedQuery.select!.push(t.children![0].text!);
-    });
-  }
-
-  const renderNode = findNodeOfType(queryNode, "RenderClause");
-  if (renderNode) {
-    let renderNameNode = findNodeOfType(renderNode, "PageRef");
-    if (!renderNameNode) {
-      renderNameNode = findNodeOfType(renderNode, "String");
-    }
-    parsedQuery.render = valueNodeToVal(renderNameNode!);
-  }
-
-  return parsedQuery;
-}
-
-export function valueNodeToVal(valNode: ParseTree): any {
-  switch (valNode.type) {
-    case "Number":
-      return +valNode.children![0].text!;
-    case "Bool":
-      return valNode.children![0].text! === "true";
-    case "Null":
-      return null;
-    case "Name":
-      return valNode.children![0].text!;
-    case "Regex": {
-      const val = valNode.children![0].text!;
-      return val.substring(1, val.length - 1);
-    }
-    case "String": {
-      const stringVal = valNode.children![0].text!;
-      return stringVal.substring(1, stringVal.length - 1);
-    }
-    case "PageRef": {
-      const pageRefVal = valNode.children![0].text!;
-      return pageRefVal.substring(2, pageRefVal.length - 2);
-    }
-    case "List": {
-      return collectNodesOfType(valNode, "Value").map((t) =>
-        valueNodeToVal(t.children![0])
-      );
-    }
-  }
-}

plugs/directive/query.test.ts

@@ -1,187 +0,0 @@
-import { assertEquals } from "../../test_deps.ts";
-import { applyQuery } from "$sb/lib/query.ts";
-
-import wikiMarkdownLang from "../../common/markdown_parser/parser.ts";
-import { parse } from "../../common/markdown_parser/parse_tree.ts";
-import { parseQuery as parseQueryQuery } from "./parser.ts";
-import { findNodeOfType, renderToText } from "../../plug-api/lib/tree.ts";
-
-function parseQuery(query: string) {
-  const lang = wikiMarkdownLang([]);
-  const mdTree = parse(
-    lang,
-    `<!-- #query ${query} -->
-  
-  <!-- /query -->`,
-  );
-  const programNode = findNodeOfType(mdTree, "Program")!;
-  return parseQueryQuery(programNode);
-}
-
-Deno.test("Test parser", () => {
-  const parsedBasicQuery = parseQuery(`page`);
-  assertEquals(parsedBasicQuery.table, "page");
-
-  const parsedQuery1 = parseQuery(
-    `task where completed = false and dueDate <= "{{today}}" order by dueDate desc limit 5`,
-  );
-  assertEquals(parsedQuery1.table, "task");
-  assertEquals(parsedQuery1.ordering.length, 1);
-  assertEquals(parsedQuery1.ordering[0].orderBy, "dueDate");
-  assertEquals(parsedQuery1.ordering[0].orderDesc, true);
-  assertEquals(parsedQuery1.limit, 5);
-  assertEquals(parsedQuery1.filter.length, 2);
-  assertEquals(parsedQuery1.filter[0], {
-    op: "=",
-    prop: "completed",
-    value: false,
-  });
-  assertEquals(parsedQuery1.filter[1], {
-    op: "<=",
-    prop: "dueDate",
-    value: "{{today}}",
-  });
-
-  const parsedQuery2 = parseQuery(`page where name =~ /interview\\/.*/"`);
-  assertEquals(parsedQuery2.table, "page");
-  assertEquals(parsedQuery2.filter.length, 1);
-  assertEquals(parsedQuery2.filter[0], {
-    op: "=~",
-    prop: "name",
-    value: "interview\\/.*",
-  });
-
-  const parsedQuery3 = parseQuery(`page where something != null`);
-  assertEquals(parsedQuery3.table, "page");
-  assertEquals(parsedQuery3.filter.length, 1);
-  assertEquals(parsedQuery3.filter[0], {
-    op: "!=",
-    prop: "something",
-    value: null,
-  });
-
-  assertEquals(parseQuery(`page select name`).select, ["name"]);
-  assertEquals(parseQuery(`page select name, age`).select, [
-    "name",
-    "age",
-  ]);
-
-  assertEquals(
-    parseQuery(`gh-events where type in ["PushEvent", "somethingElse"]`),
-    {
-      table: "gh-events",
-      ordering: [],
-      filter: [
-        {
-          op: "in",
-          prop: "type",
-          value: ["PushEvent", "somethingElse"],
-        },
-      ],
-    },
-  );
-
-  assertEquals(parseQuery(`something render [[template/table]]`), {
-    table: "something",
-    ordering: [],
-    filter: [],
-    render: "template/table",
-  });
-
-  assertEquals(parseQuery(`something render "template/table"`), {
-    table: "something",
-    ordering: [],
-    filter: [],
-    render: "template/table",
-  });
-});
-
-Deno.test("Test applyQuery", () => {
-  const data: any[] = [
-    { name: "interview/My Interview", lastModified: 1 },
-    { name: "interview/My Interview 2", lastModified: 2 },
-    { name: "Pete", age: 38 },
-    { name: "Angie", age: 28 },
-  ];
-
-  assertEquals(
-    applyQuery(parseQuery(`page where name =~ /interview\\/.*/`), data),
-    [
-      { name: "interview/My Interview", lastModified: 1 },
-      { name: "interview/My Interview 2", lastModified: 2 },
-    ],
-  );
-  assertEquals(
-    applyQuery(
-      parseQuery(`page where name =~ /interview\\/.*/ order by lastModified`),
-      data,
-    ),
-    [
-      { name: "interview/My Interview", lastModified: 1 },
-      { name: "interview/My Interview 2", lastModified: 2 },
-    ],
-  );
-  assertEquals(
-    applyQuery(
-      parseQuery(
-        `page where name  =~ /interview\\/.*/ order by lastModified desc`,
-      ),
-      data,
-    ),
-    [
-      { name: "interview/My Interview 2", lastModified: 2 },
-      { name: "interview/My Interview", lastModified: 1 },
-    ],
-  );
-  assertEquals(applyQuery(parseQuery(`page where age > 30`), data), [
-    { name: "Pete", age: 38 },
-  ]);
-  assertEquals(
-    applyQuery(parseQuery(`page where age > 28 and age < 38`), data),
-    [],
-  );
-  assertEquals(
-    applyQuery(parseQuery(`page where age > 30 select name`), data),
-    [{ name: "Pete" }],
-  );
-
-  assertEquals(
-    applyQuery(parseQuery(`page where name in ["Pete"] select name`), data),
-    [{ name: "Pete" }],
-  );
-});
-
-Deno.test("Test applyQuery with multi value", () => {
-  const data: any[] = [
-    { name: "Pete", children: ["John", "Angie"] },
-    { name: "Angie", children: ["Angie"] },
-    { name: "Steve" },
-  ];
-
-  assertEquals(
-    applyQuery(parseQuery(`page where children = "Angie"`), data),
-    [
-      { name: "Pete", children: ["John", "Angie"] },
-      { name: "Angie", children: ["Angie"] },
-    ],
-  );
-
-  assertEquals(
-    applyQuery(parseQuery(`page where children = ["Angie", "John"]`), data),
-    [
-      { name: "Pete", children: ["John", "Angie"] },
-      { name: "Angie", children: ["Angie"] },
-    ],
-  );
-});
-
-const testQuery = `<!-- #query source where a = 1 and b = "2" and c = "3" -->
-
-<!-- /query -->`;
-
-Deno.test("Query parsing and serialization", () => {
-  const lang = wikiMarkdownLang([]);
-  const mdTree = parse(lang, testQuery);
-  // console.log(JSON.stringify(mdTree, null, 2));
-  assertEquals(renderToText(mdTree), testQuery);
-});

plugs/directive/query_directive.ts

@@ -2,10 +2,10 @@ import { events } from "$sb/syscalls.ts";
 
 import { replaceTemplateVars } from "../template/template.ts";
 import { renderTemplate } from "./util.ts";
-import { parseQuery } from "./parser.ts";
 import { jsonToMDTable } from "./util.ts";
-import { ParseTree } from "$sb/lib/tree.ts";
-import type { PageMeta } from "../../web/types.ts";
+import { ParseTree, parseTreeToAST } from "$sb/lib/tree.ts";
+import { astToKvQuery } from "$sb/lib/parse-query.ts";
+import { PageMeta, Query } from "$sb/types.ts";
 
 export async function queryDirectiveRenderer(
   _directive: string,
@@ -15,11 +15,14 @@ export async function queryDirectiveRenderer(
   if (typeof query === "string") {
     throw new Error("Argument must be a ParseTree");
   }
-  const parsedQuery = parseQuery(
-    JSON.parse(replaceTemplateVars(JSON.stringify(query), pageMeta)),
+  const parsedQuery: Query = astToKvQuery(
+    parseTreeToAST(
+      JSON.parse(await replaceTemplateVars(JSON.stringify(query), pageMeta)),
+    ),
   );
+  // console.log("QUERY", parsedQuery);
 
-  const eventName = `query:${parsedQuery.table}`;
+  const eventName = `query:${parsedQuery.querySource}`;
 
   // console.log("Parsed query", parsedQuery);
   // Let's dispatch an event and see what happens
@@ -30,24 +33,23 @@ export async function queryDirectiveRenderer(
   );
   if (results.length === 0) {
     // This means there was no handler for the event which means it's unsupported
-    return `**Error:** Unsupported query source '${parsedQuery.table}'`;
-  } else if (results.length === 1) {
+    return `**Error:** Unsupported query source '${parsedQuery.querySource}'`;
+  } else {
     // console.log("Parsed query", parsedQuery);
+    const allResults = results.flat();
     if (parsedQuery.render) {
       const rendered = await renderTemplate(
         pageMeta,
         parsedQuery.render,
-        results[0],
+        allResults,
       );
       return rendered.trim();
     } else {
-      if (results[0].length === 0) {
+      if (allResults.length === 0) {
         return "No results";
       } else {
-        return jsonToMDTable(results[0]);
+        return jsonToMDTable(allResults);
       }
     }
-  } else {
-    throw new Error(`Too many query results: ${results.length}`);
   }
 }

plugs/directive/template_directive.ts

@@ -1,15 +1,13 @@
 import { queryRegex } from "$sb/lib/query.ts";
 import { ParseTree, renderToText } from "$sb/lib/tree.ts";
-import { markdown, space } from "$sb/syscalls.ts";
-import Handlebars from "handlebars";
+import { handlebars, markdown, space } from "$sb/syscalls.ts";
 
 import { replaceTemplateVars } from "../template/template.ts";
 import { extractFrontmatter } from "$sb/lib/frontmatter.ts";
 import { directiveRegex } from "./directives.ts";
 import { updateDirectives } from "./command.ts";
-import { buildHandebarOptions } from "./util.ts";
-import { PageMeta } from "../../web/types.ts";
 import { resolvePath, rewritePageRefs } from "$sb/lib/resolve.ts";
+import { PageMeta } from "$sb/types.ts";
 
 const templateRegex = /\[\[([^\]]+)\]\]\s*(.*)\s*/;
 
@@ -30,7 +28,7 @@ export async function templateDirectiveRenderer(
   let parsedArgs = {};
   if (args) {
     try {
-      parsedArgs = JSON.parse(replaceTemplateVars(args, pageMeta));
+      parsedArgs = JSON.parse(await replaceTemplateVars(args, pageMeta));
     } catch {
       throw new Error(
         `Failed to parse template instantiation arg: ${
@@ -65,11 +63,9 @@ export async function templateDirectiveRenderer(
 
   // if it's a template injection (not a literal "include")
   if (directive === "use") {
-    const templateFn = Handlebars.compile(
-      newBody,
-      { noEscape: true },
-    );
-    newBody = templateFn(parsedArgs, buildHandebarOptions(pageMeta));
+    newBody = await handlebars.renderTemplate(newBody, parsedArgs, {
+      page: pageMeta,
+    });
 
     // Recursively render directives
     const tree = await markdown.parseMarkdown(newBody);

plugs/directive/util.ts

@@ -1,8 +1,6 @@
-import Handlebars from "handlebars";
-
-import { space } from "$sb/syscalls.ts";
-import type { PageMeta } from "../../web/types.ts";
-import { handlebarHelpers } from "./handlebar_helpers.ts";
+import { handlebars, space } from "$sb/syscalls.ts";
+import { handlebarHelpers } from "../../common/syscalls/handlebar_helpers.ts";
+import { PageMeta } from "$sb/types.ts";
 
 const maxWidth = 70;
 
@@ -10,6 +8,9 @@ export function defaultJsonTransformer(_k: string, v: any) {
   if (v === undefined) {
     return "";
   }
+  if (typeof v === "string") {
+    return v.replaceAll("\n", " ").replaceAll("|", "\\|");
+  }
   return "" + v;
 }
 
@@ -86,13 +87,12 @@ export async function renderTemplate(
 ): Promise<string> {
   let templateText = await space.readPage(renderTemplate);
   templateText = `{{#each .}}\n${templateText}\n{{/each}}`;
-  const template = Handlebars.compile(templateText, { noEscape: true });
-  return template(data, buildHandebarOptions(pageMeta));
+  return handlebars.renderTemplate(templateText, data, { page: pageMeta });
 }
 
 export function buildHandebarOptions(pageMeta: PageMeta) {
   return {
-    helpers: handlebarHelpers(pageMeta.name),
+    helpers: handlebarHelpers(),
     data: { page: pageMeta },
   };
 }