Work to reduce bundles size (prebundle modules)

common/preload_modules.ts

@@ -0,0 +1,7 @@
+// These are the node modules that will be pre-bundled with SB
+// as a result they will not be included into plugos bundles and assumed to be loadable
+// via require() in the sandbox
+// Candidate modules for this are larger modules
+
+// When adding a module to this list, also manually add it to sandbox_worker.ts
+export const preloadModules = ["@lezer/lr", "yaml"];

common/spaces/constants.ts

@@ -0,0 +1,2 @@
+export const trashPrefix = "_trash/";
+export const plugPrefix = "_plug/";

common/spaces/evented_space_primitives.ts

@@ -2,6 +2,7 @@ import { SpacePrimitives } from "./space_primitives";
 import { EventHook } from "../../plugos/hooks/event";
 import { PageMeta } from "../types";
 import { Plug } from "../../plugos/plug";
+import { trashPrefix } from "./constants";
 
 export class EventedSpacePrimitives implements SpacePrimitives {
   constructor(private wrapped: SpacePrimitives, private eventHook: EventHook) {}
@@ -40,17 +41,19 @@ export class EventedSpacePrimitives implements SpacePrimitives {
       lastModified
     );
     // This can happen async
-    this.eventHook
+    if (!pageName.startsWith(trashPrefix)) {
-      .dispatchEvent("page:saved", pageName)
+      this.eventHook
-      .then(() => {
+        .dispatchEvent("page:saved", pageName)
-        return this.eventHook.dispatchEvent("page:index", {
+        .then(() => {
-          name: pageName,
+          return this.eventHook.dispatchEvent("page:index", {
-          text: text,
+            name: pageName,
+            text: text,
+          });
+        })
+        .catch((e) => {
+          console.error("Error dispatching page:saved event", e);
         });
-      })
+    }
-      .catch((e) => {
-        console.error("Error dispatching page:saved event", e);
-      });
     return newPageMeta;
   }
 

common/spaces/space.ts

@@ -4,10 +4,9 @@ import { PageMeta } from "../types";
 import { EventEmitter } from "../event";
 import { Plug } from "../../plugos/plug";
 import { Manifest } from "../manifest";
+import { plugPrefix, trashPrefix } from "./constants";
 
 const pageWatchInterval = 2000;
-const trashPrefix = "_trash/";
-const plugPrefix = "_plug/";
 
 export type SpaceEvents = {
   pageCreated: (meta: PageMeta) => void;
@@ -69,7 +68,9 @@ export class Space extends EventEmitter<SpaceEvents> {
           this.emit("pageCreated", newPageMeta);
         } else if (
           oldPageMeta &&
-          oldPageMeta.lastModified !== newPageMeta.lastModified
+          oldPageMeta.lastModified !== newPageMeta.lastModified &&
+          (!this.trashEnabled ||
+            (this.trashEnabled && !pageName.startsWith(trashPrefix)))
         ) {
           this.emit("pageChanged", newPageMeta);
         }

common/types.ts

@@ -4,3 +4,10 @@ export type PageMeta = {
   lastOpened?: number;
   created?: boolean;
 };
+
+// Used by FilterBox
+export type FilterOption = {
+  name: string;
+  orderId?: number;
+  hint?: string;
+};

plugos-silverbullet-syscall/editor.ts

@@ -1,4 +1,5 @@
 import { syscall } from "./syscall";
+import { FilterOption } from "../common/types";
 
 export function getCurrentPage(): Promise<string> {
   return syscall("editor.getCurrentPage");
@@ -32,6 +33,15 @@ export function flashNotification(message: string): Promise<void> {
   return syscall("editor.flashNotification", message);
 }
 
+export function filterBox(
+  label: string,
+  options: FilterOption[],
+  helpText: string = "",
+  placeHolder: string = ""
+): Promise<FilterOption | undefined> {
+  return syscall("editor.filterBox", label, options, helpText, placeHolder);
+}
+
 export function showRhs(html: string, flex = 1): Promise<void> {
   return syscall("editor.showRhs", html, flex);
 }

plugos/bin/plugos-bundle.ts

@@ -8,8 +8,14 @@ import yargs from "yargs";
 import { hideBin } from "yargs/helpers";
 import { Manifest } from "../types";
 import YAML from "yaml";
+import { preloadModules } from "../../common/preload_modules";
 
-async function compile(filePath: string, functionName: string, debug: boolean) {
+async function compile(
+  filePath: string,
+  functionName: string,
+  debug: boolean,
+  meta = true
+) {
   let outFile = "_out.tmp";
   let inFile = filePath;
 
@@ -23,7 +29,7 @@ async function compile(filePath: string, functionName: string, debug: boolean) {
   }
 
   // TODO: Figure out how to make source maps work correctly with eval() code
-  let js = await esbuild.build({
+  let result = await esbuild.build({
     entryPoints: [inFile],
     bundle: true,
     format: "iife",
@@ -32,8 +38,15 @@ async function compile(filePath: string, functionName: string, debug: boolean) {
     sourcemap: false, //sourceMap ? "inline" : false,
     minify: !debug,
     outfile: outFile,
+    metafile: true,
+    external: preloadModules,
   });
 
+  if (meta) {
+    let text = await esbuild.analyzeMetafile(result.metafile);
+    console.log("Bundle info for", functionName, text);
+  }
+
   let jsCode = (await readFile(outFile)).toString();
   await unlink(outFile);
   if (inFile !== filePath) {

plugos/environments/node_worker.ts

@@ -1,3 +1,5 @@
+import { preloadModules } from "../../common/preload_modules";
+
 const { parentPort, workerData } = require("worker_threads");
 // @ts-ignore
 let vm2 = `${workerData}/vm2`;
@@ -16,11 +18,17 @@ let pendingRequests = new Map<
 
 let syscallReqId = 0;
 
-// console.log("Here's crypto", crypto);
-
 let vm = new VM({
   sandbox: {
     console,
+    require: (moduleName: string): any => {
+      console.log("Loading", moduleName);
+      if (preloadModules.includes(moduleName)) {
+        return require(`${workerData}/${moduleName}`);
+      } else {
+        throw Error("Cannot import arbitrary modules");
+      }
+    },
     self: {
       syscall: (name: string, ...args: any[]) => {
         return new Promise((resolve, reject) => {

plugos/environments/sandbox_worker.ts

@@ -20,6 +20,7 @@ function workerPostMessage(msg: ControllerMessage) {
 
 declare global {
   function syscall(name: string, ...args: any[]): Promise<any>;
+  // function require(moduleName: string): any;
 }
 
 let syscallReqId = 0;
@@ -37,6 +38,20 @@ self.syscall = async (name: string, ...args: any[]) => {
   });
 };
 
+const preloadedModules: { [key: string]: any } = {
+  "@lezer/lr": require("@lezer/lr"),
+  yaml: require("yaml"),
+};
+// for (const moduleName of preloadModules) {
+//   preloadedModules[moduleName] = require(moduleName);
+// }
+
+// @ts-ignore
+self.require = (moduleName: string): any => {
+  console.log("Loading", moduleName, preloadedModules[moduleName]);
+  return preloadedModules[moduleName];
+};
+
 function wrapScript(code: string) {
   return `return (${code})["default"]`;
 }

plugs/core/core.plug.yaml

@@ -71,10 +71,6 @@ functions:
     path: "./dates.ts:insertTomorrow"
     slashCommand:
       name: tomorrow
-  indexDates:
-    path: "./dates.ts:indexDates"
-    events:
-      - page:index
   parseServerCommand:
     path: ./page.ts:parseServerPageCommand
     command:
@@ -85,3 +81,8 @@ functions:
     path: ./page.ts:parsePageCommand
     command:
       name: "Debug: Parse Document"
+
+  instantiateTemplateCommand:
+    path: ./template.ts:instantiateTemplateCommand
+    command:
+      name: "Template: Instantiate for Page"

plugs/core/dates.ts

@@ -1,7 +1,7 @@
 import { insertAtCursor } from "plugos-silverbullet-syscall/editor";
 import { IndexEvent } from "../../webapp/app_event";
 import { batchSet } from "plugos-silverbullet-syscall";
-import { whiteOutQueries } from "../query/materialized_queries";
+import { whiteOutQueries } from "../query/util";
 
 const dateMatchRegex = /(\d{4}\-\d{2}\-\d{2})/g;
 

plugs/core/item.ts

@@ -1,9 +1,9 @@
 import { IndexEvent } from "../../webapp/app_event";
-import { whiteOutQueries } from "../query/materialized_queries";
 
 import { batchSet } from "plugos-silverbullet-syscall/index";
 import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
 import { collectNodesOfType, ParseTree, renderToText } from "../../common/tree";
+import { whiteOutQueries } from "../query/util";
 
 export type Item = {
   name: string;

plugs/core/template.ts

@@ -0,0 +1,51 @@
+import { listPages, readPage, writePage } from "plugos-silverbullet-syscall/space";
+import { filterBox, navigate, prompt } from "plugos-silverbullet-syscall/editor";
+import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
+import { extractMeta } from "../query/data";
+import { renderToText } from "../../common/tree";
+import { niceDate } from "./dates";
+
+const pageTemplatePrefix = `template/page/`;
+
+export async function instantiateTemplateCommand() {
+  let allPages = await listPages();
+  let allPageTemplates = allPages.filter((pageMeta) =>
+    pageMeta.name.startsWith(pageTemplatePrefix)
+  );
+
+  let selectedTemplate = await filterBox(
+    "Template",
+    allPageTemplates,
+    "Select the template to create a new page from"
+  );
+
+  if (!selectedTemplate) {
+    return;
+  }
+  console.log("Selected template", selectedTemplate);
+
+  let { text } = await readPage(selectedTemplate.name);
+
+  let parseTree = await parseMarkdown(text);
+  let additionalPageMeta = extractMeta(parseTree, true);
+  console.log("Page meta", additionalPageMeta);
+
+  let pageName = await prompt("Name of new page", additionalPageMeta.name);
+  if (!pageName) {
+    return;
+  }
+  let pageText = replaceTemplateVars(renderToText(parseTree));
+  await writePage(pageName, pageText);
+  await navigate(pageName);
+}
+
+export function replaceTemplateVars(s: string): string {
+  return s.replaceAll(/\{\{(\w+)\}\}/g, (match, v) => {
+    switch (v) {
+      case "today":
+        return niceDate(new Date());
+        break;
+    }
+    return match;
+  });
+}

plugs/ghost/ghost.ts

@@ -1,9 +1,9 @@
 import { readPage, writePage } from "plugos-silverbullet-syscall/space";
 import { json } from "plugos-syscall/fetch";
-import YAML from "yaml";
+import { parse as parseYaml } from "yaml";
 import { invokeFunction } from "plugos-silverbullet-syscall/system";
 import { getCurrentPage, getText } from "plugos-silverbullet-syscall/editor";
-import { cleanMarkdown } from "../markdown/markdown";
+import { cleanMarkdown } from "../markdown/util";
 
 type GhostConfig = {
   url: string;
@@ -183,7 +183,13 @@ async function markdownToPost(text: string): Promise<Partial<Post>> {
 
 async function getConfig(): Promise<GhostConfig> {
   let configPage = await readPage("ghost-config");
-  return YAML.parse(configPage.text) as GhostConfig;
+  return parseYaml(configPage.text) as GhostConfig;
+  // return {
+  //   adminKey: "",
+  //   pagePrefix: "",
+  //   postPrefix: "",
+  //   url: "",
+  // };
 }
 
 export async function downloadAllPostsCommand() {

plugs/markdown/markdown.plug.yaml

@@ -6,7 +6,7 @@ functions:
       key: Ctrl-p
       mac: Cmd-p
   preview:
-    path: "./markdown.ts:updateMarkdownPreview"
+    path: "./preview.ts:updateMarkdownPreview"
     env: client
     events:
       - plug:load

plugs/markdown/markdown.ts

@@ -1,97 +1,18 @@
-import MarkdownIt from "markdown-it";
+import { hideRhs } from "plugos-silverbullet-syscall/editor";
-import { getText, hideRhs, showRhs } from "plugos-silverbullet-syscall/editor";
+import { invokeFunction } from "plugos-silverbullet-syscall/system";
 import * as clientStore from "plugos-silverbullet-syscall/clientStore";
-import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
-import { renderToText, replaceNodesMatching } from "../../common/tree";
-
-const css = `
-<style>
-body {
-  font-family: georgia,times,serif;
-  font-size: 14pt;
-  max-width: 800px;
-  margin-left: auto;
-  margin-right: auto;
-  padding-left: 20px;
-  padding-right: 20px;
-}
-
-a[href] {
-  text-decoration: none;
-}
-
-blockquote {
-  border-left: 1px solid #333;
-  margin-left: 2px;
-  padding-left: 10px;
-}
-
-hr {
-    margin: 1em 0 1em 0;
-    text-align: center;
-    border-color: #777;
-    border-width: 0;
-    border-style: dotted;
-}
-
-hr:after {
-    content: "···";
-    letter-spacing: 1em;
-}
-
-</style>
-`;
-
-var taskLists = require("markdown-it-task-lists");
-
-const md = new MarkdownIt({
-  linkify: true,
-  html: false,
-  typographer: true,
-}).use(taskLists);
 
 export async function togglePreview() {
   let currentValue = !!(await clientStore.get("enableMarkdownPreview"));
   await clientStore.set("enableMarkdownPreview", !currentValue);
   if (!currentValue) {
-    updateMarkdownPreview();
+    await invokeFunction("client", "preview");
+    // updateMarkdownPreview();
   } else {
-    hideMarkdownPreview();
+    await hideMarkdownPreview();
   }
 }
 
-function encodePageUrl(name: string): string {
-  return name.replaceAll(" ", "_");
-}
-
-export async function cleanMarkdown(text: string) {
-  let mdTree = await parseMarkdown(text);
-  replaceNodesMatching(mdTree, (n) => {
-    if (n.type === "WikiLink") {
-      const page = n.children![1].children![0].text!;
-      return {
-        // HACK
-        text: `[${page}](/${encodePageUrl(page)})`,
-      };
-    }
-    // Simply get rid of these
-    if (n.type === "CommentBlock" || n.type === "Comment") {
-      return null;
-    }
-  });
-  let html = md.render(renderToText(mdTree));
-  return html;
-}
-
-export async function updateMarkdownPreview() {
-  if (!(await clientStore.get("enableMarkdownPreview"))) {
-    return;
-  }
-  let text = await getText();
-  let html = await cleanMarkdown(text);
-  await showRhs(`<html><head>${css}</head><body>${html}</body></html>`, 2);
-}
-
 async function hideMarkdownPreview() {
   await hideRhs();
 }

plugs/markdown/preview.ts

@@ -0,0 +1,62 @@
+import MarkdownIt from "markdown-it";
+import { getText, showRhs } from "plugos-silverbullet-syscall/editor";
+import * as clientStore from "plugos-silverbullet-syscall/clientStore";
+import { cleanMarkdown } from "./util";
+
+const css = `
+<style>
+body {
+  font-family: georgia,times,serif;
+  font-size: 14pt;
+  max-width: 800px;
+  margin-left: auto;
+  margin-right: auto;
+  padding-left: 20px;
+  padding-right: 20px;
+}
+
+a[href] {
+  text-decoration: none;
+}
+
+blockquote {
+  border-left: 1px solid #333;
+  margin-left: 2px;
+  padding-left: 10px;
+}
+
+hr {
+    margin: 1em 0 1em 0;
+    text-align: center;
+    border-color: #777;
+    border-width: 0;
+    border-style: dotted;
+}
+
+hr:after {
+    content: "···";
+    letter-spacing: 1em;
+}
+
+</style>
+`;
+
+var taskLists = require("markdown-it-task-lists");
+
+const md = new MarkdownIt({
+  linkify: true,
+  html: false,
+  typographer: true,
+}).use(taskLists);
+
+export async function updateMarkdownPreview() {
+  if (!(await clientStore.get("enableMarkdownPreview"))) {
+    return;
+  }
+  let text = await getText();
+  let cleanMd = await cleanMarkdown(text);
+  await showRhs(
+    `<html><head>${css}</head><body>${md.render(cleanMd)}</body></html>`,
+    2
+  );
+}

plugs/markdown/util.ts

@@ -0,0 +1,24 @@
+import { renderToText, replaceNodesMatching } from "../../common/tree";
+import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
+
+export function encodePageUrl(name: string): string {
+  return name.replaceAll(" ", "_");
+}
+
+export async function cleanMarkdown(text: string): Promise<string> {
+  let mdTree = await parseMarkdown(text);
+  replaceNodesMatching(mdTree, (n) => {
+    if (n.type === "WikiLink") {
+      const page = n.children![1].children![0].text!;
+      return {
+        // HACK
+        text: `[${page}](/${encodePageUrl(page)})`,
+      };
+    }
+    // Simply get rid of these
+    if (n.type === "CommentBlock" || n.type === "Comment") {
+      return null;
+    }
+  });
+  return renderToText(mdTree);
+}

plugs/query/data.ts

@@ -2,17 +2,16 @@
 // data:page@pos
 
 import { IndexEvent } from "../../webapp/app_event";
-import { whiteOutQueries } from "./materialized_queries";
 import { batchSet } from "plugos-silverbullet-syscall";
 import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
-import { collectNodesOfType, findNodeOfType } from "../../common/tree";
+import { collectNodesOfType, findNodeOfType, ParseTree, replaceNodesMatching } from "../../common/tree";
-import YAML from "yaml";
+import { parse as parseYaml, parseAllDocuments } from "yaml";
+import { whiteOutQueries } from "./util";
 
 export async function indexData({ name, text }: IndexEvent) {
   let e;
   text = whiteOutQueries(text);
-  console.log("Now data indexing", name);
+  // console.log("Now data indexing", name);
-  console.log("Indexing items", name);
   let mdTree = await parseMarkdown(text);
 
   let dataObjects: { key: string; value: Object }[] = [];
@@ -33,7 +32,7 @@ export async function indexData({ name, text }: IndexEvent) {
     let codeText = codeTextNode.children![0].text!;
     try {
       // We support multiple YAML documents in one block
-      for (let doc of YAML.parseAllDocuments(codeText)) {
+      for (let doc of parseAllDocuments(codeText)) {
         if (!doc.contents) {
           continue;
         }
@@ -49,6 +48,32 @@ export async function indexData({ name, text }: IndexEvent) {
       return;
     }
   });
-  console.log("Found", dataObjects, "data objects");
+  console.log("Found", dataObjects.length, "data objects");
   await batchSet(name, dataObjects);
 }
+
+export function extractMeta(parseTree: ParseTree, remove = false): any {
+  let data = {};
+  replaceNodesMatching(parseTree, (t) => {
+    if (t.type !== "FencedCode") {
+      return;
+    }
+    let codeInfoNode = findNodeOfType(t, "CodeInfo");
+    if (!codeInfoNode) {
+      return;
+    }
+    if (codeInfoNode.children![0].text !== "meta") {
+      return;
+    }
+    let codeTextNode = findNodeOfType(t, "CodeText");
+    if (!codeTextNode) {
+      // Honestly, this shouldn't happen
+      return;
+    }
+    let codeText = codeTextNode.children![0].text!;
+    data = parseYaml(codeText);
+    return remove ? null : undefined;
+  });
+
+  return data;
+}

plugs/query/engine.test.ts

@@ -32,6 +32,15 @@ test("Test parser", () => {
     prop: "name",
     value: /interview\/.*/,
   });
+
+  let parsedQuery3 = parseQuery(`page where something != null`);
+  expect(parsedQuery3.table).toBe("page");
+  expect(parsedQuery3.filter.length).toBe(1);
+  expect(parsedQuery3.filter[0]).toStrictEqual({
+    op: "!=",
+    prop: "something",
+    value: null,
+  });
 });
 
 test("Test performing the queries", () => {

plugs/query/engine.ts

@@ -63,6 +63,9 @@ export function parseQuery(query: string): ParsedQuery {
       case "Bool":
         val = valNode.children![0].text! === "true";
         break;
+      case "Null":
+        val = null;
+        break;
       case "Name":
         val = valNode.children![0].text!;
         break;
@@ -96,12 +99,12 @@ export function applyQuery<T>(parsedQuery: ParsedQuery, records: T[]): T[] {
       for (let { op, prop, value } of parsedQuery.filter) {
         switch (op) {
           case "=":
-            if (!(recordAny[prop] === value)) {
+            if (!(recordAny[prop] == value)) {
               continue recordLoop;
             }
             break;
           case "!=":
-            if (!(recordAny[prop] !== value)) {
+            if (!(recordAny[prop] != value)) {
               continue recordLoop;
             }
             break;
@@ -130,6 +133,11 @@ export function applyQuery<T>(parsedQuery: ParsedQuery, records: T[]): T[] {
               continue recordLoop;
             }
             break;
+          case "!=~":
+            if (value.exec(recordAny[prop])) {
+              continue recordLoop;
+            }
+            break;
         }
       }
       resultRecords.push(recordAny);

plugs/query/materialized_queries.ts

@@ -3,21 +3,13 @@ import { flashNotification, getCurrentPage, reloadPage, save } from "plugos-silv
 import { listPages, readPage, writePage } from "plugos-silverbullet-syscall/space";
 import { invokeFunction } from "plugos-silverbullet-syscall/system";
 import { scanPrefixGlobal } from "plugos-silverbullet-syscall";
-import { niceDate } from "../core/dates";
 import { applyQuery, parseQuery } from "./engine";
 import { PageMeta } from "../../common/types";
 import type { Task } from "../tasks/task";
 import { Item } from "../core/item";
 import YAML from "yaml";
-
+import { replaceTemplateVars } from "../core/template";
-export const queryRegex =
+import { queryRegex } from "./util";
-  /(<!--\s*#query\s+(.+?)-->)(.+?)(<!--\s*#end\s*-->)/gs;
-
-export function whiteOutQueries(text: string): string {
-  return text.replaceAll(queryRegex, (match) =>
-    new Array(match.length + 1).join(" ")
-  );
-}
 
 async function replaceAsync(
   str: string,
@@ -46,17 +38,6 @@ export async function updateMaterializedQueriesCommand() {
   await flashNotification("Updated materialized queries");
 }
 
-function replaceTemplateVars(s: string): string {
-  return s.replaceAll(/\{\{(\w+)\}\}/g, (match, v) => {
-    switch (v) {
-      case "today":
-        return niceDate(new Date());
-        break;
-    }
-    return match;
-  });
-}
-
 // Called from client, running on server
 export async function updateMaterializedQueriesOnPage(pageName: string) {
   let { text } = await readPage(pageName);
@@ -107,7 +88,7 @@ export async function updateMaterializedQueriesOnPage(pageName: string) {
         case "item":
           let allItems: Item[] = [];
           for (let { key, page, value } of await scanPrefixGlobal("it:")) {
-            let [, pos] = key.split("@");
+            let [, pos] = key.split(":");
             allItems.push({
               ...value,
               page: page,

plugs/query/parse-query.js

@@ -3,14 +3,14 @@ import { LRParser } from "@lezer/lr";
 
 export const parser = LRParser.deserialize({
   version: 13,
-  states: "$UOVQPOOO[QQO'#C^QOQPOOOjQPO'#C`OoQQO'#CiOtQPO'#CkOOQO'#Cl'#ClOyQQO,58xO!XQPO'#CcO!pQQO'#CaOOQO'#Ca'#CaOOQO,58z,58zO#RQPO,59TOOQO,59V,59VOOQO-E6j-E6jO#WQQO,58}OjQPO,58|O#iQQO1G.oOOQO'#Cg'#CgOOQO'#Cd'#CdOOQO1G.i1G.iOOQO1G.h1G.hOOQO'#Cj'#CjOOQO7+$Z7+$Z",
+  states: "$[OVQPOOO[QQO'#C^QOQPOOOjQPO'#C`OoQQO'#CjOtQPO'#ClOOQO'#Cm'#CmOyQQO,58xO!XQPO'#CcO!sQQO'#CaOOQO'#Ca'#CaOOQO,58z,58zO#UQPO,59UOOQO,59W,59WOOQO-E6k-E6kO#ZQQO,58}OjQPO,58|O#oQQO1G.pOOQO'#Cg'#CgOOQO'#Ci'#CiOOQO'#Cd'#CdOOQO1G.i1G.iOOQO1G.h1G.hOOQO'#Ck'#CkOOQO7+$[7+$[",
-  stateData: "#}~OcOS~ORPO~OdROoSOsTOaQX~ORWO~Op[O~OX]O~OdROoSOsTOaQa~Oe_Oh_Oi_Oj_Ok_Ol_Om_O~On`OaTXdTXoTXsTX~ORaO~OXcOYcO[cOfbOgbO~OqfOrfOa]id]io]is]i~O",
+  stateData: "$T~OdOS~ORPO~OeROrSOvTObQX~ORWO~Os[O~OX]O~OeROrSOvTObQa~Of_Oj_Ok_Ol_Om_On_Oo_Op_O~Oq`ObTXeTXrTXvTX~ORaO~OXdOYdO[dOgbOhbOicO~OtgOugOb^ie^ir^iv^i~O",
-  goto: "!UaPPbPeilouPPxPe{e!ORQOTUPVRZRRYRQXRRe`Rd_Rc_RgaQVPR^V",
+  goto: "!VbPPcPfjmpvPPyPyf|f!PRQOTUPVRZRRYRQXRRf`Re_Rd_RhaQVPR^V",
-  nodeNames: "⚠ Program Query Name WhereClause LogicalExpr AndExpr FilterExpr Value Number String Bool Regex OrderClause Order LimitClause",
+  nodeNames: "⚠ Program Query Name WhereClause LogicalExpr AndExpr FilterExpr Value Number String Bool Regex Null OrderClause Order LimitClause",
-  maxTerm: 35,
+  maxTerm: 38,
   skippedNodes: [0],
   repeatNodeCount: 1,
-  tokenData: "3X~RsX^#`pq#`qr$Trs$`!P!Q$z!Q![%q!^!_%y!_!`&W!`!a&e!c!}&r#T#U&}#U#V(t#V#W&r#W#X)d#X#Y&r#Y#Z*v#Z#`&r#`#a,h#a#c&r#c#d.]#d#h&r#h#i0Q#i#k&r#k#l1d#l#o&r#y#z#`$f$g#`#BY#BZ#`$IS$I_#`$Ip$Iq$`$Iq$Ir$`$I|$JO#`$JT$JU#`$KV$KW#`&FU&FV#`~#eYc~X^#`pq#`#y#z#`$f$g#`#BY#BZ#`$IS$I_#`$I|$JO#`$JT$JU#`$KV$KW#`&FU&FV#`~$WP!_!`$Z~$`Oj~~$cUOr$`rs$us$Ip$`$Ip$Iq$u$Iq$Ir$u$Ir~$`~$zOY~~%PV[~OY$zZ]$z^!P$z!P!Q%f!Q#O$z#O#P%k#P~$z~%kO[~~%nPO~$z~%vPX~!Q![%q~&OPe~!_!`&R~&WOh~~&]Pi~#r#s&`~&eOm~~&jPl~!_!`&m~&rOk~P&wQRP!c!}&r#T#o&rR'SURP!c!}&r#T#b&r#b#c'f#c#g&r#g#h(U#h#o&rR'kSRP!c!}&r#T#W&r#W#X'w#X#o&rR(OQnQRP!c!}&r#T#o&rR(ZSRP!c!}&r#T#V&r#V#W(g#W#o&rR(nQrQRP!c!}&r#T#o&rR(ySRP!c!}&r#T#m&r#m#n)V#n#o&rR)^QpQRP!c!}&r#T#o&rR)iSRP!c!}&r#T#X&r#X#Y)u#Y#o&rR)zSRP!c!}&r#T#g&r#g#h*W#h#o&rR*]SRP!c!}&r#T#V&r#V#W*i#W#o&rR*pQqQRP!c!}&r#T#o&rR*{RRP!c!}&r#T#U+U#U#o&rR+ZSRP!c!}&r#T#`&r#`#a+g#a#o&rR+lSRP!c!}&r#T#g&r#g#h+x#h#o&rR+}SRP!c!}&r#T#X&r#X#Y,Z#Y#o&rR,bQgQRP!c!}&r#T#o&rR,mSRP!c!}&r#T#]&r#]#^,y#^#o&rR-OSRP!c!}&r#T#a&r#a#b-[#b#o&rR-aSRP!c!}&r#T#]&r#]#^-m#^#o&rR-rSRP!c!}&r#T#h&r#h#i.O#i#o&rR.VQsQRP!c!}&r#T#o&rR.bSRP!c!}&r#T#f&r#f#g.n#g#o&rR.sSRP!c!}&r#T#W&r#W#X/P#X#o&rR/USRP!c!}&r#T#X&r#X#Y/b#Y#o&rR/gSRP!c!}&r#T#f&r#f#g/s#g#o&rR/zQoQRP!c!}&r#T#o&rR0VSRP!c!}&r#T#f&r#f#g0c#g#o&rR0hSRP!c!}&r#T#i&r#i#j0t#j#o&rR0ySRP!c!}&r#T#X&r#X#Y1V#Y#o&rR1^QfQRP!c!}&r#T#o&rR1iSRP!c!}&r#T#[&r#[#]1u#]#o&rR1zSRP!c!}&r#T#X&r#X#Y2W#Y#o&rR2]SRP!c!}&r#T#f&r#f#g2i#g#o&rR2nSRP!c!}&r#T#X&r#X#Y2z#Y#o&rR3RQdQRP!c!}&r#T#o&r",
+  tokenData: "4v~RtX^#cpq#cqr$Wrs$k!P!Q%V!Q![%|!^!_&U!_!`&c!`!a&p!c!}&}#T#U'Y#U#V)P#V#W&}#W#X)o#X#Y&}#Y#Z+R#Z#`&}#`#a,s#a#b&}#b#c.h#c#d/z#d#h&}#h#i1o#i#k&}#k#l3R#l#o&}#y#z#c$f$g#c#BY#BZ#c$IS$I_#c$Ip$Iq$k$Iq$Ir$k$I|$JO#c$JT$JU#c$KV$KW#c&FU&FV#c~#hYd~X^#cpq#c#y#z#c$f$g#c#BY#BZ#c$IS$I_#c$I|$JO#c$JT$JU#c$KV$KW#c&FU&FV#c~$ZP!_!`$^~$cPl~#r#s$f~$kOp~~$nUOr$krs%Qs$Ip$k$Ip$Iq%Q$Iq$Ir%Q$Ir~$k~%VOY~~%[V[~OY%VZ]%V^!P%V!P!Q%q!Q#O%V#O#P%v#P~%V~%vO[~~%yPO~%V~&RPX~!Q![%|~&ZPf~!_!`&^~&cOj~~&hPk~#r#s&k~&pOo~~&uPn~!_!`&x~&}Om~P'SQRP!c!}&}#T#o&}R'_URP!c!}&}#T#b&}#b#c'q#c#g&}#g#h(a#h#o&}R'vSRP!c!}&}#T#W&}#W#X(S#X#o&}R(ZQqQRP!c!}&}#T#o&}R(fSRP!c!}&}#T#V&}#V#W(r#W#o&}R(yQuQRP!c!}&}#T#o&}R)USRP!c!}&}#T#m&}#m#n)b#n#o&}R)iQsQRP!c!}&}#T#o&}R)tSRP!c!}&}#T#X&}#X#Y*Q#Y#o&}R*VSRP!c!}&}#T#g&}#g#h*c#h#o&}R*hSRP!c!}&}#T#V&}#V#W*t#W#o&}R*{QtQRP!c!}&}#T#o&}R+WRRP!c!}&}#T#U+a#U#o&}R+fSRP!c!}&}#T#`&}#`#a+r#a#o&}R+wSRP!c!}&}#T#g&}#g#h,T#h#o&}R,YSRP!c!}&}#T#X&}#X#Y,f#Y#o&}R,mQhQRP!c!}&}#T#o&}R,xSRP!c!}&}#T#]&}#]#^-U#^#o&}R-ZSRP!c!}&}#T#a&}#a#b-g#b#o&}R-lSRP!c!}&}#T#]&}#]#^-x#^#o&}R-}SRP!c!}&}#T#h&}#h#i.Z#i#o&}R.bQvQRP!c!}&}#T#o&}R.mSRP!c!}&}#T#i&}#i#j.y#j#o&}R/OSRP!c!}&}#T#`&}#`#a/[#a#o&}R/aSRP!c!}&}#T#`&}#`#a/m#a#o&}R/tQiQRP!c!}&}#T#o&}R0PSRP!c!}&}#T#f&}#f#g0]#g#o&}R0bSRP!c!}&}#T#W&}#W#X0n#X#o&}R0sSRP!c!}&}#T#X&}#X#Y1P#Y#o&}R1USRP!c!}&}#T#f&}#f#g1b#g#o&}R1iQrQRP!c!}&}#T#o&}R1tSRP!c!}&}#T#f&}#f#g2Q#g#o&}R2VSRP!c!}&}#T#i&}#i#j2c#j#o&}R2hSRP!c!}&}#T#X&}#X#Y2t#Y#o&}R2{QgQRP!c!}&}#T#o&}R3WSRP!c!}&}#T#[&}#[#]3d#]#o&}R3iSRP!c!}&}#T#X&}#X#Y3u#Y#o&}R3zSRP!c!}&}#T#f&}#f#g4W#g#o&}R4]SRP!c!}&}#T#X&}#X#Y4i#Y#o&}R4pQeQRP!c!}&}#T#o&}",
   tokenizers: [0, 1],
   topRules: {"Program":[0,1]},
   tokenPrec: 0

plugs/query/parse-query.terms.js

@@ -12,6 +12,7 @@ export const
   String = 10,
   Bool = 11,
   Regex = 12,
-  OrderClause = 13,
+  Null = 13,
-  Order = 14,
+  OrderClause = 14,
-  LimitClause = 15
+  Order = 15,
+  LimitClause = 16

plugs/query/query.grammar

@@ -14,7 +14,7 @@ Order {
   "desc" | "asc"
 }
 
-Value { Number | String | Bool | Regex }
+Value { Number | String | Bool | Regex | Null }
 
 LogicalExpr { AndExpr | FilterExpr }
 
@@ -28,6 +28,7 @@ FilterExpr {
 | Name ">=" Value
 | Name ">" Value
 | Name "=~" Value
+| Name "!=~" Value
 }
 
 @skip { space }
@@ -36,6 +37,10 @@ Bool {
   "true" | "false"
 }
 
+Null {
+  "null"
+}
+
 @tokens {
   space { std.whitespace+ }
   Name { std.asciiLetter+ }

plugs/query/util.ts

@@ -0,0 +1,8 @@
+export const queryRegex =
+  /(<!--\s*#query\s+(.+?)-->)(.+?)(<!--\s*#end\s*-->)/gs;
+
+export function whiteOutQueries(text: string): string {
+  return text.replaceAll(queryRegex, (match) =>
+    new Array(match.length + 1).join(" ")
+  );
+}

plugs/tasks/task.ts

@@ -1,7 +1,5 @@
-import type { ClickEvent } from "../../webapp/app_event";
+import type { ClickEvent, IndexEvent } from "../../webapp/app_event";
-import { IndexEvent } from "../../webapp/app_event";
 
-import { whiteOutQueries } from "../query/materialized_queries";
 import { batchSet } from "plugos-silverbullet-syscall/index";
 import { readPage, writePage } from "plugos-silverbullet-syscall/space";
 import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
@@ -13,6 +11,7 @@ import {
   nodeAtPos,
   renderToText
 } from "../../common/tree";
+import { whiteOutQueries } from "../query/util";
 
 export type Task = {
   name: string;
@@ -25,7 +24,7 @@ export type Task = {
 };
 
 export async function indexTasks({ name, text }: IndexEvent) {
-  console.log("Indexing tasks");
+  // console.log("Indexing tasks");
   let tasks: { key: string; value: Task }[] = [];
   text = whiteOutQueries(text);
   let mdTree = await parseMarkdown(text);

webapp/components/command_palette.tsx

@@ -1,7 +1,8 @@
 import { isMacLike } from "../util";
-import { FilterList, Option } from "./filter";
+import { FilterList } from "./filter";
 import { faPersonRunning } from "@fortawesome/free-solid-svg-icons";
 import { AppCommand } from "../hooks/command";
+import { FilterOption } from "../../common/types";
 
 export function CommandPalette({
   commands,
@@ -10,7 +11,7 @@ export function CommandPalette({
   commands: Map<string, AppCommand>;
   onTrigger: (command: AppCommand | undefined) => void;
 }) {
-  let options: Option[] = [];
+  let options: FilterOption[] = [];
   const isMac = isMacLike();
   for (let [name, def] of commands.entries()) {
     options.push({

webapp/components/filter.tsx

@@ -1,14 +1,9 @@
 import React, { useEffect, useRef, useState } from "react";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import { IconDefinition } from "@fortawesome/free-solid-svg-icons";
+import { FilterOption } from "../../common/types";
 
-export type Option = {
+function magicSorter(a: FilterOption, b: FilterOption): number {
-  name: string;
-  orderId?: number;
-  hint?: string;
-};
-
-function magicSorter(a: Option, b: Option): number {
   if (a.orderId && b.orderId) {
     return a.orderId < b.orderId ? -1 : 1;
   }
@@ -19,7 +14,7 @@ function escapeRegExp(str: string): string {
   return str.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&");
 }
 
-function fuzzyFilter(pattern: string, options: Option[]): Option[] {
+function fuzzyFilter(pattern: string, options: FilterOption[]): FilterOption[] {
   let closeMatchRegex = escapeRegExp(pattern);
   closeMatchRegex = closeMatchRegex.split(/\s+/).join(".*?");
   closeMatchRegex = closeMatchRegex.replace(/\\\//g, ".*?\\/.*?");
@@ -51,7 +46,10 @@ function fuzzyFilter(pattern: string, options: Option[]): Option[] {
   return matches;
 }
 
-function simpleFilter(pattern: string, options: Option[]): Option[] {
+function simpleFilter(
+  pattern: string,
+  options: FilterOption[]
+): FilterOption[] {
   const lowerPattern = pattern.toLowerCase();
   return options.filter((option) => {
     return option.name.toLowerCase().includes(lowerPattern);
@@ -71,10 +69,10 @@ export function FilterList({
   newHint,
 }: {
   placeholder: string;
-  options: Option[];
+  options: FilterOption[];
   label: string;
   onKeyPress?: (key: string, currentText: string) => void;
-  onSelect: (option: Option | undefined) => void;
+  onSelect: (option: FilterOption | undefined) => void;
   allowNew?: boolean;
   completePrefix?: string;
   helpText: string;
@@ -169,7 +167,7 @@ export function FilterList({
                 onSelect(undefined);
                 break;
               case " ":
-                if (completePrefix) {
+                if (completePrefix && !text) {
                   setText(completePrefix);
                   e.preventDefault();
                 }

webapp/components/page_navigator.tsx

@@ -1,5 +1,5 @@
-import { FilterList, Option } from "./filter";
+import { FilterList } from "./filter";
-import { PageMeta } from "../../common/types";
+import { FilterOption, PageMeta } from "../../common/types";
 
 export function PageNavigator({
   allPages,
@@ -10,7 +10,7 @@ export function PageNavigator({
   onNavigate: (page: string | undefined) => void;
   currentPage?: string;
 }) {
-  let options: Option[] = [];
+  let options: FilterOption[] = [];
   for (let pageMeta of allPages) {
     if (currentPage && currentPage === pageMeta.name) {
       continue;

webapp/editor.tsx

@@ -50,6 +50,8 @@ import { markdownSyscalls } from "../common/syscalls/markdown";
 import { clientStoreSyscalls } from "./syscalls/clientStore";
 import { StatusBar } from "./components/status_bar";
 import { loadMarkdownExtensions, MDExt } from "./markdown_ext";
+import { FilterList } from "./components/filter";
+import { FilterOption } from "../common/types";
 
 class PageState {
   scrollTop: number;
@@ -245,6 +247,26 @@ export class Editor implements AppEventDispatcher {
     }, 2000);
   }
 
+  filterBox(
+    label: string,
+    options: FilterOption[],
+    helpText: string = "",
+    placeHolder: string = ""
+  ): Promise<FilterOption | undefined> {
+    return new Promise((resolve) => {
+      this.viewDispatch({
+        type: "show-filterbox",
+        options,
+        placeHolder,
+        helpText,
+        onSelect: (option) => {
+          this.viewDispatch({ type: "hide-filterbox" });
+          resolve(option);
+        },
+      });
+    });
+  }
+
   async dispatchAppEvent(name: AppEvent, data?: any): Promise<void> {
     return this.eventHook.dispatchEvent(name, data);
   }
@@ -535,6 +557,17 @@ export class Editor implements AppEventDispatcher {
             commands={viewState.commands}
           />
         )}
+        {viewState.showFilterBox && (
+          <FilterList
+            label={viewState.filterBoxPlaceHolder}
+            placeholder={viewState.filterBoxPlaceHolder}
+            options={viewState.filterBoxOptions}
+            allowNew={false}
+            // icon={faPersonRunning}
+            helpText={viewState.filterBoxHelpText}
+            onSelect={viewState.filterBoxOnSelect}
+          />
+        )}
         <TopBar
           pageName={viewState.currentPage}
           notifications={viewState.notifications}

webapp/reducer.ts

@@ -92,6 +92,24 @@ export default function reducer(
         showLHS: 0,
         lhsHTML: "",
       };
+    case "show-filterbox":
+      return {
+        ...state,
+        showFilterBox: true,
+        filterBoxOnSelect: action.onSelect,
+        filterBoxPlaceHolder: action.placeHolder,
+        filterBoxOptions: action.options,
+        filterBoxHelpText: action.helpText,
+      };
+    case "hide-filterbox":
+      return {
+        ...state,
+        showFilterBox: false,
+        filterBoxOnSelect: () => {},
+        filterBoxPlaceHolder: "",
+        filterBoxOptions: [],
+        filterBoxHelpText: "",
+      };
   }
   return state;
 }

webapp/styles/editor.scss

@@ -26,7 +26,7 @@
   .line-h1,
   .line-h2,
   .line-h3 {
-    background-color: rgba(0, 15, 52, 0.6);
+    background-color: rgba(0, 30, 77, 0.5);
     color: #fff;
     font-weight: bold;
     padding: 2px 2px;
@@ -195,9 +195,11 @@
   }
 
   .comment {
-    color: gray;
+    color: #989797;
-    background-color: rgba(210, 210, 210, 0.3);
+    background-color: rgba(210, 210, 210, 0.2);
     border-radius: 5px;
-    padding: 0 2px;
+    font-style: italic;
+    font-size: 75%;
+    line-height: 75%;
   }
 }

webapp/syscalls/editor.ts

@@ -1,6 +1,7 @@
 import { Editor } from "../editor";
 import { Transaction } from "@codemirror/state";
 import { SysCallMapping } from "../../plugos/system";
+import { FilterOption } from "../../common/types";
 
 type SyntaxNode = {
   name: string;
@@ -51,6 +52,15 @@ export function editorSyscalls(editor: Editor): SysCallMapping {
     "editor.flashNotification": (ctx, message: string) => {
       editor.flashNotification(message);
     },
+    "editor.filterBox": (
+      ctx,
+      label: string,
+      options: FilterOption[],
+      helpText: string = "",
+      placeHolder: string = ""
+    ): Promise<FilterOption | undefined> => {
+      return editor.filterBox(label, options, helpText, placeHolder);
+    },
     "editor.showRhs": (ctx, html: string, flex: number) => {
       editor.viewDispatch({
         type: "show-rhs",

webapp/syscalls/system.ts

@@ -13,6 +13,10 @@ export function systemSyscalls(space: Space): SysCallMapping {
         throw Error("No plug associated with context");
       }
 
+      if (env === "client") {
+        return ctx.plug.invoke(name, args);
+      }
+
       return space.invokeFunction(ctx.plug, env, name, args);
     },
   };

webapp/types.ts

@@ -1,5 +1,5 @@
 import { AppCommand } from "./hooks/command";
-import { PageMeta } from "../common/types";
+import { FilterOption, PageMeta } from "../common/types";
 
 export const slashCommandRegexp = /\/[\w\-]*/;
 
@@ -21,6 +21,12 @@ export type AppViewState = {
   allPages: Set<PageMeta>;
   commands: Map<string, AppCommand>;
   notifications: Notification[];
+
+  showFilterBox: boolean;
+  filterBoxPlaceHolder: string;
+  filterBoxOptions: FilterOption[];
+  filterBoxHelpText: string;
+  filterBoxOnSelect: (option: FilterOption | undefined) => void;
 };
 
 export const initialViewState: AppViewState = {
@@ -34,6 +40,11 @@ export const initialViewState: AppViewState = {
   allPages: new Set(),
   commands: new Map(),
   notifications: [],
+  showFilterBox: false,
+  filterBoxHelpText: "",
+  filterBoxOnSelect: () => {},
+  filterBoxOptions: [],
+  filterBoxPlaceHolder: "",
 };
 
 export type Action =
@@ -51,4 +62,12 @@ export type Action =
   | { type: "show-rhs"; html: string; flex: number }
   | { type: "hide-rhs" }
   | { type: "show-lhs"; html: string; flex: number }
-  | { type: "hide-lhs" };
+  | { type: "hide-lhs" }
+  | {
+      type: "show-filterbox";
+      options: FilterOption[];
+      placeHolder: string;
+      helpText: string;
+      onSelect: (option: FilterOption | undefined) => void;
+    }
+  | { type: "hide-filterbox" };