Monorepo with yarn workspaces requires yarn 3.2

packages/plugos-silverbullet-syscall/clientStore.ts

@@ -0,0 +1,13 @@
+import { syscall } from "./syscall";
+
+export async function set(key: string, value: any): Promise<void> {
+  return syscall("clientStore.set", key, value);
+}
+
+export async function get(key: string): Promise<any> {
+  return syscall("clientStore.get", key);
+}
+
+export async function del(key: string): Promise<void> {
+  return syscall("clientStore.delete", key);
+}

packages/plugos-silverbullet-syscall/editor.ts

@@ -0,0 +1,96 @@
+import { syscall } from "./syscall";
+import { FilterOption } from "../silverbullet-common/types";
+
+export function getCurrentPage(): Promise<string> {
+  return syscall("editor.getCurrentPage");
+}
+
+export function getText(): Promise<string> {
+  return syscall("editor.getText");
+}
+
+export function getCursor(): Promise<number> {
+  return syscall("editor.getCursor");
+}
+
+export function save(): Promise<void> {
+  return syscall("editor.save");
+}
+
+export function navigate(name: string, pos?: number): Promise<void> {
+  return syscall("editor.navigate", name, pos);
+}
+
+export function reloadPage(): Promise<void> {
+  return syscall("editor.reloadPage");
+}
+
+export function openUrl(url: string): Promise<void> {
+  return syscall("editor.openUrl", url);
+}
+
+export function flashNotification(message: string): Promise<void> {
+  return syscall("editor.flashNotification", message);
+}
+
+export function filterBox(
+  label: string,
+  options: FilterOption[],
+  helpText: string = "",
+  placeHolder: string = ""
+): Promise<FilterOption | undefined> {
+  return syscall("editor.filterBox", label, options, helpText, placeHolder);
+}
+
+export function showRhs(html: string, flex = 1): Promise<void> {
+  return syscall("editor.showRhs", html, flex);
+}
+
+export function hideRhs(): Promise<void> {
+  return syscall("editor.hideRhs");
+}
+
+export function showLhs(html: string, flex = 1): Promise<void> {
+  return syscall("editor.showLhs", html, flex);
+}
+
+export function hideLhs(): Promise<void> {
+  return syscall("editor.hideLhs");
+}
+
+export function insertAtPos(text: string, pos: number): Promise<void> {
+  return syscall("editor.insertAtPos", text, pos);
+}
+
+export function replaceRange(
+  from: number,
+  to: number,
+  text: string
+): Promise<void> {
+  return syscall("editor.replaceRange", from, to, text);
+}
+
+export function moveCursor(pos: number): Promise<void> {
+  return syscall("editor.moveCursor", pos);
+}
+
+export function insertAtCursor(text: string): Promise<void> {
+  return syscall("editor.insertAtCursor", text);
+}
+
+export function matchBefore(
+  re: string
+): Promise<{ from: number; to: number; text: string } | null> {
+  return syscall("editor.matchBefore", re);
+}
+
+export function dispatch(change: any): Promise<void> {
+  return syscall("editor.dispatch", change);
+}
+
+export function prompt(
+  message: string,
+  defaultValue = ""
+): Promise<string | undefined> {
+  return syscall("editor.prompt", message, defaultValue);
+}

packages/plugos-silverbullet-syscall/index.ts

@@ -0,0 +1,54 @@
+import { syscall } from "./syscall";
+
+export type KV = {
+  key: string;
+  value: any;
+};
+
+export async function set(
+  page: string,
+  key: string,
+  value: any
+): Promise<void> {
+  return syscall("index.set", page, key, value);
+}
+
+export async function batchSet(page: string, kvs: KV[]): Promise<void> {
+  return syscall("index.batchSet", page, kvs);
+}
+
+export async function get(page: string, key: string): Promise<any> {
+  return syscall("index.get", page, key);
+}
+
+export async function del(page: string, key: string): Promise<void> {
+  return syscall("index.delete", page, key);
+}
+
+export async function scanPrefixForPage(
+  page: string,
+  prefix: string
+): Promise<{ key: string; page: string; value: any }[]> {
+  return syscall("index.scanPrefixForPage", page, prefix);
+}
+
+export async function scanPrefixGlobal(
+  prefix: string
+): Promise<{ key: string; page: string; value: any }[]> {
+  return syscall("index.scanPrefixGlobal", prefix);
+}
+
+export async function clearPageIndexForPage(page: string): Promise<void> {
+  return syscall("index.clearPageIndexForPage", page);
+}
+
+export async function deletePrefixForPage(
+  page: string,
+  prefix: string
+): Promise<void> {
+  return syscall("index.deletePrefixForPage", page, prefix);
+}
+
+export async function clearPageIndex(): Promise<void> {
+  return syscall("index.clearPageIndex");
+}

packages/plugos-silverbullet-syscall/markdown.ts

@@ -0,0 +1,7 @@
+import { syscall } from "./syscall";
+
+import type { ParseTree } from "../silverbullet-common/tree";
+
+export async function parseMarkdown(text: string): Promise<ParseTree> {
+  return syscall("markdown.parseMarkdown", text);
+}

packages/plugos-silverbullet-syscall/package.json

@@ -0,0 +1,4 @@
+{
+  "name": "@silverbulletmd/plugos-silverbullet-syscall",
+  "version": "1.0.0"
+}

packages/plugos-silverbullet-syscall/space.ts

@@ -0,0 +1,20 @@
+import { syscall } from "./syscall";
+import { PageMeta } from "../silverbullet-common/types";
+
+export async function listPages(): Promise<PageMeta[]> {
+  return syscall("space.listPages");
+}
+
+export async function readPage(
+  name: string
+): Promise<{ text: string; meta: PageMeta }> {
+  return syscall("space.readPage", name);
+}
+
+export async function writePage(name: string, text: string): Promise<PageMeta> {
+  return syscall("space.writePage", name, text);
+}
+
+export async function deletePage(name: string): Promise<PageMeta> {
+  return syscall("space.deletePage", name);
+}

packages/plugos-silverbullet-syscall/syscall.ts

@@ -0,0 +1,5 @@
+declare global {
+  function syscall(name: string, ...args: any[]): Promise<any>;
+}
+
+export const syscall = self.syscall;

packages/plugos-silverbullet-syscall/system.ts

@@ -0,0 +1,9 @@
+import { syscall } from "./syscall";
+
+export async function invokeFunction(
+  env: string,
+  name: string,
+  ...args: any[]
+): Promise<any> {
+  return syscall("system.invokeFunction", env, name, ...args);
+}

packages/plugos-syscall/event.ts

@@ -0,0 +1,25 @@
+import { syscall } from "./syscall";
+
+export async function dispatch(
+  eventName: string,
+  data: any,
+  timeout?: number
+): Promise<any[]> {
+  return new Promise((resolve, reject) => {
+    let timeouter: any = -1;
+    if (timeout) {
+      timeouter = setTimeout(() => {
+        console.log("Timeout!");
+        reject("timeout");
+      }, timeout);
+    }
+    syscall("event.dispatch", eventName, data)
+      .then((r) => {
+        if (timeouter !== -1) {
+          clearTimeout(timeouter);
+        }
+        resolve(r);
+      })
+      .catch(reject);
+  });
+}

packages/plugos-syscall/fetch.ts

@@ -0,0 +1,12 @@
+import { syscall } from "./syscall";
+
+export async function json(url: RequestInfo, init: RequestInit): Promise<any> {
+  return syscall("fetch.json", url, init);
+}
+
+export async function text(
+  url: RequestInfo,
+  init: RequestInit = {}
+): Promise<string> {
+  return syscall("fetch.text", url, init);
+}

packages/plugos-syscall/package.json

@@ -0,0 +1,4 @@
+{
+  "name": "@silverbulletmd/plugos-syscall",
+  "version": "1.0.0"
+}

packages/plugos-syscall/shell.ts

@@ -0,0 +1,8 @@
+import { syscall } from "./syscall";
+
+export async function run(
+  cmd: string,
+  args: string[]
+): Promise<{ stdout: string; stderr: string }> {
+  return syscall("shell.run", cmd, args);
+}

packages/plugos-syscall/store.ts

@@ -0,0 +1,40 @@
+import { syscall } from "./syscall";
+
+export type KV = {
+  key: string;
+  value: any;
+};
+
+export async function set(key: string, value: any): Promise<void> {
+  return syscall("store.set", key, value);
+}
+
+export async function batchSet(kvs: KV[]): Promise<void> {
+  return syscall("store.batchSet", kvs);
+}
+
+export async function get(key: string): Promise<any> {
+  return syscall("store.get", key);
+}
+
+export async function del(key: string): Promise<void> {
+  return syscall("store.delete", key);
+}
+
+export async function batchDel(keys: string[]): Promise<void> {
+  return syscall("store.batchDelete", keys);
+}
+
+export async function queryPrefix(
+  prefix: string
+): Promise<{ key: string; value: any }[]> {
+  return syscall("store.scanPrefix", prefix);
+}
+
+export async function deletePrefix(prefix: string): Promise<void> {
+  return syscall("store.deletePrefix", prefix);
+}
+
+export async function deleteAll(): Promise<void> {
+  return syscall("store.deleteAll");
+}

packages/plugos-syscall/syscall.ts

@@ -0,0 +1,5 @@
+declare global {
+  function syscall(name: string, ...args: any[]): Promise<any>;
+}
+
+export const syscall = self.syscall;

packages/plugos/bin/plugos-bundle.ts

@@ -0,0 +1,150 @@
+#!/usr/bin/env node
+
+import esbuild from "esbuild";
+import { readFile, unlink, watch, writeFile } from "fs/promises";
+import path from "path";
+
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import { Manifest } from "../types";
+import YAML from "yaml";
+import { preloadModules } from "../../silverbullet-common/preload_modules";
+import { mkdirSync } from "fs";
+
+async function compile(
+  filePath: string,
+  functionName: string,
+  debug: boolean,
+  meta = false
+) {
+  let outFile = "_out.tmp";
+  let inFile = filePath;
+
+  if (functionName) {
+    // Generate a new file importing just this one function and exporting it
+    inFile = "_in.js";
+    await writeFile(
+      inFile,
+      `import {${functionName}} from "./${filePath}";export default ${functionName};`
+    );
+  }
+
+  // TODO: Figure out how to make source maps work correctly with eval() code
+  let result = await esbuild.build({
+    entryPoints: [inFile],
+    bundle: true,
+    format: "iife",
+    globalName: "mod",
+    platform: "browser",
+    sourcemap: false, //sourceMap ? "inline" : false,
+    minify: !debug,
+    outfile: outFile,
+    metafile: true,
+    external: preloadModules,
+  });
+
+  if (meta) {
+    let text = await esbuild.analyzeMetafile(result.metafile);
+    console.log("Bundle info for", functionName, text);
+  }
+
+  let jsCode = (await readFile(outFile)).toString();
+  await unlink(outFile);
+  if (inFile !== filePath) {
+    await unlink(inFile);
+  }
+  return `(() => { ${jsCode}
+  return mod;})()`;
+}
+
+async function bundle(manifestPath: string, sourceMaps: boolean) {
+  const rootPath = path.dirname(manifestPath);
+  const manifest = YAML.parse(
+    (await readFile(manifestPath)).toString()
+  ) as Manifest<any>;
+
+  for (let [name, def] of Object.entries(manifest.functions)) {
+    let jsFunctionName = "default",
+      filePath = path.join(rootPath, def.path!);
+    if (filePath.indexOf(":") !== -1) {
+      [filePath, jsFunctionName] = filePath.split(":");
+    }
+
+    def.code = await compile(filePath, jsFunctionName, sourceMaps);
+    delete def.path;
+  }
+  return manifest;
+}
+
+async function buildManifest(
+  manifestPath: string,
+  distPath: string,
+  debug: boolean
+) {
+  let generatedManifest = await bundle(manifestPath, debug);
+  const outFile =
+    manifestPath.substring(
+      0,
+      manifestPath.length - path.extname(manifestPath).length
+    ) + ".json";
+  const outPath = path.join(distPath, path.basename(outFile));
+  console.log("Emitting bundle to", outPath);
+  await writeFile(outPath, JSON.stringify(generatedManifest, null, 2));
+  return { generatedManifest, outPath };
+}
+
+async function run() {
+  let args = yargs(hideBin(process.argv))
+    .option("debug", {
+      type: "boolean",
+    })
+    .option("watch", {
+      type: "boolean",
+      alias: "w",
+    })
+    .option("dist", {
+      type: "string",
+      default: ".",
+    })
+    .parse();
+  if (args._.length === 0) {
+    console.log(
+      "Usage: plugos-bundle [--debug] [--dist <path>] <manifest.plug.yaml> <manifest2.plug.yaml> ..."
+    );
+    process.exit(1);
+  }
+
+  async function buildAll() {
+    mkdirSync(args.dist, { recursive: true });
+    for (const plugManifestPath of args._) {
+      let manifestPath = plugManifestPath as string;
+      try {
+        await buildManifest(manifestPath, args.dist, !!args.debug);
+      } catch (e) {
+        console.error(`Error building ${manifestPath}:`, e);
+      }
+    }
+  }
+
+  await buildAll();
+  if (args.watch) {
+    console.log("Watching for changes...");
+    for await (const { eventType, filename } of watch(".", {
+      recursive: true,
+    })) {
+      if (
+        filename.endsWith(".plug.yaml") ||
+        filename.endsWith(".ts") ||
+        (filename.endsWith(".js") && !filename.endsWith("_in.js"))
+      ) {
+        console.log("Change detected", eventType, filename);
+        await buildAll();
+      }
+    }
+  }
+}
+
+run().catch((e) => {
+  console.error(e);
+  process.exit(1);
+});

packages/plugos/bin/plugos-server.ts

@@ -0,0 +1,62 @@
+#!/usr/bin/env node
+
+import express from "express";
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import { DiskPlugLoader } from "../plug_loader";
+import { CronHookT, NodeCronHook } from "../hooks/node_cron";
+import shellSyscalls from "../syscalls/shell.node";
+import { System } from "../system";
+import { EndpointHook, EndpointHookT } from "../hooks/endpoint";
+import { safeRun } from "../util";
+import knex from "knex";
+import { ensureTable, storeSyscalls } from "../syscalls/store.knex_node";
+import { fetchSyscalls } from "../syscalls/fetch.node";
+import { EventHook, EventHookT } from "../hooks/event";
+import { eventSyscalls } from "../syscalls/event";
+
+let args = yargs(hideBin(process.argv))
+  .option("port", {
+    type: "number",
+    default: 1337,
+  })
+  .parse();
+
+if (!args._.length) {
+  console.error("Usage: plugos-server <path-to-plugs>");
+  process.exit(1);
+}
+
+const plugPath = args._[0] as string;
+
+const app = express();
+
+type ServerHook = EndpointHookT & CronHookT & EventHookT;
+const system = new System<ServerHook>("server");
+
+safeRun(async () => {
+  const db = knex({
+    client: "better-sqlite3",
+    connection: {
+      filename: "plugos.db",
+    },
+    useNullAsDefault: true,
+  });
+
+  await ensureTable(db, "item");
+
+  let plugLoader = new DiskPlugLoader(system, plugPath);
+  await plugLoader.loadPlugs();
+  plugLoader.watcher();
+  system.addHook(new NodeCronHook());
+  let eventHook = new EventHook();
+  system.addHook(eventHook);
+  system.registerSyscalls([], eventSyscalls(eventHook));
+  system.addHook(new EndpointHook(app, ""));
+  system.registerSyscalls([], shellSyscalls("."));
+  system.registerSyscalls([], fetchSyscalls());
+  system.registerSyscalls([], storeSyscalls(db, "item"));
+  app.listen(args.port, () => {
+    console.log(`Plugbox server listening on port ${args.port}`);
+  });
+});

packages/plugos/environments/iframe_sandbox.html

@@ -0,0 +1,8 @@
+<html>
+  <body>
+    <script type="module">
+      // Sup yo!
+      import "./sandbox_worker";
+    </script>
+  </body>
+</html>

packages/plugos/environments/iframe_sandbox.ts

@@ -0,0 +1,55 @@
+import { safeRun } from "../util";
+
+// @ts-ignore
+import sandboxHtml from "bundle-text:./iframe_sandbox.html";
+import { Sandbox } from "../sandbox";
+import { WorkerLike } from "./worker";
+import { Plug } from "../plug";
+
+class IFrameWrapper implements WorkerLike {
+  private iframe: HTMLIFrameElement;
+  onMessage?: (message: any) => Promise<void>;
+  ready: Promise<void>;
+  private messageListener: (evt: any) => void;
+
+  constructor() {
+    const iframe = document.createElement("iframe", {});
+    this.iframe = iframe;
+    iframe.style.display = "none";
+    // Let's lock this down significantly
+    iframe.setAttribute("sandbox", "allow-scripts");
+    iframe.srcdoc = sandboxHtml;
+    this.messageListener = (evt: any) => {
+      if (evt.source !== iframe.contentWindow) {
+        return;
+      }
+      let data = evt.data;
+      if (!data) return;
+      safeRun(async () => {
+        await this.onMessage!(data);
+      });
+    };
+    window.addEventListener("message", this.messageListener);
+    document.body.appendChild(iframe);
+    this.ready = new Promise((resolve) => {
+      iframe.onload = () => {
+        resolve();
+        iframe.onload = null;
+      };
+    });
+  }
+
+  postMessage(message: any): void {
+    this.iframe.contentWindow!.postMessage(message, "*");
+  }
+
+  terminate() {
+    console.log("Terminating iframe sandbox");
+    window.removeEventListener("message", this.messageListener);
+    return this.iframe.remove();
+  }
+}
+
+export function createSandbox(plug: Plug<any>) {
+  return new Sandbox(plug, new IFrameWrapper());
+}

packages/plugos/environments/node_sandbox.ts

@@ -0,0 +1,50 @@
+import { Worker } from "worker_threads";
+import { safeRun } from "../util";
+
+// @ts-ignore
+import workerCode from "bundle-text:./node_worker.ts";
+import { Sandbox } from "../sandbox";
+import { WorkerLike } from "./worker";
+import { Plug } from "../plug";
+import path from "path";
+import fs from "fs";
+
+class NodeWorkerWrapper implements WorkerLike {
+  onMessage?: (message: any) => Promise<void>;
+  ready: Promise<void>;
+  private worker: Worker;
+
+  constructor(worker: Worker) {
+    this.worker = worker;
+    worker.on("message", (message: any) => {
+      safeRun(async () => {
+        await this.onMessage!(message);
+      });
+    });
+    this.ready = new Promise((resolve) => {
+      worker.once("online", resolve);
+    });
+  }
+
+  postMessage(message: any): void {
+    this.worker.postMessage(message);
+  }
+
+  terminate(): void {
+    this.worker.terminate();
+  }
+}
+
+// Look for the node_modules directory, to be passed to the worker to find e.g. the vm2 module
+let nodeModulesDir = __dirname;
+while (!fs.existsSync(nodeModulesDir + "/node_modules/vm2")) {
+  nodeModulesDir = path.dirname(nodeModulesDir);
+}
+
+export function createSandbox(plug: Plug<any>) {
+  let worker = new Worker(workerCode, {
+    eval: true,
+    workerData: path.join(nodeModulesDir, "node_modules"),
+  });
+  return new Sandbox(plug, new NodeWorkerWrapper(worker));
+}

packages/plugos/environments/node_worker.ts

@@ -0,0 +1,117 @@
+import { preloadModules } from "../../silverbullet-common/preload_modules";
+
+const { parentPort, workerData } = require("worker_threads");
+const { VM, VMScript } = require(`${workerData}/vm2`);
+const fetch = require(`${workerData}/node-fetch`);
+const WebSocket = require(`${workerData}/ws`);
+
+// console.log("Process env", process.env);
+
+let loadedFunctions = new Map<string, Function>();
+let pendingRequests = new Map<
+  number,
+  {
+    resolve: (result: unknown) => void;
+    reject: (e: any) => void;
+  }
+>();
+
+let syscallReqId = 0;
+
+let vm = new VM({
+  sandbox: {
+    // Exposing some "safe" APIs
+    console,
+    setTimeout,
+    clearTimeout,
+    setInterval,
+    clearInterval,
+    fetch,
+    WebSocket,
+    // This is only going to be called for pre-bundled modules, we won't allow
+    // arbitrary requiring of modules
+    require: (moduleName: string): any => {
+      // console.log("Loading", moduleName);
+      if (preloadModules.includes(moduleName)) {
+        return require(`${workerData}/${moduleName}`);
+      } else {
+        throw Error("Cannot import arbitrary modules");
+      }
+    },
+    self: {
+      syscall: (name: string, ...args: any[]) => {
+        return new Promise((resolve, reject) => {
+          syscallReqId++;
+          pendingRequests.set(syscallReqId, { resolve, reject });
+          parentPort.postMessage({
+            type: "syscall",
+            id: syscallReqId,
+            name,
+            // TODO: Figure out why this is necessary (to avoide a CloneError)
+            args: JSON.parse(JSON.stringify(args)),
+          });
+        });
+      },
+    },
+  },
+});
+
+function wrapScript(code: string) {
+  return `(${code})["default"]`;
+}
+
+function safeRun(fn: any) {
+  fn().catch((e: any) => {
+    console.error(e);
+  });
+}
+
+parentPort.on("message", (data: any) => {
+  safeRun(async () => {
+    switch (data.type) {
+      case "load":
+        loadedFunctions.set(data.name, new VMScript(wrapScript(data.code)));
+        parentPort.postMessage({
+          type: "inited",
+          name: data.name,
+        });
+        break;
+      case "invoke":
+        let fn = loadedFunctions.get(data.name);
+        if (!fn) {
+          throw new Error(`Function not loaded: ${data.name}`);
+        }
+        try {
+          let r = vm.run(fn);
+          let result = await Promise.resolve(r(...data.args));
+          parentPort.postMessage({
+            type: "result",
+            id: data.id,
+            // TOOD: Figure out if this is necessary, because it's expensive
+            result: result && JSON.parse(JSON.stringify(result)),
+          });
+        } catch (e: any) {
+          parentPort.postMessage({
+            type: "result",
+            id: data.id,
+            error: e.message,
+          });
+        }
+        break;
+      case "syscall-response":
+        let syscallId = data.id;
+        const lookup = pendingRequests.get(syscallId);
+        if (!lookup) {
+          throw Error("Invalid request id");
+        }
+        pendingRequests.delete(syscallId);
+        if (data.error) {
+          console.log("Got rejection", data.error);
+          lookup.reject(new Error(data.error));
+        } else {
+          lookup.resolve(data.result);
+        }
+        break;
+    }
+  });
+});

packages/plugos/environments/sandbox_worker.ts

@@ -0,0 +1,114 @@
+import { safeRun } from "../util";
+import { ControllerMessage, WorkerMessage } from "./worker";
+
+let loadedFunctions = new Map<string, Function>();
+let pendingRequests = new Map<
+  number,
+  {
+    resolve: (result: unknown) => void;
+    reject: (e: any) => void;
+  }
+>();
+
+function workerPostMessage(msg: ControllerMessage) {
+  if (typeof window !== "undefined" && window.parent !== window) {
+    window.parent.postMessage(msg, "*");
+  } else {
+    self.postMessage(msg);
+  }
+}
+
+declare global {
+  function syscall(name: string, ...args: any[]): Promise<any>;
+  // function require(moduleName: string): any;
+}
+
+let syscallReqId = 0;
+
+self.syscall = async (name: string, ...args: any[]) => {
+  return await new Promise((resolve, reject) => {
+    syscallReqId++;
+    pendingRequests.set(syscallReqId, { resolve, reject });
+    workerPostMessage({
+      type: "syscall",
+      id: syscallReqId,
+      name,
+      args,
+    });
+  });
+};
+
+const preloadedModules: { [key: string]: any } = {
+  "@lezer/lr": require("@lezer/lr"),
+  yaml: require("yaml"),
+};
+// for (const moduleName of preloadModules) {
+//   preloadedModules[moduleName] = require(moduleName);
+// }
+
+// @ts-ignore
+self.require = (moduleName: string): any => {
+  // console.log("Loading", moduleName, preloadedModules[moduleName]);
+  return preloadedModules[moduleName];
+};
+
+function wrapScript(code: string) {
+  return `return (${code})["default"]`;
+}
+
+self.addEventListener("message", (event: { data: WorkerMessage }) => {
+  safeRun(async () => {
+    let data = event.data;
+    switch (data.type) {
+      case "load":
+        let fn2 = new Function(wrapScript(data.code!));
+        loadedFunctions.set(data.name!, fn2());
+        workerPostMessage({
+          type: "inited",
+          name: data.name,
+        });
+        break;
+      case "invoke":
+        let fn = loadedFunctions.get(data.name!);
+        if (!fn) {
+          throw new Error(`Function not loaded: ${data.name}`);
+        }
+        try {
+          let result = await Promise.resolve(fn(...(data.args || [])));
+          workerPostMessage({
+            type: "result",
+            id: data.id,
+            result: result,
+          } as ControllerMessage);
+        } catch (e: any) {
+          workerPostMessage({
+            type: "result",
+            id: data.id,
+            error: e.message,
+          });
+          throw e;
+        }
+
+        break;
+      case "syscall-response":
+        let syscallId = data.id!;
+        const lookup = pendingRequests.get(syscallId);
+        if (!lookup) {
+          console.log(
+            "Current outstanding requests",
+            pendingRequests,
+            "looking up",
+            syscallId
+          );
+          throw Error("Invalid request id");
+        }
+        pendingRequests.delete(syscallId);
+        if (data.error) {
+          lookup.reject(new Error(data.error));
+        } else {
+          lookup.resolve(data.result);
+        }
+        break;
+    }
+  });
+});

packages/plugos/environments/webworker_sandbox.ts

@@ -0,0 +1,37 @@
+import { safeRun } from "../util";
+import { Sandbox } from "../sandbox";
+import { WorkerLike } from "./worker";
+import { Plug } from "../plug";
+
+class WebWorkerWrapper implements WorkerLike {
+  private worker: Worker;
+  onMessage?: (message: any) => Promise<void>;
+  ready: Promise<void>;
+
+  constructor(worker: Worker) {
+    this.worker = worker;
+    this.worker.addEventListener("message", (evt: any) => {
+      let data = evt.data;
+      if (!data) return;
+      safeRun(async () => {
+        await this.onMessage!(data);
+      });
+    });
+    this.ready = Promise.resolve();
+  }
+  postMessage(message: any): void {
+    this.worker.postMessage(message);
+  }
+
+  terminate() {
+    return this.worker.terminate();
+  }
+}
+
+export function createSandbox(plug: Plug<any>) {
+  // ParcelJS will build this file into a worker.
+  let worker = new Worker(new URL("sandbox_worker.ts", import.meta.url), {
+    type: "module",
+  });
+  return new Sandbox(plug, new WebWorkerWrapper(worker));
+}

packages/plugos/environments/worker.ts

@@ -0,0 +1,31 @@
+export type ControllerMessageType = "inited" | "result" | "syscall";
+
+export type ControllerMessage = {
+  type: ControllerMessageType;
+  id?: number;
+  name?: string;
+  args?: any[];
+  error?: string;
+  result?: any;
+};
+
+export interface WorkerLike {
+  ready: Promise<void>;
+  onMessage?: (message: any) => Promise<void>;
+
+  postMessage(message: any): void;
+
+  terminate(): void;
+}
+
+export type WorkerMessageType = "load" | "invoke" | "syscall-response";
+
+export type WorkerMessage = {
+  type: WorkerMessageType;
+  id?: number;
+  name?: string;
+  code?: string;
+  args?: any[];
+  result?: any;
+  error?: any;
+};

packages/plugos/hooks/endpoint.test.ts

@@ -0,0 +1,49 @@
+import { createSandbox } from "../environments/node_sandbox";
+import { expect, test } from "@jest/globals";
+import { Manifest } from "../types";
+import express from "express";
+import request from "supertest";
+import { EndpointHook, EndpointHookT } from "./endpoint";
+import { System } from "../system";
+
+test("Run a plugos endpoint server", async () => {
+  let system = new System<EndpointHookT>("server");
+  let plug = await system.load(
+    "test",
+    {
+      functions: {
+        testhandler: {
+          http: {
+            path: "/",
+          },
+          code: `(() => {
+          return {
+            default: (req) => {
+              console.log("Req", req);
+              return {status: 200, body: [1, 2, 3], headers: {"Content-type": "application/json"}};
+            }
+          };
+        })()`,
+        },
+      },
+    } as Manifest<EndpointHookT>,
+    createSandbox
+  );
+
+  const app = express();
+  const port = 3123;
+
+  system.addHook(new EndpointHook(app, "/_"));
+
+  let server = app.listen(port, () => {
+    console.log(`Listening on port ${port}`);
+  });
+  let resp = await request(app)
+    .get("/_/test/?name=Pete")
+    .expect((resp) => {
+      expect(resp.status).toBe(200);
+      expect(resp.header["content-type"]).toContain("application/json");
+      expect(resp.text).toBe(JSON.stringify([1, 2, 3]));
+    });
+  server.close();
+});

packages/plugos/hooks/endpoint.ts

@@ -0,0 +1,134 @@
+import { Hook, Manifest } from "../types";
+import { Express, NextFunction, Request, Response } from "express";
+import { System } from "../system";
+
+export type EndpointRequest = {
+  method: string;
+  path: string;
+  query: { [key: string]: string };
+  headers: { [key: string]: string };
+  body: any;
+};
+
+export type EndpointResponse = {
+  status: number;
+  headers?: { [key: string]: string };
+  body: any;
+};
+
+export type EndpointHookT = {
+  http?: EndPointDef | EndPointDef[];
+};
+
+export type EndPointDef = {
+  method?: "GET" | "POST" | "PUT" | "DELETE" | "HEAD" | "OPTIONS" | "ANY";
+  path: string;
+};
+
+export class EndpointHook implements Hook<EndpointHookT> {
+  private app: Express;
+  readonly prefix: string;
+
+  constructor(app: Express, prefix: string) {
+    this.app = app;
+    this.prefix = prefix;
+  }
+
+  apply(system: System<EndpointHookT>): void {
+    this.app.use((req: Request, res: Response, next: NextFunction) => {
+      if (!req.path.startsWith(this.prefix)) {
+        return next();
+      }
+      console.log("Endpoint request", req.path);
+      Promise.resolve()
+        .then(async () => {
+          // Iterate over all loaded plugins
+          for (const [plugName, plug] of system.loadedPlugs.entries()) {
+            const manifest = plug.manifest;
+            if (!manifest) {
+              continue;
+            }
+            const functions = manifest.functions;
+            console.log("Checking plug", plugName);
+            let prefix = `${this.prefix}/${plugName}`;
+            if (!req.path.startsWith(prefix)) {
+              continue;
+            }
+            for (const [name, functionDef] of Object.entries(functions)) {
+              if (!functionDef.http) {
+                continue;
+              }
+              let endpoints = Array.isArray(functionDef.http)
+                ? functionDef.http
+                : [functionDef.http];
+              console.log(endpoints);
+              for (const { path, method } of endpoints) {
+                let prefixedPath = `${prefix}${path}`;
+                if (
+                  prefixedPath === req.path &&
+                  ((method || "GET") === req.method || method === "ANY")
+                ) {
+                  try {
+                    const response: EndpointResponse = await plug.invoke(name, [
+                      {
+                        path: req.path,
+                        method: req.method,
+                        body: req.body,
+                        query: req.query,
+                        headers: req.headers,
+                      } as EndpointRequest,
+                    ]);
+                    let resp = res.status(response.status);
+                    if (response.headers) {
+                      for (const [key, value] of Object.entries(
+                        response.headers
+                      )) {
+                        resp = resp.header(key, value);
+                      }
+                    }
+                    resp.send(response.body);
+                    return;
+                  } catch (e: any) {
+                    console.error("Error executing function", e);
+                    res.status(500).send(e.message);
+                    return;
+                  }
+                }
+              }
+            }
+          }
+          next();
+        })
+        .catch((e) => {
+          console.error(e);
+          next(e);
+        });
+    });
+  }
+
+  validateManifest(manifest: Manifest<EndpointHookT>): string[] {
+    let errors = [];
+    for (const [name, functionDef] of Object.entries(manifest.functions)) {
+      if (!functionDef.http) {
+        continue;
+      }
+      let endpoints = Array.isArray(functionDef.http)
+        ? functionDef.http
+        : [functionDef.http];
+      for (let { path, method } of endpoints) {
+        if (!path) {
+          errors.push("Path not defined for endpoint");
+        }
+        if (
+          method &&
+          ["GET", "POST", "PUT", "DELETE", "ANY"].indexOf(method) === -1
+        ) {
+          errors.push(
+            `Invalid method ${method} for end point with with ${path}`
+          );
+        }
+      }
+    }
+    return errors;
+  }
+}

packages/plugos/hooks/event.ts

@@ -0,0 +1,58 @@
+import { Hook, Manifest } from "../types";
+import { System } from "../system";
+import { safeRun } from "../util";
+
+// System events:
+// - plug:load (plugName: string)
+
+export type EventHookT = {
+  events?: string[];
+};
+
+export class EventHook implements Hook<EventHookT> {
+  private system?: System<EventHookT>;
+
+  async dispatchEvent(eventName: string, data?: any): Promise<any[]> {
+    if (!this.system) {
+      throw new Error("Event hook is not initialized");
+    }
+    let responses: any[] = [];
+    for (const plug of this.system.loadedPlugs.values()) {
+      for (const [name, functionDef] of Object.entries(
+        plug.manifest!.functions
+      )) {
+        if (functionDef.events && functionDef.events.includes(eventName)) {
+          // Only dispatch functions that can run in this environment
+          if (plug.canInvoke(name)) {
+            let result = await plug.invoke(name, [data]);
+            if (result !== undefined) {
+              responses.push(result);
+            }
+          }
+        }
+      }
+    }
+    return responses;
+  }
+
+  apply(system: System<EventHookT>): void {
+    this.system = system;
+    this.system.on({
+      plugLoaded: (name) => {
+        safeRun(async () => {
+          await this.dispatchEvent("plug:load", name);
+        });
+      },
+    });
+  }
+
+  validateManifest(manifest: Manifest<EventHookT>): string[] {
+    let errors = [];
+    for (const [name, functionDef] of Object.entries(manifest.functions)) {
+      if (functionDef.events && !Array.isArray(functionDef.events)) {
+        errors.push("'events' key must be an array of strings");
+      }
+    }
+    return errors;
+  }
+}

packages/plugos/hooks/node_cron.ts

@@ -0,0 +1,76 @@
+import { Hook, Manifest } from "../types";
+import cron, { ScheduledTask } from "node-cron";
+import { safeRun } from "../util";
+import { System } from "../system";
+
+export type CronHookT = {
+  cron?: string | string[];
+};
+
+export class NodeCronHook implements Hook<CronHookT> {
+  apply(system: System<CronHookT>): void {
+    let tasks: ScheduledTask[] = [];
+    system.on({
+      plugLoaded: (name, plug) => {
+        reloadCrons();
+      },
+      plugUnloaded(name, plug) {
+        reloadCrons();
+      },
+    });
+
+    reloadCrons();
+
+    function reloadCrons() {
+      tasks.forEach((task) => task.stop());
+      tasks = [];
+      for (let plug of system.loadedPlugs.values()) {
+        if (!plug.manifest) {
+          continue;
+        }
+        for (const [name, functionDef] of Object.entries(
+          plug.manifest.functions
+        )) {
+          if (!functionDef.cron) {
+            continue;
+          }
+          const crons = Array.isArray(functionDef.cron)
+            ? functionDef.cron
+            : [functionDef.cron];
+          for (let cronDef of crons) {
+            tasks.push(
+              cron.schedule(cronDef, () => {
+                console.log("Now acting on cron", cronDef);
+                safeRun(async () => {
+                  try {
+                    await plug.invoke(name, [cronDef]);
+                  } catch (e: any) {
+                    console.error("Execution of cron function failed", e);
+                  }
+                });
+              })
+            );
+          }
+        }
+      }
+    }
+  }
+
+  validateManifest(manifest: Manifest<CronHookT>): string[] {
+    let errors = [];
+    for (const [name, functionDef] of Object.entries(manifest.functions)) {
+      if (!functionDef.cron) {
+        continue;
+      }
+      const crons = Array.isArray(functionDef.cron)
+        ? functionDef.cron
+        : [functionDef.cron];
+      for (let cronDef of crons) {
+        if (!cron.validate(cronDef)) {
+          errors.push(`Invalid cron expression ${cronDef}`);
+        }
+      }
+    }
+    return errors;
+  }
+}

packages/plugos/package.json

@@ -0,0 +1,86 @@
+{
+  "name": "@silverbulletmd/plugos",
+  "version": "0.0.1",
+  "license": "MIT",
+  "bin": {
+    "plugos-bundle": "./dist/plugos/plugos-bundle.js",
+    "plugos-server": "./dist/plugos/plugos-server.js"
+  },
+  "scripts": {
+    "watch": "rm -rf .parcel-cache && parcel watch",
+    "build": "parcel build",
+    "clean": "rm -rf dist",
+    "test": "jest dist/test"
+  },
+  "files": [
+    "dist"
+  ],
+  "targets": {
+    "plugos": {
+      "source": [
+        "bin/plugos-bundle.ts",
+        "bin/plugos-server.ts"
+      ],
+      "outputFormat": "commonjs",
+      "isLibrary": true,
+      "context": "node"
+    },
+    "test": {
+      "source": [
+        "runtime.test.ts",
+        "hooks/endpoint.test.ts",
+        "syscalls/store.knex_node.test.ts",
+        "syscalls/store.dexie_browser.test.ts"
+      ],
+      "outputFormat": "commonjs",
+      "isLibrary": true,
+      "context": "node"
+    }
+  },
+  "dependencies": {
+    "@jest/globals": "^27.5.1",
+    "@types/cors": "^2.8.12",
+    "@types/express": "^4.17.13",
+    "@types/jsonwebtoken": "^8.5.8",
+    "better-sqlite3": "^7.5.0",
+    "body-parser": "^1.19.2",
+    "cors": "^2.8.5",
+    "dexie": "^3.2.1",
+    "esbuild": "^0.14.27",
+    "express": "^4.17.3",
+    "fake-indexeddb": "^3.1.7",
+    "jest": "^27.5.1",
+    "jsonwebtoken": "^8.5.1",
+    "knex": "^1.0.4",
+    "node-cron": "^3.0.0",
+    "node-fetch": "2",
+    "node-watch": "^0.7.3",
+    "supertest": "^6.2.2",
+    "vm2": "^3.9.9",
+    "ws": "^8.5.0",
+    "yaml": "^1.10.2",
+    "yargs": "^17.3.1"
+  },
+  "devDependencies": {
+    "@parcel/optimizer-data-url": "2.3.2",
+    "@parcel/packager-raw-url": "2.3.2",
+    "@parcel/service-worker": "2.3.2",
+    "@parcel/transformer-inline-string": "2.3.2",
+    "@parcel/transformer-sass": "2.3.2",
+    "@parcel/transformer-webmanifest": "2.3.2",
+    "@parcel/validator-typescript": "2.3.2",
+    "@types/events": "^3.0.0",
+    "@types/jest": "^27.4.1",
+    "@types/node": "^17.0.21",
+    "@types/node-cron": "^3.0.1",
+    "@types/node-fetch": "^2.6.1",
+    "@types/supertest": "^2.0.11",
+    "@types/yaml": "^1.9.7",
+    "@vscode/sqlite3": "^5.0.7",
+    "assert": "^2.0.0",
+    "events": "^3.3.0",
+    "parcel": "2.3.2",
+    "prettier": "^2.5.1",
+    "typescript": "^4.6.2"
+  }
+}

packages/plugos/plug.ts

@@ -0,0 +1,66 @@
+import { Manifest, RuntimeEnvironment } from "./types";
+import { Sandbox } from "./sandbox";
+import { System } from "./system";
+
+export class Plug<HookT> {
+  system: System<HookT>;
+  sandbox: Sandbox;
+  public manifest?: Manifest<HookT>;
+  readonly runtimeEnv: RuntimeEnvironment;
+  grantedPermissions: string[] = [];
+  name: string;
+  version: number;
+
+  constructor(
+    system: System<HookT>,
+    name: string,
+    sandboxFactory: (plug: Plug<HookT>) => Sandbox
+  ) {
+    this.system = system;
+    this.name = name;
+    this.sandbox = sandboxFactory(this);
+    this.runtimeEnv = system.runtimeEnv;
+    this.version = new Date().getTime();
+  }
+
+  async load(manifest: Manifest<HookT>) {
+    this.manifest = manifest;
+    // TODO: These need to be explicitly granted, not just taken
+    this.grantedPermissions = manifest.requiredPermissions || [];
+  }
+
+  syscall(name: string, args: any[]): Promise<any> {
+    return this.system.syscallWithContext({ plug: this }, name, args);
+  }
+
+  canInvoke(name: string) {
+    if (!this.manifest) {
+      return false;
+    }
+    const funDef = this.manifest.functions[name];
+    if (!funDef) {
+      throw new Error(`Function ${name} not found in manifest`);
+    }
+    return !funDef.env || funDef.env === this.runtimeEnv;
+  }
+
+  async invoke(name: string, args: Array<any>): Promise<any> {
+    if (!this.sandbox.isLoaded(name)) {
+      const funDef = this.manifest!.functions[name];
+      if (!funDef) {
+        throw new Error(`Function ${name} not found in manifest`);
+      }
+      if (!this.canInvoke(name)) {
+        throw new Error(
+          `Function ${name} is not available in ${this.runtimeEnv}`
+        );
+      }
+      await this.sandbox.load(name, funDef.code!);
+    }
+    return await this.sandbox.invoke(name, args);
+  }
+
+  async stop() {
+    this.sandbox.stop();
+  }
+}

packages/plugos/plug_loader.ts

@@ -0,0 +1,71 @@
+import fs from "fs/promises";
+import watch from "node-watch";
+import path from "path";
+import { createSandbox } from "./environments/node_sandbox";
+import { System } from "./system";
+
+function extractPlugName(localPath: string): string {
+  const baseName = path.basename(localPath);
+  return baseName.substring(0, baseName.length - ".plug.json".length);
+}
+
+export class DiskPlugLoader<HookT> {
+  private system: System<HookT>;
+  private plugPath: string;
+
+  constructor(system: System<HookT>, plugPath: string) {
+    this.system = system;
+    this.plugPath = plugPath;
+  }
+
+  watcher() {
+    watch(this.plugPath, (eventType, localPath) => {
+      if (!localPath.endsWith(".plug.json")) {
+        return;
+      }
+      Promise.resolve()
+        .then(async () => {
+          try {
+            // let localPath = path.join(this.plugPath, filename);
+            const plugName = extractPlugName(localPath);
+            console.log("Change detected for", plugName);
+            try {
+              await fs.stat(localPath);
+            } catch (e) {
+              // Likely removed
+              await this.system.unload(plugName);
+            }
+            const plugDef = await this.loadPlugFromFile(localPath);
+          } catch (e) {
+            console.log("Ignoring something FYI", e);
+            // ignore, error handled by loadPlug
+          }
+        })
+        .catch(console.error);
+    });
+  }
+
+  private async loadPlugFromFile(localPath: string) {
+    const plug = await fs.readFile(localPath, "utf8");
+    const plugName = extractPlugName(localPath);
+
+    console.log("Now loading plug", plugName);
+    try {
+      const plugDef = JSON.parse(plug);
+      await this.system.load(plugName, plugDef, createSandbox);
+      return plugDef;
+    } catch (e) {
+      console.error("Could not parse plugin file", e);
+      throw e;
+    }
+  }
+
+  async loadPlugs() {
+    for (let filename of await fs.readdir(this.plugPath)) {
+      if (filename.endsWith(".plug.json")) {
+        let localPath = path.join(this.plugPath, filename);
+        await this.loadPlugFromFile(localPath);
+      }
+    }
+  }
+}

packages/plugos/runtime.test.ts

@@ -0,0 +1,115 @@
+import { createSandbox } from "./environments/node_sandbox";
+import { expect, test } from "@jest/globals";
+import { System } from "./system";
+
+test("Run a Node sandbox", async () => {
+  let system = new System("server");
+  system.registerSyscalls([], {
+    addNumbers: (ctx, a, b) => {
+      return a + b;
+    },
+    failingSyscall: () => {
+      throw new Error("#fail");
+    },
+  });
+  system.registerSyscalls(["restricted"], {
+    restrictedSyscall: () => {
+      return "restricted";
+    },
+  });
+  system.registerSyscalls(["dangerous"], {
+    dangerousSyscall: () => {
+      return "yay";
+    },
+  });
+  let plug = await system.load(
+    "test",
+    {
+      requiredPermissions: ["dangerous"],
+      functions: {
+        addTen: {
+          code: `(() => {
+          return {
+            default: (n) => {
+              return n + 10;
+            }
+          };
+        })()`,
+        },
+        addNumbersSyscall: {
+          code: `(() => {
+          return {
+            default: async (a, b) => {
+              return await self.syscall("addNumbers", a, b);
+            }
+          };
+        })()`,
+        },
+        errorOut: {
+          code: `(() => {
+          return {
+            default: () => {
+              throw Error("BOOM");
+            }
+          };
+        })()`,
+        },
+        errorOutSys: {
+          code: `(() => {
+          return {
+            default: async () => {
+              await self.syscall("failingSyscall");
+            }
+          };
+        })()`,
+        },
+        restrictedTest: {
+          code: `(() => {
+          return {
+            default: async () => {
+              await self.syscall("restrictedSyscall");
+            }
+          };
+        })()`,
+        },
+        dangerousTest: {
+          code: `(() => {
+          return {
+            default: async () => {
+              return await self.syscall("dangerousSyscall");
+            }
+          };
+        })()`,
+        },
+      },
+    },
+    createSandbox
+  );
+  expect(await plug.invoke("addTen", [10])).toBe(20);
+  for (let i = 0; i < 100; i++) {
+    expect(await plug.invoke("addNumbersSyscall", [10, i])).toBe(10 + i);
+  }
+  try {
+    await plug.invoke("errorOut", []);
+    expect(true).toBe(false);
+  } catch (e: any) {
+    expect(e.message).toBe("BOOM");
+  }
+  try {
+    await plug.invoke("errorOutSys", []);
+    expect(true).toBe(false);
+  } catch (e: any) {
+    expect(e.message).toBe("#fail");
+  }
+  try {
+    await plug.invoke("restrictedTest", []);
+    expect(true).toBe(false);
+  } catch (e: any) {
+    expect(e.message).toBe(
+      "Missing permission 'restricted' for syscall restrictedSyscall"
+    );
+  }
+  expect(await plug.invoke("dangerousTest", [])).toBe("yay");
+
+  await system.unloadAll();
+});

packages/plugos/sandbox.ts

@@ -0,0 +1,108 @@
+import { ControllerMessage, WorkerLike, WorkerMessage } from "./environments/worker";
+import { Plug } from "./plug";
+
+export type SandboxFactory<HookT> = (plug: Plug<HookT>) => Sandbox;
+
+export class Sandbox {
+  protected worker: WorkerLike;
+  protected reqId = 0;
+  protected outstandingInits = new Map<string, () => void>();
+  protected outstandingInvocations = new Map<
+    number,
+    { resolve: (result: any) => void; reject: (e: any) => void }
+  >();
+  protected loadedFunctions = new Set<string>();
+  protected plug: Plug<any>;
+
+  constructor(plug: Plug<any>, worker: WorkerLike) {
+    worker.onMessage = this.onMessage.bind(this);
+    this.worker = worker;
+    this.plug = plug;
+  }
+
+  isLoaded(name: string) {
+    return this.loadedFunctions.has(name);
+  }
+
+  async load(name: string, code: string): Promise<void> {
+    await this.worker.ready;
+    let outstandingInit = this.outstandingInits.get(name);
+    if (outstandingInit) {
+      // Load already in progress, let's wait for it...
+      return new Promise((resolve) => {
+        this.outstandingInits.set(name, () => {
+          outstandingInit!();
+          resolve();
+        });
+      });
+    }
+    this.worker.postMessage({
+      type: "load",
+      name: name,
+      code: code,
+    } as WorkerMessage);
+    return new Promise((resolve) => {
+      this.outstandingInits.set(name, () => {
+        this.loadedFunctions.add(name);
+        this.outstandingInits.delete(name);
+        resolve();
+      });
+    });
+  }
+
+  async onMessage(data: ControllerMessage) {
+    switch (data.type) {
+      case "inited":
+        let initCb = this.outstandingInits.get(data.name!);
+        initCb && initCb();
+        this.outstandingInits.delete(data.name!);
+        break;
+      case "syscall":
+        try {
+          let result = await this.plug.syscall(data.name!, data.args!);
+
+          this.worker.postMessage({
+            type: "syscall-response",
+            id: data.id,
+            result: result,
+          } as WorkerMessage);
+        } catch (e: any) {
+          // console.error("Syscall fail", e);
+          this.worker.postMessage({
+            type: "syscall-response",
+            id: data.id,
+            error: e.message,
+          } as WorkerMessage);
+        }
+        break;
+      case "result":
+        let resultCbs = this.outstandingInvocations.get(data.id!);
+        this.outstandingInvocations.delete(data.id!);
+        if (data.error) {
+          resultCbs && resultCbs.reject(new Error(data.error));
+        } else {
+          resultCbs && resultCbs.resolve(data.result);
+        }
+        break;
+      default:
+        console.error("Unknown message type", data);
+    }
+  }
+
+  async invoke(name: string, args: any[]): Promise<any> {
+    this.reqId++;
+    this.worker.postMessage({
+      type: "invoke",
+      id: this.reqId,
+      name,
+      args,
+    });
+    return new Promise((resolve, reject) => {
+      this.outstandingInvocations.set(this.reqId, { resolve, reject });
+    });
+  }
+
+  stop() {
+    this.worker.terminate();
+  }
+}

packages/plugos/syscalls/event.ts

@@ -0,0 +1,10 @@
+import { SysCallMapping } from "../system";
+import { EventHook } from "../hooks/event";
+
+export function eventSyscalls(eventHook: EventHook): SysCallMapping {
+  return {
+    "event.dispatch": async (ctx, eventName: string, data: any) => {
+      return eventHook.dispatchEvent(eventName, data);
+    },
+  };
+}

packages/plugos/syscalls/fetch.node.ts

@@ -0,0 +1,15 @@
+import fetch, { RequestInfo, RequestInit } from "node-fetch";
+import { SysCallMapping } from "../system";
+
+export function fetchSyscalls(): SysCallMapping {
+  return {
+    "fetch.json": async (ctx, url: RequestInfo, init: RequestInit) => {
+      let resp = await fetch(url, init);
+      return resp.json();
+    },
+    "fetch.text": async (ctx, url: RequestInfo, init: RequestInit) => {
+      let resp = await fetch(url, init);
+      return resp.text();
+    },
+  };
+}

packages/plugos/syscalls/jwt.ts

@@ -0,0 +1,22 @@
+import jwt, { Algorithm } from "jsonwebtoken";
+import { SysCallMapping } from "../system";
+
+export function jwtSyscalls(): SysCallMapping {
+  return {
+    "jwt.jwt": (
+      ctx,
+      hexSecret: string,
+      id: string,
+      algorithm: Algorithm,
+      expiry: string,
+      audience: string
+    ): string => {
+      return jwt.sign({}, Buffer.from(hexSecret, "hex"), {
+        keyid: id,
+        algorithm: algorithm,
+        expiresIn: expiry,
+        audience: audience,
+      });
+    },
+  };
+}

packages/plugos/syscalls/shell.node.ts

@@ -0,0 +1,20 @@
+import { promisify } from "util";
+import { execFile } from "child_process";
+import type { SysCallMapping } from "../system";
+
+const execFilePromise = promisify(execFile);
+
+export default function (cwd: string): SysCallMapping {
+  return {
+    "shell.run": async (
+      ctx,
+      cmd: string,
+      args: string[]
+    ): Promise<{ stdout: string; stderr: string }> => {
+      let { stdout, stderr } = await execFilePromise(cmd, args, {
+        cwd: cwd,
+      });
+      return { stdout, stderr };
+    },
+  };
+}

packages/plugos/syscalls/store.dexie_browser.test.ts

@@ -0,0 +1,49 @@
+import { createSandbox } from "../environments/node_sandbox";
+import { expect, test } from "@jest/globals";
+import { System } from "../system";
+import { storeSyscalls } from "./store.dexie_browser";
+
+// For testing in node.js
+require("fake-indexeddb/auto");
+
+test("Test store", async () => {
+  let system = new System("server");
+  system.registerSyscalls([], storeSyscalls("test", "test"));
+  let plug = await system.load(
+    "test",
+    {
+      functions: {
+        test1: {
+          code: `(() => {
+          return {
+            default: async () => {
+              await self.syscall("store.set", "name", "Pete");
+              return await self.syscall("store.get", "name");
+            }
+          };
+        })()`,
+        },
+        test2: {
+          code: `(() => {
+          return {
+            default: async () => {
+              await self.syscall("store.set", "page1:bl:page2:10", {title: "Something", meta: 20});
+              await self.syscall("store.batchSet", [
+                 {key: "page2:bl:page3", value: {title: "Something2", meta: 10}},
+                 {key: "page2:bl:page4", value: {title: "Something3", meta: 10}},
+              ]);
+              return await self.syscall("store.queryPrefix", "page2:");
+            }
+          };
+        })()`,
+        },
+      },
+    },
+    createSandbox
+  );
+  expect(await plug.invoke("test1", [])).toBe("Pete");
+  let queryResults = await plug.invoke("test2", []);
+  expect(queryResults.length).toBe(2);
+  expect(queryResults[0].value.meta).toBe(10);
+  await system.unloadAll();
+});

packages/plugos/syscalls/store.dexie_browser.ts

@@ -0,0 +1,66 @@
+import Dexie from "dexie";
+import { SysCallMapping } from "../system";
+
+export type KV = {
+  key: string;
+  value: any;
+};
+
+export function storeSyscalls(
+  dbName: string,
+  tableName: string
+): SysCallMapping {
+  const db = new Dexie(dbName);
+  db.version(1).stores({
+    [tableName]: "key",
+  });
+  const items = db.table(tableName);
+
+  return {
+    "store.delete": async (ctx, key: string) => {
+      await items.delete(key);
+    },
+
+    "store.deletePrefix": async (ctx, prefix: string) => {
+      await items.where("key").startsWith(prefix).delete();
+    },
+
+    "store.deleteAll": async () => {
+      await items.clear();
+    },
+
+    "store.set": async (ctx, key: string, value: any) => {
+      await items.put({
+        key,
+        value,
+      });
+    },
+
+    "store.batchSet": async (ctx, kvs: KV[]) => {
+      await items.bulkPut(
+        kvs.map(({ key, value }) => ({
+          key,
+          value,
+        }))
+      );
+    },
+
+    "store.get": async (ctx, key: string): Promise<any | null> => {
+      let result = await items.get({
+        key,
+      });
+      return result ? result.value : null;
+    },
+
+    "store.queryPrefix": async (
+      ctx,
+      keyPrefix: string
+    ): Promise<{ key: string; value: any }[]> => {
+      let results = await items.where("key").startsWith(keyPrefix).toArray();
+      return results.map((result) => ({
+        key: result.key,
+        value: result.value,
+      }));
+    },
+  };
+}

packages/plugos/syscalls/store.knex_node.test.ts

@@ -0,0 +1,40 @@
+import { createSandbox } from "../environments/node_sandbox";
+import { expect, test } from "@jest/globals";
+import { System } from "../system";
+import { ensureTable, storeSyscalls } from "./store.knex_node";
+import knex from "knex";
+import fs from "fs/promises";
+
+test("Test store", async () => {
+  const db = knex({
+    client: "better-sqlite3",
+    connection: {
+      filename: "test.db",
+    },
+    useNullAsDefault: true,
+  });
+  await ensureTable(db, "test_table");
+  let system = new System("server");
+  system.registerSyscalls([], storeSyscalls(db, "test_table"));
+  let plug = await system.load(
+    "test",
+    {
+      functions: {
+        test1: {
+          code: `(() => {
+          return {
+            default: async () => {
+              await self.syscall("store.set", "name", "Pete");
+              return await self.syscall("store.get", "name");
+            }
+          };
+        })()`,
+        },
+      },
+    },
+    createSandbox
+  );
+  expect(await plug.invoke("test1", [])).toBe("Pete");
+  await system.unloadAll();
+  await fs.unlink("test.db");
+});

packages/plugos/syscalls/store.knex_node.ts

@@ -0,0 +1,82 @@
+import { Knex } from "knex";
+import { SysCallMapping } from "../system";
+
+type Item = {
+  page: string;
+  key: string;
+  value: any;
+};
+
+export type KV = {
+  key: string;
+  value: any;
+};
+
+export async function ensureTable(db: Knex<any, unknown>, tableName: string) {
+  if (!(await db.schema.hasTable(tableName))) {
+    await db.schema.createTable(tableName, (table) => {
+      table.string("key");
+      table.text("value");
+      table.primary(["key"]);
+    });
+    console.log(`Created table ${tableName}`);
+  }
+}
+
+export function storeSyscalls(
+  db: Knex<any, unknown>,
+  tableName: string
+): SysCallMapping {
+  const apiObj: SysCallMapping = {
+    "store.delete": async (ctx, key: string) => {
+      await db<Item>(tableName).where({ key }).del();
+    },
+    "store.deletePrefix": async (ctx, prefix: string) => {
+      return db<Item>(tableName).andWhereLike("key", `${prefix}%`).del();
+    },
+    "store.deleteAll": async (ctx) => {
+      await db<Item>(tableName).del();
+    },
+    "store.set": async (ctx, key: string, value: any) => {
+      let changed = await db<Item>(tableName)
+        .where({ key })
+        .update("value", JSON.stringify(value));
+      if (changed === 0) {
+        await db<Item>(tableName).insert({
+          key,
+          value: JSON.stringify(value),
+        });
+      }
+    },
+    // TODO: Optimize
+    "store.batchSet": async (ctx, kvs: KV[]) => {
+      for (let { key, value } of kvs) {
+        await apiObj["store.set"](ctx, key, value);
+      }
+    },
+    "store.batchDelete": async (ctx, keys: string[]) => {
+      for (let key of keys) {
+        await apiObj["store.delete"](ctx, key);
+      }
+    },
+    "store.get": async (ctx, key: string): Promise<any | null> => {
+      let result = await db<Item>(tableName).where({ key }).select("value");
+      if (result.length) {
+        return JSON.parse(result[0].value);
+      } else {
+        return null;
+      }
+    },
+    "store.queryPrefix": async (ctx, prefix: string) => {
+      return (
+        await db<Item>(tableName)
+          .andWhereLike("key", `${prefix}%`)
+          .select("key", "value")
+      ).map(({ key, value }) => ({
+        key,
+        value: JSON.parse(value),
+      }));
+    },
+  };
+  return apiObj;
+}

packages/plugos/syscalls/transport.ts

@@ -0,0 +1,20 @@
+import { SyscallContext, SysCallMapping } from "../system";
+
+export function proxySyscalls(
+  names: string[],
+  transportCall: (
+    ctx: SyscallContext,
+    name: string,
+    ...args: any[]
+  ) => Promise<any>
+): SysCallMapping {
+  let syscalls: SysCallMapping = {};
+
+  for (let name of names) {
+    syscalls[name] = (ctx, ...args: any[]) => {
+      return transportCall(ctx, name, ...args);
+    };
+  }
+
+  return syscalls;
+}

packages/plugos/system.ts

@@ -0,0 +1,146 @@
+import { Hook, Manifest, RuntimeEnvironment } from "./types";
+import { EventEmitter } from "../silverbullet-common/event";
+import { SandboxFactory } from "./sandbox";
+import { Plug } from "./plug";
+
+export interface SysCallMapping {
+  [key: string]: (ctx: SyscallContext, ...args: any) => Promise<any> | any;
+}
+
+export type SystemJSON<HookT> = { [key: string]: Manifest<HookT> };
+
+export type SystemEvents<HookT> = {
+  plugLoaded: (name: string, plug: Plug<HookT>) => void;
+  plugUnloaded: (name: string, plug: Plug<HookT>) => void;
+};
+
+export type SyscallContext = {
+  plug: Plug<any>;
+};
+
+type SyscallSignature = (
+  ctx: SyscallContext,
+  ...args: any[]
+) => Promise<any> | any;
+
+type Syscall = {
+  requiredPermissions: string[];
+  callback: SyscallSignature;
+};
+
+export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
+  readonly runtimeEnv: RuntimeEnvironment;
+  protected plugs = new Map<string, Plug<HookT>>();
+  protected registeredSyscalls = new Map<string, Syscall>();
+  protected enabledHooks = new Set<Hook<HookT>>();
+
+  constructor(env: RuntimeEnvironment) {
+    super();
+    this.runtimeEnv = env;
+  }
+
+  get loadedPlugs(): Map<string, Plug<HookT>> {
+    return this.plugs;
+  }
+
+  addHook(feature: Hook<HookT>) {
+    this.enabledHooks.add(feature);
+    feature.apply(this);
+  }
+
+  registerSyscalls(
+    requiredCapabilities: string[],
+    ...registrationObjects: SysCallMapping[]
+  ) {
+    for (const registrationObject of registrationObjects) {
+      for (let [name, callback] of Object.entries(registrationObject)) {
+        this.registeredSyscalls.set(name, {
+          requiredPermissions: requiredCapabilities,
+          callback,
+        });
+      }
+    }
+  }
+
+  async syscallWithContext(
+    ctx: SyscallContext,
+    name: string,
+    args: any[]
+  ): Promise<any> {
+    const syscall = this.registeredSyscalls.get(name);
+    if (!syscall) {
+      throw Error(`Unregistered syscall ${name}`);
+    }
+    for (const permission of syscall.requiredPermissions) {
+      if (!ctx.plug) {
+        throw Error(`Syscall ${name} requires permission and no plug is set`);
+      }
+      if (!ctx.plug.grantedPermissions.includes(permission)) {
+        throw Error(`Missing permission '${permission}' for syscall ${name}`);
+      }
+    }
+    return Promise.resolve(syscall.callback(ctx, ...args));
+  }
+
+  async load(
+    name: string,
+    manifest: Manifest<HookT>,
+    sandboxFactory: SandboxFactory<HookT>
+  ): Promise<Plug<HookT>> {
+    if (this.plugs.has(name)) {
+      await this.unload(name);
+    }
+    // Validate
+    let errors: string[] = [];
+    for (const feature of this.enabledHooks) {
+      errors = [...errors, ...feature.validateManifest(manifest)];
+    }
+    if (errors.length > 0) {
+      throw new Error(`Invalid manifest: ${errors.join(", ")}`);
+    }
+    // Ok, let's load this thing!
+    const plug = new Plug(this, name, sandboxFactory);
+    await plug.load(manifest);
+    this.plugs.set(name, plug);
+    this.emit("plugLoaded", name, plug);
+    return plug;
+  }
+
+  async unload(name: string) {
+    const plug = this.plugs.get(name);
+    if (!plug) {
+      throw Error(`Plug ${name} not found`);
+    }
+    await plug.stop();
+    this.emit("plugUnloaded", name, plug);
+    this.plugs.delete(name);
+  }
+
+  toJSON(): SystemJSON<HookT> {
+    let plugJSON: { [key: string]: Manifest<HookT> } = {};
+    for (let [name, plug] of this.plugs) {
+      if (!plug.manifest) {
+        continue;
+      }
+      plugJSON[name] = plug.manifest;
+    }
+    return plugJSON;
+  }
+
+  async replaceAllFromJSON(
+    json: SystemJSON<HookT>,
+    sandboxFactory: SandboxFactory<HookT>
+  ) {
+    await this.unloadAll();
+    for (let [name, manifest] of Object.entries(json)) {
+      console.log("Loading plug", name);
+      await this.load(name, manifest, sandboxFactory);
+    }
+  }
+
+  async unloadAll(): Promise<void[]> {
+    return Promise.all(
+      Array.from(this.plugs.keys()).map(this.unload.bind(this))
+    );
+  }
+}

packages/plugos/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "include": ["bin/*", "environments/*", "hooks/**", "syscalls/*", "*"],
+  "compilerOptions": {
+    "target": "esnext",
+    "strict": true,
+    "moduleResolution": "node",
+    "module": "esnext",
+    "esModuleInterop": true,
+    "allowSyntheticDefaultImports": true,
+    "resolveJsonModule": true,
+    "jsx": "react-jsx",
+    "downlevelIteration": true
+  }
+}

packages/plugos/types.ts

@@ -0,0 +1,22 @@
+import { System } from "./system";
+
+export interface Manifest<HookT> {
+  requiredPermissions?: string[];
+  functions: {
+    [key: string]: FunctionDef<HookT>;
+  };
+}
+
+export type FunctionDef<HookT> = {
+  path?: string;
+  code?: string;
+  env?: RuntimeEnvironment;
+} & HookT;
+
+export type RuntimeEnvironment = "client" | "server";
+
+export interface Hook<HookT> {
+  validateManifest(manifest: Manifest<HookT>): string[];
+
+  apply(system: System<HookT>): void;
+}

packages/plugos/util.ts

@@ -0,0 +1,6 @@
+export function safeRun(fn: () => Promise<void>) {
+  fn().catch((e) => {
+    // console.error(e);
+    throw e;
+  });
+}

packages/plugos/yarn-error.log

@@ -0,0 +1,116 @@
+Arguments: 
+  /Users/zef/.nvm/versions/node/v16.13.2/bin/node /opt/homebrew/Cellar/yarn/1.22.18/libexec/bin/yarn.js install
+
+PATH: 
+  /Users/zef/.nvm/versions/node/v16.13.2/bin:/Library/Frameworks/Python.framework/Versions/2.7/bin:/Users/zef/.local/share/solana/install/active_release/bin:/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin:/Users/zef/.cargo/bin:/Users/zef/git/silverbullet/node_modules/.bin
+
+Yarn version: 
+  1.22.18
+
+Node version: 
+  16.13.2
+
+Platform: 
+  darwin arm64
+
+Trace: 
+  SyntaxError: /Users/zef/git/silverbullet/plugos/package.json: Unexpected token ] in JSON at position 755
+      at JSON.parse (<anonymous>)
+      at /opt/homebrew/Cellar/yarn/1.22.18/libexec/lib/cli.js:1625:59
+      at Generator.next (<anonymous>)
+      at step (/opt/homebrew/Cellar/yarn/1.22.18/libexec/lib/cli.js:310:30)
+      at /opt/homebrew/Cellar/yarn/1.22.18/libexec/lib/cli.js:321:13
+
+npm manifest: 
+  {
+    "name": "plugos",
+    "version": "0.0.1",
+    "license": "MIT",
+    "bin": {
+      "plugos-bundle": "./dist/plugos-bundle.js",
+      "plugos-server": "./dist/plugos-server.js"
+    },
+    "scripts": {
+      "watch": "rm -rf .parcel-cache && parcel watch",
+      "build": "parcel build",
+      "clean": "rm -rf dist",
+      "test": "jest"
+    },
+    "files": [
+      "dist"
+    ],
+    "targets": {
+      "plugos": {
+        "source": [
+          "bin/plugos-bundle.ts",
+          "bin/plugos-server.ts"
+        ],
+        "outputFormat": "commonjs",
+        "isLibrary": true,
+        "context": "node"
+      },
+      "test": {
+        "source": [
+          "runtime.test.ts",
+          "feature/endpoint.test.ts",
+          "syscall/store.knex_node.test.ts",
+          "syscall/store.dexie_browser.test.ts",
+        ],
+        "outputFormat": "commonjs",
+        "isLibrary": true,
+        "context": "node"
+      }
+    },
+    "dependencies": {
+      "@jest/globals": "^27.5.1",
+      "@parcel/optimizer-data-url": "2.3.2",
+      "@parcel/service-worker": "^2.3.2",
+      "@parcel/transformer-inline-string": "2.3.2",
+      "@types/cors": "^2.8.12",
+      "@types/express": "^4.17.13",
+      "better-sqlite3": "^7.5.0",
+      "body-parser": "^1.19.2",
+      "cors": "^2.8.5",
+      "esbuild": "^0.14.27",
+      "express": "^4.17.3",
+      "fake-indexeddb": "^3.1.7",
+      "jest": "^27.5.1",
+      "knex": "^1.0.4",
+      "node-cron": "^3.0.0",
+      "node-fetch": "2",
+      "parcel": "^2.3.2",
+      "supertest": "^6.2.2",
+      "vm2": "^3.9.9",
+      "yaml": "^1.10.2",
+      "yargs": "^17.3.1"
+    },
+    "devDependencies": {
+      "@parcel/packager-raw-url": "2.3.2",
+      "@parcel/service-worker": "^2.3.2",
+      "@parcel/transformer-inline-string": "2.3.2",
+      "@parcel/transformer-sass": "2.3.2",
+      "@parcel/transformer-webmanifest": "2.3.2",
+      "@parcel/validator-typescript": "^2.3.2",
+      "@types/events": "^3.0.0",
+      "@types/jest": "^27.4.1",
+      "@types/node": "^17.0.21",
+      "@types/node-cron": "^3.0.1",
+      "@types/node-fetch": "^2.6.1",
+      "@types/react": "^17.0.39",
+      "@types/react-dom": "^17.0.11",
+      "@types/supertest": "^2.0.11",
+      "@types/yaml": "^1.9.7",
+      "@vscode/sqlite3": "^5.0.7",
+      "assert": "^2.0.0",
+      "events": "^3.3.0",
+      "parcel": "^2.3.2",
+      "prettier": "^2.5.1",
+      "typescript": "^4.6.2"
+    }
+  }
+
+yarn manifest: 
+  No manifest
+
+Lockfile: 
+  No lockfile

packages/plugs/core/core.plug.yaml

@@ -0,0 +1,105 @@
+syntax:
+  HashTag:
+    firstCharacters:
+      - "#"
+    regex: "#[A-Za-z\\.]+"
+    styles:
+      color: blue
+  AtMention:
+    firstCharacters:
+      - "@"
+    regex: "@[A-Za-z\\.]+"
+    styles:
+      color: blue
+  NakedURL:
+    firstCharacters:
+      - "h"
+    regex: "https?:\\/\\/[-a-zA-Z0-9@:%._\\+~#=]{1,256}([-a-zA-Z0-9()@:%_\\+.~#?&=\\/]*)"
+    styles:
+      color: "#0330cb"
+      textDecoration: underline
+functions:
+  clearPageIndex:
+    path: "./page.ts:clearPageIndex"
+    env: server
+    events:
+      - page:saved
+      - page:deleted
+  pageQueryProvider:
+    path: ./page.ts:pageQueryProvider
+    events:
+      - query:page
+  parseIndexTextRepublish:
+    path: "./page.ts:parseIndexTextRepublish"
+    events:
+      - page:index_text
+  indexLinks:
+    path: "./page.ts:indexLinks"
+    events:
+      - page:index
+  linkQueryProvider:
+    path: ./page.ts:linkQueryProvider
+    events:
+      - query:link
+  indexItems:
+    path: "./item.ts:indexItems"
+    events:
+      - page:index
+  itemQueryProvider:
+    path: ./item.ts:queryProvider
+    events:
+      - query:item
+  deletePage:
+    path: "./page.ts:deletePage"
+    command:
+      name: "Page: Delete"
+  reindexSpaceCommand:
+    path: "./page.ts:reindexCommand"
+    command:
+      name: "Space: Reindex"
+  reindexSpace:
+    path: "./page.ts:reindexSpace"
+    env: server
+  renamePage:
+    path: "./page.ts:renamePage"
+    command:
+      name: "Page: Rename"
+      mac: Cmd-Alt-r
+      key: Ctrl-Alt-r
+  pageComplete:
+    path: "./page.ts:pageComplete"
+    events:
+      - page:complete
+  linkNavigate:
+    path: "./navigate.ts:linkNavigate"
+    command:
+      name: Navigate To page
+      key: Ctrl-Enter
+      mac: Cmd-Enter
+  clickNavigate:
+    path: "./navigate.ts:clickNavigate"
+    events:
+      - page:click
+  insertToday:
+    path: "./dates.ts:insertToday"
+    slashCommand:
+      name: today
+  insertTomorrow:
+    path: "./dates.ts:insertTomorrow"
+    slashCommand:
+      name: tomorrow
+  parseServerCommand:
+    path: ./page.ts:parseServerPageCommand
+    command:
+      name: "Debug: Parse Document on Server"
+  parsePage:
+    path: ./page.ts:parsePage
+  parseCommand:
+    path: ./page.ts:parsePageCommand
+    command:
+      name: "Debug: Parse Document"
+
+  instantiateTemplateCommand:
+    path: ./template.ts:instantiateTemplateCommand
+    command:
+      name: "Template: Instantiate for Page"

packages/plugs/core/dates.ts

@@ -0,0 +1,17 @@
+import { insertAtCursor } from "@silverbulletmd/plugos-silverbullet-syscall/editor";
+
+const dateMatchRegex = /(\d{4}\-\d{2}\-\d{2})/g;
+
+export function niceDate(d: Date): string {
+  return d.toISOString().split("T")[0];
+}
+
+export async function insertToday() {
+  await insertAtCursor(niceDate(new Date()));
+}
+
+export async function insertTomorrow() {
+  let d = new Date();
+  d.setDate(d.getDate() + 1);
+  await insertAtCursor(niceDate(d));
+}

packages/plugs/core/item.ts

@@ -0,0 +1,71 @@
+import type { IndexTreeEvent } from "@silverbulletmd/web/app_event";
+
+import { batchSet, scanPrefixGlobal } from "@silverbulletmd/plugos-silverbullet-syscall/index";
+import { collectNodesOfType, ParseTree, renderToText } from "@silverbulletmd/common/tree";
+import { removeQueries } from "../query/util";
+import { applyQuery, QueryProviderEvent } from "../query/engine";
+
+export type Item = {
+  name: string;
+  nested?: string;
+  // Not stored in DB
+  page?: string;
+  pos?: number;
+};
+
+export async function indexItems({ name, tree }: IndexTreeEvent) {
+  let items: { key: string; value: Item }[] = [];
+  removeQueries(tree);
+
+  console.log("Indexing items", name);
+
+  let coll = collectNodesOfType(tree, "ListItem");
+
+  coll.forEach((n) => {
+    if (!n.children) {
+      return;
+    }
+    let textNodes: ParseTree[] = [];
+    let nested: string | undefined;
+    for (let child of n.children!.slice(1)) {
+      if (child.type === "OrderedList" || child.type === "BulletList") {
+        nested = renderToText(child);
+        break;
+      }
+      textNodes.push(child);
+    }
+    let item = textNodes.map(renderToText).join("").trim();
+    let value: Item = {
+      name: item,
+    };
+    if (nested) {
+      value.nested = nested;
+    }
+    items.push({
+      key: `it:${n.from}`,
+      value,
+    });
+  });
+  console.log("Found", items.length, "item(s)");
+  await batchSet(name, items);
+}
+
+export async function queryProvider({
+  query,
+}: QueryProviderEvent): Promise<string> {
+  let allItems: Item[] = [];
+  for (let { key, page, value } of await scanPrefixGlobal("it:")) {
+    let [, pos] = key.split(":");
+    allItems.push({
+      ...value,
+      page: page,
+      pos: +pos,
+    });
+  }
+  let markdownItems = applyQuery(query, allItems).map(
+    (item) =>
+      `* [[${item.page}@${item.pos}]] ${item.name}` +
+      (item.nested ? "\n  " + item.nested : "")
+  );
+  return markdownItems.join("\n");
+}

packages/plugs/core/markup.ts

@@ -0,0 +1,33 @@
+import { getCursor, getText, insertAtPos, replaceRange } from "@silverbulletmd/plugos-silverbullet-syscall/editor";
+
+export async function toggleH1() {
+  await togglePrefix("# ");
+}
+
+export async function toggleH2() {
+  await togglePrefix("## ");
+}
+
+function lookBack(s: string, pos: number, backString: string): boolean {
+  return s.substring(pos - backString.length, pos) === backString;
+}
+
+async function togglePrefix(prefix: string) {
+  let text = await getText();
+  let pos = await getCursor();
+  if (text[pos] === "\n") {
+    pos--;
+  }
+  while (pos > 0 && text[pos] !== "\n") {
+    if (lookBack(text, pos, prefix)) {
+      // Already has this prefix, let's flip it
+      await replaceRange(pos - prefix.length, pos, "");
+      return;
+    }
+    pos--;
+  }
+  if (pos) {
+    pos++;
+  }
+  await insertAtPos(prefix, pos);
+}

packages/plugs/core/navigate.ts

@@ -0,0 +1,51 @@
+import type { ClickEvent } from "@silverbulletmd/web/app_event";
+import {
+  getCursor,
+  getText,
+  navigate as navigateTo,
+  openUrl
+} from "@silverbulletmd/plugos-silverbullet-syscall/editor";
+import { parseMarkdown } from "@silverbulletmd/plugos-silverbullet-syscall/markdown";
+import { nodeAtPos, ParseTree } from "@silverbulletmd/common/tree";
+
+const materializedQueryPrefix = /<!--\s*#query\s+/;
+
+async function actionClickOrActionEnter(mdTree: ParseTree | null) {
+  if (!mdTree) {
+    return;
+  }
+  console.log("Attempting to navigate based on syntax node", mdTree);
+  switch (mdTree.type) {
+    case "WikiLinkPage":
+      let pageLink = mdTree.children![0].text!;
+      let pos = "0";
+      if (pageLink.includes("@")) {
+        [pageLink, pos] = pageLink.split("@");
+      }
+      await navigateTo(pageLink, +pos);
+      break;
+    case "URL":
+    case "NakedURL":
+      await openUrl(mdTree.children![0].text!);
+      break;
+    case "Link":
+      await openUrl(mdTree.children![4].children![0].text!);
+      break;
+  }
+}
+
+export async function linkNavigate() {
+  let mdTree = await parseMarkdown(await getText());
+  let newNode = nodeAtPos(mdTree, await getCursor());
+  await actionClickOrActionEnter(newNode);
+}
+
+export async function clickNavigate(event: ClickEvent) {
+  // Navigate by default, don't navigate when Ctrl or Cmd is held
+  if (event.ctrlKey || event.metaKey) {
+    return;
+  }
+  let mdTree = await parseMarkdown(await getText());
+  let newNode = nodeAtPos(mdTree, event.pos);
+  await actionClickOrActionEnter(newNode);
+}

packages/plugs/core/page.ts

@@ -0,0 +1,258 @@
+import type { IndexEvent, IndexTreeEvent } from "@silverbulletmd/web/app_event";
+import {
+  batchSet,
+  clearPageIndex as clearPageIndexSyscall,
+  clearPageIndexForPage,
+  scanPrefixGlobal,
+  set
+} from "@silverbulletmd/plugos-silverbullet-syscall/index";
+import {
+  flashNotification,
+  getCurrentPage,
+  getText,
+  matchBefore,
+  navigate,
+  prompt
+} from "@silverbulletmd/plugos-silverbullet-syscall/editor";
+
+import { dispatch } from "@silverbulletmd/plugos-syscall/event";
+import {
+  deletePage as deletePageSyscall,
+  listPages,
+  readPage,
+  writePage
+} from "@silverbulletmd/plugos-silverbullet-syscall/space";
+import { invokeFunction } from "@silverbulletmd/plugos-silverbullet-syscall/system";
+import { parseMarkdown } from "@silverbulletmd/plugos-silverbullet-syscall/markdown";
+import {
+  addParentPointers,
+  collectNodesMatching,
+  ParseTree,
+  renderToText,
+  replaceNodesMatching
+} from "@silverbulletmd/common/tree";
+import { applyQuery, QueryProviderEvent } from "../query/engine";
+import { PageMeta } from "@silverbulletmd/common/types";
+import { extractMeta } from "../query/data";
+import { jsonToMDTable } from "../query/util";
+
+// Key space:
+//   pl:toPage:pos => pageName
+//   meta => metaJson
+
+export async function indexLinks({ name, tree }: IndexTreeEvent) {
+  let backLinks: { key: string; value: string }[] = [];
+  // [[Style Links]]
+  console.log("Now indexing", name);
+  let pageMeta = extractMeta(tree);
+  if (Object.keys(pageMeta).length > 0) {
+    await set(name, "meta:", pageMeta);
+  }
+
+  collectNodesMatching(tree, (n) => n.type === "WikiLinkPage").forEach((n) => {
+    let toPage = n.children![0].text!;
+    if (toPage.includes("@")) {
+      toPage = toPage.split("@")[0];
+    }
+    backLinks.push({
+      key: `pl:${toPage}:${n.from}`,
+      value: name,
+    });
+  });
+  console.log("Found", backLinks.length, "wiki link(s)");
+  await batchSet(name, backLinks);
+}
+
+export async function pageQueryProvider({
+  query,
+}: QueryProviderEvent): Promise<string> {
+  let allPages = await listPages();
+  if (query.select) {
+    let allPageMap: Map<string, any> = new Map(
+      allPages.map((pm) => [pm.name, pm])
+    );
+    for (let { page, value } of await scanPrefixGlobal("meta:")) {
+      let p = allPageMap.get(page);
+      if (p) {
+        for (let [k, v] of Object.entries(value)) {
+          p[k] = v;
+        }
+      }
+    }
+    allPages = [...allPageMap.values()];
+    return jsonToMDTable(applyQuery(query, allPages), (k, v) =>
+      k === "name" ? `[[${v}]]` : v
+    );
+  } else {
+    return applyQuery(query, allPages)
+      .map((pageMeta: PageMeta) => `* [[${pageMeta.name}]]`)
+      .join("\n");
+  }
+}
+
+export async function linkQueryProvider({
+  query,
+  pageName,
+}: QueryProviderEvent): Promise<string> {
+  let uniqueLinks = new Set<string>();
+  for (let { value: name } of await scanPrefixGlobal(`pl:${pageName}:`)) {
+    uniqueLinks.add(name);
+  }
+  let markdownLinks = applyQuery(
+    query,
+    [...uniqueLinks].map((l) => ({ name: l }))
+  ).map((pageMeta) => `* [[${pageMeta.name}]]`);
+  return markdownLinks.join("\n");
+}
+
+export async function deletePage() {
+  let pageName = await getCurrentPage();
+  console.log("Navigating to start page");
+  await navigate("start");
+  console.log("Deleting page from space");
+  await deletePageSyscall(pageName);
+}
+
+export async function renamePage() {
+  const oldName = await getCurrentPage();
+  console.log("Old name is", oldName);
+  const newName = await prompt(`Rename ${oldName} to:`, oldName);
+  if (!newName) {
+    return;
+  }
+  console.log("New name", newName);
+
+  let pagesToUpdate = await getBackLinks(oldName);
+  console.log("All pages containing backlinks", pagesToUpdate);
+
+  let text = await getText();
+  console.log("Writing new page to space");
+  await writePage(newName, text);
+  console.log("Navigating to new page");
+  await navigate(newName);
+  console.log("Deleting page from space");
+  await deletePageSyscall(oldName);
+
+  let pageToUpdateSet = new Set<string>();
+  for (let pageToUpdate of pagesToUpdate) {
+    pageToUpdateSet.add(pageToUpdate.page);
+  }
+
+  for (let pageToUpdate of pageToUpdateSet) {
+    if (pageToUpdate === oldName) {
+      continue;
+    }
+    console.log("Now going to update links in", pageToUpdate);
+    let { text } = await readPage(pageToUpdate);
+    // console.log("Received text", text);
+    if (!text) {
+      // Page likely does not exist, but at least we can skip it
+      continue;
+    }
+    let mdTree = await parseMarkdown(text);
+    addParentPointers(mdTree);
+    replaceNodesMatching(mdTree, (n): ParseTree | undefined | null => {
+      if (n.type === "WikiLinkPage") {
+        let pageName = n.children![0].text!;
+        if (pageName === oldName) {
+          n.children![0].text = newName;
+          return n;
+        }
+        // page name with @pos position
+        if (pageName.startsWith(`${oldName}@`)) {
+          let [, pos] = pageName.split("@");
+          n.children![0].text = `${newName}@${pos}`;
+          return n;
+        }
+      }
+      return;
+    });
+    // let newText = text.replaceAll(`[[${oldName}]]`, `[[${newName}]]`);
+    let newText = renderToText(mdTree);
+    if (text !== newText) {
+      console.log("Changes made, saving...");
+      await writePage(pageToUpdate, newText);
+    }
+  }
+}
+
+type BackLink = {
+  page: string;
+  pos: number;
+};
+
+async function getBackLinks(pageName: string): Promise<BackLink[]> {
+  let allBackLinks = await scanPrefixGlobal(`pl:${pageName}:`);
+  let pagesToUpdate: BackLink[] = [];
+  for (let { key, value } of allBackLinks) {
+    let keyParts = key.split(":");
+    pagesToUpdate.push({
+      page: value,
+      pos: +keyParts[keyParts.length - 1],
+    });
+  }
+  return pagesToUpdate;
+}
+
+export async function reindexCommand() {
+  await flashNotification("Reindexing...");
+  await invokeFunction("server", "reindexSpace");
+  await flashNotification("Reindexing done");
+}
+
+// Completion
+export async function pageComplete() {
+  let prefix = await matchBefore("\\[\\[[\\w\\s]*");
+  if (!prefix) {
+    return null;
+  }
+  let allPages = await listPages();
+  return {
+    from: prefix.from + 2,
+    options: allPages.map((pageMeta) => ({
+      label: pageMeta.name,
+      type: "page",
+    })),
+  };
+}
+
+// Server functions
+export async function reindexSpace() {
+  console.log("Clearing page index...");
+  await clearPageIndexSyscall();
+  console.log("Listing all pages");
+  let pages = await listPages();
+  for (let { name } of pages) {
+    console.log("Indexing", name);
+    const { text } = await readPage(name);
+    let parsed = await parseMarkdown(text);
+    await dispatch("page:index", {
+      name,
+      tree: parsed,
+    });
+  }
+}
+
+export async function clearPageIndex(page: string) {
+  console.log("Clearing page index for page", page);
+  await clearPageIndexForPage(page);
+}
+
+export async function parseIndexTextRepublish({ name, text }: IndexEvent) {
+  await dispatch("page:index", {
+    name,
+    tree: await parseMarkdown(text),
+  });
+}
+
+export async function parseServerPageCommand() {
+  console.log(await invokeFunction("server", "parsePage", await getText()));
+}
+
+export async function parsePageCommand() {
+  parsePage(await getText());
+}
+
+export async function parsePage(text: string) {
+  console.log("AST", JSON.stringify(await parseMarkdown(text), null, 2));
+}

packages/plugs/core/server.ts

@@ -0,0 +1,9 @@
+import { EndpointRequest, EndpointResponse } from "@silverbulletmd/plugos/hooks/endpoint";
+
+export function endpointTest(req: EndpointRequest): EndpointResponse {
+  console.log("I'm running on the server!", req);
+  return {
+    status: 200,
+    body: "Hello world!",
+  };
+}

packages/plugs/core/template.ts

@@ -0,0 +1,58 @@
+import { listPages, readPage, writePage } from "@silverbulletmd/plugos-silverbullet-syscall/space";
+import { filterBox, navigate, prompt } from "@silverbulletmd/plugos-silverbullet-syscall/editor";
+import { parseMarkdown } from "@silverbulletmd/plugos-silverbullet-syscall/markdown";
+import { extractMeta } from "../query/data";
+import { renderToText } from "@silverbulletmd/common/tree";
+import { niceDate } from "./dates";
+
+const pageTemplatePrefix = `template/page/`;
+
+export async function instantiateTemplateCommand() {
+  let allPages = await listPages();
+  let allPageTemplates = allPages.filter((pageMeta) =>
+    pageMeta.name.startsWith(pageTemplatePrefix)
+  );
+
+  let selectedTemplate = await filterBox(
+    "Template",
+    allPageTemplates,
+    "Select the template to create a new page from"
+  );
+
+  if (!selectedTemplate) {
+    return;
+  }
+  console.log("Selected template", selectedTemplate);
+
+  let { text } = await readPage(selectedTemplate.name);
+
+  let parseTree = await parseMarkdown(text);
+  let additionalPageMeta = extractMeta(parseTree, true);
+  console.log("Page meta", additionalPageMeta);
+
+  let pageName = await prompt("Name of new page", additionalPageMeta.name);
+  if (!pageName) {
+    return;
+  }
+  let pageText = replaceTemplateVars(renderToText(parseTree), pageName);
+  await writePage(pageName, pageText);
+  await navigate(pageName);
+}
+
+export function replaceTemplateVars(s: string, pageName: string): string {
+  return s.replaceAll(/\{\{([^\}]+)\}\}/g, (match, v) => {
+    switch (v) {
+      case "today":
+        return niceDate(new Date());
+      case "yesterday":
+        let yesterday = new Date();
+        yesterday.setDate(yesterday.getDate() - 1);
+        return niceDate(yesterday);
+      case "lastWeek":
+        let lastWeek = new Date();
+        lastWeek.setDate(lastWeek.getDate() - 7);
+        return niceDate(lastWeek);
+    }
+    return match;
+  });
+}

packages/plugs/core/word_count_command.ts

@@ -0,0 +1,9 @@
+function countWords(str: string): number {
+  const matches = str.match(/[\w\d\'-]+/gi);
+  return matches ? matches.length : 0;
+}
+
+function readingTime(wordCount: number): number {
+  // 225 is average word reading speed for adults
+  return Math.ceil(wordCount / 225);
+}

packages/plugs/emoji/.gitignore

@@ -0,0 +1 @@
+emoji-data.txt

packages/plugs/emoji/Makefile

@@ -0,0 +1,4 @@
+build:
+	curl https://unicode.org/Public/emoji/14.0/emoji-test.txt > emoji-data.txt
+	node build.js
+	rm emoji-data.txt

packages/plugs/emoji/build.js

@@ -0,0 +1,17 @@
+// Generates emoji.json from emoji-data.txt
+const { readFileSync, writeFileSync } = require("fs");
+
+const emojiRe = /#\s([^\s]+)\s+E[^\s]+\s+(.+)$/;
+
+let text = readFileSync("emoji-data.txt", "utf-8");
+const lines = text.split("\n").filter((line) => !line.startsWith("#"));
+
+let emoji = [];
+for (const line of lines) {
+  let match = emojiRe.exec(line);
+  if (match) {
+    emoji.push([match[1], match[2].toLowerCase().replaceAll(/\W+/g, "_")]);
+  }
+}
+
+writeFileSync("emoji.json", JSON.stringify(emoji));

packages/plugs/emoji/emoji.plug.yaml

@@ -0,0 +1,5 @@
+functions:
+  emojiCompleter:
+    path: "./emoji.ts:emojiCompleter"
+    events:
+      - page:complete

packages/plugs/emoji/emoji.ts

@@ -0,0 +1,26 @@
+// @ts-ignore
+import emojis from "./emoji.json";
+import { matchBefore } from "@silverbulletmd/plugos-silverbullet-syscall/editor";
+
+const emojiMatcher = /\(([^\)]+)\)\s+(.+)$/;
+
+export async function emojiCompleter() {
+  let prefix = await matchBefore(":[\\w]+");
+  if (!prefix) {
+    return null;
+  }
+  const textPrefix = prefix.text.substring(1); // Cut off the initial :
+  let filteredEmoji = emojis.filter(([_, shortcode]) =>
+    shortcode.includes(textPrefix)
+  );
+
+  return {
+    from: prefix.from,
+    filter: false,
+    options: filteredEmoji.map(([emoji, shortcode]) => ({
+      detail: shortcode,
+      label: emoji,
+      type: "emoji",
+    })),
+  };
+}

packages/plugs/ghost/ghost.plug.yaml

@@ -0,0 +1,15 @@
+functions:
+  downloadAllPostsCommand:
+    path: "./ghost.ts:downloadAllPostsCommand"
+    command:
+      name: "Ghost: Download Posts"
+  downloadAllPosts:
+    path: "./ghost.ts:downloadAllPosts"
+    env: server
+  publishCommand:
+    path: "./ghost.ts:publishCommand"
+    command:
+      name: "Ghost: Publish"
+  publish:
+    path: "./ghost.ts:publish"
+    env: server

packages/plugs/ghost/ghost.ts

@@ -0,0 +1,232 @@
+import { readPage, writePage } from "@silverbulletmd/plugos-silverbullet-syscall/space";
+import { json } from "@silverbulletmd/plugos-syscall/fetch";
+import { invokeFunction } from "@silverbulletmd/plugos-silverbullet-syscall/system";
+import { getCurrentPage, getText } from "@silverbulletmd/plugos-silverbullet-syscall/editor";
+import { cleanMarkdown } from "../markdown/util";
+import { parseMarkdown } from "@silverbulletmd/plugos-silverbullet-syscall/markdown";
+import { extractMeta } from "../query/data";
+
+type GhostConfig = {
+  url: string;
+  adminKey: string;
+  postPrefix: string;
+  pagePrefix: string;
+};
+
+type Post = {
+  id: string;
+  uuid: string;
+  title: string;
+  slug: string;
+  mobiledoc: string;
+  status: "draft" | "published";
+  visibility: string;
+  created_at: string;
+  upblished_at: string;
+  updated_at: string;
+  tags: Tag[];
+  primary_tag: Tag;
+  url: string;
+  excerpt: string;
+};
+
+type Tag = {
+  id: string;
+  name: string;
+  slug: string;
+  description: string | null;
+};
+
+type MobileDoc = {
+  version: string;
+  atoms: any[];
+  cards: Card[];
+};
+
+type Card = any[];
+
+function mobileDocToMarkdown(doc: string): string | null {
+  let mobileDoc = JSON.parse(doc) as MobileDoc;
+  if (mobileDoc.cards.length > 0 && mobileDoc.cards[0][0] === "markdown") {
+    return mobileDoc.cards[0][1].markdown;
+  }
+  return null;
+}
+
+function markdownToMobileDoc(text: string): string {
+  return JSON.stringify({
+    version: "0.3.1",
+    atoms: [],
+    cards: [["markdown", { markdown: text }]],
+    markups: [],
+    sections: [
+      [10, 0],
+      [1, "p", []],
+    ],
+  });
+}
+
+class GhostAdmin {
+  private token?: string;
+
+  constructor(private url: string, private key: string) {}
+
+  async init() {
+    const [id, secret] = this.key.split(":");
+
+    this.token = await self.syscall(
+      "jwt.jwt",
+      secret,
+      id,
+      "HS256",
+      "5m",
+      "/v3/admin/"
+    );
+  }
+
+  async listPosts(): Promise<Post[]> {
+    let result = await json(
+      `${this.url}/ghost/api/v3/admin/posts?order=published_at+DESC`,
+      {
+        headers: {
+          Authorization: `Ghost ${this.token}`,
+        },
+      }
+    );
+
+    return result.posts;
+  }
+
+  async listMarkdownPosts(): Promise<Post[]> {
+    let markdownPosts: Post[] = [];
+    for (let post of await this.listPosts()) {
+      let mobileDoc = JSON.parse(post.mobiledoc) as MobileDoc;
+      if (mobileDoc.cards.length > 0 && mobileDoc.cards[0][0] === "markdown") {
+        markdownPosts.push(post);
+      }
+    }
+    return markdownPosts;
+  }
+
+  publishPost(post: Partial<Post>): Promise<any> {
+    return this.publish("posts", post);
+  }
+
+  publishPage(post: Partial<Post>): Promise<any> {
+    return this.publish("pages", post);
+  }
+
+  async publish(what: "pages" | "posts", post: Partial<Post>): Promise<any> {
+    let oldPostQuery = await json(
+      `${this.url}/ghost/api/v3/admin/${what}/slug/${post.slug}`,
+      {
+        headers: {
+          Authorization: `Ghost ${this.token}`,
+          "Content-Type": "application/json",
+        },
+      }
+    );
+    if (!oldPostQuery[what]) {
+      // New!
+      if (!post.status) {
+        post.status = "draft";
+      }
+      let result = await json(`${this.url}/ghost/api/v3/admin/${what}`, {
+        method: "POST",
+        headers: {
+          Authorization: `Ghost ${this.token}`,
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify({
+          [what]: [post],
+        }),
+      });
+      return result[what][0];
+    } else {
+      let oldPost: Post = oldPostQuery[what][0];
+      post.updated_at = oldPost.updated_at;
+      let result = await json(
+        `${this.url}/ghost/api/v3/admin/${what}/${oldPost.id}`,
+        {
+          method: "PUT",
+          headers: {
+            Authorization: `Ghost ${this.token}`,
+            "Content-Type": "application/json",
+          },
+          body: JSON.stringify({
+            [what]: [post],
+          }),
+        }
+      );
+      return result[what][0];
+    }
+  }
+}
+
+function postToMarkdown(post: Post): string {
+  let text = mobileDocToMarkdown(post.mobiledoc);
+  return `# ${post.title}\n${text}`;
+}
+
+const postRegex = /#\s*([^\n]+)\n([^$]+)$/;
+
+async function markdownToPost(text: string): Promise<Partial<Post>> {
+  let match = postRegex.exec(text);
+  if (match) {
+    let [, title, content] = match;
+    return {
+      title,
+      mobiledoc: markdownToMobileDoc(await cleanMarkdown(content)),
+    };
+  }
+  throw Error("Post should stat with a # header");
+}
+
+async function getConfig(): Promise<GhostConfig> {
+  let { text } = await readPage("ghost-config");
+  let parsedContent = await parseMarkdown(text);
+  let pageMeta = await extractMeta(parsedContent);
+  return pageMeta as GhostConfig;
+}
+
+export async function downloadAllPostsCommand() {
+  await invokeFunction("server", "downloadAllPosts");
+}
+
+export async function downloadAllPosts() {
+  let config = await getConfig();
+  let admin = new GhostAdmin(config.url, config.adminKey);
+  await admin.init();
+  let allPosts = await admin.listMarkdownPosts();
+  for (let post of allPosts) {
+    let text = mobileDocToMarkdown(post.mobiledoc);
+    text = `# ${post.title}\n${text}`;
+    await writePage(`${config.postPrefix}/${post.slug}`, text);
+  }
+}
+export async function publishCommand() {
+  await invokeFunction(
+    "server",
+    "publish",
+    await getCurrentPage(),
+    await getText()
+  );
+}
+
+export async function publish(name: string, text: string) {
+  let config = await getConfig();
+  let admin = new GhostAdmin(config.url, config.adminKey);
+  await admin.init();
+  let post = await markdownToPost(text);
+  if (name.startsWith(config.postPrefix)) {
+    post.slug = name.substring(config.postPrefix.length + 1);
+    await admin.publishPost(post);
+    console.log("Done!");
+  } else if (name.startsWith(config.pagePrefix)) {
+    post.slug = name.substring(config.pagePrefix.length + 1);
+    await admin.publishPage(post);
+    console.log("Done!");
+  } else {
+    console.error("Not in either the post or page prefix");
+  }
+}

packages/plugs/git/git.plug.yaml

@@ -0,0 +1,20 @@
+requiredPermissions:
+  - shell
+functions:
+  snapshotCommand:
+    path: "./git.ts:snapshotCommand"
+    env: client
+    command:
+      name: "Git: Snapshot"
+  syncCommand:
+    path: "./git.ts:syncCommand"
+    env: client
+    command:
+      name: "Git: Sync"
+  commit:
+    path: "./git.ts:commit"
+    env: server
+  sync:
+    path: "./git.ts:sync"
+    env: server
+

packages/plugs/git/git.ts

@@ -0,0 +1,45 @@
+import { run } from "@silverbulletmd/plugos-syscall/shell";
+import { flashNotification, prompt } from "@silverbulletmd/plugos-silverbullet-syscall/editor";
+import { invokeFunction } from "@silverbulletmd/plugos-silverbullet-syscall/system";
+
+export async function commit(message?: string) {
+  if (!message) {
+    message = "Snapshot";
+  }
+  console.log(
+    "Snapshotting the current space to git with commit message",
+    message
+  );
+  await run("git", ["add", "./*.md"]);
+  try {
+    await run("git", ["commit", "-a", "-m", message]);
+  } catch (e) {
+    // We can ignore, this happens when there's no changes to commit
+  }
+  console.log("Done!");
+}
+
+export async function snapshotCommand() {
+  let revName = await prompt(`Revision name:`);
+  if (!revName) {
+    revName = "Snapshot";
+  }
+  console.log("Revision name", revName);
+  await invokeFunction("server", "commit", revName);
+}
+
+export async function syncCommand() {
+  await flashNotification("Syncing with git");
+  await invokeFunction("server", "sync");
+  await flashNotification("Git sync complete!");
+}
+
+export async function sync() {
+  console.log("Going to sync with git");
+  await commit();
+  console.log("Then pulling from remote");
+  await run("git", ["pull"]);
+  console.log("And then pushing to remote");
+  await run("git", ["push"]);
+  console.log("Done!");
+}

packages/plugs/lib/util.ts

@@ -0,0 +1,14 @@
+export async function replaceAsync(
+  str: string,
+  regex: RegExp,
+  asyncFn: (match: string, ...args: any[]) => Promise<string>
+) {
+  const promises: Promise<string>[] = [];
+  str.replace(regex, (match: string, ...args: any[]): string => {
+    const promise = asyncFn(match, ...args);
+    promises.push(promise);
+    return "";
+  });
+  const data = await Promise.all(promises);
+  return str.replace(regex, () => data.shift()!);
+}

packages/plugs/markdown/markdown.plug.yaml

@@ -0,0 +1,14 @@
+functions:
+  toggle:
+    path: "./markdown.ts:togglePreview"
+    command:
+      name: "Toggle Markdown Preview"
+      key: Ctrl-p
+      mac: Cmd-p
+  preview:
+    path: "./preview.ts:updateMarkdownPreview"
+    env: client
+    events:
+      - plug:load
+      - editor:updated
+      - editor:pageSwitched

packages/plugs/markdown/markdown.ts

@@ -0,0 +1,18 @@
+import { hideRhs } from "@silverbulletmd/plugos-silverbullet-syscall/editor";
+import { invokeFunction } from "@silverbulletmd/plugos-silverbullet-syscall/system";
+import * as clientStore from "@silverbulletmd/plugos-silverbullet-syscall/clientStore";
+
+export async function togglePreview() {
+  let currentValue = !!(await clientStore.get("enableMarkdownPreview"));
+  await clientStore.set("enableMarkdownPreview", !currentValue);
+  if (!currentValue) {
+    await invokeFunction("client", "preview");
+    // updateMarkdownPreview();
+  } else {
+    await hideMarkdownPreview();
+  }
+}
+
+async function hideMarkdownPreview() {
+  await hideRhs();
+}

packages/plugs/markdown/preview.ts

@@ -0,0 +1,80 @@
+import MarkdownIt from "markdown-it";
+import { getText, showRhs } from "@silverbulletmd/plugos-silverbullet-syscall/editor";
+import * as clientStore from "@silverbulletmd/plugos-silverbullet-syscall/clientStore";
+import { cleanMarkdown } from "./util";
+
+const css = `
+<style>
+body {
+  font-family: georgia,times,serif;
+  font-size: 14pt;
+  max-width: 800px;
+  margin-left: auto;
+  margin-right: auto;
+  padding-left: 20px;
+  padding-right: 20px;
+}
+
+table {
+  width: 100%;
+  border-spacing: 0;
+}
+
+thead tr {
+    background-color: #333;
+    color: #eee;
+}
+
+th, td {
+    padding: 8px;
+}
+
+tbody tr:nth-of-type(even) {
+    background-color: #f3f3f3;
+}
+
+a[href] {
+  text-decoration: none;
+}
+
+blockquote {
+  border-left: 1px solid #333;
+  margin-left: 2px;
+  padding-left: 10px;
+}
+
+hr {
+    margin: 1em 0 1em 0;
+    text-align: center;
+    border-color: #777;
+    border-width: 0;
+    border-style: dotted;
+}
+
+hr:after {
+    content: "···";
+    letter-spacing: 1em;
+}
+
+</style>
+`;
+
+var taskLists = require("markdown-it-task-lists");
+
+const md = new MarkdownIt({
+  linkify: true,
+  html: false,
+  typographer: true,
+}).use(taskLists);
+
+export async function updateMarkdownPreview() {
+  if (!(await clientStore.get("enableMarkdownPreview"))) {
+    return;
+  }
+  let text = await getText();
+  let cleanMd = await cleanMarkdown(text);
+  await showRhs(
+    `<html><head>${css}</head><body>${md.render(cleanMd)}</body></html>`,
+    2
+  );
+}

packages/plugs/markdown/util.ts

@@ -0,0 +1,33 @@
+import { findNodeOfType, renderToText, replaceNodesMatching } from "@silverbulletmd/common/tree";
+import { parseMarkdown } from "@silverbulletmd/plugos-silverbullet-syscall/markdown";
+
+export function encodePageUrl(name: string): string {
+  return name.replaceAll(" ", "_");
+}
+
+export async function cleanMarkdown(text: string): Promise<string> {
+  let mdTree = await parseMarkdown(text);
+  replaceNodesMatching(mdTree, (n) => {
+    if (n.type === "WikiLink") {
+      const page = n.children![1].children![0].text!;
+      return {
+        // HACK
+        text: `[${page}](/${encodePageUrl(page)})`,
+      };
+    }
+    // Simply get rid of these
+    if (n.type === "CommentBlock" || n.type === "Comment") {
+      return null;
+    }
+    if (n.type === "FencedCode") {
+      let codeInfoNode = findNodeOfType(n, "CodeInfo");
+      if (!codeInfoNode) {
+        return;
+      }
+      if (codeInfoNode.children![0].text === "meta") {
+        return null;
+      }
+    }
+  });
+  return renderToText(mdTree);
+}

packages/plugs/mattermost/mattermost.plug.yaml

@@ -0,0 +1,5 @@
+functions:
+  test:
+    path: mattermost.ts:savedPostsQueryProvider
+    events:
+      - query:mm-saved

packages/plugs/mattermost/mattermost.ts

@@ -0,0 +1,80 @@
+import { Client4 } from "@mattermost/client";
+import { applyQuery, QueryProviderEvent } from "../query/engine";
+import { readPage } from "@silverbulletmd/plugos-silverbullet-syscall/space";
+import { parseMarkdown } from "@silverbulletmd/plugos-silverbullet-syscall/markdown";
+import { extractMeta } from "../query/data";
+import { niceDate } from "../core/dates";
+import { Post } from "@mattermost/types/lib/posts";
+
+type AugmentedPost = Post & {
+  // Dates we can use to filter
+  createdAt: string;
+  updatedAt: string;
+  editedAt: string;
+};
+
+// https://community.mattermost.com/private-core/pl/rbp7a7jtr3f89nzsefo6ftqt3o
+
+function mattermostDesktopUrlForPost(
+  url: string,
+  teamName: string,
+  postId: string
+) {
+  return `${url.replace("https://", "mattermost://")}/${teamName}/pl/${postId}`;
+}
+type MattermostConfig = {
+  url: string;
+  token: string;
+};
+
+async function getConfig(): Promise<MattermostConfig> {
+  let { text } = await readPage("mattermost-config");
+  let parsedContent = await parseMarkdown(text);
+  let pageMeta = await extractMeta(parsedContent);
+  return pageMeta as MattermostConfig;
+}
+
+function augmentPost(post: AugmentedPost) {
+  if (post.create_at) {
+    post.createdAt = niceDate(new Date(post.create_at));
+  }
+  if (post.update_at) {
+    post.updatedAt = niceDate(new Date(post.update_at));
+  }
+  if (post.edit_at) {
+    post.editedAt = niceDate(new Date(post.edit_at));
+  }
+}
+
+export async function savedPostsQueryProvider({
+  query,
+}: QueryProviderEvent): Promise<string> {
+  let config = await getConfig();
+  let client = new Client4();
+  client.setUrl(config.url);
+  client.setToken(config.token);
+  let me = await client.getMe();
+  let postCollection = await client.getFlaggedPosts(me.id);
+  let savedPosts: AugmentedPost[] = [];
+  for (let order of postCollection.order) {
+    let post = postCollection.posts[order];
+    augmentPost(post);
+    savedPosts.push(post);
+  }
+  let savedPostsMd = [];
+  savedPosts = applyQuery(query, savedPosts);
+  for (let savedPost of savedPosts) {
+    let channel = await client.getChannel(savedPost.channel_id);
+    let team = await client.getTeam(channel.team_id);
+    savedPostsMd.push(
+      `@${(await client.getUser(savedPost.user_id)).username} [${
+        savedPost.createdAt
+      }](${mattermostDesktopUrlForPost(
+        client.url,
+        team.name,
+        savedPost.id
+      )}):\n> ${savedPost.message.substring(0, 1000).replaceAll(/\n/g, "\n> ")}`
+    );
+  }
+  return savedPostsMd.join("\n\n");
+}

packages/plugs/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "@silverbulletmd/plugs",
+  "version": "1.0.0",
+  "scripts": {
+    "generate": "lezer-generator query/query.grammar -o query/parse-query.js",
+    "watch": "plugos-bundle -w --dist dist */*.plug.yaml",
+    "build": "plugos-bundle --dist dist */*.plug.yaml"
+  },
+  "dependencies": {
+    "@jest/globals": "^27.5.1",
+    "@lezer/generator": "^0.15.4",
+    "@lezer/lr": "^0.15.8",
+    "@mattermost/client": "^6.7.0-0",
+    "@mattermost/types": "^6.7.0-0",
+    "@silverbulletmd/plugos": "workspace:*",
+    "@silverbulletmd/plugos-silverbullet-syscall": "workspace:*",
+    "@silverbulletmd/plugos-syscall": "workspace:*",
+    "@types/yaml": "^1.9.7",
+    "markdown-it": "^12.3.2",
+    "markdown-it-task-lists": "^2.1.1",
+    "yaml": "^2.0.0"
+  },
+  "devDependencies": {
+    "@types/markdown-it": "^12.2.3"
+  }
+}

packages/plugs/query/command.ts

@@ -0,0 +1,7 @@
+import { getCursor, insertAtCursor, moveCursor } from "@silverbulletmd/plugos-silverbullet-syscall/editor";
+
+export async function insertQuery() {
+  let cursorPos = await getCursor();
+  await insertAtCursor(`<!-- #query  -->\n\n<!-- #end -->`);
+  await moveCursor(cursorPos + 12);
+}

packages/plugs/query/data.ts

@@ -0,0 +1,93 @@
+// Index key space:
+// data:page@pos
+
+import type { IndexTreeEvent } from "@silverbulletmd/web/app_event";
+import { batchSet, scanPrefixGlobal } from "@silverbulletmd/plugos-silverbullet-syscall";
+import { collectNodesOfType, findNodeOfType, ParseTree, replaceNodesMatching } from "@silverbulletmd/common/tree";
+import { parse as parseYaml, parseAllDocuments } from "yaml";
+import type { QueryProviderEvent } from "./engine";
+import { applyQuery } from "./engine";
+import { jsonToMDTable, removeQueries } from "./util";
+
+export async function indexData({ name, tree }: IndexTreeEvent) {
+  let dataObjects: { key: string; value: Object }[] = [];
+
+  removeQueries(tree);
+
+  collectNodesOfType(tree, "FencedCode").forEach((t) => {
+    let codeInfoNode = findNodeOfType(t, "CodeInfo");
+    if (!codeInfoNode) {
+      return;
+    }
+    if (codeInfoNode.children![0].text !== "data") {
+      return;
+    }
+    let codeTextNode = findNodeOfType(t, "CodeText");
+    if (!codeTextNode) {
+      // Honestly, this shouldn't happen
+      return;
+    }
+    let codeText = codeTextNode.children![0].text!;
+    try {
+      // We support multiple YAML documents in one block
+      for (let doc of parseAllDocuments(codeText)) {
+        if (!doc.contents) {
+          continue;
+        }
+        console.log(doc.contents.toJSON());
+        dataObjects.push({
+          key: `data:${name}@${t.from! + doc.range[0]}`,
+          value: doc.contents.toJSON(),
+        });
+      }
+      // console.log("Parsed data", parsedData);
+    } catch (e) {
+      console.error("Could not parse data", codeText, "error:", e);
+      return;
+    }
+  });
+  console.log("Found", dataObjects.length, "data objects");
+  await batchSet(name, dataObjects);
+}
+
+export function extractMeta(parseTree: ParseTree, remove = false): any {
+  let data = {};
+  replaceNodesMatching(parseTree, (t) => {
+    if (t.type !== "FencedCode") {
+      return;
+    }
+    let codeInfoNode = findNodeOfType(t, "CodeInfo");
+    if (!codeInfoNode) {
+      return;
+    }
+    if (codeInfoNode.children![0].text !== "meta") {
+      return;
+    }
+    let codeTextNode = findNodeOfType(t, "CodeText");
+    if (!codeTextNode) {
+      // Honestly, this shouldn't happen
+      return;
+    }
+    let codeText = codeTextNode.children![0].text!;
+    data = parseYaml(codeText);
+    return remove ? null : undefined;
+  });
+
+  return data;
+}
+
+export async function queryProvider({
+  query,
+}: QueryProviderEvent): Promise<string> {
+  let allData: any[] = [];
+  for (let { key, page, value } of await scanPrefixGlobal("data:")) {
+    let [, pos] = key.split("@");
+    allData.push({
+      ...value,
+      page: page,
+      pos: +pos,
+    });
+  }
+  let resultData = applyQuery(query, allData);
+  return jsonToMDTable(resultData);
+}

packages/plugs/query/engine.test.ts

@@ -0,0 +1,95 @@
+import { expect, test } from "@jest/globals";
+import { applyQuery, parseQuery } from "./engine";
+
+test("Test parser", () => {
+  let parsedBasicQuery = parseQuery(`page`);
+  expect(parsedBasicQuery.table).toBe("page");
+
+  let parsedQuery1 = parseQuery(
+    `task where completed = false and dueDate <= "{{today}}" order by dueDate desc limit 5`
+  );
+  expect(parsedQuery1.table).toBe("task");
+  expect(parsedQuery1.orderBy).toBe("dueDate");
+  expect(parsedQuery1.orderDesc).toBe(true);
+  expect(parsedQuery1.limit).toBe(5);
+  expect(parsedQuery1.filter.length).toBe(2);
+  expect(parsedQuery1.filter[0]).toStrictEqual({
+    op: "=",
+    prop: "completed",
+    value: false,
+  });
+  expect(parsedQuery1.filter[1]).toStrictEqual({
+    op: "<=",
+    prop: "dueDate",
+    value: "{{today}}",
+  });
+
+  let parsedQuery2 = parseQuery(`page where name =~ /interview\\/.*/"`);
+  expect(parsedQuery2.table).toBe("page");
+  expect(parsedQuery2.filter.length).toBe(1);
+  expect(parsedQuery2.filter[0]).toStrictEqual({
+    op: "=~",
+    prop: "name",
+    value: "interview\\/.*",
+  });
+
+  let parsedQuery3 = parseQuery(`page where something != null`);
+  expect(parsedQuery3.table).toBe("page");
+  expect(parsedQuery3.filter.length).toBe(1);
+  expect(parsedQuery3.filter[0]).toStrictEqual({
+    op: "!=",
+    prop: "something",
+    value: null,
+  });
+
+  expect(parseQuery(`page select name`).select).toStrictEqual(["name"]);
+  expect(parseQuery(`page select name, age`).select).toStrictEqual([
+    "name",
+    "age",
+  ]);
+});
+
+test("Test performing the queries", () => {
+  let data: any[] = [
+    { name: "interview/My Interview", lastModified: 1 },
+    { name: "interview/My Interview 2", lastModified: 2 },
+    { name: "Pete", age: 38 },
+    { name: "Angie", age: 28 },
+  ];
+
+  expect(
+    applyQuery(parseQuery(`page where name =~ /interview\\/.*/`), data)
+  ).toStrictEqual([
+    { name: "interview/My Interview", lastModified: 1 },
+    { name: "interview/My Interview 2", lastModified: 2 },
+  ]);
+  expect(
+    applyQuery(
+      parseQuery(`page where name =~ /interview\\/.*/ order by lastModified`),
+      data
+    )
+  ).toStrictEqual([
+    { name: "interview/My Interview", lastModified: 1 },
+    { name: "interview/My Interview 2", lastModified: 2 },
+  ]);
+  expect(
+    applyQuery(
+      parseQuery(
+        `page where name  =~ /interview\\/.*/ order by lastModified desc`
+      ),
+      data
+    )
+  ).toStrictEqual([
+    { name: "interview/My Interview 2", lastModified: 2 },
+    { name: "interview/My Interview", lastModified: 1 },
+  ]);
+  expect(applyQuery(parseQuery(`page where age > 30`), data)).toStrictEqual([
+    { name: "Pete", age: 38 },
+  ]);
+  expect(
+    applyQuery(parseQuery(`page where age > 28 and age < 38`), data)
+  ).toStrictEqual([]);
+  expect(
+    applyQuery(parseQuery(`page where age > 30 select name`), data)
+  ).toStrictEqual([{ name: "Pete" }]);
+});

packages/plugs/query/engine.ts

@@ -0,0 +1,193 @@
+import { collectNodesOfType, findNodeOfType, replaceNodesMatching } from "@silverbulletmd/common/tree";
+import { lezerToParseTree } from "@silverbulletmd/common/parse_tree";
+
+// @ts-ignore
+import { parser } from "./parse-query";
+
+export type QueryProviderEvent = {
+  query: ParsedQuery;
+  pageName: string;
+};
+
+export type Filter = {
+  op: string;
+  prop: string;
+  value: any;
+};
+
+export type ParsedQuery = {
+  table: string;
+  orderBy?: string;
+  orderDesc?: boolean;
+  limit?: number;
+  filter: Filter[];
+  select?: string[];
+};
+
+export function parseQuery(query: string): ParsedQuery {
+  let n = lezerToParseTree(query, parser.parse(query).topNode);
+  // Clean the tree a bit
+  replaceNodesMatching(n, (n) => {
+    if (!n.type) {
+      let trimmed = n.text!.trim();
+      if (!trimmed) {
+        return null;
+      }
+      n.text = trimmed;
+    }
+  });
+
+  // console.log("Parsed", JSON.stringify(n, null, 2));
+
+  let queryNode = n.children![0];
+  let parsedQuery: ParsedQuery = {
+    table: queryNode.children![0].children![0].text!,
+    filter: [],
+  };
+  let orderByNode = findNodeOfType(queryNode, "OrderClause");
+  if (orderByNode) {
+    let nameNode = findNodeOfType(orderByNode, "Name");
+    parsedQuery.orderBy = nameNode!.children![0].text!;
+    let orderNode = findNodeOfType(orderByNode, "Order");
+    parsedQuery.orderDesc = orderNode
+      ? orderNode.children![0].text! === "desc"
+      : false;
+  }
+  let limitNode = findNodeOfType(queryNode, "LimitClause");
+  if (limitNode) {
+    let nameNode = findNodeOfType(limitNode, "Number");
+    parsedQuery.limit = +nameNode!.children![0].text!;
+  }
+  let filterNodes = collectNodesOfType(queryNode, "FilterExpr");
+  for (let filterNode of filterNodes) {
+    let val: any = undefined;
+    let valNode = filterNode.children![2].children![0];
+    switch (valNode.type) {
+      case "Number":
+        val = valNode.children![0].text!;
+        break;
+      case "Bool":
+        val = valNode.children![0].text! === "true";
+        break;
+      case "Null":
+        val = null;
+        break;
+      case "Name":
+        val = valNode.children![0].text!;
+        break;
+      case "Regex":
+        val = valNode.children![0].text!;
+        val = val.substring(1, val.length - 1);
+        break;
+      case "String":
+        val = valNode.children![0].text!;
+        val = val.substring(1, val.length - 1);
+        break;
+    }
+    let f: Filter = {
+      prop: filterNode.children![0].children![0].text!,
+      op: filterNode.children![1].text!,
+      value: val,
+    };
+    parsedQuery.filter.push(f);
+  }
+  let selectNode = findNodeOfType(queryNode, "SelectClause");
+  if (selectNode) {
+    console.log("Select node", JSON.stringify(selectNode));
+    parsedQuery.select = [];
+    collectNodesOfType(selectNode, "Name").forEach((t) => {
+      parsedQuery.select!.push(t.children![0].text!);
+    });
+    // let nameNode = findNodeOfType(selectNode, "Number");
+    // parsedQuery.limit = +nameNode!.children![0].text!;
+  }
+
+  // console.log(JSON.stringify(queryNode, null, 2));
+  return parsedQuery;
+}
+
+export function applyQuery<T>(parsedQuery: ParsedQuery, records: T[]): T[] {
+  let resultRecords: any[] = [];
+  if (parsedQuery.filter.length === 0) {
+    resultRecords = records.slice();
+  } else {
+    recordLoop: for (let record of records) {
+      const recordAny: any = record;
+      for (let { op, prop, value } of parsedQuery.filter) {
+        switch (op) {
+          case "=":
+            if (!(recordAny[prop] == value)) {
+              continue recordLoop;
+            }
+            break;
+          case "!=":
+            if (!(recordAny[prop] != value)) {
+              continue recordLoop;
+            }
+            break;
+          case "<":
+            if (!(recordAny[prop] < value)) {
+              continue recordLoop;
+            }
+            break;
+          case "<=":
+            if (!(recordAny[prop] <= value)) {
+              continue recordLoop;
+            }
+            break;
+          case ">":
+            if (!(recordAny[prop] > value)) {
+              continue recordLoop;
+            }
+            break;
+          case ">=":
+            if (!(recordAny[prop] >= value)) {
+              continue recordLoop;
+            }
+            break;
+          case "=~":
+            // TODO: Cache regexps somehow
+            if (!new RegExp(value).exec(recordAny[prop])) {
+              continue recordLoop;
+            }
+            break;
+          case "!=~":
+            if (new RegExp(value).exec(recordAny[prop])) {
+              continue recordLoop;
+            }
+            break;
+        }
+      }
+      resultRecords.push(recordAny);
+    }
+  }
+  // Now the sorting
+  if (parsedQuery.orderBy) {
+    resultRecords = resultRecords.sort((a: any, b: any) => {
+      const orderBy = parsedQuery.orderBy!;
+      const orderDesc = parsedQuery.orderDesc!;
+      if (a[orderBy] === b[orderBy]) {
+        return 0;
+      }
+
+      if (a[orderBy] < b[orderBy]) {
+        return orderDesc ? 1 : -1;
+      } else {
+        return orderDesc ? -1 : 1;
+      }
+    });
+  }
+  if (parsedQuery.limit) {
+    resultRecords = resultRecords.slice(0, parsedQuery.limit);
+  }
+  if (parsedQuery.select) {
+    resultRecords = resultRecords.map((rec) => {
+      let newRec: any = {};
+      for (let k of parsedQuery.select!) {
+        newRec[k] = rec[k];
+      }
+      return newRec;
+    });
+  }
+  return resultRecords;
+}

packages/plugs/query/materialized_queries.ts

@@ -0,0 +1,66 @@
+import {
+  flashNotification,
+  getCurrentPage,
+  getText,
+  reloadPage,
+  save
+} from "@silverbulletmd/plugos-silverbullet-syscall/editor";
+
+import { readPage, writePage } from "@silverbulletmd/plugos-silverbullet-syscall/space";
+import { invokeFunction } from "@silverbulletmd/plugos-silverbullet-syscall/system";
+import { parseQuery } from "./engine";
+import { replaceTemplateVars } from "../core/template";
+import { queryRegex, removeQueries } from "./util";
+import { dispatch } from "@silverbulletmd/plugos-syscall/event";
+import { replaceAsync } from "../lib/util";
+import { parseMarkdown } from "@silverbulletmd/plugos-silverbullet-syscall/markdown";
+
+export async function updateMaterializedQueriesCommand() {
+  const currentPage = await getCurrentPage();
+  await save();
+  await flashNotification("Updating materialized queries...");
+  await invokeFunction(
+    "server",
+    "updateMaterializedQueriesOnPage",
+    currentPage
+  );
+  await reloadPage();
+  await flashNotification("Updated materialized queries");
+}
+
+export async function whiteOutQueriesCommand() {
+  const text = await getText();
+  const parsed = await parseMarkdown(text);
+  console.log(removeQueries(parsed));
+}
+
+// Called from client, running on server
+export async function updateMaterializedQueriesOnPage(pageName: string) {
+  let { text } = await readPage(pageName);
+
+  text = await replaceAsync(
+    text,
+    queryRegex,
+    async (fullMatch, startQuery, query, body, endQuery) => {
+      let parsedQuery = parseQuery(replaceTemplateVars(query, pageName));
+
+      console.log("Parsed query", parsedQuery);
+      // Let's dispatch an event and see what happens
+      let results = await dispatch(
+        `query:${parsedQuery.table}`,
+        { query: parsedQuery, pageName: pageName },
+        10 * 1000
+      );
+      if (results.length === 0) {
+        return `${startQuery}\n${endQuery}`;
+      } else if (results.length === 1) {
+        return `${startQuery}\n${results[0]}\n${endQuery}`;
+      } else {
+        console.error("Too many query results", results);
+        return fullMatch;
+      }
+    }
+  );
+  // console.log("New text", text);
+  await writePage(pageName, text);
+}

packages/plugs/query/parse-query.js

@@ -0,0 +1,21 @@
+// This file was generated by lezer-generator. You probably shouldn't edit it.
+import { LRParser } from "@lezer/lr";
+
+export const parser = LRParser.deserialize({
+  version: 13,
+  states:
+    "%WOVQPOOO[QQO'#C^QOQPOOOmQPO'#C`OrQQO'#CjOwQPO'#ClO|QPO'#CmOOQO'#Cn'#CnO!RQQO,58xO!dQPO'#CcO#OQQO'#CaOOQO'#Ca'#CaOOQO,58z,58zO#dQPO,59UOOQO,59W,59WO#iQQO'#DWOOQO,59X,59XOOQO-E6l-E6lO#}QQO,58}OmQPO,58|O$cQQO1G.pO$zQPO'#CoO%PQQO,59rOOQO'#Cg'#CgOOQO'#Ci'#CiOOQO'#Cd'#CdOOQO1G.i1G.iOOQO1G.h1G.hOOQO'#Ck'#CkOOQO7+$[7+$[OOQO,59Z,59ZOOQO-E6m-E6m",
+  stateData:
+    "%e~OfOS~ORPO~OgROtSOxTOyUOdQX~ORXO~Ou]O~OX^O~OR_O~OgROtSOxTOyUOdQa~OhbOlbOmbOnbOobOpbOqbOrbO~OscOdTXgTXtTXxTXyTX~ORdO~O{eOdzXgzXtzXxzXyzX~OXiOYiO[iOigOjgOkhO~OvlOwlOd^ig^it^ix^iy^i~ORnO~O{eOdzagzatzaxzayza~O",
+  goto: "!y{PP|P!P!T!W!Z!aPP!dP!d!P!g!P!P!j!pPPPPPPPPPPPPPPPPPPPPPP!vRQOTVPWR[RRZRQYRRkcRjbRibRmdQWPRaWQf_RofR`U",
+  nodeNames:
+    "⚠ Program Query Name WhereClause LogicalExpr AndExpr FilterExpr Value Number String Bool Regex Null OrderClause Order LimitClause SelectClause",
+  maxTerm: 43,
+  skippedNodes: [0],
+  repeatNodeCount: 2,
+  tokenData:
+    "=_~RxX^#opq#oqr$drs$w|}%c}!O%h!P!Q%y!Q![&p!^!_&x!_!`'V!`!a'd!c!}%h#R#S%h#T#U'q#U#V*W#V#W%h#W#X+S#X#Y%h#Y#Z-O#Z#`%h#`#a/`#a#b%h#b#c1s#c#d3o#d#g%h#g#h6S#h#i9O#i#k%h#k#l:z#l#o%h#y#z#o$f$g#o#BY#BZ#o$IS$I_#o$Ip$Iq$w$Iq$Ir$w$I|$JO#o$JT$JU#o$KV$KW#o&FU&FV#o~#tYf~X^#opq#o#y#z#o$f$g#o#BY#BZ#o$IS$I_#o$I|$JO#o$JT$JU#o$KV$KW#o&FU&FV#o~$gP!_!`$j~$oPn~#r#s$r~$wOr~~$zUOr$wrs%^s$Ip$w$Ip$Iq%^$Iq$Ir%^$Ir~$w~%cOY~~%hO{~P%mSRP}!O%h!c!}%h#R#S%h#T#o%h~&OV[~OY%yZ]%y^!P%y!P!Q&e!Q#O%y#O#P&j#P~%y~&jO[~~&mPO~%y~&uPX~!Q![&p~&}Ph~!_!`'Q~'VOl~~'[Pm~#r#s'_~'dOq~~'iPp~!_!`'l~'qOo~R'vWRP}!O%h!c!}%h#R#S%h#T#b%h#b#c(`#c#g%h#g#h)[#h#o%hR(eURP}!O%h!c!}%h#R#S%h#T#W%h#W#X(w#X#o%hR)OSsQRP}!O%h!c!}%h#R#S%h#T#o%hR)aURP}!O%h!c!}%h#R#S%h#T#V%h#V#W)s#W#o%hR)zSwQRP}!O%h!c!}%h#R#S%h#T#o%hR*]URP}!O%h!c!}%h#R#S%h#T#m%h#m#n*o#n#o%hR*vSuQRP}!O%h!c!}%h#R#S%h#T#o%hR+XURP}!O%h!c!}%h#R#S%h#T#X%h#X#Y+k#Y#o%hR+pURP}!O%h!c!}%h#R#S%h#T#g%h#g#h,S#h#o%hR,XURP}!O%h!c!}%h#R#S%h#T#V%h#V#W,k#W#o%hR,rSvQRP}!O%h!c!}%h#R#S%h#T#o%hR-TTRP}!O%h!c!}%h#R#S%h#T#U-d#U#o%hR-iURP}!O%h!c!}%h#R#S%h#T#`%h#`#a-{#a#o%hR.QURP}!O%h!c!}%h#R#S%h#T#g%h#g#h.d#h#o%hR.iURP}!O%h!c!}%h#R#S%h#T#X%h#X#Y.{#Y#o%hR/SSjQRP}!O%h!c!}%h#R#S%h#T#o%hR/eURP}!O%h!c!}%h#R#S%h#T#]%h#]#^/w#^#o%hR/|URP}!O%h!c!}%h#R#S%h#T#a%h#a#b0`#b#o%hR0eURP}!O%h!c!}%h#R#S%h#T#]%h#]#^0w#^#o%hR0|URP}!O%h!c!}%h#R#S%h#T#h%h#h#i1`#i#o%hR1gSxQRP}!O%h!c!}%h#R#S%h#T#o%hR1xURP}!O%h!c!}%h#R#S%h#T#i%h#i#j2[#j#o%hR2aURP}!O%h!c!}%h#R#S%h#T#`%h#`#a2s#a#o%hR2xURP}!O%h!c!}%h#R#S%h#T#`%h#`#a3[#a#o%hR3cSkQRP}!O%h!c!}%h#R#S%h#T#o%hR3tURP}!O%h!c!}%h#R#S%h#T#f%h#f#g4W#g#o%hR4]URP}!O%h!c!}%h#R#S%h#T#W%h#W#X4o#X#o%hR4tURP}!O%h!c!}%h#R#S%h#T#X%h#X#Y5W#Y#o%hR5]URP}!O%h!c!}%h#R#S%h#T#f%h#f#g5o#g#o%hR5vStQRP}!O%h!c!}%h#R#S%h#T#o%hR6XURP}!O%h!c!}%h#R#S%h#T#X%h#X#Y6k#Y#o%hR6pURP}!O%h!c!}%h#R#S%h#T#`%h#`#a7S#a#o%hR7XURP}!O%h!c!}%h#R#S%h#T#X%h#X#Y7k#Y#o%hR7pURP}!O%h!c!}%h#R#S%h#T#V%h#V#W8S#W#o%hR8XURP}!O%h!c!}%h#R#S%h#T#h%h#h#i8k#i#o%hR8rSyQRP}!O%h!c!}%h#R#S%h#T#o%hR9TURP}!O%h!c!}%h#R#S%h#T#f%h#f#g9g#g#o%hR9lURP}!O%h!c!}%h#R#S%h#T#i%h#i#j:O#j#o%hR:TURP}!O%h!c!}%h#R#S%h#T#X%h#X#Y:g#Y#o%hR:nSiQRP}!O%h!c!}%h#R#S%h#T#o%hR;PURP}!O%h!c!}%h#R#S%h#T#[%h#[#];c#]#o%hR;hURP}!O%h!c!}%h#R#S%h#T#X%h#X#Y;z#Y#o%hR<PURP}!O%h!c!}%h#R#S%h#T#f%h#f#g<c#g#o%hR<hURP}!O%h!c!}%h#R#S%h#T#X%h#X#Y<z#Y#o%hR=RSgQRP}!O%h!c!}%h#R#S%h#T#o%h",
+  tokenizers: [0, 1],
+  topRules: { Program: [0, 1] },
+  tokenPrec: 0,
+});

packages/plugs/query/parse-query.terms.js

@@ -0,0 +1,19 @@
+// This file was generated by lezer-generator. You probably shouldn't edit it.
+export const
+  Program = 1,
+  Query = 2,
+  Name = 3,
+  WhereClause = 4,
+  LogicalExpr = 5,
+  AndExpr = 6,
+  FilterExpr = 7,
+  Value = 8,
+  Number = 9,
+  String = 10,
+  Bool = 11,
+  Regex = 12,
+  Null = 13,
+  OrderClause = 14,
+  Order = 15,
+  LimitClause = 16,
+  SelectClause = 17

packages/plugs/query/query.grammar

@@ -0,0 +1,58 @@
+@precedence { logic @left }
+
+@top Program { Query }
+
+Query {
+    Name ( WhereClause | OrderClause  | LimitClause | SelectClause )*
+}
+
+commaSep<content> { content ("," content)* }
+
+WhereClause { "where" LogicalExpr }
+OrderClause { "order" "by" Name Order? }
+LimitClause { "limit" Number }
+SelectClause { "select" commaSep<Name> }
+
+Order {
+  "desc" | "asc"
+}
+
+Value { Number | String | Bool | Regex | Null }
+
+LogicalExpr { AndExpr | FilterExpr }
+
+AndExpr { FilterExpr !logic "and" FilterExpr }
+
+FilterExpr {
+  Name "<" Value
+| Name "<=" Value
+| Name "=" Value
+| Name "!=" Value
+| Name ">=" Value
+| Name ">" Value
+| Name "=~" Value
+| Name "!=~" Value
+}
+
+@skip { space }
+
+
+
+Bool {
+  "true" | "false"
+}
+
+Null {
+  "null"
+}
+
+@tokens {
+  space { std.whitespace+ }
+  Name { (std.asciiLetter | "-" | "_")+ }
+  String {
+    ("\"" | "“" | "”") ![\"”“]* ("\"" | "“" | "”")
+  }
+  Regex { "/" ( ![/\\\n\r] | "\\" _ )* "/"? }
+
+  Number { std.digit+ }
+}

packages/plugs/query/query.plug.yaml

@@ -0,0 +1,24 @@
+functions:
+  updateMaterializedQueriesOnPage:
+    path: ./materialized_queries.ts:updateMaterializedQueriesOnPage
+  updateMaterializedQueriesCommand:
+    path: ./materialized_queries.ts:updateMaterializedQueriesCommand
+    command:
+      name: "Materialized Queries: Update"
+      key: "Alt-q"
+  whiteOutQueriesCommand:
+    path: ./materialized_queries.ts:whiteOutQueriesCommand
+    command:
+      name: "Debug: Whiteout Queries"
+  indexData:
+    path: ./data.ts:indexData
+    events:
+      - page:index
+  dataQueryProvider:
+    path: ./data.ts:queryProvider
+    events:
+      - query:data
+  insertQueryCommand:
+    path: ./command.ts:insertQuery
+    slashCommand:
+      name: query

packages/plugs/query/util.ts

@@ -0,0 +1,70 @@
+import { addParentPointers, collectNodesMatching, ParseTree, renderToText } from "@silverbulletmd/common/tree";
+
+export const queryRegex =
+  /(<!--\s*#query\s+(.+?)-->)(.+?)(<!--\s*#end\s*-->)/gs;
+
+export const queryStartRegex = /<!--\s*#query\s+(.+?)-->/s;
+
+export const queryEndRegex = /<!--\s*#end\s*-->/s;
+
+// export function whiteOutQueries(text: string): string {
+//   return text.replaceAll(queryRegex, (match) =>
+//     new Array(match.length + 1).join(" ")
+//   );
+// }
+
+export function removeQueries(pt: ParseTree) {
+  addParentPointers(pt);
+  collectNodesMatching(pt, (t) => {
+    if (t.type !== "CommentBlock") {
+      return false;
+    }
+    let text = t.children![0].text!;
+    if (!queryStartRegex.exec(text)) {
+      return false;
+    }
+    let parentChildren = t.parent!.children!;
+    let index = parentChildren.indexOf(t);
+    let nodesToReplace: ParseTree[] = [];
+    for (let i = index + 1; i < parentChildren.length; i++) {
+      let n = parentChildren[i];
+      if (n.type === "CommentBlock") {
+        let text = n.children![0].text!;
+        if (queryEndRegex.exec(text)) {
+          break;
+        }
+      }
+      nodesToReplace.push(n);
+    }
+    let renderedText = nodesToReplace.map(renderToText).join("");
+    parentChildren.splice(index + 1, nodesToReplace.length, {
+      text: new Array(renderedText.length + 1).join(" "),
+    });
+    return true;
+  });
+}
+
+// Nicely format an array of JSON objects as a Markdown table
+export function jsonToMDTable(
+  jsonArray: any[],
+  valueTransformer?: (k: string, v: any) => string | undefined
+): string {
+  let headers = new Set<string>();
+  for (let entry of jsonArray) {
+    for (let k of Object.keys(entry)) {
+      headers.add(k);
+    }
+  }
+  let headerList = [...headers];
+  let lines = [];
+  lines.push("|" + headerList.join("|") + "|");
+  lines.push("|" + headerList.map((title) => "----").join("|") + "|");
+  for (const val of jsonArray) {
+    let el = [];
+    for (let prop of headerList) {
+      el.push(valueTransformer ? valueTransformer(prop, val[prop]) : val[prop]);
+    }
+    lines.push("|" + el.join("|") + "|");
+  }
+  return lines.join("\n");
+}

packages/plugs/tasks/task.ts

@@ -0,0 +1,203 @@
+import type { ClickEvent, IndexTreeEvent } from "@silverbulletmd/web/app_event";
+
+import { batchSet, scanPrefixGlobal } from "@silverbulletmd/plugos-silverbullet-syscall/index";
+import { readPage, writePage } from "@silverbulletmd/plugos-silverbullet-syscall/space";
+import { parseMarkdown } from "@silverbulletmd/plugos-silverbullet-syscall/markdown";
+import { dispatch, filterBox, getCursor, getText } from "@silverbulletmd/plugos-silverbullet-syscall/editor";
+import {
+  addParentPointers,
+  collectNodesMatching,
+  collectNodesOfType,
+  findNodeOfType,
+  nodeAtPos,
+  ParseTree,
+  renderToText
+} from "@silverbulletmd/common/tree";
+import { removeQueries } from "../query/util";
+import { applyQuery, QueryProviderEvent } from "../query/engine";
+import { niceDate } from "../core/dates";
+
+export type Task = {
+  name: string;
+  done: boolean;
+  deadline?: string;
+  nested?: string;
+  // Not saved in DB, just added when pulled out (from key)
+  pos?: number;
+  page?: string;
+};
+
+function getDeadline(deadlineNode: ParseTree): string {
+  return deadlineNode.children![0].text!.replace(/📅\s*/, "");
+}
+
+export async function indexTasks({ name, tree }: IndexTreeEvent) {
+  // console.log("Indexing tasks");
+  let tasks: { key: string; value: Task }[] = [];
+  removeQueries(tree);
+  collectNodesOfType(tree, "Task").forEach((n) => {
+    let task = n.children!.slice(1).map(renderToText).join("").trim();
+    let complete = n.children![0].children![0].text! !== "[ ]";
+    let value: Task = {
+      name: task,
+      done: complete,
+    };
+
+    let deadlineNode = findNodeOfType(n, "DeadlineDate");
+    if (deadlineNode) {
+      value.deadline = getDeadline(deadlineNode);
+    }
+
+    let taskIndex = n.parent!.children!.indexOf(n);
+    let nestedItems = n.parent!.children!.slice(taskIndex + 1);
+    if (nestedItems.length > 0) {
+      value.nested = nestedItems.map(renderToText).join("").trim();
+    }
+    tasks.push({
+      key: `task:${n.from}`,
+      value,
+    });
+    // console.log("Task", value);
+  });
+
+  console.log("Found", tasks.length, "task(s)");
+  await batchSet(name, tasks);
+}
+
+export async function taskToggle(event: ClickEvent) {
+  return taskToggleAtPos(event.pos);
+}
+
+async function toggleTaskMarker(node: ParseTree, moveToPos: number) {
+  let changeTo = "[x]";
+  if (node.children![0].text === "[x]" || node.children![0].text === "[X]") {
+    changeTo = "[ ]";
+  }
+  await dispatch({
+    changes: {
+      from: node.from,
+      to: node.to,
+      insert: changeTo,
+    },
+    selection: {
+      anchor: moveToPos,
+    },
+  });
+
+  let parentWikiLinks = collectNodesMatching(
+    node.parent!,
+    (n) => n.type === "WikiLinkPage"
+  );
+  for (let wikiLink of parentWikiLinks) {
+    let ref = wikiLink.children![0].text!;
+    if (ref.includes("@")) {
+      let [page, pos] = ref.split("@");
+      let text = (await readPage(page)).text;
+
+      let referenceMdTree = await parseMarkdown(text);
+      // Adding +1 to immediately hit the task marker
+      let taskMarkerNode = nodeAtPos(referenceMdTree, +pos + 1);
+
+      if (!taskMarkerNode || taskMarkerNode.type !== "TaskMarker") {
+        console.error(
+          "Reference not a task marker, out of date?",
+          taskMarkerNode
+        );
+        return;
+      }
+      taskMarkerNode.children![0].text = changeTo;
+      text = renderToText(referenceMdTree);
+      console.log("Updated reference paged text", text);
+      await writePage(page, text);
+    }
+  }
+}
+
+export async function taskToggleAtPos(pos: number) {
+  let text = await getText();
+  let mdTree = await parseMarkdown(text);
+  addParentPointers(mdTree);
+
+  let node = nodeAtPos(mdTree, pos);
+  if (node && node.type === "TaskMarker") {
+    await toggleTaskMarker(node, pos);
+  }
+}
+
+export async function taskToggleCommand() {
+  let text = await getText();
+  let pos = await getCursor();
+  let tree = await parseMarkdown(text);
+  addParentPointers(tree);
+
+  let node = nodeAtPos(tree, pos);
+  // We kwow node.type === Task (due to the task context)
+  let taskMarker = findNodeOfType(node!, "TaskMarker");
+  await toggleTaskMarker(taskMarker!, pos);
+}
+
+export async function postponeCommand() {
+  let text = await getText();
+  let pos = await getCursor();
+  let tree = await parseMarkdown(text);
+  addParentPointers(tree);
+
+  let node = nodeAtPos(tree, pos)!;
+  // We kwow node.type === DeadlineDate (due to the task context)
+  let date = getDeadline(node);
+  let option = await filterBox(
+    "Postpone for...",
+    [
+      { name: "a day", orderId: 1 },
+      { name: "a week", orderId: 2 },
+      { name: "following Monday", orderId: 3 },
+    ],
+    "Select the desired time span to delay this task"
+  );
+  if (!option) {
+    return;
+  }
+  let d = new Date(date);
+  switch (option.name) {
+    case "a day":
+      d.setDate(d.getDate() + 1);
+      break;
+    case "a week":
+      d.setDate(d.getDate() + 7);
+      break;
+    case "following Monday":
+      d.setDate(d.getDate() + ((7 - d.getDay() + 1) % 7 || 7));
+      break;
+  }
+  await dispatch({
+    changes: {
+      from: node.from,
+      to: node.to,
+      insert: `📅 ${niceDate(d)}`,
+    },
+    selection: {
+      anchor: pos,
+    },
+  });
+  // await toggleTaskMarker(taskMarker!, pos);
+}
+
+export async function queryProvider({
+  query,
+}: QueryProviderEvent): Promise<string> {
+  let allTasks: Task[] = [];
+  for (let { key, page, value } of await scanPrefixGlobal("task:")) {
+    let [, pos] = key.split(":");
+    allTasks.push({
+      ...value,
+      page: page,
+      pos: pos,
+    });
+  }
+  let markdownTasks = applyQuery(query, allTasks).map(
+    (t) =>
+      `* [${t.done ? "x" : " "}] [[${t.page}@${t.pos}]] ${t.name}` +
+      (t.nested ? "\n  " + t.nested : "")
+  );
+  return markdownTasks.join("\n");
+}

packages/plugs/tasks/tasks.plug.yaml

@@ -0,0 +1,46 @@
+syntax:
+  DeadlineDate:
+    firstCharacters:
+      - "📅"
+    regex: "📅\\s*\\d{4}\\-\\d{2}\\-\\d{2}"
+    styles:
+      backgroundColor: "rgba(22,22,22,0.07)"
+  CompletedDate:
+    firstCharacters:
+      - "✅"
+    regex: "✅\\s*\\d{4}\\-\\d{2}\\-\\d{2}"
+    styles:
+      backgroundColor: "rgba(22,22,22,0.07)"
+  RepeatInterval:
+    firstCharacters:
+      - "🔁"
+    regex: "🔁\\s*every\\s+\\w+"
+    styles:
+      backgroundColor: "rgba(22,22,22,0.07)"
+functions:
+  indexTasks:
+    path: "./task.ts:indexTasks"
+    events:
+      - page:index
+  taskToggle:
+    path: "./task.ts:taskToggle"
+    events:
+      - page:click
+  itemQueryProvider:
+    path: ./task.ts:queryProvider
+    events:
+      - query:task
+  taskToggleCommand:
+    path: ./task.ts:taskToggleCommand
+    command:
+      name: "Task: Toggle"
+      key: Alt-t
+      contexts:
+        - Task
+  taskPostponeCommand:
+    path: ./task.ts:postponeCommand
+    command:
+      name: "Task: Postpone by 1 day"
+      key: Alt-+
+      contexts:
+        - DeadlineDate

packages/silverbullet-common/event.ts

@@ -0,0 +1,20 @@
+export abstract class EventEmitter<HandlerT> {
+  private handlers: Partial<HandlerT>[] = [];
+
+  on(handlers: Partial<HandlerT>) {
+    this.handlers.push(handlers);
+  }
+
+  off(handlers: Partial<HandlerT>) {
+    this.handlers = this.handlers.filter((h) => h !== handlers);
+  }
+
+  emit(eventName: keyof HandlerT, ...args: any[]) {
+    for (let handler of this.handlers) {
+      let fn: any = handler[eventName];
+      if (fn) {
+        fn(...args);
+      }
+    }
+  }
+}

packages/silverbullet-common/manifest.ts

@@ -0,0 +1,24 @@
+import * as plugos from "../plugos/types";
+import { EndpointHookT } from "../plugos/hooks/endpoint";
+import { CronHookT } from "../plugos/hooks/node_cron";
+import { EventHookT } from "../plugos/hooks/event";
+import { CommandHookT } from "../silverbullet-webapp/hooks/command";
+import { SlashCommandHookT } from "../silverbullet-webapp/hooks/slash_command";
+
+export type SilverBulletHooks = CommandHookT &
+  SlashCommandHookT &
+  EndpointHookT &
+  CronHookT &
+  EventHookT;
+
+export type SyntaxExtensions = {
+  syntax?: { [key: string]: NodeDef };
+};
+
+export type NodeDef = {
+  firstCharacters: string[];
+  regex: string;
+  styles: { [key: string]: string };
+};
+
+export type Manifest = plugos.Manifest<SilverBulletHooks> & SyntaxExtensions;

packages/silverbullet-common/package.json

@@ -0,0 +1,4 @@
+{
+  "name": "@silverbulletmd/common",
+  "version": "1.0.0"
+}

packages/silverbullet-common/parse_tree.ts

@@ -0,0 +1,84 @@
+import type { SyntaxNode } from "@lezer/common";
+import type { Language } from "@codemirror/language";
+import { ParseTree } from "./tree";
+
+export function lezerToParseTree(
+  text: string,
+  n: SyntaxNode,
+  offset = 0
+): ParseTree {
+  let children: ParseTree[] = [];
+  let nodeText: string | undefined;
+  let child = n.firstChild;
+  while (child) {
+    children.push(lezerToParseTree(text, child));
+    child = child.nextSibling;
+  }
+
+  if (children.length === 0) {
+    children = [
+      {
+        from: n.from + offset,
+        to: n.to + offset,
+        text: text.substring(n.from, n.to),
+      },
+    ];
+  } else {
+    let newChildren: ParseTree[] = [];
+    let index = n.from;
+    for (let child of children) {
+      let s = text.substring(index, child.from);
+      if (s) {
+        newChildren.push({
+          from: index + offset,
+          to: child.from! + offset,
+          text: s,
+        });
+      }
+      newChildren.push(child);
+      index = child.to!;
+    }
+    let s = text.substring(index, n.to);
+    if (s) {
+      newChildren.push({ from: index + offset, to: n.to + offset, text: s });
+    }
+    children = newChildren;
+  }
+
+  let result: ParseTree = {
+    type: n.name,
+    from: n.from + offset,
+    to: n.to + offset,
+  };
+  if (children.length > 0) {
+    result.children = children;
+  }
+  if (nodeText) {
+    result.text = nodeText;
+  }
+  return result;
+}
+
+export function parse(language: Language, text: string): ParseTree {
+  let tree = lezerToParseTree(text, language.parser.parse(text).topNode);
+  // replaceNodesMatching(tree, (n): MarkdownTree | undefined | null => {
+  //   if (n.type === "FencedCode") {
+  //     let infoN = findNodeMatching(n, (n) => n.type === "CodeInfo");
+  //     let language = infoN!.children![0].text;
+  //     let textN = findNodeMatching(n, (n) => n.type === "CodeText");
+  //     let text = textN!.children![0].text!;
+  //
+  //     console.log(language, text);
+  //     switch (language) {
+  //       case "yaml":
+  //         let parsed = StreamLanguage.define(yaml).parser.parse(text);
+  //         let subTree = treeToAST(text, parsed.topNode, n.from);
+  //         // console.log(JSON.stringify(subTree, null, 2));
+  //         subTree.type = "yaml";
+  //         return subTree;
+  //     }
+  //   }
+  //   return;
+  // });
+  return tree;
+}

packages/silverbullet-common/preload_modules.ts

@@ -0,0 +1,7 @@
+// These are the node modules that will be pre-bundled with SB
+// as a result they will not be included into plugos bundles and assumed to be loadable
+// via require() in the sandbox
+// Candidate modules for this are larger modules
+
+// When adding a module to this list, also manually add it to sandbox_worker.ts
+export const preloadModules = ["@lezer/lr", "yaml"];

packages/silverbullet-common/spaces/constants.ts

@@ -0,0 +1,2 @@
+export const trashPrefix = "_trash/";
+export const plugPrefix = "_plug/";

packages/silverbullet-common/spaces/disk_space_primitives.ts

@@ -0,0 +1,142 @@
+import { mkdir, readdir, readFile, stat, unlink, utimes, writeFile } from "fs/promises";
+import * as path from "path";
+import { PageMeta } from "../types";
+import { SpacePrimitives } from "./space_primitives";
+import { Plug } from "../../plugos/plug";
+
+export class DiskSpacePrimitives implements SpacePrimitives {
+  rootPath: string;
+  plugPrefix: string;
+
+  constructor(rootPath: string, plugPrefix: string = "_plug/") {
+    this.rootPath = rootPath;
+    this.plugPrefix = plugPrefix;
+  }
+
+  pageNameToPath(pageName: string) {
+    if (pageName.startsWith(this.plugPrefix)) {
+      return path.join(this.rootPath, pageName + ".plug.json");
+    }
+    return path.join(this.rootPath, pageName + ".md");
+  }
+
+  pathToPageName(fullPath: string): string {
+    let extLength = fullPath.endsWith(".plug.json")
+      ? ".plug.json".length
+      : ".md".length;
+    return fullPath.substring(
+      this.rootPath.length + 1,
+      fullPath.length - extLength
+    );
+  }
+
+  async readPage(pageName: string): Promise<{ text: string; meta: PageMeta }> {
+    const localPath = this.pageNameToPath(pageName);
+    try {
+      const s = await stat(localPath);
+      return {
+        text: await readFile(localPath, "utf8"),
+        meta: {
+          name: pageName,
+          lastModified: s.mtime.getTime(),
+        },
+      };
+    } catch (e) {
+      // console.error("Error while reading page", pageName, e);
+      throw Error(`Could not read page ${pageName}`);
+    }
+  }
+
+  async writePage(
+    pageName: string,
+    text: string,
+    selfUpdate: boolean,
+    lastModified?: number
+  ): Promise<PageMeta> {
+    let localPath = this.pageNameToPath(pageName);
+    try {
+      // Ensure parent folder exists
+      await mkdir(path.dirname(localPath), { recursive: true });
+
+      // Actually write the file
+      await writeFile(localPath, text);
+
+      if (lastModified) {
+        let d = new Date(lastModified);
+        console.log("Going to set the modified time", d);
+        await utimes(localPath, d, d);
+      }
+      // Fetch new metadata
+      const s = await stat(localPath);
+      return {
+        name: pageName,
+        lastModified: s.mtime.getTime(),
+      };
+    } catch (e) {
+      console.error("Error while writing page", pageName, e);
+      throw Error(`Could not write ${pageName}`);
+    }
+  }
+
+  async getPageMeta(pageName: string): Promise<PageMeta> {
+    let localPath = this.pageNameToPath(pageName);
+    try {
+      const s = await stat(localPath);
+      return {
+        name: pageName,
+        lastModified: s.mtime.getTime(),
+      };
+    } catch (e) {
+      console.error("Error while getting page meta", pageName, e);
+      throw Error(`Could not get meta for ${pageName}`);
+    }
+  }
+
+  async deletePage(pageName: string): Promise<void> {
+    let localPath = this.pageNameToPath(pageName);
+    await unlink(localPath);
+  }
+
+  async fetchPageList(): Promise<{
+    pages: Set<PageMeta>;
+    nowTimestamp: number;
+  }> {
+    let pages = new Set<PageMeta>();
+
+    const walkPath = async (dir: string) => {
+      let files = await readdir(dir);
+      for (let file of files) {
+        const fullPath = path.join(dir, file);
+        let s = await stat(fullPath);
+        if (s.isDirectory()) {
+          await walkPath(fullPath);
+        } else {
+          if (file.endsWith(".md") || file.endsWith(".json")) {
+            pages.add({
+              name: this.pathToPageName(fullPath),
+              lastModified: s.mtime.getTime(),
+            });
+          }
+        }
+      }
+    };
+    await walkPath(this.rootPath);
+    return {
+      pages: pages,
+      nowTimestamp: Date.now(),
+    };
+  }
+
+  invokeFunction(
+    plug: Plug<any>,
+    env: string,
+    name: string,
+    args: any[]
+  ): Promise<any> {
+    return plug.invoke(name, args);
+  }
+
+  proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
+    return plug.syscall(name, args);
+  }
+}

packages/silverbullet-common/spaces/evented_space_primitives.ts

@@ -0,0 +1,68 @@
+import { SpacePrimitives } from "./space_primitives";
+import { EventHook } from "../../plugos/hooks/event";
+import { PageMeta } from "../types";
+import { Plug } from "../../plugos/plug";
+import { trashPrefix } from "./constants";
+
+export class EventedSpacePrimitives implements SpacePrimitives {
+  constructor(private wrapped: SpacePrimitives, private eventHook: EventHook) {}
+
+  fetchPageList(): Promise<{ pages: Set<PageMeta>; nowTimestamp: number }> {
+    return this.wrapped.fetchPageList();
+  }
+
+  proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
+    return this.wrapped.proxySyscall(plug, name, args);
+  }
+
+  invokeFunction(
+    plug: Plug<any>,
+    env: string,
+    name: string,
+    args: any[]
+  ): Promise<any> {
+    return this.wrapped.invokeFunction(plug, env, name, args);
+  }
+
+  readPage(pageName: string): Promise<{ text: string; meta: PageMeta }> {
+    return this.wrapped.readPage(pageName);
+  }
+
+  async writePage(
+    pageName: string,
+    text: string,
+    selfUpdate: boolean,
+    lastModified?: number
+  ): Promise<PageMeta> {
+    const newPageMeta = await this.wrapped.writePage(
+      pageName,
+      text,
+      selfUpdate,
+      lastModified
+    );
+    // This can happen async
+    if (!pageName.startsWith(trashPrefix)) {
+      this.eventHook
+        .dispatchEvent("page:saved", pageName)
+        .then(() => {
+          return this.eventHook.dispatchEvent("page:index_text", {
+            name: pageName,
+            text,
+          });
+        })
+        .catch((e) => {
+          console.error("Error dispatching page:saved event", e);
+        });
+    }
+    return newPageMeta;
+  }
+
+  getPageMeta(pageName: string): Promise<PageMeta> {
+    return this.wrapped.getPageMeta(pageName);
+  }
+
+  async deletePage(pageName: string): Promise<void> {
+    await this.eventHook.dispatchEvent("page:deleted", pageName);
+    return this.wrapped.deletePage(pageName);
+  }
+}

packages/silverbullet-common/spaces/http_space_primitives.ts

@@ -0,0 +1,142 @@
+import { PageMeta } from "../types";
+import { Plug } from "../../plugos/plug";
+import { SpacePrimitives } from "./space_primitives";
+
+export class HttpSpacePrimitives implements SpacePrimitives {
+  pageUrl: string;
+  private plugUrl: string;
+
+  constructor(url: string) {
+    this.pageUrl = url + "/fs";
+    this.plugUrl = url + "/plug";
+  }
+
+  public async fetchPageList(): Promise<{
+    pages: Set<PageMeta>;
+    nowTimestamp: number;
+  }> {
+    let req = await fetch(this.pageUrl, {
+      method: "GET",
+    });
+
+    let result = new Set<PageMeta>();
+    ((await req.json()) as any[]).forEach((meta: any) => {
+      const pageName = meta.name;
+      result.add({
+        name: pageName,
+        lastModified: meta.lastModified,
+      });
+    });
+
+    return {
+      pages: result,
+      nowTimestamp: +req.headers.get("Now-Timestamp")!,
+    };
+  }
+
+  async readPage(name: string): Promise<{ text: string; meta: PageMeta }> {
+    let res = await fetch(`${this.pageUrl}/${name}`, {
+      method: "GET",
+    });
+    if (res.headers.get("X-Status") === "404") {
+      throw new Error(`Page not found`);
+    }
+    return {
+      text: await res.text(),
+      meta: this.responseToMeta(name, res),
+    };
+  }
+
+  async writePage(
+    name: string,
+    text: string,
+    selfUpdate?: boolean,
+    lastModified?: number
+  ): Promise<PageMeta> {
+    // TODO: lastModified ignored for now
+    let res = await fetch(`${this.pageUrl}/${name}`, {
+      method: "PUT",
+      body: text,
+      headers: lastModified
+        ? {
+            "Last-Modified": "" + lastModified,
+          }
+        : undefined,
+    });
+    const newMeta = this.responseToMeta(name, res);
+    return newMeta;
+  }
+
+  async deletePage(name: string): Promise<void> {
+    let req = await fetch(`${this.pageUrl}/${name}`, {
+      method: "DELETE",
+    });
+    if (req.status !== 200) {
+      throw Error(`Failed to delete page: ${req.statusText}`);
+    }
+  }
+
+  async proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
+    let req = await fetch(`${this.plugUrl}/${plug.name}/syscall/${name}`, {
+      method: "POST",
+      headers: {
+        "Content-type": "application/json",
+      },
+      body: JSON.stringify(args),
+    });
+    if (req.status !== 200) {
+      let error = await req.text();
+      throw Error(error);
+    }
+    if (req.headers.get("Content-length") === "0") {
+      return;
+    }
+    return await req.json();
+  }
+
+  async invokeFunction(
+    plug: Plug<any>,
+    env: string,
+    name: string,
+    args: any[]
+  ): Promise<any> {
+    // Invoke locally
+    if (!env || env === "client") {
+      return plug.invoke(name, args);
+    }
+    // Or dispatch to server
+    let req = await fetch(`${this.plugUrl}/${plug.name}/function/${name}`, {
+      method: "POST",
+      headers: {
+        "Content-type": "application/json",
+      },
+      body: JSON.stringify(args),
+    });
+    if (req.status !== 200) {
+      let error = await req.text();
+      throw Error(error);
+    }
+    if (req.headers.get("Content-length") === "0") {
+      return;
+    }
+    return await req.json();
+  }
+
+  async getPageMeta(name: string): Promise<PageMeta> {
+    let res = await fetch(`${this.pageUrl}/${name}`, {
+      method: "OPTIONS",
+    });
+    if (res.headers.get("X-Status") === "404") {
+      throw new Error(`Page not found`);
+    }
+    return this.responseToMeta(name, res);
+  }
+
+  private responseToMeta(name: string, res: Response): PageMeta {
+    const meta = {
+      name,
+      lastModified: +(res.headers.get("Last-Modified") || "0"),
+    };
+    return meta;
+  }
+}

packages/silverbullet-common/spaces/indexeddb_space_primitives.ts

@@ -0,0 +1,86 @@
+import { SpacePrimitives } from "./space_primitives";
+import { PageMeta } from "../types";
+import Dexie, { Table } from "dexie";
+import { Plug } from "../../plugos/plug";
+
+type Page = {
+  name: string;
+  text: string;
+  meta: PageMeta;
+};
+
+export class IndexedDBSpacePrimitives implements SpacePrimitives {
+  private pageTable: Table<Page, string>;
+
+  constructor(dbName: string, readonly timeSkew: number = 0) {
+    const db = new Dexie(dbName);
+    db.version(1).stores({
+      page: "name",
+    });
+    this.pageTable = db.table("page");
+  }
+
+  async deletePage(name: string): Promise<void> {
+    return this.pageTable.delete(name);
+  }
+
+  async getPageMeta(name: string): Promise<PageMeta> {
+    let entry = await this.pageTable.get(name);
+    if (entry) {
+      return entry.meta;
+    } else {
+      throw Error(`Page not found`);
+    }
+  }
+
+  invokeFunction(
+    plug: Plug<any>,
+    env: string,
+    name: string,
+    args: any[]
+  ): Promise<any> {
+    return plug.invoke(name, args);
+  }
+
+  async fetchPageList(): Promise<{
+    pages: Set<PageMeta>;
+    nowTimestamp: number;
+  }> {
+    let allPages = await this.pageTable.toArray();
+    return {
+      pages: new Set(allPages.map((p) => p.meta)),
+      nowTimestamp: Date.now() + this.timeSkew,
+    };
+  }
+
+  proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
+    return plug.syscall(name, args);
+  }
+
+  async readPage(name: string): Promise<{ text: string; meta: PageMeta }> {
+    let page = await this.pageTable.get(name);
+    if (page) {
+      return page;
+    } else {
+      throw new Error("Page not found");
+    }
+  }
+
+  async writePage(
+    name: string,
+    text: string,
+    selfUpdate?: boolean,
+    lastModified?: number
+  ): Promise<PageMeta> {
+    let meta = {
+      name,
+      lastModified: lastModified ? lastModified : Date.now() + this.timeSkew,
+    };
+    await this.pageTable.put({
+      name,
+      text,
+      meta,
+    });
+    return meta;
+  }
+}

packages/silverbullet-common/spaces/space.ts

@@ -0,0 +1,244 @@
+import { SpacePrimitives } from "./space_primitives";
+import { safeRun } from "@silverbulletmd/web/util";
+import { PageMeta } from "../types";
+import { EventEmitter } from "../event";
+import { Plug } from "../../plugos/plug";
+import { Manifest } from "../manifest";
+import { plugPrefix, trashPrefix } from "./constants";
+
+const pageWatchInterval = 2000;
+
+export type SpaceEvents = {
+  pageCreated: (meta: PageMeta) => void;
+  pageChanged: (meta: PageMeta) => void;
+  pageDeleted: (name: string) => void;
+  pageListUpdated: (pages: Set<PageMeta>) => void;
+  plugLoaded: (plugName: string, plug: Manifest) => void;
+  plugUnloaded: (plugName: string) => void;
+};
+
+export class Space extends EventEmitter<SpaceEvents> {
+  pageMetaCache = new Map<string, PageMeta>();
+  watchedPages = new Set<string>();
+  private initialPageListLoad = true;
+  private saving = false;
+
+  constructor(private space: SpacePrimitives, private trashEnabled = true) {
+    super();
+    this.on({
+      pageCreated: async (pageMeta) => {
+        if (pageMeta.name.startsWith(plugPrefix)) {
+          let pageData = await this.readPage(pageMeta.name);
+          this.emit(
+            "plugLoaded",
+            pageMeta.name.substring(plugPrefix.length),
+            JSON.parse(pageData.text)
+          );
+          this.watchPage(pageMeta.name);
+        }
+      },
+      pageChanged: async (pageMeta) => {
+        if (pageMeta.name.startsWith(plugPrefix)) {
+          let pageData = await this.readPage(pageMeta.name);
+          this.emit(
+            "plugLoaded",
+            pageMeta.name.substring(plugPrefix.length),
+            JSON.parse(pageData.text)
+          );
+        }
+      },
+    });
+  }
+
+  public updatePageListAsync() {
+    safeRun(async () => {
+      let newPageList = await this.space.fetchPageList();
+      let deletedPages = new Set<string>(this.pageMetaCache.keys());
+      newPageList.pages.forEach((meta) => {
+        const pageName = meta.name;
+        const oldPageMeta = this.pageMetaCache.get(pageName);
+        const newPageMeta = {
+          name: pageName,
+          lastModified: meta.lastModified,
+        };
+        if (
+          !oldPageMeta &&
+          (pageName.startsWith(plugPrefix) || !this.initialPageListLoad)
+        ) {
+          this.emit("pageCreated", newPageMeta);
+        } else if (
+          oldPageMeta &&
+          oldPageMeta.lastModified !== newPageMeta.lastModified &&
+          (!this.trashEnabled ||
+            (this.trashEnabled && !pageName.startsWith(trashPrefix)))
+        ) {
+          this.emit("pageChanged", newPageMeta);
+        }
+        // Page found, not deleted
+        deletedPages.delete(pageName);
+
+        // Update in cache
+        this.pageMetaCache.set(pageName, newPageMeta);
+      });
+
+      for (const deletedPage of deletedPages) {
+        this.pageMetaCache.delete(deletedPage);
+        this.emit("pageDeleted", deletedPage);
+      }
+
+      this.emit("pageListUpdated", this.listPages());
+      this.initialPageListLoad = false;
+    });
+  }
+
+  watch() {
+    setInterval(() => {
+      safeRun(async () => {
+        if (this.saving) {
+          return;
+        }
+        for (const pageName of this.watchedPages) {
+          const oldMeta = this.pageMetaCache.get(pageName);
+          if (!oldMeta) {
+            // No longer in cache, meaning probably deleted let's unwatch
+            this.watchedPages.delete(pageName);
+            continue;
+          }
+          // This seems weird, but simply fetching it will compare to local cache and trigger an event if necessary
+          await this.getPageMeta(pageName);
+        }
+      });
+    }, pageWatchInterval);
+    this.updatePageListAsync();
+  }
+
+  async deletePage(name: string, deleteDate?: number): Promise<void> {
+    await this.getPageMeta(name); // Check if page exists, if not throws Error
+    if (this.trashEnabled) {
+      let pageData = await this.readPage(name);
+      // Move to trash
+      await this.writePage(
+        `${trashPrefix}${name}`,
+        pageData.text,
+        true,
+        deleteDate
+      );
+    }
+    await this.space.deletePage(name);
+
+    this.pageMetaCache.delete(name);
+    this.emit("pageDeleted", name);
+    this.emit("pageListUpdated", new Set([...this.pageMetaCache.values()]));
+  }
+
+  async getPageMeta(name: string): Promise<PageMeta> {
+    let oldMeta = this.pageMetaCache.get(name);
+    let newMeta = await this.space.getPageMeta(name);
+    if (oldMeta) {
+      if (oldMeta.lastModified !== newMeta.lastModified) {
+        // Changed on disk, trigger event
+        this.emit("pageChanged", newMeta);
+      }
+    }
+    return this.metaCacher(name, newMeta);
+  }
+
+  invokeFunction(
+    plug: Plug<any>,
+    env: string,
+    name: string,
+    args: any[]
+  ): Promise<any> {
+    return this.space.invokeFunction(plug, env, name, args);
+  }
+
+  listPages(): Set<PageMeta> {
+    return new Set(
+      [...this.pageMetaCache.values()].filter(
+        (pageMeta) =>
+          !pageMeta.name.startsWith(trashPrefix) &&
+          !pageMeta.name.startsWith(plugPrefix)
+      )
+    );
+  }
+
+  listTrash(): Set<PageMeta> {
+    return new Set(
+      [...this.pageMetaCache.values()]
+        .filter(
+          (pageMeta) =>
+            pageMeta.name.startsWith(trashPrefix) &&
+            !pageMeta.name.startsWith(plugPrefix)
+        )
+        .map((pageMeta) => ({
+          ...pageMeta,
+          name: pageMeta.name.substring(trashPrefix.length),
+        }))
+    );
+  }
+
+  listPlugs(): Set<PageMeta> {
+    return new Set(
+      [...this.pageMetaCache.values()].filter((pageMeta) =>
+        pageMeta.name.startsWith(plugPrefix)
+      )
+    );
+  }
+
+  proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
+    return this.space.proxySyscall(plug, name, args);
+  }
+
+  async readPage(name: string): Promise<{ text: string; meta: PageMeta }> {
+    let pageData = await this.space.readPage(name);
+    let previousMeta = this.pageMetaCache.get(name);
+    if (previousMeta) {
+      if (previousMeta.lastModified !== pageData.meta.lastModified) {
+        // Page changed since last cached metadata, trigger event
+        this.emit("pageChanged", pageData.meta);
+      }
+    }
+    this.pageMetaCache.set(name, pageData.meta);
+    return pageData;
+  }
+
+  watchPage(pageName: string) {
+    this.watchedPages.add(pageName);
+  }
+
+  unwatchPage(pageName: string) {
+    this.watchedPages.delete(pageName);
+  }
+
+  async writePage(
+    name: string,
+    text: string,
+    selfUpdate?: boolean,
+    lastModified?: number
+  ): Promise<PageMeta> {
+    try {
+      this.saving = true;
+      let pageMeta = await this.space.writePage(
+        name,
+        text,
+        selfUpdate,
+        lastModified
+      );
+      if (!selfUpdate) {
+        this.emit("pageChanged", pageMeta);
+      }
+      return this.metaCacher(name, pageMeta);
+    } finally {
+      this.saving = false;
+    }
+  }
+
+  fetchPageList(): Promise<{ pages: Set<PageMeta>; nowTimestamp: number }> {
+    return this.space.fetchPageList();
+  }
+
+  private metaCacher(name: string, pageMeta: PageMeta): PageMeta {
+    this.pageMetaCache.set(name, pageMeta);
+    return pageMeta;
+  }
+}

packages/silverbullet-common/spaces/space_primitives.ts

@@ -0,0 +1,25 @@
+import { Plug } from "../../plugos/plug";
+import { PageMeta } from "../types";
+
+export interface SpacePrimitives {
+  // Pages
+  fetchPageList(): Promise<{ pages: Set<PageMeta>; nowTimestamp: number }>;
+  readPage(name: string): Promise<{ text: string; meta: PageMeta }>;
+  getPageMeta(name: string): Promise<PageMeta>;
+  writePage(
+    name: string,
+    text: string,
+    selfUpdate?: boolean,
+    lastModified?: number
+  ): Promise<PageMeta>;
+  deletePage(name: string): Promise<void>;
+
+  // Plugs
+  proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any>;
+  invokeFunction(
+    plug: Plug<any>,
+    env: string,
+    name: string,
+    args: any[]
+  ): Promise<any>;
+}

packages/silverbullet-common/spaces/sync.test.ts

@@ -0,0 +1,123 @@
+import { expect, test } from "@jest/globals";
+import { IndexedDBSpacePrimitives } from "./indexeddb_space_primitives";
+import { SpaceSync } from "./sync";
+import { PageMeta } from "../types";
+import { Space } from "./space";
+
+// For testing in node.js
+require("fake-indexeddb/auto");
+
+test("Test store", async () => {
+  let primary = new Space(new IndexedDBSpacePrimitives("primary"), true);
+  let secondary = new Space(
+    new IndexedDBSpacePrimitives("secondary", -5000),
+    true
+  );
+  let sync = new SpaceSync(primary, secondary, 0, 0, "_trash/");
+
+  async function conflictResolver(pageMeta1: PageMeta, pageMeta2: PageMeta) {}
+
+  // Write one page to primary
+  await primary.writePage("start", "Hello");
+  expect((await secondary.listPages()).size).toBe(0);
+  await syncPages(conflictResolver);
+  expect((await secondary.listPages()).size).toBe(1);
+  expect((await secondary.readPage("start")).text).toBe("Hello");
+
+  // Should be a no-op
+  expect(await syncPages()).toBe(0);
+
+  // Now let's make a change on the secondary
+  await secondary.writePage("start", "Hello!!");
+  await secondary.writePage("test", "Test page");
+
+  // And sync it
+  await syncPages();
+
+  expect(primary.listPages().size).toBe(2);
+  expect(secondary.listPages().size).toBe(2);
+
+  expect((await primary.readPage("start")).text).toBe("Hello!!");
+
+  // Let's make some random edits on both ends
+  await primary.writePage("start", "1");
+  await primary.writePage("start2", "2");
+  await secondary.writePage("start3", "3");
+  await secondary.writePage("start4", "4");
+  await syncPages();
+
+  expect((await primary.listPages()).size).toBe(5);
+  expect((await secondary.listPages()).size).toBe(5);
+
+  expect(await syncPages()).toBe(0);
+
+  console.log("Deleting pages");
+  // Delete some pages
+  await primary.deletePage("start");
+  await primary.deletePage("start3");
+
+  console.log("Pages", await primary.listPages());
+  console.log("Trash", await primary.listTrash());
+
+  await syncPages();
+
+  expect((await primary.listPages()).size).toBe(3);
+  expect((await secondary.listPages()).size).toBe(3);
+
+  // No-op
+  expect(await syncPages()).toBe(0);
+
+  await secondary.deletePage("start4");
+  await primary.deletePage("start2");
+
+  await syncPages();
+
+  // Just "test" left
+  expect((await primary.listPages()).size).toBe(1);
+  expect((await secondary.listPages()).size).toBe(1);
+
+  // No-op
+  expect(await syncPages()).toBe(0);
+
+  await secondary.writePage("start", "I'm back");
+
+  await syncPages();
+
+  expect((await primary.readPage("start")).text).toBe("I'm back");
+
+  // Cause a conflict
+  await primary.writePage("start", "Hello 1");
+  await secondary.writePage("start", "Hello 2");
+
+  await syncPages(SpaceSync.primaryConflictResolver(primary, secondary));
+
+  // Sync conflicting copy back
+  await syncPages();
+
+  // Verify that primary won
+  expect((await primary.readPage("start")).text).toBe("Hello 1");
+  expect((await secondary.readPage("start")).text).toBe("Hello 1");
+
+  // test + start + start.conflicting copy
+  expect((await primary.listPages()).size).toBe(3);
+  expect((await secondary.listPages()).size).toBe(3);
+
+  async function syncPages(
+    conflictResolver?: (
+      pageMeta1: PageMeta,
+      pageMeta2: PageMeta
+    ) => Promise<void>
+  ): Promise<number> {
+    // Awesome practice: adding sleeps to fix issues!
+    await sleep(2);
+    let n = await sync.syncPages(conflictResolver);
+    await sleep(2);
+    return n;
+  }
+});
+
+function sleep(ms: number = 5): Promise<void> {
+  return new Promise((resolve) => {
+    setTimeout(resolve, ms);
+  });
+}

packages/silverbullet-common/spaces/sync.ts

@@ -0,0 +1,208 @@
+import { Space } from "./space";
+import { PageMeta } from "../types";
+import { SpacePrimitives } from "./space_primitives";
+
+export class SpaceSync {
+  constructor(
+    private primary: Space,
+    private secondary: Space,
+    public primaryLastSync: number,
+    public secondaryLastSync: number,
+    private trashPrefix: string
+  ) {}
+
+  // Strategy: Primary wins
+  public static primaryConflictResolver(
+    primary: Space,
+    secondary: Space
+  ): (pageMeta1: PageMeta, pageMeta2: PageMeta) => Promise<void> {
+    return async (pageMeta1, pageMeta2) => {
+      const pageName = pageMeta1.name;
+      const revisionPageName = `${pageName}.conflicted.${pageMeta2.lastModified}`;
+      // Copy secondary to conflict copy
+      let oldPageData = await secondary.readPage(pageName);
+      await secondary.writePage(revisionPageName, oldPageData.text);
+
+      // Write replacement on top
+      let newPageData = await primary.readPage(pageName);
+      await secondary.writePage(
+        pageName,
+        newPageData.text,
+        true,
+        newPageData.meta.lastModified
+      );
+    };
+  }
+
+  async syncablePages(
+    space: Space
+  ): Promise<{ pages: PageMeta[]; nowTimestamp: number }> {
+    let fetchResult = await space.fetchPageList();
+    return {
+      pages: [...fetchResult.pages].filter(
+        (pageMeta) => !pageMeta.name.startsWith(this.trashPrefix)
+      ),
+      nowTimestamp: fetchResult.nowTimestamp,
+    };
+  }
+
+  async trashPages(space: SpacePrimitives): Promise<PageMeta[]> {
+    return [...(await space.fetchPageList()).pages]
+      .filter((pageMeta) => pageMeta.name.startsWith(this.trashPrefix))
+      .map((pageMeta) => ({
+        ...pageMeta,
+        name: pageMeta.name.substring(this.trashPrefix.length),
+      }));
+  }
+
+  async syncPages(
+    conflictResolver?: (
+      pageMeta1: PageMeta,
+      pageMeta2: PageMeta
+    ) => Promise<void>
+  ): Promise<number> {
+    let syncOps = 0;
+
+    let { pages: primaryAllPagesSet, nowTimestamp: primarySyncTimestamp } =
+      await this.syncablePages(this.primary);
+    let allPagesPrimary = new Map(primaryAllPagesSet.map((p) => [p.name, p]));
+    let { pages: secondaryAllPagesSet, nowTimestamp: secondarySyncTimestamp } =
+      await this.syncablePages(this.secondary);
+    let allPagesSecondary = new Map(
+      secondaryAllPagesSet.map((p) => [p.name, p])
+    );
+
+    let allTrashPrimary = new Map(
+      (await this.trashPages(this.primary))
+        // Filter out old trash
+        .filter((p) => p.lastModified > this.primaryLastSync)
+        .map((p) => [p.name, p])
+    );
+    let allTrashSecondary = new Map(
+      (await this.trashPages(this.secondary))
+        // Filter out old trash
+        .filter((p) => p.lastModified > this.secondaryLastSync)
+        .map((p) => [p.name, p])
+    );
+
+    // Iterate over all pages on the primary first
+    for (let [name, pageMetaPrimary] of allPagesPrimary.entries()) {
+      let pageMetaSecondary = allPagesSecondary.get(pageMetaPrimary.name);
+      if (!pageMetaSecondary) {
+        // New page on primary
+        // Let's check it's not on the deleted list
+        if (allTrashSecondary.has(name)) {
+          // Explicitly deleted, let's skip
+          continue;
+        }
+
+        // Push from primary to secondary
+        console.log("New page on primary", name, "syncing to secondary");
+        let pageData = await this.primary.readPage(name);
+        await this.secondary.writePage(
+          name,
+          pageData.text,
+          true,
+          secondarySyncTimestamp // The reason for this is to not include it in the next sync cycle, we cannot blindly use the lastModified date due to time skew
+        );
+        syncOps++;
+      } else {
+        // Existing page
+        if (pageMetaPrimary.lastModified > this.primaryLastSync) {
+          // Primary updated since last sync
+          if (pageMetaSecondary.lastModified > this.secondaryLastSync) {
+            // Secondary also updated! CONFLICT
+            if (conflictResolver) {
+              await conflictResolver(pageMetaPrimary, pageMetaSecondary);
+            } else {
+              throw Error(
+                `Sync conflict for ${name} with no conflict resolver specified`
+              );
+            }
+          } else {
+            // Ok, not changed on secondary, push it secondary
+            console.log(
+              "Changed page on primary",
+              name,
+              "syncing to secondary"
+            );
+            let pageData = await this.primary.readPage(name);
+            await this.secondary.writePage(
+              name,
+              pageData.text,
+              false,
+              secondarySyncTimestamp
+            );
+            syncOps++;
+          }
+        } else if (pageMetaSecondary.lastModified > this.secondaryLastSync) {
+          // Secondary updated, but not primary (checked above)
+          // Push from secondary to primary
+          console.log("Changed page on secondary", name, "syncing to primary");
+          let pageData = await this.secondary.readPage(name);
+          await this.primary.writePage(
+            name,
+            pageData.text,
+            false,
+            primarySyncTimestamp
+          );
+          syncOps++;
+        } else {
+          // Neither updated, no-op
+        }
+      }
+    }
+
+    // Now do a simplified version in reverse, only detecting new pages
+    for (let [name, pageMetaSecondary] of allPagesSecondary.entries()) {
+      if (!allPagesPrimary.has(pageMetaSecondary.name)) {
+        // New page on secondary
+        // Let's check it's not on the deleted list
+        if (allTrashPrimary.has(name)) {
+          // Explicitly deleted, let's skip
+          continue;
+        }
+        // Push from secondary to primary
+        console.log("New page on secondary", name, "pushing to primary");
+        let pageData = await this.secondary.readPage(name);
+        await this.primary.writePage(
+          name,
+          pageData.text,
+          false,
+          primarySyncTimestamp
+        );
+        syncOps++;
+      }
+    }
+
+    // And finally, let's trash some pages
+    for (let pageToDelete of allTrashPrimary.values()) {
+      console.log("Deleting", pageToDelete.name, "on secondary");
+      try {
+        await this.secondary.deletePage(
+          pageToDelete.name,
+          secondarySyncTimestamp
+        );
+        syncOps++;
+      } catch (e: any) {
+        console.log("Page already gone", e.message);
+      }
+    }
+
+    for (let pageToDelete of allTrashSecondary.values()) {
+      console.log("Deleting", pageToDelete.name, "on primary");
+      try {
+        await this.primary.deletePage(pageToDelete.name, primarySyncTimestamp);
+        syncOps++;
+      } catch (e: any) {
+        console.log("Page already gone", e.message);
+      }
+    }
+
+    // Setting last sync time to the timestamps we got back when fetching the page lists on each end
+    this.primaryLastSync = primarySyncTimestamp;
+    this.secondaryLastSync = secondarySyncTimestamp;
+
+    return syncOps;
+  }
+}