WIP: CLI running of plugs

2023-08-04 18:56:55 +02:00 · 2023-08-04 18:56:55 +02:00 · 3464af0252
commit 3464af0252
parent de3e385017
35 changed files with 738 additions and 136 deletions
--- a/cli/plug_run.test.ts
+++ b/cli/plug_run.test.ts
@ -0,0 +1,38 @@
+import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
+import { compileManifest } from "../plugos/compile.ts";
+import { esbuild } from "../plugos/deps.ts";
+import { runPlug } from "./plug_run.ts";
+import assets from "../dist/plug_asset_bundle.json" assert {
+  type: "json",
+};
+import { assertEquals } from "../test_deps.ts";
+import { path } from "../common/deps.ts";
+
+Deno.test("Test plug run", async () => {
+  // const tempDir = await Deno.makeTempDir();
+
+  const assetBundle = new AssetBundle(assets);
+
+  const testFolder = path.dirname(new URL(import.meta.url).pathname);
+  const testSpaceFolder = path.join(testFolder, "test_space");
+
+  const plugFolder = path.join(testSpaceFolder, "_plug");
+  await Deno.mkdir(plugFolder, { recursive: true });
+
+  await compileManifest(
+    path.join(testFolder, "test.plug.yaml"),
+    plugFolder,
+  );
+  assertEquals(
+    await runPlug(
+      testSpaceFolder,
+      "test.run",
+      [],
+      assetBundle,
+    ),
+    "Hello",
+  );
+
+  // await Deno.remove(tempDir, { recursive: true });
+  esbuild.stop();
+});
--- a/cli/plug_run.ts
+++ b/cli/plug_run.ts
@ -0,0 +1,145 @@
+import { path } from "../common/deps.ts";
+import { PlugNamespaceHook } from "../common/hooks/plug_namespace.ts";
+import { SilverBulletHooks } from "../common/manifest.ts";
+import { loadMarkdownExtensions } from "../common/markdown_parser/markdown_ext.ts";
+import buildMarkdown from "../common/markdown_parser/parser.ts";
+import { DiskSpacePrimitives } from "../common/spaces/disk_space_primitives.ts";
+import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives.ts";
+import { FileMetaSpacePrimitives } from "../common/spaces/file_meta_space_primitives.ts";
+import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts";
+import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
+import { createSandbox } from "../plugos/environments/deno_sandbox.ts";
+import { CronHook } from "../plugos/hooks/cron.ts";
+import { EventHook } from "../plugos/hooks/event.ts";
+import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts";
+import assetSyscalls from "../plugos/syscalls/asset.ts";
+import { eventSyscalls } from "../plugos/syscalls/event.ts";
+import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts";
+import { shellSyscalls } from "../plugos/syscalls/shell.deno.ts";
+import { storeSyscalls } from "../plugos/syscalls/store.ts";
+import { System } from "../plugos/system.ts";
+import { Space } from "../web/space.ts";
+import { debugSyscalls } from "../web/syscalls/debug.ts";
+import { markdownSyscalls } from "../web/syscalls/markdown.ts";
+import { systemSyscalls } from "../web/syscalls/system.ts";
+import { yamlSyscalls } from "../web/syscalls/yaml.ts";
+import { pageIndexSyscalls } from "./syscalls/index.ts";
+import { spaceSyscalls } from "./syscalls/space.ts";
+
+export async function runPlug(
+  spacePath: string,
+  functionName: string,
+  args: string[] = [],
+  builtinAssetBundle: AssetBundle,
+  indexFirst = false,
+) {
+  spacePath = path.resolve(spacePath);
+  const system = new System<SilverBulletHooks>("cli");
+
+  // Event hook
+  const eventHook = new EventHook();
+  system.addHook(eventHook);
+
+  // Cron hook
+  const cronHook = new CronHook(system);
+  system.addHook(cronHook);
+
+  const pageIndexCalls = pageIndexSyscalls("run.db");
+
+  // TODO: Add endpoint
+
+  const plugNamespaceHook = new PlugNamespaceHook();
+  system.addHook(plugNamespaceHook);
+
+  const spacePrimitives = new FileMetaSpacePrimitives(
+    new EventedSpacePrimitives(
+      new PlugSpacePrimitives(
+        new DiskSpacePrimitives(spacePath),
+        plugNamespaceHook,
+      ),
+      eventHook,
+    ),
+    pageIndexCalls,
+  );
+  const kvStore = new JSONKVStore();
+  const space = new Space(spacePrimitives, kvStore);
+
+  // Add syscalls
+  system.registerSyscalls(
+    [],
+    eventSyscalls(eventHook),
+    spaceSyscalls(space),
+    assetSyscalls(system),
+    yamlSyscalls(),
+    storeSyscalls(kvStore),
+    systemSyscalls(undefined as any, system),
+    pageIndexCalls,
+    debugSyscalls(),
+    markdownSyscalls(buildMarkdown([])), // Will later be replaced with markdown extensions
+  );
+
+  // Syscalls that require some additional permissions
+  system.registerSyscalls(
+    ["fetch"],
+    sandboxFetchSyscalls(),
+  );
+
+  system.registerSyscalls(
+    ["shell"],
+    shellSyscalls("."),
+  );
+
+  await loadPlugsFromAssetBundle(system, builtinAssetBundle);
+
+  for (let plugPath of await space.listPlugs()) {
+    plugPath = path.resolve(spacePath, plugPath);
+    await system.load(
+      new URL(`file://${plugPath}`),
+      createSandbox,
+    );
+  }
+
+  // Load markdown syscalls based on all new syntax (if any)
+  system.registerSyscalls(
+    [],
+    markdownSyscalls(buildMarkdown(loadMarkdownExtensions(system))),
+  );
+
+  if (indexFirst) {
+    await system.loadedPlugs.get("core")!.invoke("reindexSpace", []);
+  }
+
+  const [plugName, funcName] = functionName.split(".");
+
+  const plug = system.loadedPlugs.get(plugName);
+  if (!plug) {
+    throw new Error(`Plug ${plugName} not found`);
+  }
+  const result = await plug.invoke(funcName, args);
+
+  await system.unloadAll();
+  await pageIndexCalls["index.close"]({} as any);
+  return result;
+}
+
+async function loadPlugsFromAssetBundle(
+  system: System<any>,
+  assetBundle: AssetBundle,
+) {
+  const tempDir = await Deno.makeTempDir();
+  try {
+    for (const filePath of assetBundle.listFiles()) {
+      if (filePath.endsWith(".plug.js")) {
+        const plugPath = path.join(tempDir, filePath);
+        await Deno.mkdir(path.dirname(plugPath), { recursive: true });
+        await Deno.writeFile(plugPath, assetBundle.readFileSync(filePath));
+        await system.load(
+          new URL(`file://${plugPath}`),
+          createSandbox,
+        );
+      }
+    }
+  } finally {
+    await Deno.remove(tempDir, { recursive: true });
+  }
+}
--- a/cli/plug_test.ts
+++ b/cli/plug_test.ts
@ -0,0 +1,7 @@
+import { index } from "$sb/silverbullet-syscall/mod.ts";
+
+export async function run() {
+  console.log("Hello from plug_test.ts");
+  console.log(await index.queryPrefix(`tag:`));
+  return "Hello";
+}
--- a/cli/syscalls/index.test.ts
+++ b/cli/syscalls/index.test.ts
@ -0,0 +1,34 @@
+import { assertEquals } from "../../test_deps.ts";
+import { pageIndexSyscalls } from "./index.ts";
+
+Deno.test("Test KV index", async () => {
+  const ctx: any = {};
+  const calls = pageIndexSyscalls();
+  await calls["index.set"](ctx, "page", "test", "value");
+  assertEquals(await calls["index.get"](ctx, "page", "test"), "value");
+  await calls["index.delete"](ctx, "page", "test");
+  assertEquals(await calls["index.get"](ctx, "page", "test"), null);
+  await calls["index.batchSet"](ctx, "page", [{
+    key: "attr:test",
+    value: "value",
+  }, {
+    key: "attr:test2",
+    value: "value2",
+  }, { key: "random", value: "value3" }]);
+  await calls["index.batchSet"](ctx, "page2", [{
+    key: "attr:test",
+    value: "value",
+  }, {
+    key: "attr:test2",
+    value: "value2",
+  }, { key: "random", value: "value3" }]);
+  let results = await calls["index.queryPrefix"](ctx, "attr:");
+  assertEquals(results.length, 4);
+  await calls["index.clearPageIndexForPage"](ctx, "page");
+  results = await calls["index.queryPrefix"](ctx, "attr:");
+  assertEquals(results.length, 2);
+  await calls["index.clearPageIndex"](ctx);
+  results = await calls["index.queryPrefix"](ctx, "");
+  assertEquals(results.length, 0);
+  await calls["index.close"](ctx);
+});
--- a/cli/syscalls/index.ts
+++ b/cli/syscalls/index.ts
@ -0,0 +1,103 @@
+/// <reference lib="deno.unstable" />
+
+import type { SysCallMapping } from "../../plugos/system.ts";
+
+type Item = {
+  page: string;
+  key: string;
+  value: any;
+};
+
+export type KV = {
+  key: string;
+  value: any;
+};
+
+// Keyspace:
+// ["index", page, key] -> value
+// ["indexByKey", key, page] -> value
+
+/**
+ * Implements the index syscalls using Deno's KV store.
+ * @param dbFile
+ * @returns
+ */
+export function pageIndexSyscalls(dbFile?: string): SysCallMapping {
+  const kv = Deno.openKv(dbFile);
+  const apiObj: SysCallMapping = {
+    "index.set": async (_ctx, page: string, key: string, value: any) => {
+      const res = await (await kv).atomic()
+        .set(["index", page, key], value)
+        .set(["indexByKey", key, page], value)
+        .commit();
+      if (!res.ok) {
+        throw res;
+      }
+    },
+    "index.batchSet": async (_ctx, page: string, kvs: KV[]) => {
+      // await items.bulkPut(kvs);
+      for (const { key, value } of kvs) {
+        await apiObj["index.set"](_ctx, page, key, value);
+      }
+    },
+    "index.delete": async (_ctx, page: string, key: string) => {
+      const res = await (await kv).atomic()
+        .delete(["index", page, key])
+        .delete(["indexByKey", key, page])
+        .commit();
+      if (!res.ok) {
+        throw res;
+      }
+    },
+    "index.get": async (_ctx, page: string, key: string) => {
+      return (await (await kv).get(["index", page, key])).value;
+    },
+    "index.queryPrefix": async (_ctx, prefix: string) => {
+      const results: { key: string; page: string; value: any }[] = [];
+      for await (
+        const result of (await kv).list({
+          start: ["indexByKey", prefix],
+          end: [
+            "indexByKey",
+            prefix.slice(0, -1) +
+            // This is a hack to get the next character in the ASCII table (e.g. "a" -> "b")
+            String.fromCharCode(prefix.charCodeAt(prefix.length - 1) + 1),
+          ],
+        })
+      ) {
+        results.push({
+          key: result.key[1] as string,
+          page: result.key[2] as string,
+          value: result.value,
+        });
+      }
+      return results;
+    },
+    "index.clearPageIndexForPage": async (ctx, page: string) => {
+      await apiObj["index.deletePrefixForPage"](ctx, page, "");
+    },
+    "index.deletePrefixForPage": async (_ctx, page: string, prefix: string) => {
+      for await (
+        const result of (await kv).list({
+          start: ["index", page, prefix],
+          end: ["index", page, prefix + "~"],
+        })
+      ) {
+        await apiObj["index.delete"](_ctx, page, result.key[2]);
+      }
+    },
+    "index.clearPageIndex": async (ctx) => {
+      for await (
+        const result of (await kv).list({
+          prefix: ["index"],
+        })
+      ) {
+        await apiObj["index.delete"](ctx, result.key[1], result.key[2]);
+      }
+    },
+    "index.close": async () => {
+      (await kv).close();
+    },
+  };
+  return apiObj;
+}
--- a/cli/syscalls/space.ts
+++ b/cli/syscalls/space.ts
@ -0,0 +1,61 @@
+import { SysCallMapping } from "../../plugos/system.ts";
+import type { Space } from "../../web/space.ts";
+import { AttachmentMeta, PageMeta } from "../../web/types.ts";
+
+/**
+ * Almost the same as web/syscalls/space.ts except leaving out client-specific stuff
+ */
+export function spaceSyscalls(space: Space): SysCallMapping {
+  return {
+    "space.listPages": (): Promise<PageMeta[]> => {
+      return space.fetchPageList();
+    },
+    "space.readPage": async (
+      _ctx,
+      name: string,
+    ): Promise<string> => {
+      return (await space.readPage(name)).text;
+    },
+    "space.getPageMeta": (_ctx, name: string): Promise<PageMeta> => {
+      return space.getPageMeta(name);
+    },
+    "space.writePage": (
+      _ctx,
+      name: string,
+      text: string,
+    ): Promise<PageMeta> => {
+      return space.writePage(name, text);
+    },
+    "space.deletePage": async (_ctx, name: string) => {
+      await space.deletePage(name);
+    },
+    "space.listPlugs": (): Promise<string[]> => {
+      return space.listPlugs();
+    },
+    "space.listAttachments": async (): Promise<AttachmentMeta[]> => {
+      return await space.fetchAttachmentList();
+    },
+    "space.readAttachment": async (
+      _ctx,
+      name: string,
+    ): Promise<Uint8Array> => {
+      return (await space.readAttachment(name)).data;
+    },
+    "space.getAttachmentMeta": async (
+      _ctx,
+      name: string,
+    ): Promise<AttachmentMeta> => {
+      return await space.getAttachmentMeta(name);
+    },
+    "space.writeAttachment": (
+      _ctx,
+      name: string,
+      data: Uint8Array,
+    ): Promise<AttachmentMeta> => {
+      return space.writeAttachment(name, data);
+    },
+    "space.deleteAttachment": async (_ctx, name: string) => {
+      await space.deleteAttachment(name);
+    },
+  };
+}
--- a/cli/test.plug.yaml
+++ b/cli/test.plug.yaml
@ -0,0 +1,6 @@
+name: test
+requiredPermissions:
+- shell
+functions:
+    run:
+        path: plug_test.ts:run
--- a/cmd/plug_run.ts
+++ b/cmd/plug_run.ts
@ -0,0 +1,34 @@
+import { runPlug } from "../cli/plug_run.ts";
+import { path } from "../common/deps.ts";
+import assets from "../dist/plug_asset_bundle.json" assert {
+  type: "json",
+};
+import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
+
+export async function plugRunCommand(
+  {
+    noIndex,
+  }: {
+    noIndex: boolean;
+  },
+  spacePath: string,
+  functionName: string,
+  ...args: string[]
+) {
+  spacePath = path.resolve(spacePath);
+  console.log("Space path", spacePath);
+  console.log("Function to run:", functionName, "with arguments", args);
+  try {
+    const result = await runPlug(
+      spacePath,
+      functionName,
+      args,
+      new AssetBundle(assets),
+      !noIndex,
+    );
+    console.log("Output", result);
+  } catch (e: any) {
+    console.error(e.message);
+    Deno.exit(1);
+  }
+}
--- a/deno.jsonc
+++ b/deno.jsonc
@ -2,7 +2,7 @@
  "tasks": {
    "clean": "rm -rf dist dist_client_bundle dist_plug_bundle website_build",
    "deep-clean-mac": "rm -f deno.lock && rm -rf $HOME/Library/Caches/deno && deno task clean",
-    "install": "deno install -f -A --importmap import_map.json silverbullet.ts",
+    "install": "deno install -f --unstable -A --importmap import_map.json silverbullet.ts",
    "check": "find . -name '*.ts*' | xargs deno check",
    "test": "deno test -A --unstable",
    "build": "deno run -A build_plugs.ts && deno run -A --unstable build_web.ts",
--- a/plugos/environments/deno_sandbox.ts
+++ b/plugos/environments/deno_sandbox.ts
@ -6,8 +6,8 @@ export function createSandbox<HookT>(plug: Plug<HookT>): Sandbox<HookT> {
  return new Sandbox(plug, {
    deno: {
      permissions: {
-        // Disallow network access
-        net: false,
+        // Allow network access
+        net: true,
        // This is required for console logging to work, apparently?
        env: true,
        // No talking to native code
--- a/plugos/lib/kv_store.deno_kv.test.ts
+++ b/plugos/lib/kv_store.deno_kv.test.ts
@ -0,0 +1,29 @@
+import { assertEquals } from "../../test_deps.ts";
+import { DenoKVStore } from "./kv_store.deno_kv.ts";
+
+Deno.test("Test KV index", async () => {
+  const kv = new DenoKVStore();
+  await kv.init("test.db");
+
+  await kv.set("name", "Peter");
+  assertEquals(await kv.get("name"), "Peter");
+  await kv.del("name");
+  assertEquals(await kv.has("name"), false);
+
+  await kv.batchSet([
+    { key: "page:hello", value: "Hello" },
+    { key: "page:hello2", value: "Hello 2" },
+    { key: "page:hello3", value: "Hello 3" },
+    { key: "something", value: "Something" },
+  ]);
+
+  const results = await kv.queryPrefix("page:");
+  assertEquals(results.length, 3);
+
+  assertEquals(await kv.batchGet(["page:hello", "page:hello3"]), [
+    "Hello",
+    "Hello 3",
+  ]);
+
+  await kv.delete();
+});
--- a/plugos/lib/kv_store.deno_kv.ts
+++ b/plugos/lib/kv_store.deno_kv.ts
@ -0,0 +1,102 @@
+/// <reference lib="deno.unstable" />
+
+import { KV, KVStore } from "./kv_store.ts";
+
+export class DenoKVStore implements KVStore {
+  kv!: Deno.Kv;
+  path: string | undefined;
+
+  async init(path?: string) {
+    this.path = path;
+    this.kv = await Deno.openKv(path);
+  }
+
+  close() {
+    this.kv.close();
+  }
+
+  async delete() {
+    this.kv.close();
+    if (this.path) {
+      await Deno.remove(this.path);
+    }
+  }
+
+  async del(key: string): Promise<void> {
+    const res = await this.kv.atomic()
+      .delete([key])
+      .commit();
+    if (!res.ok) {
+      throw res;
+    }
+  }
+  async deletePrefix(prefix: string): Promise<void> {
+    for await (
+      const result of this.kv.list({
+        start: [prefix],
+        end: [endRange(prefix)],
+      })
+    ) {
+      await this.del(result.key[0] as string);
+    }
+  }
+  async deleteAll(): Promise<void> {
+    for await (
+      const result of this.kv.list({ prefix: [] })
+    ) {
+      await this.del(result.key[0] as string);
+    }
+  }
+  async set(key: string, value: any): Promise<void> {
+    const res = await this.kv.atomic()
+      .set([key], value)
+      .commit();
+    if (!res.ok) {
+      throw res;
+    }
+  }
+  async batchSet(kvs: KV[]): Promise<void> {
+    for (const { key, value } of kvs) {
+      await this.set(key, value);
+    }
+  }
+  async batchDelete(keys: string[]): Promise<void> {
+    for (const key of keys) {
+      await this.del(key);
+    }
+  }
+  batchGet(keys: string[]): Promise<any[]> {
+    const results: Promise<any>[] = [];
+    for (const key of keys) {
+      results.push(this.get(key));
+    }
+    return Promise.all(results);
+  }
+  async get(key: string): Promise<any> {
+    return (await this.kv.get([key])).value;
+  }
+  async has(key: string): Promise<boolean> {
+    return (await this.kv.get([key])).value !== null;
+  }
+  async queryPrefix(keyPrefix: string): Promise<{ key: string; value: any }[]> {
+    const results: { key: string; value: any }[] = [];
+    for await (
+      const result of (this.kv).list({
+        start: [keyPrefix],
+        end: [endRange(keyPrefix)],
+      })
+    ) {
+      results.push({
+        key: result.key[0] as string,
+        value: result.value as any,
+      });
+    }
+    return results;
+  }
+}
+
+function endRange(prefix: string) {
+  const lastChar = prefix[prefix.length - 1];
+  const nextLastChar = String.fromCharCode(lastChar.charCodeAt(0) + 1);
+  return prefix.slice(0, -1) + nextLastChar;
+}
--- a/plugos/plug.ts
+++ b/plugos/plug.ts
@ -1,10 +1,10 @@
-import { Manifest, RuntimeEnvironment } from "./types.ts";
+import { Manifest } from "./types.ts";
 import { Sandbox } from "./sandbox.ts";
 import { System } from "./system.ts";
 import { AssetBundle, AssetJson } from "./asset_bundle/bundle.ts";

 export class Plug<HookT> {
-  readonly runtimeEnv?: RuntimeEnvironment;
+  readonly runtimeEnv?: string;

  public grantedPermissions: string[] = [];
  public sandbox: Sandbox<HookT>;
--- a/plugos/syscalls/fetch.ts
+++ b/plugos/syscalls/fetch.ts
@ -0,0 +1,25 @@
+import type { SysCallMapping } from "../../plugos/system.ts";
+import {
+  ProxyFetchRequest,
+  ProxyFetchResponse,
+} from "../../common/proxy_fetch.ts";
+import { base64Encode } from "../asset_bundle/base64.ts";
+
+export function sandboxFetchSyscalls(): SysCallMapping {
+  return {
+    "sandboxFetch.fetch": async (
+      _ctx,
+      url: string,
+      options: ProxyFetchRequest,
+    ): Promise<ProxyFetchResponse> => {
+      // console.log("Got sandbox fetch ", url);
+      const resp = await fetch(url, options);
+      return {
+        status: resp.status,
+        ok: resp.ok,
+        headers: Object.fromEntries(resp.headers.entries()),
+        base64Body: base64Encode(new Uint8Array(await resp.arrayBuffer())),
+      };
+    },
+  };
+}
--- a/plugos/syscalls/shell.deno.ts
+++ b/plugos/syscalls/shell.deno.ts
@ -1,21 +1,21 @@
 import type { SysCallMapping } from "../system.ts";

-export default function (cwd: string): SysCallMapping {
+export function shellSyscalls(cwd: string): SysCallMapping {
  return {
    "shell.run": async (
      _ctx,
      cmd: string,
      args: string[],
    ): Promise<{ stdout: string; stderr: string }> => {
-      const p = Deno.run({
-        cmd: [cmd, ...args],
-        cwd: cwd,
+      const p = new Deno.Command(cmd, {
+        args: args,
+        cwd,
        stdout: "piped",
        stderr: "piped",
      });
-      await p.status();
-      const stdout = new TextDecoder().decode(await p.output());
-      const stderr = new TextDecoder().decode(await p.stderrOutput());
+      const output = await p.output();
+      const stdout = new TextDecoder().decode(output.stdout);
+      const stderr = new TextDecoder().decode(output.stderr);

      return { stdout, stderr };
    },
--- a/plugos/syscalls/store.dexie_browser.ts
+++ b/plugos/syscalls/store.dexie_browser.ts
@ -1,9 +1,8 @@
 import { SysCallMapping } from "../system.ts";
-import { DexieKVStore } from "../lib/kv_store.dexie.ts";
-import { KV } from "../lib/kv_store.ts";
+import { KV, KVStore } from "../lib/kv_store.ts";

 export function storeSyscalls(
-  db: DexieKVStore,
+  db: KVStore,
 ): SysCallMapping {
  return {
    "store.delete": (_ctx, key: string) => {
--- a/plugos/system.ts
+++ b/plugos/system.ts
@ -1,4 +1,4 @@
-import { Hook, RuntimeEnvironment } from "./types.ts";
+import { Hook } from "./types.ts";
 import { EventEmitter } from "./event.ts";
 import type { SandboxFactory } from "./sandbox.ts";
 import { Plug } from "./plug.ts";
@ -32,7 +32,7 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
  protected registeredSyscalls = new Map<string, Syscall>();
  protected enabledHooks = new Set<Hook<HookT>>();

-  constructor(readonly env?: RuntimeEnvironment) {
+  constructor(readonly env?: string) {
    super();
  }

--- a/plugos/types.ts
+++ b/plugos/types.ts
@ -17,11 +17,9 @@ export type FunctionDef<HookT> = {
  // Reuse an
  // Format: plugName.functionName
  redirect?: string;
-  env?: RuntimeEnvironment;
+  env?: string;
 } & HookT;

-export type RuntimeEnvironment = "client" | "server";
-
 export interface Hook<HookT> {
  validateManifest(manifest: Manifest<HookT>): string[];
  apply(system: System<HookT>): void;
--- a/plugs/core/core.plug.yaml
+++ b/plugs/core/core.plug.yaml
@ -52,6 +52,8 @@ functions:
    path: "./page.ts:reindexCommand"
    command:
      name: "Space: Reindex"
+  reindexSpace:
+    path: "./page.ts:reindexSpace"
  deletePage:
    path: "./page.ts:deletePage"
    command:
--- a/plugs/directive/command.ts
+++ b/plugs/directive/command.ts
@ -1,5 +1,6 @@
 import { editor, markdown, space, sync } from "$sb/silverbullet-syscall/mod.ts";
 import {
+  ParseTree,
  removeParentPointers,
  renderToText,
  traverseTree,
@ -36,42 +37,7 @@ export async function updateDirectivesOnPageCommand() {

  await editor.save();

-  // Collect all directives and their body replacements
-  const replacements: { fullMatch: string; textPromise: Promise<string> }[] =
-    [];
-
-  // Convenience array to wait for all promises to resolve
-  const allPromises: Promise<string>[] = [];
-
-  removeParentPointers(tree);
-
-  traverseTree(tree, (tree) => {
-    if (tree.type !== "Directive") {
-      return false;
-    }
-    const fullMatch = text.substring(tree.from!, tree.to!);
-    try {
-      const promise = renderDirectives(pageMeta, tree);
-      replacements.push({
-        textPromise: promise,
-        fullMatch,
-      });
-      allPromises.push(promise);
-    } catch (e: any) {
-      replacements.push({
-        fullMatch,
-        textPromise: Promise.resolve(
-          `${renderToText(tree.children![0])}\n**ERROR:** ${e.message}\n${
-            renderToText(tree.children![tree.children!.length - 1])
-          }`,
-        ),
-      });
-    }
-    return true;
-  });
-
-  // Wait for all to have processed
-  await Promise.all(allPromises);
+  const replacements = await findReplacements(tree, text, pageMeta);

  // Iterate again and replace the bodies. Iterating again (not using previous positions)
  // because text may have changed in the mean time (directive processing may take some time)
@ -111,27 +77,27 @@ export async function updateDirectivesOnPageCommand() {
  }
 }

-export async function updateDirectives(
-  pageMeta: PageMeta,
+async function findReplacements(
+  tree: ParseTree,
  text: string,
+  pageMeta: PageMeta,
 ) {
-  const tree = await markdown.parseMarkdown(text);
  // Collect all directives and their body replacements
  const replacements: { fullMatch: string; textPromise: Promise<string> }[] =
    [];

+  // Convenience array to wait for all promises to resolve
  const allPromises: Promise<string>[] = [];

+  removeParentPointers(tree);
+
  traverseTree(tree, (tree) => {
    if (tree.type !== "Directive") {
      return false;
    }
    const fullMatch = text.substring(tree.from!, tree.to!);
    try {
-      const promise = renderDirectives(
-        pageMeta,
-        tree,
-      );
+      const promise = renderDirectives(pageMeta, tree);
      replacements.push({
        textPromise: promise,
        fullMatch,
@ -153,9 +119,51 @@ export async function updateDirectives(
  // Wait for all to have processed
  await Promise.all(allPromises);

+  return replacements;
+}
+
+export async function updateDirectivesInSpace() {
+  const allPages = await space.listPages();
+  let counter = 0;
+  for (const page of allPages) {
+    counter++;
+    console.log(
+      `Updating directives in page [${counter}/${allPages.length}]`,
+      page.name,
+    );
+    try {
+      await updateDirectivesForPage(page.name);
+    } catch (e: any) {
+      console.error("Error while updating directives on page", page.name, e);
+    }
+  }
+}
+
+async function updateDirectivesForPage(
+  pageName: string,
+) {
+  const pageMeta = await space.getPageMeta(pageName);
+  const currentText = await space.readPage(pageName);
+  const newText = await updateDirectives(pageMeta, currentText);
+  if (newText !== currentText) {
+    console.info("Content of page changed, saving.");
+    await space.writePage(pageName, newText);
+  }
+}
+
+export async function updateDirectives(
+  pageMeta: PageMeta,
+  text: string,
+) {
+  const tree = await markdown.parseMarkdown(text);
+  const replacements = await findReplacements(tree, text, pageMeta);
+
  // Iterate again and replace the bodies.
  for (const replacement of replacements) {
-    text = text.replace(replacement.fullMatch, await replacement.textPromise);
+    text = text.replace(
+      replacement.fullMatch,
+      await replacement.textPromise,
+    );
  }
  return text;
 }
--- a/plugs/directive/directive.plug.yaml
+++ b/plugs/directive/directive.plug.yaml
@ -9,6 +9,8 @@ functions:
      key: "Alt-q"
    events:
      - editor:pageLoaded
+  updateDirectivesInSpace:
+    path: ./command.ts:updateDirectivesInSpace
  indexData:
    path: ./data.ts:indexData
    events:
--- a/plugs/directive/directives.ts
+++ b/plugs/directive/directives.ts
@ -1,5 +1,4 @@
 import { ParseTree, renderToText } from "$sb/lib/tree.ts";
-import { sync } from "../../plug-api/silverbullet-syscall/mod.ts";
 import { PageMeta } from "../../web/types.ts";

 import { evalDirectiveRenderer } from "./eval_directive.ts";
--- a/silverbullet.ts
+++ b/silverbullet.ts
@ -11,6 +11,7 @@ import { userAdd } from "./cmd/user_add.ts";
 import { userPasswd } from "./cmd/user_passwd.ts";
 import { userDelete } from "./cmd/user_delete.ts";
 import { userChgrp } from "./cmd/user_chgrp.ts";
+import { plugRunCommand } from "./cmd/plug_run.ts";

 await new Command()
  .name("silverbullet")
@ -66,6 +67,13 @@ await new Command()
  .option("--importmap <path:string>", "Path to import map file to use")
  .option("--runtimeUrl <url:string>", "URL to worker_runtime.ts to use")
  .action(plugCompileCommand)
+  // plug:run
+  .command("plug:run", "Run a PlugOS function from the CLI")
+  .arguments("<spacePath> <function> [...args:string]")
+  .option("--noIndex [type:boolean]", "Do not run a full space index first", {
+    default: false,
+  })
+  .action(plugRunCommand)
  .command("user:add", "Add a new user to an authentication file")
  .arguments("[username:string]")
  .option(
--- a/web/client_system.ts
+++ b/web/client_system.ts
@ -8,7 +8,7 @@ import { createSandbox } from "../plugos/environments/webworker_sandbox.ts";

 import assetSyscalls from "../plugos/syscalls/asset.ts";
 import { eventSyscalls } from "../plugos/syscalls/event.ts";
-import { storeSyscalls } from "../plugos/syscalls/store.dexie_browser.ts";
+import { storeSyscalls } from "../plugos/syscalls/store.ts";
 import { SysCallMapping, System } from "../plugos/system.ts";
 import type { Client } from "./client.ts";
 import { CodeWidgetHook } from "./hooks/code_widget.ts";
--- a/web/space.ts
+++ b/web/space.ts
@ -4,8 +4,8 @@ import { EventEmitter } from "../plugos/event.ts";
 import { plugPrefix } from "../common/spaces/constants.ts";
 import { safeRun } from "../common/util.ts";
 import { AttachmentMeta, PageMeta } from "./types.ts";
-import { DexieKVStore } from "../plugos/lib/kv_store.dexie.ts";
 import { throttle } from "../common/async_util.ts";
+import { KVStore } from "../plugos/lib/kv_store.ts";

 export type SpaceEvents = {
  pageCreated: (meta: PageMeta) => void;
@ -46,7 +46,7 @@ export class Space extends EventEmitter<SpaceEvents> {

  constructor(
    readonly spacePrimitives: SpacePrimitives,
-    private kvStore: DexieKVStore,
+    private kvStore: KVStore,
  ) {
    super();
    this.kvStore.get("imageHeightCache").then((cache) => {
--- a/website/Attributes.md
+++ b/website/Attributes.md
@ -33,7 +33,7 @@ Example query:
 <!-- #query page where name = "Attributes" -->
 |name      |lastModified |contentType  |size|perm|pageAttribute|
 |----------|-------------|-------------|----|--|-----|
-|Attributes|1690384301337|text/markdown|1591|rw|hello|
+|Attributes|1691165890257|text/markdown|1609|rw|hello|
 <!-- /query -->

 This attaches an attribute to an item:
@ -57,5 +57,5 @@ Example query:
 <!-- #query task where page = "Attributes" and taskAttribute = "hello" -->
 |name|done |taskAttribute|page      |pos |
 |----|-----|-----|----------|----|
-|Task|false|hello|Attributes|1352|
+|Task|false|hello|Attributes|1355|
 <!-- /query -->
--- a/website/Getting
+++ b/website/Getting
@ -24,17 +24,17 @@ Don’t believe me, check this out, here’s a list of (max 10) pages in your sp

 <!-- #query page select name order by lastModified desc limit 10 -->
 |name              |
-|-------------------------|
-|CHANGELOG                |
-|🔨 Development           |
-|Server                   |
-|Raspberry Pi Installation|
-|STYLES                   |
+|------------------|
+|🔌 Directive/Query|
+|Attributes        |
 |Getting Started   |
-|Sandbox                  |
-|SETTINGS                 |
-|SilverBullet             |
-|🔌 Core/Templates        |
+|🔌 Core/Tags      |
+|🔌 Github         |
+|🔌 Mattermost     |
+|🔌 Git            |
+|🔌 Ghost          |
+|🔌 Share          |
+|Install           |
 <!-- /query -->

 That said, the directive used wrapping this page section is `#use` which uses the content of another page as a template and inlines it. Directives recalculate their bodies in two scenarios:
--- a/website/Install.md
+++ b/website/Install.md
@ -20,13 +20,13 @@ This consists of two steps (unless Deno is already installed — in which case w
 With Deno installed, run:

 ```shell
-deno install -f --name silverbullet -A https://get.silverbullet.md
+deno install -f --name silverbullet --unstable -A https://get.silverbullet.md
 ```

 This will give you (and when you use `silverbullet upgrade`) the latest stable release. If you prefer to live on the bleeding edge, you can install using the following command instead:

 ```shell
-deno install -f --name silverbullet -A https://silverbullet.md/silverbullet.js
+deno install -f --name silverbullet --unstable -A https://silverbullet.md/silverbullet.js
 ```

 This will install `silverbullet` into your `~/.deno/bin` folder (which should already be in your `$PATH` if you followed the Deno install instructions).
--- a/Core/Tags.md
+++ b/Core/Tags.md
@ -18,7 +18,7 @@ and be queried:
 <!-- #query item where tags = "core-tag" -->
 |name                           |tags    |page        |pos|
 |-------------------------------|--------|------------|---|
-|This is a tagged item #core-tag|core-tag|🔌 Core/Tags|486|
+|This is a tagged item #core-tag|core-tag|🔌 Core/Tags|493|
 <!-- /query -->

 and **tags**:
@ -28,5 +28,5 @@ and **tags**:
 And they can be queried this way:

 <!-- #query task where tags = "core-tag" render [[template/task]] -->
-* [ ] [[🔌 Core/Tags@783]] This is a tagged task #core-tag
+* [ ] [[🔌 Core/Tags@804]] This is a tagged task #core-tag
 <!-- /query -->
--- a/Directive/Query.md
+++ b/Directive/Query.md
@ -88,8 +88,8 @@ Example:
 <!-- #query data where age > 20 and country = "Italy" -->
 |name|age|city |country|page              |pos |
 |----|--|-----|-----|------------------|----|
-|John|50|Milan|Italy|🔌 Directive/Query|3293|
-|Jane|53|Rome |Italy|🔌 Directive/Query|3294|
+|John|50|Milan|Italy|🔌 Directive/Query|2933|
+|Jane|53|Rome |Italy|🔌 Directive/Query|2934|
 <!-- /query -->

 #### 4.2 Plugs’ data sources
@ -156,11 +156,11 @@ For the sake of simplicity, we will use the `page` data source and limit the res
 **Result:** Look at the data. This is more than we need. The query even gives us template pages. Let's try to limit it in the next step.

 <!-- #query page limit 3 -->
-|name          |lastModified |contentType  |size|perm|
-|--------------|-------------|-------------|----|--|
-|API           |1688987324351|text/markdown|1405|rw|
-|Authelia      |1688482500313|text/markdown|866 |rw|
-|Authentication|1686682290943|text/markdown|1730|rw|
+|name                                     |lastModified |contentType  |size |perm|
+|--|--|--|--|--|
+|Authentication                           |1686682290943|text/markdown|1730 |rw|
+|Guide/Deployment/Cloudflare and Portainer|1690298800145|text/markdown|12899|rw|
+|Markdown                                 |1676121406520|text/markdown|1178 |rw|
 <!-- /query -->


@ -171,13 +171,13 @@ For the sake of simplicity, we will use the `page` data source and limit the res
 **Result:** Okay, this is what we wanted but there is also information such as `perm`, `type` and `lastModified` that we don't need.

 <!-- #query page where type = "plug" order by lastModified desc limit 5 -->
-|name        |lastModified |contentType  |size|perm|type|repo                                                  |uri                                                       |author              |
-|--|--|--|--|--|--|--|--|--|
-|🔌 Directive|1688987324365|text/markdown|2607|rw|plug|https://github.com/silverbulletmd/silverbullet        |                                                          |                    |
-|🔌 KaTeX    |1687099068396|text/markdown|1342|rw|plug|https://github.com/silverbulletmd/silverbullet-katex  |github:silverbulletmd/silverbullet-katex/katex.plug.js    |Zef Hemel           |
-|🔌 Core     |1687094809367|text/markdown|402 |rw|plug|https://github.com/silverbulletmd/silverbullet        |                                                          |                    |
-|🔌 Twitter  |1685105433212|text/markdown|1266|rw|plug|https://github.com/silverbulletmd/silverbullet-twitter|github:silverbulletmd/silverbullet-twitter/twitter.plug.js|SilverBullet Authors|
-|🔌 Mermaid  |1685105423879|text/markdown|1096|rw|plug|https://github.com/silverbulletmd/silverbullet-mermaid|github:silverbulletmd/silverbullet-mermaid/mermaid.plug.js|Zef Hemel           |
+|name         |lastModified |contentType  |size|perm|type|uri                                                               |repo                                                     |author   |share-support|
+|--|--|--|--|--|--|--|--|--|--|
+|🔌 Github    |1691137925014|text/markdown|2206|rw|plug|github:silverbulletmd/silverbullet-github/github.plug.js          |https://github.com/silverbulletmd/silverbullet-github    |Zef Hemel|true|
+|🔌 Mattermost|1691137924741|text/markdown|3535|rw|plug|github:silverbulletmd/silverbullet-mattermost/mattermost.plug.json|https://github.com/silverbulletmd/silverbullet-mattermost|Zef Hemel|true|
+|🔌 Git       |1691137924435|text/markdown|1112|rw|plug|github:silverbulletmd/silverbullet-git/git.plug.js                |https://github.com/silverbulletmd/silverbullet-git       |Zef Hemel|    |
+|🔌 Ghost     |1691137922296|text/markdown|1733|rw|plug|github:silverbulletmd/silverbullet-ghost/ghost.plug.js            |https://github.com/silverbulletmd/silverbullet-ghost     |Zef Hemel|true|
+|🔌 Share     |1691137921643|text/markdown|693 |rw|plug|                                                                  |https://github.com/silverbulletmd/silverbullet           |         |    |
 <!-- /query -->

 #### 6.3 Query to select only certain fields
@ -191,11 +191,11 @@ from a visual perspective.
 <!-- #query page select name, author, repo where type = "plug" order by lastModified desc limit 5 -->
 |name         |author   |repo                                                     |
 |--|--|--|
-|🔌 Directive|                    |https://github.com/silverbulletmd/silverbullet        |
-|🔌 KaTeX    |Zef Hemel           |https://github.com/silverbulletmd/silverbullet-katex  |
-|🔌 Core     |                    |https://github.com/silverbulletmd/silverbullet        |
-|🔌 Twitter  |SilverBullet Authors|https://github.com/silverbulletmd/silverbullet-twitter|
-|🔌 Mermaid  |Zef Hemel           |https://github.com/silverbulletmd/silverbullet-mermaid|
+|🔌 Github    |Zef Hemel|https://github.com/silverbulletmd/silverbullet-github    |
+|🔌 Mattermost|Zef Hemel|https://github.com/silverbulletmd/silverbullet-mattermost|
+|🔌 Git       |Zef Hemel|https://github.com/silverbulletmd/silverbullet-git       |
+|🔌 Ghost     |Zef Hemel|https://github.com/silverbulletmd/silverbullet-ghost     |
+|🔌 Share     |         |https://github.com/silverbulletmd/silverbullet           |
 <!-- /query -->

 #### 6.4 Display the data in a format defined by a template
@ -205,11 +205,11 @@ from a visual perspective.
 **Result:** Here you go. This is the result we would like to achieve 🎉. Did you see how I used `render` and `template/plug` in a query? 🚀

 <!-- #query page where type = "plug" order by lastModified desc limit 5 render [[template/plug]] -->
-* [[🔌 Directive]] 
-* [[🔌 KaTeX]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-katex))
-* [[🔌 Core]] 
-* [[🔌 Twitter]] by **SilverBullet Authors** ([repo](https://github.com/silverbulletmd/silverbullet-twitter))
-* [[🔌 Mermaid]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-mermaid))
+* [[🔌 Github]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-github))
+* [[🔌 Mattermost]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-mattermost))
+* [[🔌 Git]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-git))
+* [[🔌 Ghost]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-ghost))
+* [[🔌 Share]]
 <!-- /query -->

 PS: You don't need to select only certain fields to use templates. Templates are
--- a/website/🔌
+++ b/website/🔌
@ -9,8 +9,7 @@ share-support: true
 <!-- #include [[https://raw.githubusercontent.com/silverbulletmd/silverbullet-ghost/main/README.md]] -->
 # Ghost plug for Silver Bullet

-This allows you to publish your pages as [Ghost](https://ghost.org/) pages or
-posts. I use it to publish [Zef+](https://zef.plus).
+This allows you to publish your pages as [Ghost](https://ghost.org/) pages or posts. I use it to publish [Zef+](https://zef.plus).

 ## Configuration

@ -22,9 +21,7 @@ In your `SETTINGS` specify the following settings:
        url: https://your-ghost-blog.ghost.io
    ```

-Then, create a Custom Integration (in your Ghost control panel under Settings >
-Advanced > Integrations > Add Custom Integration). Enter a name (whatever you
-want), then copy the full Admin API Key in your `SECRETS` file, mirroring the
+Then, create a Custom Integration (in your Ghost control panel under Settings > Advanced > Integrations > Add Custom Integration). Enter a name (whatever you want), then copy the full Admin API Key in your `SECRETS` file, mirroring the
 structure of SETTINGS:

    ```yaml
@ -34,10 +31,8 @@ structure of SETTINGS:

 ## Usage

-The plugin hooks into Silver Bullet's
-[Share infrastructure](https://silverbullet.md/%F0%9F%94%8C_Share). Therefore to
-share a page as either a Ghost page or post, add a `$share` front matter key.
-For posts this should take the shape of:
+The plugin hooks into Silver Bullet's [Share infrastructure](https://silverbullet.md/%F0%9F%94%8C_Share). Therefore to
+share a page as either a Ghost page or post, add a `$share` front matter key. For posts this should take the shape of:

    ---
    $share:
@ -51,8 +46,7 @@ And for pages:
    - ghost:myblog:page:my-page-slug
    ---

-Now, when you {[Share: Publish]} (Cmd-s/Ctrl-s) your post will automatically be
-created (as a draft) or updated if it already exists.
+Now, when you {[Share: Publish]} (Cmd-s/Ctrl-s) your post will automatically be created (as a draft) or updated if it already exists.

 Enjoy!

--- a/website/🔌
+++ b/website/🔌
@ -7,28 +7,37 @@ author: Zef Hemel

 <!-- #include [[https://raw.githubusercontent.com/silverbulletmd/silverbullet-git/main/README.md]] -->
 # SilverBullet plug for Git
+
 Very basic in functionality, it assumes you have git configured for push and pull in your space. What it does, roughly speaking:

-`Git : Sync`:
-* Adds all *.md files in your folder to git
-* It commits them with a "Snapshot" commit message
-* It `git pull`s changes from the remote server
-* It `git push`es changes to the remote server
+{[Git: Sync]}:

-`Git: Snapshot`:
-* Asks you for a commit message
-* Commits
+- Adds all files in your folder to git
+- It commits them with a "Snapshot" commit message
+- It `git pull`s changes from the remote server
+- It `git push`es changes to the remote server
+
+{[Git: Snapshot]}:
+
+- Asks you for a commit message
+- Commits
+
+{[Github: Clone]}:
+
+Clones into your space from a Github repository. This will do authentication based on a [personal access token](https://github.com/settings/tokens).

 ## Installation
+
 Open your `PLUGS` note in SilverBullet and add this plug to the list:

 ```
- github:silverbulletmd/silverbullet-git/git.plug.json
+- github:silverbulletmd/silverbullet-git/git.plug.js
 ```

 Then run the `Plugs: Update` command and off you go!

 ## To Build
+
 ```shell
 deno task build
 ```
--- a/website/🔌
+++ b/website/🔌
@ -17,7 +17,7 @@ Provides various integrations with Github:
 Open your `PLUGS` note in SilverBullet and add this plug to the list:

 ```
- github:silverbulletmd/silverbullet-github/github.plug.json
+- github:silverbulletmd/silverbullet-github/github.plug.js
 ```

 Then run the `Plugs: Update` command and off you go!
--- a/Mattermost.md
+++ b/Mattermost.md
@ -7,12 +7,12 @@ share-support: true
 ---

 <!-- #include [[https://raw.githubusercontent.com/silverbulletmd/silverbullet-mattermost/main/README.md]] -->
-# Mattermost for SilverBullet
+# Mattermost for Silver Bullet
 This plug provides various integrations with the [Mattermost suite](https://www.mattermost.com) of products. Please follow the installation, configuration sections, and have a look at the example.

 Features:

-* Integration with [SilverBullet Share](https://silverbullet.md/%F0%9F%94%8C_Share), allowing you to publish and update a page as a post on Mattermost, as well as load existing posts into SB as a page using the {[Share: Mattermost Post: Publish]} (to publish an existing page as a Mattermost post) and {[Share: Mattermost Post: Load]} (to load an existing post into SB) commands.
+* Integration with [Silver Bullet Share](https://silverbullet.md/%F0%9F%94%8C_Share), allowing you to publish and update a page as a post on Mattermost, as well as load existing posts into SB as a page using the {[Share: Mattermost Post: Publish]} (to publish an existing page as a Mattermost post) and {[Share: Mattermost Post: Load]} (to load an existing post into SB) commands.
 * Access your saved posts via the `mm-saved` query provider
 * Unfurl support for posts (after dumping a permalink URL to a post in a page, use the {[Link: Unfurl]} command).
 * Boards support is WIP
@ -46,7 +46,7 @@ In `SECRETS` provide a Mattermost personal access token (or hijack one from your

 * `mm-saved` fetches (by default 15) saved posts in Mattermost, you need to add a `where server = "community"` (with server name) clause to your query to select the mattermost server to query.

-To make the `mm-saved` query results look good, it's recommended you render your query results with a template. Here is one to start with: you can keep it in e.g., `templates/mm-saved`:
+To make the `mm-saved` query results look good, it's recommended you render your query results a template. Here is one to start with, you can keep it in e.g. `templates/mm-saved`:

    [{{username}}]({{url}}) in {{#if channelName}}**{{channelName}}**{{else}}a DM{{/if}} at _{{updatedAt}}_ {[Unsave]}:

--- a/website/🔌
+++ b/website/🔌
@ -11,7 +11,6 @@ Specific implementations for sharing are implemented in other plugs, specificall
 <!-- #query page where share-support = true render [[template/page]] -->
 * [[🔌 Ghost]]
 * [[🔌 Markdown]]
-* [[🔌 Collab]]
 * [[🔌 Mattermost]]
 * [[🔌 Github]]
 <!-- /query -->