Complete redo of content indexing and querying (#517)

Complete redo of data store
Introduces live queries and live templates

web/syscalls/clientStore.ts

@@ -1,19 +1,20 @@
-import { KVStore } from "../../plugos/lib/kv_store.ts";
-import { storeSyscalls } from "../../plugos/syscalls/store.ts";
-import { proxySyscalls } from "../../plugos/syscalls/transport.ts";
 import { SysCallMapping } from "../../plugos/system.ts";
+import { DataStore } from "../../plugos/lib/datastore.ts";
+import { KvKey } from "$sb/types.ts";
 
 export function clientStoreSyscalls(
-  db: KVStore,
+  ds: DataStore,
+  prefix: KvKey = ["client"],
 ): SysCallMapping {
-  const localStoreCalls = storeSyscalls(db);
-  return proxySyscalls(
-    ["clientStore.get", "clientStore.set", "clientStore.delete"],
-    (ctx, name, ...args) => {
-      return localStoreCalls[name.replace("clientStore.", "store.")](
-        ctx,
-        ...args,
-      );
+  return {
+    "clientStore.get": (ctx, key: string): Promise<any> => {
+      return ds.get([...prefix, ctx.plug!.name!, key]);
     },
-  );
+    "clientStore.set": (ctx, key: string, val: any): Promise<void> => {
+      return ds.set([...prefix, ctx.plug!.name!, key], val);
+    },
+    "clientStore.delete": (ctx, key: string): Promise<void> => {
+      return ds.delete([...prefix, ctx.plug!.name!, key]);
+    },
+  };
 }

web/syscalls/datastore.proxy.ts

@@ -0,0 +1,15 @@
+import type { SysCallMapping } from "../../plugos/system.ts";
+import type { Client } from "../client.ts";
+import { proxySyscalls } from "./util.ts";
+
+export function dataStoreProxySyscalls(client: Client): SysCallMapping {
+  return proxySyscalls(client, [
+    "datastore.delete",
+    "datastore.set",
+    "datastore.batchSet",
+    "datastore.batchDelete",
+    "datastore.batchGet",
+    "datastore.get",
+    "datastore.query",
+  ]);
+}

web/syscalls/editor.ts

@@ -89,12 +89,20 @@ export function editorSyscalls(editor: Client): SysCallMapping {
         id: id as any,
         config: { html, script, mode },
       });
+      setTimeout(() => {
+        // Dummy dispatch to rerender the editor and toggle the panel
+        editor.editorView.dispatch({});
+      });
     },
     "editor.hidePanel": (_ctx, id: string) => {
       editor.ui.viewDispatch({
         type: "hide-panel",
         id: id as any,
       });
+      setTimeout(() => {
+        // Dummy dispatch to rerender the editor and toggle the panel
+        editor.editorView.dispatch({});
+      });
     },
     "editor.insertAtPos": (_ctx, text: string, pos: number) => {
       editor.editorView.dispatch({

web/syscalls/index.proxy.ts

@@ -1,16 +0,0 @@
-import { SysCallMapping } from "../../plugos/system.ts";
-import { Client } from "../client.ts";
-import { proxySyscalls } from "./util.ts";
-
-export function indexProxySyscalls(client: Client): SysCallMapping {
-  return proxySyscalls(client, [
-    "index.set",
-    "index.batchSet",
-    "index.delete",
-    "index.get",
-    "index.queryPrefix",
-    "index.clearPageIndexForPage",
-    "index.deletePrefixForPage",
-    "index.clearPageIndex",
-  ]);
-}

web/syscalls/index.ts

@@ -1,65 +0,0 @@
-import type { SysCallMapping } from "../../plugos/system.ts";
-import Dexie from "dexie";
-
-type Item = {
-  page: string;
-  key: string;
-  value: any;
-};
-
-export type KV = {
-  key: string;
-  value: any;
-};
-
-export function pageIndexSyscalls(
-  dbName: string,
-  indexedDB?: any,
-  IDBKeyRange?: any,
-): SysCallMapping {
-  const db = new Dexie(dbName, {
-    indexedDB,
-    IDBKeyRange,
-  });
-  db.version(1).stores({
-    "index": "[page+key], page, key",
-  });
-  const items = db.table<Item, { key: string; page: string }>("index");
-  const apiObj: SysCallMapping = {
-    "index.set": (_ctx, page: string, key: string, value: any) => {
-      return items.put({ page, key, value });
-    },
-    "index.batchSet": async (_ctx, page: string, kvs: KV[]) => {
-      // await items.bulkPut(kvs);
-      if (kvs.length === 0) {
-        return;
-      }
-      const values = kvs.flatMap((kv) => ({
-        page,
-        key: kv.key,
-        value: kv.value,
-      }));
-      await items.bulkPut(values);
-    },
-    "index.delete": (_ctx, page: string, key: string) => {
-      return items.delete({ page, key });
-    },
-    "index.get": async (_ctx, page: string, key: string) => {
-      return (await items.get({ page, key }))?.value;
-    },
-    "index.queryPrefix": (_ctx, prefix: string) => {
-      return items.where("key").startsWith(prefix).toArray();
-    },
-    "index.clearPageIndexForPage": async (ctx, page: string) => {
-      await apiObj["index.deletePrefixForPage"](ctx, page, "");
-    },
-    "index.deletePrefixForPage": (_ctx, page: string, prefix: string) => {
-      return items.where({ page }).and((it) => it.key.startsWith(prefix))
-        .delete();
-    },
-    "index.clearPageIndex": () => {
-      return items.clear();
-    },
-  };
-  return apiObj;
-}

web/syscalls/markdown.ts

@@ -1,12 +0,0 @@
-import { SysCallMapping } from "../../plugos/system.ts";
-import { parse } from "../../common/markdown_parser/parse_tree.ts";
-import { Language } from "../deps.ts";
-import type { ParseTree } from "$sb/lib/tree.ts";
-
-export function markdownSyscalls(lang: Language): SysCallMapping {
-  return {
-    "markdown.parseMarkdown": (_ctx, text: string): ParseTree => {
-      return parse(lang, text);
-    },
-  };
-}

web/syscalls/space.ts

@@ -1,7 +1,6 @@
 import { Client } from "../client.ts";
 import { SysCallMapping } from "../../plugos/system.ts";
-import { AttachmentMeta, PageMeta } from "../types.ts";
-import { FileMeta } from "$sb/types.ts";
+import { AttachmentMeta, FileMeta, PageMeta } from "$sb/types.ts";
 
 export function spaceSyscalls(editor: Client): SysCallMapping {
   return {

web/syscalls/store.proxy.ts

@@ -1,18 +0,0 @@
-import type { SysCallMapping } from "../../plugos/system.ts";
-import type { Client } from "../client.ts";
-import { proxySyscalls } from "./util.ts";
-
-export function storeProxySyscalls(client: Client): SysCallMapping {
-  return proxySyscalls(client, [
-    "store.delete",
-    "store.deletePrefix",
-    "store.deleteAll",
-    "store.set",
-    "store.batchSet",
-    "store.batchDelete",
-    "store.batchGet",
-    "store.get",
-    "store.has",
-    "store.queryPrefix",
-  ]);
-}

web/syscalls/system.ts

@@ -14,10 +14,6 @@ export function systemSyscalls(
       name: string,
       ...args: any[]
     ) => {
-      if (!ctx.plug) {
-        throw Error("No plug associated with context");
-      }
-
       if (name === "server" || name === "client") {
         // Backwards compatibility mode (previously there was an 'env' argument)
         name = args[0];
@@ -25,7 +21,9 @@ export function systemSyscalls(
       }
 
       let plug: Plug<any> | undefined = ctx.plug;
-      if (name.indexOf(".") !== -1) {
+      const fullName = name;
+      // console.log("Invoking function", fullName, "on plug", plug);
+      if (name.includes(".")) {
         // plug name in the name
         const [plugName, functionName] = name.split(".");
         plug = system.loadedPlugs.get(plugName);
@@ -34,7 +32,7 @@ export function systemSyscalls(
         }
         name = functionName;
       }
-      const functionDef = plug.manifest!.functions[name];
+      const functionDef = plug?.manifest!.functions[name];
       if (!functionDef) {
         throw Error(`Function ${name} not found`);
       }
@@ -43,7 +41,12 @@ export function systemSyscalls(
         functionDef.env !== system.env
       ) {
         // Proxy to another environment
-        return proxySyscall(ctx, client.remoteSpacePrimitives, name, args);
+        return proxySyscall(
+          ctx,
+          client.remoteSpacePrimitives,
+          "system.invokeFunction",
+          [fullName, ...args],
+        );
       }
       return plug.invoke(name, args);
     },

web/syscalls/util.ts

@@ -1,4 +1,3 @@
-import { plugCompileCommand } from "../../cmd/plug_compile.ts";
 import { HttpSpacePrimitives } from "../../common/spaces/http_space_primitives.ts";
 import { SyscallContext, SysCallMapping } from "../../plugos/system.ts";
 import { SyscallResponse } from "../../server/rpc.ts";

web/syscalls/widget.ts

@@ -0,0 +1,22 @@
+import { SysCallMapping } from "../../plugos/system.ts";
+import { Client } from "../client.ts";
+
+export function widgetSyscalls(
+  client: Client,
+): SysCallMapping {
+  return {
+    "widget.render": (
+      _ctx,
+      lang: string,
+      body: string,
+    ): Promise<{ html: string; script: string }> => {
+      const langCallback = client.system.codeWidgetHook.codeWidgetCallbacks.get(
+        lang,
+      );
+      if (!langCallback) {
+        throw new Error(`Code widget ${lang} not found`);
+      }
+      return langCallback(body);
+    },
+  };
+}

web/syscalls/yaml.ts

@@ -1,16 +0,0 @@
-import { SysCallMapping } from "../../plugos/system.ts";
-import { YAML } from "../deps.ts";
-
-export function yamlSyscalls(): SysCallMapping {
-  return {
-    "yaml.parse": (_ctx, text: string): any => {
-      return YAML.parse(text);
-    },
-    "yaml.stringify": (_ctx, obj: any): string => {
-      return YAML.stringify(obj, {
-        noArrayIndent: true,
-        noCompatMode: true,
-      });
-    },
-  };
-}