Complete redo of content indexing and querying (#517)

Complete redo of data store
Introduces live queries and live templates

web/boot.ts

@@ -1,5 +1,4 @@
 import { safeRun } from "../common/util.ts";
-import { IndexedDBKvPrimitives } from "../plugos/lib/indexeddb_kv_primitives.ts";
 import { Client } from "./client.ts";
 
 const syncMode = window.silverBulletConfig.syncOnly ||

web/client.ts

@@ -4,10 +4,11 @@ import {
   CompletionResult,
   EditorView,
   gitIgnoreCompiler,
+  SyntaxNode,
   syntaxTree,
 } from "../common/deps.ts";
 import { fileMetaToPageMeta, Space } from "./space.ts";
-import { FilterOption, PageMeta } from "./types.ts";
+import { FilterOption } from "./types.ts";
 import { parseYamlSettings } from "../common/util.ts";
 import { EventHook } from "../plugos/hooks/event.ts";
 import { AppCommand } from "./hooks/command.ts";
@@ -18,8 +19,6 @@ import { AppViewState, BuiltinSettings } from "./types.ts";
 import type { AppEvent, CompleteEvent } from "../plug-api/app_event.ts";
 import { throttle } from "$sb/lib/async.ts";
 import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts";
-import { IndexedDBSpacePrimitives } from "../common/spaces/indexeddb_space_primitives.ts";
-import { FileMetaSpacePrimitives } from "../common/spaces/file_meta_space_primitives.ts";
 import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives.ts";
 import {
   ISyncService,
@@ -28,7 +27,6 @@ import {
   SyncService,
 } from "./sync_service.ts";
 import { simpleHash } from "../common/crypto.ts";
-import { DexieKVStore } from "../plugos/lib/kv_store.dexie.ts";
 import { SyncStatus } from "../common/spaces/sync.ts";
 import { HttpSpacePrimitives } from "../common/spaces/http_space_primitives.ts";
 import { FallbackSpacePrimitives } from "../common/spaces/fallback_space_primitives.ts";
@@ -38,11 +36,14 @@ import { ClientSystem } from "./client_system.ts";
 import { createEditorState } from "./editor_state.ts";
 import { OpenPages } from "./open_pages.ts";
 import { MainUI } from "./editor_ui.tsx";
-import { DexieMQ } from "../plugos/lib/mq.dexie.ts";
 import { cleanPageRef } from "$sb/lib/resolve.ts";
 import { expandPropertyNames } from "$sb/lib/json.ts";
 import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
-import { FileMeta } from "$sb/types.ts";
+import { FileMeta, PageMeta } from "$sb/types.ts";
+import { DataStore } from "../plugos/lib/datastore.ts";
+import { IndexedDBKvPrimitives } from "../plugos/lib/indexeddb_kv_primitives.ts";
+import { DataStoreMQ } from "../plugos/lib/mq.datastore.ts";
+import { DataStoreSpacePrimitives } from "../common/spaces/datastore_space_primitives.ts";
 const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/;
 
 const autoSaveInterval = 1000;
@@ -60,8 +61,8 @@ declare global {
 
 // TODO: Oh my god, need to refactor this
 export class Client {
-  system: ClientSystem;
-  editorView: EditorView;
+  system!: ClientSystem;
+  editorView!: EditorView;
   private pageNavigator!: PathPageNavigator;
 
   private dbPrefix: string;
@@ -78,22 +79,34 @@ export class Client {
       .catch((e) => console.error("Error dispatching editor:updated event", e));
   }, 1000);
 
+  debouncedPlugsUpdatedEvent = throttle(async () => {
+    // To register new commands, update editor state based on new plugs
+    this.rebuildEditorState();
+    await this.dispatchAppEvent(
+      "editor:pageLoaded",
+      this.currentPage,
+      undefined,
+      true,
+    );
+  }, 1000);
+
   // Track if plugs have been updated since sync cycle
   fullSyncCompleted = false;
 
-  syncService: ISyncService;
+  syncService!: ISyncService;
   settings!: BuiltinSettings;
-  kvStore: DexieKVStore;
-  mq: DexieMQ;
 
   // Event bus used to communicate between components
-  eventHook: EventHook;
+  eventHook!: EventHook;
 
-  ui: MainUI;
-  openPages: OpenPages;
+  ui!: MainUI;
+  openPages!: OpenPages;
+  stateDataStore!: DataStore;
+  spaceDataStore!: DataStore;
+  mq!: DataStoreMQ;
 
   constructor(
-    parent: Element,
+    private parent: Element,
     public syncMode = false,
   ) {
     if (!syncMode) {
@@ -101,15 +114,21 @@ export class Client {
     }
     // Generate a semi-unique prefix for the database so not to reuse databases for different space paths
     this.dbPrefix = "" + simpleHash(window.silverBulletConfig.spaceFolderPath);
+  }
 
-    this.kvStore = new DexieKVStore(
-      `${this.dbPrefix}_store`,
-      "data",
-      globalThis.indexedDB,
-      globalThis.IDBKeyRange,
+  /**
+   * Initialize the client
+   * This is a separated from the constructor to allow for async initialization
+   */
+  async init() {
+    const stateKvPrimitives = new IndexedDBKvPrimitives(
+      `${this.dbPrefix}_state`,
     );
+    await stateKvPrimitives.init();
+    this.stateDataStore = new DataStore(stateKvPrimitives);
 
-    this.mq = new DexieMQ(`${this.dbPrefix}_mq`, indexedDB, IDBKeyRange);
+    // Setup message queue
+    this.mq = new DataStoreMQ(this.stateDataStore);
 
     setInterval(() => {
       // Timeout after 5s, retries 3 times, otherwise drops the message (no DLQ)
@@ -122,19 +141,18 @@ export class Client {
     // Instantiate a PlugOS system
     this.system = new ClientSystem(
       this,
-      this.kvStore,
       this.mq,
-      this.dbPrefix,
+      this.stateDataStore,
       this.eventHook,
     );
 
-    const localSpacePrimitives = this.initSpace();
+    const localSpacePrimitives = await this.initSpace();
 
     this.syncService = this.syncMode
       ? new SyncService(
         localSpacePrimitives,
         this.plugSpaceRemotePrimitives,
-        this.kvStore,
+        this.stateDataStore,
         this.eventHook,
         (path) => {
           // TODO: At some point we should remove the data.db exception here
@@ -148,7 +166,7 @@ export class Client {
       : new NoSyncSyncService(this.space);
 
     this.ui = new MainUI(this);
-    this.ui.render(parent);
+    this.ui.render(this.parent);
 
     this.editorView = new EditorView({
       state: createEditorState(this, "", "", false),
@@ -160,13 +178,8 @@ export class Client {
     this.focus();
 
     // This constructor will always be followed by an (async) invocatition of init()
-  }
+    await this.system.init();
 
-  /**
-   * Initialize the client
-   * This is a separated from the constructor to allow for async initialization
-   */
-  async init() {
     // Load settings
     this.settings = await this.loadSettings();
 
@@ -193,7 +206,6 @@ export class Client {
     await this.dispatchAppEvent("editor:init");
 
     setInterval(() => {
-      // console.log("Syncing page", this.currentPage, "in background");
       try {
         this.syncService.syncFile(`${this.currentPage!}.md`).catch((e: any) => {
           console.error("Interval sync error", e);
@@ -201,7 +213,6 @@ export class Client {
       } catch (e: any) {
         console.error("Interval sync error", e);
       }
-      // console.log("End of kick-off of background sync of", this.currentPage);
     }, pageSyncInterval);
   }
 
@@ -218,23 +229,12 @@ export class Client {
         // "sync:success" is called with a number of operations only from syncSpace(), not from syncing individual pages
         this.fullSyncCompleted = true;
       }
-      if (this.system.plugsUpdated) {
-        // To register new commands, update editor state based on new plugs
-        this.rebuildEditorState();
-        this.dispatchAppEvent(
-          "editor:pageLoaded",
-          this.currentPage,
-          undefined,
-          true,
-        );
-        if (operations) {
-          // Likely initial sync so let's show visually that we're synced now
-          // this.flashNotification(`Synced ${operations} files`, "info");
-          this.showProgress(100);
-        }
+      // if (this.system.plugsUpdated) {
+      if (operations) {
+        // Likely initial sync so let's show visually that we're synced now
+        this.showProgress(100);
       }
-      // Reset for next sync cycle
-      this.system.plugsUpdated = false;
+      // }
 
       this.ui.viewDispatch({ type: "sync-change", syncSuccess: true });
     });
@@ -276,28 +276,27 @@ export class Client {
         if (typeof pos === "string") {
           console.log("Navigating to anchor", pos);
 
-          // We're going to look up the anchor through a direct page store query...
-          // TODO: This should be extracted
-          const posLookup = await this.system.localSyscall(
-            "index.get",
-            [
-              pageName,
-              `a:${pageName}:${pos}`,
-            ],
+          // We're going to look up the anchor through a API invocation
+          const matchingAnchor = await this.system.system.localSyscall(
+            "index",
+            "system.invokeFunction",
+            ["getObjectByRef", pageName, "anchor", `${pageName}@${pos}`],
           );
 
-          if (!posLookup) {
+          if (!matchingAnchor) {
             return this.flashNotification(
-              `Could not find anchor @${pos}`,
+              `Could not find anchor $${pos}`,
               "error",
             );
           } else {
-            pos = +posLookup;
+            pos = matchingAnchor.pos as number;
           }
         }
-        this.editorView.dispatch({
-          selection: { anchor: pos },
-          effects: EditorView.scrollIntoView(pos, { y: "start" }),
+        setTimeout(() => {
+          this.editorView.dispatch({
+            selection: { anchor: pos as number },
+            effects: EditorView.scrollIntoView(pos as number, { y: "start" }),
+          });
         });
       } else if (!stateRestored) {
         // Somewhat ad-hoc way to determine if the document contains frontmatter and if so, putting the cursor _after it_.
@@ -318,13 +317,16 @@ export class Client {
           scrollIntoView: true,
         });
       }
-      await this.kvStore.set("lastOpenedPage", pageName);
+      await this.stateDataStore.set(["client", "lastOpenedPage"], pageName);
     });
 
     if (location.hash === "#boot") {
       (async () => {
         // Cold start PWA load
-        const lastPage = await this.kvStore.get("lastOpenedPage");
+        const lastPage = await this.stateDataStore.get([
+          "client",
+          "lastOpenedPage",
+        ]);
         if (lastPage) {
           await this.navigate(lastPage);
         }
@@ -332,7 +334,7 @@ export class Client {
     }
   }
 
-  initSpace(): SpacePrimitives {
+  async initSpace(): Promise<SpacePrimitives> {
     this.remoteSpacePrimitives = new HttpSpacePrimitives(
       location.origin,
       window.silverBulletConfig.spaceFolderPath,
@@ -348,20 +350,20 @@ export class Client {
     let localSpacePrimitives: SpacePrimitives | undefined;
 
     if (this.syncMode) {
+      // We'll store the space files in a separate data store
+      const spaceKvPrimitives = new IndexedDBKvPrimitives(
+        `${this.dbPrefix}_synced_space`,
+      );
+      await spaceKvPrimitives.init();
+
       localSpacePrimitives = new FilteredSpacePrimitives(
-        new FileMetaSpacePrimitives(
-          new EventedSpacePrimitives(
-            // Using fallback space primitives here to allow (by default) local reads to "fall through" to HTTP when files aren't synced yet
-            new FallbackSpacePrimitives(
-              new IndexedDBSpacePrimitives(
-                `${this.dbPrefix}_space`,
-                globalThis.indexedDB,
-              ),
-              this.plugSpaceRemotePrimitives,
-            ),
-            this.eventHook,
+        new EventedSpacePrimitives(
+          // Using fallback space primitives here to allow (by default) local reads to "fall through" to HTTP when files aren't synced yet
+          new FallbackSpacePrimitives(
+            new DataStoreSpacePrimitives(new DataStore(spaceKvPrimitives)),
+            this.plugSpaceRemotePrimitives,
           ),
-          this.system.indexSyscalls,
+          this.eventHook,
         ),
         (meta) => fileFilterFn(meta.name),
         // Run when a list of files has been retrieved
@@ -381,7 +383,11 @@ export class Client {
       );
     }
 
-    this.space = new Space(localSpacePrimitives, this.kvStore, this.eventHook);
+    this.space = new Space(
+      localSpacePrimitives,
+      this.stateDataStore,
+      this.eventHook,
+    );
 
     this.eventHook.addLocalListener("file:changed", (path: string) => {
       // Only reload when watching the current page (to avoid reloading when switching pages)
@@ -585,6 +591,7 @@ export class Client {
   async loadPlugs() {
     await this.system.reloadPlugsFromSpace(this.space);
     this.rebuildEditorState();
+    await this.eventHook.dispatchEvent("system:ready");
     await this.dispatchAppEvent("plugs:loaded");
   }
 
@@ -627,13 +634,20 @@ export class Client {
     const linePrefix = line.text.slice(0, selection.from - line.from);
 
     const parentNodes: string[] = [];
-    const currentNode = syntaxTree(editorState).resolveInner(selection.from);
+    const sTree = syntaxTree(editorState);
+    const currentNode = sTree.resolveInner(selection.from);
     if (currentNode) {
-      let node = currentNode;
-      while (node.parent) {
-        parentNodes.push(node.parent.name);
+      let node: SyntaxNode | null = currentNode;
+      do {
+        if (node.name === "FencedCode") {
+          const code = editorState.sliceDoc(node.from + 3, node.to);
+          const fencedCodeLanguage = code.split("\n")[0];
+          parentNodes.push(`FencedCode:${fencedCodeLanguage}`);
+        } else {
+          parentNodes.push(node.name);
+        }
         node = node.parent;
-      }
+      } while (node);
     }
 
     const results = await this.dispatchAppEvent(eventName, {
@@ -742,9 +756,6 @@ export class Client {
     let doc;
     try {
       doc = await this.space.readPage(pageName);
-      if (doc.meta.contentType.startsWith("text/html")) {
-        throw new Error("Got HTML page, not markdown");
-      }
     } catch (e: any) {
       if (e.message.includes("Not found")) {
         // Not found, new page

web/client_system.ts

@@ -3,13 +3,11 @@ import { Manifest, SilverBulletHooks } from "../common/manifest.ts";
 import buildMarkdown from "../common/markdown_parser/parser.ts";
 import { CronHook } from "../plugos/hooks/cron.ts";
 import { EventHook } from "../plugos/hooks/event.ts";
-import { DexieKVStore } from "../plugos/lib/kv_store.dexie.ts";
 import { createSandbox } from "../plugos/environments/webworker_sandbox.ts";
 
 import assetSyscalls from "../plugos/syscalls/asset.ts";
 import { eventSyscalls } from "../plugos/syscalls/event.ts";
-import { storeSyscalls } from "../plugos/syscalls/store.ts";
-import { SysCallMapping, System } from "../plugos/system.ts";
+import { System } from "../plugos/system.ts";
 import type { Client } from "./client.ts";
 import { CodeWidgetHook } from "./hooks/code_widget.ts";
 import { CommandHook } from "./hooks/command.ts";
@@ -18,40 +16,41 @@ import { clientStoreSyscalls } from "./syscalls/clientStore.ts";
 import { debugSyscalls } from "./syscalls/debug.ts";
 import { editorSyscalls } from "./syscalls/editor.ts";
 import { sandboxFetchSyscalls } from "./syscalls/fetch.ts";
-import { pageIndexSyscalls } from "./syscalls/index.ts";
-import { markdownSyscalls } from "./syscalls/markdown.ts";
+import { markdownSyscalls } from "../common/syscalls/markdown.ts";
 import { shellSyscalls } from "./syscalls/shell.ts";
 import { spaceSyscalls } from "./syscalls/space.ts";
 import { syncSyscalls } from "./syscalls/sync.ts";
 import { systemSyscalls } from "./syscalls/system.ts";
-import { yamlSyscalls } from "./syscalls/yaml.ts";
+import { yamlSyscalls } from "../common/syscalls/yaml.ts";
 import { Space } from "./space.ts";
 import {
   loadMarkdownExtensions,
   MDExt,
 } from "../common/markdown_parser/markdown_ext.ts";
-import { DexieMQ } from "../plugos/lib/mq.dexie.ts";
 import { MQHook } from "../plugos/hooks/mq.ts";
-import { mqSyscalls } from "../plugos/syscalls/mq.dexie.ts";
-import { indexProxySyscalls } from "./syscalls/index.proxy.ts";
-import { storeProxySyscalls } from "./syscalls/store.proxy.ts";
+import { mqSyscalls } from "../plugos/syscalls/mq.ts";
 import { mqProxySyscalls } from "./syscalls/mq.proxy.ts";
+import { dataStoreProxySyscalls } from "./syscalls/datastore.proxy.ts";
+import { dataStoreSyscalls } from "../plugos/syscalls/datastore.ts";
+import { DataStore } from "../plugos/lib/datastore.ts";
+import { MessageQueue } from "../plugos/lib/mq.ts";
+import { languageSyscalls } from "../common/syscalls/language.ts";
+import { handlebarsSyscalls } from "../common/syscalls/handlebars.ts";
+import { widgetSyscalls } from "./syscalls/widget.ts";
 
 export class ClientSystem {
   commandHook: CommandHook;
   slashCommandHook: SlashCommandHook;
   namespaceHook: PlugNamespaceHook;
-  indexSyscalls: SysCallMapping;
   codeWidgetHook: CodeWidgetHook;
-  plugsUpdated = false;
   mdExtensions: MDExt[] = [];
   system: System<SilverBulletHooks>;
 
   constructor(
     private client: Client,
-    private kvStore: DexieKVStore,
-    private mq: DexieMQ,
-    dbPrefix: string,
+    private mq: MessageQueue,
+    private ds: DataStore,
+    // private dbPrefix: string,
     private eventHook: EventHook,
   ) {
     // Only set environment to "client" when running in thin client mode, otherwise we run everything locally (hybrid)
@@ -67,18 +66,6 @@ export class ClientSystem {
     const cronHook = new CronHook(this.system);
     this.system.addHook(cronHook);
 
-    if (!client.syncMode) {
-      // In non-sync mode, proxy these to the server
-      this.indexSyscalls = indexProxySyscalls(client);
-    } else {
-      // In sync mode, run them locally
-      this.indexSyscalls = pageIndexSyscalls(
-        `${dbPrefix}_page_index`,
-        globalThis.indexedDB,
-        globalThis.IDBKeyRange,
-      );
-    }
-
     // Code widget hook
     this.codeWidgetHook = new CodeWidgetHook();
     this.system.addHook(this.codeWidgetHook);
@@ -93,7 +80,7 @@ export class ClientSystem {
     this.commandHook = new CommandHook();
     this.commandHook.on({
       commandsUpdated: (commandMap) => {
-        this.client.ui.viewDispatch({
+        this.client.ui?.viewDispatch({
           type: "update-commands",
           commands: commandMap,
         });
@@ -118,7 +105,7 @@ export class ClientSystem {
           // If there are syntax extensions, rebuild the markdown parser immediately
           this.updateMarkdownParser();
         }
-        this.plugsUpdated = true;
+        this.client.debouncedPlugsUpdatedEvent();
       }
     });
 
@@ -138,17 +125,9 @@ export class ClientSystem {
     // this.eventHook.addLocalListener("file:deleted", (file) => {
     //   console.log("File deleted", file);
     // });
-
-    this.registerSyscalls();
   }
 
-  registerSyscalls() {
-    const storeCalls = this.client.syncMode
-      // In sync mode handle locally
-      ? storeSyscalls(this.kvStore)
-      // In non-sync mode proxy to server
-      : storeProxySyscalls(this.client);
-
+  async init() {
     // Slash command hook
     this.slashCommandHook = new SlashCommandHook(this.client);
     this.system.addHook(this.slashCommandHook);
@@ -163,16 +142,20 @@ export class ClientSystem {
       markdownSyscalls(buildMarkdown(this.mdExtensions)),
       assetSyscalls(this.system),
       yamlSyscalls(),
+      handlebarsSyscalls(),
+      widgetSyscalls(this.client),
+      languageSyscalls(),
       this.client.syncMode
         // In sync mode handle locally
         ? mqSyscalls(this.mq)
         // In non-sync mode proxy to server
         : mqProxySyscalls(this.client),
-      storeCalls,
-      this.indexSyscalls,
+      this.client.syncMode
+        ? dataStoreSyscalls(this.ds)
+        : dataStoreProxySyscalls(this.client),
       debugSyscalls(),
       syncSyscalls(this.client),
-      clientStoreSyscalls(this.kvStore),
+      clientStoreSyscalls(this.ds),
     );
 
     // Syscalls that require some additional permissions

web/cm_plugins/directive.ts

@@ -19,11 +19,11 @@ export function directivePlugin() {
           return;
         }
 
-        const cursorInRange = isCursorInRange(state, [from, to]);
+        const cursorInRange = isCursorInRange(state, [parent.from, parent.to]);
 
         if (type.name === "DirectiveStart") {
           if (cursorInRange) {
-            // Cursor outside this directive
+            // Cursor inside this directive
             widgets.push(
               Decoration.line({ class: "sb-directive-start" }).range(from),
             );

web/cm_plugins/fenced_code.ts

@@ -1,5 +1,4 @@
 import { WidgetContent } from "../../plug-api/app_event.ts";
-import { panelHtml } from "../components/panel.tsx";
 import { Decoration, EditorState, syntaxTree, WidgetType } from "../deps.ts";
 import type { Client } from "../client.ts";
 import { CodeWidgetCallback } from "../hooks/code_widget.ts";
@@ -8,8 +7,11 @@ import {
   invisibleDecoration,
   isCursorInRange,
 } from "./util.ts";
+import { createWidgetSandboxIFrame } from "../components/widget_sandbox_iframe.ts";
 
 class IFrameWidget extends WidgetType {
+  iframe?: HTMLIFrameElement;
+
   constructor(
     readonly from: number,
     readonly to: number,
@@ -21,71 +23,44 @@ class IFrameWidget extends WidgetType {
   }
 
   toDOM(): HTMLElement {
-    const iframe = document.createElement("iframe");
-    iframe.srcdoc = panelHtml;
-    // iframe.style.height = "0";
-
-    const messageListener = (evt: any) => {
-      if (evt.source !== iframe.contentWindow) {
-        return;
-      }
-      const data = evt.data;
-      if (!data) {
-        return;
-      }
-      switch (data.type) {
-        case "event":
-          this.editor.dispatchAppEvent(data.name, ...data.args);
-          break;
-        case "setHeight":
-          iframe.style.height = data.height + "px";
-          break;
-        case "setBody":
-          this.editor.editorView.dispatch({
-            changes: {
-              from: this.from,
-              to: this.to,
-              insert: data.body,
-            },
-          });
-          break;
-        case "blur":
-          this.editor.editorView.dispatch({
-            selection: { anchor: this.from },
-          });
-          this.editor.focus();
-          break;
-      }
-    };
-
-    iframe.onload = () => {
-      // Subscribe to message event on global object (to receive messages from iframe)
-      globalThis.addEventListener("message", messageListener);
-      // Only run this code once
-      iframe.onload = null;
-      this.codeWidgetCallback(this.bodyText).then(
-        (widgetContent: WidgetContent) => {
-          if (widgetContent.html) {
-            iframe.contentWindow!.postMessage({
-              type: "html",
-              html: widgetContent.html,
-              script: widgetContent.script,
+    const iframe = createWidgetSandboxIFrame(
+      this.editor,
+      this.bodyText,
+      this.codeWidgetCallback(this.bodyText),
+      (message) => {
+        switch (message.type) {
+          case "blur":
+            this.editor.editorView.dispatch({
+              selection: { anchor: this.from },
             });
-          } else if (widgetContent.url) {
-            iframe.contentWindow!.location.href = widgetContent.url;
-            if (widgetContent.height) {
-              iframe.style.height = widgetContent.height + "px";
-            }
-            if (widgetContent.width) {
-              iframe.style.width = widgetContent.width + "px";
-            }
-          }
-        },
-      );
-    };
+            this.editor.focus();
+
+            break;
+          case "reload":
+            this.codeWidgetCallback(this.bodyText).then(
+              (widgetContent: WidgetContent) => {
+                iframe.contentWindow!.postMessage({
+                  type: "html",
+                  html: widgetContent.html,
+                  script: widgetContent.script,
+                  theme: document.getElementsByTagName("html")[0].dataset.theme,
+                });
+              },
+            );
+            break;
+        }
+      },
+    );
+
     return iframe;
   }
 
+  get estimatedHeight(): number {
+    const cachedHeight = this.editor.space.getCachedWidgetHeight(this.bodyText);
+    // console.log("Calling estimated height", cachedHeight);
+    return cachedHeight || 150;
+  }
+
   eq(other: WidgetType): boolean {
     return (
       other instanceof IFrameWidget &&

web/cm_plugins/post_script.ts

@@ -0,0 +1,46 @@
+import { Decoration, EditorState, WidgetType } from "../deps.ts";
+import type { Client } from "../client.ts";
+import { decoratorStateField } from "./util.ts";
+import { PanelConfig } from "../types.ts";
+import { createWidgetSandboxIFrame } from "../components/widget_sandbox_iframe.ts";
+
+class IFrameWidget extends WidgetType {
+  constructor(
+    readonly editor: Client,
+    readonly panel: PanelConfig,
+  ) {
+    super();
+  }
+
+  toDOM(): HTMLElement {
+    const iframe = createWidgetSandboxIFrame(this.editor, null, this.panel);
+    iframe.classList.add("sb-ps-iframe");
+    return iframe;
+  }
+
+  eq(other: WidgetType): boolean {
+    return this.panel.html ===
+        (other as IFrameWidget).panel.html &&
+      this.panel.script ===
+        (other as IFrameWidget).panel.script;
+  }
+}
+
+export function postScriptPlugin(editor: Client) {
+  return decoratorStateField((state: EditorState) => {
+    const widgets: any[] = [];
+    if (editor.ui.viewState.panels.ps.html) {
+      widgets.push(
+        Decoration.widget({
+          widget: new IFrameWidget(
+            editor,
+            editor.ui.viewState.panels.ps,
+          ),
+          side: 1,
+          block: true,
+        }).range(state.doc.length),
+      );
+    }
+    return Decoration.set(widgets);
+  });
+}

web/components/page_navigator.tsx

@@ -1,6 +1,7 @@
 import { FilterList } from "./filter.tsx";
-import { FilterOption, PageMeta } from "../types.ts";
+import { FilterOption } from "../types.ts";
 import { CompletionContext, CompletionResult } from "../deps.ts";
+import { PageMeta } from "$sb/types.ts";
 
 export function PageNavigator({
   allPages,
@@ -51,7 +52,7 @@ export function PageNavigator({
       darkMode={darkMode}
       completer={completer}
       allowNew={true}
-      helpText="Start typing the page name to filter results, press <code>Return</code> to open."
+      helpText="Press <code>Enter</code> to open the selected page, or <code>Shift-Enter</code> to create a new page."
       newHint="Create page"
       completePrefix={completePrefix}
       onSelect={(opt) => {

web/components/panel.tsx

@@ -1,97 +1,7 @@
 import { useEffect, useRef } from "../deps.ts";
 import { Client } from "../client.ts";
 import { PanelConfig } from "../types.ts";
-
-export const panelHtml = `<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <base target="_top">
-<script>
-const pendingRequests = new Map();
-let syscallReqId = 0;
-
-self.syscall = async (name, ...args) => {
-  return await new Promise((resolve, reject) => {
-    syscallReqId++;
-    pendingRequests.set(syscallReqId, { resolve, reject });
-    window.parent.postMessage({
-      type: "syscall",
-      id: syscallReqId,
-      name,
-      args,
-    }, "*");
-  });
-};
-
-window.addEventListener("message", (message) => {
-  const data = message.data;
-  switch (data.type) {
-    case "html":
-      document.body.innerHTML = data.html;
-      if (data.script) {
-        try {
-          eval(data.script);
-        } catch (e) {
-          console.error("Error evaling script", e);
-        }
-      }
-      break;
-    case "syscall-response":
-      {
-        const syscallId = data.id;
-        const lookup = pendingRequests.get(syscallId);
-        if (!lookup) {
-          console.log(
-            "Current outstanding requests",
-            pendingRequests,
-            "looking up",
-            syscallId,
-          );
-          throw Error("Invalid request id");
-        }
-        pendingRequests.delete(syscallId);
-        if (data.error) {
-          lookup.reject(new Error(data.error));
-        } else {
-          lookup.resolve(data.result);
-        }
-      }
-
-      break;
-  }
-});
-
-// DEPRECATED: Use syscall("event.dispatch", ...) instead
-function sendEvent(name, ...args) {
-  window.parent.postMessage({ type: "event", name, args, }, "*");
-}
-function api(obj) {
-  window.parent.postMessage(obj, "*");
-}
-function updateHeight() {
-  api({
-    type: "setHeight", 
-    height: document.documentElement.offsetHeight,
-  });
-}
-
-function loadJsByUrl(url) {
-  const script = document.createElement("script");
-  script.src = url;
-
-  return new Promise((resolve) => {
-    script.onload = resolve;
-  
-    document.documentElement.firstChild.appendChild(script);
-  });
-}
-</script>
-</head>
-<body>
-Loading...
-</body>
-</html>`;
+import { panelHtml } from "./panel_html.ts";
 
 export function Panel({
   config,

web/components/panel_html.ts

@@ -0,0 +1,103 @@
+export const panelHtml = `<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <base target="_top">
+<script>
+const pendingRequests = new Map();
+let syscallReqId = 0;
+
+self.syscall = async (name, ...args) => {
+  return await new Promise((resolve, reject) => {
+    syscallReqId++;
+    pendingRequests.set(syscallReqId, { resolve, reject });
+    window.parent.postMessage({
+      type: "syscall",
+      id: syscallReqId,
+      name,
+      args,
+    }, "*");
+  });
+};
+
+window.addEventListener("message", (message) => {
+  const data = message.data;
+  switch (data.type) {
+    case "html":
+      document.body.innerHTML = data.html;
+      if(data.theme) {
+        document.getElementsByTagName("html")[0].setAttribute("data-theme", data.theme);
+      }
+      if (data.script) {
+        try {
+          eval(data.script);
+        } catch (e) {
+          console.error("Error evaling script", e);
+        }
+      }
+      break;
+    case "syscall-response":
+      {
+        const syscallId = data.id;
+        const lookup = pendingRequests.get(syscallId);
+        if (!lookup) {
+          console.log(
+            "Current outstanding requests",
+            pendingRequests,
+            "looking up",
+            syscallId,
+          );
+          throw Error("Invalid request id");
+        }
+        pendingRequests.delete(syscallId);
+        if (data.error) {
+          lookup.reject(new Error(data.error));
+        } else {
+          lookup.resolve(data.result);
+        }
+      }
+
+      break;
+  }
+});
+
+function api(obj) {
+  window.parent.postMessage(obj, "*");
+}
+
+let oldHeight = undefined;
+let heightChecks = 0;
+function updateHeight() {
+  const body = document.body, html = document.documentElement;
+  let height = Math.max(body.scrollHeight, body.offsetHeight, html.clientHeight, html.scrollHeight, html.offsetHeight);
+  heightChecks++;
+  if(height !== oldHeight) {
+    oldHeight = height;
+    api({
+      type: "setHeight", 
+      height: height,
+    });
+  }
+  if(heightChecks < 25) {
+    setTimeout(updateHeight, 100);
+  }
+}
+setTimeout(() => {
+  updateHeight();
+});
+
+function loadJsByUrl(url) {
+  const script = document.createElement("script");
+  script.src = url;
+
+  return new Promise((resolve) => {
+    script.onload = resolve;
+    document.documentElement.firstChild.appendChild(script);
+  });
+}
+</script>
+</head>
+<body>
+
+</body>
+</html>`;

web/components/widget_sandbox_iframe.ts

@@ -0,0 +1,96 @@
+import { WidgetContent } from "$sb/app_event.ts";
+import { Client } from "../client.ts";
+import { panelHtml } from "./panel_html.ts";
+
+export function createWidgetSandboxIFrame(
+  client: Client,
+  widgetHeightCacheKey: string | null,
+  content: WidgetContent | Promise<WidgetContent>,
+  onMessage?: (message: any) => void,
+) {
+  const iframe = document.createElement("iframe");
+  iframe.srcdoc = panelHtml;
+  // iframe.style.height = "150px";
+
+  const messageListener = (evt: any) => {
+    (async () => {
+      if (evt.source !== iframe.contentWindow) {
+        return;
+      }
+      const data = evt.data;
+      if (!data) {
+        return;
+      }
+      switch (data.type) {
+        case "syscall": {
+          const { id, name, args } = data;
+          try {
+            const result = await client.system.localSyscall(name, args);
+            if (!iframe.contentWindow) {
+              // iFrame already went away
+              return;
+            }
+            iframe.contentWindow!.postMessage({
+              type: "syscall-response",
+              id,
+              result,
+            });
+          } catch (e: any) {
+            if (!iframe.contentWindow) {
+              // iFrame already went away
+              return;
+            }
+            iframe.contentWindow!.postMessage({
+              type: "syscall-response",
+              id,
+              error: e.message,
+            });
+          }
+          break;
+        }
+        case "setHeight":
+          iframe.style.height = data.height + "px";
+          if (widgetHeightCacheKey) {
+            client.space.setCachedWidgetHeight(
+              widgetHeightCacheKey,
+              data.height,
+            );
+          }
+          break;
+        default:
+          if (onMessage) {
+            onMessage(data);
+          }
+      }
+    })().catch((e) => {
+      console.error("Message listener error", e);
+    });
+  };
+
+  iframe.onload = () => {
+    // Subscribe to message event on global object (to receive messages from iframe)
+    globalThis.addEventListener("message", messageListener);
+    // Only run this code once
+    iframe.onload = null;
+    Promise.resolve(content).then((content) => {
+      if (content.html) {
+        iframe.contentWindow!.postMessage({
+          type: "html",
+          html: content.html,
+          script: content.script,
+          theme: document.getElementsByTagName("html")[0].dataset.theme,
+        });
+      } else if (content.url) {
+        iframe.contentWindow!.location.href = content.url;
+        if (content.height) {
+          iframe.style.height = content.height + "px";
+        }
+        if (content.width) {
+          iframe.style.width = content.width + "px";
+        }
+      }
+    }).catch(console.error);
+  };
+
+  return iframe;
+}

web/editor_state.ts

@@ -3,14 +3,10 @@ import { readonlyMode } from "./cm_plugins/readonly.ts";
 import customMarkdownStyle from "./style.ts";
 import {
   autocompletion,
-  cLanguage,
   closeBrackets,
   closeBracketsKeymap,
   codeFolding,
   completionKeymap,
-  cppLanguage,
-  csharpLanguage,
-  dartLanguage,
   drawSelection,
   dropCursor,
   EditorState,
@@ -18,37 +14,18 @@ import {
   highlightSpecialChars,
   history,
   historyKeymap,
-  htmlLanguage,
   indentOnInput,
   indentWithTab,
-  javaLanguage,
-  javascriptLanguage,
-  jsonLanguage,
   KeyBinding,
   keymap,
-  kotlinLanguage,
   LanguageDescription,
   LanguageSupport,
   markdown,
-  objectiveCLanguage,
-  objectiveCppLanguage,
-  postgresqlLanguage,
-  protobufLanguage,
-  pythonLanguage,
-  rustLanguage,
-  scalaLanguage,
   searchKeymap,
-  shellLanguage,
-  sqlLanguage,
   standardKeymap,
-  StreamLanguage,
   syntaxHighlighting,
-  tomlLanguage,
-  typescriptLanguage,
   ViewPlugin,
   ViewUpdate,
-  xmlLanguage,
-  yamlLanguage,
 } from "../common/deps.ts";
 import { Client } from "./client.ts";
 import { vim } from "./deps.ts";
@@ -63,6 +40,8 @@ import {
   pasteLinkExtension,
 } from "./cm_plugins/editor_paste.ts";
 import { TextChange } from "$sb/lib/change.ts";
+import { postScriptPlugin } from "./cm_plugins/post_script.ts";
+import { languageFor } from "../common/languages.ts";
 
 export function createEditorState(
   editor: Client,
@@ -124,143 +103,17 @@ export function createEditorState(
       // The uber markdown mode
       markdown({
         base: markdownLanguage,
-        codeLanguages: [
-          LanguageDescription.of({
-            name: "yaml",
-            alias: ["meta", "data", "embed"],
-            support: new LanguageSupport(StreamLanguage.define(yamlLanguage)),
-          }),
-          LanguageDescription.of({
-            name: "javascript",
-            alias: ["js"],
-            support: new LanguageSupport(javascriptLanguage),
-          }),
-          LanguageDescription.of({
-            name: "typescript",
-            alias: ["ts"],
-            support: new LanguageSupport(typescriptLanguage),
-          }),
-          LanguageDescription.of({
-            name: "sql",
-            alias: ["sql"],
-            support: new LanguageSupport(StreamLanguage.define(sqlLanguage)),
-          }),
-          LanguageDescription.of({
-            name: "postgresql",
-            alias: ["pgsql", "postgres"],
-            support: new LanguageSupport(
-              StreamLanguage.define(postgresqlLanguage),
-            ),
-          }),
-          LanguageDescription.of({
-            name: "rust",
-            alias: ["rs"],
-            support: new LanguageSupport(StreamLanguage.define(rustLanguage)),
-          }),
-          LanguageDescription.of({
-            name: "css",
-            support: new LanguageSupport(StreamLanguage.define(sqlLanguage)),
-          }),
-          LanguageDescription.of({
-            name: "html",
-            support: new LanguageSupport(htmlLanguage),
-          }),
-          LanguageDescription.of({
-            name: "python",
-            alias: ["py"],
-            support: new LanguageSupport(
-              StreamLanguage.define(pythonLanguage),
-            ),
-          }),
-          LanguageDescription.of({
-            name: "protobuf",
-            alias: ["proto"],
-            support: new LanguageSupport(
-              StreamLanguage.define(protobufLanguage),
-            ),
-          }),
-          LanguageDescription.of({
-            name: "shell",
-            alias: ["sh", "bash", "zsh", "fish"],
-            support: new LanguageSupport(
-              StreamLanguage.define(shellLanguage),
-            ),
-          }),
-          LanguageDescription.of({
-            name: "swift",
-            support: new LanguageSupport(StreamLanguage.define(rustLanguage)),
-          }),
-          LanguageDescription.of({
-            name: "toml",
-            support: new LanguageSupport(StreamLanguage.define(tomlLanguage)),
-          }),
-          LanguageDescription.of({
-            name: "json",
-            support: new LanguageSupport(StreamLanguage.define(jsonLanguage)),
-          }),
-          LanguageDescription.of({
-            name: "xml",
-            support: new LanguageSupport(StreamLanguage.define(xmlLanguage)),
-          }),
-          LanguageDescription.of({
-            name: "c",
-            support: new LanguageSupport(StreamLanguage.define(cLanguage)),
-          }),
-          LanguageDescription.of({
-            name: "cpp",
-            alias: ["c++", "cxx"],
-            support: new LanguageSupport(StreamLanguage.define(cppLanguage)),
-          }),
-          LanguageDescription.of({
-            name: "java",
-            support: new LanguageSupport(StreamLanguage.define(javaLanguage)),
-          }),
-          LanguageDescription.of({
-            name: "csharp",
-            alias: ["c#", "cs"],
-            support: new LanguageSupport(
-              StreamLanguage.define(csharpLanguage),
-            ),
-          }),
-          LanguageDescription.of({
-            name: "scala",
-            alias: ["sc"],
-            support: new LanguageSupport(
-              StreamLanguage.define(scalaLanguage),
-            ),
-          }),
-          LanguageDescription.of({
-            name: "kotlin",
-            alias: ["kt", "kts"],
-            support: new LanguageSupport(
-              StreamLanguage.define(kotlinLanguage),
-            ),
-          }),
-          LanguageDescription.of({
-            name: "objc",
-            alias: ["objective-c", "objectivec"],
-            support: new LanguageSupport(
-              StreamLanguage.define(objectiveCLanguage),
-            ),
-          }),
-          LanguageDescription.of({
-            name: "objcpp",
-            alias: [
-              "objc++",
-              "objective-cpp",
-              "objectivecpp",
-              "objective-c++",
-              "objectivec++",
-            ],
-            support: new LanguageSupport(
-              StreamLanguage.define(objectiveCppLanguage),
-            ),
-          }),
-          LanguageDescription.of({
-            name: "dart",
-            support: new LanguageSupport(StreamLanguage.define(dartLanguage)),
-          }),
-        ],
+        codeLanguages: (info) => {
+          const lang = languageFor(info);
+          if (lang) {
+            return LanguageDescription.of({
+              name: info,
+              support: new LanguageSupport(lang),
+            });
+          }
+
+          return null;
+        },
         addKeymap: true,
       }),
       markdownLanguage.data.of({
@@ -286,6 +139,7 @@ export function createEditorState(
       indentOnInput(),
       ...cleanModePlugins(editor),
       EditorView.lineWrapping,
+      postScriptPlugin(editor),
       lineWrapper([
         { selector: "ATXHeading1", class: "sb-line-h1" },
         { selector: "ATXHeading2", class: "sb-line-h2" },

web/service_worker.ts

@@ -1,8 +1,7 @@
-import Dexie from "https://esm.sh/v120/dexie@3.2.2/dist/dexie.js";
-
-import type { FileContent } from "../common/spaces/indexeddb_space_primitives.ts";
+import type { FileContent } from "../common/spaces/datastore_space_primitives.ts";
 import { simpleHash } from "../common/crypto.ts";
-import { FileMeta } from "$sb/types.ts";
+import { DataStore } from "../plugos/lib/datastore.ts";
+import { IndexedDBKvPrimitives } from "../plugos/lib/indexeddb_kv_primitives.ts";
 
 const CACHE_NAME = "{{CACHE_NAME}}";
 
@@ -61,9 +60,9 @@ self.addEventListener("activate", (event: any) => {
   );
 });
 
-let db: Dexie | undefined;
-let fileContentTable: Dexie.Table<FileContent, string> | undefined;
-let fileMetatable: Dexie.Table<FileMeta, string> | undefined;
+let ds: DataStore | undefined;
+const filesMetaPrefix = ["file", "meta"];
+const filesContentPrefix = ["file", "content"];
 
 self.addEventListener("fetch", (event: any) => {
   const url = new URL(event.request.url);
@@ -89,7 +88,7 @@ self.addEventListener("fetch", (event: any) => {
         return cachedResponse;
       }
 
-      if (!fileContentTable) {
+      if (!ds) {
         // Not initialzed yet, or in thin client mode, let's just proxy
         return fetch(request);
       }
@@ -124,18 +123,14 @@ async function handleLocalFileRequest(
   request: Request,
   pathname: string,
 ): Promise<Response> {
-  if (!db?.isOpen()) {
-    console.log("Detected that the DB was closed, reopening");
-    await db!.open();
-  }
+  // if (!db?.isOpen()) {
+  //   console.log("Detected that the DB was closed, reopening");
+  //   await db!.open();
+  // }
   const path = decodeURIComponent(pathname.slice(1));
-  const data = await fileContentTable!.get(path);
+  const data = await ds?.get<FileContent>([...filesContentPrefix, path]);
   if (data) {
     // console.log("Serving from space", path);
-    if (!data.meta) {
-      // Legacy database not fully synced yet
-      data.meta = (await fileMetatable!.get(path))!;
-    }
     return new Response(
       data.data,
       {
@@ -177,7 +172,7 @@ self.addEventListener("message", (event: any) => {
     caches.delete(CACHE_NAME)
       .then(() => {
         console.log("[Service worker]", "Cache deleted");
-        db?.close();
+        // ds?.close();
         event.source.postMessage({ type: "cacheFlushed" });
       });
   }
@@ -186,15 +181,10 @@ self.addEventListener("message", (event: any) => {
     const dbPrefix = "" + simpleHash(spaceFolderPath);
 
     // Setup space
-    db = new Dexie(`${dbPrefix}_space`, {
-      indexedDB: globalThis.indexedDB,
+    const kv = new IndexedDBKvPrimitives(`${dbPrefix}_synced_space`);
+    kv.init().then(() => {
+      ds = new DataStore(kv);
+      console.log("Datastore in service worker initialized...");
     });
-    db.version(1).stores({
-      fileMeta: "name",
-      fileContent: "name",
-    });
-
-    fileContentTable = db.table<FileContent, string>("fileContent");
-    fileMetatable = db.table<FileMeta, string>("fileMeta");
   }
 });

web/space.ts

@@ -1,33 +1,49 @@
 import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
 import { plugPrefix } from "../common/spaces/constants.ts";
 import { safeRun } from "../common/util.ts";
-import { AttachmentMeta, PageMeta } from "./types.ts";
 
-import { KVStore } from "../plugos/lib/kv_store.ts";
-import { FileMeta } from "$sb/types.ts";
+import { AttachmentMeta, FileMeta, PageMeta } from "$sb/types.ts";
 import { EventHook } from "../plugos/hooks/event.ts";
 import { throttle } from "$sb/lib/async.ts";
+import { DataStore } from "../plugos/lib/datastore.ts";
+import { LimitedMap } from "../common/limited_map.ts";
 
 const pageWatchInterval = 5000;
 
 export class Space {
-  imageHeightCache: Record<string, number> = {};
-  // pageMetaCache = new Map<string, PageMeta>();
+  imageHeightCache = new LimitedMap<number>(100); // url -> height
+  widgetHeightCache = new LimitedMap<number>(100); // bodytext -> height
   cachedPageList: PageMeta[] = [];
 
-  debouncedCacheFlush = throttle(() => {
-    this.kvStore.set("imageHeightCache", this.imageHeightCache).catch(
+  debouncedImageCacheFlush = throttle(() => {
+    this.ds.set(["cache", "imageHeight"], this.imageHeightCache).catch(
       console.error,
     );
     console.log("Flushed image height cache to store");
   }, 5000);
 
   setCachedImageHeight(url: string, height: number) {
-    this.imageHeightCache[url] = height;
-    this.debouncedCacheFlush();
+    this.imageHeightCache.set(url, height);
+    this.debouncedImageCacheFlush();
   }
   getCachedImageHeight(url: string): number {
-    return this.imageHeightCache[url] ?? -1;
+    return this.imageHeightCache.get(url) ?? -1;
+  }
+
+  debouncedWidgetCacheFlush = throttle(() => {
+    this.ds.set(["cache", "widgetHeight"], this.widgetHeightCache.toJSON())
+      .catch(
+        console.error,
+      );
+    console.log("Flushed widget height cache to store");
+  }, 5000);
+
+  setCachedWidgetHeight(bodyText: string, height: number) {
+    this.widgetHeightCache.set(bodyText, height);
+    this.debouncedWidgetCacheFlush();
+  }
+  getCachedWidgetHeight(bodyText: string): number {
+    return this.widgetHeightCache.get(bodyText) ?? -1;
   }
 
   // We do watch files in the background to detect changes
@@ -40,16 +56,20 @@ export class Space {
 
   constructor(
     readonly spacePrimitives: SpacePrimitives,
-    private kvStore: KVStore,
+    private ds: DataStore,
     private eventHook: EventHook,
   ) {
     // super();
-    this.kvStore.get("imageHeightCache").then((cache) => {
-      if (cache) {
-        // console.log("Loaded image height cache from KV store", cache);
-        this.imageHeightCache = cache;
-      }
-    });
+    this.ds.batchGet([["cache", "imageHeight"], ["cache", "widgetHeight"]])
+      .then(([imageCache, widgetCache]) => {
+        if (imageCache) {
+          this.imageHeightCache = new LimitedMap(100, imageCache);
+        }
+        if (widgetCache) {
+          // console.log("Loaded widget cache from store", widgetCache);
+          this.widgetHeightCache = new LimitedMap(100, widgetCache);
+        }
+      });
     eventHook.addLocalListener("file:listed", (files: FileMeta[]) => {
       this.cachedPageList = files.filter(this.isListedPage).map(
         fileMetaToPageMeta,

web/style.ts

@@ -21,7 +21,7 @@ export default function highlightStyles(mdExtension: MDExt[]) {
     { tag: ct.AttributeTag, class: "sb-frontmatter" },
     { tag: ct.AttributeNameTag, class: "sb-atom" },
     { tag: ct.TaskTag, class: "sb-task" },
-    { tag: ct.TaskMarkerTag, class: "sb-task-marker" },
+    { tag: ct.TaskMarkTag, class: "sb-task-mark" },
     { tag: ct.TaskStateTag, class: "sb-task-state" },
     { tag: ct.CodeInfoTag, class: "sb-code-info" },
     { tag: ct.CommentTag, class: "sb-comment" },

web/styles/colors.scss

@@ -378,7 +378,7 @@
     color: var(--editor-wiki-link-page-color); // #8f96c2;
   }
 
-  .sb-task-marker {
+  .sb-task-mark {
     color: var(--editor-task-marker-color);
   }
 

web/styles/editor.scss

@@ -308,7 +308,7 @@
     cursor: pointer;
   }
 
-  .sb-task-marker {
+  .sb-task-mark {
     font-size: 91%;
   }
 
@@ -448,9 +448,11 @@
     iframe {
       border: 0;
       width: 100%;
+      max-width: 100%;
       padding: 0;
       margin: 0;
-      max-width: 100%;
+      border: 1px solid var(--editor-directive-background-color);
+      border-radius: 5px;
     }
   }
 

web/styles/main.scss

@@ -74,10 +74,8 @@ body {
       }
 
       .sb-notifications {
-        position: fixed;
-        bottom: 0;
-        left: 5px;
-        right: 5px;
+        float: right;
+        margin-top: 8px;
         font-size: 15px;
         z-index: 100;
 
@@ -87,66 +85,74 @@ body {
           border-radius: 5px;
         }
       }
+    }
 
-      #sb-current-page {
-        flex: 1;
+    #sb-current-page {
+      flex: 1;
 
-        overflow: hidden;
-        white-space: nowrap;
-        text-align: left;
-        display: block;
+      overflow: hidden;
+      white-space: nowrap;
+      text-align: left;
+      display: block;
 
-        .cm-scroller {
-          font-family: var(--ui-font);
-        }
+      .cm-scroller {
+        font-family: var(--ui-font);
+      }
 
-        .cm-content {
+      .cm-content {
+        padding: 0;
+
+        .cm-line {
           padding: 0;
-
-          .cm-line {
-            padding: 0;
-          }
         }
       }
     }
-
-    .sb-actions {
-      text-align: right;
-      position: absolute;
-      right: 15px;
-      top: 0;
-    }
-
-    .progress-wrapper {
-      display: inline-block;
-      position: relative;
-      top: -6px;
-      padding: 4px;
-      background-color: var(--top-background-color);
-      margin-right: -2px;
-    }
-
-    .progress-bar {
-      display: flex;
-      justify-content: center;
-      align-items: center;
-
-      width: 20px;
-      height: 20px;
-      border-radius: 50%;
-      font-size: 6px;
-    }
-
-    // .progress-bar::before {
-    //   content: "66%";
-    // }
   }
 
-  .sb-panel {
-    flex: 1;
+  .sb-actions {
+    text-align: right;
+    position: absolute;
+    right: 15px;
+    top: 0;
   }
+
+  .progress-wrapper {
+    display: inline-block;
+    position: relative;
+    top: -6px;
+    padding: 4px;
+    background-color: var(--top-background-color);
+    margin-right: -2px;
+  }
+
+  .progress-bar {
+    display: flex;
+    justify-content: center;
+    align-items: center;
+
+    width: 20px;
+    height: 20px;
+    border-radius: 50%;
+    font-size: 6px;
+  }
+
+  // .progress-bar::before {
+  //   content: "66%";
+  // }
 }
 
+.sb-panel {
+  flex: 1;
+}
+
+.sb-ps-iframe {
+  width: 100%;
+  margin-top: 10px;
+  border: 1px solid var(--editor-directive-background-color);
+  border-radius: 5px;
+}
+
+
 #sb-main {
   display: flex;
   flex-direction: row;

web/sync_service.ts

@@ -6,22 +6,22 @@ import {
   SyncStatusItem,
 } from "../common/spaces/sync.ts";
 import { EventHook } from "../plugos/hooks/event.ts";
-import { KVStore } from "../plugos/lib/kv_store.ts";
+import { DataStore } from "../plugos/lib/datastore.ts";
 import { Space } from "./space.ts";
 
 // Keeps the current sync snapshot
-const syncSnapshotKey = "syncSnapshot";
+const syncSnapshotKey = ["sync", "snapshot"];
 
 // Keeps the start time of an ongoing sync, is reset once the sync is done
-const syncStartTimeKey = "syncStartTime";
+const syncStartTimeKey = ["sync", "startTime"];
 
 // Keeps the start time of the last full sync cycle
-const syncLastFullCycleKey = "syncLastFullCycle";
+const syncLastFullCycleKey = ["sync", "lastFullCycle"];
 
 // Keeps the last time an activity was registered, used to detect if a sync is still alive and whether a new one should be started already
-const syncLastActivityKey = "syncLastActivity";
+const syncLastActivityKey = ["sync", "lastActivity"];
 
-const syncInitialFullSyncCompletedKey = "syncInitialFullSyncCompleted";
+const syncInitialFullSyncCompletedKey = ["sync", "initialFullSyncCompleted"];
 
 // maximum time between two activities before we consider a sync crashed
 const syncMaxIdleTimeout = 1000 * 27;
@@ -53,7 +53,7 @@ export class SyncService implements ISyncService {
   constructor(
     readonly localSpacePrimitives: SpacePrimitives,
     readonly remoteSpace: SpacePrimitives,
-    private kvStore: KVStore,
+    private ds: DataStore,
     private eventHook: EventHook,
     private isSyncCandidate: (path: string) => boolean,
   ) {
@@ -91,30 +91,30 @@ export class SyncService implements ISyncService {
   }
 
   async isSyncing(): Promise<boolean> {
-    const startTime = await this.kvStore.get(syncStartTimeKey);
+    const startTime = await this.ds.get(syncStartTimeKey);
     if (!startTime) {
       return false;
     }
     // Sync is running, but is it still alive?
-    const lastActivity = await this.kvStore.get(syncLastActivityKey)!;
+    const lastActivity = await this.ds.get(syncLastActivityKey)!;
     if (Date.now() - lastActivity > syncMaxIdleTimeout) {
       // It's been too long since the last activity, let's consider this one crashed and
       // reset the sync start state
-      await this.kvStore.del(syncStartTimeKey);
+      await this.ds.delete(syncStartTimeKey);
       console.info("Sync without activity for too long, resetting");
       return false;
     }
     return true;
   }
 
-  hasInitialSyncCompleted(): Promise<boolean> {
+  async hasInitialSyncCompleted(): Promise<boolean> {
     // Initial sync has happened when sync progress has been reported at least once, but the syncStartTime has been reset (which happens after sync finishes)
-    return this.kvStore.has(syncInitialFullSyncCompletedKey);
+    return !!(await this.ds.get(syncInitialFullSyncCompletedKey));
   }
 
   async registerSyncStart(fullSync: boolean): Promise<void> {
     // Assumption: this is called after an isSyncing() check
-    await this.kvStore.batchSet([
+    await this.ds.batchSet([
       {
         key: syncStartTimeKey,
         value: Date.now(),
@@ -135,23 +135,23 @@ export class SyncService implements ISyncService {
   async registerSyncProgress(status?: SyncStatus): Promise<void> {
     // Emit a sync event at most every 2s
     if (status && this.lastReportedSyncStatus < Date.now() - 2000) {
-      this.eventHook.dispatchEvent("sync:progress", status);
+      await this.eventHook.dispatchEvent("sync:progress", status);
       this.lastReportedSyncStatus = Date.now();
       await this.saveSnapshot(status.snapshot);
     }
-    await this.kvStore.set(syncLastActivityKey, Date.now());
+    await this.ds.set(syncLastActivityKey, Date.now());
   }
 
   async registerSyncStop(isFullSync: boolean): Promise<void> {
     await this.registerSyncProgress();
-    await this.kvStore.del(syncStartTimeKey);
+    await this.ds.delete(syncStartTimeKey);
     if (isFullSync) {
-      await this.kvStore.set(syncInitialFullSyncCompletedKey, true);
+      await this.ds.set(syncInitialFullSyncCompletedKey, true);
     }
   }
 
   async getSnapshot(): Promise<Map<string, SyncStatusItem>> {
-    const snapshot = (await this.kvStore.get(syncSnapshotKey)) || {};
+    const snapshot = (await this.ds.get(syncSnapshotKey)) || {};
     return new Map<string, SyncStatusItem>(
       Object.entries(snapshot),
     );
@@ -194,8 +194,7 @@ export class SyncService implements ISyncService {
     setInterval(async () => {
       try {
         if (!await this.isSyncing()) {
-          const lastFullCycle =
-            (await this.kvStore.get(syncLastFullCycleKey)) || 0;
+          const lastFullCycle = (await this.ds.get(syncLastFullCycleKey)) || 0;
           if (lastFullCycle && Date.now() - lastFullCycle > spaceSyncInterval) {
             // It's been a while since the last full cycle, let's sync the whole space
             await this.syncSpace();
@@ -223,11 +222,11 @@ export class SyncService implements ISyncService {
       );
       await this.saveSnapshot(snapshot);
       await this.registerSyncStop(true);
-      this.eventHook.dispatchEvent("sync:success", operations);
+      await this.eventHook.dispatchEvent("sync:success", operations);
     } catch (e: any) {
       await this.saveSnapshot(snapshot);
       await this.registerSyncStop(false);
-      this.eventHook.dispatchEvent("sync:error", e.message);
+      await this.eventHook.dispatchEvent("sync:error", e.message);
       console.error("Sync error", e.message);
     }
     return operations;
@@ -302,7 +301,7 @@ export class SyncService implements ISyncService {
   }
 
   async saveSnapshot(snapshot: Map<string, SyncStatusItem>) {
-    await this.kvStore.set(syncSnapshotKey, Object.fromEntries(snapshot));
+    await this.ds.set(syncSnapshotKey, Object.fromEntries(snapshot));
   }
 
   public async plugAwareConflictResolver(

web/syscalls/clientStore.ts

@@ -1,19 +1,20 @@
-import { KVStore } from "../../plugos/lib/kv_store.ts";
-import { storeSyscalls } from "../../plugos/syscalls/store.ts";
-import { proxySyscalls } from "../../plugos/syscalls/transport.ts";
 import { SysCallMapping } from "../../plugos/system.ts";
+import { DataStore } from "../../plugos/lib/datastore.ts";
+import { KvKey } from "$sb/types.ts";
 
 export function clientStoreSyscalls(
-  db: KVStore,
+  ds: DataStore,
+  prefix: KvKey = ["client"],
 ): SysCallMapping {
-  const localStoreCalls = storeSyscalls(db);
-  return proxySyscalls(
-    ["clientStore.get", "clientStore.set", "clientStore.delete"],
-    (ctx, name, ...args) => {
-      return localStoreCalls[name.replace("clientStore.", "store.")](
-        ctx,
-        ...args,
-      );
+  return {
+    "clientStore.get": (ctx, key: string): Promise<any> => {
+      return ds.get([...prefix, ctx.plug!.name!, key]);
     },
-  );
+    "clientStore.set": (ctx, key: string, val: any): Promise<void> => {
+      return ds.set([...prefix, ctx.plug!.name!, key], val);
+    },
+    "clientStore.delete": (ctx, key: string): Promise<void> => {
+      return ds.delete([...prefix, ctx.plug!.name!, key]);
+    },
+  };
 }

web/syscalls/datastore.proxy.ts

@@ -0,0 +1,15 @@
+import type { SysCallMapping } from "../../plugos/system.ts";
+import type { Client } from "../client.ts";
+import { proxySyscalls } from "./util.ts";
+
+export function dataStoreProxySyscalls(client: Client): SysCallMapping {
+  return proxySyscalls(client, [
+    "datastore.delete",
+    "datastore.set",
+    "datastore.batchSet",
+    "datastore.batchDelete",
+    "datastore.batchGet",
+    "datastore.get",
+    "datastore.query",
+  ]);
+}

web/syscalls/editor.ts

@@ -89,12 +89,20 @@ export function editorSyscalls(editor: Client): SysCallMapping {
         id: id as any,
         config: { html, script, mode },
       });
+      setTimeout(() => {
+        // Dummy dispatch to rerender the editor and toggle the panel
+        editor.editorView.dispatch({});
+      });
     },
     "editor.hidePanel": (_ctx, id: string) => {
       editor.ui.viewDispatch({
         type: "hide-panel",
         id: id as any,
       });
+      setTimeout(() => {
+        // Dummy dispatch to rerender the editor and toggle the panel
+        editor.editorView.dispatch({});
+      });
     },
     "editor.insertAtPos": (_ctx, text: string, pos: number) => {
       editor.editorView.dispatch({

web/syscalls/index.proxy.ts

@@ -1,16 +0,0 @@
-import { SysCallMapping } from "../../plugos/system.ts";
-import { Client } from "../client.ts";
-import { proxySyscalls } from "./util.ts";
-
-export function indexProxySyscalls(client: Client): SysCallMapping {
-  return proxySyscalls(client, [
-    "index.set",
-    "index.batchSet",
-    "index.delete",
-    "index.get",
-    "index.queryPrefix",
-    "index.clearPageIndexForPage",
-    "index.deletePrefixForPage",
-    "index.clearPageIndex",
-  ]);
-}

web/syscalls/index.ts

@@ -1,65 +0,0 @@
-import type { SysCallMapping } from "../../plugos/system.ts";
-import Dexie from "dexie";
-
-type Item = {
-  page: string;
-  key: string;
-  value: any;
-};
-
-export type KV = {
-  key: string;
-  value: any;
-};
-
-export function pageIndexSyscalls(
-  dbName: string,
-  indexedDB?: any,
-  IDBKeyRange?: any,
-): SysCallMapping {
-  const db = new Dexie(dbName, {
-    indexedDB,
-    IDBKeyRange,
-  });
-  db.version(1).stores({
-    "index": "[page+key], page, key",
-  });
-  const items = db.table<Item, { key: string; page: string }>("index");
-  const apiObj: SysCallMapping = {
-    "index.set": (_ctx, page: string, key: string, value: any) => {
-      return items.put({ page, key, value });
-    },
-    "index.batchSet": async (_ctx, page: string, kvs: KV[]) => {
-      // await items.bulkPut(kvs);
-      if (kvs.length === 0) {
-        return;
-      }
-      const values = kvs.flatMap((kv) => ({
-        page,
-        key: kv.key,
-        value: kv.value,
-      }));
-      await items.bulkPut(values);
-    },
-    "index.delete": (_ctx, page: string, key: string) => {
-      return items.delete({ page, key });
-    },
-    "index.get": async (_ctx, page: string, key: string) => {
-      return (await items.get({ page, key }))?.value;
-    },
-    "index.queryPrefix": (_ctx, prefix: string) => {
-      return items.where("key").startsWith(prefix).toArray();
-    },
-    "index.clearPageIndexForPage": async (ctx, page: string) => {
-      await apiObj["index.deletePrefixForPage"](ctx, page, "");
-    },
-    "index.deletePrefixForPage": (_ctx, page: string, prefix: string) => {
-      return items.where({ page }).and((it) => it.key.startsWith(prefix))
-        .delete();
-    },
-    "index.clearPageIndex": () => {
-      return items.clear();
-    },
-  };
-  return apiObj;
-}

web/syscalls/markdown.ts

@@ -1,12 +0,0 @@
-import { SysCallMapping } from "../../plugos/system.ts";
-import { parse } from "../../common/markdown_parser/parse_tree.ts";
-import { Language } from "../deps.ts";
-import type { ParseTree } from "$sb/lib/tree.ts";
-
-export function markdownSyscalls(lang: Language): SysCallMapping {
-  return {
-    "markdown.parseMarkdown": (_ctx, text: string): ParseTree => {
-      return parse(lang, text);
-    },
-  };
-}

web/syscalls/space.ts

@@ -1,7 +1,6 @@
 import { Client } from "../client.ts";
 import { SysCallMapping } from "../../plugos/system.ts";
-import { AttachmentMeta, PageMeta } from "../types.ts";
-import { FileMeta } from "$sb/types.ts";
+import { AttachmentMeta, FileMeta, PageMeta } from "$sb/types.ts";
 
 export function spaceSyscalls(editor: Client): SysCallMapping {
   return {

web/syscalls/store.proxy.ts

@@ -1,18 +0,0 @@
-import type { SysCallMapping } from "../../plugos/system.ts";
-import type { Client } from "../client.ts";
-import { proxySyscalls } from "./util.ts";
-
-export function storeProxySyscalls(client: Client): SysCallMapping {
-  return proxySyscalls(client, [
-    "store.delete",
-    "store.deletePrefix",
-    "store.deleteAll",
-    "store.set",
-    "store.batchSet",
-    "store.batchDelete",
-    "store.batchGet",
-    "store.get",
-    "store.has",
-    "store.queryPrefix",
-  ]);
-}

web/syscalls/system.ts

@@ -14,10 +14,6 @@ export function systemSyscalls(
       name: string,
       ...args: any[]
     ) => {
-      if (!ctx.plug) {
-        throw Error("No plug associated with context");
-      }
-
       if (name === "server" || name === "client") {
         // Backwards compatibility mode (previously there was an 'env' argument)
         name = args[0];
@@ -25,7 +21,9 @@ export function systemSyscalls(
       }
 
       let plug: Plug<any> | undefined = ctx.plug;
-      if (name.indexOf(".") !== -1) {
+      const fullName = name;
+      // console.log("Invoking function", fullName, "on plug", plug);
+      if (name.includes(".")) {
         // plug name in the name
         const [plugName, functionName] = name.split(".");
         plug = system.loadedPlugs.get(plugName);
@@ -34,7 +32,7 @@ export function systemSyscalls(
         }
         name = functionName;
       }
-      const functionDef = plug.manifest!.functions[name];
+      const functionDef = plug?.manifest!.functions[name];
       if (!functionDef) {
         throw Error(`Function ${name} not found`);
       }
@@ -43,7 +41,12 @@ export function systemSyscalls(
         functionDef.env !== system.env
       ) {
         // Proxy to another environment
-        return proxySyscall(ctx, client.remoteSpacePrimitives, name, args);
+        return proxySyscall(
+          ctx,
+          client.remoteSpacePrimitives,
+          "system.invokeFunction",
+          [fullName, ...args],
+        );
       }
       return plug.invoke(name, args);
     },

web/syscalls/util.ts

@@ -1,4 +1,3 @@
-import { plugCompileCommand } from "../../cmd/plug_compile.ts";
 import { HttpSpacePrimitives } from "../../common/spaces/http_space_primitives.ts";
 import { SyscallContext, SysCallMapping } from "../../plugos/system.ts";
 import { SyscallResponse } from "../../server/rpc.ts";

web/syscalls/widget.ts

@@ -0,0 +1,22 @@
+import { SysCallMapping } from "../../plugos/system.ts";
+import { Client } from "../client.ts";
+
+export function widgetSyscalls(
+  client: Client,
+): SysCallMapping {
+  return {
+    "widget.render": (
+      _ctx,
+      lang: string,
+      body: string,
+    ): Promise<{ html: string; script: string }> => {
+      const langCallback = client.system.codeWidgetHook.codeWidgetCallbacks.get(
+        lang,
+      );
+      if (!langCallback) {
+        throw new Error(`Code widget ${lang} not found`);
+      }
+      return langCallback(body);
+    },
+  };
+}

web/syscalls/yaml.ts

@@ -1,16 +0,0 @@
-import { SysCallMapping } from "../../plugos/system.ts";
-import { YAML } from "../deps.ts";
-
-export function yamlSyscalls(): SysCallMapping {
-  return {
-    "yaml.parse": (_ctx, text: string): any => {
-      return YAML.parse(text);
-    },
-    "yaml.stringify": (_ctx, obj: any): string => {
-      return YAML.stringify(obj, {
-        noArrayIndent: true,
-        noCompatMode: true,
-      });
-    },
-  };
-}

web/types.ts

@@ -1,21 +1,7 @@
 import { Manifest } from "../common/manifest.ts";
+import { PageMeta } from "$sb/types.ts";
 import { AppCommand } from "./hooks/command.ts";
 
-export type PageMeta = {
-  name: string;
-  lastModified: number;
-  lastOpened?: number;
-  perm: "ro" | "rw";
-} & Record<string, any>;
-
-export type AttachmentMeta = {
-  name: string;
-  contentType: string;
-  lastModified: number;
-  size: number;
-  perm: "ro" | "rw";
-};
-
 // Used by FilterBox
 export type FilterOption = {
   name: string;
@@ -104,6 +90,7 @@ export const initialViewState: AppViewState = {
     rhs: {},
     bhs: {},
     modal: {},
+    ps: {},
   },
   allPages: [],
   commands: new Map(),