PlugOS refactor and other tweaks (#631)

* Prep for in-process plug loading (e.g. for CF workers, Deno Deploy)
* Prototype of fixed in-process loading plugs
* Fix: buttons not to scroll with content
* Better positioning of modal especially on mobile
* Move query caching outside query
* Fix annoying mouse behavior when filter box appears
* Page navigator search tweaks

web/client.ts

@@ -46,11 +46,12 @@ import { DataStoreSpacePrimitives } from "../common/spaces/datastore_space_primi
 import {
   EncryptedSpacePrimitives,
 } from "../common/spaces/encrypted_space_primitives.ts";
-import { LimitedMap } from "../common/limited_map.ts";
+
 import {
   ensureSpaceIndex,
   markFullSpaceIndexComplete,
 } from "../common/space_index.ts";
+import { LimitedMap } from "$sb/lib/limited_map.ts";
 const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/;
 
 const autoSaveInterval = 1000;
@@ -135,7 +136,7 @@ export class Client {
       `${this.dbPrefix}_state`,
     );
     await stateKvPrimitives.init();
-    this.stateDataStore = new DataStore(stateKvPrimitives, true);
+    this.stateDataStore = new DataStore(stateKvPrimitives);
 
     // Setup message queue
     this.mq = new DataStoreMQ(this.stateDataStore);
@@ -316,9 +317,13 @@ export class Client {
 
             // We're going to look up the anchor through a API invocation
             const matchingAnchor = await this.system.system.localSyscall(
-              "index",
               "system.invokeFunction",
-              ["getObjectByRef", pageName, "anchor", `${pageName}$${pos}`],
+              [
+                "index.getObjectByRef",
+                pageName,
+                "anchor",
+                `${pageName}$${pos}`,
+              ],
             );
 
             if (!matchingAnchor) {

web/client_system.ts

@@ -133,10 +133,9 @@ export class ClientSystem {
           console.log("Plug updated, reloading", plugName, "from", path);
           this.system.unload(path);
           const plug = await this.system.load(
-            new URL(`/${path}`, location.href),
             plugName,
+            createSandbox(new URL(`/${path}`, location.href)),
             newHash,
-            createSandbox,
           );
           if ((plug.manifest! as Manifest).syntax) {
             // If there are syntax extensions, rebuild the markdown parser immediately
@@ -201,10 +200,9 @@ export class ClientSystem {
       try {
         const plugName = plugNameExtractRegex.exec(plugMeta.name)![1];
         await this.system.load(
-          new URL(plugMeta.name, location.origin),
           plugName,
+          createSandbox(new URL(plugMeta.name, location.origin)),
           plugMeta.lastModified,
-          createSandbox,
         );
       } catch (e: any) {
         console.error(
@@ -228,14 +226,13 @@ export class ClientSystem {
   }
 
   localSyscall(name: string, args: any[]) {
-    return this.system.localSyscall("editor", name, args);
+    return this.system.localSyscall(name, args);
   }
 
   queryObjects<T>(tag: string, query: Query): Promise<T[]> {
-    return this.system.localSyscall(
-      "index",
+    return this.localSyscall(
       "system.invokeFunction",
-      ["queryObjects", tag, query],
+      ["index.queryObjects", tag, query],
     );
   }
 }

web/cm_plugins/markdown_widget.ts

@@ -132,7 +132,7 @@ export class MarkdownWidget extends WidgetType {
       buttons.filter((button) => !button.widgetTarget).map((button, idx) =>
         `<button data-button="${idx}" title="${button.description}">${button.svg}</button> `
       ).join("")
-    }</div>${html}`;
+    }</div><div class="content">${html}</div>`;
   }
 
   private attachListeners(div: HTMLElement, buttons?: CodeWidgetButton[]) {

web/components/basic_modals.tsx

@@ -19,43 +19,41 @@ export function Prompt({
 }) {
   const [text, setText] = useState(defaultValue || "");
   const returnEl = (
-    <div className="sb-modal-wrapper">
-      <div className="sb-modal-box">
-        <div className="sb-prompt">
-          <label>{message}</label>
-          <MiniEditor
-            text={defaultValue || ""}
-            vimMode={vimMode}
-            vimStartInInsertMode={true}
-            focus={true}
-            darkMode={darkMode}
-            completer={completer}
-            onEnter={(text) => {
-              callback(text);
-              return true;
-            }}
-            onEscape={() => {
-              callback();
-            }}
-            onChange={(text) => {
-              setText(text);
-            }}
-          />
-          <button
-            onClick={() => {
-              callback(text);
-            }}
-          >
-            Ok
-          </button>
-          <button
-            onClick={() => {
-              callback();
-            }}
-          >
-            Cancel
-          </button>
-        </div>
+    <div className="sb-modal-box">
+      <div className="sb-prompt">
+        <label>{message}</label>
+        <MiniEditor
+          text={defaultValue || ""}
+          vimMode={vimMode}
+          vimStartInInsertMode={true}
+          focus={true}
+          darkMode={darkMode}
+          completer={completer}
+          onEnter={(text) => {
+            callback(text);
+            return true;
+          }}
+          onEscape={() => {
+            callback();
+          }}
+          onChange={(text) => {
+            setText(text);
+          }}
+        />
+        <button
+          onClick={() => {
+            callback(text);
+          }}
+        >
+          Ok
+        </button>
+        <button
+          onClick={() => {
+            callback();
+          }}
+        >
+          Cancel
+        </button>
       </div>
     </div>
   );

web/components/filter.tsx

@@ -95,115 +95,115 @@ export function FilterList({
   }, []);
 
   const returnEl = (
-    <div className="sb-modal-wrapper">
-      <div className="sb-modal-box">
-        <div
-          className="sb-header"
-          onClick={(e) => {
-            // Allow tapping/clicking the header without closing it
-            e.stopPropagation();
+    <div className="sb-modal-box">
+      <div
+        className="sb-header"
+        onClick={(e) => {
+          // Allow tapping/clicking the header without closing it
+          e.stopPropagation();
+        }}
+      >
+        <label>{label}</label>
+        <MiniEditor
+          text={text}
+          vimMode={vimMode}
+          vimStartInInsertMode={true}
+          focus={true}
+          darkMode={darkMode}
+          completer={completer}
+          placeholderText={placeholder}
+          onEnter={(_newText, shiftDown) => {
+            onSelect(
+              shiftDown ? { name: text } : matchingOptions[selectedOption],
+            );
+            return true;
           }}
-        >
-          <label>{label}</label>
-          <MiniEditor
-            text={text}
-            vimMode={vimMode}
-            vimStartInInsertMode={true}
-            focus={true}
-            darkMode={darkMode}
-            completer={completer}
-            placeholderText={placeholder}
-            onEnter={(_newText, shiftDown) => {
-              onSelect(
-                shiftDown ? { name: text } : matchingOptions[selectedOption],
-              );
-              return true;
-            }}
-            onEscape={() => {
-              onSelect(undefined);
-            }}
-            onChange={(text) => {
-              setText(text);
-            }}
-            onKeyUp={(view, e) => {
-              // This event is triggered after the key has been processed by CM already
-              if (onKeyPress) {
-                onKeyPress(e.key, view.state.sliceDoc());
-              }
-              return false;
-            }}
-            onKeyDown={(view, e) => {
-              switch (e.key) {
-                case "ArrowUp":
-                  setSelectionOption(Math.max(0, selectedOption - 1));
+          onEscape={() => {
+            onSelect(undefined);
+          }}
+          onChange={(text) => {
+            setText(text);
+          }}
+          onKeyUp={(view, e) => {
+            // This event is triggered after the key has been processed by CM already
+            if (onKeyPress) {
+              onKeyPress(e.key, view.state.sliceDoc());
+            }
+            return false;
+          }}
+          onKeyDown={(view, e) => {
+            switch (e.key) {
+              case "ArrowUp":
+                setSelectionOption(Math.max(0, selectedOption - 1));
+                return true;
+              case "ArrowDown":
+                setSelectionOption(
+                  Math.min(matchingOptions.length - 1, selectedOption + 1),
+                );
+                return true;
+              case "PageUp":
+                setSelectionOption(Math.max(0, selectedOption - 5));
+                return true;
+              case "PageDown":
+                setSelectionOption(Math.max(0, selectedOption + 5));
+                return true;
+              case "Home":
+                setSelectionOption(0);
+                return true;
+              case "End":
+                setSelectionOption(matchingOptions.length - 1);
+                return true;
+              case " ": {
+                const text = view.state.sliceDoc();
+                if (completePrefix && text === "") {
+                  setText(completePrefix);
+                  // updateFilter(completePrefix);
                   return true;
-                case "ArrowDown":
-                  setSelectionOption(
-                    Math.min(matchingOptions.length - 1, selectedOption + 1),
-                  );
-                  return true;
-                case "PageUp":
-                  setSelectionOption(Math.max(0, selectedOption - 5));
-                  return true;
-                case "PageDown":
-                  setSelectionOption(Math.max(0, selectedOption + 5));
-                  return true;
-                case "Home":
-                  setSelectionOption(0);
-                  return true;
-                case "End":
-                  setSelectionOption(matchingOptions.length - 1);
-                  return true;
-                case " ": {
-                  const text = view.state.sliceDoc();
-                  if (completePrefix && text === "") {
-                    setText(completePrefix);
-                    // updateFilter(completePrefix);
-                    return true;
-                  }
-                  break;
                 }
+                break;
               }
-              return false;
-            }}
-          />
-        </div>
-        <div
-          className="sb-help-text"
-          dangerouslySetInnerHTML={{ __html: helpText }}
-        >
-        </div>
-        <div className="sb-result-list">
-          {matchingOptions && matchingOptions.length > 0
-            ? matchingOptions.map((option, idx) => (
-              <div
-                key={"" + idx}
-                ref={selectedOption === idx ? selectedElementRef : undefined}
-                className={selectedOption === idx
-                  ? "sb-selected-option"
-                  : "sb-option"}
-                onMouseOver={(e) => {
+            }
+            return false;
+          }}
+        />
+      </div>
+      <div
+        className="sb-help-text"
+        dangerouslySetInnerHTML={{ __html: helpText }}
+      >
+      </div>
+      <div className="sb-result-list">
+        {matchingOptions && matchingOptions.length > 0
+          ? matchingOptions.map((option, idx) => (
+            <div
+              key={"" + idx}
+              ref={selectedOption === idx ? selectedElementRef : undefined}
+              className={selectedOption === idx
+                ? "sb-selected-option"
+                : "sb-option"}
+              onMouseMove={(e) => {
+                if (selectedOption !== idx) {
                   setSelectionOption(idx);
-                }}
-                onClick={(e) => {
-                  e.stopPropagation();
-                  onSelect(option);
-                }}
-              >
-                {Icon && (
-                  <span className="sb-icon">
-                    <Icon width={16} height={16} />
-                  </span>
-                )}
-                <span className="sb-name">
-                  {option.name}
+                }
+              }}
+              onClick={(e) => {
+                e.stopPropagation();
+                onSelect(option);
+              }}
+            >
+              {Icon && (
+                <span className="sb-icon">
+                  <Icon width={16} height={16} />
                 </span>
-                {option.hint && <span className="sb-hint">{option.hint}</span>}
-                <div className="sb-description">{option.description}</div>
-              </div>
-            ))
-            : null}
-        </div>
+              )}
+              <span className="sb-name">
+                {option.name}
+              </span>
+              {option.hint && <span className="sb-hint">{option.hint}</span>}
+              <div className="sb-description">{option.description}</div>
+            </div>
+          ))
+          : null}
       </div>
     </div>
   );

web/components/fuse_search.ts

@@ -28,17 +28,18 @@ export const fuzzySearchAndSort = (
       weight: 0.3,
     }, {
       name: "baseName",
-      weight: 0.7,
+      weight: 1,
     }, {
       name: "displayName",
-      weight: 0.3,
+      weight: 0.7,
     }, {
       name: "aliases",
-      weight: 0.7,
+      weight: 0.5,
     }],
     includeScore: true,
     shouldSort: true,
     isCaseSensitive: false,
+    ignoreLocation: true,
     threshold: 0.6,
     sortFn: (a, b): number => {
       if (a.score === b.score) {

web/components/top_bar.tsx

@@ -142,8 +142,8 @@ export function TopBar({
                   <button
                     onClick={(e) => {
                       e.preventDefault();
-                      actionButton.callback();
                       e.stopPropagation();
+                      actionButton.callback();
                     }}
                     title={actionButton.description}
                     className={actionButton.class}

web/space.ts

@@ -4,9 +4,6 @@ import { safeRun } from "../common/util.ts";
 
 import { AttachmentMeta, FileMeta, PageMeta } from "$sb/types.ts";
 import { EventHook } from "../plugos/hooks/event.ts";
-import { throttle } from "$sb/lib/async.ts";
-import { DataStore } from "../plugos/lib/datastore.ts";
-import { LimitedMap } from "../common/limited_map.ts";
 
 const pageWatchInterval = 5000;
 

web/styles/editor.scss

@@ -428,12 +428,12 @@
     margin-top: 10px;
   }
 
-  .sb-markdown-top-widget:has(*) {
+  .sb-markdown-top-widget:has(*) .content {
     max-height: 500px;
   }
 
   @media screen and (max-height: 1000px) {
-    .sb-markdown-top-widget:has(*) {
+    .sb-markdown-top-widget:has(*) .content {
       max-height: 300px;
     }
   }
@@ -441,13 +441,16 @@
   .sb-markdown-widget,
   .sb-markdown-top-widget:has(*),
   .sb-markdown-bottom-widget:has(*) {
-    overflow-y: auto;
     border: 1px solid var(--editor-widget-background-color);
     border-radius: 5px;
     white-space: normal;
     position: relative;
     min-height: 48px;
 
+    .content {
+      overflow-y: auto;
+    }
+
     ul,
     ol {
       margin-top: 0;

web/styles/modals.scss

@@ -1,20 +1,16 @@
-.sb-modal-wrapper {
-  position: absolute;
-  margin: auto;
-  max-width: 500px;
-  height: 600px;
-  left: 0;
-  right: 0;
-  top: 0;
-  bottom: 0;
-  max-height: 290px;
-  z-index: 100;
-}
-
 .sb-modal-box {
+  position: absolute;
+  // At the toppest of the toppest
+  z-index: 1000;
+
+  top: 60px;
+  left: 50%;
+  transform: translateX(-50%);
+  width: 700px;
+  max-width: 90%;
+
   border-radius: 8px;
   overflow: hidden;
-  margin: 10px;
 
   .cm-content {
     padding: 0;
@@ -53,7 +49,7 @@
   }
 
   .sb-result-list {
-    max-height: 216px;
+    max-height: 250px;
     overflow-y: scroll;
 
     .sb-icon {

web/syscalls/clientStore.ts

@@ -7,14 +7,14 @@ export function clientStoreSyscalls(
   prefix: KvKey = ["client"],
 ): SysCallMapping {
   return {
-    "clientStore.get": (ctx, key: string): Promise<any> => {
-      return ds.get([...prefix, ctx.plug!.name!, key]);
+    "clientStore.get": (_ctx, key: string): Promise<any> => {
+      return ds.get([...prefix, key]);
     },
-    "clientStore.set": (ctx, key: string, val: any): Promise<void> => {
-      return ds.set([...prefix, ctx.plug!.name!, key], val);
+    "clientStore.set": (_ctx, key: string, val: any): Promise<void> => {
+      return ds.set([...prefix, key], val);
     },
-    "clientStore.delete": (ctx, key: string): Promise<void> => {
-      return ds.delete([...prefix, ctx.plug!.name!, key]);
+    "clientStore.delete": (_ctx, key: string): Promise<void> => {
+      return ds.delete([...prefix, key]);
     },
   };
 }

web/syscalls/datastore.proxy.ts

@@ -1,48 +1,17 @@
 import { KvQuery } from "$sb/types.ts";
-import { LimitedMap } from "../../common/limited_map.ts";
+import { LimitedMap } from "../../plug-api/lib/limited_map.ts";
 import type { SysCallMapping } from "../../plugos/system.ts";
 import type { Client } from "../client.ts";
 import { proxySyscall, proxySyscalls } from "./util.ts";
 
 export function dataStoreProxySyscalls(client: Client): SysCallMapping {
-  const syscalls = proxySyscalls(client, [
+  return proxySyscalls(client, [
     "datastore.delete",
     "datastore.set",
     "datastore.batchSet",
     "datastore.batchDelete",
     "datastore.batchGet",
+    "datastore.query",
     "datastore.get",
   ]);
-  // Add a cache for datastore.query
-  const queryCache = new LimitedMap<any>(5);
-  syscalls["datastore.query"] = async (ctx, query: KvQuery) => {
-    let cacheKey: string | undefined;
-    const cacheSecs = query.cacheSecs;
-    // Should we do caching?
-    if (cacheSecs) {
-      // Remove the cacheSecs from the query
-      query = { ...query, cacheSecs: undefined };
-      cacheKey = JSON.stringify(query);
-      const cachedResult = queryCache.get(cacheKey);
-      if (cachedResult) {
-        // Let's use the cached result
-        return cachedResult;
-      }
-    }
-
-    const result = await proxySyscall(
-      ctx,
-      client.httpSpacePrimitives,
-      "datastore.query",
-      [
-        query,
-      ],
-    );
-    if (cacheKey) {
-      // Store in the cache
-      queryCache.set(cacheKey, result, cacheSecs! * 1000);
-    }
-    return result;
-  };
-  return syscalls;
 }

web/syscalls/editor.ts

@@ -220,10 +220,7 @@ export function editorSyscalls(editor: Client): SysCallMapping {
     ): Promise<string | undefined> => {
       return editor.prompt(message, defaultValue);
     },
-    "editor.confirm": (
-      _ctx,
-      message: string,
-    ): Promise<boolean> => {
+    "editor.confirm": (_ctx, message: string): Promise<boolean> => {
       return editor.confirm(message);
     },
     "editor.getUiOption": (_ctx, key: string): any => {

web/syscalls/space.ts

@@ -7,10 +7,7 @@ export function spaceSyscalls(editor: Client): SysCallMapping {
     "space.listPages": (): Promise<PageMeta[]> => {
       return editor.space.fetchPageList();
     },
-    "space.readPage": async (
-      _ctx,
-      name: string,
-    ): Promise<string> => {
+    "space.readPage": async (_ctx, name: string): Promise<string> => {
       return (await editor.space.readPage(name)).text;
     },
     "space.getPageMeta": (_ctx, name: string): Promise<PageMeta> => {
@@ -39,10 +36,7 @@ export function spaceSyscalls(editor: Client): SysCallMapping {
     "space.listAttachments": async (): Promise<AttachmentMeta[]> => {
       return await editor.space.fetchAttachmentList();
     },
-    "space.readAttachment": async (
-      _ctx,
-      name: string,
-    ): Promise<Uint8Array> => {
+    "space.readAttachment": async (_ctx, name: string): Promise<Uint8Array> => {
       return (await editor.space.readAttachment(name)).data;
     },
     "space.getAttachmentMeta": async (

web/syscalls/system.ts

@@ -1,4 +1,3 @@
-import type { Plug } from "../../plugos/plug.ts";
 import { SysCallMapping, System } from "../../plugos/system.ts";
 import type { Client } from "../client.ts";
 import { CommandDef } from "../hooks/command.ts";
@@ -11,30 +10,20 @@ export function systemSyscalls(
   const api: SysCallMapping = {
     "system.invokeFunction": (
       ctx,
-      name: string,
+      fullName: string, // plug.function
       ...args: any[]
     ) => {
-      if (name === "server" || name === "client") {
-        // Backwards compatibility mode (previously there was an 'env' argument)
-        name = args[0];
-        args = args.slice(1);
+      const [plugName, functionName] = fullName.split(".");
+      if (!plugName || !functionName) {
+        throw Error(`Invalid function name ${fullName}`);
       }
-
-      let plug: Plug<any> | undefined = ctx.plug;
-      const fullName = name;
-      // console.log("Invoking function", fullName, "on plug", plug);
-      if (name.includes(".")) {
-        // plug name in the name
-        const [plugName, functionName] = name.split(".");
-        plug = system.loadedPlugs.get(plugName);
-        if (!plug) {
-          throw Error(`Plug ${plugName} not found`);
-        }
-        name = functionName;
+      const plug = system.loadedPlugs.get(plugName);
+      if (!plug) {
+        throw Error(`Plug ${plugName} not found`);
       }
-      const functionDef = plug?.manifest!.functions[name];
+      const functionDef = plug.manifest!.functions[functionName];
       if (!functionDef) {
-        throw Error(`Function ${name} not found`);
+        throw Error(`Function ${functionName} not found`);
       }
       if (
         client && functionDef.env && system.env &&
@@ -48,7 +37,7 @@ export function systemSyscalls(
           [fullName, ...args],
         );
       }
-      return plug.invoke(name, args);
+      return plug.invoke(functionName, args);
     },
     "system.invokeCommand": (_ctx, name: string, args?: string[]) => {
       if (!client) {

web/syscalls/util.ts

@@ -18,8 +18,11 @@ export async function proxySyscall(
   name: string,
   args: any[],
 ): Promise<any> {
+  if (!ctx.plug) {
+    throw new Error(`Cannot proxy ${name} syscall without plug context`);
+  }
   const resp = await httpSpacePrimitives.authenticatedFetch(
-    `${httpSpacePrimitives.url}/.rpc/${ctx.plug.name}/${name}`,
+    `${httpSpacePrimitives.url}/.rpc/${ctx.plug}/${name}`,
     {
       method: "POST",
       body: JSON.stringify(args),