PlugOS refactor and other tweaks (#631)

* Prep for in-process plug loading (e.g. for CF workers, Deno Deploy)
* Prototype of fixed in-process loading plugs
* Fix: buttons not to scroll with content
* Better positioning of modal especially on mobile
* Move query caching outside query
* Fix annoying mouse behavior when filter box appears
* Page navigator search tweaks

cli/plug_run.test.ts

@@ -2,9 +2,7 @@ import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
 import { compileManifest } from "../plugos/compile.ts";
 import { esbuild } from "../plugos/deps.ts";
 import { runPlug } from "./plug_run.ts";
-import assets from "../dist/plug_asset_bundle.json" assert {
+import assets from "../dist/plug_asset_bundle.json" with { type: "json" };
-  type: "json",
-};
 import { assertEquals } from "../test_deps.ts";
 import { path } from "../common/deps.ts";
 

cmd/plug_run.ts

@@ -1,6 +1,6 @@
 import { runPlug } from "../cli/plug_run.ts";
 import { path } from "../common/deps.ts";
-import assets from "../dist/plug_asset_bundle.json" assert {
+import assets from "../dist/plug_asset_bundle.json" with {
   type: "json",
 };
 import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";

cmd/server.ts

@@ -1,8 +1,8 @@
 import { HttpServer } from "../server/http_server.ts";
-import clientAssetBundle from "../dist/client_asset_bundle.json" assert {
+import clientAssetBundle from "../dist/client_asset_bundle.json" with {
   type: "json",
 };
-import plugAssetBundle from "../dist/plug_asset_bundle.json" assert {
+import plugAssetBundle from "../dist/plug_asset_bundle.json" with {
   type: "json",
 };
 import { AssetBundle, AssetJson } from "../plugos/asset_bundle/bundle.ts";

common/space_index.ts

@@ -4,7 +4,7 @@ import { System } from "../plugos/system.ts";
 const indexVersionKey = ["$indexVersion"];
 
 // Bump this one every time a full reinxex is needed
-const desiredIndexVersion = 2;
+const desiredIndexVersion = 3;
 
 let indexOngoing = false;
 

common/syscalls/language.ts

@@ -16,9 +16,7 @@ export function languageSyscalls(): SysCallMapping {
       }
       return parse(lang, code);
     },
-    "language.listLanguages": (
+    "language.listLanguages": (): string[] => {
-      _ctx,
-    ): string[] => {
       return Object.keys(builtinLanguages);
     },
   };

plug-api/lib/async.ts

@@ -37,24 +37,24 @@ export class PromiseQueue {
     resolve: (value: any) => void;
     reject: (error: any) => void;
   }[] = [];
-  private running = false;
+  private processing = false;
 
   runInQueue(fn: () => Promise<any>): Promise<any> {
     return new Promise((resolve, reject) => {
       this.queue.push({ fn, resolve, reject });
-      if (!this.running) {
+      if (!this.processing) {
-        this.run();
+        this.process();
       }
     });
   }
 
-  private async run(): Promise<void> {
+  private async process(): Promise<void> {
     if (this.queue.length === 0) {
-      this.running = false;
+      this.processing = false;
       return;
     }
 
-    this.running = true;
+    this.processing = true;
     const { fn, resolve, reject } = this.queue.shift()!;
 
     try {
@@ -64,7 +64,7 @@ export class PromiseQueue {
       reject(error);
     }
 
-    this.run(); // Continue processing the next promise in the queue
+    this.process(); // Continue processing the next promise in the queue
   }
 }
 

plug-api/lib/frontmatter.ts

@@ -7,6 +7,7 @@ import {
   replaceNodesMatchingAsync,
   traverseTreeAsync,
 } from "$sb/lib/tree.ts";
+import { expandPropertyNames } from "$sb/lib/json.ts";
 
 export type FrontMatter = { tags?: string[] } & Record<string, any>;
 
@@ -116,6 +117,8 @@ export async function extractFrontmatter(
   data.tags = [...new Set([...tags.map((t) => t.replace(/^#/, ""))])];
 
   // console.log("Extracted tags", data.tags);
+  // Expand property names (e.g. "foo.bar" => { foo: { bar: true } })
+  data = expandPropertyNames(data);
 
   return data;
 }

plug-api/lib/limited_map.test.ts

@@ -1,5 +1,5 @@
 import { sleep } from "$sb/lib/async.ts";
-import { assertEquals } from "../test_deps.ts";
+import { assertEquals } from "../../test_deps.ts";
 import { LimitedMap } from "./limited_map.ts";
 
 Deno.test("limited map", async () => {

plug-api/lib/limited_map.ts

@@ -1,4 +1,8 @@
-type LimitedMapRecord<V> = { value: V; la: number };
+type LimitedMapRecord<V> = {
+  value: V;
+  la: number;
+  expTimer?: number;
+};
 
 export class LimitedMap<V> {
   private map: Map<string, LimitedMapRecord<V>>;
@@ -16,8 +20,13 @@ export class LimitedMap<V> {
    * @param ttl time to live (in ms)
    */
   set(key: string, value: V, ttl?: number) {
+    const entry: LimitedMapRecord<V> = { value, la: Date.now() };
     if (ttl) {
-      setTimeout(() => {
+      const existingEntry = this.map.get(key);
+      if (existingEntry?.expTimer) {
+        clearTimeout(existingEntry.expTimer);
+      }
+      entry.expTimer = setTimeout(() => {
         this.map.delete(key);
       }, ttl);
     }
@@ -26,7 +35,7 @@ export class LimitedMap<V> {
       const oldestKey = this.getOldestKey();
       this.map.delete(oldestKey!);
     }
-    this.map.set(key, { value, la: Date.now() });
+    this.map.set(key, entry);
   }
 
   get(key: string): V | undefined {

plug-api/lib/memory_cache.test.ts

@@ -0,0 +1,17 @@
+import { sleep } from "$sb/lib/async.ts";
+import { ttlCache } from "$sb/lib/memory_cache.ts";
+import { assertEquals } from "../../test_deps.ts";
+
+Deno.test("Memory cache", async () => {
+  let calls = 0;
+  async function expensiveFunction(key: string) {
+    calls++;
+    await sleep(1);
+    return key;
+  }
+  assertEquals("key", await ttlCache("key", expensiveFunction, 0.01));
+  assertEquals(1, calls);
+  assertEquals("key", await ttlCache("key", expensiveFunction, 0.01));
+  assertEquals(1, calls);
+  await sleep(10);
+});

plug-api/lib/memory_cache.ts

@@ -0,0 +1,21 @@
+import { LimitedMap } from "$sb/lib/limited_map.ts";
+
+const cache = new LimitedMap<any>(50);
+
+export async function ttlCache<K, V>(
+  key: K,
+  fn: (key: K) => Promise<V>,
+  ttlSecs?: number,
+): Promise<V> {
+  if (!ttlSecs) {
+    return fn(key);
+  }
+  const serializedKey = JSON.stringify(key);
+  const cached = cache.get(serializedKey);
+  if (cached) {
+    return cached;
+  }
+  const result = await fn(key);
+  cache.set(serializedKey, result, ttlSecs * 1000);
+  return result;
+}

plug-api/plugos-syscall/asset.ts

@@ -2,10 +2,11 @@ import { base64DecodeDataUrl } from "../../plugos/asset_bundle/base64.ts";
 import { syscall } from "./syscall.ts";
 
 export async function readAsset(
+  plugName: string,
   name: string,
   encoding: "utf8" | "dataurl" = "utf8",
 ): Promise<string> {
-  const dataUrl = await syscall("asset.readAsset", name) as string;
+  const dataUrl = await syscall("asset.readAsset", plugName, name) as string;
   switch (encoding) {
     case "utf8":
       return new TextDecoder().decode(base64DecodeDataUrl(dataUrl));

plug-api/types.ts

@@ -73,11 +73,6 @@ export type Query = {
   render?: string;
   renderAll?: boolean;
   distinct?: boolean;
-
-  /**
-   * When set, the DS implementation _may_ cache the result for the given number of seconds.
-   */
-  cacheSecs?: number;
 };
 
 export type KvQuery = Omit<Query, "querySource"> & {

plugos/hooks/endpoint.test.ts

@@ -16,12 +16,7 @@ Deno.test("Run a plugos endpoint server", async () => {
     tempDir,
   );
 
-  await system.load(
+  await system.load("test", createSandbox(new URL(`file://${workerPath}`)));
-    new URL(`file://${workerPath}`),
-    "test",
-    0,
-    createSandbox,
-  );
 
   const app = new Hono();
   const port = 3123;

plugos/hooks/event.ts

@@ -70,7 +70,7 @@ export class EventHook implements Hook<EventHookT> {
                     }
                   } catch (e: any) {
                     console.error(
-                      `Error dispatching event ${eventName} to plug ${plug.name}: ${e.message}`,
+                      `Error dispatching event ${eventName} to ${plug.name}.${name}: ${e.message}`,
                     );
                   }
                 })());

plugos/hooks/mq.ts

@@ -1,6 +1,5 @@
 import { Hook, Manifest } from "../types.ts";
 import { System } from "../system.ts";
-import { fullQueueName } from "../lib/mq_util.ts";
 import { MQMessage } from "$sb/types.ts";
 import { MessageQueue } from "../lib/mq.ts";
 import { throttle } from "$sb/lib/async.ts";
@@ -61,7 +60,7 @@ export class MQHook implements Hook<MQHookT> {
         }
         const subscriptions = functionDef.mqSubscriptions;
         for (const subscriptionDef of subscriptions) {
-          const queue = fullQueueName(plug.name!, subscriptionDef.queue);
+          const queue = subscriptionDef.queue;
           // console.log("Subscribing to queue", queue);
           this.subscriptions.push(
             this.mq.subscribe(

plugos/lib/datastore.test.ts

@@ -7,7 +7,7 @@ import { assertEquals } from "https://deno.land/std@0.165.0/testing/asserts.ts";
 import { PrefixedKvPrimitives } from "./prefixed_kv_primitives.ts";
 
 async function test(db: KvPrimitives) {
-  const datastore = new DataStore(new PrefixedKvPrimitives(db, ["ds"]), false, {
+  const datastore = new DataStore(new PrefixedKvPrimitives(db, ["ds"]), {
     count: (arr: any[]) => arr.length,
   });
   await datastore.set(["user", "peter"], { name: "Peter" });

plugos/lib/datastore.ts

@@ -2,18 +2,13 @@ import { applyQueryNoFilterKV, evalQueryExpression } from "$sb/lib/query.ts";
 import { FunctionMap, KV, KvKey, KvQuery } from "$sb/types.ts";
 import { builtinFunctions } from "$sb/lib/builtin_query_functions.ts";
 import { KvPrimitives } from "./kv_primitives.ts";
-import { LimitedMap } from "../../common/limited_map.ts";
-
 /**
  * This is the data store class you'll actually want to use, wrapping the primitives
  * in a more user-friendly way
  */
 export class DataStore {
-  private cache = new LimitedMap<any>(20);
-
   constructor(
     readonly kv: KvPrimitives,
-    private enableCache = false,
     private functionMap: FunctionMap = builtinFunctions,
   ) {
   }
@@ -63,21 +58,6 @@ export class DataStore {
   }
 
   async query<T = any>(query: KvQuery): Promise<KV<T>[]> {
-    let cacheKey: string | undefined;
-    const cacheSecs = query.cacheSecs;
-    // Should we do caching?
-    if (cacheSecs && this.enableCache) {
-      // Remove the cacheSecs from the query
-      query = { ...query, cacheSecs: undefined };
-      console.log("Going to cache query", query);
-      cacheKey = JSON.stringify(query);
-      const cachedResult = this.cache.get(cacheKey);
-      if (cachedResult) {
-        // Let's use the cached result
-        return cachedResult;
-      }
-    }
-
     const results: KV<T>[] = [];
     let itemCount = 0;
     // Accumulate results
@@ -104,12 +84,7 @@ export class DataStore {
       }
     }
     // Apply order by, limit, and select
-    const finalResult = applyQueryNoFilterKV(query, results, this.functionMap);
+    return applyQueryNoFilterKV(query, results, this.functionMap);
-    if (cacheKey) {
-      // Store in the cache
-      this.cache.set(cacheKey, finalResult, cacheSecs! * 1000);
-    }
-    return finalResult;
   }
 
   async queryDelete(query: KvQuery): Promise<void> {

plugos/lib/mq_util.ts

@@ -1,7 +0,0 @@
-// Adds a plug name to a queue name if it doesn't already have one.
-export function fullQueueName(plugName: string, queueName: string) {
-  if (queueName.includes(".")) {
-    return queueName;
-  }
-  return plugName + "." + queueName;
-}

plugos/plug.ts

@@ -21,7 +21,6 @@ export class Plug<HookT> {
 
   constructor(
     private system: System<HookT>,
-    public workerUrl: URL | undefined,
     readonly name: string,
     private hash: number,
     private sandboxFactory: SandboxFactory<HookT>,
@@ -44,7 +43,7 @@ export class Plug<HookT> {
 
   // Invoke a syscall
   syscall(name: string, args: any[]): Promise<any> {
-    return this.system.syscallWithContext({ plug: this }, name, args);
+    return this.system.syscall({ plug: this.name }, name, args);
   }
 
   /**

plugos/runtime.test.ts

@@ -1,20 +1,25 @@
 import { createSandbox } from "./sandboxes/deno_worker_sandbox.ts";
 import { System } from "./system.ts";
-import { assertEquals } from "../test_deps.ts";
+import { assert, assertEquals } from "../test_deps.ts";
 import { compileManifest } from "./compile.ts";
 import { esbuild } from "./deps.ts";
+import {
+  createSandbox as createNoSandbox,
+  runWithSystemLock,
+} from "./sandboxes/no_sandbox.ts";
+import { sleep } from "$sb/lib/async.ts";
+import { SysCallMapping } from "./system.ts";
 
 Deno.test("Run a deno sandbox", async () => {
   const system = new System("server");
   system.registerSyscalls([], {
     addNumbers: (_ctx, a, b) => {
-      console.log("This is the context", _ctx.plug.name);
       return a + b;
     },
     failingSyscall: () => {
       throw new Error("#fail");
     },
-  });
+  } as SysCallMapping);
   system.registerSyscalls(["restricted"], {
     restrictedSyscall: () => {
       return "restricted";
@@ -34,10 +39,8 @@ Deno.test("Run a deno sandbox", async () => {
   );
 
   const plug = await system.load(
-    new URL(`file://${workerPath}`),
     "test",
-    0,
+    createSandbox(new URL(`file://${workerPath}`)),
-    createSandbox,
   );
 
   assertEquals({
@@ -52,12 +55,31 @@ Deno.test("Run a deno sandbox", async () => {
     `file://${workerPath}`
   );
 
-  const plug2 = await system.loadNoSandbox("test", plugExport);
+  const plug2 = await system.load("test", createNoSandbox(plugExport));
 
-  assertEquals({
+  let running = false;
-    addedNumbers: 3,
+  await Promise.all([
-    yamlMessage: "hello: world\n",
+    runWithSystemLock(system, async () => {
-  }, await plug2.invoke("boot", []));
+      console.log("Starting first run");
+      running = true;
+      await sleep(5);
+      assertEquals({
+        addedNumbers: 3,
+        yamlMessage: "hello: world\n",
+      }, await plug2.invoke("boot", []));
+      console.log("Done first run");
+      running = false;
+    }),
+    runWithSystemLock(system, async () => {
+      assert(!running);
+      console.log("Starting second run");
+      assertEquals({
+        addedNumbers: 3,
+        yamlMessage: "hello: world\n",
+      }, await plug2.invoke("boot", []));
+      console.log("Done second run");
+    }),
+  ]);
 
   await system.unloadAll();
 

plugos/sandboxes/deno_worker_sandbox.ts

@@ -1,26 +1,26 @@
 import { WorkerSandbox } from "./worker_sandbox.ts";
-import { Plug } from "../plug.ts";
+import type { SandboxFactory } from "./sandbox.ts";
-import { Sandbox } from "./sandbox.ts";
 
 // Uses Deno's permissions to lock the worker down significantly
-export function createSandbox<HookT>(plug: Plug<HookT>): Sandbox<HookT> {
+export function createSandbox<HookT>(workerUrl: URL): SandboxFactory<HookT> {
-  return new WorkerSandbox(plug, {
+  return (plug) =>
-    deno: {
+    new WorkerSandbox(plug, workerUrl, {
-      permissions: {
+      deno: {
-        // Allow network access
+        permissions: {
-        net: true,
+          // Allow network access
-        // This is required for console logging to work, apparently?
+          net: true,
-        env: true,
+          // This is required for console logging to work, apparently?
-        // No talking to native code
+          env: true,
-        ffi: false,
+          // No talking to native code
-        // No invocation of shell commands
+          ffi: false,
-        run: false,
+          // No invocation of shell commands
-        // No read access to the file system
+          run: false,
-        read: false,
+          // No read access to the file system
-        // No write access to the file system
+          read: false,
-        write: false,
+          // No write access to the file system
+          write: false,
+        },
       },
-    },
+      // Have to do this because the "deno" option is not standard and doesn't typecheck yet
-    // Have to do this because the "deno" option is not standard and doesn't typecheck yet
+    });
-  } as any);
 }

plugos/sandboxes/no_sandbox.ts

@@ -2,6 +2,38 @@ import { PromiseQueue } from "$sb/lib/async.ts";
 import { Plug } from "../plug.ts";
 import { Sandbox } from "./sandbox.ts";
 import { Manifest } from "../types.ts";
+import { System } from "../system.ts";
+import { SandboxFactory } from "./sandbox.ts";
+
+/**
+ * This implements a "no sandbox" sandbox that actually runs code the main thread, without any isolation.
+ * This is useful for (often serverless) environments like CloudFlare workers and Deno Deploy that do not support workers.
+ * Since these environments often also don't allow dynamic loading (or even eval'ing) of code, plug code needs to be
+ * imported as a regular ESM module (which is possible).
+ *
+ * To make this work, a global `syscall` function needs to be injected into the global scope.
+ * Since a syscall relies on a System, we need to track the active System in a global variable.
+ * The issue with this is that it means that only a single System can be active at a given time per JS process.
+ * To enforce this, we have a runWithSystemLock function that can be used to run code in a System-locked context, effectively queuing the execution of tasks sequentially.
+ * This isn't great, but it's the best we can do.
+ *
+ * Luckily, in the only contexts in which you need to run plugs this way are serverless, where code will be
+ * run in a bunch of isolates with hopefully low parallelism of requests per isolate.
+ */
+
+/**
+ * A type representing the `plug` export of a plug, used via e.g. `import { plug } from "./some.plug.js`
+ * Values of this type are passed into the `noSandboxFactory` function when called on a system.load
+ */
+export type PlugExport = {
+  manifest: Manifest<any>;
+  functionMapping: Record<string, (...args: any[]) => any>;
+};
+
+// The global variable tracking the currently active system (if any)
+let activeSystem:
+  | System<any>
+  | undefined;
 
 // We need to hard inject the syscall function into the global scope
 declare global {
@@ -9,60 +41,76 @@ declare global {
     syscall(name: string, ...args: any[]): Promise<any>;
   }
 }
-
-export type PlugExport<HookT> = {
-  manifest: Manifest<HookT>;
-  functionMapping: Record<string, Function>;
-};
-
-const functionQueue = new PromiseQueue();
-
-let activePlug: Plug<any> | undefined;
-
 // @ts-ignore: globalThis
 globalThis.syscall = (name: string, ...args: any[]): Promise<any> => {
-  if (!activePlug) {
+  if (!activeSystem) {
-    throw new Error("No active plug");
+    throw new Error(`No currently active system, can't invoke syscall ${name}`);
   }
-  console.log("Calling syscall", name, args);
+  // Invoke syscall with no active plug set (because we don't know which plug is invoking the syscall)
-  return activePlug.syscall(name, args);
+  return activeSystem.syscall({}, name, args);
 };
 
+// Global sequential task queue for running tasks in a System-locked context
+const taskQueue = new PromiseQueue();
+
+/**
+ * Schedules a task to run in a System-locked context
+ * in effect this will ensure only one such context is active at a given time allowing for no parallelism
+ * @param system to activate while running the task
+ * @param task callback to run
+ * @returns the result of the task once it completes
+ */
+export function runWithSystemLock(
+  system: System<any>,
+  task: () => Promise<any>,
+): Promise<any> {
+  return taskQueue.runInQueue(async () => {
+    // Set the global active system, which is used by the syscall function
+    activeSystem = system;
+    try {
+      // Run the logic, note putting the await here is crucial to make sure the `finally` block runs at the right time
+      return await task();
+    } finally {
+      // And then reset the global active system whether the thing blew up or not
+      activeSystem = undefined;
+    }
+  });
+}
+
+/**
+ * Implements a no-sandbox sandbox that runs code in the main thread
+ */
 export class NoSandbox<HookT> implements Sandbox<HookT> {
-  manifest?: Manifest<HookT> | undefined;
+  manifest: Manifest<HookT>;
+
   constructor(
-    private plug: Plug<HookT>,
+    readonly plug: Plug<HookT>,
-    private plugExport: PlugExport<HookT>,
+    readonly plugExport: PlugExport,
   ) {
     this.manifest = plugExport.manifest;
     plug.manifest = this.manifest;
   }
 
   init(): Promise<void> {
+    // Nothing to do
     return Promise.resolve();
   }
 
   invoke(name: string, args: any[]): Promise<any> {
-    activePlug = this.plug;
+    const fn = this.plugExport.functionMapping[name];
-    return functionQueue.runInQueue(async () => {
+    if (!fn) {
-      try {
+      throw new Error(`Function not defined: ${name}`);
-        const fn = this.plugExport.functionMapping[name];
+    }
-        if (!fn) {
+    return Promise.resolve(fn(...args));
-          throw new Error(`Function not loaded: ${name}`);
-        }
-        return await fn(...args);
-      } finally {
-        activePlug = undefined;
-      }
-    });
   }
 
   stop() {
+    // Nothing to do
   }
 }
 
-export function noSandboxFactory<HookT>(
+export function createSandbox<HookT>(
-  plugExport: PlugExport<HookT>,
+  plugExport: PlugExport,
-): (plug: Plug<HookT>) => Sandbox<HookT> {
+): SandboxFactory<HookT> {
-  return (plug: Plug<HookT>) => new NoSandbox(plug, plugExport);
+  return (plug: Plug<any>) => new NoSandbox(plug, plugExport);
 }

plugos/sandboxes/web_worker_sandbox.ts

@@ -1,7 +1,7 @@
 import { WorkerSandbox } from "./worker_sandbox.ts";
 import type { Plug } from "../plug.ts";
-import { Sandbox } from "./sandbox.ts";
+import type { SandboxFactory } from "./sandbox.ts";
 
-export function createSandbox<HookT>(plug: Plug<HookT>): Sandbox<HookT> {
+export function createSandbox<HookT>(workerUrl: URL): SandboxFactory<HookT> {
-  return new WorkerSandbox(plug);
+  return (plug: Plug<HookT>) => new WorkerSandbox(plug, workerUrl);
 }

plugos/sandboxes/worker_sandbox.ts

@@ -21,6 +21,7 @@ export class WorkerSandbox<HookT> implements Sandbox<HookT> {
 
   constructor(
     readonly plug: Plug<HookT>,
+    public workerUrl: URL,
     private workerOptions = {},
   ) {
   }
@@ -35,7 +36,7 @@ export class WorkerSandbox<HookT> implements Sandbox<HookT> {
       console.warn("Double init of sandbox, ignoring");
       return Promise.resolve();
     }
-    this.worker = new Worker(this.plug.workerUrl!, {
+    this.worker = new Worker(this.workerUrl, {
       ...this.workerOptions,
       type: "module",
     });

plugos/syscalls/asset.ts

@@ -2,11 +2,8 @@ import { SysCallMapping, System } from "../system.ts";
 
 export default function assetSyscalls(system: System<any>): SysCallMapping {
   return {
-    "asset.readAsset": (
+    "asset.readAsset": (_ctx, plugName: string, name: string): string => {
-      ctx,
+      return system.loadedPlugs.get(plugName)!.assets!.readFileAsDataUrl(
-      name: string,
-    ): string => {
-      return system.loadedPlugs.get(ctx.plug.name!)!.assets!.readFileAsDataUrl(
         name,
       );
     },

plugos/syscalls/datastore.ts

@@ -1,75 +1,47 @@
 import { KV, KvKey, KvQuery } from "$sb/types.ts";
 import type { DataStore } from "../lib/datastore.ts";
-import type { SyscallContext, SysCallMapping } from "../system.ts";
+import type { SysCallMapping } from "../system.ts";
 
 /**
  * Exposes the datastore API to plugs, but scoping everything to a prefix based on the plug's name
  * @param ds the datastore to wrap
  * @param prefix prefix to scope all keys to to which the plug name will be appended
  */
-export function dataStoreSyscalls(
+export function dataStoreSyscalls(ds: DataStore): SysCallMapping {
-  ds: DataStore,
-  prefix: KvKey = ["ds"],
-): SysCallMapping {
   return {
-    "datastore.delete": (ctx, key: KvKey) => {
+    "datastore.delete": (_ctx, key: KvKey) => {
-      return ds.delete(applyPrefix(ctx, key));
+      return ds.delete(key);
     },
 
-    "datastore.set": (ctx, key: KvKey, value: any) => {
+    "datastore.set": (_ctx, key: KvKey, value: any) => {
-      return ds.set(applyPrefix(ctx, key), value);
+      return ds.set(key, value);
     },
 
-    "datastore.batchSet": (ctx, kvs: KV[]) => {
+    "datastore.batchSet": (_ctx, kvs: KV[]) => {
-      return ds.batchSet(
+      return ds.batchSet(kvs);
-        kvs.map((kv) => ({ key: applyPrefix(ctx, kv.key), value: kv.value })),
-      );
     },
 
-    "datastore.batchDelete": (ctx, keys: KvKey[]) => {
+    "datastore.batchDelete": (_ctx, keys: KvKey[]) => {
-      return ds.batchDelete(keys.map((k) => applyPrefix(ctx, k)));
+      return ds.batchDelete(keys);
     },
 
     "datastore.batchGet": (
-      ctx,
+      _ctx,
       keys: KvKey[],
     ): Promise<(any | undefined)[]> => {
-      return ds.batchGet(keys.map((k) => applyPrefix(ctx, k)));
+      return ds.batchGet(keys);
     },
 
-    "datastore.get": (ctx, key: KvKey): Promise<any | null> => {
+    "datastore.get": (_ctx, key: KvKey): Promise<any | null> => {
-      return ds.get(applyPrefix(ctx, key));
+      return ds.get(key);
     },
 
-    "datastore.query": async (
+    "datastore.query": async (_ctx, query: KvQuery): Promise<KV[]> => {
-      ctx,
+      return (await ds.query(query));
-      query: KvQuery,
-    ): Promise<KV[]> => {
-      return (await ds.query({
-        ...query,
-        prefix: applyPrefix(ctx, query.prefix),
-      })).map((kv) => ({
-        key: stripPrefix(kv.key),
-        value: kv.value,
-      }));
     },
 
-    "datastore.queryDelete": (
+    "datastore.queryDelete": (_ctx, query: KvQuery): Promise<void> => {
-      ctx,
+      return ds.queryDelete(query);
-      query: KvQuery,
-    ): Promise<void> => {
-      return ds.queryDelete({
-        ...query,
-        prefix: applyPrefix(ctx, query.prefix),
-      });
     },
   };
-
-  function applyPrefix(ctx: SyscallContext, key?: KvKey): KvKey {
-    return [...prefix, ctx.plug.name!, ...(key ? key : [])];
-  }
-
-  function stripPrefix(key: KvKey): KvKey {
-    return key.slice(prefix.length + 1);
-  }
 }

plugos/syscalls/fs.deno.test.ts

@@ -3,15 +3,13 @@ import { assert } from "../../test_deps.ts";
 import { path } from "../deps.ts";
 import fileSystemSyscalls from "./fs.deno.ts";
 
-const fakeCtx = {} as any;
-
 Deno.test("Test FS operations", async () => {
   const thisFolder = path.resolve(
     path.dirname(new URL(import.meta.url).pathname),
   );
   const syscalls = fileSystemSyscalls(thisFolder);
   const allFiles: FileMeta[] = await syscalls["fs.listFiles"](
-    fakeCtx,
+    {},
     thisFolder,
     true,
   );

plugos/syscalls/mq.ts

@@ -1,25 +1,24 @@
 import { SysCallMapping } from "../system.ts";
-import { fullQueueName } from "../lib/mq_util.ts";
 import { MessageQueue } from "../lib/mq.ts";
 
 export function mqSyscalls(
   mq: MessageQueue,
 ): SysCallMapping {
   return {
-    "mq.send": (ctx, queue: string, body: any) => {
+    "mq.send": (_ctx, queue: string, body: any) => {
-      return mq.send(fullQueueName(ctx.plug.name!, queue), body);
+      return mq.send(queue, body);
     },
-    "mq.batchSend": (ctx, queue: string, bodies: any[]) => {
+    "mq.batchSend": (_ctx, queue: string, bodies: any[]) => {
-      return mq.batchSend(fullQueueName(ctx.plug.name!, queue), bodies);
+      return mq.batchSend(queue, bodies);
     },
-    "mq.ack": (ctx, queue: string, id: string) => {
+    "mq.ack": (_ctx, queue: string, id: string) => {
-      return mq.ack(fullQueueName(ctx.plug.name!, queue), id);
+      return mq.ack(queue, id);
     },
-    "mq.batchAck": (ctx, queue: string, ids: string[]) => {
+    "mq.batchAck": (_ctx, queue: string, ids: string[]) => {
-      return mq.batchAck(fullQueueName(ctx.plug.name!, queue), ids);
+      return mq.batchAck(queue, ids);
     },
-    "mq.getQueueStats": (ctx, queue: string) => {
+    "mq.getQueueStats": (_ctx, queue: string) => {
-      return mq.getQueueStats(fullQueueName(ctx.plug.name!, queue));
+      return mq.getQueueStats(queue);
     },
   };
 }

plugos/syscalls/transport.ts

@@ -1,20 +0,0 @@
-import { SyscallContext, SysCallMapping } from "../system.ts";
-
-export function proxySyscalls(
-  names: string[],
-  transportCall: (
-    ctx: SyscallContext,
-    name: string,
-    ...args: any[]
-  ) => Promise<any>,
-): SysCallMapping {
-  const syscalls: SysCallMapping = {};
-
-  for (const name of names) {
-    syscalls[name] = (ctx, ...args: any[]) => {
-      return transportCall(ctx, name, ...args);
-    };
-  }
-
-  return syscalls;
-}

plugos/system.ts

@@ -3,7 +3,6 @@ import { EventEmitter } from "./event.ts";
 import type { SandboxFactory } from "./sandboxes/sandbox.ts";
 import { Plug } from "./plug.ts";
 import { InMemoryManifestCache, ManifestCache } from "./manifest_cache.ts";
-import { noSandboxFactory, PlugExport } from "./sandboxes/no_sandbox.ts";
 
 export interface SysCallMapping {
   [key: string]: (ctx: SyscallContext, ...args: any) => Promise<any> | any;
@@ -16,11 +15,12 @@ export type SystemEvents<HookT> = {
 
 // Passed to every syscall, allows to pass in additional context that the syscall may use
 export type SyscallContext = {
-  plug: Plug<any>;
+  // This is the plug that is invoking the syscall,
+  // which may be undefined where this cannot be determined (e.g. when running in a NoSandbox)
+  plug?: string;
 };
 
 type SyscallSignature = (
-  ctx: SyscallContext,
   ...args: any[]
 ) => Promise<any> | any;
 
@@ -75,7 +75,11 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
     }
   }
 
-  syscallWithContext(
+  localSyscall(name: string, args: any): Promise<any> {
+    return this.syscall({}, name, args);
+  }
+
+  syscall(
     ctx: SyscallContext,
     name: string,
     args: any[],
@@ -84,36 +88,29 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
     if (!syscall) {
       throw Error(`Unregistered syscall ${name}`);
     }
-    for (const permission of syscall.requiredPermissions) {
+    if (ctx.plug) {
-      if (!ctx.plug) {
+      // Only when running in a plug context do we check permissions
-        throw Error(`Syscall ${name} requires permission and no plug is set`);
+      const plug = this.loadedPlugs.get(ctx.plug!);
+      if (!plug) {
+        throw new Error(
+          `Plug ${ctx.plug} not found while attempting to invoke ${name}}`,
+        );
       }
-      if (!ctx.plug.grantedPermissions.includes(permission)) {
+      for (const permission of syscall.requiredPermissions) {
-        throw Error(`Missing permission '${permission}' for syscall ${name}`);
+        if (!plug.grantedPermissions.includes(permission)) {
+          throw Error(`Missing permission '${permission}' for syscall ${name}`);
+        }
       }
     }
     return Promise.resolve(syscall.callback(ctx, ...args));
   }
 
-  localSyscall(
-    contextPlugName: string,
-    syscallName: string,
-    args: any[],
-  ): Promise<any> {
-    return this.syscallWithContext(
-      { plug: this.plugs.get(contextPlugName)! },
-      syscallName,
-      args,
-    );
-  }
-
   async load(
-    workerUrl: URL,
     name: string,
-    hash: number,
     sandboxFactory: SandboxFactory<HookT>,
+    hash = -1,
   ): Promise<Plug<HookT>> {
-    const plug = new Plug(this, workerUrl, name, hash, sandboxFactory);
+    const plug = new Plug(this, name, hash, sandboxFactory);
 
     // Wait for worker to boot, and pass back its manifest
     await plug.ready;
@@ -139,44 +136,6 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
     return plug;
   }
 
-  /**
-   * Loads a plug without a sandbox, which means it will run in the same context as the caller
-   * @param name
-   * @param plugExport extracted via e.g. `import { plug } from "./some.plug.js`
-   * @returns Plug instance
-   */
-  async loadNoSandbox(
-    name: string,
-    plugExport: PlugExport<HookT>,
-  ): Promise<Plug<HookT>> {
-    const plug = new Plug(
-      this,
-      undefined,
-      name,
-      -1,
-      noSandboxFactory(plugExport),
-    );
-
-    const manifest = plugExport.manifest;
-
-    // Validate the manifest
-    let errors: string[] = [];
-    for (const feature of this.enabledHooks) {
-      errors = [...errors, ...feature.validateManifest(plug.manifest!)];
-    }
-    if (errors.length > 0) {
-      throw new Error(`Invalid manifest: ${errors.join(", ")}`);
-    }
-    if (this.plugs.has(manifest.name)) {
-      this.unload(manifest.name);
-    }
-    console.log("Activated plug without sandbox", manifest.name);
-    this.plugs.set(manifest.name, plug);
-
-    await this.emit("plugLoaded", plug);
-    return plug;
-  }
-
   unload(name: string) {
     const plug = this.plugs.get(name);
     if (!plug) {

plugos/worker_runtime.ts

@@ -9,6 +9,20 @@ declare global {
   function syscall(name: string, ...args: any[]): Promise<any>;
 }
 
+// Are we running in a (web) worker?
+
+// Determines if we're running in a web worker environment (Deno or browser)
+// - in a browser's main threads, typeof window is "object"
+// - in a browser's worker threads, typeof window === "undefined"
+// - in Deno's main thread typeof window === "object"
+// - in Deno's workers typeof window === "undefined
+// - in Cloudflare workers typeof window === "undefined", but typeof globalThis.WebSocketPair is defined
+const runningAsWebWorker = typeof window === "undefined" &&
+  // @ts-ignore: globalThis
+  typeof globalThis.WebSocketPair === "undefined";
+
+// console.log("Running as web worker:", runningAsWebWorker);
+
 if (typeof Deno === "undefined") {
   // @ts-ignore: Deno hack
   self.Deno = {
@@ -35,13 +49,11 @@ const pendingRequests = new Map<
 
 let syscallReqId = 0;
 
-const workerMode = typeof window === "undefined";
-
 function workerPostMessage(msg: ControllerMessage) {
   self.postMessage(msg);
 }
 
-if (workerMode) {
+if (runningAsWebWorker) {
   globalThis.syscall = async (name: string, ...args: any[]) => {
     return await new Promise((resolve, reject) => {
       syscallReqId++;
@@ -61,7 +73,7 @@ export function setupMessageListener(
   functionMapping: Record<string, Function>,
   manifest: any,
 ) {
-  if (!workerMode) {
+  if (!runningAsWebWorker) {
     // Don't do any of this stuff if this is not a web worker
     // This caters to the NoSandbox run mode
     return;
@@ -163,10 +175,10 @@ export async function sandboxFetch(
   return syscall("sandboxFetch.fetch", reqInfo, options);
 }
 
+// @ts-ignore: monkey patching fetch
+globalThis.nativeFetch = globalThis.fetch;
 // Monkey patch fetch()
-
 export function monkeyPatchFetch() {
-  globalThis.nativeFetch = globalThis.fetch;
   // @ts-ignore: monkey patching fetch
   globalThis.fetch = async function (
     reqInfo: RequestInfo,
@@ -192,4 +204,6 @@ export function monkeyPatchFetch() {
   };
 }
 
-monkeyPatchFetch();
+if (runningAsWebWorker) {
+  monkeyPatchFetch();
+}

plugs/editor/complete.ts

@@ -18,9 +18,7 @@ export async function pageComplete(completeEvent: CompleteEvent) {
       completeEvent.linePrefix,
     );
   const tagToQuery = isInTemplateContext ? "template" : "page";
-  let allPages: PageMeta[] = await queryObjects<PageMeta>(tagToQuery, {
+  let allPages: PageMeta[] = await queryObjects<PageMeta>(tagToQuery, {}, 5);
-    cacheSecs: 5,
-  });
   const prefix = match[1];
   if (prefix.startsWith("!")) {
     // Federation prefix, let's first see if we're matching anything from federation that is locally synced

plugs/editor/editor.ts

@@ -31,6 +31,6 @@ export async function moveToPosCommand() {
   await editor.moveCursor(pos);
 }
 
-export async function customFlashMessage(_ctx: any, message: string) {
+export async function customFlashMessage(_def: any, message: string) {
   await editor.flashNotification(message);
 }

plugs/index/anchor.ts

@@ -43,10 +43,7 @@ export async function anchorComplete(completeEvent: CompleteEvent) {
     // "bare" anchor, match any page for completion purposes
     filter = undefined;
   }
-  const allAnchors = await queryObjects<AnchorObject>("anchor", {
+  const allAnchors = await queryObjects<AnchorObject>("anchor", { filter }, 5);
-    filter,
-    cacheSecs: 5,
-  });
   return {
     from: completeEvent.pos - match[1].length,
     options: allAnchors.map((a) => ({

plugs/index/api.ts

@@ -3,6 +3,7 @@ import { KV, KvKey, KvQuery, ObjectQuery, ObjectValue } from "$sb/types.ts";
 import { QueryProviderEvent } from "$sb/app_event.ts";
 import { builtins } from "./builtins.ts";
 import { AttributeObject, determineType } from "./attributes.ts";
+import { ttlCache } from "$sb/lib/memory_cache.ts";
 
 const indexKey = "idx";
 const pageKey = "ridx";
@@ -159,15 +160,18 @@ function cleanKey(ref: string, page: string) {
   }
 }
 
-export async function queryObjects<T>(
+export function queryObjects<T>(
   tag: string,
   query: ObjectQuery,
+  ttlSecs?: number,
 ): Promise<ObjectValue<T>[]> {
-  return (await datastore.query({
+  return ttlCache(query, async () => {
-    ...query,
+    return (await datastore.query({
-    prefix: [indexKey, tag],
+      ...query,
-    distinct: true,
+      prefix: [indexKey, tag],
-  })).map(({ value }) => value);
+      distinct: true,
+    })).map(({ value }) => value);
+  }, ttlSecs);
 }
 
 export async function query(

plugs/index/attributes.ts

@@ -52,8 +52,7 @@ export async function objectAttributeCompleter(
     select: [{ name: "name" }, { name: "attributeType" }, { name: "tag" }, {
       name: "readOnly",
     }],
-    cacheSecs: 5,
+  }, 5);
-  });
   return allAttributes.map((value) => {
     return {
       name: value.name,

plugs/index/command.ts

@@ -6,7 +6,7 @@ import { isTemplate } from "$sb/lib/cheap_yaml.ts";
 
 export async function reindexCommand() {
   await editor.flashNotification("Performing full page reindex...");
-  await system.invokeFunction("reindexSpace");
+  await system.invokeFunction("index.reindexSpace");
   await editor.flashNotification("Done with page index!");
 }
 

plugs/index/plug_api.ts

@@ -1,5 +1,6 @@
 import { KV, KvQuery, ObjectQuery, ObjectValue } from "$sb/types.ts";
 import { invokeFunction } from "$sb/silverbullet-syscall/system.ts";
+import { ttlCache } from "$sb/lib/memory_cache.ts";
 
 export function indexObjects<T>(
   page: string,
@@ -21,8 +22,13 @@ export function query(
 export function queryObjects<T>(
   tag: string,
   query: ObjectQuery,
+  ttlSecs?: number,
 ): Promise<ObjectValue<T>[]> {
-  return invokeFunction("index.queryObjects", tag, query);
+  return ttlCache(
+    query,
+    () => invokeFunction("index.queryObjects", tag, query),
+    ttlSecs, // no-op when undefined
+  );
 }
 
 export function getObjectByRef<T>(

plugs/index/tags.ts

@@ -73,8 +73,7 @@ export async function tagComplete(completeEvent: CompleteEvent) {
     filter: ["=", ["attr", "parent"], ["string", parent]],
     select: [{ name: "name" }],
     distinct: true,
-    cacheSecs: 5,
+  }, 5);
-  });
 
   if (parent === "page") {
     // Also add template, even though that would otherwise not appear because has "builtin" as a parent

plugs/markdown/markdown_render.test.ts

@@ -9,16 +9,16 @@ import { renderMarkdownToHtml } from "./markdown_render.ts";
 Deno.test("Markdown render", async () => {
   const system = new System<any>("server");
   await system.load(
-    new URL("../../dist_plug_bundle/_plug/editor.plug.js", import.meta.url),
     "editor",
-    0,
+    createSandbox(
-    createSandbox,
+      new URL("../../dist_plug_bundle/_plug/editor.plug.js", import.meta.url),
+    ),
   );
   await system.load(
-    new URL("../../dist_plug_bundle/_plug/tasks.plug.js", import.meta.url),
     "tasks",
-    0,
+    createSandbox(
-    createSandbox,
+      new URL("../../dist_plug_bundle/_plug/tasks.plug.js", import.meta.url),
+    ),
   );
   const lang = buildMarkdown(loadMarkdownExtensions(system));
   const testFile = Deno.readTextFileSync(

plugs/markdown/preview.ts

@@ -11,8 +11,8 @@ export async function updateMarkdownPreview() {
   const text = await editor.getText();
   const mdTree = await markdown.parseMarkdown(text);
   // const cleanMd = await cleanMarkdown(text);
-  const css = await asset.readAsset("assets/preview.css");
+  const css = await asset.readAsset("markdown", "assets/preview.css");
-  const js = await asset.readAsset("assets/preview.js");
+  const js = await asset.readAsset("markdown", "assets/preview.js");
 
   await expandCodeWidgets(mdTree, currentPage);
   const html = renderMarkdownToHtml(mdTree, {

plugs/tasks/complete.ts

@@ -9,9 +9,7 @@ export async function completeTaskState(completeEvent: CompleteEvent) {
   if (!taskMatch) {
     return null;
   }
-  const allStates = await queryObjects<TaskStateObject>("taskstate", {
+  const allStates = await queryObjects<TaskStateObject>("taskstate", {}, 5);
-    cacheSecs: 5,
-  });
   const states = [...new Set(allStates.map((s) => s.state))];
 
   return {

plugs/template/complete.ts

@@ -56,8 +56,7 @@ export async function templateSlashComplete(
       "boolean",
       false,
     ]]],
-    cacheSecs: 5,
+  }, 5);
-  });
   return allTemplates.map((template) => ({
     label: template.trigger!,
     detail: "template",

server/crypto.ts

@@ -17,6 +17,7 @@ export class JWTIssuer {
   async init(authString: string) {
     const [secret] = await this.kv.batchGet([[jwtSecretKey]]);
     if (!secret) {
+      console.log("Generating new JWT secret key");
       return this.generateNewKey();
     } else {
       this.key = await crypto.subtle.importKey(
@@ -34,6 +35,9 @@ export class JWTIssuer {
     ]]);
     const newAuthHash = await this.hashSHA256(authString);
     if (currentAuthHash && currentAuthHash !== newAuthHash) {
+      console.log(
+        "Authentication has changed since last run, so invalidating all existing tokens",
+      );
       // It has, so we need to generate a new key to invalidate all existing tokens
       await this.generateNewKey();
     }

server/http_server.ts

@@ -344,6 +344,16 @@ export class HttpServer {
       }),
     );
 
+    // For when the day comes...
+    // this.app.use("*", async (c, next) => {
+    //   // if (["POST", "PUT", "DELETE"].includes(c.req.method)) {
+    //   const spaceServer = await this.ensureSpaceServer(c.req);
+    //   return runWithSystemLock(spaceServer.system!, async () => {
+    //     await next();
+    //   });
+    //   // }
+    // });
+
     // File list
     this.app.get(
       "/index.json",
@@ -382,10 +392,10 @@ export class HttpServer {
     });
 
     // RPC syscall
-    this.app.post("/.rpc/:plug/:syscall", async (c) => {
+    this.app.post("/.rpc/:plugName/:syscall", async (c) => {
       const req = c.req;
-      const plugName = req.param("plug")!;
       const syscall = req.param("syscall")!;
+      const plugName = req.param("plugName")!;
       const spaceServer = await this.ensureSpaceServer(req);
       const body = await req.json();
       try {
@@ -394,11 +404,11 @@ export class HttpServer {
         }
         const args: string[] = body;
         try {
-          const plug = spaceServer.system!.loadedPlugs.get(plugName);
+          const result = await spaceServer.system!.syscall(
-          if (!plug) {
+            { plug: plugName },
-            throw new Error(`Plug ${plugName} not found`);
+            syscall,
-          }
+            args,
-          const result = await plug.syscall(syscall, args);
+          );
           return c.json({
             result: result,
           });

server/server_system.ts

@@ -35,6 +35,20 @@ import { KvPrimitives } from "../plugos/lib/kv_primitives.ts";
 import { ShellBackend } from "./shell_backend.ts";
 import { ensureSpaceIndex } from "../common/space_index.ts";
 
+// // Important: load this before the actual plugs
+// import {
+//   createSandbox as noSandboxFactory,
+//   runWithSystemLock,
+// } from "../plugos/sandboxes/no_sandbox.ts";
+
+// // Load list of builtin plugs
+// import { plug as plugIndex } from "../dist_plug_bundle/_plug/index.plug.js";
+// import { plug as plugFederation } from "../dist_plug_bundle/_plug/federation.plug.js";
+// import { plug as plugQuery } from "../dist_plug_bundle/_plug/query.plug.js";
+// import { plug as plugSearch } from "../dist_plug_bundle/_plug/search.plug.js";
+// import { plug as plugTasks } from "../dist_plug_bundle/_plug/tasks.plug.js";
+// import { plug as plugTemplate } from "../dist_plug_bundle/_plug/template.plug.js";
+
 const fileListInterval = 30 * 1000; // 30s
 
 const plugNameExtractRegex = /([^/]+)\.plug\.js$/;
@@ -138,28 +152,29 @@ export class ServerSystem {
     );
 
     this.listInterval = setInterval(() => {
+      // runWithSystemLock(this.system, async () => {
+      //   await space.updatePageList();
+      // });
       space.updatePageList().catch(console.error);
     }, fileListInterval);
 
-    eventHook.addLocalListener("file:changed", (path, localChange) => {
+    eventHook.addLocalListener("file:changed", async (path, localChange) => {
-      (async () => {
+      if (!localChange && path.endsWith(".md")) {
-        if (!localChange && path.endsWith(".md")) {
+        const pageName = path.slice(0, -3);
-          const pageName = path.slice(0, -3);
+        const data = await this.spacePrimitives.readFile(path);
-          const data = await this.spacePrimitives.readFile(path);
+        console.log("Outside page change: reindexing", pageName);
-          console.log("Outside page change: reindexing", pageName);
+        // Change made outside of editor, trigger reindex
-          // Change made outside of editor, trigger reindex
+        await eventHook.dispatchEvent("page:index_text", {
-          await eventHook.dispatchEvent("page:index_text", {
+          name: pageName,
-            name: pageName,
+          text: new TextDecoder().decode(data.data),
-            text: new TextDecoder().decode(data.data),
+        });
-          });
+      }
-        }
 
-        if (path.startsWith("_plug/") && path.endsWith(".plug.js")) {
+      if (path.startsWith("_plug/") && path.endsWith(".plug.js")) {
-          console.log("Plug updated, reloading:", path);
+        console.log("Plug updated, reloading:", path);
-          this.system.unload(path);
+        this.system.unload(path);
-          await this.loadPlugFromSpace(path);
+        await this.loadPlugFromSpace(path);
-        }
+      }
-      })().catch(console.error);
     });
 
     // Ensure a valid index
@@ -168,10 +183,19 @@ export class ServerSystem {
       await indexPromise;
     }
 
+    // await runWithSystemLock(this.system, async () => {
     await eventHook.dispatchEvent("system:ready");
+    // });
   }
 
   async loadPlugs() {
+    // await this.system.load("index", noSandboxFactory(plugIndex));
+    // await this.system.load("federation", noSandboxFactory(plugFederation));
+    // await this.system.load("query", noSandboxFactory(plugQuery));
+    // await this.system.load("search", noSandboxFactory(plugSearch));
+    // await this.system.load("tasks", noSandboxFactory(plugTasks));
+    // await this.system.load("template", noSandboxFactory(plugTemplate));
+
     for (const { name } of await this.spacePrimitives.fetchFileList()) {
       if (plugNameExtractRegex.test(name)) {
         await this.loadPlugFromSpace(name);
@@ -183,11 +207,12 @@ export class ServerSystem {
     const { meta, data } = await this.spacePrimitives.readFile(path);
     const plugName = path.match(plugNameExtractRegex)![1];
     return this.system.load(
-      // Base64 encoding this to support `deno compile` mode
-      new URL(base64EncodedDataUrl("application/javascript", data)),
       plugName,
+      createSandbox(
+        // Base64 encoding this to support `deno compile` mode
+        new URL(base64EncodedDataUrl("application/javascript", data)),
+      ),
       meta.lastModified,
-      createSandbox,
     );
   }
 

server/syscalls/shell.ts

@@ -1,4 +1,3 @@
-import { shell } from "$sb/syscalls.ts";
 import { SysCallMapping } from "../../plugos/system.ts";
 import { ShellResponse } from "../../server/rpc.ts";
 import { ShellBackend } from "../shell_backend.ts";

server/syscalls/space.ts

@@ -10,10 +10,7 @@ export function spaceSyscalls(space: Space): SysCallMapping {
     "space.listPages": (): Promise<PageMeta[]> => {
       return space.fetchPageList();
     },
-    "space.readPage": async (
+    "space.readPage": async (_ctx, name: string): Promise<string> => {
-      _ctx,
-      name: string,
-    ): Promise<string> => {
       return (await space.readPage(name)).text;
     },
     "space.getPageMeta": (_ctx, name: string): Promise<PageMeta> => {
@@ -35,10 +32,7 @@ export function spaceSyscalls(space: Space): SysCallMapping {
     "space.listAttachments": async (): Promise<AttachmentMeta[]> => {
       return await space.fetchAttachmentList();
     },
-    "space.readAttachment": async (
+    "space.readAttachment": async (_ctx, name: string): Promise<Uint8Array> => {
-      _ctx,
-      name: string,
-    ): Promise<Uint8Array> => {
       return (await space.readAttachment(name)).data;
     },
     "space.getAttachmentMeta": async (

web/client.ts

@@ -46,11 +46,12 @@ import { DataStoreSpacePrimitives } from "../common/spaces/datastore_space_primi
 import {
   EncryptedSpacePrimitives,
 } from "../common/spaces/encrypted_space_primitives.ts";
-import { LimitedMap } from "../common/limited_map.ts";
+
 import {
   ensureSpaceIndex,
   markFullSpaceIndexComplete,
 } from "../common/space_index.ts";
+import { LimitedMap } from "$sb/lib/limited_map.ts";
 const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/;
 
 const autoSaveInterval = 1000;
@@ -135,7 +136,7 @@ export class Client {
       `${this.dbPrefix}_state`,
     );
     await stateKvPrimitives.init();
-    this.stateDataStore = new DataStore(stateKvPrimitives, true);
+    this.stateDataStore = new DataStore(stateKvPrimitives);
 
     // Setup message queue
     this.mq = new DataStoreMQ(this.stateDataStore);
@@ -316,9 +317,13 @@ export class Client {
 
             // We're going to look up the anchor through a API invocation
             const matchingAnchor = await this.system.system.localSyscall(
-              "index",
               "system.invokeFunction",
-              ["getObjectByRef", pageName, "anchor", `${pageName}$${pos}`],
+              [
+                "index.getObjectByRef",
+                pageName,
+                "anchor",
+                `${pageName}$${pos}`,
+              ],
             );
 
             if (!matchingAnchor) {

web/client_system.ts

@@ -133,10 +133,9 @@ export class ClientSystem {
           console.log("Plug updated, reloading", plugName, "from", path);
           this.system.unload(path);
           const plug = await this.system.load(
-            new URL(`/${path}`, location.href),
             plugName,
+            createSandbox(new URL(`/${path}`, location.href)),
             newHash,
-            createSandbox,
           );
           if ((plug.manifest! as Manifest).syntax) {
             // If there are syntax extensions, rebuild the markdown parser immediately
@@ -201,10 +200,9 @@ export class ClientSystem {
       try {
         const plugName = plugNameExtractRegex.exec(plugMeta.name)![1];
         await this.system.load(
-          new URL(plugMeta.name, location.origin),
           plugName,
+          createSandbox(new URL(plugMeta.name, location.origin)),
           plugMeta.lastModified,
-          createSandbox,
         );
       } catch (e: any) {
         console.error(
@@ -228,14 +226,13 @@ export class ClientSystem {
   }
 
   localSyscall(name: string, args: any[]) {
-    return this.system.localSyscall("editor", name, args);
+    return this.system.localSyscall(name, args);
   }
 
   queryObjects<T>(tag: string, query: Query): Promise<T[]> {
-    return this.system.localSyscall(
+    return this.localSyscall(
-      "index",
       "system.invokeFunction",
-      ["queryObjects", tag, query],
+      ["index.queryObjects", tag, query],
     );
   }
 }

web/cm_plugins/markdown_widget.ts

@@ -132,7 +132,7 @@ export class MarkdownWidget extends WidgetType {
       buttons.filter((button) => !button.widgetTarget).map((button, idx) =>
         `<button data-button="${idx}" title="${button.description}">${button.svg}</button> `
       ).join("")
-    }</div>${html}`;
+    }</div><div class="content">${html}</div>`;
   }
 
   private attachListeners(div: HTMLElement, buttons?: CodeWidgetButton[]) {

web/components/basic_modals.tsx

@@ -19,43 +19,41 @@ export function Prompt({
 }) {
   const [text, setText] = useState(defaultValue || "");
   const returnEl = (
-    <div className="sb-modal-wrapper">
+    <div className="sb-modal-box">
-      <div className="sb-modal-box">
+      <div className="sb-prompt">
-        <div className="sb-prompt">
+        <label>{message}</label>
-          <label>{message}</label>
+        <MiniEditor
-          <MiniEditor
+          text={defaultValue || ""}
-            text={defaultValue || ""}
+          vimMode={vimMode}
-            vimMode={vimMode}
+          vimStartInInsertMode={true}
-            vimStartInInsertMode={true}
+          focus={true}
-            focus={true}
+          darkMode={darkMode}
-            darkMode={darkMode}
+          completer={completer}
-            completer={completer}
+          onEnter={(text) => {
-            onEnter={(text) => {
+            callback(text);
-              callback(text);
+            return true;
-              return true;
+          }}
-            }}
+          onEscape={() => {
-            onEscape={() => {
+            callback();
-              callback();
+          }}
-            }}
+          onChange={(text) => {
-            onChange={(text) => {
+            setText(text);
-              setText(text);
+          }}
-            }}
+        />
-          />
+        <button
-          <button
+          onClick={() => {
-            onClick={() => {
+            callback(text);
-              callback(text);
+          }}
-            }}
+        >
-          >
+          Ok
-            Ok
+        </button>
-          </button>
+        <button
-          <button
+          onClick={() => {
-            onClick={() => {
+            callback();
-              callback();
+          }}
-            }}
+        >
-          >
+          Cancel
-            Cancel
+        </button>
-          </button>
-        </div>
       </div>
     </div>
   );

web/components/filter.tsx

@@ -95,115 +95,115 @@ export function FilterList({
   }, []);
 
   const returnEl = (
-    <div className="sb-modal-wrapper">
+    <div className="sb-modal-box">
-      <div className="sb-modal-box">
+      <div
-        <div
+        className="sb-header"
-          className="sb-header"
+        onClick={(e) => {
-          onClick={(e) => {
+          // Allow tapping/clicking the header without closing it
-            // Allow tapping/clicking the header without closing it
+          e.stopPropagation();
-            e.stopPropagation();
+        }}
+      >
+        <label>{label}</label>
+        <MiniEditor
+          text={text}
+          vimMode={vimMode}
+          vimStartInInsertMode={true}
+          focus={true}
+          darkMode={darkMode}
+          completer={completer}
+          placeholderText={placeholder}
+          onEnter={(_newText, shiftDown) => {
+            onSelect(
+              shiftDown ? { name: text } : matchingOptions[selectedOption],
+            );
+            return true;
           }}
-        >
+          onEscape={() => {
-          <label>{label}</label>
+            onSelect(undefined);
-          <MiniEditor
+          }}
-            text={text}
+          onChange={(text) => {
-            vimMode={vimMode}
+            setText(text);
-            vimStartInInsertMode={true}
+          }}
-            focus={true}
+          onKeyUp={(view, e) => {
-            darkMode={darkMode}
+            // This event is triggered after the key has been processed by CM already
-            completer={completer}
+            if (onKeyPress) {
-            placeholderText={placeholder}
+              onKeyPress(e.key, view.state.sliceDoc());
-            onEnter={(_newText, shiftDown) => {
+            }
-              onSelect(
+            return false;
-                shiftDown ? { name: text } : matchingOptions[selectedOption],
+          }}
-              );
+          onKeyDown={(view, e) => {
-              return true;
+            switch (e.key) {
-            }}
+              case "ArrowUp":
-            onEscape={() => {
+                setSelectionOption(Math.max(0, selectedOption - 1));
-              onSelect(undefined);
+                return true;
-            }}
+              case "ArrowDown":
-            onChange={(text) => {
+                setSelectionOption(
-              setText(text);
+                  Math.min(matchingOptions.length - 1, selectedOption + 1),
-            }}
+                );
-            onKeyUp={(view, e) => {
+                return true;
-              // This event is triggered after the key has been processed by CM already
+              case "PageUp":
-              if (onKeyPress) {
+                setSelectionOption(Math.max(0, selectedOption - 5));
-                onKeyPress(e.key, view.state.sliceDoc());
+                return true;
-              }
+              case "PageDown":
-              return false;
+                setSelectionOption(Math.max(0, selectedOption + 5));
-            }}
+                return true;
-            onKeyDown={(view, e) => {
+              case "Home":
-              switch (e.key) {
+                setSelectionOption(0);
-                case "ArrowUp":
+                return true;
-                  setSelectionOption(Math.max(0, selectedOption - 1));
+              case "End":
+                setSelectionOption(matchingOptions.length - 1);
+                return true;
+              case " ": {
+                const text = view.state.sliceDoc();
+                if (completePrefix && text === "") {
+                  setText(completePrefix);
+                  // updateFilter(completePrefix);
                   return true;
-                case "ArrowDown":
-                  setSelectionOption(
-                    Math.min(matchingOptions.length - 1, selectedOption + 1),
-                  );
-                  return true;
-                case "PageUp":
-                  setSelectionOption(Math.max(0, selectedOption - 5));
-                  return true;
-                case "PageDown":
-                  setSelectionOption(Math.max(0, selectedOption + 5));
-                  return true;
-                case "Home":
-                  setSelectionOption(0);
-                  return true;
-                case "End":
-                  setSelectionOption(matchingOptions.length - 1);
-                  return true;
-                case " ": {
-                  const text = view.state.sliceDoc();
-                  if (completePrefix && text === "") {
-                    setText(completePrefix);
-                    // updateFilter(completePrefix);
-                    return true;
-                  }
-                  break;
                 }
+                break;
               }
-              return false;
+            }
-            }}
+            return false;
-          />
+          }}
-        </div>
+        />
-        <div
+      </div>
-          className="sb-help-text"
+      <div
-          dangerouslySetInnerHTML={{ __html: helpText }}
+        className="sb-help-text"
-        >
+        dangerouslySetInnerHTML={{ __html: helpText }}
-        </div>
+      >
-        <div className="sb-result-list">
+      </div>
-          {matchingOptions && matchingOptions.length > 0
+      <div className="sb-result-list">
-            ? matchingOptions.map((option, idx) => (
+        {matchingOptions && matchingOptions.length > 0
-              <div
+          ? matchingOptions.map((option, idx) => (
-                key={"" + idx}
+            <div
-                ref={selectedOption === idx ? selectedElementRef : undefined}
+              key={"" + idx}
-                className={selectedOption === idx
+              ref={selectedOption === idx ? selectedElementRef : undefined}
-                  ? "sb-selected-option"
+              className={selectedOption === idx
-                  : "sb-option"}
+                ? "sb-selected-option"
-                onMouseOver={(e) => {
+                : "sb-option"}
+              onMouseMove={(e) => {
+                if (selectedOption !== idx) {
                   setSelectionOption(idx);
-                }}
+                }
-                onClick={(e) => {
+              }}
-                  e.stopPropagation();
+              onClick={(e) => {
-                  onSelect(option);
+                e.stopPropagation();
-                }}
+                onSelect(option);
-              >
+              }}
-                {Icon && (
+            >
-                  <span className="sb-icon">
+              {Icon && (
-                    <Icon width={16} height={16} />
+                <span className="sb-icon">
-                  </span>
+                  <Icon width={16} height={16} />
-                )}
-                <span className="sb-name">
-                  {option.name}
                 </span>
-                {option.hint && <span className="sb-hint">{option.hint}</span>}
+              )}
-                <div className="sb-description">{option.description}</div>
+              <span className="sb-name">
-              </div>
+                {option.name}
-            ))
+              </span>
-            : null}
+              {option.hint && <span className="sb-hint">{option.hint}</span>}
-        </div>
+              <div className="sb-description">{option.description}</div>
+            </div>
+          ))
+          : null}
       </div>
     </div>
   );

web/components/fuse_search.ts

@@ -28,17 +28,18 @@ export const fuzzySearchAndSort = (
       weight: 0.3,
     }, {
       name: "baseName",
-      weight: 0.7,
+      weight: 1,
     }, {
       name: "displayName",
-      weight: 0.3,
+      weight: 0.7,
     }, {
       name: "aliases",
-      weight: 0.7,
+      weight: 0.5,
     }],
     includeScore: true,
     shouldSort: true,
     isCaseSensitive: false,
+    ignoreLocation: true,
     threshold: 0.6,
     sortFn: (a, b): number => {
       if (a.score === b.score) {

web/components/top_bar.tsx

@@ -142,8 +142,8 @@ export function TopBar({
                   <button
                     onClick={(e) => {
                       e.preventDefault();
-                      actionButton.callback();
                       e.stopPropagation();
+                      actionButton.callback();
                     }}
                     title={actionButton.description}
                     className={actionButton.class}

web/space.ts

@@ -4,9 +4,6 @@ import { safeRun } from "../common/util.ts";
 
 import { AttachmentMeta, FileMeta, PageMeta } from "$sb/types.ts";
 import { EventHook } from "../plugos/hooks/event.ts";
-import { throttle } from "$sb/lib/async.ts";
-import { DataStore } from "../plugos/lib/datastore.ts";
-import { LimitedMap } from "../common/limited_map.ts";
 
 const pageWatchInterval = 5000;
 

web/styles/editor.scss

@@ -428,12 +428,12 @@
     margin-top: 10px;
   }
 
-  .sb-markdown-top-widget:has(*) {
+  .sb-markdown-top-widget:has(*) .content {
     max-height: 500px;
   }
 
   @media screen and (max-height: 1000px) {
-    .sb-markdown-top-widget:has(*) {
+    .sb-markdown-top-widget:has(*) .content {
       max-height: 300px;
     }
   }
@@ -441,13 +441,16 @@
   .sb-markdown-widget,
   .sb-markdown-top-widget:has(*),
   .sb-markdown-bottom-widget:has(*) {
-    overflow-y: auto;
     border: 1px solid var(--editor-widget-background-color);
     border-radius: 5px;
     white-space: normal;
     position: relative;
     min-height: 48px;
 
+    .content {
+      overflow-y: auto;
+    }
+
     ul,
     ol {
       margin-top: 0;

web/styles/modals.scss

@@ -1,20 +1,16 @@
-.sb-modal-wrapper {
-  position: absolute;
-  margin: auto;
-  max-width: 500px;
-  height: 600px;
-  left: 0;
-  right: 0;
-  top: 0;
-  bottom: 0;
-  max-height: 290px;
-  z-index: 100;
-}
-
 .sb-modal-box {
+  position: absolute;
+  // At the toppest of the toppest
+  z-index: 1000;
+
+  top: 60px;
+  left: 50%;
+  transform: translateX(-50%);
+  width: 700px;
+  max-width: 90%;
+
   border-radius: 8px;
   overflow: hidden;
-  margin: 10px;
 
   .cm-content {
     padding: 0;
@@ -53,7 +49,7 @@
   }
 
   .sb-result-list {
-    max-height: 216px;
+    max-height: 250px;
     overflow-y: scroll;
 
     .sb-icon {

web/syscalls/clientStore.ts

@@ -7,14 +7,14 @@ export function clientStoreSyscalls(
   prefix: KvKey = ["client"],
 ): SysCallMapping {
   return {
-    "clientStore.get": (ctx, key: string): Promise<any> => {
+    "clientStore.get": (_ctx, key: string): Promise<any> => {
-      return ds.get([...prefix, ctx.plug!.name!, key]);
+      return ds.get([...prefix, key]);
     },
-    "clientStore.set": (ctx, key: string, val: any): Promise<void> => {
+    "clientStore.set": (_ctx, key: string, val: any): Promise<void> => {
-      return ds.set([...prefix, ctx.plug!.name!, key], val);
+      return ds.set([...prefix, key], val);
     },
-    "clientStore.delete": (ctx, key: string): Promise<void> => {
+    "clientStore.delete": (_ctx, key: string): Promise<void> => {
-      return ds.delete([...prefix, ctx.plug!.name!, key]);
+      return ds.delete([...prefix, key]);
     },
   };
 }

web/syscalls/datastore.proxy.ts

@@ -1,48 +1,17 @@
 import { KvQuery } from "$sb/types.ts";
-import { LimitedMap } from "../../common/limited_map.ts";
+import { LimitedMap } from "../../plug-api/lib/limited_map.ts";
 import type { SysCallMapping } from "../../plugos/system.ts";
 import type { Client } from "../client.ts";
 import { proxySyscall, proxySyscalls } from "./util.ts";
 
 export function dataStoreProxySyscalls(client: Client): SysCallMapping {
-  const syscalls = proxySyscalls(client, [
+  return proxySyscalls(client, [
     "datastore.delete",
     "datastore.set",
     "datastore.batchSet",
     "datastore.batchDelete",
     "datastore.batchGet",
+    "datastore.query",
     "datastore.get",
   ]);
-  // Add a cache for datastore.query
-  const queryCache = new LimitedMap<any>(5);
-  syscalls["datastore.query"] = async (ctx, query: KvQuery) => {
-    let cacheKey: string | undefined;
-    const cacheSecs = query.cacheSecs;
-    // Should we do caching?
-    if (cacheSecs) {
-      // Remove the cacheSecs from the query
-      query = { ...query, cacheSecs: undefined };
-      cacheKey = JSON.stringify(query);
-      const cachedResult = queryCache.get(cacheKey);
-      if (cachedResult) {
-        // Let's use the cached result
-        return cachedResult;
-      }
-    }
-
-    const result = await proxySyscall(
-      ctx,
-      client.httpSpacePrimitives,
-      "datastore.query",
-      [
-        query,
-      ],
-    );
-    if (cacheKey) {
-      // Store in the cache
-      queryCache.set(cacheKey, result, cacheSecs! * 1000);
-    }
-    return result;
-  };
-  return syscalls;
 }

web/syscalls/editor.ts

@@ -220,10 +220,7 @@ export function editorSyscalls(editor: Client): SysCallMapping {
     ): Promise<string | undefined> => {
       return editor.prompt(message, defaultValue);
     },
-    "editor.confirm": (
+    "editor.confirm": (_ctx, message: string): Promise<boolean> => {
-      _ctx,
-      message: string,
-    ): Promise<boolean> => {
       return editor.confirm(message);
     },
     "editor.getUiOption": (_ctx, key: string): any => {

web/syscalls/space.ts

@@ -7,10 +7,7 @@ export function spaceSyscalls(editor: Client): SysCallMapping {
     "space.listPages": (): Promise<PageMeta[]> => {
       return editor.space.fetchPageList();
     },
-    "space.readPage": async (
+    "space.readPage": async (_ctx, name: string): Promise<string> => {
-      _ctx,
-      name: string,
-    ): Promise<string> => {
       return (await editor.space.readPage(name)).text;
     },
     "space.getPageMeta": (_ctx, name: string): Promise<PageMeta> => {
@@ -39,10 +36,7 @@ export function spaceSyscalls(editor: Client): SysCallMapping {
     "space.listAttachments": async (): Promise<AttachmentMeta[]> => {
       return await editor.space.fetchAttachmentList();
     },
-    "space.readAttachment": async (
+    "space.readAttachment": async (_ctx, name: string): Promise<Uint8Array> => {
-      _ctx,
-      name: string,
-    ): Promise<Uint8Array> => {
       return (await editor.space.readAttachment(name)).data;
     },
     "space.getAttachmentMeta": async (

web/syscalls/system.ts

@@ -1,4 +1,3 @@
-import type { Plug } from "../../plugos/plug.ts";
 import { SysCallMapping, System } from "../../plugos/system.ts";
 import type { Client } from "../client.ts";
 import { CommandDef } from "../hooks/command.ts";
@@ -11,30 +10,20 @@ export function systemSyscalls(
   const api: SysCallMapping = {
     "system.invokeFunction": (
       ctx,
-      name: string,
+      fullName: string, // plug.function
       ...args: any[]
     ) => {
-      if (name === "server" || name === "client") {
+      const [plugName, functionName] = fullName.split(".");
-        // Backwards compatibility mode (previously there was an 'env' argument)
+      if (!plugName || !functionName) {
-        name = args[0];
+        throw Error(`Invalid function name ${fullName}`);
-        args = args.slice(1);
       }
-
+      const plug = system.loadedPlugs.get(plugName);
-      let plug: Plug<any> | undefined = ctx.plug;
+      if (!plug) {
-      const fullName = name;
+        throw Error(`Plug ${plugName} not found`);
-      // console.log("Invoking function", fullName, "on plug", plug);
-      if (name.includes(".")) {
-        // plug name in the name
-        const [plugName, functionName] = name.split(".");
-        plug = system.loadedPlugs.get(plugName);
-        if (!plug) {
-          throw Error(`Plug ${plugName} not found`);
-        }
-        name = functionName;
       }
-      const functionDef = plug?.manifest!.functions[name];
+      const functionDef = plug.manifest!.functions[functionName];
       if (!functionDef) {
-        throw Error(`Function ${name} not found`);
+        throw Error(`Function ${functionName} not found`);
       }
       if (
         client && functionDef.env && system.env &&
@@ -48,7 +37,7 @@ export function systemSyscalls(
           [fullName, ...args],
         );
       }
-      return plug.invoke(name, args);
+      return plug.invoke(functionName, args);
     },
     "system.invokeCommand": (_ctx, name: string, args?: string[]) => {
       if (!client) {

web/syscalls/util.ts

@@ -18,8 +18,11 @@ export async function proxySyscall(
   name: string,
   args: any[],
 ): Promise<any> {
+  if (!ctx.plug) {
+    throw new Error(`Cannot proxy ${name} syscall without plug context`);
+  }
   const resp = await httpSpacePrimitives.authenticatedFetch(
-    `${httpSpacePrimitives.url}/.rpc/${ctx.plug.name}/${name}`,
+    `${httpSpacePrimitives.url}/.rpc/${ctx.plug}/${name}`,
     {
       method: "POST",
       body: JSON.stringify(args),