PlugOS refactor and other tweaks (#631)

* Prep for in-process plug loading (e.g. for CF workers, Deno Deploy)
* Prototype of fixed in-process loading plugs
* Fix: buttons not to scroll with content
* Better positioning of modal especially on mobile
* Move query caching outside query
* Fix annoying mouse behavior when filter box appears
* Page navigator search tweaks

server/crypto.ts

@@ -17,6 +17,7 @@ export class JWTIssuer {
   async init(authString: string) {
     const [secret] = await this.kv.batchGet([[jwtSecretKey]]);
     if (!secret) {
+      console.log("Generating new JWT secret key");
       return this.generateNewKey();
     } else {
       this.key = await crypto.subtle.importKey(
@@ -34,6 +35,9 @@ export class JWTIssuer {
     ]]);
     const newAuthHash = await this.hashSHA256(authString);
     if (currentAuthHash && currentAuthHash !== newAuthHash) {
+      console.log(
+        "Authentication has changed since last run, so invalidating all existing tokens",
+      );
       // It has, so we need to generate a new key to invalidate all existing tokens
       await this.generateNewKey();
     }

server/http_server.ts

@@ -344,6 +344,16 @@ export class HttpServer {
       }),
     );
 
+    // For when the day comes...
+    // this.app.use("*", async (c, next) => {
+    //   // if (["POST", "PUT", "DELETE"].includes(c.req.method)) {
+    //   const spaceServer = await this.ensureSpaceServer(c.req);
+    //   return runWithSystemLock(spaceServer.system!, async () => {
+    //     await next();
+    //   });
+    //   // }
+    // });
+
     // File list
     this.app.get(
       "/index.json",
@@ -382,10 +392,10 @@ export class HttpServer {
     });
 
     // RPC syscall
-    this.app.post("/.rpc/:plug/:syscall", async (c) => {
+    this.app.post("/.rpc/:plugName/:syscall", async (c) => {
       const req = c.req;
-      const plugName = req.param("plug")!;
       const syscall = req.param("syscall")!;
+      const plugName = req.param("plugName")!;
       const spaceServer = await this.ensureSpaceServer(req);
       const body = await req.json();
       try {
@@ -394,11 +404,11 @@ export class HttpServer {
         }
         const args: string[] = body;
         try {
-          const plug = spaceServer.system!.loadedPlugs.get(plugName);
-          if (!plug) {
-            throw new Error(`Plug ${plugName} not found`);
-          }
-          const result = await plug.syscall(syscall, args);
+          const result = await spaceServer.system!.syscall(
+            { plug: plugName },
+            syscall,
+            args,
+          );
           return c.json({
             result: result,
           });

server/server_system.ts

@@ -35,6 +35,20 @@ import { KvPrimitives } from "../plugos/lib/kv_primitives.ts";
 import { ShellBackend } from "./shell_backend.ts";
 import { ensureSpaceIndex } from "../common/space_index.ts";
 
+// // Important: load this before the actual plugs
+// import {
+//   createSandbox as noSandboxFactory,
+//   runWithSystemLock,
+// } from "../plugos/sandboxes/no_sandbox.ts";
+
+// // Load list of builtin plugs
+// import { plug as plugIndex } from "../dist_plug_bundle/_plug/index.plug.js";
+// import { plug as plugFederation } from "../dist_plug_bundle/_plug/federation.plug.js";
+// import { plug as plugQuery } from "../dist_plug_bundle/_plug/query.plug.js";
+// import { plug as plugSearch } from "../dist_plug_bundle/_plug/search.plug.js";
+// import { plug as plugTasks } from "../dist_plug_bundle/_plug/tasks.plug.js";
+// import { plug as plugTemplate } from "../dist_plug_bundle/_plug/template.plug.js";
+
 const fileListInterval = 30 * 1000; // 30s
 
 const plugNameExtractRegex = /([^/]+)\.plug\.js$/;
@@ -138,28 +152,29 @@ export class ServerSystem {
     );
 
     this.listInterval = setInterval(() => {
+      // runWithSystemLock(this.system, async () => {
+      //   await space.updatePageList();
+      // });
       space.updatePageList().catch(console.error);
     }, fileListInterval);
 
-    eventHook.addLocalListener("file:changed", (path, localChange) => {
-      (async () => {
-        if (!localChange && path.endsWith(".md")) {
-          const pageName = path.slice(0, -3);
-          const data = await this.spacePrimitives.readFile(path);
-          console.log("Outside page change: reindexing", pageName);
-          // Change made outside of editor, trigger reindex
-          await eventHook.dispatchEvent("page:index_text", {
-            name: pageName,
-            text: new TextDecoder().decode(data.data),
-          });
-        }
+    eventHook.addLocalListener("file:changed", async (path, localChange) => {
+      if (!localChange && path.endsWith(".md")) {
+        const pageName = path.slice(0, -3);
+        const data = await this.spacePrimitives.readFile(path);
+        console.log("Outside page change: reindexing", pageName);
+        // Change made outside of editor, trigger reindex
+        await eventHook.dispatchEvent("page:index_text", {
+          name: pageName,
+          text: new TextDecoder().decode(data.data),
+        });
+      }
 
-        if (path.startsWith("_plug/") && path.endsWith(".plug.js")) {
-          console.log("Plug updated, reloading:", path);
-          this.system.unload(path);
-          await this.loadPlugFromSpace(path);
-        }
-      })().catch(console.error);
+      if (path.startsWith("_plug/") && path.endsWith(".plug.js")) {
+        console.log("Plug updated, reloading:", path);
+        this.system.unload(path);
+        await this.loadPlugFromSpace(path);
+      }
     });
 
     // Ensure a valid index
@@ -168,10 +183,19 @@ export class ServerSystem {
       await indexPromise;
     }
 
+    // await runWithSystemLock(this.system, async () => {
     await eventHook.dispatchEvent("system:ready");
+    // });
   }
 
   async loadPlugs() {
+    // await this.system.load("index", noSandboxFactory(plugIndex));
+    // await this.system.load("federation", noSandboxFactory(plugFederation));
+    // await this.system.load("query", noSandboxFactory(plugQuery));
+    // await this.system.load("search", noSandboxFactory(plugSearch));
+    // await this.system.load("tasks", noSandboxFactory(plugTasks));
+    // await this.system.load("template", noSandboxFactory(plugTemplate));
+
     for (const { name } of await this.spacePrimitives.fetchFileList()) {
       if (plugNameExtractRegex.test(name)) {
         await this.loadPlugFromSpace(name);
@@ -183,11 +207,12 @@ export class ServerSystem {
     const { meta, data } = await this.spacePrimitives.readFile(path);
     const plugName = path.match(plugNameExtractRegex)![1];
     return this.system.load(
-      // Base64 encoding this to support `deno compile` mode
-      new URL(base64EncodedDataUrl("application/javascript", data)),
       plugName,
+      createSandbox(
+        // Base64 encoding this to support `deno compile` mode
+        new URL(base64EncodedDataUrl("application/javascript", data)),
+      ),
       meta.lastModified,
-      createSandbox,
     );
   }
 

server/syscalls/shell.ts

@@ -1,4 +1,3 @@
-import { shell } from "$sb/syscalls.ts";
 import { SysCallMapping } from "../../plugos/system.ts";
 import { ShellResponse } from "../../server/rpc.ts";
 import { ShellBackend } from "../shell_backend.ts";

server/syscalls/space.ts

@@ -10,10 +10,7 @@ export function spaceSyscalls(space: Space): SysCallMapping {
     "space.listPages": (): Promise<PageMeta[]> => {
       return space.fetchPageList();
     },
-    "space.readPage": async (
-      _ctx,
-      name: string,
-    ): Promise<string> => {
+    "space.readPage": async (_ctx, name: string): Promise<string> => {
       return (await space.readPage(name)).text;
     },
     "space.getPageMeta": (_ctx, name: string): Promise<PageMeta> => {
@@ -35,10 +32,7 @@ export function spaceSyscalls(space: Space): SysCallMapping {
     "space.listAttachments": async (): Promise<AttachmentMeta[]> => {
       return await space.fetchAttachmentList();
     },
-    "space.readAttachment": async (
-      _ctx,
-      name: string,
-    ): Promise<Uint8Array> => {
+    "space.readAttachment": async (_ctx, name: string): Promise<Uint8Array> => {
       return (await space.readAttachment(name)).data;
     },
     "space.getAttachmentMeta": async (