Plug sandbox rework

plugos/compile.ts

@@ -4,9 +4,9 @@ import { bundleAssets } from "./asset_bundle/builder.ts";
 import { Manifest } from "./types.ts";
 import { version } from "../version.ts";
 
-// const workerRuntimeUrl = new URL("./worker_runtime.ts", import.meta.url);
+const workerRuntimeUrl = new URL("./worker_runtime.ts", import.meta.url);
-const workerRuntimeUrl =
+// const workerRuntimeUrl =
-  `https://deno.land/x/silverbullet@${version}/plugos/worker_runtime.ts`;
+//   `https://deno.land/x/silverbullet@${version}/plugos/worker_runtime.ts`;
 
 export type CompileOptions = {
   debug?: boolean;
@@ -64,7 +64,7 @@ ${
   }
 
 // Function mapping
-export const functionMapping = {
+const functionMapping = {
 ${
     Object.entries(manifest.functions).map(([funcName, def]) => {
       if (!def.path) {
@@ -75,8 +75,11 @@ ${
   }
 };
 
+// Manifest
 const manifest = ${JSON.stringify(manifest, null, 2)};
 
+export const plug = {manifest, functionMapping};
+
 setupMessageListener(functionMapping, manifest);
 `;
 
@@ -89,7 +92,7 @@ setupMessageListener(functionMapping, manifest);
   const result = await esbuild.build({
     entryPoints: [path.basename(inFile)],
     bundle: true,
-    format: "iife",
+    format: "esm",
     globalName: "mod",
     platform: "browser",
     sourcemap: options.debug ? "linked" : false,

plugos/environments/webworker_sandbox.ts

@@ -1,6 +0,0 @@
-import { Sandbox } from "../sandbox.ts";
-import type { Plug } from "../plug.ts";
-
-export function createSandbox<HookT>(plug: Plug<HookT>): Sandbox<HookT> {
-  return new Sandbox(plug);
-}

plugos/hooks/endpoint.test.ts

@@ -1,4 +1,4 @@
-import { createSandbox } from "../environments/deno_sandbox.ts";
+import { createSandbox } from "../sandboxes/deno_worker_sandbox.ts";
 import { EndpointHook, EndpointHookT } from "./endpoint.ts";
 import { System } from "../system.ts";
 

plugos/manifest_cache.ts

@@ -37,15 +37,16 @@ export class InMemoryManifestCache<T> implements ManifestCache<T> {
   }>();
 
   async getManifest(plug: Plug<T>, hash: number): Promise<Manifest<T>> {
-    const cached = this.cache.get(plug.workerUrl.href);
+    const cached = this.cache.get(plug.name);
     if (cached && cached.hash === hash) {
       // console.log("Using memory cached manifest for", plug.name);
       return cached.manifest;
     }
     await plug.sandbox.init();
     const manifest = plug.sandbox.manifest!;
+
     // Deliverately removing the assets from the manifest to preserve space, will be re-added upon load of actual worker
-    this.cache.set(plug.name!, {
+    this.cache.set(plug.name, {
       manifest: { ...manifest, assets: undefined },
       hash,
     });

plugos/plug.ts

@@ -1,7 +1,7 @@
 import { Manifest } from "./types.ts";
-import { Sandbox } from "./sandbox.ts";
 import { System } from "./system.ts";
 import { AssetBundle } from "./asset_bundle/bundle.ts";
+import { Sandbox, SandboxFactory } from "./sandboxes/sandbox.ts";
 
 export class Plug<HookT> {
   readonly runtimeEnv?: string;
@@ -21,10 +21,10 @@ export class Plug<HookT> {
 
   constructor(
     private system: System<HookT>,
-    public workerUrl: URL,
+    public workerUrl: URL | undefined,
     readonly name: string,
     private hash: number,
-    private sandboxFactory: (plug: Plug<HookT>) => Sandbox<HookT>,
+    private sandboxFactory: SandboxFactory<HookT>,
   ) {
     this.runtimeEnv = system.env;
 

plugos/runtime.test.ts

@@ -1,4 +1,4 @@
-import { createSandbox } from "./environments/deno_sandbox.ts";
+import { createSandbox } from "./sandboxes/deno_worker_sandbox.ts";
 import { System } from "./system.ts";
 import { assertEquals } from "../test_deps.ts";
 import { compileManifest } from "./compile.ts";
@@ -8,6 +8,7 @@ Deno.test("Run a deno sandbox", async () => {
   const system = new System("server");
   system.registerSyscalls([], {
     addNumbers: (_ctx, a, b) => {
+      console.log("This is the context", _ctx.plug.name);
       return a + b;
     },
     failingSyscall: () => {
@@ -39,9 +40,24 @@ Deno.test("Run a deno sandbox", async () => {
     createSandbox,
   );
 
-  console.log("Plug", plug.manifest);
+  assertEquals({
+    addedNumbers: 3,
+    yamlMessage: "hello: world\n",
+  }, await plug.invoke("boot", []));
 
-  assertEquals("hello", await plug.invoke("boot", []));
+  await system.unloadAll();
+
+  // Now load directly from module
+  const { plug: plugExport } = await import(
+    `file://${workerPath}`
+  );
+
+  const plug2 = await system.loadNoSandbox("test", plugExport);
+
+  assertEquals({
+    addedNumbers: 3,
+    yamlMessage: "hello: world\n",
+  }, await plug2.invoke("boot", []));
 
   await system.unloadAll();
 

plugos/sandboxes/deno_worker_sandbox.ts

@@ -1,9 +1,10 @@
-import { Sandbox } from "../sandbox.ts";
+import { WorkerSandbox } from "./worker_sandbox.ts";
 import { Plug } from "../plug.ts";
+import { Sandbox } from "./sandbox.ts";
 
 // Uses Deno's permissions to lock the worker down significantly
 export function createSandbox<HookT>(plug: Plug<HookT>): Sandbox<HookT> {
-  return new Sandbox(plug, {
+  return new WorkerSandbox(plug, {
     deno: {
       permissions: {
         // Allow network access

plugos/sandboxes/no_sandbox.ts

@@ -0,0 +1,68 @@
+import { PromiseQueue } from "$sb/lib/async.ts";
+import { Plug } from "../plug.ts";
+import { Sandbox } from "./sandbox.ts";
+import { Manifest } from "../types.ts";
+
+// We need to hard inject the syscall function into the global scope
+declare global {
+  interface globalThis {
+    syscall(name: string, ...args: any[]): Promise<any>;
+  }
+}
+
+export type PlugExport<HookT> = {
+  manifest: Manifest<HookT>;
+  functionMapping: Record<string, Function>;
+};
+
+const functionQueue = new PromiseQueue();
+
+let activePlug: Plug<any> | undefined;
+
+// @ts-ignore: globalThis
+globalThis.syscall = (name: string, ...args: any[]): Promise<any> => {
+  if (!activePlug) {
+    throw new Error("No active plug");
+  }
+  console.log("Calling syscall", name, args);
+  return activePlug.syscall(name, args);
+};
+
+export class NoSandbox<HookT> implements Sandbox<HookT> {
+  manifest?: Manifest<HookT> | undefined;
+  constructor(
+    private plug: Plug<HookT>,
+    private plugExport: PlugExport<HookT>,
+  ) {
+    this.manifest = plugExport.manifest;
+    plug.manifest = this.manifest;
+  }
+
+  init(): Promise<void> {
+    return Promise.resolve();
+  }
+
+  invoke(name: string, args: any[]): Promise<any> {
+    activePlug = this.plug;
+    return functionQueue.runInQueue(async () => {
+      try {
+        const fn = this.plugExport.functionMapping[name];
+        if (!fn) {
+          throw new Error(`Function not loaded: ${name}`);
+        }
+        return await fn(...args);
+      } finally {
+        activePlug = undefined;
+      }
+    });
+  }
+
+  stop() {
+  }
+}
+
+export function noSandboxFactory<HookT>(
+  plugExport: PlugExport<HookT>,
+): (plug: Plug<HookT>) => Sandbox<HookT> {
+  return (plug: Plug<HookT>) => new NoSandbox(plug, plugExport);
+}

plugos/sandboxes/sandbox.ts

@@ -0,0 +1,11 @@
+import { Plug } from "../plug.ts";
+import { Manifest } from "../types.ts";
+
+export type SandboxFactory<HookT> = (plug: Plug<HookT>) => Sandbox<HookT>;
+
+export interface Sandbox<HookT> {
+  manifest?: Manifest<HookT>;
+  init(): Promise<void>;
+  invoke(name: string, args: any[]): Promise<any>;
+  stop(): void;
+}

plugos/sandboxes/web_worker_sandbox.ts

@@ -0,0 +1,7 @@
+import { WorkerSandbox } from "./worker_sandbox.ts";
+import type { Plug } from "../plug.ts";
+import { Sandbox } from "./sandbox.ts";
+
+export function createSandbox<HookT>(plug: Plug<HookT>): Sandbox<HookT> {
+  return new WorkerSandbox(plug);
+}

plugos/sandboxes/worker_sandbox.ts

@@ -1,15 +1,14 @@
-import { Manifest } from "./types.ts";
+import { Manifest } from "../types.ts";
-import { ControllerMessage, WorkerMessage } from "./protocol.ts";
+import { ControllerMessage, WorkerMessage } from "../protocol.ts";
-import { Plug } from "./plug.ts";
+import { Plug } from "../plug.ts";
-import { AssetBundle, AssetJson } from "./asset_bundle/bundle.ts";
+import { AssetBundle, AssetJson } from "../asset_bundle/bundle.ts";
-
+import { Sandbox } from "./sandbox.ts";
-export type SandboxFactory<HookT> = (plug: Plug<HookT>) => Sandbox<HookT>;
 
 /**
  * Represents a "safe" execution environment for plug code
  * Effectively this wraps a web worker, the reason to have this split from Plugs is to allow plugs to manage multiple sandboxes, e.g. for performance in the future
  */
-export class Sandbox<HookT> {
+export class WorkerSandbox<HookT> implements Sandbox<HookT> {
   private worker?: Worker;
   private reqId = 0;
   private outstandingInvocations = new Map<
@@ -36,7 +35,7 @@ export class Sandbox<HookT> {
       console.warn("Double init of sandbox, ignoring");
       return Promise.resolve();
     }
-    this.worker = new Worker(this.plug.workerUrl, {
+    this.worker = new Worker(this.plug.workerUrl!, {
       ...this.workerOptions,
       type: "module",
     });

plugos/system.ts

@@ -1,8 +1,9 @@
 import { Hook } from "./types.ts";
 import { EventEmitter } from "./event.ts";
-import type { SandboxFactory } from "./sandbox.ts";
+import type { SandboxFactory } from "./sandboxes/sandbox.ts";
 import { Plug } from "./plug.ts";
 import { InMemoryManifestCache, ManifestCache } from "./manifest_cache.ts";
+import { noSandboxFactory, PlugExport } from "./sandboxes/no_sandbox.ts";
 
 export interface SysCallMapping {
   [key: string]: (ctx: SyscallContext, ...args: any) => Promise<any> | any;
@@ -138,6 +139,44 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
     return plug;
   }
 
+  /**
+   * Loads a plug without a sandbox, which means it will run in the same context as the caller
+   * @param name
+   * @param plugExport extracted via e.g. `import { plug } from "./some.plug.js`
+   * @returns Plug instance
+   */
+  async loadNoSandbox(
+    name: string,
+    plugExport: PlugExport<HookT>,
+  ): Promise<Plug<HookT>> {
+    const plug = new Plug(
+      this,
+      undefined,
+      name,
+      -1,
+      noSandboxFactory(plugExport),
+    );
+
+    const manifest = plugExport.manifest;
+
+    // Validate the manifest
+    let errors: string[] = [];
+    for (const feature of this.enabledHooks) {
+      errors = [...errors, ...feature.validateManifest(plug.manifest!)];
+    }
+    if (errors.length > 0) {
+      throw new Error(`Invalid manifest: ${errors.join(", ")}`);
+    }
+    if (this.plugs.has(manifest.name)) {
+      this.unload(manifest.name);
+    }
+    console.log("Activated plug without sandbox", manifest.name);
+    this.plugs.set(manifest.name, plug);
+
+    await this.emit("plugLoaded", plug);
+    return plug;
+  }
+
   unload(name: string) {
     const plug = this.plugs.get(name);
     if (!plug) {

plugos/test_func.test.ts

@@ -1,10 +1,12 @@
 import * as YAML from "https://deno.land/std@0.184.0/yaml/mod.ts";
 import { EndpointRequest, EndpointResponse } from "./hooks/endpoint.ts";
 
-export function hello() {
+export async function hello() {
-  console.log(YAML.stringify({ hello: "world" }));
+  const numbers = await syscall("addNumbers", 1, 2);
-
+  return {
-  return "hello";
+    yamlMessage: YAML.stringify({ hello: "world" }),
+    addedNumbers: numbers,
+  };
 }
 
 export function endpoint(req: EndpointRequest): EndpointResponse {

plugos/worker_runtime.ts

@@ -35,28 +35,37 @@ const pendingRequests = new Map<
 
 let syscallReqId = 0;
 
+const workerMode = typeof window === "undefined";
+
 function workerPostMessage(msg: ControllerMessage) {
   self.postMessage(msg);
 }
 
-self.syscall = async (name: string, ...args: any[]) => {
+if (workerMode) {
-  return await new Promise((resolve, reject) => {
+  globalThis.syscall = async (name: string, ...args: any[]) => {
-    syscallReqId++;
+    return await new Promise((resolve, reject) => {
-    pendingRequests.set(syscallReqId, { resolve, reject });
+      syscallReqId++;
-    workerPostMessage({
+      pendingRequests.set(syscallReqId, { resolve, reject });
-      type: "sys",
+      workerPostMessage({
-      id: syscallReqId,
+        type: "sys",
-      name,
+        id: syscallReqId,
-      args,
+        name,
+        args,
+      });
     });
-  });
+  };
-};
+}
 
 export function setupMessageListener(
   // deno-lint-ignore ban-types
   functionMapping: Record<string, Function>,
   manifest: any,
 ) {
+  if (!workerMode) {
+    // Don't do any of this stuff if this is not a web worker
+    // This caters to the NoSandbox run mode
+    return;
+  }
   self.addEventListener("message", (event: { data: WorkerMessage }) => {
     (async () => {
       const data = event.data;

plugs/markdown/markdown_render.test.ts

@@ -2,7 +2,7 @@ import buildMarkdown from "../../common/markdown_parser/parser.ts";
 import { parse } from "../../common/markdown_parser/parse_tree.ts";
 import { System } from "../../plugos/system.ts";
 
-import { createSandbox } from "../../plugos/environments/deno_sandbox.ts";
+import { createSandbox } from "../../plugos/sandboxes/deno_worker_sandbox.ts";
 import { loadMarkdownExtensions } from "../../common/markdown_parser/markdown_ext.ts";
 import { renderMarkdownToHtml } from "./markdown_render.ts";
 

server/server_system.ts

@@ -4,7 +4,7 @@ import { loadMarkdownExtensions } from "../common/markdown_parser/markdown_ext.t
 import buildMarkdown from "../common/markdown_parser/parser.ts";
 import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives.ts";
 import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts";
-import { createSandbox } from "../plugos/environments/webworker_sandbox.ts";
+import { createSandbox } from "../plugos/sandboxes/web_worker_sandbox.ts";
 import { CronHook } from "../plugos/hooks/cron.ts";
 import { EventHook } from "../plugos/hooks/event.ts";
 import { MQHook } from "../plugos/hooks/mq.ts";

web/client_system.ts

@@ -3,7 +3,7 @@ import { Manifest, SilverBulletHooks } from "../common/manifest.ts";
 import buildMarkdown from "../common/markdown_parser/parser.ts";
 import { CronHook } from "../plugos/hooks/cron.ts";
 import { EventHook } from "../plugos/hooks/event.ts";
-import { createSandbox } from "../plugos/environments/webworker_sandbox.ts";
+import { createSandbox } from "../plugos/sandboxes/web_worker_sandbox.ts";
 
 import assetSyscalls from "../plugos/syscalls/asset.ts";
 import { eventSyscalls } from "../plugos/syscalls/event.ts";