Tons of work

plugbox/src/node_sandbox.ts

@@ -0,0 +1,92 @@
+import { ControllerMessage, WorkerMessage } from "./types";
+import { System, Sandbox } from "./runtime";
+
+import { Worker } from "worker_threads";
+
+function wrapScript(code: string): string {
+  return `${code}["default"]`;
+}
+
+export class NodeSandbox implements Sandbox {
+  worker: Worker;
+  private reqId = 0;
+
+  outstandingInits = new Map<string, () => void>();
+  outstandingInvocations = new Map<
+    number,
+    { resolve: (result: any) => void; reject: (e: any) => void }
+  >();
+  loadedFunctions = new Set<string>();
+
+  constructor(readonly system: System<any>, workerScript: string) {
+    this.worker = new Worker(workerScript);
+
+    this.worker.on("message", this.onmessage.bind(this));
+  }
+
+  isLoaded(name: string): boolean {
+    return this.loadedFunctions.has(name);
+  }
+
+  async load(name: string, code: string): Promise<void> {
+    this.worker.postMessage({
+      type: "load",
+      name: name,
+      code: code,
+    } as WorkerMessage);
+    return new Promise((resolve) => {
+      this.loadedFunctions.add(name);
+      this.outstandingInits.set(name, resolve);
+    });
+  }
+
+  async onmessage(data: ControllerMessage) {
+    // let data = evt.data;
+    // let data = JSON.parse(msg) as ControllerMessage;
+    switch (data.type) {
+      case "inited":
+        let initCb = this.outstandingInits.get(data.name!);
+        initCb && initCb();
+        this.outstandingInits.delete(data.name!);
+        break;
+      case "syscall":
+        let result = await this.system.syscall(data.name!, data.args!);
+
+        this.worker.postMessage({
+          type: "syscall-response",
+          id: data.id,
+          data: result,
+        } as WorkerMessage);
+        break;
+      case "result":
+        let resultCb = this.outstandingInvocations.get(data.id!);
+        this.outstandingInvocations.delete(data.id!);
+        resultCb && resultCb.resolve(data.result);
+        break;
+      case "error":
+        let errCb = this.outstandingInvocations.get(data.result.id!);
+        this.outstandingInvocations.delete(data.id!);
+        errCb && errCb.reject(data.reason);
+        break;
+      default:
+        console.error("Unknown message type", data);
+    }
+  }
+
+  async invoke(name: string, args: any[]): Promise<any> {
+    this.reqId++;
+    this.worker.postMessage({
+      type: "invoke",
+      id: this.reqId,
+      name,
+      args,
+    });
+    return new Promise((resolve, reject) => {
+      this.outstandingInvocations.set(this.reqId, { resolve, reject });
+    });
+  }
+
+  stop() {
+    this.worker.terminate();
+  }
+}

plugbox/src/node_worker.ts

@@ -0,0 +1,88 @@
+import { VM, VMScript } from "vm2";
+import { parentPort } from "worker_threads";
+
+let loadedFunctions = new Map();
+let pendingRequests = new Map();
+
+let reqId = 0; // Syscall request ID
+
+let vm = new VM({
+  sandbox: {
+    console: console,
+    syscall: (name: string, args: any[]) => {
+      return new Promise((resolve, reject) => {
+        reqId++;
+        pendingRequests.set(reqId, resolve);
+        parentPort!.postMessage({
+          type: "syscall",
+          id: reqId,
+          name,
+          // TODO: Figure out why this is necessary (to avoide a CloneError)
+          args: JSON.parse(JSON.stringify(args)),
+        });
+      });
+    },
+  },
+});
+
+function wrapScript(code: string) {
+  return `${code}["default"]`;
+}
+
+function safeRun(fn: () => Promise<any>) {
+  fn().catch((e) => {
+    console.error(e);
+  });
+}
+
+parentPort!.on("message", (data) => {
+  safeRun(async () => {
+    switch (data.type) {
+      case "load":
+        console.log("Booting", data.name);
+        loadedFunctions.set(data.name, new VMScript(wrapScript(data.code)));
+        parentPort!.postMessage({
+          type: "inited",
+          name: data.name,
+        });
+        break;
+      case "invoke":
+        let fn = loadedFunctions.get(data.name);
+        if (!fn) {
+          throw new Error(`Function not loaded: ${data.name}`);
+        }
+        try {
+          let r = vm.run(fn);
+          let result = await Promise.resolve(r(...data.args));
+          parentPort!.postMessage({
+            type: "result",
+            id: data.id,
+            result: result,
+          });
+        } catch (e: any) {
+          parentPort!.postMessage({
+            type: "error",
+            id: data.id,
+            reason: e.message,
+          });
+          throw e;
+        }
+        break;
+      case "syscall-response":
+        let syscallId = data.id;
+        const lookup = pendingRequests.get(syscallId);
+        if (!lookup) {
+          console.log(
+            "Current outstanding requests",
+            pendingRequests,
+            "looking up",
+            syscallId
+          );
+          throw Error("Invalid request id");
+        }
+        pendingRequests.delete(syscallId);
+        lookup(data.data);
+        break;
+    }
+  });
+});

plugbox/src/runtime.test.ts

@@ -0,0 +1,47 @@
+import { NodeSandbox } from "./node_sandbox";
+import { System } from "./runtime";
+import { test, expect } from "@jest/globals";
+
+test("Run a Node sandbox", async () => {
+  let system = new System();
+  system.registerSyscalls({
+    addNumbers: (a, b) => {
+      return a + b;
+    },
+  });
+  let plug = await system.load(
+    "test",
+    {
+      functions: {
+        addTen: {
+          code: `(() => {
+          return {
+            default: (n) => {
+              return n + 10;
+            }
+          };
+        })()`,
+        },
+        addNumbersSyscall: {
+          code: `(() => {
+          return {
+            default: async (a, b) => {
+              return await(syscall("addNumbers", [a, b]));
+            }
+          };
+        })()`,
+        },
+      },
+      hooks: {
+        events: {},
+      },
+    },
+    new NodeSandbox(system, __dirname + "/../dist/node_worker.js")
+  );
+  expect(await plug.invoke("addTen", [10])).toBe(20);
+  for (let i = 0; i < 100; i++) {
+    expect(await plug.invoke("addNumbersSyscall", [10, i])).toBe(10 + i);
+  }
+  console.log(plug.sandbox);
+  await system.stop();
+});

plugbox/src/runtime.ts

@@ -1,9 +1,8 @@
 import { Manifest } from "./types";
-import { WebworkerSandbox } from "./worker_sandbox";
+// import { WebworkerSandbox } from "./worker_sandbox";
 
 interface SysCallMapping {
-  // TODO: Better typing
-  [key: string]: any;
+  [key: string]: (...args: any) => Promise<any> | any;
 }
 
 export interface Sandbox {
@@ -13,19 +12,14 @@ export interface Sandbox {
   stop(): void;
 }
 
-export interface PlugLoader<HookT> {
-  load(name: string, manifest: Manifest<HookT>): Promise<void>;
-}
-
 export class Plug<HookT> {
   system: System<HookT>;
-  // private runningFunctions: Map<string, FunctionWorker>;
-  functionWorker: WebworkerSandbox;
+  sandbox: Sandbox;
   public manifest?: Manifest<HookT>;
 
-  constructor(system: System<HookT>, name: string) {
+  constructor(system: System<HookT>, name: string, sandbox: Sandbox) {
     this.system = system;
-    this.functionWorker = new WebworkerSandbox(this);
+    this.sandbox = sandbox;
   }
 
   async load(manifest: Manifest<HookT>) {
@@ -34,13 +28,11 @@ export class Plug<HookT> {
   }
 
   async invoke(name: string, args: Array<any>): Promise<any> {
-    if (!this.functionWorker.isLoaded(name)) {
-      await this.functionWorker.load(
-        name,
-        this.manifest!.functions[name].code!
-      );
+    if (!this.sandbox.isLoaded(name)) {
+      await this.sandbox.load(name, this.manifest!.functions[name].code!);
     }
-    return await this.functionWorker.invoke(name, args);
+    console.log("Loaded", name);
+    return await this.sandbox.invoke(name, args);
   }
 
   async dispatchEvent(name: string, data?: any): Promise<any[]> {
@@ -58,18 +50,13 @@ export class Plug<HookT> {
   }
 
   async stop() {
-    this.functionWorker.stop();
+    this.sandbox.stop();
   }
 }
 
 export class System<HookT> {
-  protected plugs: Map<string, Plug<HookT>>;
-  registeredSyscalls: SysCallMapping;
-
-  constructor() {
-    this.plugs = new Map<string, Plug<HookT>>();
-    this.registeredSyscalls = {};
-  }
+  protected plugs = new Map<string, Plug<HookT>>();
+  registeredSyscalls: SysCallMapping = {};
 
   registerSyscalls(...registrationObjects: SysCallMapping[]) {
     for (const registrationObject of registrationObjects) {
@@ -90,8 +77,12 @@ export class System<HookT> {
     return Promise.resolve(callback(...args));
   }
 
-  async load(name: string, manifest: Manifest<HookT>): Promise<Plug<HookT>> {
-    const plug = new Plug(this, name);
+  async load(
+    name: string,
+    manifest: Manifest<HookT>,
+    sandbox: Sandbox
+  ): Promise<Plug<HookT>> {
+    const plug = new Plug(this, name, sandbox);
     await plug.load(manifest);
     this.plugs.set(name, plug);
     return plug;

plugbox/src/sandbox_worker.ts

@@ -1,12 +1,13 @@
-declare global {
-  function syscall(id: number, name: string, args: any[]): Promise<any>;
-}
 import { ControllerMessage, WorkerMessage, WorkerMessageType } from "./types";
 import { safeRun } from "./util";
 
 let loadedFunctions = new Map<string, Function>();
 let pendingRequests = new Map<number, (result: unknown) => void>();
 
+declare global {
+  function syscall(id: number, name: string, args: any[]): Promise<any>;
+}
+
 self.syscall = async (id: number, name: string, args: any[]) => {
   return await new Promise((resolve, reject) => {
     pendingRequests.set(id, resolve);
@@ -19,22 +20,6 @@ self.syscall = async (id: number, name: string, args: any[]) => {
   });
 };
 
-self.addEventListener("result", (event) => {
-  let customEvent = event as CustomEvent;
-  self.postMessage({
-    type: "result",
-    result: customEvent.detail,
-  });
-});
-
-self.addEventListener("app-error", (event) => {
-  let customEvent = event as CustomEvent;
-  self.postMessage({
-    type: "error",
-    reason: customEvent.detail,
-  });
-});
-
 function wrapScript(code: string): string {
   return `const fn = ${code};
 return fn["default"].apply(null, arguments);`;

plugbox/src/worker_sandbox.ts

@@ -1,5 +1,5 @@
 import { ControllerMessage, WorkerMessage } from "./types";
-import { Plug, Sandbox } from "./runtime";
+import { Plug, Sandbox, System } from "./runtime";
 
 export class WebworkerSandbox implements Sandbox {
   private worker: Worker;
@@ -12,7 +12,7 @@ export class WebworkerSandbox implements Sandbox {
   >();
   private loadedFunctions = new Set<string>();
 
-  constructor(readonly plug: Plug<any>) {
+  constructor(readonly system: System<any>) {
     this.worker = new Worker(new URL("sandbox_worker.ts", import.meta.url), {
       type: "module",
     });
@@ -46,7 +46,7 @@ export class WebworkerSandbox implements Sandbox {
         this.outstandingInits.delete(data.name!);
         break;
       case "syscall":
-        let result = await this.plug.system.syscall(data.name!, data.args!);
+        let result = await this.system.syscall(data.name!, data.args!);
 
         this.worker.postMessage({
           type: "syscall-response",
@@ -56,10 +56,12 @@ export class WebworkerSandbox implements Sandbox {
         break;
       case "result":
         let resultCb = this.outstandingInvocations.get(data.id!);
+        this.outstandingInvocations.delete(data.id!);
         resultCb && resultCb.resolve(data.result);
         break;
       case "error":
         let errCb = this.outstandingInvocations.get(data.result.id!);
+        this.outstandingInvocations.delete(data.id!);
         errCb && errCb.reject(data.reason);
         break;
       default: