Enabled back-end running of functions, moved indexing to server.

2022-03-18 14:59:04 +01:00
parent 97b9d3a3e0
commit 8bff6d98e1
27 changed files with 538 additions and 367 deletions
@@ -40,7 +40,8 @@ export default ${functionName};`
  if (inFile !== filePath) {
    await unlink(inFile);
  }
-  return jsCode;
+  // Strip final ';'
+  return jsCode.substring(0, jsCode.length - 2);
 }

 async function bundle(manifestPath, sourceMaps) {
@@ -8,8 +8,7 @@
  },
  "scripts": {
    "check": "tsc --noEmit",
-    "test": "jest",
-    "build-worker": "tsc src/node_worker.ts --outDir dist --module nodenext"
+    "test": "jest"
  },
  "dependencies": {
    "esbuild": "^0.14.24",
@@ -27,7 +26,6 @@
    "events": "^3.3.0",
    "jest": "^27.5.1",
    "parcel": "^2.3.2",
-    "parceljs": "^0.0.1",
    "path-browserify": "^1.0.1",
    "ts-jest": "^27.1.3",
    "util": "^0.12.4",
@@ -1,7 +1,8 @@
 <html>
  <body>
    <script type="module">
-      import "./function_worker";
+      // Sup yo!
+      import "./sandbox_worker";
    </script>
  </body>
 </html>
@@ -0,0 +1,43 @@
+import { ControllerMessage, WorkerLike, WorkerMessage } from "./types";
+import { Sandbox, System } from "./runtime";
+import { safeRun } from "./util";
+
+// @ts-ignore
+import sandboxHtml from "bundle-text:./iframe_sandbox.html";
+
+class IFrameWrapper implements WorkerLike {
+  private iframe: HTMLIFrameElement;
+  onMessage?: (message: any) => Promise<void>;
+
+  constructor() {
+    const iframe = document.createElement("iframe", {});
+    this.iframe = iframe;
+    iframe.style.display = "none";
+    // Let's lock this down significantly
+    iframe.setAttribute("sandbox", "allow-scripts");
+    iframe.srcdoc = sandboxHtml;
+    window.addEventListener("message", (evt: any) => {
+      if (evt.source !== iframe.contentWindow) {
+        return;
+      }
+      let data = evt.data;
+      if (!data) return;
+      safeRun(async () => {
+        await this.onMessage!(data);
+      });
+    });
+    document.body.appendChild(iframe);
+  }
+
+  postMessage(message: any): void {
+    this.iframe.contentWindow!.postMessage(message, "*");
+  }
+
+  terminate() {
+    return this.iframe.remove();
+  }
+}
+
+export function createSandbox(system: System<any>) {
+  return new Sandbox(system, new IFrameWrapper());
+}
@@ -1,91 +1,42 @@
-import { ControllerMessage, WorkerMessage } from "./types";
+import { ControllerMessage, WorkerLike, WorkerMessage } from "./types";
 import { System, Sandbox } from "./runtime";

 import { Worker } from "worker_threads";
+import * as fs from "fs";
+import { safeRun } from "./util";

-function wrapScript(code: string): string {
-  return `${code}["default"]`;
-}
+// ParcelJS will simply inline this into the bundle.
+const workerCode = fs.readFileSync(__dirname + "/node_worker.js", "utf-8");

-export class NodeSandbox implements Sandbox {
-  worker: Worker;
-  private reqId = 0;
+class NodeWorkerWrapper implements WorkerLike {
+  onMessage?: (message: any) => Promise<void>;
+  private worker: Worker;

-  outstandingInits = new Map<string, () => void>();
-  outstandingInvocations = new Map<
-    number,
-    { resolve: (result: any) => void; reject: (e: any) => void }
-  >();
-  loadedFunctions = new Set<string>();
-
-  constructor(readonly system: System<any>, workerScript: string) {
-    this.worker = new Worker(workerScript);
-    this.worker.on("message", this.onmessage.bind(this));
-  }
-
-  isLoaded(name: string): boolean {
-    return this.loadedFunctions.has(name);
-  }
-
-  async load(name: string, code: string): Promise<void> {
-    this.worker.postMessage({
-      type: "load",
-      name: name,
-      code: code,
-    } as WorkerMessage);
-    return new Promise((resolve) => {
-      this.loadedFunctions.add(name);
-      this.outstandingInits.set(name, resolve);
+  constructor(worker: Worker) {
+    this.worker = worker;
+    worker.on("message", (message: any) => {
+      safeRun(async () => {
+        await this.onMessage!(message);
+      });
    });
  }

-  async onmessage(data: ControllerMessage) {
-    // let data = evt.data;
-    // let data = JSON.parse(msg) as ControllerMessage;
-    switch (data.type) {
-      case "inited":
-        let initCb = this.outstandingInits.get(data.name!);
-        initCb && initCb();
-        this.outstandingInits.delete(data.name!);
-        break;
-      case "syscall":
-        let result = await this.system.syscall(data.name!, data.args!);
-
-        this.worker.postMessage({
-          type: "syscall-response",
-          id: data.id,
-          data: result,
-        } as WorkerMessage);
-        break;
-      case "result":
-        let resultCb = this.outstandingInvocations.get(data.id!);
-        this.outstandingInvocations.delete(data.id!);
-        resultCb && resultCb.resolve(data.result);
-        break;
-      case "error":
-        let errCb = this.outstandingInvocations.get(data.result.id!);
-        this.outstandingInvocations.delete(data.id!);
-        errCb && errCb.reject(data.reason);
-        break;
-      default:
-        console.error("Unknown message type", data);
-    }
+  postMessage(message: any): void {
+    this.worker.postMessage(message);
  }

-  async invoke(name: string, args: any[]): Promise<any> {
-    this.reqId++;
-    this.worker.postMessage({
-      type: "invoke",
-      id: this.reqId,
-      name,
-      args,
-    });
-    return new Promise((resolve, reject) => {
-      this.outstandingInvocations.set(this.reqId, { resolve, reject });
-    });
-  }
-
-  stop() {
+  terminate(): void {
    this.worker.terminate();
  }
 }
+
+export function createSandbox(system: System<any>) {
+  return new Sandbox(
+    system,
+    new NodeWorkerWrapper(
+      new Worker(workerCode, {
+        eval: true,
+      })
+    )
+  );
+}
@@ -1,88 +0,0 @@
-import { VM, VMScript } from "vm2";
-import { parentPort } from "worker_threads";
-
-let loadedFunctions = new Map();
-let pendingRequests = new Map();
-
-let reqId = 0; // Syscall request ID
-
-let vm = new VM({
-  sandbox: {
-    console: console,
-    syscall: (name: string, args: any[]) => {
-      return new Promise((resolve, reject) => {
-        reqId++;
-        pendingRequests.set(reqId, resolve);
-        parentPort!.postMessage({
-          type: "syscall",
-          id: reqId,
-          name,
-          // TODO: Figure out why this is necessary (to avoide a CloneError)
-          args: JSON.parse(JSON.stringify(args)),
-        });
-      });
-    },
-  },
-});
-
-function wrapScript(code: string) {
-  return `${code}["default"]`;
-}
-
-function safeRun(fn: () => Promise<any>) {
-  fn().catch((e) => {
-    console.error(e);
-  });
-}
-
-parentPort!.on("message", (data) => {
-  safeRun(async () => {
-    switch (data.type) {
-      case "load":
-        console.log("Booting", data.name);
-        loadedFunctions.set(data.name, new VMScript(wrapScript(data.code)));
-        parentPort!.postMessage({
-          type: "inited",
-          name: data.name,
-        });
-        break;
-      case "invoke":
-        let fn = loadedFunctions.get(data.name);
-        if (!fn) {
-          throw new Error(`Function not loaded: ${data.name}`);
-        }
-        try {
-          let r = vm.run(fn);
-          let result = await Promise.resolve(r(...data.args));
-          parentPort!.postMessage({
-            type: "result",
-            id: data.id,
-            result: result,
-          });
-        } catch (e: any) {
-          parentPort!.postMessage({
-            type: "error",
-            id: data.id,
-            reason: e.message,
-          });
-          throw e;
-        }
-        break;
-      case "syscall-response":
-        let syscallId = data.id;
-        const lookup = pendingRequests.get(syscallId);
-        if (!lookup) {
-          console.log(
-            "Current outstanding requests",
-            pendingRequests,
-            "looking up",
-            syscallId
-          );
-          throw Error("Invalid request id");
-        }
-        pendingRequests.delete(syscallId);
-        lookup(data.data);
-        break;
-    }
-  });
-});
@@ -1,4 +1,4 @@
-import { NodeSandbox } from "./node_sandbox";
+import { createSandbox } from "./node_sandbox";
 import { System } from "./runtime";
 import { test, expect } from "@jest/globals";

@@ -8,6 +8,9 @@ test("Run a Node sandbox", async () => {
    addNumbers: (a, b) => {
      return a + b;
    },
+    failingSyscall: () => {
+      throw new Error("#fail");
+    },
  });
  let plug = await system.load(
    "test",
@@ -26,7 +29,25 @@ test("Run a Node sandbox", async () => {
          code: `(() => {
          return {
            default: async (a, b) => {
-              return await(syscall("addNumbers", [a, b]));
+              return await self.syscall(1, "addNumbers", [a, b]);
+            }
+          };
+        })()`,
+        },
+        errorOut: {
+          code: `(() => {
+          return {
+            default: () => {
+              throw Error("BOOM");
+            }
+          };
+        })()`,
+        },
+        errorOutSys: {
+          code: `(() => {
+          return {
+            default: async () => {
+              await self.syscall(2, "failingSyscall", []);
            }
          };
        })()`,
@@ -36,12 +57,23 @@ test("Run a Node sandbox", async () => {
        events: {},
      },
    },
-    new NodeSandbox(system, __dirname + "/../dist/node_worker.js")
+    createSandbox(system)
  );
  expect(await plug.invoke("addTen", [10])).toBe(20);
  for (let i = 0; i < 100; i++) {
    expect(await plug.invoke("addNumbersSyscall", [10, i])).toBe(10 + i);
  }
-  // console.log(plug.sandbox);
+  try {
+    await plug.invoke("errorOut", []);
+    expect(true).toBe(false);
+  } catch (e: any) {
+    expect(e.message).toBe("BOOM");
+  }
+  try {
+    await plug.invoke("errorOutSys", []);
+    expect(true).toBe(false);
+  } catch (e: any) {
+    expect(e.message).toBe("#fail");
+  }
  await system.stop();
 });
@@ -1,15 +1,101 @@
-import { Manifest } from "./types";
-// import { WebworkerSandbox } from "./worker_sandbox";
+import {
+  ControllerMessage,
+  Manifest,
+  WorkerLike,
+  WorkerMessage,
+} from "./types";

 interface SysCallMapping {
  [key: string]: (...args: any) => Promise<any> | any;
 }

-export interface Sandbox {
-  isLoaded(name: string): boolean;
-  load(name: string, code: string): Promise<void>;
-  invoke(name: string, args: any[]): Promise<any>;
-  stop(): void;
+export class Sandbox {
+  protected worker: WorkerLike;
+  protected reqId = 0;
+  protected outstandingInits = new Map<string, () => void>();
+  protected outstandingInvocations = new Map<
+    number,
+    { resolve: (result: any) => void; reject: (e: any) => void }
+  >();
+  protected loadedFunctions = new Set<string>();
+  protected system: System<any>;
+
+  constructor(system: System<any>, worker: WorkerLike) {
+    worker.onMessage = this.onMessage.bind(this);
+    this.worker = worker;
+    this.system = system;
+  }
+
+  isLoaded(name: string) {
+    return this.loadedFunctions.has(name);
+  }
+
+  async load(name: string, code: string): Promise<void> {
+    this.worker.postMessage({
+      type: "load",
+      name: name,
+      code: code,
+    } as WorkerMessage);
+    return new Promise((resolve) => {
+      this.loadedFunctions.add(name);
+      this.outstandingInits.set(name, resolve);
+    });
+  }
+
+  async onMessage(data: ControllerMessage) {
+    switch (data.type) {
+      case "inited":
+        let initCb = this.outstandingInits.get(data.name!);
+        initCb && initCb();
+        this.outstandingInits.delete(data.name!);
+        break;
+      case "syscall":
+        try {
+          let result = await this.system.syscall(data.name!, data.args!);
+
+          this.worker.postMessage({
+            type: "syscall-response",
+            id: data.id,
+            result: result,
+          } as WorkerMessage);
+        } catch (e: any) {
+          this.worker.postMessage({
+            type: "syscall-response",
+            id: data.id,
+            error: e.message,
+          } as WorkerMessage);
+        }
+        break;
+      case "result":
+        let resultCbs = this.outstandingInvocations.get(data.id!);
+        this.outstandingInvocations.delete(data.id!);
+        if (data.error) {
+          resultCbs && resultCbs.reject(new Error(data.error));
+        } else {
+          resultCbs && resultCbs.resolve(data.result);
+        }
+        break;
+      default:
+        console.error("Unknown message type", data);
+    }
+  }
+
+  async invoke(name: string, args: any[]): Promise<any> {
+    this.reqId++;
+    this.worker.postMessage({
+      type: "invoke",
+      id: this.reqId,
+      name,
+      args,
+    });
+    return new Promise((resolve, reject) => {
+      this.outstandingInvocations.set(this.reqId, { resolve, reject });
+    });
+  }
+
+  stop() {
+    this.worker.terminate();
+  }
 }

 export class Plug<HookT> {
@@ -29,6 +115,10 @@ export class Plug<HookT> {

  async invoke(name: string, args: Array<any>): Promise<any> {
    if (!this.sandbox.isLoaded(name)) {
+      const funDef = this.manifest!.functions[name];
+      if (!funDef) {
+        throw new Error(`Function ${name} not found in manifest`);
+      }
      await this.sandbox.load(name, this.manifest!.functions[name].code!);
    }
    return await this.sandbox.invoke(name, args);
@@ -87,11 +177,17 @@ export class System<HookT> {
    return plug;
  }

+  async dispatchEvent(name: string, data?: any): Promise<any[]> {
+    let promises = [];
+    for (let plug of this.plugs.values()) {
+      promises.push(plug.dispatchEvent(name, data));
+    }
+    return await Promise.all(promises);
+  }
+
  async stop(): Promise<void[]> {
    return Promise.all(
      Array.from(this.plugs.values()).map((plug) => plug.stop())
    );
  }
 }
-
-console.log("Starting");
@@ -2,21 +2,38 @@ import { ControllerMessage, WorkerMessage, WorkerMessageType } from "./types";
 import { safeRun } from "./util";

 let loadedFunctions = new Map<string, Function>();
-let pendingRequests = new Map<number, (result: unknown) => void>();
+let pendingRequests = new Map<
+  number,
+  {
+    resolve: (result: unknown) => void;
+    reject: (e: any) => void;
+  }
+>();

 declare global {
  function syscall(id: number, name: string, args: any[]): Promise<any>;
 }

+let postMessage = self.postMessage.bind(self);
+
+if (window.parent !== window) {
+  console.log("running in an iframe");
+  postMessage = window.parent.postMessage.bind(window.parent);
+  // postMessage({ type: "test" }, "*");
+}
+
 self.syscall = async (id: number, name: string, args: any[]) => {
  return await new Promise((resolve, reject) => {
-    pendingRequests.set(id, resolve);
-    self.postMessage({
-      type: "syscall",
-      id,
-      name,
-      args,
-    });
+    pendingRequests.set(id, { resolve, reject });
+    postMessage(
+      {
+        type: "syscall",
+        id,
+        name,
+        args,
+      },
+      "*"
+    );
  });
 };

@@ -26,6 +43,7 @@ return fn["default"].apply(null, arguments);`;
 }

 self.addEventListener("message", (event: { data: WorkerMessage }) => {
+  // console.log("Got a message", event.data);
  safeRun(async () => {
    let messageEvent = event;
    let data = messageEvent.data;
@@ -33,10 +51,13 @@ self.addEventListener("message", (event: { data: WorkerMessage }) => {
      case "load":
        console.log("Booting", data.name);
        loadedFunctions.set(data.name!, new Function(wrapScript(data.code!)));
-        self.postMessage({
-          type: "inited",
-          name: data.name,
-        } as ControllerMessage);
+        postMessage(
+          {
+            type: "inited",
+            name: data.name,
+          } as ControllerMessage,
+          "*"
+        );
        break;
      case "invoke":
        let fn = loadedFunctions.get(data.name!);
@@ -45,17 +66,23 @@ self.addEventListener("message", (event: { data: WorkerMessage }) => {
        }
        try {
          let result = await Promise.resolve(fn(...(data.args || [])));
-          self.postMessage({
-            type: "result",
-            id: data.id,
-            result: result,
-          } as ControllerMessage);
+          postMessage(
+            {
+              type: "result",
+              id: data.id,
+              result: result,
+            } as ControllerMessage,
+            "*"
+          );
        } catch (e: any) {
-          self.postMessage({
-            type: "error",
-            id: data.id,
-            reason: e.message,
-          } as ControllerMessage);
+          postMessage(
+            {
+              type: "result",
+              id: data.id,
+              error: e.message,
+            } as ControllerMessage,
+            "*"
+          );
          throw e;
        }

@@ -73,7 +100,11 @@ self.addEventListener("message", (event: { data: WorkerMessage }) => {
          throw Error("Invalid request id");
        }
        pendingRequests.delete(syscallId);
-        lookup(data.data);
+        if (data.error) {
+          lookup.reject(new Error(data.error));
+        } else {
+          lookup.resolve(data.result);
+        }
        break;
    }
  });
@@ -10,18 +10,19 @@ export type WorkerMessage = {
  name?: string;
  code?: string;
  args?: any[];
-  data?: any;
+  result?: any;
+  error?: any;
 };

-export type ControllerMessageType = "inited" | "result" | "error" | "syscall";
+export type ControllerMessageType = "inited" | "result" | "syscall";

 export type ControllerMessage = {
  type: ControllerMessageType;
  id?: number;
  name?: string;
-  reason?: string;
  args?: any[];
-  result: any;
+  error?: string;
+  result?: any;
 };

 export interface Manifest<HookT> {
@@ -35,3 +36,9 @@ export interface FunctionDef {
  path?: string;
  code?: string;
 }
+
+export interface WorkerLike {
+  onMessage?: (message: any) => Promise<void>;
+  postMessage(message: any): void;
+  terminate(): void;
+}
@@ -0,0 +1,34 @@
+import { ControllerMessage, WorkerLike, WorkerMessage } from "./types";
+import { Sandbox, System } from "./runtime";
+import { safeRun } from "./util";
+
+class WebWorkerWrapper implements WorkerLike {
+  private worker: Worker;
+  onMessage?: (message: any) => Promise<void>;
+
+  constructor(worker: Worker) {
+    this.worker = worker;
+    this.worker.addEventListener("message", (evt: any) => {
+      let data = evt.data;
+      if (!data) return;
+      safeRun(async () => {
+        await this.onMessage!(data);
+      });
+    });
+  }
+  postMessage(message: any): void {
+    this.worker.postMessage(message);
+  }
+
+  terminate() {
+    return this.worker.terminate();
+  }
+}
+
+export function createSandbox(system: System<any>) {
+  // ParcelJS will build this file into a worker.
+  let worker = new Worker(new URL("sandbox_worker.ts", import.meta.url), {
+    type: "module",
+  });
+  return new Sandbox(system, new WebWorkerWrapper(worker));
+}
@@ -1,88 +0,0 @@
-import { ControllerMessage, WorkerMessage } from "./types";
-import { Sandbox, System } from "./runtime";
-
-export class WebworkerSandbox implements Sandbox {
-  private worker: Worker;
-  private reqId = 0;
-
-  private outstandingInits = new Map<string, () => void>();
-  private outstandingInvocations = new Map<
-    number,
-    { resolve: (result: any) => void; reject: (e: any) => void }
-  >();
-  private loadedFunctions = new Set<string>();
-
-  constructor(readonly system: System<any>) {
-    this.worker = new Worker(new URL("sandbox_worker.ts", import.meta.url), {
-      type: "module",
-    });
-
-    this.worker.onmessage = this.onmessage.bind(this);
-  }
-
-  isLoaded(name: string) {
-    return this.loadedFunctions.has(name);
-  }
-
-  async load(name: string, code: string): Promise<void> {
-    this.worker.postMessage({
-      type: "load",
-      name: name,
-      code: code,
-    } as WorkerMessage);
-    return new Promise((resolve) => {
-      this.loadedFunctions.add(name);
-      this.outstandingInits.set(name, resolve);
-    });
-  }
-
-  async onmessage(evt: { data: ControllerMessage }) {
-    let data = evt.data;
-    if (!data) return;
-    switch (data.type) {
-      case "inited":
-        let initCb = this.outstandingInits.get(data.name!);
-        initCb && initCb();
-        this.outstandingInits.delete(data.name!);
-        break;
-      case "syscall":
-        let result = await this.system.syscall(data.name!, data.args!);
-
-        this.worker.postMessage({
-          type: "syscall-response",
-          id: data.id,
-          data: result,
-        } as WorkerMessage);
-        break;
-      case "result":
-        let resultCb = this.outstandingInvocations.get(data.id!);
-        this.outstandingInvocations.delete(data.id!);
-        resultCb && resultCb.resolve(data.result);
-        break;
-      case "error":
-        let errCb = this.outstandingInvocations.get(data.result.id!);
-        this.outstandingInvocations.delete(data.id!);
-        errCb && errCb.reject(data.reason);
-        break;
-      default:
-        console.error("Unknown message type", data);
-    }
-  }
-
-  async invoke(name: string, args: any[]): Promise<any> {
-    this.reqId++;
-    this.worker.postMessage({
-      type: "invoke",
-      id: this.reqId,
-      name,
-      args,
-    });
-    return new Promise((resolve, reject) => {
-      this.outstandingInvocations.set(this.reqId, { resolve, reject });
-    });
-  }
-
-  stop() {
-    this.worker.terminate();
-  }
-}