Monorepo with yarn workspaces requires yarn 3.2

2022-04-21 13:57:45 +02:00
parent 32f3501773
commit 1f842ec1d6
167 changed files with 10424 additions and 8263 deletions
@@ -0,0 +1,8 @@
+<html>
+  <body>
+    <script type="module">
+      // Sup yo!
+      import "./sandbox_worker";
+    </script>
+  </body>
+</html>
@@ -0,0 +1,55 @@
+import { safeRun } from "../util";
+
+// @ts-ignore
+import sandboxHtml from "bundle-text:./iframe_sandbox.html";
+import { Sandbox } from "../sandbox";
+import { WorkerLike } from "./worker";
+import { Plug } from "../plug";
+
+class IFrameWrapper implements WorkerLike {
+  private iframe: HTMLIFrameElement;
+  onMessage?: (message: any) => Promise<void>;
+  ready: Promise<void>;
+  private messageListener: (evt: any) => void;
+
+  constructor() {
+    const iframe = document.createElement("iframe", {});
+    this.iframe = iframe;
+    iframe.style.display = "none";
+    // Let's lock this down significantly
+    iframe.setAttribute("sandbox", "allow-scripts");
+    iframe.srcdoc = sandboxHtml;
+    this.messageListener = (evt: any) => {
+      if (evt.source !== iframe.contentWindow) {
+        return;
+      }
+      let data = evt.data;
+      if (!data) return;
+      safeRun(async () => {
+        await this.onMessage!(data);
+      });
+    };
+    window.addEventListener("message", this.messageListener);
+    document.body.appendChild(iframe);
+    this.ready = new Promise((resolve) => {
+      iframe.onload = () => {
+        resolve();
+        iframe.onload = null;
+      };
+    });
+  }
+
+  postMessage(message: any): void {
+    this.iframe.contentWindow!.postMessage(message, "*");
+  }
+
+  terminate() {
+    console.log("Terminating iframe sandbox");
+    window.removeEventListener("message", this.messageListener);
+    return this.iframe.remove();
+  }
+}
+
+export function createSandbox(plug: Plug<any>) {
+  return new Sandbox(plug, new IFrameWrapper());
+}
@@ -0,0 +1,50 @@
+import { Worker } from "worker_threads";
+import { safeRun } from "../util";
+
+// @ts-ignore
+import workerCode from "bundle-text:./node_worker.ts";
+import { Sandbox } from "../sandbox";
+import { WorkerLike } from "./worker";
+import { Plug } from "../plug";
+import path from "path";
+import fs from "fs";
+
+class NodeWorkerWrapper implements WorkerLike {
+  onMessage?: (message: any) => Promise<void>;
+  ready: Promise<void>;
+  private worker: Worker;
+
+  constructor(worker: Worker) {
+    this.worker = worker;
+    worker.on("message", (message: any) => {
+      safeRun(async () => {
+        await this.onMessage!(message);
+      });
+    });
+    this.ready = new Promise((resolve) => {
+      worker.once("online", resolve);
+    });
+  }
+
+  postMessage(message: any): void {
+    this.worker.postMessage(message);
+  }
+
+  terminate(): void {
+    this.worker.terminate();
+  }
+}
+
+// Look for the node_modules directory, to be passed to the worker to find e.g. the vm2 module
+let nodeModulesDir = __dirname;
+while (!fs.existsSync(nodeModulesDir + "/node_modules/vm2")) {
+  nodeModulesDir = path.dirname(nodeModulesDir);
+}
+
+export function createSandbox(plug: Plug<any>) {
+  let worker = new Worker(workerCode, {
+    eval: true,
+    workerData: path.join(nodeModulesDir, "node_modules"),
+  });
+  return new Sandbox(plug, new NodeWorkerWrapper(worker));
+}
@@ -0,0 +1,117 @@
+import { preloadModules } from "../../silverbullet-common/preload_modules";
+
+const { parentPort, workerData } = require("worker_threads");
+const { VM, VMScript } = require(`${workerData}/vm2`);
+const fetch = require(`${workerData}/node-fetch`);
+const WebSocket = require(`${workerData}/ws`);
+
+// console.log("Process env", process.env);
+
+let loadedFunctions = new Map<string, Function>();
+let pendingRequests = new Map<
+  number,
+  {
+    resolve: (result: unknown) => void;
+    reject: (e: any) => void;
+  }
+>();
+
+let syscallReqId = 0;
+
+let vm = new VM({
+  sandbox: {
+    // Exposing some "safe" APIs
+    console,
+    setTimeout,
+    clearTimeout,
+    setInterval,
+    clearInterval,
+    fetch,
+    WebSocket,
+    // This is only going to be called for pre-bundled modules, we won't allow
+    // arbitrary requiring of modules
+    require: (moduleName: string): any => {
+      // console.log("Loading", moduleName);
+      if (preloadModules.includes(moduleName)) {
+        return require(`${workerData}/${moduleName}`);
+      } else {
+        throw Error("Cannot import arbitrary modules");
+      }
+    },
+    self: {
+      syscall: (name: string, ...args: any[]) => {
+        return new Promise((resolve, reject) => {
+          syscallReqId++;
+          pendingRequests.set(syscallReqId, { resolve, reject });
+          parentPort.postMessage({
+            type: "syscall",
+            id: syscallReqId,
+            name,
+            // TODO: Figure out why this is necessary (to avoide a CloneError)
+            args: JSON.parse(JSON.stringify(args)),
+          });
+        });
+      },
+    },
+  },
+});
+
+function wrapScript(code: string) {
+  return `(${code})["default"]`;
+}
+
+function safeRun(fn: any) {
+  fn().catch((e: any) => {
+    console.error(e);
+  });
+}
+
+parentPort.on("message", (data: any) => {
+  safeRun(async () => {
+    switch (data.type) {
+      case "load":
+        loadedFunctions.set(data.name, new VMScript(wrapScript(data.code)));
+        parentPort.postMessage({
+          type: "inited",
+          name: data.name,
+        });
+        break;
+      case "invoke":
+        let fn = loadedFunctions.get(data.name);
+        if (!fn) {
+          throw new Error(`Function not loaded: ${data.name}`);
+        }
+        try {
+          let r = vm.run(fn);
+          let result = await Promise.resolve(r(...data.args));
+          parentPort.postMessage({
+            type: "result",
+            id: data.id,
+            // TOOD: Figure out if this is necessary, because it's expensive
+            result: result && JSON.parse(JSON.stringify(result)),
+          });
+        } catch (e: any) {
+          parentPort.postMessage({
+            type: "result",
+            id: data.id,
+            error: e.message,
+          });
+        }
+        break;
+      case "syscall-response":
+        let syscallId = data.id;
+        const lookup = pendingRequests.get(syscallId);
+        if (!lookup) {
+          throw Error("Invalid request id");
+        }
+        pendingRequests.delete(syscallId);
+        if (data.error) {
+          console.log("Got rejection", data.error);
+          lookup.reject(new Error(data.error));
+        } else {
+          lookup.resolve(data.result);
+        }
+        break;
+    }
+  });
+});
@@ -0,0 +1,114 @@
+import { safeRun } from "../util";
+import { ControllerMessage, WorkerMessage } from "./worker";
+
+let loadedFunctions = new Map<string, Function>();
+let pendingRequests = new Map<
+  number,
+  {
+    resolve: (result: unknown) => void;
+    reject: (e: any) => void;
+  }
+>();
+
+function workerPostMessage(msg: ControllerMessage) {
+  if (typeof window !== "undefined" && window.parent !== window) {
+    window.parent.postMessage(msg, "*");
+  } else {
+    self.postMessage(msg);
+  }
+}
+
+declare global {
+  function syscall(name: string, ...args: any[]): Promise<any>;
+  // function require(moduleName: string): any;
+}
+
+let syscallReqId = 0;
+
+self.syscall = async (name: string, ...args: any[]) => {
+  return await new Promise((resolve, reject) => {
+    syscallReqId++;
+    pendingRequests.set(syscallReqId, { resolve, reject });
+    workerPostMessage({
+      type: "syscall",
+      id: syscallReqId,
+      name,
+      args,
+    });
+  });
+};
+
+const preloadedModules: { [key: string]: any } = {
+  "@lezer/lr": require("@lezer/lr"),
+  yaml: require("yaml"),
+};
+// for (const moduleName of preloadModules) {
+//   preloadedModules[moduleName] = require(moduleName);
+// }
+
+// @ts-ignore
+self.require = (moduleName: string): any => {
+  // console.log("Loading", moduleName, preloadedModules[moduleName]);
+  return preloadedModules[moduleName];
+};
+
+function wrapScript(code: string) {
+  return `return (${code})["default"]`;
+}
+
+self.addEventListener("message", (event: { data: WorkerMessage }) => {
+  safeRun(async () => {
+    let data = event.data;
+    switch (data.type) {
+      case "load":
+        let fn2 = new Function(wrapScript(data.code!));
+        loadedFunctions.set(data.name!, fn2());
+        workerPostMessage({
+          type: "inited",
+          name: data.name,
+        });
+        break;
+      case "invoke":
+        let fn = loadedFunctions.get(data.name!);
+        if (!fn) {
+          throw new Error(`Function not loaded: ${data.name}`);
+        }
+        try {
+          let result = await Promise.resolve(fn(...(data.args || [])));
+          workerPostMessage({
+            type: "result",
+            id: data.id,
+            result: result,
+          } as ControllerMessage);
+        } catch (e: any) {
+          workerPostMessage({
+            type: "result",
+            id: data.id,
+            error: e.message,
+          });
+          throw e;
+        }
+
+        break;
+      case "syscall-response":
+        let syscallId = data.id!;
+        const lookup = pendingRequests.get(syscallId);
+        if (!lookup) {
+          console.log(
+            "Current outstanding requests",
+            pendingRequests,
+            "looking up",
+            syscallId
+          );
+          throw Error("Invalid request id");
+        }
+        pendingRequests.delete(syscallId);
+        if (data.error) {
+          lookup.reject(new Error(data.error));
+        } else {
+          lookup.resolve(data.result);
+        }
+        break;
+    }
+  });
+});
@@ -0,0 +1,37 @@
+import { safeRun } from "../util";
+import { Sandbox } from "../sandbox";
+import { WorkerLike } from "./worker";
+import { Plug } from "../plug";
+
+class WebWorkerWrapper implements WorkerLike {
+  private worker: Worker;
+  onMessage?: (message: any) => Promise<void>;
+  ready: Promise<void>;
+
+  constructor(worker: Worker) {
+    this.worker = worker;
+    this.worker.addEventListener("message", (evt: any) => {
+      let data = evt.data;
+      if (!data) return;
+      safeRun(async () => {
+        await this.onMessage!(data);
+      });
+    });
+    this.ready = Promise.resolve();
+  }
+  postMessage(message: any): void {
+    this.worker.postMessage(message);
+  }
+
+  terminate() {
+    return this.worker.terminate();
+  }
+}
+
+export function createSandbox(plug: Plug<any>) {
+  // ParcelJS will build this file into a worker.
+  let worker = new Worker(new URL("sandbox_worker.ts", import.meta.url), {
+    type: "module",
+  });
+  return new Sandbox(plug, new WebWorkerWrapper(worker));
+}
@@ -0,0 +1,31 @@
+export type ControllerMessageType = "inited" | "result" | "syscall";
+
+export type ControllerMessage = {
+  type: ControllerMessageType;
+  id?: number;
+  name?: string;
+  args?: any[];
+  error?: string;
+  result?: any;
+};
+
+export interface WorkerLike {
+  ready: Promise<void>;
+  onMessage?: (message: any) => Promise<void>;
+
+  postMessage(message: any): void;
+
+  terminate(): void;
+}
+
+export type WorkerMessageType = "load" | "invoke" | "syscall-response";
+
+export type WorkerMessage = {
+  type: WorkerMessageType;
+  id?: number;
+  name?: string;
+  code?: string;
+  args?: any[];
+  result?: any;
+  error?: any;
+};