Dependency builds for plugos

packages/plugos/environments/custom_logger.ts

@@ -43,6 +43,9 @@ export class ConsoleLogger {
         case "number":
           pieces.push("" + val);
           break;
+        case "undefined":
+          pieces.push("undefined");
+          break;
         default:
           try {
             let s = JSON.stringify(val, null, 2);

packages/plugos/environments/node_sandbox.ts

@@ -41,15 +41,11 @@ while (!fs.existsSync(nodeModulesDir + "/node_modules/vm2")) {
   nodeModulesDir = path.dirname(nodeModulesDir);
 }
 
-export function createSandbox(
-  plug: Plug<any>,
-  preloadedModules: string[] = []
-) {
+export function createSandbox(plug: Plug<any>) {
   let worker = new Worker(workerCode, {
     eval: true,
     workerData: {
       nodeModulesPath: path.join(nodeModulesDir, "node_modules"),
-      preloadedModules,
     },
   });
   return new Sandbox(plug, new NodeWorkerWrapper(worker));

packages/plugos/environments/node_worker.ts

@@ -2,7 +2,7 @@ import { ConsoleLogger } from "./custom_logger";
 
 const {
   parentPort,
-  workerData: { preloadedModules, nodeModulesPath },
+  workerData: { nodeModulesPath },
 } = require("worker_threads");
 
 const { VM, VMScript } = require(`${nodeModulesPath}/vm2`);
@@ -26,6 +26,8 @@ let consoleLogger = new ConsoleLogger((level, message) => {
   });
 }, false);
 
+let loadedModules = new Map<string, any>();
+
 let vm = new VM({
   sandbox: {
     // Exposing some "safe" APIs
@@ -39,9 +41,14 @@ let vm = new VM({
     // This is only going to be called for pre-bundled modules, we won't allow
     // arbitrary requiring of modules
     require: (moduleName: string): any => {
-      // console.log("Loading", moduleName);
-      if (preloadedModules.includes(moduleName)) {
-        return require(`${nodeModulesPath}/${moduleName}`);
+      // console.log("Loading module", moduleName);
+      // if (preloadedModules.includes(moduleName)) {
+      //   return require(`${nodeModulesPath}/${moduleName}`);
+      // } else
+      if (loadedModules.has(moduleName)) {
+        let mod = loadedModules.get(moduleName);
+        // console.log("And it has the value", mod);
+        return mod;
       } else {
         throw Error(`Cannot import arbitrary modules like ${moduleName}`);
       }
@@ -84,6 +91,20 @@ parentPort.on("message", (data: any) => {
           name: data.name,
         });
         break;
+      case "load-dependency":
+        // console.log("Asked to load dep", data.name);
+        try {
+          let r = vm.run(data.code);
+          // console.log("Loaded dependency", r);
+          loadedModules.set(data.name, r);
+          parentPort.postMessage({
+            type: "dependency-inited",
+            name: data.name,
+          });
+        } catch (e: any) {
+          console.error("Could not load dependency", e.message);
+        }
+        break;
       case "invoke":
         let fn = loadedFunctions.get(data.name);
         if (!fn) {

packages/plugos/environments/sandbox_worker.ts

@@ -39,19 +39,12 @@ self.syscall = async (name: string, ...args: any[]) => {
   });
 };
 
-const preloadedModules: { [key: string]: any } = {
-  "@lezer/lr": require("@lezer/lr"),
-  yaml: require("yaml"),
-  handlebars: require("handlebars/dist/handlebars"),
-};
-// for (const moduleName of preloadModules) {
-//   preloadedModules[moduleName] = require(moduleName);
-// }
+let loadedModules = new Map<string, any>();
 
 // @ts-ignore
 self.require = (moduleName: string): any => {
-  // console.log("Loading", moduleName, preloadedModules[moduleName]);
-  return preloadedModules[moduleName];
+  // console.log("Loading", moduleName, loadedModules.get(moduleName));
+  return loadedModules.get(moduleName);
 };
 
 // @ts-ignore
@@ -75,6 +68,17 @@ self.addEventListener("message", (event: { data: WorkerMessage }) => {
           name: data.name,
         });
         break;
+      case "load-dependency":
+        // console.log("Received dep", data.name);
+        let fn3 = new Function(`return ${data.code!}`);
+        let v = fn3();
+        loadedModules.set(data.name!, v);
+        // console.log("Dep val", v);
+        workerPostMessage({
+          type: "dependency-inited",
+          name: data.name,
+        });
+        break;
       case "invoke":
         let fn = loadedFunctions.get(data.name!);
         if (!fn) {

packages/plugos/environments/worker.ts

@@ -1,6 +1,12 @@
 import type { LogLevel } from "./custom_logger";
 
-export type ControllerMessageType = "inited" | "result" | "syscall" | "log";
+export type ControllerMessageType =
+  | "inited"
+  | "dependency-inited"
+  | "result"
+  | "syscall"
+  | "log";
+
 export type ControllerMessage = {
   type: ControllerMessageType;
   id?: number;
@@ -21,7 +27,11 @@ export interface WorkerLike {
   terminate(): void;
 }
 
-export type WorkerMessageType = "load" | "invoke" | "syscall-response";
+export type WorkerMessageType =
+  | "load"
+  | "load-dependency"
+  | "invoke"
+  | "syscall-response";
 
 export type WorkerMessage = {
   type: WorkerMessageType;