Pre plug format change

plugbox/bin/plugbox-bundle.ts

@@ -7,6 +7,7 @@ import path from "path";
 import yargs from "yargs";
 import { hideBin } from "yargs/helpers";
 import { Manifest } from "../types";
+import { watchFile } from "fs";
 
 async function compile(filePath: string, functionName: string, debug: boolean) {
   let outFile = "out.js";
@@ -63,18 +64,49 @@ async function bundle(manifestPath: string, sourceMaps: boolean) {
   }
   return manifest;
 }
+
+async function buildManifest(
+  manifestPath: string,
+  distPath: string,
+  debug: boolean
+) {
+  let generatedManifest = await bundle(manifestPath, debug);
+  const outPath = path.join(distPath, path.basename(manifestPath));
+  console.log("Emitting bundle to", outPath);
+  await writeFile(outPath, JSON.stringify(generatedManifest, null, 2));
+  return { generatedManifest, outPath };
+}
+
 async function run() {
-  let args = await yargs(hideBin(process.argv))
+  let args = yargs(hideBin(process.argv))
     .option("debug", {
       type: "boolean",
     })
+    .option("watch", {
+      type: "boolean",
+      alias: "w",
+    })
+    .option("dist", {
+      type: "string",
+      default: ".",
+    })
     .parse();
-
-  let generatedManifest = await bundle(args._[0] as string, !!args.debug);
-  await writeFile(
-    args._[1] as string,
-    JSON.stringify(generatedManifest, null, 2)
-  );
+  if (args._.length === 0) {
+    console.log(
+      "Usage: plugbox-bundle [--debug] [--dist <path>] <manifest.plug.json> <manifest2.plug.json> ..."
+    );
+    process.exit(1);
+  }
+  for (const plugManifestPath of args._) {
+    let manifestPath = plugManifestPath as string;
+    await buildManifest(manifestPath, args.dist, !!args.debug);
+    if (args.watch) {
+      watchFile(manifestPath, { interval: 1000 }, async () => {
+        console.log("Rebuilding", manifestPath);
+        await buildManifest(manifestPath, args.dist, !!args.debug);
+      });
+    }
+  }
 }
 
 run().catch((e) => {

plugbox/bin/plugbox-server.ts

@@ -0,0 +1,66 @@
+#!/usr/bin/env node
+
+import express from "express";
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import { DiskPlugLoader } from "../plug_loader";
+import { CronHook, NodeCronFeature } from "../feature/node_cron";
+import shellSyscalls from "../syscall/shell.node";
+import { System } from "../system";
+import { EndpointFeature, EndpointHook } from "../feature/endpoint";
+import { safeRun } from "../util";
+import knex from "knex";
+import {
+  ensureTable,
+  storeReadSyscalls,
+  storeWriteSyscalls,
+} from "../syscall/store.knex_node";
+import { fetchSyscalls } from "../syscall/fetch.node";
+
+let args = yargs(hideBin(process.argv))
+  .option("port", {
+    type: "number",
+    default: 1337,
+  })
+  .parse();
+
+if (!args._.length) {
+  console.error("Usage: plugbox-server <path-to-plugs>");
+  process.exit(1);
+}
+
+const plugPath = args._[0] as string;
+
+const app = express();
+
+type ServerHook = EndpointHook & CronHook;
+const system = new System<ServerHook>("server");
+
+safeRun(async () => {
+  const db = knex({
+    client: "better-sqlite3",
+    connection: {
+      filename: "plugbox.db",
+    },
+    useNullAsDefault: true,
+  });
+
+  await ensureTable(db, "item");
+
+  let plugLoader = new DiskPlugLoader(system, plugPath);
+  await plugLoader.loadPlugs();
+  plugLoader.watcher();
+  system.addFeature(new NodeCronFeature());
+  system.addFeature(new EndpointFeature(app, ""));
+  system.registerSyscalls("shell", [], shellSyscalls("."));
+  system.registerSyscalls("fetch", [], fetchSyscalls());
+  system.registerSyscalls(
+    "store",
+    [],
+    storeWriteSyscalls(db, "item"),
+    storeReadSyscalls(db, "item")
+  );
+  app.listen(args.port, () => {
+    console.log(`Plugbox server listening on port ${args.port}`);
+  });
+});

plugbox/environment/node_sandbox.ts

@@ -6,6 +6,8 @@ import workerCode from "bundle-text:./node_worker.ts";
 import { Sandbox } from "../sandbox";
 import { WorkerLike } from "./worker";
 import { Plug } from "../plug";
+import path from "path";
+import fs from "fs";
 
 class NodeWorkerWrapper implements WorkerLike {
   onMessage?: (message: any) => Promise<void>;
@@ -33,16 +35,16 @@ class NodeWorkerWrapper implements WorkerLike {
   }
 }
 
+// Look for the node_modules directory, to be passed to the worker to find e.g. the vm2 module
+let nodeModulesDir = __dirname;
+while (!fs.existsSync(nodeModulesDir + "/node_modules")) {
+  nodeModulesDir = path.dirname(nodeModulesDir);
+}
+
 export function createSandbox(plug: Plug<any>) {
   let worker = new Worker(workerCode, {
     eval: true,
+    workerData: path.join(nodeModulesDir, "node_modules"),
   });
-  return new Sandbox(
-    plug,
-    new NodeWorkerWrapper(
-      new Worker(workerCode, {
-        eval: true,
-      })
-    )
-  );
+  return new Sandbox(plug, new NodeWorkerWrapper(worker));
 }

plugbox/environment/node_worker.ts

@@ -1,5 +1,8 @@
-const { VM, VMScript } = require("vm2");
-const { parentPort } = require("worker_threads");
+const { parentPort, workerData } = require("worker_threads");
+let vm2 = `${workerData}/vm2`;
+const { VM, VMScript } = require(vm2);
+
+// console.log("Process env", process.env);
 
 let loadedFunctions = new Map<string, Function>();
 let pendingRequests = new Map<

plugbox/feature/endpoint.test.ts

@@ -33,7 +33,7 @@ test("Run a plugbox endpoint server", async () => {
   const app = express();
   const port = 3123;
 
-  system.addFeature(new EndpointFeature(app));
+  system.addFeature(new EndpointFeature(app, "/_"));
 
   let server = app.listen(port, () => {
     console.log(`Listening on port ${port}`);

plugbox/feature/endpoint.ts

@@ -26,20 +26,21 @@ export type EndPointDef = {
   handler: string; // function name
 };
 
-const endPointPrefix = "/_";
-
 export class EndpointFeature implements Feature<EndpointHook> {
   private app: Express;
+  private prefix: string;
 
-  constructor(app: Express) {
+  constructor(app: Express, prefix: string) {
     this.app = app;
+    this.prefix = prefix;
   }
 
   apply(system: System<EndpointHook>): void {
     this.app.use((req: Request, res: Response, next: NextFunction) => {
-      if (!req.path.startsWith(endPointPrefix)) {
+      if (!req.path.startsWith(this.prefix)) {
         return next();
       }
+      console.log("Endpoint request", req.path);
       Promise.resolve()
         .then(async () => {
           for (const [plugName, plug] of system.loadedPlugs.entries()) {
@@ -48,8 +49,10 @@ export class EndpointFeature implements Feature<EndpointHook> {
               continue;
             }
             const endpoints = manifest.hooks?.endpoints;
+            console.log("Checking plug", plugName, endpoints);
             if (endpoints) {
-              let prefix = `${endPointPrefix}/${plugName}`;
+              let prefix = `${this.prefix}/${plugName}`;
+              console.log("Need prefix", prefix, "got", req.path);
               if (!req.path.startsWith(prefix)) {
                 continue;
               }