Refactoring

plugbox/environment/iframe_sandbox.ts

@@ -3,8 +3,8 @@ import { safeRun } from "../util";
 // @ts-ignore
 import sandboxHtml from "bundle-text:./iframe_sandbox.html";
 import { Sandbox } from "../sandbox";
-import { System } from "../system";
 import { WorkerLike } from "./worker";
+import { Plug } from "../plug";
 
 class IFrameWrapper implements WorkerLike {
   private iframe: HTMLIFrameElement;
@@ -49,6 +49,6 @@ class IFrameWrapper implements WorkerLike {
   }
 }
 
-export function createSandbox(system: System<any>) {
-  return new Sandbox(system, new IFrameWrapper());
+export function createSandbox(plug: Plug<any>) {
+  return new Sandbox(plug, new IFrameWrapper());
 }

plugbox/environment/node_sandbox.ts

@@ -4,8 +4,8 @@ import { safeRun } from "../util";
 // @ts-ignore
 import workerCode from "bundle-text:./node_worker.ts";
 import { Sandbox } from "../sandbox";
-import { System } from "../system";
 import { WorkerLike } from "./worker";
+import { Plug } from "../plug";
 
 class NodeWorkerWrapper implements WorkerLike {
   onMessage?: (message: any) => Promise<void>;
@@ -33,12 +33,12 @@ class NodeWorkerWrapper implements WorkerLike {
   }
 }
 
-export function createSandbox(system: System<any>) {
+export function createSandbox(plug: Plug<any>) {
   let worker = new Worker(workerCode, {
     eval: true,
   });
   return new Sandbox(
-    system,
+    plug,
     new NodeWorkerWrapper(
       new Worker(workerCode, {
         eval: true,

plugbox/environment/webworker_sandbox.ts

@@ -1,7 +1,7 @@
 import { safeRun } from "../util";
 import { Sandbox } from "../sandbox";
-import { System } from "../system";
 import { WorkerLike } from "./worker";
+import { Plug } from "../plug";
 
 class WebWorkerWrapper implements WorkerLike {
   private worker: Worker;
@@ -28,10 +28,10 @@ class WebWorkerWrapper implements WorkerLike {
   }
 }
 
-export function createSandbox(system: System<any>) {
+export function createSandbox(plug: Plug<any>) {
   // ParcelJS will build this file into a worker.
   let worker = new Worker(new URL("sandbox_worker.ts", import.meta.url), {
     type: "module",
   });
-  return new Sandbox(system, new WebWorkerWrapper(worker));
+  return new Sandbox(plug, new WebWorkerWrapper(worker));
 }

plugbox/feature/endpoint.test.ts

@@ -27,7 +27,7 @@ test("Run a plugbox endpoint server", async () => {
         endpoints: [{ method: "GET", path: "/", handler: "testhandler" }],
       },
     } as Manifest<EndpointHook>,
-    createSandbox(system)
+    createSandbox
   );
 
   const app = express();

plugbox/feature/event.ts

@@ -0,0 +1,43 @@
+import { Feature, Manifest } from "../types";
+import { System } from "../system";
+
+export type EventHook = {
+  events?: { [key: string]: string[] };
+};
+
+export class EventFeature implements Feature<EventHook> {
+  private system?: System<EventHook>;
+
+  async dispatchEvent(name: string, data?: any): Promise<any[]> {
+    if (!this.system) {
+      throw new Error("EventFeature is not initialized");
+    }
+    let promises: Promise<any>[] = [];
+    for (const plug of this.system.loadedPlugs.values()) {
+      if (!plug.manifest!.hooks?.events) {
+        continue;
+      }
+      let functionsToSpawn = plug.manifest!.hooks.events[name];
+      if (functionsToSpawn) {
+        functionsToSpawn.forEach((functionToSpawn) => {
+          // Only dispatch functions on events when they're allowed to be invoked in this environment
+          if (plug.canInvoke(functionToSpawn)) {
+            promises.push(plug.invoke(functionToSpawn, [data]));
+          }
+        });
+      }
+    }
+    return Promise.all(promises);
+  }
+
+  apply(system: System<EventHook>): void {
+    this.system = system;
+    system.on({
+      plugLoaded: (name, plug) => {},
+    });
+  }
+
+  validateManifest(manifest: Manifest<EventHook>): string[] {
+    return [];
+  }
+}

plugbox/plug.ts

@@ -7,16 +7,28 @@ export class Plug<HookT> {
   sandbox: Sandbox;
   public manifest?: Manifest<HookT>;
   readonly runtimeEnv: RuntimeEnvironment;
+  grantedPermissions: string[] = [];
+  name: string;
 
-  constructor(system: System<HookT>, name: string, sandbox: Sandbox) {
+  constructor(
+    system: System<HookT>,
+    name: string,
+    sandboxFactory: (plug: Plug<HookT>) => Sandbox
+  ) {
     this.system = system;
-    this.sandbox = sandbox;
+    this.name = name;
+    this.sandbox = sandboxFactory(this);
     this.runtimeEnv = system.runtimeEnv;
   }
 
   async load(manifest: Manifest<HookT>) {
     this.manifest = manifest;
-    await this.dispatchEvent("load");
+    // TODO: These need to be explicitly granted, not just taken
+    this.grantedPermissions = manifest.requiredPermissions || [];
+  }
+
+  syscall(name: string, args: any[]): Promise<any> {
+    return this.system.syscallWithContext({ plug: this }, name, args);
   }
 
   canInvoke(name: string) {
@@ -46,27 +58,6 @@ export class Plug<HookT> {
     return await this.sandbox.invoke(name, args);
   }
 
-  async dispatchEvent(name: string, data?: any): Promise<any[]> {
-    if (!this.manifest!.hooks?.events) {
-      return [];
-    }
-    let functionsToSpawn = this.manifest!.hooks.events[name];
-    if (functionsToSpawn) {
-      return await Promise.all(
-        functionsToSpawn.map((functionToSpawn: string) => {
-          // Only dispatch functions on events when they're allowed to be invoked in this environment
-          if (this.canInvoke(functionToSpawn)) {
-            return this.invoke(functionToSpawn, [data]);
-          } else {
-            return Promise.resolve();
-          }
-        })
-      );
-    } else {
-      return [];
-    }
-  }
-
   async stop() {
     this.sandbox.stop();
   }

plugbox/plug_loader.ts

@@ -20,9 +20,7 @@ export class DiskPlugLoader<HookT> {
 
   watcher() {
     safeRun(async () => {
-      for await (const { filename, eventType } of watch(this.plugPath, {
-        recursive: true,
-      })) {
+      for await (const { filename, eventType } of watch(this.plugPath)) {
         if (!filename.endsWith(".plug.json")) {
           return;
         }
@@ -50,7 +48,7 @@ export class DiskPlugLoader<HookT> {
     console.log("Now loading plug", plugName);
     try {
       const plugDef = JSON.parse(plug);
-      await this.system.load(plugName, plugDef, createSandbox(this.system));
+      await this.system.load(plugName, plugDef, createSandbox);
       return plugDef;
     } catch (e) {
       console.error("Could not parse plugin file", e);

plugbox/runtime.test.ts

@@ -4,17 +4,28 @@ import { System } from "./system";
 
 test("Run a Node sandbox", async () => {
   let system = new System("server");
-  system.registerSyscalls({
-    addNumbers: (a, b) => {
+  system.registerSyscalls("", [], {
+    addNumbers: (ctx, a, b) => {
       return a + b;
     },
     failingSyscall: () => {
       throw new Error("#fail");
     },
   });
+  system.registerSyscalls("", ["restricted"], {
+    restrictedSyscall: () => {
+      return "restricted";
+    },
+  });
+  system.registerSyscalls("", ["dangerous"], {
+    dangerousSyscall: () => {
+      return "yay";
+    },
+  });
   let plug = await system.load(
     "test",
     {
+      requiredPermissions: ["dangerous"],
       functions: {
         addTen: {
           code: `(() => {
@@ -52,12 +63,30 @@ test("Run a Node sandbox", async () => {
           };
         })()`,
         },
+        restrictedTest: {
+          code: `(() => {
+          return {
+            default: async () => {
+              await self.syscall("restrictedSyscall");
+            }
+          };
+        })()`,
+        },
+        dangerousTest: {
+          code: `(() => {
+          return {
+            default: async () => {
+              return await self.syscall("dangerousSyscall");
+            }
+          };
+        })()`,
+        },
       },
       hooks: {
         events: {},
       },
     },
-    createSandbox(system)
+    createSandbox
   );
   expect(await plug.invoke("addTen", [10])).toBe(20);
   for (let i = 0; i < 100; i++) {
@@ -75,5 +104,15 @@ test("Run a Node sandbox", async () => {
   } catch (e: any) {
     expect(e.message).toBe("#fail");
   }
+  try {
+    await plug.invoke("restrictedTest", []);
+    expect(true).toBe(false);
+  } catch (e: any) {
+    expect(e.message).toBe(
+      "Missing permission 'restricted' for syscall restrictedSyscall"
+    );
+  }
+  expect(await plug.invoke("dangerousTest", [])).toBe("yay");
+
   await system.unloadAll();
 });

plugbox/sandbox.ts

@@ -1,9 +1,11 @@
-import { System } from "./system";
 import {
   ControllerMessage,
   WorkerLike,
   WorkerMessage,
 } from "./environment/worker";
+import { Plug } from "./plug";
+
+export type SandboxFactory<HookT> = (plug: Plug<HookT>) => Sandbox;
 
 export class Sandbox {
   protected worker: WorkerLike;
@@ -14,12 +16,12 @@ export class Sandbox {
     { resolve: (result: any) => void; reject: (e: any) => void }
   >();
   protected loadedFunctions = new Set<string>();
-  protected system: System<any>;
+  protected plug: Plug<any>;
 
-  constructor(system: System<any>, worker: WorkerLike) {
+  constructor(plug: Plug<any>, worker: WorkerLike) {
     worker.onMessage = this.onMessage.bind(this);
     this.worker = worker;
-    this.system = system;
+    this.plug = plug;
   }
 
   isLoaded(name: string) {
@@ -48,7 +50,7 @@ export class Sandbox {
         break;
       case "syscall":
         try {
-          let result = await this.system.syscall(data.name!, data.args!);
+          let result = await this.plug.syscall(data.name!, data.args!);
 
           this.worker.postMessage({
             type: "syscall-response",

plugbox/syscall/fetch.node.ts

@@ -0,0 +1,15 @@
+import fetch, { RequestInfo, RequestInit } from "node-fetch";
+import { SysCallMapping } from "../system";
+
+export function fetchSyscalls(): SysCallMapping {
+  return {
+    async fetchJson(ctx, url: RequestInfo, init: RequestInit) {
+      let resp = await fetch(url, init);
+      return resp.json();
+    },
+    async fetchText(ctx, url: RequestInfo, init: RequestInit) {
+      let resp = await fetch(url, init);
+      return resp.text();
+    },
+  };
+}

plugbox/syscall/shell.node.ts

@@ -0,0 +1,20 @@
+import { promisify } from "util";
+import { execFile } from "child_process";
+import type { SysCallMapping } from "../system";
+
+const execFilePromise = promisify(execFile);
+
+export default function (cwd: string): SysCallMapping {
+  return {
+    run: async (
+      ctx,
+      cmd: string,
+      args: string[]
+    ): Promise<{ stdout: string; stderr: string }> => {
+      let { stdout, stderr } = await execFilePromise(cmd, args, {
+        cwd: cwd,
+      });
+      return { stdout, stderr };
+    },
+  };
+}

plugbox/syscall/store.dexie_browser.test.ts

@@ -0,0 +1,50 @@
+import { createSandbox } from "../environment/node_sandbox";
+import { expect, test } from "@jest/globals";
+import { System } from "../system";
+import { storeSyscalls } from "./store.dexie_browser";
+
+// For testing in node.js
+require("fake-indexeddb/auto");
+
+test("Test store", async () => {
+  let system = new System("server");
+  system.registerSyscalls("store", [], storeSyscalls("test", "test"));
+  let plug = await system.load(
+    "test",
+    {
+      hooks: {},
+      functions: {
+        test1: {
+          code: `(() => {
+          return {
+            default: async () => {
+              await self.syscall("store.set", "name", "Pete");
+              return await self.syscall("store.get", "name");
+            }
+          };
+        })()`,
+        },
+        test2: {
+          code: `(() => {
+          return {
+            default: async () => {
+              await self.syscall("store.set", "page1:bl:page2:10", {title: "Something", meta: 20});
+              await self.syscall("store.batchSet", [
+                 {key: "page2:bl:page3", value: {title: "Something2", meta: 10}},
+                 {key: "page2:bl:page4", value: {title: "Something3", meta: 10}},
+              ]);
+              return await self.syscall("store.queryPrefix", "page2:");
+            }
+          };
+        })()`,
+        },
+      },
+    },
+    createSandbox
+  );
+  expect(await plug.invoke("test1", [])).toBe("Pete");
+  let queryResults = await plug.invoke("test2", []);
+  expect(queryResults.length).toBe(2);
+  expect(queryResults[0].value.meta).toBe(10);
+  await system.unloadAll();
+});

plugbox/syscall/store.dexie_browser.ts

@@ -0,0 +1,66 @@
+import Dexie from "dexie";
+import { SysCallMapping } from "../system";
+
+export type KV = {
+  key: string;
+  value: any;
+};
+
+export function storeSyscalls(
+  dbName: string,
+  tableName: string
+): SysCallMapping {
+  const db = new Dexie(dbName);
+  db.version(1).stores({
+    test: "key",
+  });
+  const items = db.table(tableName);
+
+  return {
+    async delete(ctx, key: string) {
+      await items.delete(key);
+    },
+
+    async deletePrefix(ctx, prefix: string) {
+      await items.where("key").startsWith(prefix).delete();
+    },
+
+    async deleteAll() {
+      await items.clear();
+    },
+
+    async set(ctx, key: string, value: any) {
+      await items.put({
+        key,
+        value,
+      });
+    },
+
+    async batchSet(ctx, kvs: KV[]) {
+      await items.bulkPut(
+        kvs.map(({ key, value }) => ({
+          key,
+          value,
+        }))
+      );
+    },
+
+    async get(ctx, key: string): Promise<any | null> {
+      let result = await items.get({
+        key,
+      });
+      return result ? result.value : null;
+    },
+
+    async queryPrefix(
+      ctx,
+      keyPrefix: string
+    ): Promise<{ key: string; value: any }[]> {
+      let results = await items.where("key").startsWith(keyPrefix).toArray();
+      return results.map((result) => ({
+        key: result.key,
+        value: result.value,
+      }));
+    },
+  };
+}

plugbox/syscall/store.knex_node.test.ts

@@ -0,0 +1,50 @@
+import { createSandbox } from "../environment/node_sandbox";
+import { expect, test } from "@jest/globals";
+import { System } from "../system";
+import {
+  ensureTable,
+  storeReadSyscalls,
+  storeWriteSyscalls,
+} from "./store.knex_node";
+import knex from "knex";
+import fs from "fs/promises";
+
+test("Test store", async () => {
+  const db = knex({
+    client: "better-sqlite3",
+    connection: {
+      filename: "test.db",
+    },
+    useNullAsDefault: true,
+  });
+  await ensureTable(db, "test_table");
+  let system = new System("server");
+  system.registerSyscalls(
+    "store",
+    [],
+    storeWriteSyscalls(db, "test_table"),
+    storeReadSyscalls(db, "test_table")
+  );
+  let plug = await system.load(
+    "test",
+    {
+      hooks: {},
+      functions: {
+        test1: {
+          code: `(() => {
+          return {
+            default: async () => {
+              await self.syscall("store.set", "name", "Pete");
+              return await self.syscall("store.get", "name");
+            }
+          };
+        })()`,
+        },
+      },
+    },
+    createSandbox
+  );
+  expect(await plug.invoke("test1", [])).toBe("Pete");
+  await system.unloadAll();
+  await fs.unlink("test.db");
+});

plugbox/syscall/store.knex_node.ts

@@ -0,0 +1,84 @@
+import { Knex } from "knex";
+import { SysCallMapping } from "../system";
+
+type Item = {
+  page: string;
+  key: string;
+  value: any;
+};
+
+export type KV = {
+  key: string;
+  value: any;
+};
+
+export async function ensureTable(db: Knex<any, unknown>, tableName: string) {
+  if (!(await db.schema.hasTable(tableName))) {
+    await db.schema.createTable(tableName, (table) => {
+      table.string("key");
+      table.text("value");
+      table.primary(["key"]);
+    });
+    console.log(`Created table ${tableName}`);
+  }
+}
+
+export function storeWriteSyscalls(
+  db: Knex<any, unknown>,
+  tableName: string
+): SysCallMapping {
+  const apiObj: SysCallMapping = {
+    delete: async (ctx, page: string, key: string) => {
+      await db<Item>(tableName).where({ page, key }).del();
+    },
+    deletePrefix: async (ctx, prefix: string) => {
+      return db<Item>(tableName).andWhereLike("key", `${prefix}%`).del();
+    },
+    deleteAll: async (ctx) => {
+      await db<Item>(tableName).del();
+    },
+    set: async (ctx, key: string, value: any) => {
+      let changed = await db<Item>(tableName)
+        .where({ key })
+        .update("value", JSON.stringify(value));
+      if (changed === 0) {
+        await db<Item>(tableName).insert({
+          key,
+          value: JSON.stringify(value),
+        });
+      }
+    },
+    batchSet: async (ctx, kvs: KV[]) => {
+      for (let { key, value } of kvs) {
+        await apiObj["store.set"](ctx, key, value);
+      }
+    },
+  };
+  return apiObj;
+}
+
+export function storeReadSyscalls(
+  db: Knex<any, unknown>,
+  tableName: string
+): SysCallMapping {
+  return {
+    get: async (ctx, key: string): Promise<any | null> => {
+      let result = await db<Item>(tableName).where({ key }).select("value");
+      if (result.length) {
+        return JSON.parse(result[0].value);
+      } else {
+        return null;
+      }
+    },
+    queryPrefix: async (ctx, prefix: string) => {
+      return (
+        await db<Item>(tableName)
+          .andWhereLike("key", `${prefix}%`)
+          .select("key", "value")
+      ).map(({ key, value }) => ({
+        key,
+        value: JSON.parse(value),
+      }));
+    },
+  };
+}

plugbox/syscall/transport.ts

@@ -0,0 +1,16 @@
+import { SysCallMapping } from "../system";
+
+export function transportSyscalls(
+  names: string[],
+  transportCall: (name: string, ...args: any[]) => Promise<any>
+): SysCallMapping {
+  let syscalls: SysCallMapping = {};
+
+  for (let name of names) {
+    syscalls[name] = (ctx, ...args: any[]) => {
+      return transportCall(name, ...args);
+    };
+  }
+
+  return syscalls;
+}

plugbox/system.ts

@@ -1,10 +1,10 @@
 import { Feature, Manifest, RuntimeEnvironment } from "./types";
 import { EventEmitter } from "../common/event";
-import { Sandbox } from "./sandbox";
+import { SandboxFactory } from "./sandbox";
 import { Plug } from "./plug";
 
 export interface SysCallMapping {
-  [key: string]: (...args: any) => Promise<any> | any;
+  [key: string]: (ctx: SyscallContext, ...args: any) => Promise<any> | any;
 }
 
 export type SystemJSON<HookT> = { [key: string]: Manifest<HookT> };
@@ -14,9 +14,23 @@ export type SystemEvents<HookT> = {
   plugUnloaded: (name: string, plug: Plug<HookT>) => void;
 };
 
+type SyscallContext = {
+  plug: Plug<any> | null;
+};
+
+type SyscallSignature = (
+  ctx: SyscallContext,
+  ...args: any[]
+) => Promise<any> | any;
+
+type Syscall = {
+  requiredPermissions: string[];
+  callback: SyscallSignature;
+};
+
 export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
   protected plugs = new Map<string, Plug<HookT>>();
-  registeredSyscalls: SysCallMapping = {};
+  protected registeredSyscalls = new Map<string, Syscall>();
   protected enabledFeatures = new Set<Feature<HookT>>();
 
   readonly runtimeEnv: RuntimeEnvironment;
@@ -31,29 +45,46 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
     feature.apply(this);
   }
 
-  registerSyscalls(...registrationObjects: SysCallMapping[]) {
+  registerSyscalls(
+    namespace: string,
+    requiredCapabilities: string[],
+    ...registrationObjects: SysCallMapping[]
+  ) {
     for (const registrationObject of registrationObjects) {
-      for (let [name, def] of Object.entries(registrationObject)) {
-        this.registeredSyscalls[name] = def;
+      for (let [name, callback] of Object.entries(registrationObject)) {
+        const callName = namespace ? `${namespace}.${name}` : name;
+        this.registeredSyscalls.set(callName, {
+          requiredPermissions: requiredCapabilities,
+          callback,
+        });
       }
     }
   }
 
-  async syscall(name: string, args: any[]): Promise<any> {
-    const callback = this.registeredSyscalls[name];
-    if (!name) {
+  async syscallWithContext(
+    ctx: SyscallContext,
+    name: string,
+    args: any[]
+  ): Promise<any> {
+    const syscall = this.registeredSyscalls.get(name);
+    if (!syscall) {
       throw Error(`Unregistered syscall ${name}`);
     }
-    if (!callback) {
-      throw Error(`Registered but not implemented syscall ${name}`);
+    for (const permission of syscall.requiredPermissions) {
+      if (!ctx.plug) {
+        throw Error(`Syscall ${name} requires permission and no plug is set`);
+      }
+      if (!ctx.plug.grantedPermissions.includes(permission)) {
+        throw Error(`Missing permission '${permission}' for syscall ${name}`);
+      }
     }
-    return Promise.resolve(callback(...args));
+    return Promise.resolve(syscall.callback(ctx, ...args));
   }
 
   async load(
     name: string,
     manifest: Manifest<HookT>,
-    sandbox: Sandbox
+    sandboxFactory: SandboxFactory<HookT>
   ): Promise<Plug<HookT>> {
     if (this.plugs.has(name)) {
       await this.unload(name);
@@ -67,7 +98,7 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
       throw new Error(`Invalid manifest: ${errors.join(", ")}`);
     }
     // Ok, let's load this thing!
-    const plug = new Plug(this, name, sandbox);
+    const plug = new Plug(this, name, sandboxFactory);
     await plug.load(manifest);
     this.plugs.set(name, plug);
     this.emit("plugLoaded", name, plug);
@@ -84,16 +115,6 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
     this.plugs.delete(name);
   }
 
-  async dispatchEvent(name: string, data?: any): Promise<any[]> {
-    let promises = [];
-    for (let plug of this.plugs.values()) {
-      for (let result of await plug.dispatchEvent(name, data)) {
-        promises.push(result);
-      }
-    }
-    return await Promise.all(promises);
-  }
-
   get loadedPlugs(): Map<string, Plug<HookT>> {
     return this.plugs;
   }
@@ -111,12 +132,12 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
 
   async replaceAllFromJSON(
     json: SystemJSON<HookT>,
-    sandboxFactory: () => Sandbox
+    sandboxFactory: SandboxFactory<HookT>
   ) {
     await this.unloadAll();
     for (let [name, manifest] of Object.entries(json)) {
       console.log("Loading plug", name);
-      await this.load(name, manifest, sandboxFactory());
+      await this.load(name, manifest, sandboxFactory);
     }
   }
 

plugbox/types.ts

@@ -1,7 +1,8 @@
 import { System } from "./system";
 
 export interface Manifest<HookT> {
-  hooks: HookT & EventHook;
+  requiredPermissions?: string[];
+  hooks: HookT;
   functions: {
     [key: string]: FunctionDef;
   };
@@ -15,10 +16,6 @@ export interface FunctionDef {
 
 export type RuntimeEnvironment = "client" | "server";
 
-export type EventHook = {
-  events?: { [key: string]: string[] };
-};
-
 export interface Feature<HookT> {
   validateManifest(manifest: Manifest<HookT>): string[];