Migrate to Deno (#86)

Big bang migration to Deno 🤯

plugos/asset_bundle.ts

@@ -0,0 +1,51 @@
+import { mime } from "../server/deps.ts";
+import { AssetBundle } from "./asset_bundle_reader.ts";
+import { base64Encode } from "./base64.ts";
+import { globToRegExp, path, walk } from "./deps.ts";
+
+export async function bundleAssets(
+  rootPath: string,
+  patterns: string[],
+): Promise<AssetBundle> {
+  const bundle: AssetBundle = {};
+  for await (
+    const file of walk(rootPath, {
+      match: patterns.map((pat) => globToRegExp(pat)),
+    })
+  ) {
+    await loadIntoBundle(file.path, "", bundle);
+  }
+  return bundle;
+}
+
+export async function bundleFolder(rootPath: string, bundlePath: string) {
+  const bundle: AssetBundle = {};
+  await Deno.mkdir(path.dirname(bundlePath), { recursive: true });
+  for await (
+    const { path: filePath } of walk(rootPath, { includeDirs: false })
+  ) {
+    console.log("Bundling", filePath);
+    await loadIntoBundle(filePath, `${rootPath}/`, bundle);
+  }
+  await Deno.writeTextFile(bundlePath, JSON.stringify(bundle, null, 2));
+}
+
+async function loadIntoBundle(
+  filePath: string,
+  rootPath: string,
+  bundle: AssetBundle,
+) {
+  const b64content = base64Encode(await Deno.readFile(filePath));
+  const s = await Deno.stat(filePath);
+  const cleanPath = filePath.substring(rootPath.length);
+  bundle[cleanPath] = {
+    meta: {
+      name: cleanPath,
+      contentType: mime.getType(cleanPath) || "application/octet-stream",
+      size: s.size,
+      lastModified: s.mtime!.getTime(),
+      perm: "ro",
+    },
+    data: b64content,
+  };
+}

plugos/asset_bundle_reader.ts

@@ -0,0 +1,40 @@
+import { base64Decode } from "./base64.ts";
+
+export type FileMeta = {
+  name: string;
+  lastModified: number;
+  contentType: string;
+  size: number;
+  perm: "ro" | "rw";
+};
+
+export type AssetBundle = Record<string, { meta: FileMeta; data: string }>;
+
+export function assetReadFileSync(
+  bundle: AssetBundle,
+  path: string,
+): ArrayBuffer {
+  const content = bundle[path];
+  if (!content) {
+    throw new Error(`No such file ${path}`);
+  }
+  return base64Decode(content.data);
+}
+
+export function assetStatSync(
+  bundle: AssetBundle,
+  path: string,
+): FileMeta {
+  const content = bundle[path];
+  if (!content) {
+    throw new Error(`No such file ${path}`);
+  }
+  return content.meta;
+}
+
+export function assetReadTextFileSync(
+  bundle: AssetBundle,
+  path: string,
+): string {
+  return new TextDecoder().decode(assetReadFileSync(bundle, path));
+}

plugos/base64.test.ts

@@ -0,0 +1,12 @@
+import { assertEquals } from "../test_deps.ts";
+import { base64Decode } from "./base64.ts";
+import { base64Encode } from "./base64.ts";
+
+Deno.test("Base 64 encoding", () => {
+  const buf = new Uint8Array(3);
+  buf[0] = 1;
+  buf[1] = 2;
+  buf[2] = 3;
+
+  assertEquals(buf, base64Decode(base64Encode(buf)));
+});

plugos/base64.ts

@@ -0,0 +1,20 @@
+import { buf } from "https://deno.land/x/sqlite3@0.6.1/src/util.ts";
+
+export function base64Decode(s: string): Uint8Array {
+  const binString = atob(s);
+  const len = binString.length;
+  const bytes = new Uint8Array(len);
+  for (let i = 0; i < len; i++) {
+    bytes[i] = binString.charCodeAt(i);
+  }
+  return bytes;
+}
+
+export function base64Encode(buffer: Uint8Array): string {
+  let binary = "";
+  const len = buffer.byteLength;
+  for (let i = 0; i < len; i++) {
+    binary += String.fromCharCode(buffer[i]);
+  }
+  return btoa(binary);
+}

plugos/bin/plugos-bundle.ts

@@ -0,0 +1,163 @@
+#!/usr/bin/env deno
+
+import { Manifest } from "../types.ts";
+import { YAML } from "../../common/deps.ts";
+import {
+  compile,
+  CompileOptions,
+  esbuild,
+  sandboxCompileModule,
+} from "../compile.ts";
+import { path } from "../../server/deps.ts";
+
+import * as flags from "https://deno.land/std@0.158.0/flags/mod.ts";
+import { bundleAssets } from "../../plugos/asset_bundle.ts";
+
+export async function bundle(
+  manifestPath: string,
+  options: CompileOptions = {},
+): Promise<Manifest<any>> {
+  const rootPath = path.dirname(manifestPath);
+  const manifest = YAML.parse(
+    await Deno.readTextFile(manifestPath),
+  ) as Manifest<any>;
+
+  if (!manifest.name) {
+    throw new Error(`Missing 'name' in ${manifestPath}`);
+  }
+
+  const allModulesToExclude = options.excludeModules
+    ? options.excludeModules.slice()
+    : [];
+
+  // Dependencies
+  for (let [name, moduleSpec] of Object.entries(manifest.dependencies || {})) {
+    manifest.dependencies![name] = await sandboxCompileModule(moduleSpec);
+    allModulesToExclude.push(name);
+  }
+
+  // Assets
+  const assetsBundle = await bundleAssets(
+    rootPath,
+    manifest.assets as string[] || [],
+  );
+  manifest.assets = assetsBundle;
+
+  // Functions
+
+  for (let [name, def] of Object.entries(manifest.functions || {})) {
+    let jsFunctionName = "default",
+      filePath = path.join(rootPath, def.path!);
+    if (filePath.indexOf(":") !== -1) {
+      [filePath, jsFunctionName] = filePath.split(":");
+    }
+
+    def.code = await compile(
+      filePath,
+      jsFunctionName,
+      {
+        ...options,
+        excludeModules: allModulesToExclude,
+      },
+    );
+    delete def.path;
+  }
+  return manifest;
+}
+
+async function buildManifest(
+  manifestPath: string,
+  distPath: string,
+  options: CompileOptions = {},
+) {
+  const generatedManifest = await bundle(manifestPath, options);
+  const outFile = manifestPath.substring(
+    0,
+    manifestPath.length - path.extname(manifestPath).length,
+  ) + ".json";
+  const outPath = path.join(distPath, path.basename(outFile));
+  console.log("Emitting bundle to", outPath);
+  await Deno.writeTextFile(outPath, JSON.stringify(generatedManifest, null, 2));
+  return { generatedManifest, outPath };
+}
+
+async function bundleRun(
+  manifestFiles: string[],
+  dist: string,
+  watch: boolean,
+  options: CompileOptions = {},
+) {
+  // console.log("Args", arguments);
+  let building = false;
+  async function buildAll() {
+    if (building) {
+      return;
+    }
+    console.log("Building", manifestFiles);
+    building = true;
+    Deno.mkdirSync(dist, { recursive: true });
+    for (const plugManifestPath of manifestFiles) {
+      const manifestPath = plugManifestPath as string;
+      try {
+        await buildManifest(
+          manifestPath,
+          dist,
+          options,
+        );
+      } catch (e) {
+        console.error(`Error building ${manifestPath}:`, e);
+      }
+    }
+    console.log("Done.");
+    building = false;
+  }
+
+  await buildAll();
+
+  if (watch) {
+    const watcher = Deno.watchFs(manifestFiles.map((p) => path.dirname(p)));
+    for await (const event of watcher) {
+      if (event.paths.length > 0) {
+        if (event.paths[0].endsWith(".json")) {
+          continue;
+        }
+      }
+      console.log("Change detected, rebuilding...");
+      buildAll();
+    }
+  }
+}
+
+if (import.meta.main) {
+  const args = flags.parse(Deno.args, {
+    boolean: ["debug", "watch"],
+    string: ["dist", "exclude", "importmap"],
+    alias: { w: "watch" },
+    // collect: ["exclude"],
+  });
+
+  if (args._.length === 0) {
+    console.log(
+      "Usage: plugos-bundle [--debug] [--dist <path>] [--importmap import_map.json] [--exclude=package1,package2] <manifest.plug.yaml> <manifest2.plug.yaml> ...",
+    );
+    Deno.exit(1);
+  }
+
+  if (!args.dist) {
+    args.dist = path.resolve(".");
+  }
+
+  await bundleRun(
+    args._ as string[],
+    args.dist,
+    args.watch,
+    {
+      debug: args.debug,
+      excludeModules: args.exclude ? args.exclude.split(",") : undefined,
+      importMap: args.importmap
+        ? new URL(args.importmap, `file://${Deno.cwd()}/`)
+        : undefined,
+    },
+  );
+  esbuild.stop();
+}

plugos/bin/plugos-server.ts

@@ -0,0 +1,60 @@
+#!/usr/bin/env node
+
+import express from "express";
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import { DiskPlugLoader } from "../plug_loader";
+import { CronHookT, NodeCronHook } from "../hooks/node_cron.ts";
+import shellSyscalls from "../syscalls/shell.node.ts";
+import { System } from "../system.ts";
+import { EndpointHook, EndpointHookT } from "../hooks/endpoint.ts";
+import { safeRun } from "../util.ts";
+import knex from "knex";
+import { ensureTable, storeSyscalls } from "../syscalls/store.knex_node";
+import { EventHook, EventHookT } from "../hooks/event.ts";
+import { eventSyscalls } from "../syscalls/event.ts";
+
+let args = yargs(hideBin(process.argv))
+  .option("port", {
+    type: "number",
+    default: 1337,
+  })
+  .parse();
+
+if (!args._.length) {
+  console.error("Usage: plugos-server <path-to-plugs>");
+  process.exit(1);
+}
+
+const plugPath = args._[0] as string;
+
+const app = express();
+
+type ServerHook = EndpointHookT & CronHookT & EventHookT;
+const system = new System<ServerHook>("server");
+
+safeRun(async () => {
+  const db = knex({
+    client: "better-sqlite3",
+    connection: {
+      filename: "plugos.db",
+    },
+    useNullAsDefault: true,
+  });
+
+  await ensureTable(db, "item");
+
+  let plugLoader = new DiskPlugLoader(system, plugPath);
+  await plugLoader.loadPlugs();
+  plugLoader.watcher();
+  system.addHook(new NodeCronHook());
+  let eventHook = new EventHook();
+  system.addHook(eventHook);
+  system.registerSyscalls([], eventSyscalls(eventHook));
+  system.addHook(new EndpointHook(app, ""));
+  system.registerSyscalls([], shellSyscalls("."));
+  system.registerSyscalls([], storeSyscalls(db, "item"));
+  app.listen(args.port, () => {
+    console.log(`Plugbox server listening on port ${args.port}`);
+  });
+});

plugos/compile.ts

@@ -0,0 +1,131 @@
+// import { esbuild } from "../../mod.ts";
+import * as esbuildWasm from "https://deno.land/x/esbuild@v0.14.54/wasm.js";
+import * as esbuildNative from "https://deno.land/x/esbuild@v0.14.54/mod.js";
+
+export const esbuild: typeof esbuildWasm = Deno.run === undefined
+  ? esbuildWasm
+  : esbuildNative;
+
+import { path } from "../server/deps.ts";
+import { denoPlugin } from "../esbuild_deno_loader/mod.ts";
+import { patchDenoLibJS } from "../common/hack.ts";
+
+export type CompileOptions = {
+  debug?: boolean;
+  excludeModules?: string[];
+  meta?: boolean;
+  importMap?: URL;
+};
+
+export async function compile(
+  filePath: string,
+  functionName: string | undefined = undefined,
+  options: CompileOptions = {},
+): Promise<string> {
+  const outFile = await Deno.makeTempFile({ suffix: ".js" });
+  let inFile = filePath;
+
+  if (functionName) {
+    // Generate a new file importing just this one function and exporting it
+    inFile = await Deno.makeTempFile({ suffix: ".ts" });
+    await Deno.writeTextFile(
+      inFile,
+      `import {${functionName}} from "${
+        path.resolve(filePath)
+      }";export default ${functionName};`,
+    );
+  }
+
+  // console.log("External modules", excludeModules);
+
+  try {
+    // TODO: Figure out how to make source maps work correctly with eval() code
+    const result = await esbuild.build({
+      entryPoints: [path.basename(inFile)],
+      bundle: true,
+      format: "iife",
+      globalName: "mod",
+      platform: "browser",
+      sourcemap: false, //debug ? "inline" : false,
+      minify: !options.debug,
+      outfile: outFile,
+      metafile: true,
+      external: options.excludeModules || [],
+      treeShaking: true,
+      plugins: [
+        denoPlugin({
+          importMapURL: options.importMap ||
+            new URL("./../import_map.json", import.meta.url),
+        }),
+      ],
+      loader: {
+        ".css": "text",
+        ".md": "text",
+        ".txt": "text",
+        ".html": "text",
+        ".hbs": "text",
+        ".png": "dataurl",
+        ".gif": "dataurl",
+        ".jpg": "dataurl",
+      },
+      absWorkingDir: path.resolve(path.dirname(inFile)),
+    });
+
+    if (options.meta) {
+      const text = await esbuild.analyzeMetafile(result.metafile);
+      console.log("Bundle info for", functionName, text);
+    }
+
+    let jsCode = await Deno.readTextFile(outFile);
+    jsCode = patchDenoLibJS(jsCode);
+    await Deno.remove(outFile);
+    return `(() => { ${jsCode} return mod;})()`;
+  } finally {
+    if (inFile !== filePath) {
+      await Deno.remove(inFile);
+    }
+  }
+}
+
+export async function compileModule(
+  cwd: string,
+  moduleName: string,
+  options: CompileOptions = {},
+): Promise<string> {
+  const inFile = path.resolve(cwd, "_in.ts");
+  await Deno.writeTextFile(inFile, `export * from "${moduleName}";`);
+  const code = await compile(inFile, undefined, options);
+  await Deno.remove(inFile);
+  return code;
+}
+
+export async function sandboxCompile(
+  filename: string,
+  code: string,
+  functionName?: string,
+  options: CompileOptions = {},
+): Promise<string> {
+  const tmpDir = await Deno.makeTempDir();
+
+  await Deno.writeTextFile(`${tmpDir}/${filename}`, code);
+  const jsCode = await compile(
+    `${tmpDir}/${filename}`,
+    functionName,
+    options,
+  );
+  await Deno.remove(tmpDir, { recursive: true });
+  return jsCode;
+}
+
+export async function sandboxCompileModule(
+  moduleUrl: string,
+  options: CompileOptions = {},
+): Promise<string> {
+  await Deno.writeTextFile(
+    "_mod.ts",
+    `module.exports = require("${moduleUrl}");`,
+  );
+  const code = await compile("_mod.ts", undefined, options);
+  await Deno.remove("_mod.ts");
+  return code;
+}

plugos/deps.ts

@@ -0,0 +1,3 @@
+export { globToRegExp } from "https://deno.land/std@0.158.0/path/glob.ts";
+export { walk } from "https://deno.land/std@0.159.0/fs/mod.ts";
+export * as path from "https://deno.land/std@0.158.0/path/mod.ts";

plugos/environments/custom_logger.ts

@@ -0,0 +1,64 @@
+export type LogLevel = "info" | "warn" | "error" | "log";
+
+export class ConsoleLogger {
+  print: boolean;
+  callback: (level: LogLevel, entry: string) => void;
+
+  constructor(
+    callback: (level: LogLevel, entry: string) => void,
+    print: boolean = true
+  ) {
+    this.print = print;
+    this.callback = callback;
+  }
+
+  log(...args: any[]): void {
+    this.push("log", args);
+  }
+
+  warn(...args: any[]): void {
+    this.push("warn", args);
+  }
+
+  error(...args: any[]): void {
+    this.push("error", args);
+  }
+
+  info(...args: any[]): void {
+    this.push("info", args);
+  }
+
+  push(level: LogLevel, args: any[]) {
+    this.callback(level, this.logMessage(args));
+    if (this.print) {
+      console[level](...args);
+    }
+  }
+
+  logMessage(values: any[]): string {
+    let pieces: string[] = [];
+    for (let val of values) {
+      switch (typeof val) {
+        case "string":
+        case "number":
+          pieces.push("" + val);
+          break;
+        case "undefined":
+          pieces.push("undefined");
+          break;
+        default:
+          try {
+            let s = JSON.stringify(val, null, 2);
+            if (s.length > 500) {
+              s = s.substring(0, 500) + "...";
+            }
+            pieces.push(s);
+          } catch {
+            // May be cyclical reference
+            pieces.push("[circular object]");
+          }
+      }
+    }
+    return pieces.join(" ");
+  }
+}

plugos/environments/deno_sandbox.ts

@@ -0,0 +1,42 @@
+import { safeRun } from "../util.ts";
+
+// @ts-ignore
+// import workerCode from "bundle-text:./node_worker.ts";
+import { Sandbox } from "../sandbox.ts";
+import { WorkerLike } from "./worker.ts";
+import { Plug } from "../plug.ts";
+
+class DenoWorkerWrapper implements WorkerLike {
+  private worker: Worker;
+  onMessage?: (message: any) => Promise<void>;
+  ready: Promise<void>;
+
+  constructor(worker: Worker) {
+    this.worker = worker;
+    this.worker.addEventListener("message", (evt: any) => {
+      let data = evt.data;
+      if (!data) return;
+      safeRun(async () => {
+        await this.onMessage!(data);
+      });
+    });
+    this.ready = Promise.resolve();
+  }
+  postMessage(message: any): void {
+    this.worker.postMessage(message);
+  }
+
+  terminate() {
+    return this.worker.terminate();
+  }
+}
+
+export function createSandbox(plug: Plug<any>) {
+  let worker = new Worker(
+    new URL("./sandbox_worker.ts", import.meta.url).href,
+    {
+      type: "module",
+    }
+  );
+  return new Sandbox(plug, new DenoWorkerWrapper(worker));
+}

plugos/environments/sandbox_worker.ts

@@ -0,0 +1,159 @@
+import { safeRun } from "../util.ts";
+import { ConsoleLogger } from "./custom_logger.ts";
+import { ControllerMessage, WorkerMessage } from "./worker.ts";
+
+if (typeof Deno === "undefined") {
+  // @ts-ignore: Deno hack
+  self.Deno = {
+    args: [],
+    // @ts-ignore: Deno hack
+    build: {
+      arch: "x86_64",
+    },
+    env: {
+      // @ts-ignore: Deno hack
+      get() {
+      },
+    },
+  };
+}
+
+let loadedFunctions = new Map<string, Function>();
+let pendingRequests = new Map<
+  number,
+  {
+    resolve: (result: unknown) => void;
+    reject: (e: any) => void;
+  }
+>();
+
+function workerPostMessage(msg: ControllerMessage) {
+  if (typeof window !== "undefined" && window.parent !== window) {
+    window.parent.postMessage(msg, "*");
+  } else {
+    self.postMessage(msg);
+  }
+}
+
+declare global {
+  function syscall(name: string, ...args: any[]): Promise<any>;
+  // function require(moduleName: string): any;
+}
+
+let syscallReqId = 0;
+
+self.syscall = async (name: string, ...args: any[]) => {
+  return await new Promise((resolve, reject) => {
+    syscallReqId++;
+    pendingRequests.set(syscallReqId, { resolve, reject });
+    workerPostMessage({
+      type: "syscall",
+      id: syscallReqId,
+      name,
+      args,
+    });
+  });
+};
+
+let loadedModules = new Map<string, any>();
+
+// @ts-ignore: global to load dynamic imports
+self.require = (moduleName: string): any => {
+  // console.log("Requiring", moduleName, loadedModules.get(moduleName));
+  const mod = loadedModules.get(moduleName);
+  if (!mod) {
+    throw new Error(
+      `Dynamically importing non-preloaded library ${moduleName}`,
+    );
+  }
+  return mod;
+};
+
+// @ts-ignore
+self.console = new ConsoleLogger((level, message) => {
+  workerPostMessage({ type: "log", level, message });
+}, false);
+
+function wrapScript(code: string) {
+  return `return (${code})["default"]`;
+}
+
+self.addEventListener("message", (event: { data: WorkerMessage }) => {
+  safeRun(async () => {
+    let data = event.data;
+    switch (data.type) {
+      case "load":
+        {
+          let fn2 = new Function(wrapScript(data.code!));
+          loadedFunctions.set(data.name!, fn2());
+          workerPostMessage({
+            type: "inited",
+            name: data.name,
+          });
+        }
+
+        break;
+      case "load-dependency":
+        {
+          // console.log("Received dep", data.name);
+          let fn3 = new Function(`return ${data.code!}`);
+          let v = fn3();
+          loadedModules.set(data.name!, v);
+          // console.log("Dep val", v);
+          workerPostMessage({
+            type: "dependency-inited",
+            name: data.name,
+          });
+        }
+        break;
+      case "invoke":
+        {
+          let fn = loadedFunctions.get(data.name!);
+          if (!fn) {
+            throw new Error(`Function not loaded: ${data.name}`);
+          }
+          try {
+            let result = await Promise.resolve(fn(...(data.args || [])));
+            workerPostMessage({
+              type: "result",
+              id: data.id,
+              result: result,
+            } as ControllerMessage);
+          } catch (e: any) {
+            workerPostMessage({
+              type: "result",
+              id: data.id,
+              error: e.message,
+              stack: e.stack,
+            });
+            // console.error("Error invoking function", data.name, e.message);
+            // throw e;
+          }
+        }
+
+        break;
+      case "syscall-response":
+        {
+          let syscallId = data.id!;
+          const lookup = pendingRequests.get(syscallId);
+          if (!lookup) {
+            console.log(
+              "Current outstanding requests",
+              pendingRequests,
+              "looking up",
+              syscallId,
+            );
+            throw Error("Invalid request id");
+          }
+          pendingRequests.delete(syscallId);
+          if (data.error) {
+            lookup.reject(new Error(data.error));
+          } else {
+            lookup.resolve(data.result);
+          }
+        }
+
+        break;
+    }
+  });
+});

plugos/environments/webworker_sandbox.ts

@@ -0,0 +1,41 @@
+import { safeRun } from "../util.ts";
+import { Sandbox } from "../sandbox.ts";
+import { WorkerLike } from "./worker.ts";
+import type { Plug } from "../plug.ts";
+
+class WebWorkerWrapper implements WorkerLike {
+  private worker: Worker;
+  onMessage?: (message: any) => Promise<void>;
+  ready: Promise<void>;
+
+  constructor(worker: Worker) {
+    this.worker = worker;
+    this.worker.addEventListener("message", (evt: any) => {
+      let data = evt.data;
+      if (!data) return;
+      safeRun(async () => {
+        await this.onMessage!(data);
+      });
+    });
+    this.ready = Promise.resolve();
+  }
+  postMessage(message: any): void {
+    this.worker.postMessage(message);
+  }
+
+  terminate() {
+    return this.worker.terminate();
+  }
+}
+
+export function createSandbox(plug: Plug<any>) {
+  const worker = new Worker(
+    import.meta.url
+      ? new URL("sandbox_worker.ts", import.meta.url)
+      : new URL("worker.js", location.origin),
+    {
+      type: "module",
+    },
+  );
+  return new Sandbox(plug, new WebWorkerWrapper(worker));
+}

plugos/environments/worker.ts

@@ -0,0 +1,45 @@
+import type { LogLevel } from "./custom_logger.ts";
+
+export type ControllerMessageType =
+  | "inited"
+  | "dependency-inited"
+  | "result"
+  | "syscall"
+  | "log";
+
+export type ControllerMessage = {
+  type: ControllerMessageType;
+  id?: number;
+  name?: string;
+  args?: any[];
+  error?: string;
+  stack?: string;
+  level?: LogLevel;
+  message?: string;
+  result?: any;
+};
+
+export interface WorkerLike {
+  ready: Promise<void>;
+  onMessage?: (message: any) => Promise<void>;
+
+  postMessage(message: any): void;
+
+  terminate(): void;
+}
+
+export type WorkerMessageType =
+  | "load"
+  | "load-dependency"
+  | "invoke"
+  | "syscall-response";
+
+export type WorkerMessage = {
+  type: WorkerMessageType;
+  id?: number;
+  name?: string;
+  code?: string;
+  args?: any[];
+  result?: any;
+  error?: any;
+};

plugos/event.ts

@@ -0,0 +1,20 @@
+export abstract class EventEmitter<HandlerT> {
+  private handlers: Partial<HandlerT>[] = [];
+
+  on(handlers: Partial<HandlerT>) {
+    this.handlers.push(handlers);
+  }
+
+  off(handlers: Partial<HandlerT>) {
+    this.handlers = this.handlers.filter((h) => h !== handlers);
+  }
+
+  async emit(eventName: keyof HandlerT, ...args: any[]): Promise<void> {
+    for (const handler of this.handlers) {
+      const fn: any = handler[eventName];
+      if (fn) {
+        await Promise.resolve(fn(...args));
+      }
+    }
+  }
+}

plugos/hooks/cron.deno.ts

@@ -0,0 +1,78 @@
+import { Hook, Manifest } from "../types.ts";
+import { Cron } from "https://cdn.jsdelivr.net/gh/hexagon/croner@4/src/croner.js";
+import { safeRun } from "../util.ts";
+import { System } from "../system.ts";
+
+export type CronHookT = {
+  cron?: string | string[];
+};
+
+export class DenoCronHook implements Hook<CronHookT> {
+  apply(system: System<CronHookT>): void {
+    let tasks: Cron[] = [];
+    system.on({
+      plugLoaded: () => {
+        reloadCrons();
+      },
+      plugUnloaded() {
+        reloadCrons();
+      },
+    });
+
+    reloadCrons();
+
+    function reloadCrons() {
+      tasks.forEach((task) => task.stop());
+      tasks = [];
+      for (let plug of system.loadedPlugs.values()) {
+        if (!plug.manifest) {
+          continue;
+        }
+        for (
+          const [name, functionDef] of Object.entries(
+            plug.manifest.functions,
+          )
+        ) {
+          if (!functionDef.cron) {
+            continue;
+          }
+          const crons = Array.isArray(functionDef.cron)
+            ? functionDef.cron
+            : [functionDef.cron];
+          for (const cronDef of crons) {
+            tasks.push(
+              new Cron(cronDef, () => {
+                console.log("Now acting on cron", cronDef);
+                safeRun(async () => {
+                  try {
+                    await plug.invoke(name, [cronDef]);
+                  } catch (e: any) {
+                    console.error("Execution of cron function failed", e);
+                  }
+                });
+              }),
+            );
+          }
+        }
+      }
+    }
+  }
+
+  validateManifest(manifest: Manifest<CronHookT>): string[] {
+    let errors: string[] = [];
+    for (const [name, functionDef] of Object.entries(manifest.functions)) {
+      if (!functionDef.cron) {
+        continue;
+      }
+      const crons = Array.isArray(functionDef.cron)
+        ? functionDef.cron
+        : [functionDef.cron];
+      for (let cronDef of crons) {
+        // if (!cron.validate(cronDef)) {
+        //   errors.push(`Invalid cron expression ${cronDef}`);
+        // }
+      }
+    }
+    return errors;
+  }
+}

plugos/hooks/endpoint.test.ts

@@ -0,0 +1,48 @@
+import { createSandbox } from "../environments/deno_sandbox.ts";
+import { Manifest } from "../types.ts";
+import { EndpointHook, EndpointHookT } from "./endpoint.ts";
+import { System } from "../system.ts";
+
+import { Application } from "../../server/deps.ts";
+import { assertEquals } from "../../test_deps.ts";
+
+Deno.test("Run a plugos endpoint server", async () => {
+  let system = new System<EndpointHookT>("server");
+  let plug = await system.load(
+    {
+      name: "test",
+      functions: {
+        testhandler: {
+          http: {
+            path: "/",
+          },
+          code: `(() => {
+          return {
+            default: (req) => {
+              console.log("Req", req);
+              return {status: 200, body: [1, 2, 3], headers: {"Content-type": "application/json"}};
+            }
+          };
+        })()`,
+        },
+      },
+    } as Manifest<EndpointHookT>,
+    createSandbox,
+  );
+
+  const app = new Application();
+  const port = 3123;
+
+  system.addHook(new EndpointHook(app, "/_"));
+
+  const controller = new AbortController();
+  app.listen({ port: port, signal: controller.signal });
+
+  const res = await fetch(`http://localhost:${port}/_/test/?name=Pete`);
+  assertEquals(res.status, 200);
+  assertEquals(res.headers.get("Content-type"), "application/json");
+  assertEquals(await res.json(), [1, 2, 3]);
+  console.log("Aborting");
+  controller.abort();
+  await system.unloadAll();
+});

plugos/hooks/endpoint.ts

@@ -0,0 +1,136 @@
+import { Hook, Manifest } from "../types.ts";
+import { System } from "../system.ts";
+import { Application } from "../../server/deps.ts";
+
+export type EndpointRequest = {
+  method: string;
+  path: string;
+  query: { [key: string]: string };
+  headers: { [key: string]: string };
+  body: any;
+};
+
+export type EndpointResponse = {
+  status: number;
+  headers?: { [key: string]: string };
+  body: any;
+};
+
+export type EndpointHookT = {
+  http?: EndPointDef | EndPointDef[];
+};
+
+export type EndPointDef = {
+  method?: "GET" | "POST" | "PUT" | "DELETE" | "HEAD" | "OPTIONS" | "ANY";
+  path: string;
+};
+
+export class EndpointHook implements Hook<EndpointHookT> {
+  private app: Application;
+  readonly prefix: string;
+
+  constructor(app: Application, prefix: string) {
+    this.app = app;
+    this.prefix = prefix;
+  }
+
+  apply(system: System<EndpointHookT>): void {
+    this.app.use(async (ctx, next) => {
+      const req = ctx.request;
+      const requestPath = ctx.request.url.pathname;
+      if (!requestPath.startsWith(this.prefix)) {
+        return next();
+      }
+      console.log("Endpoint request", requestPath);
+      // Iterate over all loaded plugins
+      for (const [plugName, plug] of system.loadedPlugs.entries()) {
+        const manifest = plug.manifest;
+        if (!manifest) {
+          continue;
+        }
+        const functions = manifest.functions;
+        console.log("Checking plug", plugName);
+        let prefix = `${this.prefix}/${plugName}`;
+        if (!requestPath.startsWith(prefix)) {
+          continue;
+        }
+        for (const [name, functionDef] of Object.entries(functions)) {
+          if (!functionDef.http) {
+            continue;
+          }
+          let endpoints = Array.isArray(functionDef.http)
+            ? functionDef.http
+            : [functionDef.http];
+          console.log(endpoints);
+          for (const { path, method } of endpoints) {
+            let prefixedPath = `${prefix}${path}`;
+            if (
+              prefixedPath === requestPath &&
+              ((method || "GET") === req.method || method === "ANY")
+            ) {
+              try {
+                const response: EndpointResponse = await plug.invoke(name, [
+                  {
+                    path: req.url.pathname,
+                    method: req.method,
+                    body: req.body(),
+                    query: Object.fromEntries(
+                      req.url.searchParams.entries(),
+                    ),
+                    headers: Object.fromEntries(req.headers.entries()),
+                  } as EndpointRequest,
+                ]);
+                if (response.headers) {
+                  for (
+                    const [key, value] of Object.entries(
+                      response.headers,
+                    )
+                  ) {
+                    ctx.response.headers.set(key, value);
+                  }
+                }
+                ctx.response.status = response.status;
+                ctx.response.body = response.body;
+                console.log("Sent result");
+                return;
+              } catch (e: any) {
+                console.error("Error executing function", e);
+                ctx.response.status = 500;
+                ctx.response.body = e.message;
+                return;
+              }
+            }
+          }
+        }
+      }
+      // console.log("Shouldn't get here");
+      next();
+    });
+  }
+
+  validateManifest(manifest: Manifest<EndpointHookT>): string[] {
+    let errors = [];
+    for (const [name, functionDef] of Object.entries(manifest.functions)) {
+      if (!functionDef.http) {
+        continue;
+      }
+      let endpoints = Array.isArray(functionDef.http)
+        ? functionDef.http
+        : [functionDef.http];
+      for (let { path, method } of endpoints) {
+        if (!path) {
+          errors.push("Path not defined for endpoint");
+        }
+        if (
+          method &&
+          ["GET", "POST", "PUT", "DELETE", "ANY"].indexOf(method) === -1
+        ) {
+          errors.push(
+            `Invalid method ${method} for end point with with ${path}`,
+          );
+        }
+      }
+    }
+    return errors;
+  }
+}

plugos/hooks/event.ts

@@ -0,0 +1,104 @@
+import type { Hook, Manifest } from "../types.ts";
+import { System } from "../system.ts";
+import { safeRun } from "../util.ts";
+
+// System events:
+// - plug:load (plugName: string)
+
+export type EventHookT = {
+  events?: string[];
+};
+
+export class EventHook implements Hook<EventHookT> {
+  private system?: System<EventHookT>;
+  public localListeners: Map<string, ((data: any) => any)[]> = new Map();
+
+  addLocalListener(eventName: string, callback: (data: any) => any) {
+    if (!this.localListeners.has(eventName)) {
+      this.localListeners.set(eventName, []);
+    }
+    this.localListeners.get(eventName)!.push(callback);
+  }
+
+  // Pull all events listened to
+  listEvents(): string[] {
+    if (!this.system) {
+      throw new Error("Event hook is not initialized");
+    }
+    const eventNames = new Set<string>();
+    for (const plug of this.system.loadedPlugs.values()) {
+      for (const functionDef of Object.values(plug.manifest!.functions)) {
+        if (functionDef.events) {
+          for (const eventName of functionDef.events) {
+            eventNames.add(eventName);
+          }
+        }
+      }
+    }
+    for (const eventName of this.localListeners.keys()) {
+      eventNames.add(eventName);
+    }
+
+    return [...eventNames];
+  }
+
+  async dispatchEvent(eventName: string, data?: any): Promise<any[]> {
+    if (!this.system) {
+      throw new Error("Event hook is not initialized");
+    }
+    const responses: any[] = [];
+    for (const plug of this.system.loadedPlugs.values()) {
+      for (
+        const [name, functionDef] of Object.entries(
+          plug.manifest!.functions,
+        )
+      ) {
+        if (functionDef.events && functionDef.events.includes(eventName)) {
+          // Only dispatch functions that can run in this environment
+          if (plug.canInvoke(name)) {
+            const result = await plug.invoke(name, [data]);
+            if (result !== undefined) {
+              responses.push(result);
+            }
+          }
+        }
+      }
+    }
+    const localListeners = this.localListeners.get(eventName);
+    if (localListeners) {
+      for (const localListener of localListeners) {
+        const result = await Promise.resolve(localListener(data));
+        if (result) {
+          responses.push(result);
+        }
+      }
+    }
+
+    return responses;
+  }
+
+  apply(system: System<EventHookT>): void {
+    this.system = system;
+    this.system.on({
+      plugLoaded: (plug) => {
+        safeRun(async () => {
+          await this.dispatchEvent("plug:load", plug.name);
+        });
+      },
+    });
+  }
+
+  validateManifest(manifest: Manifest<EventHookT>): string[] {
+    const errors = [];
+    for (
+      const [_, functionDef] of Object.entries(
+        manifest.functions || {},
+      )
+    ) {
+      if (functionDef.events && !Array.isArray(functionDef.events)) {
+        errors.push("'events' key must be an array of strings");
+      }
+    }
+    return errors;
+  }
+}

plugos/plug.ts

@@ -0,0 +1,69 @@
+import { Manifest, RuntimeEnvironment } from "./types.ts";
+import { Sandbox } from "./sandbox.ts";
+import { System } from "./system.ts";
+
+export class Plug<HookT> {
+  system: System<HookT>;
+  sandbox: Sandbox;
+  public manifest?: Manifest<HookT>;
+  readonly runtimeEnv: RuntimeEnvironment;
+  grantedPermissions: string[] = [];
+  name: string;
+  version: number;
+
+  constructor(
+    system: System<HookT>,
+    name: string,
+    sandboxFactory: (plug: Plug<HookT>) => Sandbox
+  ) {
+    this.system = system;
+    this.name = name;
+    this.sandbox = sandboxFactory(this);
+    this.runtimeEnv = system.runtimeEnv;
+    this.version = new Date().getTime();
+  }
+
+  async load(manifest: Manifest<HookT>) {
+    this.manifest = manifest;
+    // TODO: These need to be explicitly granted, not just taken
+    this.grantedPermissions = manifest.requiredPermissions || [];
+    for (const [dep, code] of Object.entries(manifest.dependencies || {})) {
+      await this.sandbox.loadDependency(dep, code);
+    }
+  }
+
+  syscall(name: string, args: any[]): Promise<any> {
+    return this.system.syscallWithContext({ plug: this }, name, args);
+  }
+
+  canInvoke(name: string) {
+    if (!this.manifest) {
+      return false;
+    }
+    const funDef = this.manifest.functions[name];
+    if (!funDef) {
+      throw new Error(`Function ${name} not found in manifest`);
+    }
+    return !funDef.env || funDef.env === this.runtimeEnv;
+  }
+
+  async invoke(name: string, args: Array<any>): Promise<any> {
+    if (!this.sandbox.isLoaded(name)) {
+      const funDef = this.manifest!.functions[name];
+      if (!funDef) {
+        throw new Error(`Function ${name} not found in manifest`);
+      }
+      if (!this.canInvoke(name)) {
+        throw new Error(
+          `Function ${name} is not available in ${this.runtimeEnv}`
+        );
+      }
+      await this.sandbox.load(name, funDef.code!);
+    }
+    return await this.sandbox.invoke(name, args);
+  }
+
+  async stop() {
+    this.sandbox.stop();
+  }
+}

plugos/runtime.test.ts

@@ -0,0 +1,164 @@
+import { createSandbox } from "./environments/deno_sandbox.ts";
+import { System } from "./system.ts";
+
+import {
+  assert,
+  assertEquals,
+} from "https://deno.land/std@0.158.0/testing/asserts.ts";
+
+Deno.test("Run a deno sandbox", async () => {
+  const system = new System("server");
+  system.registerSyscalls([], {
+    addNumbers: (_ctx, a, b) => {
+      return a + b;
+    },
+    failingSyscall: () => {
+      throw new Error("#fail");
+    },
+  });
+  system.registerSyscalls(["restricted"], {
+    restrictedSyscall: () => {
+      return "restricted";
+    },
+  });
+  system.registerSyscalls(["dangerous"], {
+    dangerousSyscall: () => {
+      return "yay";
+    },
+  });
+  const plug = await system.load(
+    {
+      name: "test",
+      requiredPermissions: ["dangerous"],
+      functions: {
+        addTen: {
+          code: `(() => {
+          return {
+            default: (n) => {
+              return n + 10;
+            }
+          };
+        })()`,
+        },
+        addNumbersSyscall: {
+          code: `(() => {
+          return {
+            default: async (a, b) => {
+              return await self.syscall("addNumbers", a, b);
+            }
+          };
+        })()`,
+        },
+        errorOut: {
+          code: `(() => {
+          return {
+            default: () => {
+              throw Error("BOOM");
+            }
+          };
+        })()`,
+        },
+        errorOutSys: {
+          code: `(() => {
+          return {
+            default: async () => {
+              await self.syscall("failingSyscall");
+            }
+          };
+        })()`,
+        },
+        restrictedTest: {
+          code: `(() => {
+          return {
+            default: async () => {
+              await self.syscall("restrictedSyscall");
+            }
+          };
+        })()`,
+        },
+        dangerousTest: {
+          code: `(() => {
+          return {
+            default: async () => {
+              return await self.syscall("dangerousSyscall");
+            }
+          };
+        })()`,
+        },
+      },
+    },
+    createSandbox,
+  );
+  assertEquals(await plug.invoke("addTen", [10]), 20);
+  for (let i = 0; i < 100; i++) {
+    assertEquals(await plug.invoke("addNumbersSyscall", [10, i]), 10 + i);
+  }
+  try {
+    await plug.invoke("errorOut", []);
+    assert(false);
+  } catch (e: any) {
+    assert(e.message.indexOf("BOOM") !== -1);
+  }
+  try {
+    await plug.invoke("errorOutSys", []);
+    assert(false);
+  } catch (e: any) {
+    assert(e.message.indexOf("#fail") !== -1);
+  }
+  try {
+    await plug.invoke("restrictedTest", []);
+    assert(false);
+  } catch (e: any) {
+    assert(
+      e.message.indexOf(
+        "Missing permission 'restricted' for syscall restrictedSyscall",
+      ) !== -1,
+    );
+  }
+  assertEquals(await plug.invoke("dangerousTest", []), "yay");
+
+  await system.unloadAll();
+});
+
+import { bundle as plugOsBundle } from "./bin/plugos-bundle.ts";
+import { esbuild } from "./compile.ts";
+const __dirname = new URL(".", import.meta.url).pathname;
+
+Deno.test("Preload dependencies", async () => {
+  const globalModules = await plugOsBundle(
+    `${__dirname}../plugs/global.plug.yaml`,
+  );
+  // const globalModules = JSON.parse(
+  //   Deno.readTextFileSync(`${tmpDist}/global.plug.json`),
+  // );
+  const testPlugManifest = await plugOsBundle(
+    `${__dirname}test.plug.yaml`,
+    { excludeModules: Object.keys(globalModules.dependencies!) },
+  );
+  esbuild.stop();
+
+  const system = new System("server");
+  system.on({
+    plugLoaded: async (plug) => {
+      for (
+        const [modName, code] of Object.entries(globalModules.dependencies!)
+      ) {
+        await plug.sandbox.loadDependency(modName, code as string);
+      }
+    },
+  });
+
+  // Load test module
+  console.log("Loading test module");
+  const testPlug = await system.load(
+    testPlugManifest,
+    createSandbox,
+  );
+  console.log("Running");
+
+  const result = await testPlug.invoke("boot", []);
+
+  console.log("Result", result);
+
+  await system.unloadAll();
+});

plugos/sandbox.ts

@@ -0,0 +1,157 @@
+import type { LogLevel } from "./environments/custom_logger.ts";
+import {
+  ControllerMessage,
+  WorkerLike,
+  WorkerMessage,
+} from "./environments/worker.ts";
+import { Plug } from "./plug.ts";
+
+export type SandboxFactory<HookT> = (plug: Plug<HookT>) => Sandbox;
+
+export type LogEntry = {
+  level: LogLevel;
+  message: string;
+  date: number;
+};
+
+export class Sandbox {
+  protected worker: WorkerLike;
+  protected reqId = 0;
+  protected outstandingInits = new Map<string, () => void>();
+  protected outstandingDependencyInits = new Map<string, () => void>();
+  protected outstandingInvocations = new Map<
+    number,
+    { resolve: (result: any) => void; reject: (e: any) => void }
+  >();
+  protected loadedFunctions = new Set<string>();
+  protected plug: Plug<any>;
+  public logBuffer: LogEntry[] = [];
+  public maxLogBufferSize = 100;
+
+  constructor(plug: Plug<any>, worker: WorkerLike) {
+    worker.onMessage = this.onMessage.bind(this);
+    this.worker = worker;
+    this.plug = plug;
+  }
+
+  isLoaded(name: string) {
+    return this.loadedFunctions.has(name);
+  }
+
+  async load(name: string, code: string): Promise<void> {
+    await this.worker.ready;
+    let outstandingInit = this.outstandingInits.get(name);
+    if (outstandingInit) {
+      // Load already in progress, let's wait for it...
+      return new Promise((resolve) => {
+        this.outstandingInits.set(name, () => {
+          outstandingInit!();
+          resolve();
+        });
+      });
+    }
+    this.worker.postMessage({
+      type: "load",
+      name: name,
+      code: code,
+    } as WorkerMessage);
+    return new Promise((resolve) => {
+      this.outstandingInits.set(name, () => {
+        this.loadedFunctions.add(name);
+        this.outstandingInits.delete(name);
+        resolve();
+      });
+    });
+  }
+
+  loadDependency(name: string, code: string): Promise<void> {
+    // console.log("Loading dependency", name);
+    this.worker.postMessage({
+      type: "load-dependency",
+      name: name,
+      code: code,
+    } as WorkerMessage);
+    return new Promise((resolve) => {
+      // console.log("Loaded dependency", name);
+      this.outstandingDependencyInits.set(name, () => {
+        this.outstandingDependencyInits.delete(name);
+        resolve();
+      });
+    });
+  }
+
+  async onMessage(data: ControllerMessage) {
+    switch (data.type) {
+      case "inited":
+        let initCb = this.outstandingInits.get(data.name!);
+        initCb && initCb();
+        this.outstandingInits.delete(data.name!);
+        break;
+      case "dependency-inited":
+        let depInitCb = this.outstandingDependencyInits.get(data.name!);
+        depInitCb && depInitCb();
+        this.outstandingDependencyInits.delete(data.name!);
+        break;
+      case "syscall":
+        try {
+          let result = await this.plug.syscall(data.name!, data.args!);
+
+          this.worker.postMessage({
+            type: "syscall-response",
+            id: data.id,
+            result: result,
+          } as WorkerMessage);
+        } catch (e: any) {
+          // console.error("Syscall fail", e);
+          this.worker.postMessage({
+            type: "syscall-response",
+            id: data.id,
+            error: e.message,
+          } as WorkerMessage);
+        }
+        break;
+      case "result":
+        let resultCbs = this.outstandingInvocations.get(data.id!);
+        this.outstandingInvocations.delete(data.id!);
+        if (data.error) {
+          resultCbs &&
+            resultCbs.reject(
+              new Error(`${data.error}\nStack trace: ${data.stack}`),
+            );
+        } else {
+          resultCbs && resultCbs.resolve(data.result);
+        }
+        break;
+      case "log":
+        this.logBuffer.push({
+          level: data.level!,
+          message: data.message!,
+          date: Date.now(),
+        });
+        if (this.logBuffer.length > this.maxLogBufferSize) {
+          this.logBuffer.shift();
+        }
+        console.log(`[Sandbox ${data.level}]`, data.message);
+        break;
+      default:
+        console.error("Unknown message type", data);
+    }
+  }
+
+  async invoke(name: string, args: any[]): Promise<any> {
+    this.reqId++;
+    this.worker.postMessage({
+      type: "invoke",
+      id: this.reqId,
+      name,
+      args,
+    });
+    return new Promise((resolve, reject) => {
+      this.outstandingInvocations.set(this.reqId, { resolve, reject });
+    });
+  }
+
+  stop() {
+    this.worker.terminate();
+  }
+}

plugos/syscalls/asset.ts

@@ -0,0 +1,14 @@
+import { SysCallMapping, System } from "../system.ts";
+import type { AssetBundle, FileMeta } from "../asset_bundle_reader.ts";
+
+export default function assetSyscalls(system: System<any>): SysCallMapping {
+  return {
+    "asset.readAsset": (
+      ctx,
+      name: string,
+    ): { data: string; meta: FileMeta } => {
+      return (system.loadedPlugs.get(ctx.plug.name)!.manifest!
+        .assets as AssetBundle)[name];
+    },
+  };
+}

plugos/syscalls/esbuild.ts

@@ -0,0 +1,35 @@
+import { sandboxCompile, sandboxCompileModule } from "../compile.ts";
+import { SysCallMapping } from "../system.ts";
+
+// TODO: FIgure out a better way to do this
+const builtinModules = ["yaml", "handlebars"];
+
+export function esbuildSyscalls(): SysCallMapping {
+  return {
+    "esbuild.compile": async (
+      _ctx,
+      filename: string,
+      code: string,
+      functionName?: string,
+      excludeModules: string[] = [],
+    ): Promise<string> => {
+      return await sandboxCompile(
+        filename,
+        code,
+        functionName,
+        {
+          debug: true,
+          excludeModules: [...builtinModules, ...excludeModules],
+        },
+      );
+    },
+    "esbuild.compileModule": async (
+      _ctx,
+      moduleName: string,
+    ): Promise<string> => {
+      return await sandboxCompileModule(moduleName, {
+        excludeModules: builtinModules,
+      });
+    },
+  };
+}

plugos/syscalls/event.ts

@@ -0,0 +1,13 @@
+import { SysCallMapping } from "../system.ts";
+import { EventHook } from "../hooks/event.ts";
+
+export function eventSyscalls(eventHook: EventHook): SysCallMapping {
+  return {
+    "event.dispatch": (_ctx, eventName: string, data: any) => {
+      return eventHook.dispatchEvent(eventName, data);
+    },
+    "event.list": () => {
+      return eventHook.listEvents();
+    },
+  };
+}

plugos/syscalls/fs.deno.ts

@@ -0,0 +1,108 @@
+import type { SysCallMapping } from "../system.ts";
+import { mime, path } from "../../server/deps.ts";
+import { base64Decode, base64Encode } from "../../plugos/base64.ts";
+import type { FileMeta } from "../asset_bundle_reader.ts";
+
+export default function fileSystemSyscalls(root = "/"): SysCallMapping {
+  function resolvedPath(p: string): string {
+    p = path.resolve(root, p);
+    if (!p.startsWith(root)) {
+      throw Error("Path outside root, not allowed");
+    }
+    return p;
+  }
+
+  return {
+    "fs.readFile": async (
+      _ctx,
+      filePath: string,
+      encoding: "utf8" | "dataurl" = "utf8",
+    ): Promise<{ text: string; meta: FileMeta }> => {
+      const p = resolvedPath(filePath);
+      let text = "";
+      if (encoding === "utf8") {
+        text = await Deno.readTextFile(p);
+      } else {
+        text = `data:application/octet-stream,${
+          base64Encode(await Deno.readFile(p))
+        }`;
+      }
+      const s = await Deno.stat(p);
+      return {
+        text,
+        meta: {
+          name: filePath,
+          lastModified: s.mtime!.getTime(),
+          contentType: mime.getType(filePath) || "application/octet-stream",
+          size: s.size,
+          perm: "rw",
+        },
+      };
+    },
+    "fs.getFileMeta": async (_ctx, filePath: string): Promise<FileMeta> => {
+      const p = resolvedPath(filePath);
+      const s = await Deno.stat(p);
+      return {
+        name: filePath,
+        lastModified: s.mtime!.getTime(),
+        contentType: mime.getType(filePath) || "application/octet-stream",
+        size: s.size,
+        perm: "rw",
+      };
+    },
+    "fs.writeFile": async (
+      _ctx,
+      filePath: string,
+      text: string,
+      encoding: "utf8" | "dataurl" = "utf8",
+    ): Promise<FileMeta> => {
+      const p = resolvedPath(filePath);
+      await Deno.mkdir(path.dirname(p), { recursive: true });
+      if (encoding === "utf8") {
+        await Deno.writeTextFile(p, text);
+      } else {
+        await Deno.writeFile(p, base64Decode(text.split(",")[1]));
+      }
+      const s = await Deno.stat(p);
+      return {
+        name: filePath,
+        lastModified: s.mtime!.getTime(),
+        contentType: mime.getType(filePath) || "application/octet-stream",
+        size: s.size,
+        perm: "rw",
+      };
+    },
+    "fs.deleteFile": async (_ctx, filePath: string): Promise<void> => {
+      await Deno.remove(resolvedPath(filePath));
+    },
+    "fs.listFiles": async (
+      _ctx,
+      dirPath: string,
+      recursive: boolean,
+    ): Promise<FileMeta[]> => {
+      dirPath = resolvedPath(dirPath);
+      const allFiles: FileMeta[] = [];
+
+      async function walkPath(dir: string) {
+        const files = await Deno.readDir(dir);
+        for await (const file of files) {
+          const fullPath = path.join(dir, file.name);
+          const s = await Deno.stat(fullPath);
+          if (s.isDirectory && recursive) {
+            await walkPath(fullPath);
+          } else {
+            allFiles.push({
+              name: fullPath.substring(dirPath.length + 1),
+              lastModified: s.mtime!.getTime(),
+              contentType: mime.getType(fullPath) || "application/octet-stream",
+              size: s.size,
+              perm: "rw",
+            });
+          }
+        }
+      }
+      await walkPath(dirPath);
+      return allFiles;
+    },
+  };
+}

plugos/syscalls/fulltext.knex_sqlite.ts

@@ -0,0 +1,57 @@
+import { SQLite } from "../../server/deps.ts";
+import { SysCallMapping } from "../system.ts";
+import { asyncExecute, asyncQuery } from "./store.deno.ts";
+
+type Item = {
+  key: string;
+  value: string;
+};
+
+export function ensureFTSTable(
+  db: SQLite,
+  tableName: string,
+) {
+  const stmt = db.prepare(
+    `SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
+  );
+  const result = stmt.all(tableName);
+  if (result.length === 0) {
+    asyncExecute(
+      db,
+      `CREATE VIRTUAL TABLE ${tableName} USING fts5(key, value);`,
+    );
+
+    console.log(`Created fts5 table ${tableName}`);
+  }
+  return Promise.resolve();
+}
+
+export function fullTextSearchSyscalls(
+  db: SQLite,
+  tableName: string,
+): SysCallMapping {
+  return {
+    "fulltext.index": async (_ctx, key: string, value: string) => {
+      await asyncExecute(db, `DELETE FROM ${tableName} WHERE key = ?`, key);
+      await asyncExecute(
+        db,
+        `INSERT INTO ${tableName} (key, value) VALUES (?, ?)`,
+        key,
+        value,
+      );
+    },
+    "fulltext.delete": async (_ctx, key: string) => {
+      await asyncExecute(db, `DELETE FROM ${tableName} WHERE key = ?`, key);
+    },
+    "fulltext.search": async (_ctx, phrase: string, limit: number) => {
+      return (
+        await asyncQuery<any>(
+          db,
+          `SELECT key, rank FROM ${tableName} WHERE value MATCH ? ORDER BY key, rank LIMIT ?`,
+          phrase,
+          limit,
+        )
+      ).map((item) => ({ name: item.key, rank: item.rank }));
+    },
+  };
+}

plugos/syscalls/sandbox.ts

@@ -0,0 +1,15 @@
+import { LogEntry } from "../sandbox.ts";
+import { SysCallMapping, System } from "../system.ts";
+
+export default function sandboxSyscalls(system: System<any>): SysCallMapping {
+  return {
+    "sandbox.getLogs": (): LogEntry[] => {
+      let allLogs: LogEntry[] = [];
+      for (const plug of system.loadedPlugs.values()) {
+        allLogs = allLogs.concat(plug.sandbox.logBuffer);
+      }
+      allLogs = allLogs.sort((a, b) => a.date - b.date);
+      return allLogs;
+    },
+  };
+}

plugos/syscalls/shell.node.ts

@@ -0,0 +1,23 @@
+import type { SysCallMapping } from "../system.ts";
+
+export default function (cwd: string): SysCallMapping {
+  return {
+    "shell.run": async (
+      _ctx,
+      cmd: string,
+      args: string[],
+    ): Promise<{ stdout: string; stderr: string }> => {
+      const p = Deno.run({
+        cmd: [cmd, ...args],
+        cwd: cwd,
+        stdout: "piped",
+        stderr: "piped",
+      });
+      await p.status();
+      const stdout = new TextDecoder().decode(await p.output());
+      const stderr = new TextDecoder().decode(await p.stderrOutput());
+
+      return { stdout, stderr };
+    },
+  };
+}

plugos/syscalls/store.deno.test.ts

@@ -0,0 +1,106 @@
+import { assertEquals } from "../../test_deps.ts";
+import { SQLite } from "../../server/deps.ts";
+import { createSandbox } from "../environments/deno_sandbox.ts";
+import { System } from "../system.ts";
+import { ensureTable, storeSyscalls } from "./store.deno.ts";
+
+Deno.test("Test store", async () => {
+  const db = new SQLite(":memory:");
+  await ensureTable(db, "test_table");
+  const system = new System("server");
+  const syscalls = storeSyscalls(db, "test_table");
+  system.registerSyscalls([], syscalls);
+  const plug = await system.load(
+    {
+      name: "test",
+      functions: {
+        test1: {
+          code: `(() => {
+          return {
+            default: async () => {
+              await self.syscall("store.set", "name", "Pete");
+              return await self.syscall("store.get", "name");
+            }
+          };
+        })()`,
+        },
+      },
+    },
+    createSandbox,
+  );
+  assertEquals(await plug.invoke("test1", []), "Pete");
+  await system.unloadAll();
+
+  const dummyCtx: any = {};
+
+  await syscalls["store.deleteAll"](dummyCtx);
+  await syscalls["store.batchSet"](dummyCtx, [
+    {
+      key: "pete",
+      value: {
+        age: 20,
+        firstName: "Pete",
+        lastName: "Roberts",
+      },
+    },
+    {
+      key: "petejr",
+      value: {
+        age: 8,
+        firstName: "Pete Jr",
+        lastName: "Roberts",
+      },
+    },
+    {
+      key: "petesr",
+      value: {
+        age: 78,
+        firstName: "Pete Sr",
+        lastName: "Roberts",
+      },
+    },
+  ]);
+
+  let allRoberts = await syscalls["store.query"](dummyCtx, {
+    filter: [{ op: "=", prop: "lastName", value: "Roberts" }],
+    orderBy: "age",
+    orderDesc: true,
+  });
+
+  assertEquals(allRoberts.length, 3);
+  assertEquals(allRoberts[0].key, "petesr");
+
+  allRoberts = await syscalls["store.query"](dummyCtx, {
+    filter: [{ op: "=", prop: "lastName", value: "Roberts" }],
+    orderBy: "age",
+    limit: 1,
+  });
+
+  assertEquals(allRoberts.length, 1);
+  assertEquals(allRoberts[0].key, "petejr");
+
+  allRoberts = await syscalls["store.query"](dummyCtx, {
+    filter: [
+      { op: ">", prop: "age", value: 10 },
+      { op: "<", prop: "age", value: 30 },
+    ],
+    orderBy: "age",
+  });
+
+  assertEquals(allRoberts.length, 1);
+  assertEquals(allRoberts[0].key, "pete");
+
+  // Delete the middle one
+
+  await syscalls["store.deleteQuery"](dummyCtx, {
+    filter: [
+      { op: ">", prop: "age", value: 10 },
+      { op: "<", prop: "age", value: 30 },
+    ],
+  });
+
+  allRoberts = await syscalls["store.query"](dummyCtx, {});
+  assertEquals(allRoberts.length, 2);
+
+  db.close();
+});

plugos/syscalls/store.deno.ts

@@ -0,0 +1,177 @@
+import { SQLite } from "../../server/deps.ts";
+import { SysCallMapping } from "../system.ts";
+
+export type Item = {
+  page: string;
+  key: string;
+  value: any;
+};
+
+export type KV = {
+  key: string;
+  value: any;
+};
+
+export function ensureTable(db: SQLite, tableName: string) {
+  const stmt = db.prepare(
+    `SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
+  );
+  const result = stmt.all(tableName);
+  if (result.length === 0) {
+    db.exec(`CREATE TABLE ${tableName} (key STRING PRIMARY KEY, value TEXT);`);
+    console.log(`Created table ${tableName}`);
+  }
+  return Promise.resolve();
+}
+
+export type Query = {
+  filter?: Filter[];
+  orderBy?: string;
+  orderDesc?: boolean;
+  limit?: number;
+  select?: string[];
+};
+
+export type Filter = {
+  op: string;
+  prop: string;
+  value: any;
+};
+
+export function queryToSql(
+  query: Query,
+): { sql: string; params: any[] } {
+  const whereClauses: string[] = [];
+  const clauses: string[] = [];
+  const params: any[] = [];
+  if (query.filter) {
+    for (const filter of query.filter) {
+      whereClauses.push(
+        `json_extract(value, '$.${filter.prop}') ${filter.op} ?`,
+      );
+      params.push(filter.value);
+    }
+  }
+  if (query.orderBy) {
+    clauses.push(
+      `ORDER BY json_extract(value, '$.${query.orderBy}') ${
+        query.orderDesc ? "desc" : "asc"
+      }`,
+    );
+  }
+  if (query.limit) {
+    clauses.push(`LIMIT ${query.limit}`);
+  }
+  return {
+    sql: whereClauses.length > 0
+      ? `WHERE ${whereClauses.join(" AND ")} ${clauses.join(" ")}`
+      : clauses.join(" "),
+    params,
+  };
+}
+
+export function asyncQuery<T extends Record<string, unknown>>(
+  db: SQLite,
+  query: string,
+  ...params: any[]
+): Promise<T[]> {
+  // console.log("Querying", query, params);
+  return Promise.resolve(db.prepare(query).all<T>(params));
+}
+
+export function asyncExecute(
+  db: SQLite,
+  query: string,
+  ...params: any[]
+): Promise<number> {
+  // console.log("Exdecting", query, params);
+  return Promise.resolve(db.exec(query, params));
+}
+
+export function storeSyscalls(
+  db: SQLite,
+  tableName: string,
+): SysCallMapping {
+  const apiObj: SysCallMapping = {
+    "store.delete": async (_ctx, key: string) => {
+      await asyncExecute(db, `DELETE FROM ${tableName} WHERE key = ?`, key);
+    },
+    "store.deletePrefix": async (_ctx, prefix: string) => {
+      await asyncExecute(
+        db,
+        `DELETE FROM ${tableName} WHERE key LIKE "${prefix}%"`,
+      );
+    },
+    "store.deleteQuery": async (_ctx, query: Query) => {
+      const { sql, params } = queryToSql(query);
+      await asyncExecute(db, `DELETE FROM ${tableName} ${sql}`, ...params);
+    },
+    "store.deleteAll": async () => {
+      await asyncExecute(db, `DELETE FROM ${tableName}`);
+    },
+    "store.set": async (_ctx, key: string, value: any) => {
+      await asyncExecute(
+        db,
+        `UPDATE ${tableName} SET value = ? WHERE key = ?`,
+        JSON.stringify(value),
+        key,
+      );
+      if (db.changes === 0) {
+        await asyncExecute(
+          db,
+          `INSERT INTO ${tableName} (key, value) VALUES (?, ?)`,
+          key,
+          JSON.stringify(value),
+        );
+      }
+    },
+    // TODO: Optimize
+    "store.batchSet": async (ctx, kvs: KV[]) => {
+      for (const { key, value } of kvs) {
+        await apiObj["store.set"](ctx, key, value);
+      }
+    },
+    "store.batchDelete": async (ctx, keys: string[]) => {
+      for (const key of keys) {
+        await apiObj["store.delete"](ctx, key);
+      }
+    },
+    "store.get": async (_ctx, key: string): Promise<any | null> => {
+      const result = await asyncQuery<Item>(
+        db,
+        `SELECT value FROM ${tableName} WHERE key = ?`,
+        key,
+      );
+      if (result.length) {
+        return JSON.parse(result[0].value);
+      } else {
+        return null;
+      }
+    },
+    "store.queryPrefix": async (_ctx, prefix: string) => {
+      return (
+        await asyncQuery<Item>(
+          db,
+          `SELECT key, value FROM ${tableName} WHERE key LIKE "${prefix}%"`,
+        )
+      ).map(({ key, value }) => ({
+        key,
+        value: JSON.parse(value),
+      }));
+    },
+    "store.query": async (_ctx, query: Query) => {
+      const { sql, params } = queryToSql(query);
+      return (
+        await asyncQuery<Item>(
+          db,
+          `SELECT key, value FROM ${tableName} ${sql}`,
+          ...params,
+        )
+      ).map(({ key, value }: { key: string; value: string }) => ({
+        key,
+        value: JSON.parse(value),
+      }));
+    },
+  };
+  return apiObj;
+}

plugos/syscalls/store.dexie_browser.ts

@@ -0,0 +1,66 @@
+import Dexie from "https://esm.sh/dexie@3.2.2";
+import { SysCallMapping } from "../system.ts";
+
+export type KV = {
+  key: string;
+  value: any;
+};
+
+export function storeSyscalls(
+  dbName: string,
+  tableName: string,
+): SysCallMapping {
+  const db = new Dexie(dbName);
+  db.version(1).stores({
+    [tableName]: "key",
+  });
+  const items = db.table(tableName);
+
+  return {
+    "store.delete": async (_ctx, key: string) => {
+      await items.delete(key);
+    },
+
+    "store.deletePrefix": async (_ctx, prefix: string) => {
+      await items.where("key").startsWith(prefix).delete();
+    },
+
+    "store.deleteAll": async () => {
+      await items.clear();
+    },
+
+    "store.set": async (_ctx, key: string, value: any) => {
+      await items.put({
+        key,
+        value,
+      });
+    },
+
+    "store.batchSet": async (_ctx, kvs: KV[]) => {
+      await items.bulkPut(
+        kvs.map(({ key, value }) => ({
+          key,
+          value,
+        })),
+      );
+    },
+
+    "store.get": async (_ctx, key: string): Promise<any | null> => {
+      const result = await items.get({
+        key,
+      });
+      return result ? result.value : null;
+    },
+
+    "store.queryPrefix": async (
+      _ctx,
+      keyPrefix: string,
+    ): Promise<{ key: string; value: any }[]> => {
+      const results = await items.where("key").startsWith(keyPrefix).toArray();
+      return results.map((result) => ({
+        key: result.key,
+        value: result.value,
+      }));
+    },
+  };
+}

plugos/syscalls/transport.ts

@@ -0,0 +1,20 @@
+import { SyscallContext, SysCallMapping } from "../system.ts";
+
+export function proxySyscalls(
+  names: string[],
+  transportCall: (
+    ctx: SyscallContext,
+    name: string,
+    ...args: any[]
+  ) => Promise<any>,
+): SysCallMapping {
+  const syscalls: SysCallMapping = {};
+
+  for (const name of names) {
+    syscalls[name] = (ctx, ...args: any[]) => {
+      return transportCall(ctx, name, ...args);
+    };
+  }
+
+  return syscalls;
+}

plugos/system.ts

@@ -0,0 +1,161 @@
+import { Hook, Manifest, RuntimeEnvironment } from "./types.ts";
+import { EventEmitter } from "./event.ts";
+import { SandboxFactory } from "./sandbox.ts";
+import { Plug } from "./plug.ts";
+
+export interface SysCallMapping {
+  [key: string]: (ctx: SyscallContext, ...args: any) => Promise<any> | any;
+}
+
+export type SystemJSON<HookT> = Manifest<HookT>[];
+
+export type SystemEvents<HookT> = {
+  plugLoaded: (plug: Plug<HookT>) => void | Promise<void>;
+  plugUnloaded: (name: string) => void | Promise<void>;
+};
+
+export type SyscallContext = {
+  plug: Plug<any>;
+};
+
+type SyscallSignature = (
+  ctx: SyscallContext,
+  ...args: any[]
+) => Promise<any> | any;
+
+type Syscall = {
+  requiredPermissions: string[];
+  callback: SyscallSignature;
+};
+
+export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
+  readonly runtimeEnv: RuntimeEnvironment;
+  protected plugs = new Map<string, Plug<HookT>>();
+  protected registeredSyscalls = new Map<string, Syscall>();
+  protected enabledHooks = new Set<Hook<HookT>>();
+
+  constructor(env: RuntimeEnvironment) {
+    super();
+    this.runtimeEnv = env;
+  }
+
+  get loadedPlugs(): Map<string, Plug<HookT>> {
+    return this.plugs;
+  }
+
+  addHook(feature: Hook<HookT>) {
+    this.enabledHooks.add(feature);
+    feature.apply(this);
+  }
+
+  registerSyscalls(
+    requiredCapabilities: string[],
+    ...registrationObjects: SysCallMapping[]
+  ) {
+    for (const registrationObject of registrationObjects) {
+      for (const [name, callback] of Object.entries(registrationObject)) {
+        this.registeredSyscalls.set(name, {
+          requiredPermissions: requiredCapabilities,
+          callback,
+        });
+      }
+    }
+  }
+
+  syscallWithContext(
+    ctx: SyscallContext,
+    name: string,
+    args: any[],
+  ): Promise<any> {
+    const syscall = this.registeredSyscalls.get(name);
+    if (!syscall) {
+      throw Error(`Unregistered syscall ${name}`);
+    }
+    for (const permission of syscall.requiredPermissions) {
+      if (!ctx.plug) {
+        throw Error(`Syscall ${name} requires permission and no plug is set`);
+      }
+      if (!ctx.plug.grantedPermissions.includes(permission)) {
+        throw Error(`Missing permission '${permission}' for syscall ${name}`);
+      }
+    }
+    return Promise.resolve(syscall.callback(ctx, ...args));
+  }
+
+  localSyscall(
+    contextPlugName: string,
+    syscallName: string,
+    args: any[],
+  ): Promise<any> {
+    return this.syscallWithContext(
+      // Mock the plug
+      { plug: { name: contextPlugName } as any },
+      syscallName,
+      args,
+    );
+  }
+
+  async load(
+    manifest: Manifest<HookT>,
+    sandboxFactory: SandboxFactory<HookT>,
+  ): Promise<Plug<HookT>> {
+    const name = manifest.name;
+    if (this.plugs.has(name)) {
+      await this.unload(name);
+    }
+    // Validate
+    let errors: string[] = [];
+    for (const feature of this.enabledHooks) {
+      errors = [...errors, ...feature.validateManifest(manifest)];
+    }
+    if (errors.length > 0) {
+      throw new Error(`Invalid manifest: ${errors.join(", ")}`);
+    }
+    // Ok, let's load this thing!
+    const plug = new Plug(this, name, sandboxFactory);
+    console.log("Loading", name);
+    await plug.load(manifest);
+    this.plugs.set(name, plug);
+    await this.emit("plugLoaded", plug);
+    return plug;
+  }
+
+  async unload(name: string) {
+    // console.log("Unloading", name);
+    const plug = this.plugs.get(name);
+    if (!plug) {
+      throw Error(`Plug ${name} not found`);
+    }
+    await plug.stop();
+    this.emit("plugUnloaded", name);
+    this.plugs.delete(name);
+  }
+
+  toJSON(): SystemJSON<HookT> {
+    const plugJSON: Manifest<HookT>[] = [];
+    for (const [_, plug] of this.plugs) {
+      if (!plug.manifest) {
+        continue;
+      }
+      plugJSON.push(plug.manifest);
+    }
+    return plugJSON;
+  }
+
+  async replaceAllFromJSON(
+    json: SystemJSON<HookT>,
+    sandboxFactory: SandboxFactory<HookT>,
+  ) {
+    await this.unloadAll();
+    for (const manifest of json) {
+      // console.log("Loading plug", manifest.name);
+      await this.load(manifest, sandboxFactory);
+    }
+  }
+
+  unloadAll(): Promise<void[]> {
+    return Promise.all(
+      Array.from(this.plugs.keys()).map(this.unload.bind(this)),
+    );
+  }
+}

plugos/test.plug.yaml

@@ -0,0 +1,4 @@
+name: test
+functions:
+  boot:
+    path: "./test_func.test.ts:hello"

plugos/test_func.test.ts

@@ -0,0 +1,7 @@
+import * as YAML from "https://deno.land/std/encoding/yaml.ts";
+
+export function hello() {
+  console.log(YAML.stringify({ hello: "world" }));
+
+  return "hello";
+}

plugos/types.ts

@@ -0,0 +1,28 @@
+import { AssetBundle } from "../plugos/asset_bundle_reader.ts";
+import { System } from "./system.ts";
+
+export interface Manifest<HookT> {
+  name: string;
+  requiredPermissions?: string[];
+  assets?: string[] | AssetBundle;
+  dependencies?: {
+    [key: string]: string;
+  };
+  functions: {
+    [key: string]: FunctionDef<HookT>;
+  };
+}
+
+export type FunctionDef<HookT> = {
+  path?: string;
+  code?: string;
+  env?: RuntimeEnvironment;
+} & HookT;
+
+export type RuntimeEnvironment = "client" | "server";
+
+export interface Hook<HookT> {
+  validateManifest(manifest: Manifest<HookT>): string[];
+
+  apply(system: System<HookT>): void;
+}

plugos/util.ts

@@ -0,0 +1,7 @@
+export function safeRun(fn: () => Promise<void>) {
+  fn().catch((e: any) => {
+    console.error("Caught error", e.message);
+
+    // throw e;
+  });
+}