Rebranded plugbox to PlugOS (plugos)

This commit is contained in:
Zef Hemel
2022-03-27 11:31:12 +02:00
parent 621e55dbcf
commit cb809d2d03
45 changed files with 42 additions and 51 deletions
+121
View File
@@ -0,0 +1,121 @@
#!/usr/bin/env node
import esbuild from "esbuild";
import { readFile, unlink, writeFile } from "fs/promises";
import path from "path";
import yargs from "yargs";
import { hideBin } from "yargs/helpers";
import { Manifest } from "../types";
import { watchFile } from "fs";
import YAML from "yaml";
async function compile(filePath: string, functionName: string, debug: boolean) {
let outFile = "out.js";
let inFile = filePath;
if (functionName) {
// Generate a new file importing just this one function and exporting it
inFile = "in.js";
await writeFile(
inFile,
`import {${functionName}} from "./${filePath}";
export default ${functionName};`
);
}
// TODO: Figure out how to make source maps work correctly with eval() code
let js = await esbuild.build({
entryPoints: [inFile],
bundle: true,
format: "iife",
globalName: "mod",
platform: "neutral",
sourcemap: false, //sourceMap ? "inline" : false,
minify: !debug,
outfile: outFile,
});
let jsCode = (await readFile(outFile)).toString();
jsCode = jsCode.replace(/^var mod ?= ?/, "");
await unlink(outFile);
if (inFile !== filePath) {
await unlink(inFile);
}
// Strip final ';'
return jsCode.substring(0, jsCode.length - 2);
}
async function bundle(manifestPath: string, sourceMaps: boolean) {
const rootPath = path.dirname(manifestPath);
const manifest = YAML.parse(
(await readFile(manifestPath)).toString()
) as Manifest<any>;
for (let [name, def] of Object.entries(manifest.functions)) {
let jsFunctionName = "default",
filePath = path.join(rootPath, def.path!);
if (filePath.indexOf(":") !== -1) {
[filePath, jsFunctionName] = filePath.split(":");
}
def.code = await compile(filePath, jsFunctionName, sourceMaps);
delete def.path;
}
return manifest;
}
async function buildManifest(
manifestPath: string,
distPath: string,
debug: boolean
) {
let generatedManifest = await bundle(manifestPath, debug);
const outFile =
manifestPath.substring(
0,
manifestPath.length - path.extname(manifestPath).length
) + ".json";
const outPath = path.join(distPath, path.basename(outFile));
console.log("Emitting bundle to", outPath);
await writeFile(outPath, JSON.stringify(generatedManifest, null, 2));
return { generatedManifest, outPath };
}
async function run() {
let args = yargs(hideBin(process.argv))
.option("debug", {
type: "boolean",
})
.option("watch", {
type: "boolean",
alias: "w",
})
.option("dist", {
type: "string",
default: ".",
})
.parse();
if (args._.length === 0) {
console.log(
"Usage: plugos-bundle [--debug] [--dist <path>] <manifest.plug.yaml> <manifest2.plug.yaml> ..."
);
process.exit(1);
}
for (const plugManifestPath of args._) {
let manifestPath = plugManifestPath as string;
await buildManifest(manifestPath, args.dist, !!args.debug);
if (args.watch) {
watchFile(manifestPath, { interval: 1000 }, async () => {
console.log("Rebuilding", manifestPath);
await buildManifest(manifestPath, args.dist, !!args.debug);
});
}
}
}
run().catch((e) => {
console.error(e);
process.exit(1);
});
+71
View File
@@ -0,0 +1,71 @@
#!/usr/bin/env node
import express from "express";
import yargs from "yargs";
import { hideBin } from "yargs/helpers";
import { DiskPlugLoader } from "../plug_loader";
import { CronHook, NodeCronFeature } from "../feature/node_cron";
import shellSyscalls from "../syscall/shell.node";
import { System } from "../system";
import { EndpointFeature, EndpointHook } from "../feature/endpoint";
import { safeRun } from "../util";
import knex from "knex";
import {
ensureTable,
storeReadSyscalls,
storeWriteSyscalls,
} from "../syscall/store.knex_node";
import { fetchSyscalls } from "../syscall/fetch.node";
import { EventFeature, EventHook } from "../feature/event";
import { eventSyscalls } from "../syscall/event";
let args = yargs(hideBin(process.argv))
.option("port", {
type: "number",
default: 1337,
})
.parse();
if (!args._.length) {
console.error("Usage: plugos-server <path-to-plugs>");
process.exit(1);
}
const plugPath = args._[0] as string;
const app = express();
type ServerHook = EndpointHook & CronHook & EventHook;
const system = new System<ServerHook>("server");
safeRun(async () => {
const db = knex({
client: "better-sqlite3",
connection: {
filename: "plugos.db",
},
useNullAsDefault: true,
});
await ensureTable(db, "item");
let plugLoader = new DiskPlugLoader(system, plugPath);
await plugLoader.loadPlugs();
plugLoader.watcher();
system.addFeature(new NodeCronFeature());
let eventFeature = new EventFeature();
system.addFeature(eventFeature);
system.registerSyscalls("event", [], eventSyscalls(eventFeature));
system.addFeature(new EndpointFeature(app, ""));
system.registerSyscalls("shell", [], shellSyscalls("."));
system.registerSyscalls("fetch", [], fetchSyscalls());
system.registerSyscalls(
"store",
[],
storeWriteSyscalls(db, "item"),
storeReadSyscalls(db, "item")
);
app.listen(args.port, () => {
console.log(`Plugbox server listening on port ${args.port}`);
});
});
+8
View File
@@ -0,0 +1,8 @@
<html>
<body>
<script type="module">
// Sup yo!
import "./sandbox_worker";
</script>
</body>
</html>
+54
View File
@@ -0,0 +1,54 @@
import { safeRun } from "../util";
// @ts-ignore
import sandboxHtml from "bundle-text:./iframe_sandbox.html";
import { Sandbox } from "../sandbox";
import { WorkerLike } from "./worker";
import { Plug } from "../plug";
class IFrameWrapper implements WorkerLike {
private iframe: HTMLIFrameElement;
onMessage?: (message: any) => Promise<void>;
ready: Promise<void>;
private messageListener: (evt: any) => void;
constructor() {
const iframe = document.createElement("iframe", {});
this.iframe = iframe;
iframe.style.display = "none";
// Let's lock this down significantly
iframe.setAttribute("sandbox", "allow-scripts");
iframe.srcdoc = sandboxHtml;
this.messageListener = (evt: any) => {
if (evt.source !== iframe.contentWindow) {
return;
}
let data = evt.data;
if (!data) return;
safeRun(async () => {
await this.onMessage!(data);
});
};
window.addEventListener("message", this.messageListener);
document.body.appendChild(iframe);
this.ready = new Promise((resolve) => {
iframe.onload = () => {
resolve();
iframe.onload = null;
};
});
}
postMessage(message: any): void {
this.iframe.contentWindow!.postMessage(message, "*");
}
terminate() {
window.removeEventListener("message", this.messageListener);
return this.iframe.remove();
}
}
export function createSandbox(plug: Plug<any>) {
return new Sandbox(plug, new IFrameWrapper());
}
+50
View File
@@ -0,0 +1,50 @@
import { Worker } from "worker_threads";
import { safeRun } from "../util";
// @ts-ignore
import workerCode from "bundle-text:./node_worker.ts";
import { Sandbox } from "../sandbox";
import { WorkerLike } from "./worker";
import { Plug } from "../plug";
import path from "path";
import fs from "fs";
class NodeWorkerWrapper implements WorkerLike {
onMessage?: (message: any) => Promise<void>;
private worker: Worker;
ready: Promise<void>;
constructor(worker: Worker) {
this.worker = worker;
worker.on("message", (message: any) => {
safeRun(async () => {
await this.onMessage!(message);
});
});
this.ready = new Promise((resolve) => {
worker.once("online", resolve);
});
}
postMessage(message: any): void {
this.worker.postMessage(message);
}
terminate(): void {
this.worker.terminate();
}
}
// Look for the node_modules directory, to be passed to the worker to find e.g. the vm2 module
let nodeModulesDir = __dirname;
while (!fs.existsSync(nodeModulesDir + "/node_modules")) {
nodeModulesDir = path.dirname(nodeModulesDir);
}
export function createSandbox(plug: Plug<any>) {
let worker = new Worker(workerCode, {
eval: true,
workerData: path.join(nodeModulesDir, "node_modules"),
});
return new Sandbox(plug, new NodeWorkerWrapper(worker));
}
+104
View File
@@ -0,0 +1,104 @@
const { parentPort, workerData } = require("worker_threads");
let vm2 = `${workerData}/vm2`;
const { VM, VMScript } = require(vm2);
// console.log("Process env", process.env);
let loadedFunctions = new Map<string, Function>();
let pendingRequests = new Map<
number,
{
resolve: (result: unknown) => void;
reject: (e: any) => void;
}
>();
let syscallReqId = 0;
let vm = new VM({
sandbox: {
console: console,
self: {
syscall: (name: string, ...args: any[]) => {
return new Promise((resolve, reject) => {
syscallReqId++;
pendingRequests.set(syscallReqId, { resolve, reject });
parentPort.postMessage({
type: "syscall",
id: syscallReqId,
name,
// TODO: Figure out why this is necessary (to avoide a CloneError)
args: JSON.parse(JSON.stringify(args)),
});
});
},
},
},
});
function wrapScript(code: string) {
return `(${code})["default"]`;
}
function safeRun(fn: any) {
fn().catch((e: any) => {
console.error(e);
});
}
parentPort.on("message", (data: any) => {
safeRun(async () => {
switch (data.type) {
case "load":
console.log("Booting", data.name);
loadedFunctions.set(data.name, new VMScript(wrapScript(data.code)));
parentPort.postMessage({
type: "inited",
name: data.name,
});
break;
case "invoke":
let fn = loadedFunctions.get(data.name);
if (!fn) {
throw new Error(`Function not loaded: ${data.name}`);
}
try {
let r = vm.run(fn);
let result = await Promise.resolve(r(...data.args));
parentPort.postMessage({
type: "result",
id: data.id,
// TOOD: Figure out if this is necessary, because it's expensive
result: result && JSON.parse(JSON.stringify(result)),
});
} catch (e: any) {
// console.log("ERROR", e);
parentPort.postMessage({
type: "result",
id: data.id,
error: e.message,
});
}
break;
case "syscall-response":
let syscallId = data.id;
const lookup = pendingRequests.get(syscallId);
if (!lookup) {
console.log(
"Current outstanding requests",
pendingRequests,
"looking up",
syscallId
);
throw Error("Invalid request id");
}
pendingRequests.delete(syscallId);
if (data.error) {
lookup.reject(new Error(data.error));
} else {
lookup.resolve(data.result);
}
break;
}
});
});
+110
View File
@@ -0,0 +1,110 @@
import { safeRun } from "../util";
import { ControllerMessage, WorkerMessage } from "./worker";
let loadedFunctions = new Map<string, Function>();
let pendingRequests = new Map<
number,
{
resolve: (result: unknown) => void;
reject: (e: any) => void;
}
>();
let postMessage = self.postMessage.bind(self);
if (window.parent !== window) {
postMessage = window.parent.postMessage.bind(window.parent);
}
declare global {
function syscall(name: string, ...args: any[]): Promise<any>;
}
let syscallReqId = 0;
self.syscall = async (name: string, ...args: any[]) => {
return await new Promise((resolve, reject) => {
syscallReqId++;
pendingRequests.set(syscallReqId, { resolve, reject });
postMessage(
{
type: "syscall",
id: syscallReqId,
name,
args,
},
"*"
);
});
};
function wrapScript(code: string): string {
return `const fn = ${code};
return fn["default"].apply(null, arguments);`;
}
self.addEventListener("message", (event: { data: WorkerMessage }) => {
safeRun(async () => {
let messageEvent = event;
let data = messageEvent.data;
switch (data.type) {
case "load":
console.log("Booting", data.name);
loadedFunctions.set(data.name!, new Function(wrapScript(data.code!)));
postMessage(
{
type: "inited",
name: data.name,
} as ControllerMessage,
"*"
);
break;
case "invoke":
let fn = loadedFunctions.get(data.name!);
if (!fn) {
throw new Error(`Function not loaded: ${data.name}`);
}
try {
let result = await Promise.resolve(fn(...(data.args || [])));
postMessage(
{
type: "result",
id: data.id,
result: result,
} as ControllerMessage,
"*"
);
} catch (e: any) {
postMessage(
{
type: "result",
id: data.id,
error: e.message,
} as ControllerMessage,
"*"
);
throw e;
}
break;
case "syscall-response":
let syscallId = data.id!;
const lookup = pendingRequests.get(syscallId);
if (!lookup) {
console.log(
"Current outstanding requests",
pendingRequests,
"looking up",
syscallId
);
throw Error("Invalid request id");
}
pendingRequests.delete(syscallId);
if (data.error) {
lookup.reject(new Error(data.error));
} else {
lookup.resolve(data.result);
}
break;
}
});
});
+37
View File
@@ -0,0 +1,37 @@
import { safeRun } from "../util";
import { Sandbox } from "../sandbox";
import { WorkerLike } from "./worker";
import { Plug } from "../plug";
class WebWorkerWrapper implements WorkerLike {
private worker: Worker;
onMessage?: (message: any) => Promise<void>;
ready: Promise<void>;
constructor(worker: Worker) {
this.worker = worker;
this.worker.addEventListener("message", (evt: any) => {
let data = evt.data;
if (!data) return;
safeRun(async () => {
await this.onMessage!(data);
});
});
this.ready = Promise.resolve();
}
postMessage(message: any): void {
this.worker.postMessage(message);
}
terminate() {
return this.worker.terminate();
}
}
export function createSandbox(plug: Plug<any>) {
// ParcelJS will build this file into a worker.
let worker = new Worker(new URL("sandbox_worker.ts", import.meta.url), {
type: "module",
});
return new Sandbox(plug, new WebWorkerWrapper(worker));
}
+31
View File
@@ -0,0 +1,31 @@
export type ControllerMessageType = "inited" | "result" | "syscall";
export type ControllerMessage = {
type: ControllerMessageType;
id?: number;
name?: string;
args?: any[];
error?: string;
result?: any;
};
export interface WorkerLike {
ready: Promise<void>;
onMessage?: (message: any) => Promise<void>;
postMessage(message: any): void;
terminate(): void;
}
export type WorkerMessageType = "load" | "invoke" | "syscall-response";
export type WorkerMessage = {
type: WorkerMessageType;
id?: number;
name?: string;
code?: string;
args?: any[];
result?: any;
error?: any;
};
+49
View File
@@ -0,0 +1,49 @@
import { createSandbox } from "../environment/node_sandbox";
import { expect, test } from "@jest/globals";
import { Manifest } from "../types";
import express from "express";
import request from "supertest";
import { EndpointFeature, EndpointHook } from "./endpoint";
import { System } from "../system";
test("Run a plugos endpoint server", async () => {
let system = new System<EndpointHook>("server");
let plug = await system.load(
"test",
{
functions: {
testhandler: {
http: {
path: "/",
},
code: `(() => {
return {
default: (req) => {
console.log("Req", req);
return {status: 200, body: [1, 2, 3], headers: {"Content-type": "application/json"}};
}
};
})()`,
},
},
} as Manifest<EndpointHook>,
createSandbox
);
const app = express();
const port = 3123;
system.addFeature(new EndpointFeature(app, "/_"));
let server = app.listen(port, () => {
console.log(`Listening on port ${port}`);
});
let resp = await request(app)
.get("/_/test/?name=Pete")
.expect((resp) => {
expect(resp.status).toBe(200);
expect(resp.header["content-type"]).toContain("application/json");
expect(resp.text).toBe(JSON.stringify([1, 2, 3]));
});
server.close();
});
+134
View File
@@ -0,0 +1,134 @@
import { Feature, Manifest } from "../types";
import { Express, NextFunction, Request, Response } from "express";
import { System } from "../system";
export type EndpointRequest = {
method: string;
path: string;
query: { [key: string]: string };
headers: { [key: string]: string };
body: any;
};
export type EndpointResponse = {
status: number;
headers?: { [key: string]: string };
body: any;
};
export type EndpointHook = {
http?: EndPointDef | EndPointDef[];
};
export type EndPointDef = {
method?: "GET" | "POST" | "PUT" | "DELETE" | "HEAD" | "OPTIONS" | "ANY";
path: string;
};
export class EndpointFeature implements Feature<EndpointHook> {
private app: Express;
private prefix: string;
constructor(app: Express, prefix: string) {
this.app = app;
this.prefix = prefix;
}
apply(system: System<EndpointHook>): void {
this.app.use((req: Request, res: Response, next: NextFunction) => {
if (!req.path.startsWith(this.prefix)) {
return next();
}
console.log("Endpoint request", req.path);
Promise.resolve()
.then(async () => {
// Iterate over all loaded plugins
for (const [plugName, plug] of system.loadedPlugs.entries()) {
const manifest = plug.manifest;
if (!manifest) {
continue;
}
const functions = manifest.functions;
console.log("Checking plug", plugName);
let prefix = `${this.prefix}/${plugName}`;
if (!req.path.startsWith(prefix)) {
continue;
}
for (const [name, functionDef] of Object.entries(functions)) {
if (!functionDef.http) {
continue;
}
let endpoints = Array.isArray(functionDef.http)
? functionDef.http
: [functionDef.http];
console.log(endpoints);
for (const { path, method } of endpoints) {
let prefixedPath = `${prefix}${path}`;
if (
prefixedPath === req.path &&
((method || "GET") === req.method || method === "ANY")
) {
try {
const response: EndpointResponse = await plug.invoke(name, [
{
path: req.path,
method: req.method,
body: req.body,
query: req.query,
headers: req.headers,
} as EndpointRequest,
]);
let resp = res.status(response.status);
if (response.headers) {
for (const [key, value] of Object.entries(
response.headers
)) {
resp = resp.header(key, value);
}
}
resp.send(response.body);
return;
} catch (e: any) {
console.error("Error executing function", e);
res.status(500).send(e.message);
return;
}
}
}
}
}
next();
})
.catch((e) => {
console.error(e);
next(e);
});
});
}
validateManifest(manifest: Manifest<EndpointHook>): string[] {
let errors = [];
for (const [name, functionDef] of Object.entries(manifest.functions)) {
if (!functionDef.http) {
continue;
}
let endpoints = Array.isArray(functionDef.http)
? functionDef.http
: [functionDef.http];
for (let { path, method } of endpoints) {
if (!path) {
errors.push("Path not defined for endpoint");
}
if (
method &&
["GET", "POST", "PUT", "DELETE", "ANY"].indexOf(method) === -1
) {
errors.push(
`Invalid method ${method} for end point with with ${path}`
);
}
}
}
return errors;
}
}
+41
View File
@@ -0,0 +1,41 @@
import { Feature, Manifest } from "../types";
import { System } from "../system";
export type EventHook = {
events?: string[];
};
export class EventFeature implements Feature<EventHook> {
private system?: System<EventHook>;
async dispatchEvent(eventName: string, data?: any): Promise<any[]> {
if (!this.system) {
throw new Error("EventFeature is not initialized");
}
let promises: Promise<any>[] = [];
for (const plug of this.system.loadedPlugs.values()) {
for (const [name, functionDef] of Object.entries(
plug.manifest!.functions
)) {
if (functionDef.events && functionDef.events.includes(eventName)) {
promises.push(plug.invoke(name, [data]));
}
}
}
return Promise.all(promises);
}
apply(system: System<EventHook>): void {
this.system = system;
}
validateManifest(manifest: Manifest<EventHook>): string[] {
let errors = [];
for (const [name, functionDef] of Object.entries(manifest.functions)) {
if (functionDef.events && !Array.isArray(functionDef.events)) {
errors.push("'events' key must be an array of strings");
}
}
return errors;
}
}
+76
View File
@@ -0,0 +1,76 @@
import { Feature, Manifest } from "../types";
import cron, { ScheduledTask } from "node-cron";
import { safeRun } from "../util";
import { System } from "../system";
export type CronHook = {
cron?: string | string[];
};
export class NodeCronFeature implements Feature<CronHook> {
apply(system: System<CronHook>): void {
let tasks: ScheduledTask[] = [];
system.on({
plugLoaded: (name, plug) => {
reloadCrons();
},
plugUnloaded(name, plug) {
reloadCrons();
},
});
reloadCrons();
function reloadCrons() {
tasks.forEach((task) => task.stop());
tasks = [];
for (let plug of system.loadedPlugs.values()) {
if (!plug.manifest) {
continue;
}
for (const [name, functionDef] of Object.entries(
plug.manifest.functions
)) {
if (!functionDef.cron) {
continue;
}
const crons = Array.isArray(functionDef.cron)
? functionDef.cron
: [functionDef.cron];
for (let cronDef of crons) {
tasks.push(
cron.schedule(cronDef, () => {
console.log("Now acting on cron", cronDef);
safeRun(async () => {
try {
await plug.invoke(name, [cronDef]);
} catch (e: any) {
console.error("Execution of cron function failed", e);
}
});
})
);
}
}
}
}
}
validateManifest(manifest: Manifest<CronHook>): string[] {
let errors = [];
for (const [name, functionDef] of Object.entries(manifest.functions)) {
if (!functionDef.cron) {
continue;
}
const crons = Array.isArray(functionDef.cron)
? functionDef.cron
: [functionDef.cron];
for (let cronDef of crons) {
if (!cron.validate(cronDef)) {
errors.push(`Invalid cron expression ${cronDef}`);
}
}
}
return errors;
}
}
+64
View File
@@ -0,0 +1,64 @@
import { Manifest, RuntimeEnvironment } from "./types";
import { Sandbox } from "./sandbox";
import { System } from "./system";
export class Plug<HookT> {
system: System<HookT>;
sandbox: Sandbox;
public manifest?: Manifest<HookT>;
readonly runtimeEnv: RuntimeEnvironment;
grantedPermissions: string[] = [];
name: string;
constructor(
system: System<HookT>,
name: string,
sandboxFactory: (plug: Plug<HookT>) => Sandbox
) {
this.system = system;
this.name = name;
this.sandbox = sandboxFactory(this);
this.runtimeEnv = system.runtimeEnv;
}
async load(manifest: Manifest<HookT>) {
this.manifest = manifest;
// TODO: These need to be explicitly granted, not just taken
this.grantedPermissions = manifest.requiredPermissions || [];
}
syscall(name: string, args: any[]): Promise<any> {
return this.system.syscallWithContext({ plug: this }, name, args);
}
canInvoke(name: string) {
if (!this.manifest) {
return false;
}
const funDef = this.manifest.functions[name];
if (!funDef) {
throw new Error(`Function ${name} not found in manifest`);
}
return !funDef.env || funDef.env === this.runtimeEnv;
}
async invoke(name: string, args: Array<any>): Promise<any> {
if (!this.sandbox.isLoaded(name)) {
const funDef = this.manifest!.functions[name];
if (!funDef) {
throw new Error(`Function ${name} not found in manifest`);
}
if (!this.canInvoke(name)) {
throw new Error(
`Function ${name} is not available in ${this.runtimeEnv}`
);
}
await this.sandbox.load(name, funDef.code!);
}
return await this.sandbox.invoke(name, args);
}
async stop() {
this.sandbox.stop();
}
}
+67
View File
@@ -0,0 +1,67 @@
import fs, { watch } from "fs/promises";
import path from "path";
import { createSandbox } from "./environment/node_sandbox";
import { safeRun } from "../server/util";
import { System } from "./system";
function extractPlugName(localPath: string): string {
const baseName = path.basename(localPath);
return baseName.substring(0, baseName.length - ".plug.json".length);
}
export class DiskPlugLoader<HookT> {
private system: System<HookT>;
private plugPath: string;
constructor(system: System<HookT>, plugPath: string) {
this.system = system;
this.plugPath = plugPath;
}
watcher() {
safeRun(async () => {
for await (const { filename, eventType } of watch(this.plugPath)) {
if (!filename.endsWith(".plug.json")) {
return;
}
try {
let localPath = path.join(this.plugPath, filename);
const plugName = extractPlugName(localPath);
try {
await fs.stat(localPath);
} catch (e) {
// Likely removed
await this.system.unload(plugName);
}
const plugDef = await this.loadPlugFromFile(localPath);
} catch {
// ignore, error handled by loadPlug
}
}
});
}
private async loadPlugFromFile(localPath: string) {
const plug = await fs.readFile(localPath, "utf8");
const plugName = extractPlugName(localPath);
console.log("Now loading plug", plugName);
try {
const plugDef = JSON.parse(plug);
await this.system.load(plugName, plugDef, createSandbox);
return plugDef;
} catch (e) {
console.error("Could not parse plugin file", e);
throw e;
}
}
async loadPlugs() {
for (let filename of await fs.readdir(this.plugPath)) {
if (filename.endsWith(".plug.json")) {
let localPath = path.join(this.plugPath, filename);
await this.loadPlugFromFile(localPath);
}
}
}
}
+115
View File
@@ -0,0 +1,115 @@
import { createSandbox } from "./environment/node_sandbox";
import { expect, test } from "@jest/globals";
import { System } from "./system";
test("Run a Node sandbox", async () => {
let system = new System("server");
system.registerSyscalls("", [], {
addNumbers: (ctx, a, b) => {
return a + b;
},
failingSyscall: () => {
throw new Error("#fail");
},
});
system.registerSyscalls("", ["restricted"], {
restrictedSyscall: () => {
return "restricted";
},
});
system.registerSyscalls("", ["dangerous"], {
dangerousSyscall: () => {
return "yay";
},
});
let plug = await system.load(
"test",
{
requiredPermissions: ["dangerous"],
functions: {
addTen: {
code: `(() => {
return {
default: (n) => {
return n + 10;
}
};
})()`,
},
addNumbersSyscall: {
code: `(() => {
return {
default: async (a, b) => {
return await self.syscall("addNumbers", a, b);
}
};
})()`,
},
errorOut: {
code: `(() => {
return {
default: () => {
throw Error("BOOM");
}
};
})()`,
},
errorOutSys: {
code: `(() => {
return {
default: async () => {
await self.syscall("failingSyscall");
}
};
})()`,
},
restrictedTest: {
code: `(() => {
return {
default: async () => {
await self.syscall("restrictedSyscall");
}
};
})()`,
},
dangerousTest: {
code: `(() => {
return {
default: async () => {
return await self.syscall("dangerousSyscall");
}
};
})()`,
},
},
},
createSandbox
);
expect(await plug.invoke("addTen", [10])).toBe(20);
for (let i = 0; i < 100; i++) {
expect(await plug.invoke("addNumbersSyscall", [10, i])).toBe(10 + i);
}
try {
await plug.invoke("errorOut", []);
expect(true).toBe(false);
} catch (e: any) {
expect(e.message).toBe("BOOM");
}
try {
await plug.invoke("errorOutSys", []);
expect(true).toBe(false);
} catch (e: any) {
expect(e.message).toBe("#fail");
}
try {
await plug.invoke("restrictedTest", []);
expect(true).toBe(false);
} catch (e: any) {
expect(e.message).toBe(
"Missing permission 'restricted' for syscall restrictedSyscall"
);
}
expect(await plug.invoke("dangerousTest", [])).toBe("yay");
await system.unloadAll();
});
+98
View File
@@ -0,0 +1,98 @@
import {
ControllerMessage,
WorkerLike,
WorkerMessage,
} from "./environment/worker";
import { Plug } from "./plug";
export type SandboxFactory<HookT> = (plug: Plug<HookT>) => Sandbox;
export class Sandbox {
protected worker: WorkerLike;
protected reqId = 0;
protected outstandingInits = new Map<string, () => void>();
protected outstandingInvocations = new Map<
number,
{ resolve: (result: any) => void; reject: (e: any) => void }
>();
protected loadedFunctions = new Set<string>();
protected plug: Plug<any>;
constructor(plug: Plug<any>, worker: WorkerLike) {
worker.onMessage = this.onMessage.bind(this);
this.worker = worker;
this.plug = plug;
}
isLoaded(name: string) {
return this.loadedFunctions.has(name);
}
async load(name: string, code: string): Promise<void> {
await this.worker.ready;
this.worker.postMessage({
type: "load",
name: name,
code: code,
} as WorkerMessage);
return new Promise((resolve) => {
this.loadedFunctions.add(name);
this.outstandingInits.set(name, resolve);
});
}
async onMessage(data: ControllerMessage) {
switch (data.type) {
case "inited":
let initCb = this.outstandingInits.get(data.name!);
initCb && initCb();
this.outstandingInits.delete(data.name!);
break;
case "syscall":
try {
let result = await this.plug.syscall(data.name!, data.args!);
this.worker.postMessage({
type: "syscall-response",
id: data.id,
result: result,
} as WorkerMessage);
} catch (e: any) {
this.worker.postMessage({
type: "syscall-response",
id: data.id,
error: e.message,
} as WorkerMessage);
}
break;
case "result":
let resultCbs = this.outstandingInvocations.get(data.id!);
this.outstandingInvocations.delete(data.id!);
if (data.error) {
resultCbs && resultCbs.reject(new Error(data.error));
} else {
resultCbs && resultCbs.resolve(data.result);
}
break;
default:
console.error("Unknown message type", data);
}
}
async invoke(name: string, args: any[]): Promise<any> {
this.reqId++;
this.worker.postMessage({
type: "invoke",
id: this.reqId,
name,
args,
});
return new Promise((resolve, reject) => {
this.outstandingInvocations.set(this.reqId, { resolve, reject });
});
}
stop() {
this.worker.terminate();
}
}
+10
View File
@@ -0,0 +1,10 @@
import { SysCallMapping } from "../system";
import { EventFeature } from "../feature/event";
export function eventSyscalls(eventFeature: EventFeature): SysCallMapping {
return {
async dispatch(ctx, eventName: string, data: any) {
return eventFeature.dispatchEvent(eventName, data);
},
};
}
+15
View File
@@ -0,0 +1,15 @@
import fetch, { RequestInfo, RequestInit } from "node-fetch";
import { SysCallMapping } from "../system";
export function fetchSyscalls(): SysCallMapping {
return {
async json(ctx, url: RequestInfo, init: RequestInit) {
let resp = await fetch(url, init);
return resp.json();
},
async text(ctx, url: RequestInfo, init: RequestInit) {
let resp = await fetch(url, init);
return resp.text();
},
};
}
+20
View File
@@ -0,0 +1,20 @@
import { promisify } from "util";
import { execFile } from "child_process";
import type { SysCallMapping } from "../system";
const execFilePromise = promisify(execFile);
export default function (cwd: string): SysCallMapping {
return {
run: async (
ctx,
cmd: string,
args: string[]
): Promise<{ stdout: string; stderr: string }> => {
let { stdout, stderr } = await execFilePromise(cmd, args, {
cwd: cwd,
});
return { stdout, stderr };
},
};
}
@@ -0,0 +1,49 @@
import { createSandbox } from "../environment/node_sandbox";
import { expect, test } from "@jest/globals";
import { System } from "../system";
import { storeSyscalls } from "./store.dexie_browser";
// For testing in node.js
require("fake-indexeddb/auto");
test("Test store", async () => {
let system = new System("server");
system.registerSyscalls("store", [], storeSyscalls("test", "test"));
let plug = await system.load(
"test",
{
functions: {
test1: {
code: `(() => {
return {
default: async () => {
await self.syscall("store.set", "name", "Pete");
return await self.syscall("store.get", "name");
}
};
})()`,
},
test2: {
code: `(() => {
return {
default: async () => {
await self.syscall("store.set", "page1:bl:page2:10", {title: "Something", meta: 20});
await self.syscall("store.batchSet", [
{key: "page2:bl:page3", value: {title: "Something2", meta: 10}},
{key: "page2:bl:page4", value: {title: "Something3", meta: 10}},
]);
return await self.syscall("store.queryPrefix", "page2:");
}
};
})()`,
},
},
},
createSandbox
);
expect(await plug.invoke("test1", [])).toBe("Pete");
let queryResults = await plug.invoke("test2", []);
expect(queryResults.length).toBe(2);
expect(queryResults[0].value.meta).toBe(10);
await system.unloadAll();
});
+66
View File
@@ -0,0 +1,66 @@
import Dexie from "dexie";
import { SysCallMapping } from "../system";
export type KV = {
key: string;
value: any;
};
export function storeSyscalls(
dbName: string,
tableName: string
): SysCallMapping {
const db = new Dexie(dbName);
db.version(1).stores({
test: "key",
});
const items = db.table(tableName);
return {
async delete(ctx, key: string) {
await items.delete(key);
},
async deletePrefix(ctx, prefix: string) {
await items.where("key").startsWith(prefix).delete();
},
async deleteAll() {
await items.clear();
},
async set(ctx, key: string, value: any) {
await items.put({
key,
value,
});
},
async batchSet(ctx, kvs: KV[]) {
await items.bulkPut(
kvs.map(({ key, value }) => ({
key,
value,
}))
);
},
async get(ctx, key: string): Promise<any | null> {
let result = await items.get({
key,
});
return result ? result.value : null;
},
async queryPrefix(
ctx,
keyPrefix: string
): Promise<{ key: string; value: any }[]> {
let results = await items.where("key").startsWith(keyPrefix).toArray();
return results.map((result) => ({
key: result.key,
value: result.value,
}));
},
};
}
+49
View File
@@ -0,0 +1,49 @@
import { createSandbox } from "../environment/node_sandbox";
import { expect, test } from "@jest/globals";
import { System } from "../system";
import {
ensureTable,
storeReadSyscalls,
storeWriteSyscalls,
} from "./store.knex_node";
import knex from "knex";
import fs from "fs/promises";
test("Test store", async () => {
const db = knex({
client: "better-sqlite3",
connection: {
filename: "test.db",
},
useNullAsDefault: true,
});
await ensureTable(db, "test_table");
let system = new System("server");
system.registerSyscalls(
"store",
[],
storeWriteSyscalls(db, "test_table"),
storeReadSyscalls(db, "test_table")
);
let plug = await system.load(
"test",
{
functions: {
test1: {
code: `(() => {
return {
default: async () => {
await self.syscall("store.set", "name", "Pete");
return await self.syscall("store.get", "name");
}
};
})()`,
},
},
},
createSandbox
);
expect(await plug.invoke("test1", [])).toBe("Pete");
await system.unloadAll();
await fs.unlink("test.db");
});
+84
View File
@@ -0,0 +1,84 @@
import { Knex } from "knex";
import { SysCallMapping } from "../system";
type Item = {
page: string;
key: string;
value: any;
};
export type KV = {
key: string;
value: any;
};
export async function ensureTable(db: Knex<any, unknown>, tableName: string) {
if (!(await db.schema.hasTable(tableName))) {
await db.schema.createTable(tableName, (table) => {
table.string("key");
table.text("value");
table.primary(["key"]);
});
console.log(`Created table ${tableName}`);
}
}
export function storeWriteSyscalls(
db: Knex<any, unknown>,
tableName: string
): SysCallMapping {
const apiObj: SysCallMapping = {
delete: async (ctx, page: string, key: string) => {
await db<Item>(tableName).where({ page, key }).del();
},
deletePrefix: async (ctx, prefix: string) => {
return db<Item>(tableName).andWhereLike("key", `${prefix}%`).del();
},
deleteAll: async (ctx) => {
await db<Item>(tableName).del();
},
set: async (ctx, key: string, value: any) => {
let changed = await db<Item>(tableName)
.where({ key })
.update("value", JSON.stringify(value));
if (changed === 0) {
await db<Item>(tableName).insert({
key,
value: JSON.stringify(value),
});
}
},
batchSet: async (ctx, kvs: KV[]) => {
for (let { key, value } of kvs) {
await apiObj["store.set"](ctx, key, value);
}
},
};
return apiObj;
}
export function storeReadSyscalls(
db: Knex<any, unknown>,
tableName: string
): SysCallMapping {
return {
get: async (ctx, key: string): Promise<any | null> => {
let result = await db<Item>(tableName).where({ key }).select("value");
if (result.length) {
return JSON.parse(result[0].value);
} else {
return null;
}
},
queryPrefix: async (ctx, prefix: string) => {
return (
await db<Item>(tableName)
.andWhereLike("key", `${prefix}%`)
.select("key", "value")
).map(({ key, value }) => ({
key,
value: JSON.parse(value),
}));
},
};
}
+16
View File
@@ -0,0 +1,16 @@
import { SysCallMapping } from "../system";
export function transportSyscalls(
names: string[],
transportCall: (name: string, ...args: any[]) => Promise<any>
): SysCallMapping {
let syscalls: SysCallMapping = {};
for (let name of names) {
syscalls[name] = (ctx, ...args: any[]) => {
return transportCall(name, ...args);
};
}
return syscalls;
}
+149
View File
@@ -0,0 +1,149 @@
import { Feature, Manifest, RuntimeEnvironment } from "./types";
import { EventEmitter } from "../common/event";
import { SandboxFactory } from "./sandbox";
import { Plug } from "./plug";
export interface SysCallMapping {
[key: string]: (ctx: SyscallContext, ...args: any) => Promise<any> | any;
}
export type SystemJSON<HookT> = { [key: string]: Manifest<HookT> };
export type SystemEvents<HookT> = {
plugLoaded: (name: string, plug: Plug<HookT>) => void;
plugUnloaded: (name: string, plug: Plug<HookT>) => void;
};
type SyscallContext = {
plug: Plug<any> | null;
};
type SyscallSignature = (
ctx: SyscallContext,
...args: any[]
) => Promise<any> | any;
type Syscall = {
requiredPermissions: string[];
callback: SyscallSignature;
};
export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
protected plugs = new Map<string, Plug<HookT>>();
protected registeredSyscalls = new Map<string, Syscall>();
protected enabledFeatures = new Set<Feature<HookT>>();
readonly runtimeEnv: RuntimeEnvironment;
constructor(env: RuntimeEnvironment) {
super();
this.runtimeEnv = env;
}
addFeature(feature: Feature<HookT>) {
this.enabledFeatures.add(feature);
feature.apply(this);
}
registerSyscalls(
namespace: string,
requiredCapabilities: string[],
...registrationObjects: SysCallMapping[]
) {
for (const registrationObject of registrationObjects) {
for (let [name, callback] of Object.entries(registrationObject)) {
const callName = namespace ? `${namespace}.${name}` : name;
this.registeredSyscalls.set(callName, {
requiredPermissions: requiredCapabilities,
callback,
});
}
}
}
async syscallWithContext(
ctx: SyscallContext,
name: string,
args: any[]
): Promise<any> {
const syscall = this.registeredSyscalls.get(name);
if (!syscall) {
throw Error(`Unregistered syscall ${name}`);
}
for (const permission of syscall.requiredPermissions) {
if (!ctx.plug) {
throw Error(`Syscall ${name} requires permission and no plug is set`);
}
if (!ctx.plug.grantedPermissions.includes(permission)) {
throw Error(`Missing permission '${permission}' for syscall ${name}`);
}
}
return Promise.resolve(syscall.callback(ctx, ...args));
}
async load(
name: string,
manifest: Manifest<HookT>,
sandboxFactory: SandboxFactory<HookT>
): Promise<Plug<HookT>> {
if (this.plugs.has(name)) {
await this.unload(name);
}
// Validate
let errors: string[] = [];
for (const feature of this.enabledFeatures) {
errors = [...errors, ...feature.validateManifest(manifest)];
}
if (errors.length > 0) {
throw new Error(`Invalid manifest: ${errors.join(", ")}`);
}
// Ok, let's load this thing!
const plug = new Plug(this, name, sandboxFactory);
await plug.load(manifest);
this.plugs.set(name, plug);
this.emit("plugLoaded", name, plug);
return plug;
}
async unload(name: string) {
const plug = this.plugs.get(name);
if (!plug) {
throw Error(`Plug ${name} not found`);
}
await plug.stop();
this.emit("plugUnloaded", name, plug);
this.plugs.delete(name);
}
get loadedPlugs(): Map<string, Plug<HookT>> {
return this.plugs;
}
toJSON(): SystemJSON<HookT> {
let plugJSON: { [key: string]: Manifest<HookT> } = {};
for (let [name, plug] of this.plugs) {
if (!plug.manifest) {
continue;
}
plugJSON[name] = plug.manifest;
}
return plugJSON;
}
async replaceAllFromJSON(
json: SystemJSON<HookT>,
sandboxFactory: SandboxFactory<HookT>
) {
await this.unloadAll();
for (let [name, manifest] of Object.entries(json)) {
console.log("Loading plug", name);
await this.load(name, manifest, sandboxFactory);
}
}
async unloadAll(): Promise<void[]> {
return Promise.all(
Array.from(this.plugs.keys()).map(this.unload.bind(this))
);
}
}
+22
View File
@@ -0,0 +1,22 @@
import { System } from "./system";
export interface Manifest<HookT> {
requiredPermissions?: string[];
functions: {
[key: string]: FunctionDef<HookT>;
};
}
export type FunctionDef<HookT> = {
path?: string;
code?: string;
env?: RuntimeEnvironment;
} & HookT;
export type RuntimeEnvironment = "client" | "server";
export interface Feature<HookT> {
validateManifest(manifest: Manifest<HookT>): string[];
apply(system: System<HookT>): void;
}
+6
View File
@@ -0,0 +1,6 @@
export function safeRun(fn: () => Promise<void>) {
fn().catch((e) => {
// console.error(e);
throw e;
});
}