Rebranded plugbox to PlugOS (plugos)
This commit is contained in:
Executable
+121
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import esbuild from "esbuild";
|
||||
import { readFile, unlink, writeFile } from "fs/promises";
|
||||
import path from "path";
|
||||
|
||||
import yargs from "yargs";
|
||||
import { hideBin } from "yargs/helpers";
|
||||
import { Manifest } from "../types";
|
||||
import { watchFile } from "fs";
|
||||
import YAML from "yaml";
|
||||
|
||||
async function compile(filePath: string, functionName: string, debug: boolean) {
|
||||
let outFile = "out.js";
|
||||
|
||||
let inFile = filePath;
|
||||
|
||||
if (functionName) {
|
||||
// Generate a new file importing just this one function and exporting it
|
||||
inFile = "in.js";
|
||||
await writeFile(
|
||||
inFile,
|
||||
`import {${functionName}} from "./${filePath}";
|
||||
export default ${functionName};`
|
||||
);
|
||||
}
|
||||
|
||||
// TODO: Figure out how to make source maps work correctly with eval() code
|
||||
let js = await esbuild.build({
|
||||
entryPoints: [inFile],
|
||||
bundle: true,
|
||||
format: "iife",
|
||||
globalName: "mod",
|
||||
platform: "neutral",
|
||||
sourcemap: false, //sourceMap ? "inline" : false,
|
||||
minify: !debug,
|
||||
outfile: outFile,
|
||||
});
|
||||
|
||||
let jsCode = (await readFile(outFile)).toString();
|
||||
jsCode = jsCode.replace(/^var mod ?= ?/, "");
|
||||
await unlink(outFile);
|
||||
if (inFile !== filePath) {
|
||||
await unlink(inFile);
|
||||
}
|
||||
// Strip final ';'
|
||||
return jsCode.substring(0, jsCode.length - 2);
|
||||
}
|
||||
|
||||
async function bundle(manifestPath: string, sourceMaps: boolean) {
|
||||
const rootPath = path.dirname(manifestPath);
|
||||
const manifest = YAML.parse(
|
||||
(await readFile(manifestPath)).toString()
|
||||
) as Manifest<any>;
|
||||
|
||||
for (let [name, def] of Object.entries(manifest.functions)) {
|
||||
let jsFunctionName = "default",
|
||||
filePath = path.join(rootPath, def.path!);
|
||||
if (filePath.indexOf(":") !== -1) {
|
||||
[filePath, jsFunctionName] = filePath.split(":");
|
||||
}
|
||||
|
||||
def.code = await compile(filePath, jsFunctionName, sourceMaps);
|
||||
delete def.path;
|
||||
}
|
||||
return manifest;
|
||||
}
|
||||
|
||||
async function buildManifest(
|
||||
manifestPath: string,
|
||||
distPath: string,
|
||||
debug: boolean
|
||||
) {
|
||||
let generatedManifest = await bundle(manifestPath, debug);
|
||||
const outFile =
|
||||
manifestPath.substring(
|
||||
0,
|
||||
manifestPath.length - path.extname(manifestPath).length
|
||||
) + ".json";
|
||||
const outPath = path.join(distPath, path.basename(outFile));
|
||||
console.log("Emitting bundle to", outPath);
|
||||
await writeFile(outPath, JSON.stringify(generatedManifest, null, 2));
|
||||
return { generatedManifest, outPath };
|
||||
}
|
||||
|
||||
async function run() {
|
||||
let args = yargs(hideBin(process.argv))
|
||||
.option("debug", {
|
||||
type: "boolean",
|
||||
})
|
||||
.option("watch", {
|
||||
type: "boolean",
|
||||
alias: "w",
|
||||
})
|
||||
.option("dist", {
|
||||
type: "string",
|
||||
default: ".",
|
||||
})
|
||||
.parse();
|
||||
if (args._.length === 0) {
|
||||
console.log(
|
||||
"Usage: plugos-bundle [--debug] [--dist <path>] <manifest.plug.yaml> <manifest2.plug.yaml> ..."
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
for (const plugManifestPath of args._) {
|
||||
let manifestPath = plugManifestPath as string;
|
||||
await buildManifest(manifestPath, args.dist, !!args.debug);
|
||||
if (args.watch) {
|
||||
watchFile(manifestPath, { interval: 1000 }, async () => {
|
||||
console.log("Rebuilding", manifestPath);
|
||||
await buildManifest(manifestPath, args.dist, !!args.debug);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
run().catch((e) => {
|
||||
console.error(e);
|
||||
process.exit(1);
|
||||
});
|
||||
Executable
+71
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import express from "express";
|
||||
import yargs from "yargs";
|
||||
import { hideBin } from "yargs/helpers";
|
||||
import { DiskPlugLoader } from "../plug_loader";
|
||||
import { CronHook, NodeCronFeature } from "../feature/node_cron";
|
||||
import shellSyscalls from "../syscall/shell.node";
|
||||
import { System } from "../system";
|
||||
import { EndpointFeature, EndpointHook } from "../feature/endpoint";
|
||||
import { safeRun } from "../util";
|
||||
import knex from "knex";
|
||||
import {
|
||||
ensureTable,
|
||||
storeReadSyscalls,
|
||||
storeWriteSyscalls,
|
||||
} from "../syscall/store.knex_node";
|
||||
import { fetchSyscalls } from "../syscall/fetch.node";
|
||||
import { EventFeature, EventHook } from "../feature/event";
|
||||
import { eventSyscalls } from "../syscall/event";
|
||||
|
||||
let args = yargs(hideBin(process.argv))
|
||||
.option("port", {
|
||||
type: "number",
|
||||
default: 1337,
|
||||
})
|
||||
.parse();
|
||||
|
||||
if (!args._.length) {
|
||||
console.error("Usage: plugos-server <path-to-plugs>");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const plugPath = args._[0] as string;
|
||||
|
||||
const app = express();
|
||||
|
||||
type ServerHook = EndpointHook & CronHook & EventHook;
|
||||
const system = new System<ServerHook>("server");
|
||||
|
||||
safeRun(async () => {
|
||||
const db = knex({
|
||||
client: "better-sqlite3",
|
||||
connection: {
|
||||
filename: "plugos.db",
|
||||
},
|
||||
useNullAsDefault: true,
|
||||
});
|
||||
|
||||
await ensureTable(db, "item");
|
||||
|
||||
let plugLoader = new DiskPlugLoader(system, plugPath);
|
||||
await plugLoader.loadPlugs();
|
||||
plugLoader.watcher();
|
||||
system.addFeature(new NodeCronFeature());
|
||||
let eventFeature = new EventFeature();
|
||||
system.addFeature(eventFeature);
|
||||
system.registerSyscalls("event", [], eventSyscalls(eventFeature));
|
||||
system.addFeature(new EndpointFeature(app, ""));
|
||||
system.registerSyscalls("shell", [], shellSyscalls("."));
|
||||
system.registerSyscalls("fetch", [], fetchSyscalls());
|
||||
system.registerSyscalls(
|
||||
"store",
|
||||
[],
|
||||
storeWriteSyscalls(db, "item"),
|
||||
storeReadSyscalls(db, "item")
|
||||
);
|
||||
app.listen(args.port, () => {
|
||||
console.log(`Plugbox server listening on port ${args.port}`);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,8 @@
|
||||
<html>
|
||||
<body>
|
||||
<script type="module">
|
||||
// Sup yo!
|
||||
import "./sandbox_worker";
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,54 @@
|
||||
import { safeRun } from "../util";
|
||||
|
||||
// @ts-ignore
|
||||
import sandboxHtml from "bundle-text:./iframe_sandbox.html";
|
||||
import { Sandbox } from "../sandbox";
|
||||
import { WorkerLike } from "./worker";
|
||||
import { Plug } from "../plug";
|
||||
|
||||
class IFrameWrapper implements WorkerLike {
|
||||
private iframe: HTMLIFrameElement;
|
||||
onMessage?: (message: any) => Promise<void>;
|
||||
ready: Promise<void>;
|
||||
private messageListener: (evt: any) => void;
|
||||
|
||||
constructor() {
|
||||
const iframe = document.createElement("iframe", {});
|
||||
this.iframe = iframe;
|
||||
iframe.style.display = "none";
|
||||
// Let's lock this down significantly
|
||||
iframe.setAttribute("sandbox", "allow-scripts");
|
||||
iframe.srcdoc = sandboxHtml;
|
||||
this.messageListener = (evt: any) => {
|
||||
if (evt.source !== iframe.contentWindow) {
|
||||
return;
|
||||
}
|
||||
let data = evt.data;
|
||||
if (!data) return;
|
||||
safeRun(async () => {
|
||||
await this.onMessage!(data);
|
||||
});
|
||||
};
|
||||
window.addEventListener("message", this.messageListener);
|
||||
document.body.appendChild(iframe);
|
||||
this.ready = new Promise((resolve) => {
|
||||
iframe.onload = () => {
|
||||
resolve();
|
||||
iframe.onload = null;
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
postMessage(message: any): void {
|
||||
this.iframe.contentWindow!.postMessage(message, "*");
|
||||
}
|
||||
|
||||
terminate() {
|
||||
window.removeEventListener("message", this.messageListener);
|
||||
return this.iframe.remove();
|
||||
}
|
||||
}
|
||||
|
||||
export function createSandbox(plug: Plug<any>) {
|
||||
return new Sandbox(plug, new IFrameWrapper());
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { Worker } from "worker_threads";
|
||||
import { safeRun } from "../util";
|
||||
|
||||
// @ts-ignore
|
||||
import workerCode from "bundle-text:./node_worker.ts";
|
||||
import { Sandbox } from "../sandbox";
|
||||
import { WorkerLike } from "./worker";
|
||||
import { Plug } from "../plug";
|
||||
import path from "path";
|
||||
import fs from "fs";
|
||||
|
||||
class NodeWorkerWrapper implements WorkerLike {
|
||||
onMessage?: (message: any) => Promise<void>;
|
||||
private worker: Worker;
|
||||
ready: Promise<void>;
|
||||
|
||||
constructor(worker: Worker) {
|
||||
this.worker = worker;
|
||||
worker.on("message", (message: any) => {
|
||||
safeRun(async () => {
|
||||
await this.onMessage!(message);
|
||||
});
|
||||
});
|
||||
this.ready = new Promise((resolve) => {
|
||||
worker.once("online", resolve);
|
||||
});
|
||||
}
|
||||
|
||||
postMessage(message: any): void {
|
||||
this.worker.postMessage(message);
|
||||
}
|
||||
|
||||
terminate(): void {
|
||||
this.worker.terminate();
|
||||
}
|
||||
}
|
||||
|
||||
// Look for the node_modules directory, to be passed to the worker to find e.g. the vm2 module
|
||||
let nodeModulesDir = __dirname;
|
||||
while (!fs.existsSync(nodeModulesDir + "/node_modules")) {
|
||||
nodeModulesDir = path.dirname(nodeModulesDir);
|
||||
}
|
||||
|
||||
export function createSandbox(plug: Plug<any>) {
|
||||
let worker = new Worker(workerCode, {
|
||||
eval: true,
|
||||
workerData: path.join(nodeModulesDir, "node_modules"),
|
||||
});
|
||||
return new Sandbox(plug, new NodeWorkerWrapper(worker));
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
const { parentPort, workerData } = require("worker_threads");
|
||||
let vm2 = `${workerData}/vm2`;
|
||||
const { VM, VMScript } = require(vm2);
|
||||
|
||||
// console.log("Process env", process.env);
|
||||
|
||||
let loadedFunctions = new Map<string, Function>();
|
||||
let pendingRequests = new Map<
|
||||
number,
|
||||
{
|
||||
resolve: (result: unknown) => void;
|
||||
reject: (e: any) => void;
|
||||
}
|
||||
>();
|
||||
|
||||
let syscallReqId = 0;
|
||||
|
||||
let vm = new VM({
|
||||
sandbox: {
|
||||
console: console,
|
||||
self: {
|
||||
syscall: (name: string, ...args: any[]) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
syscallReqId++;
|
||||
pendingRequests.set(syscallReqId, { resolve, reject });
|
||||
parentPort.postMessage({
|
||||
type: "syscall",
|
||||
id: syscallReqId,
|
||||
name,
|
||||
// TODO: Figure out why this is necessary (to avoide a CloneError)
|
||||
args: JSON.parse(JSON.stringify(args)),
|
||||
});
|
||||
});
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
function wrapScript(code: string) {
|
||||
return `(${code})["default"]`;
|
||||
}
|
||||
|
||||
function safeRun(fn: any) {
|
||||
fn().catch((e: any) => {
|
||||
console.error(e);
|
||||
});
|
||||
}
|
||||
|
||||
parentPort.on("message", (data: any) => {
|
||||
safeRun(async () => {
|
||||
switch (data.type) {
|
||||
case "load":
|
||||
console.log("Booting", data.name);
|
||||
loadedFunctions.set(data.name, new VMScript(wrapScript(data.code)));
|
||||
parentPort.postMessage({
|
||||
type: "inited",
|
||||
name: data.name,
|
||||
});
|
||||
break;
|
||||
case "invoke":
|
||||
let fn = loadedFunctions.get(data.name);
|
||||
if (!fn) {
|
||||
throw new Error(`Function not loaded: ${data.name}`);
|
||||
}
|
||||
try {
|
||||
let r = vm.run(fn);
|
||||
let result = await Promise.resolve(r(...data.args));
|
||||
parentPort.postMessage({
|
||||
type: "result",
|
||||
id: data.id,
|
||||
// TOOD: Figure out if this is necessary, because it's expensive
|
||||
result: result && JSON.parse(JSON.stringify(result)),
|
||||
});
|
||||
} catch (e: any) {
|
||||
// console.log("ERROR", e);
|
||||
parentPort.postMessage({
|
||||
type: "result",
|
||||
id: data.id,
|
||||
error: e.message,
|
||||
});
|
||||
}
|
||||
break;
|
||||
case "syscall-response":
|
||||
let syscallId = data.id;
|
||||
const lookup = pendingRequests.get(syscallId);
|
||||
if (!lookup) {
|
||||
console.log(
|
||||
"Current outstanding requests",
|
||||
pendingRequests,
|
||||
"looking up",
|
||||
syscallId
|
||||
);
|
||||
throw Error("Invalid request id");
|
||||
}
|
||||
pendingRequests.delete(syscallId);
|
||||
if (data.error) {
|
||||
lookup.reject(new Error(data.error));
|
||||
} else {
|
||||
lookup.resolve(data.result);
|
||||
}
|
||||
break;
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,110 @@
|
||||
import { safeRun } from "../util";
|
||||
import { ControllerMessage, WorkerMessage } from "./worker";
|
||||
|
||||
let loadedFunctions = new Map<string, Function>();
|
||||
let pendingRequests = new Map<
|
||||
number,
|
||||
{
|
||||
resolve: (result: unknown) => void;
|
||||
reject: (e: any) => void;
|
||||
}
|
||||
>();
|
||||
let postMessage = self.postMessage.bind(self);
|
||||
|
||||
if (window.parent !== window) {
|
||||
postMessage = window.parent.postMessage.bind(window.parent);
|
||||
}
|
||||
|
||||
declare global {
|
||||
function syscall(name: string, ...args: any[]): Promise<any>;
|
||||
}
|
||||
|
||||
let syscallReqId = 0;
|
||||
|
||||
self.syscall = async (name: string, ...args: any[]) => {
|
||||
return await new Promise((resolve, reject) => {
|
||||
syscallReqId++;
|
||||
pendingRequests.set(syscallReqId, { resolve, reject });
|
||||
postMessage(
|
||||
{
|
||||
type: "syscall",
|
||||
id: syscallReqId,
|
||||
name,
|
||||
args,
|
||||
},
|
||||
"*"
|
||||
);
|
||||
});
|
||||
};
|
||||
|
||||
function wrapScript(code: string): string {
|
||||
return `const fn = ${code};
|
||||
return fn["default"].apply(null, arguments);`;
|
||||
}
|
||||
|
||||
self.addEventListener("message", (event: { data: WorkerMessage }) => {
|
||||
safeRun(async () => {
|
||||
let messageEvent = event;
|
||||
let data = messageEvent.data;
|
||||
switch (data.type) {
|
||||
case "load":
|
||||
console.log("Booting", data.name);
|
||||
loadedFunctions.set(data.name!, new Function(wrapScript(data.code!)));
|
||||
postMessage(
|
||||
{
|
||||
type: "inited",
|
||||
name: data.name,
|
||||
} as ControllerMessage,
|
||||
"*"
|
||||
);
|
||||
break;
|
||||
case "invoke":
|
||||
let fn = loadedFunctions.get(data.name!);
|
||||
if (!fn) {
|
||||
throw new Error(`Function not loaded: ${data.name}`);
|
||||
}
|
||||
try {
|
||||
let result = await Promise.resolve(fn(...(data.args || [])));
|
||||
postMessage(
|
||||
{
|
||||
type: "result",
|
||||
id: data.id,
|
||||
result: result,
|
||||
} as ControllerMessage,
|
||||
"*"
|
||||
);
|
||||
} catch (e: any) {
|
||||
postMessage(
|
||||
{
|
||||
type: "result",
|
||||
id: data.id,
|
||||
error: e.message,
|
||||
} as ControllerMessage,
|
||||
"*"
|
||||
);
|
||||
throw e;
|
||||
}
|
||||
|
||||
break;
|
||||
case "syscall-response":
|
||||
let syscallId = data.id!;
|
||||
const lookup = pendingRequests.get(syscallId);
|
||||
if (!lookup) {
|
||||
console.log(
|
||||
"Current outstanding requests",
|
||||
pendingRequests,
|
||||
"looking up",
|
||||
syscallId
|
||||
);
|
||||
throw Error("Invalid request id");
|
||||
}
|
||||
pendingRequests.delete(syscallId);
|
||||
if (data.error) {
|
||||
lookup.reject(new Error(data.error));
|
||||
} else {
|
||||
lookup.resolve(data.result);
|
||||
}
|
||||
break;
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,37 @@
|
||||
import { safeRun } from "../util";
|
||||
import { Sandbox } from "../sandbox";
|
||||
import { WorkerLike } from "./worker";
|
||||
import { Plug } from "../plug";
|
||||
|
||||
class WebWorkerWrapper implements WorkerLike {
|
||||
private worker: Worker;
|
||||
onMessage?: (message: any) => Promise<void>;
|
||||
ready: Promise<void>;
|
||||
|
||||
constructor(worker: Worker) {
|
||||
this.worker = worker;
|
||||
this.worker.addEventListener("message", (evt: any) => {
|
||||
let data = evt.data;
|
||||
if (!data) return;
|
||||
safeRun(async () => {
|
||||
await this.onMessage!(data);
|
||||
});
|
||||
});
|
||||
this.ready = Promise.resolve();
|
||||
}
|
||||
postMessage(message: any): void {
|
||||
this.worker.postMessage(message);
|
||||
}
|
||||
|
||||
terminate() {
|
||||
return this.worker.terminate();
|
||||
}
|
||||
}
|
||||
|
||||
export function createSandbox(plug: Plug<any>) {
|
||||
// ParcelJS will build this file into a worker.
|
||||
let worker = new Worker(new URL("sandbox_worker.ts", import.meta.url), {
|
||||
type: "module",
|
||||
});
|
||||
return new Sandbox(plug, new WebWorkerWrapper(worker));
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
export type ControllerMessageType = "inited" | "result" | "syscall";
|
||||
|
||||
export type ControllerMessage = {
|
||||
type: ControllerMessageType;
|
||||
id?: number;
|
||||
name?: string;
|
||||
args?: any[];
|
||||
error?: string;
|
||||
result?: any;
|
||||
};
|
||||
|
||||
export interface WorkerLike {
|
||||
ready: Promise<void>;
|
||||
onMessage?: (message: any) => Promise<void>;
|
||||
|
||||
postMessage(message: any): void;
|
||||
|
||||
terminate(): void;
|
||||
}
|
||||
|
||||
export type WorkerMessageType = "load" | "invoke" | "syscall-response";
|
||||
|
||||
export type WorkerMessage = {
|
||||
type: WorkerMessageType;
|
||||
id?: number;
|
||||
name?: string;
|
||||
code?: string;
|
||||
args?: any[];
|
||||
result?: any;
|
||||
error?: any;
|
||||
};
|
||||
@@ -0,0 +1,49 @@
|
||||
import { createSandbox } from "../environment/node_sandbox";
|
||||
import { expect, test } from "@jest/globals";
|
||||
import { Manifest } from "../types";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { EndpointFeature, EndpointHook } from "./endpoint";
|
||||
import { System } from "../system";
|
||||
|
||||
test("Run a plugos endpoint server", async () => {
|
||||
let system = new System<EndpointHook>("server");
|
||||
let plug = await system.load(
|
||||
"test",
|
||||
{
|
||||
functions: {
|
||||
testhandler: {
|
||||
http: {
|
||||
path: "/",
|
||||
},
|
||||
code: `(() => {
|
||||
return {
|
||||
default: (req) => {
|
||||
console.log("Req", req);
|
||||
return {status: 200, body: [1, 2, 3], headers: {"Content-type": "application/json"}};
|
||||
}
|
||||
};
|
||||
})()`,
|
||||
},
|
||||
},
|
||||
} as Manifest<EndpointHook>,
|
||||
createSandbox
|
||||
);
|
||||
|
||||
const app = express();
|
||||
const port = 3123;
|
||||
|
||||
system.addFeature(new EndpointFeature(app, "/_"));
|
||||
|
||||
let server = app.listen(port, () => {
|
||||
console.log(`Listening on port ${port}`);
|
||||
});
|
||||
let resp = await request(app)
|
||||
.get("/_/test/?name=Pete")
|
||||
.expect((resp) => {
|
||||
expect(resp.status).toBe(200);
|
||||
expect(resp.header["content-type"]).toContain("application/json");
|
||||
expect(resp.text).toBe(JSON.stringify([1, 2, 3]));
|
||||
});
|
||||
server.close();
|
||||
});
|
||||
@@ -0,0 +1,134 @@
|
||||
import { Feature, Manifest } from "../types";
|
||||
import { Express, NextFunction, Request, Response } from "express";
|
||||
import { System } from "../system";
|
||||
|
||||
export type EndpointRequest = {
|
||||
method: string;
|
||||
path: string;
|
||||
query: { [key: string]: string };
|
||||
headers: { [key: string]: string };
|
||||
body: any;
|
||||
};
|
||||
|
||||
export type EndpointResponse = {
|
||||
status: number;
|
||||
headers?: { [key: string]: string };
|
||||
body: any;
|
||||
};
|
||||
|
||||
export type EndpointHook = {
|
||||
http?: EndPointDef | EndPointDef[];
|
||||
};
|
||||
|
||||
export type EndPointDef = {
|
||||
method?: "GET" | "POST" | "PUT" | "DELETE" | "HEAD" | "OPTIONS" | "ANY";
|
||||
path: string;
|
||||
};
|
||||
|
||||
export class EndpointFeature implements Feature<EndpointHook> {
|
||||
private app: Express;
|
||||
private prefix: string;
|
||||
|
||||
constructor(app: Express, prefix: string) {
|
||||
this.app = app;
|
||||
this.prefix = prefix;
|
||||
}
|
||||
|
||||
apply(system: System<EndpointHook>): void {
|
||||
this.app.use((req: Request, res: Response, next: NextFunction) => {
|
||||
if (!req.path.startsWith(this.prefix)) {
|
||||
return next();
|
||||
}
|
||||
console.log("Endpoint request", req.path);
|
||||
Promise.resolve()
|
||||
.then(async () => {
|
||||
// Iterate over all loaded plugins
|
||||
for (const [plugName, plug] of system.loadedPlugs.entries()) {
|
||||
const manifest = plug.manifest;
|
||||
if (!manifest) {
|
||||
continue;
|
||||
}
|
||||
const functions = manifest.functions;
|
||||
console.log("Checking plug", plugName);
|
||||
let prefix = `${this.prefix}/${plugName}`;
|
||||
if (!req.path.startsWith(prefix)) {
|
||||
continue;
|
||||
}
|
||||
for (const [name, functionDef] of Object.entries(functions)) {
|
||||
if (!functionDef.http) {
|
||||
continue;
|
||||
}
|
||||
let endpoints = Array.isArray(functionDef.http)
|
||||
? functionDef.http
|
||||
: [functionDef.http];
|
||||
console.log(endpoints);
|
||||
for (const { path, method } of endpoints) {
|
||||
let prefixedPath = `${prefix}${path}`;
|
||||
if (
|
||||
prefixedPath === req.path &&
|
||||
((method || "GET") === req.method || method === "ANY")
|
||||
) {
|
||||
try {
|
||||
const response: EndpointResponse = await plug.invoke(name, [
|
||||
{
|
||||
path: req.path,
|
||||
method: req.method,
|
||||
body: req.body,
|
||||
query: req.query,
|
||||
headers: req.headers,
|
||||
} as EndpointRequest,
|
||||
]);
|
||||
let resp = res.status(response.status);
|
||||
if (response.headers) {
|
||||
for (const [key, value] of Object.entries(
|
||||
response.headers
|
||||
)) {
|
||||
resp = resp.header(key, value);
|
||||
}
|
||||
}
|
||||
resp.send(response.body);
|
||||
return;
|
||||
} catch (e: any) {
|
||||
console.error("Error executing function", e);
|
||||
res.status(500).send(e.message);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
next();
|
||||
})
|
||||
.catch((e) => {
|
||||
console.error(e);
|
||||
next(e);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
validateManifest(manifest: Manifest<EndpointHook>): string[] {
|
||||
let errors = [];
|
||||
for (const [name, functionDef] of Object.entries(manifest.functions)) {
|
||||
if (!functionDef.http) {
|
||||
continue;
|
||||
}
|
||||
let endpoints = Array.isArray(functionDef.http)
|
||||
? functionDef.http
|
||||
: [functionDef.http];
|
||||
for (let { path, method } of endpoints) {
|
||||
if (!path) {
|
||||
errors.push("Path not defined for endpoint");
|
||||
}
|
||||
if (
|
||||
method &&
|
||||
["GET", "POST", "PUT", "DELETE", "ANY"].indexOf(method) === -1
|
||||
) {
|
||||
errors.push(
|
||||
`Invalid method ${method} for end point with with ${path}`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
import { Feature, Manifest } from "../types";
|
||||
import { System } from "../system";
|
||||
|
||||
export type EventHook = {
|
||||
events?: string[];
|
||||
};
|
||||
|
||||
export class EventFeature implements Feature<EventHook> {
|
||||
private system?: System<EventHook>;
|
||||
|
||||
async dispatchEvent(eventName: string, data?: any): Promise<any[]> {
|
||||
if (!this.system) {
|
||||
throw new Error("EventFeature is not initialized");
|
||||
}
|
||||
let promises: Promise<any>[] = [];
|
||||
for (const plug of this.system.loadedPlugs.values()) {
|
||||
for (const [name, functionDef] of Object.entries(
|
||||
plug.manifest!.functions
|
||||
)) {
|
||||
if (functionDef.events && functionDef.events.includes(eventName)) {
|
||||
promises.push(plug.invoke(name, [data]));
|
||||
}
|
||||
}
|
||||
}
|
||||
return Promise.all(promises);
|
||||
}
|
||||
|
||||
apply(system: System<EventHook>): void {
|
||||
this.system = system;
|
||||
}
|
||||
|
||||
validateManifest(manifest: Manifest<EventHook>): string[] {
|
||||
let errors = [];
|
||||
for (const [name, functionDef] of Object.entries(manifest.functions)) {
|
||||
if (functionDef.events && !Array.isArray(functionDef.events)) {
|
||||
errors.push("'events' key must be an array of strings");
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
import { Feature, Manifest } from "../types";
|
||||
import cron, { ScheduledTask } from "node-cron";
|
||||
import { safeRun } from "../util";
|
||||
import { System } from "../system";
|
||||
|
||||
export type CronHook = {
|
||||
cron?: string | string[];
|
||||
};
|
||||
|
||||
export class NodeCronFeature implements Feature<CronHook> {
|
||||
apply(system: System<CronHook>): void {
|
||||
let tasks: ScheduledTask[] = [];
|
||||
system.on({
|
||||
plugLoaded: (name, plug) => {
|
||||
reloadCrons();
|
||||
},
|
||||
plugUnloaded(name, plug) {
|
||||
reloadCrons();
|
||||
},
|
||||
});
|
||||
|
||||
reloadCrons();
|
||||
|
||||
function reloadCrons() {
|
||||
tasks.forEach((task) => task.stop());
|
||||
tasks = [];
|
||||
for (let plug of system.loadedPlugs.values()) {
|
||||
if (!plug.manifest) {
|
||||
continue;
|
||||
}
|
||||
for (const [name, functionDef] of Object.entries(
|
||||
plug.manifest.functions
|
||||
)) {
|
||||
if (!functionDef.cron) {
|
||||
continue;
|
||||
}
|
||||
const crons = Array.isArray(functionDef.cron)
|
||||
? functionDef.cron
|
||||
: [functionDef.cron];
|
||||
for (let cronDef of crons) {
|
||||
tasks.push(
|
||||
cron.schedule(cronDef, () => {
|
||||
console.log("Now acting on cron", cronDef);
|
||||
safeRun(async () => {
|
||||
try {
|
||||
await plug.invoke(name, [cronDef]);
|
||||
} catch (e: any) {
|
||||
console.error("Execution of cron function failed", e);
|
||||
}
|
||||
});
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
validateManifest(manifest: Manifest<CronHook>): string[] {
|
||||
let errors = [];
|
||||
for (const [name, functionDef] of Object.entries(manifest.functions)) {
|
||||
if (!functionDef.cron) {
|
||||
continue;
|
||||
}
|
||||
const crons = Array.isArray(functionDef.cron)
|
||||
? functionDef.cron
|
||||
: [functionDef.cron];
|
||||
for (let cronDef of crons) {
|
||||
if (!cron.validate(cronDef)) {
|
||||
errors.push(`Invalid cron expression ${cronDef}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import { Manifest, RuntimeEnvironment } from "./types";
|
||||
import { Sandbox } from "./sandbox";
|
||||
import { System } from "./system";
|
||||
|
||||
export class Plug<HookT> {
|
||||
system: System<HookT>;
|
||||
sandbox: Sandbox;
|
||||
public manifest?: Manifest<HookT>;
|
||||
readonly runtimeEnv: RuntimeEnvironment;
|
||||
grantedPermissions: string[] = [];
|
||||
name: string;
|
||||
|
||||
constructor(
|
||||
system: System<HookT>,
|
||||
name: string,
|
||||
sandboxFactory: (plug: Plug<HookT>) => Sandbox
|
||||
) {
|
||||
this.system = system;
|
||||
this.name = name;
|
||||
this.sandbox = sandboxFactory(this);
|
||||
this.runtimeEnv = system.runtimeEnv;
|
||||
}
|
||||
|
||||
async load(manifest: Manifest<HookT>) {
|
||||
this.manifest = manifest;
|
||||
// TODO: These need to be explicitly granted, not just taken
|
||||
this.grantedPermissions = manifest.requiredPermissions || [];
|
||||
}
|
||||
|
||||
syscall(name: string, args: any[]): Promise<any> {
|
||||
return this.system.syscallWithContext({ plug: this }, name, args);
|
||||
}
|
||||
|
||||
canInvoke(name: string) {
|
||||
if (!this.manifest) {
|
||||
return false;
|
||||
}
|
||||
const funDef = this.manifest.functions[name];
|
||||
if (!funDef) {
|
||||
throw new Error(`Function ${name} not found in manifest`);
|
||||
}
|
||||
return !funDef.env || funDef.env === this.runtimeEnv;
|
||||
}
|
||||
|
||||
async invoke(name: string, args: Array<any>): Promise<any> {
|
||||
if (!this.sandbox.isLoaded(name)) {
|
||||
const funDef = this.manifest!.functions[name];
|
||||
if (!funDef) {
|
||||
throw new Error(`Function ${name} not found in manifest`);
|
||||
}
|
||||
if (!this.canInvoke(name)) {
|
||||
throw new Error(
|
||||
`Function ${name} is not available in ${this.runtimeEnv}`
|
||||
);
|
||||
}
|
||||
await this.sandbox.load(name, funDef.code!);
|
||||
}
|
||||
return await this.sandbox.invoke(name, args);
|
||||
}
|
||||
|
||||
async stop() {
|
||||
this.sandbox.stop();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import fs, { watch } from "fs/promises";
|
||||
import path from "path";
|
||||
import { createSandbox } from "./environment/node_sandbox";
|
||||
import { safeRun } from "../server/util";
|
||||
import { System } from "./system";
|
||||
|
||||
function extractPlugName(localPath: string): string {
|
||||
const baseName = path.basename(localPath);
|
||||
return baseName.substring(0, baseName.length - ".plug.json".length);
|
||||
}
|
||||
|
||||
export class DiskPlugLoader<HookT> {
|
||||
private system: System<HookT>;
|
||||
private plugPath: string;
|
||||
|
||||
constructor(system: System<HookT>, plugPath: string) {
|
||||
this.system = system;
|
||||
this.plugPath = plugPath;
|
||||
}
|
||||
|
||||
watcher() {
|
||||
safeRun(async () => {
|
||||
for await (const { filename, eventType } of watch(this.plugPath)) {
|
||||
if (!filename.endsWith(".plug.json")) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
let localPath = path.join(this.plugPath, filename);
|
||||
const plugName = extractPlugName(localPath);
|
||||
try {
|
||||
await fs.stat(localPath);
|
||||
} catch (e) {
|
||||
// Likely removed
|
||||
await this.system.unload(plugName);
|
||||
}
|
||||
const plugDef = await this.loadPlugFromFile(localPath);
|
||||
} catch {
|
||||
// ignore, error handled by loadPlug
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private async loadPlugFromFile(localPath: string) {
|
||||
const plug = await fs.readFile(localPath, "utf8");
|
||||
const plugName = extractPlugName(localPath);
|
||||
|
||||
console.log("Now loading plug", plugName);
|
||||
try {
|
||||
const plugDef = JSON.parse(plug);
|
||||
await this.system.load(plugName, plugDef, createSandbox);
|
||||
return plugDef;
|
||||
} catch (e) {
|
||||
console.error("Could not parse plugin file", e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async loadPlugs() {
|
||||
for (let filename of await fs.readdir(this.plugPath)) {
|
||||
if (filename.endsWith(".plug.json")) {
|
||||
let localPath = path.join(this.plugPath, filename);
|
||||
await this.loadPlugFromFile(localPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
import { createSandbox } from "./environment/node_sandbox";
|
||||
import { expect, test } from "@jest/globals";
|
||||
import { System } from "./system";
|
||||
|
||||
test("Run a Node sandbox", async () => {
|
||||
let system = new System("server");
|
||||
system.registerSyscalls("", [], {
|
||||
addNumbers: (ctx, a, b) => {
|
||||
return a + b;
|
||||
},
|
||||
failingSyscall: () => {
|
||||
throw new Error("#fail");
|
||||
},
|
||||
});
|
||||
system.registerSyscalls("", ["restricted"], {
|
||||
restrictedSyscall: () => {
|
||||
return "restricted";
|
||||
},
|
||||
});
|
||||
system.registerSyscalls("", ["dangerous"], {
|
||||
dangerousSyscall: () => {
|
||||
return "yay";
|
||||
},
|
||||
});
|
||||
let plug = await system.load(
|
||||
"test",
|
||||
{
|
||||
requiredPermissions: ["dangerous"],
|
||||
functions: {
|
||||
addTen: {
|
||||
code: `(() => {
|
||||
return {
|
||||
default: (n) => {
|
||||
return n + 10;
|
||||
}
|
||||
};
|
||||
})()`,
|
||||
},
|
||||
addNumbersSyscall: {
|
||||
code: `(() => {
|
||||
return {
|
||||
default: async (a, b) => {
|
||||
return await self.syscall("addNumbers", a, b);
|
||||
}
|
||||
};
|
||||
})()`,
|
||||
},
|
||||
errorOut: {
|
||||
code: `(() => {
|
||||
return {
|
||||
default: () => {
|
||||
throw Error("BOOM");
|
||||
}
|
||||
};
|
||||
})()`,
|
||||
},
|
||||
errorOutSys: {
|
||||
code: `(() => {
|
||||
return {
|
||||
default: async () => {
|
||||
await self.syscall("failingSyscall");
|
||||
}
|
||||
};
|
||||
})()`,
|
||||
},
|
||||
restrictedTest: {
|
||||
code: `(() => {
|
||||
return {
|
||||
default: async () => {
|
||||
await self.syscall("restrictedSyscall");
|
||||
}
|
||||
};
|
||||
})()`,
|
||||
},
|
||||
dangerousTest: {
|
||||
code: `(() => {
|
||||
return {
|
||||
default: async () => {
|
||||
return await self.syscall("dangerousSyscall");
|
||||
}
|
||||
};
|
||||
})()`,
|
||||
},
|
||||
},
|
||||
},
|
||||
createSandbox
|
||||
);
|
||||
expect(await plug.invoke("addTen", [10])).toBe(20);
|
||||
for (let i = 0; i < 100; i++) {
|
||||
expect(await plug.invoke("addNumbersSyscall", [10, i])).toBe(10 + i);
|
||||
}
|
||||
try {
|
||||
await plug.invoke("errorOut", []);
|
||||
expect(true).toBe(false);
|
||||
} catch (e: any) {
|
||||
expect(e.message).toBe("BOOM");
|
||||
}
|
||||
try {
|
||||
await plug.invoke("errorOutSys", []);
|
||||
expect(true).toBe(false);
|
||||
} catch (e: any) {
|
||||
expect(e.message).toBe("#fail");
|
||||
}
|
||||
try {
|
||||
await plug.invoke("restrictedTest", []);
|
||||
expect(true).toBe(false);
|
||||
} catch (e: any) {
|
||||
expect(e.message).toBe(
|
||||
"Missing permission 'restricted' for syscall restrictedSyscall"
|
||||
);
|
||||
}
|
||||
expect(await plug.invoke("dangerousTest", [])).toBe("yay");
|
||||
|
||||
await system.unloadAll();
|
||||
});
|
||||
@@ -0,0 +1,98 @@
|
||||
import {
|
||||
ControllerMessage,
|
||||
WorkerLike,
|
||||
WorkerMessage,
|
||||
} from "./environment/worker";
|
||||
import { Plug } from "./plug";
|
||||
|
||||
export type SandboxFactory<HookT> = (plug: Plug<HookT>) => Sandbox;
|
||||
|
||||
export class Sandbox {
|
||||
protected worker: WorkerLike;
|
||||
protected reqId = 0;
|
||||
protected outstandingInits = new Map<string, () => void>();
|
||||
protected outstandingInvocations = new Map<
|
||||
number,
|
||||
{ resolve: (result: any) => void; reject: (e: any) => void }
|
||||
>();
|
||||
protected loadedFunctions = new Set<string>();
|
||||
protected plug: Plug<any>;
|
||||
|
||||
constructor(plug: Plug<any>, worker: WorkerLike) {
|
||||
worker.onMessage = this.onMessage.bind(this);
|
||||
this.worker = worker;
|
||||
this.plug = plug;
|
||||
}
|
||||
|
||||
isLoaded(name: string) {
|
||||
return this.loadedFunctions.has(name);
|
||||
}
|
||||
|
||||
async load(name: string, code: string): Promise<void> {
|
||||
await this.worker.ready;
|
||||
this.worker.postMessage({
|
||||
type: "load",
|
||||
name: name,
|
||||
code: code,
|
||||
} as WorkerMessage);
|
||||
return new Promise((resolve) => {
|
||||
this.loadedFunctions.add(name);
|
||||
this.outstandingInits.set(name, resolve);
|
||||
});
|
||||
}
|
||||
|
||||
async onMessage(data: ControllerMessage) {
|
||||
switch (data.type) {
|
||||
case "inited":
|
||||
let initCb = this.outstandingInits.get(data.name!);
|
||||
initCb && initCb();
|
||||
this.outstandingInits.delete(data.name!);
|
||||
break;
|
||||
case "syscall":
|
||||
try {
|
||||
let result = await this.plug.syscall(data.name!, data.args!);
|
||||
|
||||
this.worker.postMessage({
|
||||
type: "syscall-response",
|
||||
id: data.id,
|
||||
result: result,
|
||||
} as WorkerMessage);
|
||||
} catch (e: any) {
|
||||
this.worker.postMessage({
|
||||
type: "syscall-response",
|
||||
id: data.id,
|
||||
error: e.message,
|
||||
} as WorkerMessage);
|
||||
}
|
||||
break;
|
||||
case "result":
|
||||
let resultCbs = this.outstandingInvocations.get(data.id!);
|
||||
this.outstandingInvocations.delete(data.id!);
|
||||
if (data.error) {
|
||||
resultCbs && resultCbs.reject(new Error(data.error));
|
||||
} else {
|
||||
resultCbs && resultCbs.resolve(data.result);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
console.error("Unknown message type", data);
|
||||
}
|
||||
}
|
||||
|
||||
async invoke(name: string, args: any[]): Promise<any> {
|
||||
this.reqId++;
|
||||
this.worker.postMessage({
|
||||
type: "invoke",
|
||||
id: this.reqId,
|
||||
name,
|
||||
args,
|
||||
});
|
||||
return new Promise((resolve, reject) => {
|
||||
this.outstandingInvocations.set(this.reqId, { resolve, reject });
|
||||
});
|
||||
}
|
||||
|
||||
stop() {
|
||||
this.worker.terminate();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import { SysCallMapping } from "../system";
|
||||
import { EventFeature } from "../feature/event";
|
||||
|
||||
export function eventSyscalls(eventFeature: EventFeature): SysCallMapping {
|
||||
return {
|
||||
async dispatch(ctx, eventName: string, data: any) {
|
||||
return eventFeature.dispatchEvent(eventName, data);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import fetch, { RequestInfo, RequestInit } from "node-fetch";
|
||||
import { SysCallMapping } from "../system";
|
||||
|
||||
export function fetchSyscalls(): SysCallMapping {
|
||||
return {
|
||||
async json(ctx, url: RequestInfo, init: RequestInit) {
|
||||
let resp = await fetch(url, init);
|
||||
return resp.json();
|
||||
},
|
||||
async text(ctx, url: RequestInfo, init: RequestInit) {
|
||||
let resp = await fetch(url, init);
|
||||
return resp.text();
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import { promisify } from "util";
|
||||
import { execFile } from "child_process";
|
||||
import type { SysCallMapping } from "../system";
|
||||
|
||||
const execFilePromise = promisify(execFile);
|
||||
|
||||
export default function (cwd: string): SysCallMapping {
|
||||
return {
|
||||
run: async (
|
||||
ctx,
|
||||
cmd: string,
|
||||
args: string[]
|
||||
): Promise<{ stdout: string; stderr: string }> => {
|
||||
let { stdout, stderr } = await execFilePromise(cmd, args, {
|
||||
cwd: cwd,
|
||||
});
|
||||
return { stdout, stderr };
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import { createSandbox } from "../environment/node_sandbox";
|
||||
import { expect, test } from "@jest/globals";
|
||||
import { System } from "../system";
|
||||
import { storeSyscalls } from "./store.dexie_browser";
|
||||
|
||||
// For testing in node.js
|
||||
require("fake-indexeddb/auto");
|
||||
|
||||
test("Test store", async () => {
|
||||
let system = new System("server");
|
||||
system.registerSyscalls("store", [], storeSyscalls("test", "test"));
|
||||
let plug = await system.load(
|
||||
"test",
|
||||
{
|
||||
functions: {
|
||||
test1: {
|
||||
code: `(() => {
|
||||
return {
|
||||
default: async () => {
|
||||
await self.syscall("store.set", "name", "Pete");
|
||||
return await self.syscall("store.get", "name");
|
||||
}
|
||||
};
|
||||
})()`,
|
||||
},
|
||||
test2: {
|
||||
code: `(() => {
|
||||
return {
|
||||
default: async () => {
|
||||
await self.syscall("store.set", "page1:bl:page2:10", {title: "Something", meta: 20});
|
||||
await self.syscall("store.batchSet", [
|
||||
{key: "page2:bl:page3", value: {title: "Something2", meta: 10}},
|
||||
{key: "page2:bl:page4", value: {title: "Something3", meta: 10}},
|
||||
]);
|
||||
return await self.syscall("store.queryPrefix", "page2:");
|
||||
}
|
||||
};
|
||||
})()`,
|
||||
},
|
||||
},
|
||||
},
|
||||
createSandbox
|
||||
);
|
||||
expect(await plug.invoke("test1", [])).toBe("Pete");
|
||||
let queryResults = await plug.invoke("test2", []);
|
||||
expect(queryResults.length).toBe(2);
|
||||
expect(queryResults[0].value.meta).toBe(10);
|
||||
await system.unloadAll();
|
||||
});
|
||||
@@ -0,0 +1,66 @@
|
||||
import Dexie from "dexie";
|
||||
import { SysCallMapping } from "../system";
|
||||
|
||||
export type KV = {
|
||||
key: string;
|
||||
value: any;
|
||||
};
|
||||
|
||||
export function storeSyscalls(
|
||||
dbName: string,
|
||||
tableName: string
|
||||
): SysCallMapping {
|
||||
const db = new Dexie(dbName);
|
||||
db.version(1).stores({
|
||||
test: "key",
|
||||
});
|
||||
const items = db.table(tableName);
|
||||
|
||||
return {
|
||||
async delete(ctx, key: string) {
|
||||
await items.delete(key);
|
||||
},
|
||||
|
||||
async deletePrefix(ctx, prefix: string) {
|
||||
await items.where("key").startsWith(prefix).delete();
|
||||
},
|
||||
|
||||
async deleteAll() {
|
||||
await items.clear();
|
||||
},
|
||||
|
||||
async set(ctx, key: string, value: any) {
|
||||
await items.put({
|
||||
key,
|
||||
value,
|
||||
});
|
||||
},
|
||||
|
||||
async batchSet(ctx, kvs: KV[]) {
|
||||
await items.bulkPut(
|
||||
kvs.map(({ key, value }) => ({
|
||||
key,
|
||||
value,
|
||||
}))
|
||||
);
|
||||
},
|
||||
|
||||
async get(ctx, key: string): Promise<any | null> {
|
||||
let result = await items.get({
|
||||
key,
|
||||
});
|
||||
return result ? result.value : null;
|
||||
},
|
||||
|
||||
async queryPrefix(
|
||||
ctx,
|
||||
keyPrefix: string
|
||||
): Promise<{ key: string; value: any }[]> {
|
||||
let results = await items.where("key").startsWith(keyPrefix).toArray();
|
||||
return results.map((result) => ({
|
||||
key: result.key,
|
||||
value: result.value,
|
||||
}));
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import { createSandbox } from "../environment/node_sandbox";
|
||||
import { expect, test } from "@jest/globals";
|
||||
import { System } from "../system";
|
||||
import {
|
||||
ensureTable,
|
||||
storeReadSyscalls,
|
||||
storeWriteSyscalls,
|
||||
} from "./store.knex_node";
|
||||
import knex from "knex";
|
||||
import fs from "fs/promises";
|
||||
|
||||
test("Test store", async () => {
|
||||
const db = knex({
|
||||
client: "better-sqlite3",
|
||||
connection: {
|
||||
filename: "test.db",
|
||||
},
|
||||
useNullAsDefault: true,
|
||||
});
|
||||
await ensureTable(db, "test_table");
|
||||
let system = new System("server");
|
||||
system.registerSyscalls(
|
||||
"store",
|
||||
[],
|
||||
storeWriteSyscalls(db, "test_table"),
|
||||
storeReadSyscalls(db, "test_table")
|
||||
);
|
||||
let plug = await system.load(
|
||||
"test",
|
||||
{
|
||||
functions: {
|
||||
test1: {
|
||||
code: `(() => {
|
||||
return {
|
||||
default: async () => {
|
||||
await self.syscall("store.set", "name", "Pete");
|
||||
return await self.syscall("store.get", "name");
|
||||
}
|
||||
};
|
||||
})()`,
|
||||
},
|
||||
},
|
||||
},
|
||||
createSandbox
|
||||
);
|
||||
expect(await plug.invoke("test1", [])).toBe("Pete");
|
||||
await system.unloadAll();
|
||||
await fs.unlink("test.db");
|
||||
});
|
||||
@@ -0,0 +1,84 @@
|
||||
import { Knex } from "knex";
|
||||
import { SysCallMapping } from "../system";
|
||||
|
||||
type Item = {
|
||||
page: string;
|
||||
key: string;
|
||||
value: any;
|
||||
};
|
||||
|
||||
export type KV = {
|
||||
key: string;
|
||||
value: any;
|
||||
};
|
||||
|
||||
export async function ensureTable(db: Knex<any, unknown>, tableName: string) {
|
||||
if (!(await db.schema.hasTable(tableName))) {
|
||||
await db.schema.createTable(tableName, (table) => {
|
||||
table.string("key");
|
||||
table.text("value");
|
||||
table.primary(["key"]);
|
||||
});
|
||||
console.log(`Created table ${tableName}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function storeWriteSyscalls(
|
||||
db: Knex<any, unknown>,
|
||||
tableName: string
|
||||
): SysCallMapping {
|
||||
const apiObj: SysCallMapping = {
|
||||
delete: async (ctx, page: string, key: string) => {
|
||||
await db<Item>(tableName).where({ page, key }).del();
|
||||
},
|
||||
deletePrefix: async (ctx, prefix: string) => {
|
||||
return db<Item>(tableName).andWhereLike("key", `${prefix}%`).del();
|
||||
},
|
||||
deleteAll: async (ctx) => {
|
||||
await db<Item>(tableName).del();
|
||||
},
|
||||
set: async (ctx, key: string, value: any) => {
|
||||
let changed = await db<Item>(tableName)
|
||||
.where({ key })
|
||||
.update("value", JSON.stringify(value));
|
||||
if (changed === 0) {
|
||||
await db<Item>(tableName).insert({
|
||||
key,
|
||||
value: JSON.stringify(value),
|
||||
});
|
||||
}
|
||||
},
|
||||
batchSet: async (ctx, kvs: KV[]) => {
|
||||
for (let { key, value } of kvs) {
|
||||
await apiObj["store.set"](ctx, key, value);
|
||||
}
|
||||
},
|
||||
};
|
||||
return apiObj;
|
||||
}
|
||||
|
||||
export function storeReadSyscalls(
|
||||
db: Knex<any, unknown>,
|
||||
tableName: string
|
||||
): SysCallMapping {
|
||||
return {
|
||||
get: async (ctx, key: string): Promise<any | null> => {
|
||||
let result = await db<Item>(tableName).where({ key }).select("value");
|
||||
if (result.length) {
|
||||
return JSON.parse(result[0].value);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
},
|
||||
queryPrefix: async (ctx, prefix: string) => {
|
||||
return (
|
||||
await db<Item>(tableName)
|
||||
.andWhereLike("key", `${prefix}%`)
|
||||
.select("key", "value")
|
||||
).map(({ key, value }) => ({
|
||||
key,
|
||||
value: JSON.parse(value),
|
||||
}));
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { SysCallMapping } from "../system";
|
||||
|
||||
export function transportSyscalls(
|
||||
names: string[],
|
||||
transportCall: (name: string, ...args: any[]) => Promise<any>
|
||||
): SysCallMapping {
|
||||
let syscalls: SysCallMapping = {};
|
||||
|
||||
for (let name of names) {
|
||||
syscalls[name] = (ctx, ...args: any[]) => {
|
||||
return transportCall(name, ...args);
|
||||
};
|
||||
}
|
||||
|
||||
return syscalls;
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
import { Feature, Manifest, RuntimeEnvironment } from "./types";
|
||||
import { EventEmitter } from "../common/event";
|
||||
import { SandboxFactory } from "./sandbox";
|
||||
import { Plug } from "./plug";
|
||||
|
||||
export interface SysCallMapping {
|
||||
[key: string]: (ctx: SyscallContext, ...args: any) => Promise<any> | any;
|
||||
}
|
||||
|
||||
export type SystemJSON<HookT> = { [key: string]: Manifest<HookT> };
|
||||
|
||||
export type SystemEvents<HookT> = {
|
||||
plugLoaded: (name: string, plug: Plug<HookT>) => void;
|
||||
plugUnloaded: (name: string, plug: Plug<HookT>) => void;
|
||||
};
|
||||
|
||||
type SyscallContext = {
|
||||
plug: Plug<any> | null;
|
||||
};
|
||||
|
||||
type SyscallSignature = (
|
||||
ctx: SyscallContext,
|
||||
...args: any[]
|
||||
) => Promise<any> | any;
|
||||
|
||||
type Syscall = {
|
||||
requiredPermissions: string[];
|
||||
callback: SyscallSignature;
|
||||
};
|
||||
|
||||
export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
|
||||
protected plugs = new Map<string, Plug<HookT>>();
|
||||
protected registeredSyscalls = new Map<string, Syscall>();
|
||||
protected enabledFeatures = new Set<Feature<HookT>>();
|
||||
|
||||
readonly runtimeEnv: RuntimeEnvironment;
|
||||
|
||||
constructor(env: RuntimeEnvironment) {
|
||||
super();
|
||||
this.runtimeEnv = env;
|
||||
}
|
||||
|
||||
addFeature(feature: Feature<HookT>) {
|
||||
this.enabledFeatures.add(feature);
|
||||
feature.apply(this);
|
||||
}
|
||||
|
||||
registerSyscalls(
|
||||
namespace: string,
|
||||
requiredCapabilities: string[],
|
||||
...registrationObjects: SysCallMapping[]
|
||||
) {
|
||||
for (const registrationObject of registrationObjects) {
|
||||
for (let [name, callback] of Object.entries(registrationObject)) {
|
||||
const callName = namespace ? `${namespace}.${name}` : name;
|
||||
this.registeredSyscalls.set(callName, {
|
||||
requiredPermissions: requiredCapabilities,
|
||||
callback,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async syscallWithContext(
|
||||
ctx: SyscallContext,
|
||||
name: string,
|
||||
args: any[]
|
||||
): Promise<any> {
|
||||
const syscall = this.registeredSyscalls.get(name);
|
||||
if (!syscall) {
|
||||
throw Error(`Unregistered syscall ${name}`);
|
||||
}
|
||||
for (const permission of syscall.requiredPermissions) {
|
||||
if (!ctx.plug) {
|
||||
throw Error(`Syscall ${name} requires permission and no plug is set`);
|
||||
}
|
||||
if (!ctx.plug.grantedPermissions.includes(permission)) {
|
||||
throw Error(`Missing permission '${permission}' for syscall ${name}`);
|
||||
}
|
||||
}
|
||||
return Promise.resolve(syscall.callback(ctx, ...args));
|
||||
}
|
||||
|
||||
async load(
|
||||
name: string,
|
||||
manifest: Manifest<HookT>,
|
||||
sandboxFactory: SandboxFactory<HookT>
|
||||
): Promise<Plug<HookT>> {
|
||||
if (this.plugs.has(name)) {
|
||||
await this.unload(name);
|
||||
}
|
||||
// Validate
|
||||
let errors: string[] = [];
|
||||
for (const feature of this.enabledFeatures) {
|
||||
errors = [...errors, ...feature.validateManifest(manifest)];
|
||||
}
|
||||
if (errors.length > 0) {
|
||||
throw new Error(`Invalid manifest: ${errors.join(", ")}`);
|
||||
}
|
||||
// Ok, let's load this thing!
|
||||
const plug = new Plug(this, name, sandboxFactory);
|
||||
await plug.load(manifest);
|
||||
this.plugs.set(name, plug);
|
||||
this.emit("plugLoaded", name, plug);
|
||||
return plug;
|
||||
}
|
||||
|
||||
async unload(name: string) {
|
||||
const plug = this.plugs.get(name);
|
||||
if (!plug) {
|
||||
throw Error(`Plug ${name} not found`);
|
||||
}
|
||||
await plug.stop();
|
||||
this.emit("plugUnloaded", name, plug);
|
||||
this.plugs.delete(name);
|
||||
}
|
||||
|
||||
get loadedPlugs(): Map<string, Plug<HookT>> {
|
||||
return this.plugs;
|
||||
}
|
||||
|
||||
toJSON(): SystemJSON<HookT> {
|
||||
let plugJSON: { [key: string]: Manifest<HookT> } = {};
|
||||
for (let [name, plug] of this.plugs) {
|
||||
if (!plug.manifest) {
|
||||
continue;
|
||||
}
|
||||
plugJSON[name] = plug.manifest;
|
||||
}
|
||||
return plugJSON;
|
||||
}
|
||||
|
||||
async replaceAllFromJSON(
|
||||
json: SystemJSON<HookT>,
|
||||
sandboxFactory: SandboxFactory<HookT>
|
||||
) {
|
||||
await this.unloadAll();
|
||||
for (let [name, manifest] of Object.entries(json)) {
|
||||
console.log("Loading plug", name);
|
||||
await this.load(name, manifest, sandboxFactory);
|
||||
}
|
||||
}
|
||||
|
||||
async unloadAll(): Promise<void[]> {
|
||||
return Promise.all(
|
||||
Array.from(this.plugs.keys()).map(this.unload.bind(this))
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import { System } from "./system";
|
||||
|
||||
export interface Manifest<HookT> {
|
||||
requiredPermissions?: string[];
|
||||
functions: {
|
||||
[key: string]: FunctionDef<HookT>;
|
||||
};
|
||||
}
|
||||
|
||||
export type FunctionDef<HookT> = {
|
||||
path?: string;
|
||||
code?: string;
|
||||
env?: RuntimeEnvironment;
|
||||
} & HookT;
|
||||
|
||||
export type RuntimeEnvironment = "client" | "server";
|
||||
|
||||
export interface Feature<HookT> {
|
||||
validateManifest(manifest: Manifest<HookT>): string[];
|
||||
|
||||
apply(system: System<HookT>): void;
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
export function safeRun(fn: () => Promise<void>) {
|
||||
fn().catch((e) => {
|
||||
// console.error(e);
|
||||
throw e;
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user