Moving everything into a single repo and build

This commit is contained in:
Zef Hemel
2022-03-18 17:30:58 +01:00
parent d5528fef2a
commit 4e570191a6
73 changed files with 9 additions and 17644 deletions
-77
View File
@@ -1,77 +0,0 @@
#!/usr/bin/env node
import esbuild from "esbuild";
import { readFile, unlink, writeFile } from "fs/promises";
import path from "path";
import yargs from "yargs";
import { hideBin } from "yargs/helpers";
async function compile(filePath, functionName, debug) {
let outFile = "out.js";
let inFile = filePath;
if (functionName) {
// Generate a new file importing just this one function and exporting it
inFile = "in.js";
await writeFile(
inFile,
`import {${functionName}} from "./${filePath}";
export default ${functionName};`
);
}
// TODO: Figure out how to make source maps work correctly with eval() code
let js = await esbuild.build({
entryPoints: [inFile],
bundle: true,
format: "iife",
globalName: "mod",
platform: "neutral",
sourcemap: false, //sourceMap ? "inline" : false,
minify: !debug,
outfile: outFile,
});
let jsCode = (await readFile(outFile)).toString();
jsCode = jsCode.replace(/^var mod ?= ?/, "");
await unlink(outFile);
if (inFile !== filePath) {
await unlink(inFile);
}
// Strip final ';'
return jsCode.substring(0, jsCode.length - 2);
}
async function bundle(manifestPath, sourceMaps) {
const rootPath = path.dirname(manifestPath);
const manifest = JSON.parse((await readFile(manifestPath)).toString());
for (let [name, def] of Object.entries(manifest.functions)) {
let jsFunctionName = def.functionName,
filePath = path.join(rootPath, def.path);
if (filePath.indexOf(":") !== -1) {
[filePath, jsFunctionName] = filePath.split(":");
}
def.code = await compile(filePath, jsFunctionName, sourceMaps);
delete def.path;
}
return manifest;
}
async function run() {
let args = await yargs(hideBin(process.argv))
.option("debug", {
type: "boolean",
})
.parse();
let generatedManifest = await bundle(args._[0], !!args.debug);
await writeFile(args._[1], JSON.stringify(generatedManifest, null, 2));
}
run().catch((e) => {
console.error(e);
process.exit(1);
});
-13
View File
@@ -1,13 +0,0 @@
export default {
extensionsToTreatAsEsm: [".ts"],
preset: "ts-jest/presets/default-esm", // or other ESM presets
globals: {
"ts-jest": {
useESM: true,
tsconfig: "<rootDir>/tsconfig.json",
},
},
moduleNameMapper: {
"^(\\.{1,2}/.*)\\.js$": "$1",
},
};
-34
View File
@@ -1,34 +0,0 @@
{
"name": "plugbox",
"version": "1.0.0",
"type": "module",
"license": "MIT",
"bin": {
"plugbox-bundle": "./bin/plugbox-bundle.mjs"
},
"scripts": {
"check": "tsc --noEmit",
"test": "jest"
},
"dependencies": {
"esbuild": "^0.14.24",
"idb": "^7.0.0",
"typescript": "^4.7.0-dev.20220313",
"vm2": "^3.9.9",
"yargs": "^17.3.1"
},
"devDependencies": {
"@jest/globals": "^27.5.1",
"@types/jest": "^27.4.1",
"@types/node": "^17.0.21",
"@types/yargs": "^17.0.9",
"buffer": "^6.0.3",
"events": "^3.3.0",
"jest": "^27.5.1",
"parcel": "^2.3.2",
"path-browserify": "^1.0.1",
"ts-jest": "^27.1.3",
"util": "^0.12.4",
"vm-browserify": "^1.1.2"
}
}
-8
View File
@@ -1,8 +0,0 @@
<html>
<body>
<script type="module">
// Sup yo!
import "./sandbox_worker";
</script>
</body>
</html>
-43
View File
@@ -1,43 +0,0 @@
import { ControllerMessage, WorkerLike, WorkerMessage } from "./types";
import { Sandbox, System } from "./runtime";
import { safeRun } from "./util";
// @ts-ignore
import sandboxHtml from "bundle-text:./iframe_sandbox.html";
class IFrameWrapper implements WorkerLike {
private iframe: HTMLIFrameElement;
onMessage?: (message: any) => Promise<void>;
constructor() {
const iframe = document.createElement("iframe", {});
this.iframe = iframe;
iframe.style.display = "none";
// Let's lock this down significantly
iframe.setAttribute("sandbox", "allow-scripts");
iframe.srcdoc = sandboxHtml;
window.addEventListener("message", (evt: any) => {
if (evt.source !== iframe.contentWindow) {
return;
}
let data = evt.data;
if (!data) return;
safeRun(async () => {
await this.onMessage!(data);
});
});
document.body.appendChild(iframe);
}
postMessage(message: any): void {
this.iframe.contentWindow!.postMessage(message, "*");
}
terminate() {
return this.iframe.remove();
}
}
export function createSandbox(system: System<any>) {
return new Sandbox(system, new IFrameWrapper());
}
-42
View File
@@ -1,42 +0,0 @@
import { ControllerMessage, WorkerLike, WorkerMessage } from "./types";
import { System, Sandbox } from "./runtime";
import { Worker } from "worker_threads";
import * as fs from "fs";
import { safeRun } from "./util";
// ParcelJS will simply inline this into the bundle.
const workerCode = fs.readFileSync(__dirname + "/node_worker.js", "utf-8");
class NodeWorkerWrapper implements WorkerLike {
onMessage?: (message: any) => Promise<void>;
private worker: Worker;
constructor(worker: Worker) {
this.worker = worker;
worker.on("message", (message: any) => {
safeRun(async () => {
await this.onMessage!(message);
});
});
}
postMessage(message: any): void {
this.worker.postMessage(message);
}
terminate(): void {
this.worker.terminate();
}
}
export function createSandbox(system: System<any>) {
return new Sandbox(
system,
new NodeWorkerWrapper(
new Worker(workerCode, {
eval: true,
})
)
);
}
-79
View File
@@ -1,79 +0,0 @@
import { createSandbox } from "./node_sandbox";
import { System } from "./runtime";
import { test, expect } from "@jest/globals";
test("Run a Node sandbox", async () => {
let system = new System();
system.registerSyscalls({
addNumbers: (a, b) => {
return a + b;
},
failingSyscall: () => {
throw new Error("#fail");
},
});
let plug = await system.load(
"test",
{
functions: {
addTen: {
code: `(() => {
return {
default: (n) => {
return n + 10;
}
};
})()`,
},
addNumbersSyscall: {
code: `(() => {
return {
default: async (a, b) => {
return await self.syscall(1, "addNumbers", [a, b]);
}
};
})()`,
},
errorOut: {
code: `(() => {
return {
default: () => {
throw Error("BOOM");
}
};
})()`,
},
errorOutSys: {
code: `(() => {
return {
default: async () => {
await self.syscall(2, "failingSyscall", []);
}
};
})()`,
},
},
hooks: {
events: {},
},
},
createSandbox(system)
);
expect(await plug.invoke("addTen", [10])).toBe(20);
for (let i = 0; i < 100; i++) {
expect(await plug.invoke("addNumbersSyscall", [10, i])).toBe(10 + i);
}
try {
await plug.invoke("errorOut", []);
expect(true).toBe(false);
} catch (e: any) {
expect(e.message).toBe("BOOM");
}
try {
await plug.invoke("errorOutSys", []);
expect(true).toBe(false);
} catch (e: any) {
expect(e.message).toBe("#fail");
}
await system.stop();
});
-193
View File
@@ -1,193 +0,0 @@
import {
ControllerMessage,
Manifest,
WorkerLike,
WorkerMessage,
} from "./types";
interface SysCallMapping {
[key: string]: (...args: any) => Promise<any> | any;
}
export class Sandbox {
protected worker: WorkerLike;
protected reqId = 0;
protected outstandingInits = new Map<string, () => void>();
protected outstandingInvocations = new Map<
number,
{ resolve: (result: any) => void; reject: (e: any) => void }
>();
protected loadedFunctions = new Set<string>();
protected system: System<any>;
constructor(system: System<any>, worker: WorkerLike) {
worker.onMessage = this.onMessage.bind(this);
this.worker = worker;
this.system = system;
}
isLoaded(name: string) {
return this.loadedFunctions.has(name);
}
async load(name: string, code: string): Promise<void> {
this.worker.postMessage({
type: "load",
name: name,
code: code,
} as WorkerMessage);
return new Promise((resolve) => {
this.loadedFunctions.add(name);
this.outstandingInits.set(name, resolve);
});
}
async onMessage(data: ControllerMessage) {
switch (data.type) {
case "inited":
let initCb = this.outstandingInits.get(data.name!);
initCb && initCb();
this.outstandingInits.delete(data.name!);
break;
case "syscall":
try {
let result = await this.system.syscall(data.name!, data.args!);
this.worker.postMessage({
type: "syscall-response",
id: data.id,
result: result,
} as WorkerMessage);
} catch (e: any) {
this.worker.postMessage({
type: "syscall-response",
id: data.id,
error: e.message,
} as WorkerMessage);
}
break;
case "result":
let resultCbs = this.outstandingInvocations.get(data.id!);
this.outstandingInvocations.delete(data.id!);
if (data.error) {
resultCbs && resultCbs.reject(new Error(data.error));
} else {
resultCbs && resultCbs.resolve(data.result);
}
break;
default:
console.error("Unknown message type", data);
}
}
async invoke(name: string, args: any[]): Promise<any> {
this.reqId++;
this.worker.postMessage({
type: "invoke",
id: this.reqId,
name,
args,
});
return new Promise((resolve, reject) => {
this.outstandingInvocations.set(this.reqId, { resolve, reject });
});
}
stop() {
this.worker.terminate();
}
}
export class Plug<HookT> {
system: System<HookT>;
sandbox: Sandbox;
public manifest?: Manifest<HookT>;
constructor(system: System<HookT>, name: string, sandbox: Sandbox) {
this.system = system;
this.sandbox = sandbox;
}
async load(manifest: Manifest<HookT>) {
this.manifest = manifest;
await this.dispatchEvent("load");
}
async invoke(name: string, args: Array<any>): Promise<any> {
if (!this.sandbox.isLoaded(name)) {
const funDef = this.manifest!.functions[name];
if (!funDef) {
throw new Error(`Function ${name} not found in manifest`);
}
await this.sandbox.load(name, this.manifest!.functions[name].code!);
}
return await this.sandbox.invoke(name, args);
}
async dispatchEvent(name: string, data?: any): Promise<any[]> {
let functionsToSpawn = this.manifest!.hooks.events[name];
if (functionsToSpawn) {
return await Promise.all(
functionsToSpawn.map(
async (functionToSpawn: string) =>
await this.invoke(functionToSpawn, [data])
)
);
} else {
return [];
}
}
async stop() {
this.sandbox.stop();
}
}
export class System<HookT> {
protected plugs = new Map<string, Plug<HookT>>();
registeredSyscalls: SysCallMapping = {};
registerSyscalls(...registrationObjects: SysCallMapping[]) {
for (const registrationObject of registrationObjects) {
for (let p in registrationObject) {
this.registeredSyscalls[p] = registrationObject[p];
}
}
}
async syscall(name: string, args: Array<any>): Promise<any> {
const callback = this.registeredSyscalls[name];
if (!name) {
throw Error(`Unregistered syscall ${name}`);
}
if (!callback) {
throw Error(`Registered but not implemented syscall ${name}`);
}
return Promise.resolve(callback(...args));
}
async load(
name: string,
manifest: Manifest<HookT>,
sandbox: Sandbox
): Promise<Plug<HookT>> {
const plug = new Plug(this, name, sandbox);
await plug.load(manifest);
this.plugs.set(name, plug);
return plug;
}
async dispatchEvent(name: string, data?: any): Promise<any[]> {
let promises = [];
for (let plug of this.plugs.values()) {
promises.push(plug.dispatchEvent(name, data));
}
return await Promise.all(promises);
}
async stop(): Promise<void[]> {
return Promise.all(
Array.from(this.plugs.values()).map((plug) => plug.stop())
);
}
}
-111
View File
@@ -1,111 +0,0 @@
import { ControllerMessage, WorkerMessage, WorkerMessageType } from "./types";
import { safeRun } from "./util";
let loadedFunctions = new Map<string, Function>();
let pendingRequests = new Map<
number,
{
resolve: (result: unknown) => void;
reject: (e: any) => void;
}
>();
declare global {
function syscall(id: number, name: string, args: any[]): Promise<any>;
}
let postMessage = self.postMessage.bind(self);
if (window.parent !== window) {
console.log("running in an iframe");
postMessage = window.parent.postMessage.bind(window.parent);
// postMessage({ type: "test" }, "*");
}
self.syscall = async (id: number, name: string, args: any[]) => {
return await new Promise((resolve, reject) => {
pendingRequests.set(id, { resolve, reject });
postMessage(
{
type: "syscall",
id,
name,
args,
},
"*"
);
});
};
function wrapScript(code: string): string {
return `const fn = ${code};
return fn["default"].apply(null, arguments);`;
}
self.addEventListener("message", (event: { data: WorkerMessage }) => {
// console.log("Got a message", event.data);
safeRun(async () => {
let messageEvent = event;
let data = messageEvent.data;
switch (data.type) {
case "load":
console.log("Booting", data.name);
loadedFunctions.set(data.name!, new Function(wrapScript(data.code!)));
postMessage(
{
type: "inited",
name: data.name,
} as ControllerMessage,
"*"
);
break;
case "invoke":
let fn = loadedFunctions.get(data.name!);
if (!fn) {
throw new Error(`Function not loaded: ${data.name}`);
}
try {
let result = await Promise.resolve(fn(...(data.args || [])));
postMessage(
{
type: "result",
id: data.id,
result: result,
} as ControllerMessage,
"*"
);
} catch (e: any) {
postMessage(
{
type: "result",
id: data.id,
error: e.message,
} as ControllerMessage,
"*"
);
throw e;
}
break;
case "syscall-response":
let syscallId = data.id!;
const lookup = pendingRequests.get(syscallId);
if (!lookup) {
console.log(
"Current outstanding requests",
pendingRequests,
"looking up",
syscallId
);
throw Error("Invalid request id");
}
pendingRequests.delete(syscallId);
if (data.error) {
lookup.reject(new Error(data.error));
} else {
lookup.resolve(data.result);
}
break;
}
});
});
-44
View File
@@ -1,44 +0,0 @@
export type EventHook = {
events: { [key: string]: string[] };
};
export type WorkerMessageType = "load" | "invoke" | "syscall-response";
export type WorkerMessage = {
type: WorkerMessageType;
id?: number;
name?: string;
code?: string;
args?: any[];
result?: any;
error?: any;
};
export type ControllerMessageType = "inited" | "result" | "syscall";
export type ControllerMessage = {
type: ControllerMessageType;
id?: number;
name?: string;
args?: any[];
error?: string;
result?: any;
};
export interface Manifest<HookT> {
hooks: HookT & EventHook;
functions: {
[key: string]: FunctionDef;
};
}
export interface FunctionDef {
path?: string;
code?: string;
}
export interface WorkerLike {
onMessage?: (message: any) => Promise<void>;
postMessage(message: any): void;
terminate(): void;
}
-6
View File
@@ -1,6 +0,0 @@
export function safeRun(fn: () => Promise<void>) {
fn().catch((e) => {
// console.error(e);
throw e;
});
}
-34
View File
@@ -1,34 +0,0 @@
import { ControllerMessage, WorkerLike, WorkerMessage } from "./types";
import { Sandbox, System } from "./runtime";
import { safeRun } from "./util";
class WebWorkerWrapper implements WorkerLike {
private worker: Worker;
onMessage?: (message: any) => Promise<void>;
constructor(worker: Worker) {
this.worker = worker;
this.worker.addEventListener("message", (evt: any) => {
let data = evt.data;
if (!data) return;
safeRun(async () => {
await this.onMessage!(data);
});
});
}
postMessage(message: any): void {
this.worker.postMessage(message);
}
terminate() {
return this.worker.terminate();
}
}
export function createSandbox(system: System<any>) {
// ParcelJS will build this file into a worker.
let worker = new Worker(new URL("sandbox_worker.ts", import.meta.url), {
type: "module",
});
return new Sandbox(system, new WebWorkerWrapper(worker));
}
-11
View File
@@ -1,11 +0,0 @@
{
"include": ["src/**/*"],
"compilerOptions": {
"target": "esnext",
"strict": true,
"moduleResolution": "node",
"module": "esnext",
"esModuleInterop": true,
"allowSyntheticDefaultImports": true
}
}
-4370
View File
File diff suppressed because it is too large Load Diff