1
0

Massive cleanup and plugbox cleanup

This commit is contained in:
Zef Hemel 2022-03-23 15:41:12 +01:00
parent a916088215
commit 09d07d587f
34 changed files with 695 additions and 525 deletions

View File

@ -1,5 +1,6 @@
import * as plugbox from "../plugbox/types";
import { EndpointHook } from "../plugbox/types";
import { EndpointHook } from "../plugbox/feature/endpoint";
import { CronHook } from "../plugbox/feature/node_cron";
export type CommandDef = {
// Function name to invoke
@ -18,6 +19,7 @@ export type SilverBulletHooks = {
commands?: {
[key: string]: CommandDef;
};
} & plugbox.EndpointHook;
} & EndpointHook &
CronHook;
export type Manifest = plugbox.Manifest<SilverBulletHooks>;

View File

@ -9,7 +9,7 @@
"plugbox-bundle": "./dist/bundler/plugbox-bundle.js"
},
"scripts": {
"watch": "parcel watch",
"watch": "rm -rf .parcel-cache && parcel watch",
"build": "parcel build",
"clean": "rm -rf dist",
"plugs": "node dist/bundler/plugbox-bundle.js plugs/core/core.plug.json plugs/dist/core.plug.json",
@ -40,7 +40,7 @@
"test": {
"source": [
"plugbox/runtime.test.ts",
"plugbox/endpoint.test.ts",
"plugbox/feature/endpoint.test.ts",
"server/api.test.ts"
],
"outputFormat": "commonjs",

View File

@ -1,11 +0,0 @@
import { System } from "./runtime";
import { CronHook } from "./types";
import cron from "node-cron";
export function cronSystem(system: System<CronHook>) {
let task = cron.schedule("* * * * *", () => {
});
// @ts-ignore
task.destroy();
}

View File

@ -1,85 +0,0 @@
import { System } from "./runtime";
import { EndpointHook } from "./types";
import express from "express";
export type EndpointRequest = {
method: string;
path: string;
query: { [key: string]: string };
headers: { [key: string]: string };
body: any;
};
export type EndpointResponse = {
status: number;
headers?: { [key: string]: string };
body: any;
};
const endPointPrefix = "/_";
export function exposeSystem(system: System<EndpointHook>) {
return (
req: express.Request,
res: express.Response,
next: express.NextFunction
) => {
if (!req.path.startsWith(endPointPrefix)) {
return next();
}
Promise.resolve()
.then(async () => {
for (const [plugName, plug] of system.loadedPlugs.entries()) {
const manifest = plug.manifest;
if (!manifest) {
continue;
}
const endpoints = manifest.hooks?.endpoints;
if (endpoints) {
let prefix = `${endPointPrefix}/${plugName}`;
if (!req.path.startsWith(prefix)) {
continue;
}
for (const { path, method, handler } of endpoints) {
let prefixedPath = `${prefix}${path}`;
if (prefixedPath === req.path && method === req.method) {
try {
const response: EndpointResponse = await plug.invoke(
handler,
[
{
path: req.path,
method: req.method,
body: req.body,
query: req.query,
headers: req.headers,
} as EndpointRequest,
]
);
let resp = res.status(response.status);
if (response.headers) {
for (const [key, value] of Object.entries(
response.headers
)) {
resp = resp.header(key, value);
}
}
resp.send(response.body);
return;
} catch (e: any) {
console.error("Error executing function", e);
res.status(500).send(e.message);
return;
}
}
}
}
}
next();
})
.catch((e) => {
console.error(e);
next(e);
});
};
}

View File

@ -1,9 +1,10 @@
import { ControllerMessage, WorkerLike, WorkerMessage } from "./types";
import { Sandbox, System } from "./runtime";
import { safeRun } from "./util";
import { safeRun } from "../util";
// @ts-ignore
import sandboxHtml from "bundle-text:./iframe_sandbox.html";
import { Sandbox } from "../sandbox";
import { System } from "../system";
import { WorkerLike } from "./worker";
class IFrameWrapper implements WorkerLike {
private iframe: HTMLIFrameElement;

View File

@ -1,12 +1,11 @@
import { ControllerMessage, WorkerLike, WorkerMessage } from "./types";
import { System, Sandbox } from "./runtime";
import { Worker } from "worker_threads";
import * as fs from "fs";
import { safeRun } from "./util";
import { safeRun } from "../util";
// @ts-ignore
import workerCode from "bundle-text:./node_worker.ts";
import { Sandbox } from "../sandbox";
import { System } from "../system";
import { WorkerLike } from "./worker";
class NodeWorkerWrapper implements WorkerLike {
onMessage?: (message: any) => Promise<void>;

View File

@ -1,5 +1,5 @@
import { ControllerMessage, WorkerMessage, WorkerMessageType } from "./types";
import { safeRun } from "./util";
import { safeRun } from "../util";
import { ControllerMessage, WorkerMessage } from "./worker";
let loadedFunctions = new Map<string, Function>();
let pendingRequests = new Map<

View File

@ -1,6 +1,7 @@
import { ControllerMessage, WorkerLike, WorkerMessage } from "./types";
import { Sandbox, System } from "./runtime";
import { safeRun } from "./util";
import { safeRun } from "../util";
import { Sandbox } from "../sandbox";
import { System } from "../system";
import { WorkerLike } from "./worker";
class WebWorkerWrapper implements WorkerLike {
private worker: Worker;

View File

@ -0,0 +1,30 @@
export type ControllerMessageType = "inited" | "result" | "syscall";
export type ControllerMessage = {
type: ControllerMessageType;
id?: number;
name?: string;
args?: any[];
error?: string;
result?: any;
};
export interface WorkerLike {
ready: Promise<void>;
onMessage?: (message: any) => Promise<void>;
postMessage(message: any): void;
terminate(): void;
}
export type WorkerMessageType = "load" | "invoke" | "syscall-response";
export type WorkerMessage = {
type: WorkerMessageType;
id?: number;
name?: string;
code?: string;
args?: any[];
result?: any;
error?: any;
};

View File

@ -1,13 +1,13 @@
import { createSandbox } from "./node_sandbox";
import { System } from "./runtime";
import { test, expect } from "@jest/globals";
import { EndPointDef, EndpointHook, Manifest } from "./types";
import { createSandbox } from "../environment/node_sandbox";
import { expect, test } from "@jest/globals";
import { Manifest } from "../types";
import express from "express";
import request from "supertest";
import { exposeSystem } from "./endpoints";
import { EndpointFeature, EndpointHook } from "./endpoint";
import { System } from "../system";
test("Run a plugbox endpoint server", async () => {
let system = new System<EndpointHook>();
let system = new System<EndpointHook>("server");
let plug = await system.load(
"test",
{
@ -32,7 +32,9 @@ test("Run a plugbox endpoint server", async () => {
const app = express();
const port = 3123;
app.use(exposeSystem(system));
system.addFeature(new EndpointFeature(app));
let server = app.listen(port, () => {
console.log(`Listening on port ${port}`);
});

120
plugbox/feature/endpoint.ts Normal file
View File

@ -0,0 +1,120 @@
import { Feature, Manifest } from "../types";
import { Express, NextFunction, Request, Response } from "express";
import { System } from "../system";
export type EndpointRequest = {
method: string;
path: string;
query: { [key: string]: string };
headers: { [key: string]: string };
body: any;
};
export type EndpointResponse = {
status: number;
headers?: { [key: string]: string };
body: any;
};
export type EndpointHook = {
endpoints?: EndPointDef[];
};
export type EndPointDef = {
method: "GET" | "POST" | "PUT" | "DELETE" | "HEAD" | "OPTIONS";
path: string;
handler: string; // function name
};
const endPointPrefix = "/_";
export class EndpointFeature implements Feature<EndpointHook> {
private app: Express;
constructor(app: Express) {
this.app = app;
}
apply(system: System<EndpointHook>): void {
this.app.use((req: Request, res: Response, next: NextFunction) => {
if (!req.path.startsWith(endPointPrefix)) {
return next();
}
Promise.resolve()
.then(async () => {
for (const [plugName, plug] of system.loadedPlugs.entries()) {
const manifest = plug.manifest;
if (!manifest) {
continue;
}
const endpoints = manifest.hooks?.endpoints;
if (endpoints) {
let prefix = `${endPointPrefix}/${plugName}`;
if (!req.path.startsWith(prefix)) {
continue;
}
for (const { path, method, handler } of endpoints) {
let prefixedPath = `${prefix}${path}`;
if (prefixedPath === req.path && method === req.method) {
try {
const response: EndpointResponse = await plug.invoke(
handler,
[
{
path: req.path,
method: req.method,
body: req.body,
query: req.query,
headers: req.headers,
} as EndpointRequest,
]
);
let resp = res.status(response.status);
if (response.headers) {
for (const [key, value] of Object.entries(
response.headers
)) {
resp = resp.header(key, value);
}
}
resp.send(response.body);
return;
} catch (e: any) {
console.error("Error executing function", e);
res.status(500).send(e.message);
return;
}
}
}
}
}
next();
})
.catch((e) => {
console.error(e);
next(e);
});
});
}
validateManifest(manifest: Manifest<EndpointHook>): string[] {
const endpoints = manifest.hooks.endpoints;
let errors = [];
if (endpoints) {
for (let { method, path, handler } of endpoints) {
if (!path) {
errors.push("Path not defined for endpoint");
}
if (["GET", "POST", "PUT", "DELETE"].indexOf(method) === -1) {
errors.push(
`Invalid method ${method} for end point with with ${path}`
);
}
if (!manifest.functions[handler]) {
errors.push(`Endpoint handler function ${handler} not found`);
}
}
}
return errors;
}
}

View File

@ -0,0 +1,70 @@
import { Feature, Manifest } from "../types";
import cron, { ScheduledTask } from "node-cron";
import { safeRun } from "../util";
import { System } from "../system";
export type CronHook = {
crons?: CronDef[];
};
export type CronDef = {
cron: string;
handler: string; // function name
};
export class NodeCronFeature implements Feature<CronHook> {
apply(system: System<CronHook>): void {
let tasks: ScheduledTask[] = [];
system.on({
plugLoaded: (name, plug) => {
reloadCrons();
},
plugUnloaded(name, plug) {
reloadCrons();
},
});
reloadCrons();
function reloadCrons() {
// ts-ignore
tasks.forEach((task) => task.stop());
tasks = [];
for (let plug of system.loadedPlugs.values()) {
const crons = plug.manifest?.hooks?.crons;
if (crons) {
for (let cronDef of crons) {
tasks.push(
cron.schedule(cronDef.cron, () => {
console.log("Now acting on cron", cronDef.cron);
safeRun(async () => {
try {
await plug.invoke(cronDef.handler, []);
} catch (e: any) {
console.error("Execution of cron function failed", e);
}
});
})
);
}
}
}
}
}
validateManifest(manifest: Manifest<CronHook>): string[] {
const crons = manifest.hooks.crons;
let errors = [];
if (crons) {
for (let cronDef of crons) {
if (!cron.validate(cronDef.cron)) {
errors.push(`Invalid cron expression ${cronDef.cron}`);
}
if (!manifest.functions[cronDef.handler]) {
errors.push(`Cron handler function ${cronDef.handler} not found`);
}
}
}
return errors;
}
}

73
plugbox/plug.ts Normal file
View File

@ -0,0 +1,73 @@
import { Manifest, RuntimeEnvironment } from "./types";
import { Sandbox } from "./sandbox";
import { System } from "./system";
export class Plug<HookT> {
system: System<HookT>;
sandbox: Sandbox;
public manifest?: Manifest<HookT>;
readonly runtimeEnv: RuntimeEnvironment;
constructor(system: System<HookT>, name: string, sandbox: Sandbox) {
this.system = system;
this.sandbox = sandbox;
this.runtimeEnv = system.runtimeEnv;
}
async load(manifest: Manifest<HookT>) {
this.manifest = manifest;
await this.dispatchEvent("load");
}
canInvoke(name: string) {
if (!this.manifest) {
return false;
}
const funDef = this.manifest.functions[name];
if (!funDef) {
throw new Error(`Function ${name} not found in manifest`);
}
return !funDef.env || funDef.env === this.runtimeEnv;
}
async invoke(name: string, args: Array<any>): Promise<any> {
if (!this.sandbox.isLoaded(name)) {
const funDef = this.manifest!.functions[name];
if (!funDef) {
throw new Error(`Function ${name} not found in manifest`);
}
if (!this.canInvoke(name)) {
throw new Error(
`Function ${name} is not available in ${this.runtimeEnv}`
);
}
await this.sandbox.load(name, funDef.code!);
}
return await this.sandbox.invoke(name, args);
}
async dispatchEvent(name: string, data?: any): Promise<any[]> {
if (!this.manifest!.hooks?.events) {
return [];
}
let functionsToSpawn = this.manifest!.hooks.events[name];
if (functionsToSpawn) {
return await Promise.all(
functionsToSpawn.map((functionToSpawn: string) => {
// Only dispatch functions on events when they're allowed to be invoked in this environment
if (this.canInvoke(functionToSpawn)) {
return this.invoke(functionToSpawn, [data]);
} else {
return Promise.resolve();
}
})
);
} else {
return [];
}
}
async stop() {
this.sandbox.stop();
}
}

View File

@ -1,8 +1,8 @@
import fs, { stat, watch } from "fs/promises";
import fs, { watch } from "fs/promises";
import path from "path";
import { createSandbox } from "./node_sandbox";
import { System } from "./runtime";
import { createSandbox } from "./environment/node_sandbox";
import { safeRun } from "../server/util";
import { System } from "./system";
function extractPlugName(localPath: string): string {
const baseName = path.basename(localPath);
@ -34,10 +34,8 @@ export class DiskPlugLoader<HookT> {
} catch (e) {
// Likely removed
await this.system.unload(plugName);
this.system.emit("plugRemoved", plugName);
}
const plugDef = await this.loadPlugFromFile(localPath);
this.system.emit("plugUpdated", plugName, plugDef);
} catch {
// ignore, error handled by loadPlug
}

View File

@ -1,9 +1,9 @@
import { createSandbox } from "./node_sandbox";
import { System } from "./runtime";
import { test, expect } from "@jest/globals";
import { createSandbox } from "./environment/node_sandbox";
import { expect, test } from "@jest/globals";
import { System } from "./system";
test("Run a Node sandbox", async () => {
let system = new System();
let system = new System("server");
system.registerSyscalls({
addNumbers: (a, b) => {
return a + b;

View File

@ -1,248 +0,0 @@
import {
ControllerMessage,
Manifest,
WorkerLike,
WorkerMessage,
} from "./types";
import { EventEmitter } from "../common/event";
interface SysCallMapping {
[key: string]: (...args: any) => Promise<any> | any;
}
export class Sandbox {
protected worker: WorkerLike;
protected reqId = 0;
protected outstandingInits = new Map<string, () => void>();
protected outstandingInvocations = new Map<
number,
{ resolve: (result: any) => void; reject: (e: any) => void }
>();
protected loadedFunctions = new Set<string>();
protected system: System<any>;
constructor(system: System<any>, worker: WorkerLike) {
worker.onMessage = this.onMessage.bind(this);
this.worker = worker;
this.system = system;
}
isLoaded(name: string) {
return this.loadedFunctions.has(name);
}
async load(name: string, code: string): Promise<void> {
await this.worker.ready;
this.worker.postMessage({
type: "load",
name: name,
code: code,
} as WorkerMessage);
return new Promise((resolve) => {
this.loadedFunctions.add(name);
this.outstandingInits.set(name, resolve);
});
}
async onMessage(data: ControllerMessage) {
switch (data.type) {
case "inited":
let initCb = this.outstandingInits.get(data.name!);
initCb && initCb();
this.outstandingInits.delete(data.name!);
break;
case "syscall":
try {
let result = await this.system.syscall(data.name!, data.args!);
this.worker.postMessage({
type: "syscall-response",
id: data.id,
result: result,
} as WorkerMessage);
} catch (e: any) {
this.worker.postMessage({
type: "syscall-response",
id: data.id,
error: e.message,
} as WorkerMessage);
}
break;
case "result":
let resultCbs = this.outstandingInvocations.get(data.id!);
this.outstandingInvocations.delete(data.id!);
if (data.error) {
resultCbs && resultCbs.reject(new Error(data.error));
} else {
resultCbs && resultCbs.resolve(data.result);
}
break;
default:
console.error("Unknown message type", data);
}
}
async invoke(name: string, args: any[]): Promise<any> {
this.reqId++;
this.worker.postMessage({
type: "invoke",
id: this.reqId,
name,
args,
});
return new Promise((resolve, reject) => {
this.outstandingInvocations.set(this.reqId, { resolve, reject });
});
}
stop() {
this.worker.terminate();
}
}
export class Plug<HookT> {
system: System<HookT>;
sandbox: Sandbox;
public manifest?: Manifest<HookT>;
constructor(system: System<HookT>, name: string, sandbox: Sandbox) {
this.system = system;
this.sandbox = sandbox;
}
async load(manifest: Manifest<HookT>) {
this.manifest = manifest;
await this.dispatchEvent("load");
}
async invoke(name: string, args: Array<any>): Promise<any> {
if (!this.sandbox.isLoaded(name)) {
const funDef = this.manifest!.functions[name];
if (!funDef) {
throw new Error(`Function ${name} not found in manifest`);
}
await this.sandbox.load(name, funDef.code!);
}
return await this.sandbox.invoke(name, args);
}
async dispatchEvent(name: string, data?: any): Promise<any[]> {
if (!this.manifest!.hooks?.events) {
return [];
}
let functionsToSpawn = this.manifest!.hooks.events[name];
if (functionsToSpawn) {
return await Promise.all(
functionsToSpawn.map((functionToSpawn: string) =>
this.invoke(functionToSpawn, [data])
)
);
} else {
return [];
}
}
async stop() {
this.sandbox.stop();
}
}
export type SystemJSON<HookT> = { [key: string]: Manifest<HookT> };
export type SystemEvents<HookT> = {
plugUpdated: (name: string, plug: Plug<HookT>) => void;
plugRemoved: (name: string) => void;
};
export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
protected plugs = new Map<string, Plug<HookT>>();
registeredSyscalls: SysCallMapping = {};
constructor() {
super();
}
registerSyscalls(...registrationObjects: SysCallMapping[]) {
for (const registrationObject of registrationObjects) {
for (let p in registrationObject) {
this.registeredSyscalls[p] = registrationObject[p];
}
}
}
async syscall(name: string, args: Array<any>): Promise<any> {
const callback = this.registeredSyscalls[name];
if (!name) {
throw Error(`Unregistered syscall ${name}`);
}
if (!callback) {
throw Error(`Registered but not implemented syscall ${name}`);
}
return Promise.resolve(callback(...args));
}
async load(
name: string,
manifest: Manifest<HookT>,
sandbox: Sandbox
): Promise<Plug<HookT>> {
if (this.plugs.has(name)) {
await this.unload(name);
}
const plug = new Plug(this, name, sandbox);
await plug.load(manifest);
this.plugs.set(name, plug);
return plug;
}
async unload(name: string) {
const plug = this.plugs.get(name);
if (!plug) {
throw Error(`Plug ${name} not found`);
}
await plug.stop();
this.plugs.delete(name);
}
async dispatchEvent(name: string, data?: any): Promise<any[]> {
let promises = [];
for (let plug of this.plugs.values()) {
for (let result of await plug.dispatchEvent(name, data)) {
promises.push(result);
}
}
return await Promise.all(promises);
}
get loadedPlugs(): Map<string, Plug<HookT>> {
return this.plugs;
}
toJSON(): SystemJSON<HookT> {
let plugJSON: { [key: string]: Manifest<HookT> } = {};
for (let [name, plug] of this.plugs) {
if (!plug.manifest) {
continue;
}
plugJSON[name] = plug.manifest;
}
return plugJSON;
}
async replaceAllFromJSON(
json: SystemJSON<HookT>,
sandboxFactory: () => Sandbox
) {
await this.unloadAll();
for (let [name, manifest] of Object.entries(json)) {
console.log("Loading plug", name);
await this.load(name, manifest, sandboxFactory());
}
}
async unloadAll(): Promise<void[]> {
return Promise.all(
Array.from(this.plugs.keys()).map(this.unload.bind(this))
);
}
}

96
plugbox/sandbox.ts Normal file
View File

@ -0,0 +1,96 @@
import { System } from "./system";
import {
ControllerMessage,
WorkerLike,
WorkerMessage,
} from "./environment/worker";
export class Sandbox {
protected worker: WorkerLike;
protected reqId = 0;
protected outstandingInits = new Map<string, () => void>();
protected outstandingInvocations = new Map<
number,
{ resolve: (result: any) => void; reject: (e: any) => void }
>();
protected loadedFunctions = new Set<string>();
protected system: System<any>;
constructor(system: System<any>, worker: WorkerLike) {
worker.onMessage = this.onMessage.bind(this);
this.worker = worker;
this.system = system;
}
isLoaded(name: string) {
return this.loadedFunctions.has(name);
}
async load(name: string, code: string): Promise<void> {
await this.worker.ready;
this.worker.postMessage({
type: "load",
name: name,
code: code,
} as WorkerMessage);
return new Promise((resolve) => {
this.loadedFunctions.add(name);
this.outstandingInits.set(name, resolve);
});
}
async onMessage(data: ControllerMessage) {
switch (data.type) {
case "inited":
let initCb = this.outstandingInits.get(data.name!);
initCb && initCb();
this.outstandingInits.delete(data.name!);
break;
case "syscall":
try {
let result = await this.system.syscall(data.name!, data.args!);
this.worker.postMessage({
type: "syscall-response",
id: data.id,
result: result,
} as WorkerMessage);
} catch (e: any) {
this.worker.postMessage({
type: "syscall-response",
id: data.id,
error: e.message,
} as WorkerMessage);
}
break;
case "result":
let resultCbs = this.outstandingInvocations.get(data.id!);
this.outstandingInvocations.delete(data.id!);
if (data.error) {
resultCbs && resultCbs.reject(new Error(data.error));
} else {
resultCbs && resultCbs.resolve(data.result);
}
break;
default:
console.error("Unknown message type", data);
}
}
async invoke(name: string, args: any[]): Promise<any> {
this.reqId++;
this.worker.postMessage({
type: "invoke",
id: this.reqId,
name,
args,
});
return new Promise((resolve, reject) => {
this.outstandingInvocations.set(this.reqId, { resolve, reject });
});
}
stop() {
this.worker.terminate();
}
}

127
plugbox/system.ts Normal file
View File

@ -0,0 +1,127 @@
import { Feature, Manifest, RuntimeEnvironment } from "./types";
import { EventEmitter } from "../common/event";
import { Sandbox } from "./sandbox";
import { Plug } from "./plug";
interface SysCallMapping {
[key: string]: (...args: any) => Promise<any> | any;
}
export type SystemJSON<HookT> = { [key: string]: Manifest<HookT> };
export type SystemEvents<HookT> = {
plugLoaded: (name: string, plug: Plug<HookT>) => void;
plugUnloaded: (name: string, plug: Plug<HookT>) => void;
};
export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
protected plugs = new Map<string, Plug<HookT>>();
registeredSyscalls: SysCallMapping = {};
protected enabledFeatures = new Set<Feature<HookT>>();
readonly runtimeEnv: RuntimeEnvironment;
constructor(env: RuntimeEnvironment) {
super();
this.runtimeEnv = env;
}
addFeature(feature: Feature<HookT>) {
this.enabledFeatures.add(feature);
feature.apply(this);
}
registerSyscalls(...registrationObjects: SysCallMapping[]) {
for (const registrationObject of registrationObjects) {
for (let p in registrationObject) {
this.registeredSyscalls[p] = registrationObject[p];
}
}
}
async syscall(name: string, args: Array<any>): Promise<any> {
const callback = this.registeredSyscalls[name];
if (!name) {
throw Error(`Unregistered syscall ${name}`);
}
if (!callback) {
throw Error(`Registered but not implemented syscall ${name}`);
}
return Promise.resolve(callback(...args));
}
async load(
name: string,
manifest: Manifest<HookT>,
sandbox: Sandbox
): Promise<Plug<HookT>> {
if (this.plugs.has(name)) {
await this.unload(name);
}
// Validate
let errors: string[] = [];
for (const feature of this.enabledFeatures) {
errors = [...errors, ...feature.validateManifest(manifest)];
}
if (errors.length > 0) {
throw new Error(`Invalid manifest: ${errors.join(", ")}`);
}
// Ok, let's load this thing!
const plug = new Plug(this, name, sandbox);
await plug.load(manifest);
this.plugs.set(name, plug);
this.emit("plugLoaded", name, plug);
return plug;
}
async unload(name: string) {
const plug = this.plugs.get(name);
if (!plug) {
throw Error(`Plug ${name} not found`);
}
await plug.stop();
this.emit("plugUnloaded", name, plug);
this.plugs.delete(name);
}
async dispatchEvent(name: string, data?: any): Promise<any[]> {
let promises = [];
for (let plug of this.plugs.values()) {
for (let result of await plug.dispatchEvent(name, data)) {
promises.push(result);
}
}
return await Promise.all(promises);
}
get loadedPlugs(): Map<string, Plug<HookT>> {
return this.plugs;
}
toJSON(): SystemJSON<HookT> {
let plugJSON: { [key: string]: Manifest<HookT> } = {};
for (let [name, plug] of this.plugs) {
if (!plug.manifest) {
continue;
}
plugJSON[name] = plug.manifest;
}
return plugJSON;
}
async replaceAllFromJSON(
json: SystemJSON<HookT>,
sandboxFactory: () => Sandbox
) {
await this.unloadAll();
for (let [name, manifest] of Object.entries(json)) {
console.log("Loading plug", name);
await this.load(name, manifest, sandboxFactory());
}
}
async unloadAll(): Promise<void[]> {
return Promise.all(
Array.from(this.plugs.keys()).map(this.unload.bind(this))
);
}
}

View File

@ -1,25 +1,4 @@
export type WorkerMessageType = "load" | "invoke" | "syscall-response";
export type WorkerMessage = {
type: WorkerMessageType;
id?: number;
name?: string;
code?: string;
args?: any[];
result?: any;
error?: any;
};
export type ControllerMessageType = "inited" | "result" | "syscall";
export type ControllerMessage = {
type: ControllerMessageType;
id?: number;
name?: string;
args?: any[];
error?: string;
result?: any;
};
import { System } from "./system";
export interface Manifest<HookT> {
hooks: HookT & EventHook;
@ -31,33 +10,17 @@ export interface Manifest<HookT> {
export interface FunctionDef {
path?: string;
code?: string;
env?: RuntimeEnvironment;
}
export type RuntimeEnvironment = "client" | "server";
export type EventHook = {
events?: { [key: string]: string[] };
};
export type EndpointHook = {
endpoints?: EndPointDef[];
};
export type EndPointDef = {
method: "GET" | "POST" | "PUT" | "DELETE" | "HEAD" | "OPTIONS";
path: string;
handler: string; // function name
};
export interface Feature<HookT> {
validateManifest(manifest: Manifest<HookT>): string[];
export type CronHook = {
crons?: CronDef[];
};
export type CronDef = {
cron: string;
handler: string; // function name
};
export interface WorkerLike {
ready: Promise<void>;
onMessage?: (message: any) => Promise<void>;
postMessage(message: any): void;
terminate(): void;
apply(system: System<HookT>): void;
}

View File

@ -45,6 +45,12 @@
"path": "/",
"handler": "endpointTest"
}
],
"crons": [
{
"cron": "*/15 * * * *",
"handler": "gitSnapshot"
}
]
},
"functions": {
@ -88,7 +94,12 @@
"path": "./server.ts:endpointTest"
},
"welcome": {
"path": "./server.ts:welcome"
"path": "./server.ts:welcome",
"env": "server"
},
"gitSnapshot": {
"path": "./git.ts:commit",
"env": "server"
}
}
}

12
plugs/core/git.ts Normal file
View File

@ -0,0 +1,12 @@
import { syscall } from "./lib/syscall";
export async function commit() {
console.log("Snapshotting the current space to git");
await syscall("shell.run", "git", ["add", "./*.md"]);
try {
await syscall("shell.run", "git", ["commit", "-a", "-m", "Snapshot"]);
} catch (e) {
// We can ignore, this happens when there's no changes to commit
}
console.log("Done!");
}

View File

@ -57,10 +57,10 @@ export async function renamePage() {
let text = await syscall("editor.getText");
console.log("Writing new page to space");
await syscall("space.writePage", newName, text);
console.log("Deleting page from space");
await syscall("space.deletePage", oldName);
console.log("Navigating to new page");
await syscall("editor.navigate", newName);
console.log("Deleting page from space");
await syscall("space.deletePage", oldName);
let pageToUpdateSet = new Set<string>();
for (let pageToUpdate of pagesToUpdate) {

View File

@ -1,4 +1,7 @@
import { EndpointRequest, EndpointResponse } from "../../plugbox/endpoints";
import {
EndpointRequest,
EndpointResponse,
} from "../../plugbox/feature/endpoint";
export function endpointTest(req: EndpointRequest): EndpointResponse {
console.log("I'm running on the server!", req);
@ -9,7 +12,5 @@ export function endpointTest(req: EndpointRequest): EndpointResponse {
}
export function welcome() {
for (var i = 0; i < 10; i++) {
console.log("Welcome to you all!!!");
}
console.log("Hello world!");
}

View File

@ -1,4 +1,4 @@
import { test, expect, beforeAll, afterAll, describe } from "@jest/globals";
import { afterAll, beforeAll, describe, expect, test } from "@jest/globals";
import { createServer } from "http";
import { io as Client } from "socket.io-client";
@ -7,7 +7,7 @@ import { SocketServer } from "./api_server";
import * as path from "path";
import * as fs from "fs";
import { SilverBulletHooks } from "../common/manifest";
import { System } from "../plugbox/runtime";
import { System } from "../plugbox/system";
describe("Server test", () => {
let io: Server,
@ -43,7 +43,7 @@ describe("Server test", () => {
socketServer = new SocketServer(
tmpDir,
io,
new System<SilverBulletHooks>()
new System<SilverBulletHooks>("server")
);
clientSocket.on("connect", done);
await socketServer.init();

View File

@ -3,17 +3,20 @@ import { Page } from "./types";
import * as path from "path";
import { IndexApi } from "./index_api";
import { PageApi } from "./page_api";
import { System } from "../plugbox/runtime";
import { SilverBulletHooks } from "../common/manifest";
import pageIndexSyscalls from "./syscalls/page_index";
import { safeRun } from "./util";
import { System } from "../plugbox/system";
export class ClientConnection {
openPages = new Set<string>();
constructor(readonly sock: Socket) {}
}
export interface ApiProvider {
init(): Promise<void>;
api(): Object;
}
@ -62,13 +65,19 @@ export class SocketServer {
socket.on("disconnect", () => {
console.log("Disconnected", socket.id);
clientConn.openPages.forEach(disconnectPageSocket);
clientConn.openPages.forEach((pageName) => {
safeRun(async () => {
await disconnectPageSocket(pageName);
});
});
this.connectedSockets.delete(socket);
});
socket.on("page.closePage", (pageName: string) => {
console.log("Client closed page", pageName);
disconnectPageSocket(pageName);
safeRun(async () => {
await disconnectPageSocket(pageName);
});
clientConn.openPages.delete(pageName);
});
@ -87,12 +96,12 @@ export class SocketServer {
});
};
const disconnectPageSocket = (pageName: string) => {
const disconnectPageSocket = async (pageName: string) => {
let page = this.openPages.get(pageName);
if (page) {
for (let client of page.clientStates) {
if (client.socket === socket) {
(this.apis.get("page")! as PageApi).disconnectClient(
await (this.apis.get("page")! as PageApi).disconnectClient(
client,
page
);

View File

@ -1,4 +1,4 @@
import { readdir, readFile, stat, unlink, writeFile } from "fs/promises";
import { mkdir, readdir, readFile, stat, unlink, writeFile } from "fs/promises";
import * as path from "path";
import { PageMeta } from "./types";
@ -48,7 +48,7 @@ export class DiskStorage {
},
};
} catch (e) {
// console.error("Error while writing page", pageName, e);
// console.error("Error while reading page", pageName, e);
throw Error(`Could not read page ${pageName}`);
}
}
@ -56,9 +56,13 @@ export class DiskStorage {
async writePage(pageName: string, text: string): Promise<PageMeta> {
let localPath = path.join(this.rootPath, pageName + ".md");
try {
// Ensure parent folder exists
await mkdir(path.dirname(localPath), { recursive: true });
// Actually write the file
await writeFile(localPath, text);
// console.log(`Wrote to ${localPath}`);
// Fetch new metadata
const s = await stat(localPath);
return {
name: pageName,

View File

@ -1,8 +1,8 @@
import { Express } from "express";
import { System } from "../plugbox/runtime";
import { SilverBulletHooks } from "../common/manifest";
import { exposeSystem } from "../plugbox/endpoints";
import { EndpointFeature } from "../plugbox/feature/endpoint";
import { readFile } from "fs/promises";
import { System } from "../plugbox/system";
export class ExpressServer {
app: Express;
@ -19,7 +19,7 @@ export class ExpressServer {
this.rootPath = rootPath;
this.system = system;
app.use(exposeSystem(this.system));
system.addFeature(new EndpointFeature(app));
// Fallback, serve index.html
let cachedIndex: string | undefined = undefined;

View File

@ -9,8 +9,8 @@ import fs from "fs";
import path from "path";
import { stat } from "fs/promises";
import { Cursor, cursorEffect } from "../webapp/cursorEffect";
import { System } from "../plugbox/runtime";
import { SilverBulletHooks } from "../common/manifest";
import { System } from "../plugbox/system";
export class PageApi implements ApiProvider {
openPages: Map<string, Page>;
@ -34,17 +34,18 @@ export class PageApi implements ApiProvider {
async init(): Promise<void> {
this.fileWatcher();
// TODO: Move this elsewhere, this doesn't belong here
this.system.on({
plugUpdated: (plugName, plugDef) => {
plugLoaded: (plugName, plugDef) => {
console.log("Plug updated on disk, broadcasting to all clients");
this.connectedSockets.forEach((socket) => {
socket.emit("plugUpdated", plugName, plugDef);
socket.emit("plugLoaded", plugName, plugDef.manifest);
});
},
plugRemoved: (plugName) => {
plugUnloaded: (plugName) => {
console.log("Plug removed on disk, broadcasting to all clients");
this.connectedSockets.forEach((socket) => {
socket.emit("plugRemoved", plugName);
socket.emit("plugUnloaded", plugName);
});
},
});
@ -60,20 +61,18 @@ export class PageApi implements ApiProvider {
});
}
flushPageToDisk(name: string, page: Page) {
safeRun(async () => {
async flushPageToDisk(name: string, page: Page) {
let meta = await this.pageStore.writePage(name, page.text.sliceString(0));
console.log(`Wrote page ${name} to disk`);
page.meta = meta;
});
}
disconnectClient(client: ClientPageState, page: Page) {
async disconnectClient(client: ClientPageState, page: Page) {
console.log("Disconnecting client");
page.clientStates.delete(client);
if (page.clientStates.size === 0) {
console.log("No more clients for", page.name, "flushing");
this.flushPageToDisk(page.name, page);
await this.flushPageToDisk(page.name, page);
this.openPages.delete(page.name);
} else {
page.cursors.delete(client.socket.id);
@ -214,16 +213,22 @@ export class PageApi implements ApiProvider {
// Throttle
if (!page.saveTimer) {
page.saveTimer = setTimeout(() => {
safeRun(async () => {
if (page) {
console.log("Indexing", pageName);
console.log(
"Persisting",
pageName,
" to disk and indexing."
);
await this.flushPageToDisk(pageName, page);
this.system.dispatchEvent("page:index", {
await this.system.dispatchEvent("page:index", {
name: pageName,
text: page.text.sliceString(0),
});
this.flushPageToDisk(pageName, page);
page.saveTimer = undefined;
}
});
}, 1000);
}
}

View File

@ -4,10 +4,12 @@ import { Server } from "socket.io";
import { SocketServer } from "./api_server";
import yargs from "yargs";
import { hideBin } from "yargs/helpers";
import { System } from "../plugbox/runtime";
import { SilverBulletHooks } from "../common/manifest";
import { ExpressServer } from "./express_server";
import { DiskPlugLoader } from "../plugbox/plug_loader";
import { NodeCronFeature } from "../plugbox/feature/node_cron";
import shellSyscalls from "./syscalls/shell";
import { System } from "../plugbox/system";
let args = yargs(hideBin(process.argv))
.option("debug", {
@ -23,7 +25,7 @@ const pagesPath = args._[0] as string;
const app = express();
const server = http.createServer(app);
const system = new System<SilverBulletHooks>();
const system = new System<SilverBulletHooks>("server");
const io = new Server(server, {
cors: {
@ -52,6 +54,8 @@ expressServer
);
await plugLoader.loadPlugs();
plugLoader.watcher();
system.registerSyscalls(shellSyscalls(pagesPath));
system.addFeature(new NodeCronFeature());
server.listen(port, () => {
console.log(`Server listening on port ${port}`);
});

View File

@ -21,8 +21,7 @@ import {
} from "@codemirror/view";
import React, { useEffect, useReducer } from "react";
import ReactDOM from "react-dom";
import { Plug, System } from "../plugbox/runtime";
import { createSandbox as createIFrameSandbox } from "../plugbox/iframe_sandbox";
import { createSandbox as createIFrameSandbox } from "../plugbox/environment/iframe_sandbox";
import { AppEvent, AppEventDispatcher, ClickEvent } from "./app_event";
import { CollabDocument, collabExtension } from "./collab";
import * as commands from "./commands";
@ -38,7 +37,6 @@ import reducer from "./reducer";
import { smartQuoteKeymap } from "./smart_quotes";
import { Space } from "./space";
import customMarkdownStyle from "./style";
import dbSyscalls from "./syscalls/db.localstorage";
import editorSyscalls from "./syscalls/editor.browser";
import indexerSyscalls from "./syscalls/indexer.native";
import spaceSyscalls from "./syscalls/space.native";
@ -51,6 +49,7 @@ import {
} from "./types";
import { SilverBulletHooks } from "../common/manifest";
import { safeRun } from "./util";
import { System } from "../plugbox/system";
class PageState {
scrollTop: number;
@ -63,19 +62,17 @@ class PageState {
}
export class Editor implements AppEventDispatcher {
private system = new System<SilverBulletHooks>();
private system = new System<SilverBulletHooks>("client");
openPages = new Map<string, PageState>();
editorCommands = new Map<string, AppCommand>();
editorView?: EditorView;
viewState: AppViewState;
viewDispatch: React.Dispatch<Action>;
openPages: Map<string, PageState>;
space: Space;
editorCommands: Map<string, AppCommand>;
navigationResolve?: (val: undefined) => void;
pageNavigator: IPageNavigator;
constructor(space: Space, parent: Element) {
this.editorCommands = new Map();
this.openPages = new Map();
this.space = space;
this.viewState = initialViewState;
this.viewDispatch = () => {};
@ -90,7 +87,6 @@ export class Editor implements AppEventDispatcher {
this.pageNavigator = new PathPageNavigator();
this.system.registerSyscalls(
dbSyscalls,
editorSyscalls(this),
spaceSyscalls(this),
indexerSyscalls(this.space)
@ -98,7 +94,6 @@ export class Editor implements AppEventDispatcher {
}
async init() {
await this.loadPlugs();
this.focus();
this.pageNavigator.subscribe(async (pageName) => {
@ -134,34 +129,26 @@ export class Editor implements AppEventDispatcher {
},
loadSystem: (systemJSON) => {
safeRun(async () => {
console.log("Received SYSTEM", systemJSON);
await this.system.replaceAllFromJSON(systemJSON, () =>
createIFrameSandbox(this.system)
);
console.log("Loaded plugs, now updating editor comands");
this.editorCommands = new Map<string, AppCommand>();
for (let plug of this.system.loadedPlugs.values()) {
this.buildCommands(plug);
}
this.viewDispatch({
type: "update-commands",
commands: this.editorCommands,
});
this.buildAllCommands();
});
},
plugUpdated: (plugName, plug) => {
plugLoaded: (plugName, plug) => {
safeRun(async () => {
console.log("Plug updated", plugName);
console.log("Plug load", plugName);
await this.system.load(
plugName,
plug,
createIFrameSandbox(this.system)
);
this.buildAllCommands();
});
},
plugRemoved: (plugName) => {
plugUnloaded: (plugName) => {
safeRun(async () => {
console.log("Plug removed", plugName);
console.log("Plug unload", plugName);
await this.system.unload(plugName);
});
},
@ -172,6 +159,27 @@ export class Editor implements AppEventDispatcher {
}
}
private buildAllCommands() {
console.log("Loaded plugs, now updating editor commands");
this.editorCommands.clear();
for (let plug of this.system.loadedPlugs.values()) {
const cmds = plug.manifest!.hooks.commands;
for (let name in cmds) {
let cmd = cmds[name];
this.editorCommands.set(name, {
command: cmd,
run: async (arg): Promise<any> => {
return await plug.invoke(cmd.invoke, [arg]);
},
});
}
}
this.viewDispatch({
type: "update-commands",
commands: this.editorCommands,
});
}
flashNotification(message: string) {
let id = Math.floor(Math.random() * 1000000);
this.viewDispatch({
@ -190,29 +198,6 @@ export class Editor implements AppEventDispatcher {
}, 2000);
}
async loadPlugs() {
const system = new System<SilverBulletHooks>();
system.registerSyscalls(
dbSyscalls,
editorSyscalls(this),
spaceSyscalls(this),
indexerSyscalls(this.space)
);
}
private buildCommands(plug: Plug<SilverBulletHooks>) {
const cmds = plug.manifest!.hooks.commands;
for (let name in cmds) {
let cmd = cmds[name];
this.editorCommands.set(name, {
command: cmd,
run: async (arg): Promise<any> => {
return await plug.invoke(cmd.invoke, [arg]);
},
});
}
}
async dispatchAppEvent(name: AppEvent, data?: any): Promise<any[]> {
return this.system.dispatchEvent(name, data);
}
@ -349,7 +334,6 @@ export class Editor implements AppEventDispatcher {
async plugCompleter(): Promise<CompletionResult | null> {
let allCompletionResults = await this.dispatchAppEvent("editor:complete");
console.log("Completion results", allCompletionResults);
if (allCompletionResults.length === 1) {
return allCompletionResults[0];
} else if (allCompletionResults.length > 1) {
@ -398,8 +382,8 @@ export class Editor implements AppEventDispatcher {
this.editorView!.focus();
}
navigate(name: string) {
this.pageNavigator.navigate(name);
async navigate(name: string) {
await this.pageNavigator.navigate(name);
}
async loadPage(pageName: string) {

View File

@ -2,7 +2,9 @@ import { safeRun } from "./util";
export interface IPageNavigator {
subscribe(pageLoadCallback: (pageName: string) => Promise<void>): void;
navigate(page: string): void;
navigate(page: string): Promise<void>;
getCurrentPage(): string;
}

View File

@ -6,8 +6,8 @@ import { ChangeSet, Text, Transaction } from "@codemirror/state";
import { CollabDocument, CollabEvents } from "./collab";
import { cursorEffect } from "./cursorEffect";
import { EventEmitter } from "../common/event";
import { SystemJSON } from "../plugbox/runtime";
import { Manifest } from "../common/manifest";
import { SystemJSON } from "../plugbox/system";
export type SpaceEvents = {
connect: () => void;
@ -16,8 +16,8 @@ export type SpaceEvents = {
pageDeleted: (name: string) => void;
pageListUpdated: (pages: Set<PageMeta>) => void;
loadSystem: (systemJSON: SystemJSON<any>) => void;
plugUpdated: (plugName: string, plug: Manifest) => void;
plugRemoved: (plugName: string) => void;
plugLoaded: (plugName: string, plug: Manifest) => void;
plugUnloaded: (plugName: string) => void;
} & CollabEvents;
export type KV = {
@ -41,8 +41,8 @@ export class Space extends EventEmitter<SpaceEvents> {
"pageChanged",
"pageDeleted",
"loadSystem",
"plugUpdated",
"plugRemoved",
"plugLoaded",
"plugUnloaded",
].forEach((eventName) => {
socket.on(eventName, (...args) => {
this.emit(eventName as keyof SpaceEvents, ...args);