From 8bff6d98e10e769657ed780ab39c6df7480b2dcc Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Fri, 18 Mar 2022 14:59:04 +0100 Subject: [PATCH] Enabled back-end running of functions, moved indexing to server. --- plugbox/bin/plugbox-bundle.mjs | 3 +- plugbox/package.json | 4 +- .../src/{sandbox.html => iframe_sandbox.html} | 3 +- plugbox/src/iframe_sandbox.ts | 43 +++++++ plugbox/src/node_sandbox.ts | 105 +++++----------- plugbox/src/node_worker.ts | 88 -------------- plugbox/src/runtime.test.ts | 40 +++++- plugbox/src/runtime.ts | 114 ++++++++++++++++-- plugbox/src/sandbox_worker.ts | 77 ++++++++---- plugbox/src/types.ts | 15 ++- plugbox/src/webworker_sandbox.ts | 34 ++++++ plugbox/src/worker_sandbox.ts | 88 -------------- plugs/core/core.plug.json | 3 + plugs/core/server.ts | 5 + server/package.json | 3 + server/src/api.test.ts | 2 +- server/src/{api.ts => api_server.ts} | 45 +++++-- server/src/index_api.ts | 57 ++------- server/src/page_api.ts | 28 ++++- server/src/server.ts | 2 +- server/src/syscalls/page_index.ts | 82 +++++++++++++ server/yarn.lock | 42 ++++++- tsconfig.json | 3 +- webapp/package.json | 4 +- webapp/src/editor.tsx | 5 +- webapp/src/indexer.ts | 3 +- webapp/yarn.lock | 7 ++ 27 files changed, 538 insertions(+), 367 deletions(-) rename plugbox/src/{sandbox.html => iframe_sandbox.html} (60%) create mode 100644 plugbox/src/iframe_sandbox.ts delete mode 100644 plugbox/src/node_worker.ts create mode 100644 plugbox/src/webworker_sandbox.ts delete mode 100644 plugbox/src/worker_sandbox.ts create mode 100644 plugs/core/server.ts rename server/src/{api.ts => api_server.ts} (73%) create mode 100644 server/src/syscalls/page_index.ts diff --git a/plugbox/bin/plugbox-bundle.mjs b/plugbox/bin/plugbox-bundle.mjs index 178df30..6b6ddec 100755 --- a/plugbox/bin/plugbox-bundle.mjs +++ b/plugbox/bin/plugbox-bundle.mjs @@ -40,7 +40,8 @@ export default ${functionName};` if (inFile !== filePath) { await unlink(inFile); } - return jsCode; + // Strip final ';' + return jsCode.substring(0, jsCode.length - 2); } async function bundle(manifestPath, sourceMaps) { diff --git a/plugbox/package.json b/plugbox/package.json index d11d332..a540657 100644 --- a/plugbox/package.json +++ b/plugbox/package.json @@ -8,8 +8,7 @@ }, "scripts": { "check": "tsc --noEmit", - "test": "jest", - "build-worker": "tsc src/node_worker.ts --outDir dist --module nodenext" + "test": "jest" }, "dependencies": { "esbuild": "^0.14.24", @@ -27,7 +26,6 @@ "events": "^3.3.0", "jest": "^27.5.1", "parcel": "^2.3.2", - "parceljs": "^0.0.1", "path-browserify": "^1.0.1", "ts-jest": "^27.1.3", "util": "^0.12.4", diff --git a/plugbox/src/sandbox.html b/plugbox/src/iframe_sandbox.html similarity index 60% rename from plugbox/src/sandbox.html rename to plugbox/src/iframe_sandbox.html index 8f6045b..ca0b2fc 100644 --- a/plugbox/src/sandbox.html +++ b/plugbox/src/iframe_sandbox.html @@ -1,7 +1,8 @@ diff --git a/plugbox/src/iframe_sandbox.ts b/plugbox/src/iframe_sandbox.ts new file mode 100644 index 0000000..ddaadaf --- /dev/null +++ b/plugbox/src/iframe_sandbox.ts @@ -0,0 +1,43 @@ +import { ControllerMessage, WorkerLike, WorkerMessage } from "./types"; +import { Sandbox, System } from "./runtime"; +import { safeRun } from "./util"; + +// @ts-ignore +import sandboxHtml from "bundle-text:./iframe_sandbox.html"; + +class IFrameWrapper implements WorkerLike { + private iframe: HTMLIFrameElement; + onMessage?: (message: any) => Promise; + + constructor() { + const iframe = document.createElement("iframe", {}); + this.iframe = iframe; + iframe.style.display = "none"; + // Let's lock this down significantly + iframe.setAttribute("sandbox", "allow-scripts"); + iframe.srcdoc = sandboxHtml; + window.addEventListener("message", (evt: any) => { + if (evt.source !== iframe.contentWindow) { + return; + } + let data = evt.data; + if (!data) return; + safeRun(async () => { + await this.onMessage!(data); + }); + }); + document.body.appendChild(iframe); + } + + postMessage(message: any): void { + this.iframe.contentWindow!.postMessage(message, "*"); + } + + terminate() { + return this.iframe.remove(); + } +} + +export function createSandbox(system: System) { + return new Sandbox(system, new IFrameWrapper()); +} diff --git a/plugbox/src/node_sandbox.ts b/plugbox/src/node_sandbox.ts index 0149737..019ad99 100644 --- a/plugbox/src/node_sandbox.ts +++ b/plugbox/src/node_sandbox.ts @@ -1,91 +1,42 @@ -import { ControllerMessage, WorkerMessage } from "./types"; +import { ControllerMessage, WorkerLike, WorkerMessage } from "./types"; import { System, Sandbox } from "./runtime"; import { Worker } from "worker_threads"; +import * as fs from "fs"; +import { safeRun } from "./util"; -function wrapScript(code: string): string { - return `${code}["default"]`; -} +// ParcelJS will simply inline this into the bundle. +const workerCode = fs.readFileSync(__dirname + "/node_worker.js", "utf-8"); -export class NodeSandbox implements Sandbox { - worker: Worker; - private reqId = 0; +class NodeWorkerWrapper implements WorkerLike { + onMessage?: (message: any) => Promise; + private worker: Worker; - outstandingInits = new Map void>(); - outstandingInvocations = new Map< - number, - { resolve: (result: any) => void; reject: (e: any) => void } - >(); - loadedFunctions = new Set(); - - constructor(readonly system: System, workerScript: string) { - this.worker = new Worker(workerScript); - this.worker.on("message", this.onmessage.bind(this)); - } - - isLoaded(name: string): boolean { - return this.loadedFunctions.has(name); - } - - async load(name: string, code: string): Promise { - this.worker.postMessage({ - type: "load", - name: name, - code: code, - } as WorkerMessage); - return new Promise((resolve) => { - this.loadedFunctions.add(name); - this.outstandingInits.set(name, resolve); + constructor(worker: Worker) { + this.worker = worker; + worker.on("message", (message: any) => { + safeRun(async () => { + await this.onMessage!(message); + }); }); } - async onmessage(data: ControllerMessage) { - // let data = evt.data; - // let data = JSON.parse(msg) as ControllerMessage; - switch (data.type) { - case "inited": - let initCb = this.outstandingInits.get(data.name!); - initCb && initCb(); - this.outstandingInits.delete(data.name!); - break; - case "syscall": - let result = await this.system.syscall(data.name!, data.args!); - - this.worker.postMessage({ - type: "syscall-response", - id: data.id, - data: result, - } as WorkerMessage); - break; - case "result": - let resultCb = this.outstandingInvocations.get(data.id!); - this.outstandingInvocations.delete(data.id!); - resultCb && resultCb.resolve(data.result); - break; - case "error": - let errCb = this.outstandingInvocations.get(data.result.id!); - this.outstandingInvocations.delete(data.id!); - errCb && errCb.reject(data.reason); - break; - default: - console.error("Unknown message type", data); - } + postMessage(message: any): void { + this.worker.postMessage(message); } - async invoke(name: string, args: any[]): Promise { - this.reqId++; - this.worker.postMessage({ - type: "invoke", - id: this.reqId, - name, - args, - }); - return new Promise((resolve, reject) => { - this.outstandingInvocations.set(this.reqId, { resolve, reject }); - }); - } - - stop() { + terminate(): void { this.worker.terminate(); } } + +export function createSandbox(system: System) { + return new Sandbox( + system, + new NodeWorkerWrapper( + new Worker(workerCode, { + eval: true, + }) + ) + ); +} diff --git a/plugbox/src/node_worker.ts b/plugbox/src/node_worker.ts deleted file mode 100644 index a56fd2f..0000000 --- a/plugbox/src/node_worker.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { VM, VMScript } from "vm2"; -import { parentPort } from "worker_threads"; - -let loadedFunctions = new Map(); -let pendingRequests = new Map(); - -let reqId = 0; // Syscall request ID - -let vm = new VM({ - sandbox: { - console: console, - syscall: (name: string, args: any[]) => { - return new Promise((resolve, reject) => { - reqId++; - pendingRequests.set(reqId, resolve); - parentPort!.postMessage({ - type: "syscall", - id: reqId, - name, - // TODO: Figure out why this is necessary (to avoide a CloneError) - args: JSON.parse(JSON.stringify(args)), - }); - }); - }, - }, -}); - -function wrapScript(code: string) { - return `${code}["default"]`; -} - -function safeRun(fn: () => Promise) { - fn().catch((e) => { - console.error(e); - }); -} - -parentPort!.on("message", (data) => { - safeRun(async () => { - switch (data.type) { - case "load": - console.log("Booting", data.name); - loadedFunctions.set(data.name, new VMScript(wrapScript(data.code))); - parentPort!.postMessage({ - type: "inited", - name: data.name, - }); - break; - case "invoke": - let fn = loadedFunctions.get(data.name); - if (!fn) { - throw new Error(`Function not loaded: ${data.name}`); - } - try { - let r = vm.run(fn); - let result = await Promise.resolve(r(...data.args)); - parentPort!.postMessage({ - type: "result", - id: data.id, - result: result, - }); - } catch (e: any) { - parentPort!.postMessage({ - type: "error", - id: data.id, - reason: e.message, - }); - throw e; - } - break; - case "syscall-response": - let syscallId = data.id; - const lookup = pendingRequests.get(syscallId); - if (!lookup) { - console.log( - "Current outstanding requests", - pendingRequests, - "looking up", - syscallId - ); - throw Error("Invalid request id"); - } - pendingRequests.delete(syscallId); - lookup(data.data); - break; - } - }); -}); diff --git a/plugbox/src/runtime.test.ts b/plugbox/src/runtime.test.ts index 5960bcf..2b00d35 100644 --- a/plugbox/src/runtime.test.ts +++ b/plugbox/src/runtime.test.ts @@ -1,4 +1,4 @@ -import { NodeSandbox } from "./node_sandbox"; +import { createSandbox } from "./node_sandbox"; import { System } from "./runtime"; import { test, expect } from "@jest/globals"; @@ -8,6 +8,9 @@ test("Run a Node sandbox", async () => { addNumbers: (a, b) => { return a + b; }, + failingSyscall: () => { + throw new Error("#fail"); + }, }); let plug = await system.load( "test", @@ -26,7 +29,25 @@ test("Run a Node sandbox", async () => { code: `(() => { return { default: async (a, b) => { - return await(syscall("addNumbers", [a, b])); + return await self.syscall(1, "addNumbers", [a, b]); + } + }; + })()`, + }, + errorOut: { + code: `(() => { + return { + default: () => { + throw Error("BOOM"); + } + }; + })()`, + }, + errorOutSys: { + code: `(() => { + return { + default: async () => { + await self.syscall(2, "failingSyscall", []); } }; })()`, @@ -36,12 +57,23 @@ test("Run a Node sandbox", async () => { events: {}, }, }, - new NodeSandbox(system, __dirname + "/../dist/node_worker.js") + createSandbox(system) ); expect(await plug.invoke("addTen", [10])).toBe(20); for (let i = 0; i < 100; i++) { expect(await plug.invoke("addNumbersSyscall", [10, i])).toBe(10 + i); } - // console.log(plug.sandbox); + try { + await plug.invoke("errorOut", []); + expect(true).toBe(false); + } catch (e: any) { + expect(e.message).toBe("BOOM"); + } + try { + await plug.invoke("errorOutSys", []); + expect(true).toBe(false); + } catch (e: any) { + expect(e.message).toBe("#fail"); + } await system.stop(); }); diff --git a/plugbox/src/runtime.ts b/plugbox/src/runtime.ts index f050fca..bf01e9c 100644 --- a/plugbox/src/runtime.ts +++ b/plugbox/src/runtime.ts @@ -1,15 +1,101 @@ -import { Manifest } from "./types"; -// import { WebworkerSandbox } from "./worker_sandbox"; +import { + ControllerMessage, + Manifest, + WorkerLike, + WorkerMessage, +} from "./types"; interface SysCallMapping { [key: string]: (...args: any) => Promise | any; } -export interface Sandbox { - isLoaded(name: string): boolean; - load(name: string, code: string): Promise; - invoke(name: string, args: any[]): Promise; - stop(): void; +export class Sandbox { + protected worker: WorkerLike; + protected reqId = 0; + protected outstandingInits = new Map void>(); + protected outstandingInvocations = new Map< + number, + { resolve: (result: any) => void; reject: (e: any) => void } + >(); + protected loadedFunctions = new Set(); + protected system: System; + + constructor(system: System, worker: WorkerLike) { + worker.onMessage = this.onMessage.bind(this); + this.worker = worker; + this.system = system; + } + + isLoaded(name: string) { + return this.loadedFunctions.has(name); + } + + async load(name: string, code: string): Promise { + this.worker.postMessage({ + type: "load", + name: name, + code: code, + } as WorkerMessage); + return new Promise((resolve) => { + this.loadedFunctions.add(name); + this.outstandingInits.set(name, resolve); + }); + } + + async onMessage(data: ControllerMessage) { + switch (data.type) { + case "inited": + let initCb = this.outstandingInits.get(data.name!); + initCb && initCb(); + this.outstandingInits.delete(data.name!); + break; + case "syscall": + try { + let result = await this.system.syscall(data.name!, data.args!); + + this.worker.postMessage({ + type: "syscall-response", + id: data.id, + result: result, + } as WorkerMessage); + } catch (e: any) { + this.worker.postMessage({ + type: "syscall-response", + id: data.id, + error: e.message, + } as WorkerMessage); + } + break; + case "result": + let resultCbs = this.outstandingInvocations.get(data.id!); + this.outstandingInvocations.delete(data.id!); + if (data.error) { + resultCbs && resultCbs.reject(new Error(data.error)); + } else { + resultCbs && resultCbs.resolve(data.result); + } + break; + default: + console.error("Unknown message type", data); + } + } + + async invoke(name: string, args: any[]): Promise { + this.reqId++; + this.worker.postMessage({ + type: "invoke", + id: this.reqId, + name, + args, + }); + return new Promise((resolve, reject) => { + this.outstandingInvocations.set(this.reqId, { resolve, reject }); + }); + } + + stop() { + this.worker.terminate(); + } } export class Plug { @@ -29,6 +115,10 @@ export class Plug { async invoke(name: string, args: Array): Promise { if (!this.sandbox.isLoaded(name)) { + const funDef = this.manifest!.functions[name]; + if (!funDef) { + throw new Error(`Function ${name} not found in manifest`); + } await this.sandbox.load(name, this.manifest!.functions[name].code!); } return await this.sandbox.invoke(name, args); @@ -87,11 +177,17 @@ export class System { return plug; } + async dispatchEvent(name: string, data?: any): Promise { + let promises = []; + for (let plug of this.plugs.values()) { + promises.push(plug.dispatchEvent(name, data)); + } + return await Promise.all(promises); + } + async stop(): Promise { return Promise.all( Array.from(this.plugs.values()).map((plug) => plug.stop()) ); } } - -console.log("Starting"); diff --git a/plugbox/src/sandbox_worker.ts b/plugbox/src/sandbox_worker.ts index 91bd04f..2415726 100644 --- a/plugbox/src/sandbox_worker.ts +++ b/plugbox/src/sandbox_worker.ts @@ -2,21 +2,38 @@ import { ControllerMessage, WorkerMessage, WorkerMessageType } from "./types"; import { safeRun } from "./util"; let loadedFunctions = new Map(); -let pendingRequests = new Map void>(); +let pendingRequests = new Map< + number, + { + resolve: (result: unknown) => void; + reject: (e: any) => void; + } +>(); declare global { function syscall(id: number, name: string, args: any[]): Promise; } +let postMessage = self.postMessage.bind(self); + +if (window.parent !== window) { + console.log("running in an iframe"); + postMessage = window.parent.postMessage.bind(window.parent); + // postMessage({ type: "test" }, "*"); +} + self.syscall = async (id: number, name: string, args: any[]) => { return await new Promise((resolve, reject) => { - pendingRequests.set(id, resolve); - self.postMessage({ - type: "syscall", - id, - name, - args, - }); + pendingRequests.set(id, { resolve, reject }); + postMessage( + { + type: "syscall", + id, + name, + args, + }, + "*" + ); }); }; @@ -26,6 +43,7 @@ return fn["default"].apply(null, arguments);`; } self.addEventListener("message", (event: { data: WorkerMessage }) => { + // console.log("Got a message", event.data); safeRun(async () => { let messageEvent = event; let data = messageEvent.data; @@ -33,10 +51,13 @@ self.addEventListener("message", (event: { data: WorkerMessage }) => { case "load": console.log("Booting", data.name); loadedFunctions.set(data.name!, new Function(wrapScript(data.code!))); - self.postMessage({ - type: "inited", - name: data.name, - } as ControllerMessage); + postMessage( + { + type: "inited", + name: data.name, + } as ControllerMessage, + "*" + ); break; case "invoke": let fn = loadedFunctions.get(data.name!); @@ -45,17 +66,23 @@ self.addEventListener("message", (event: { data: WorkerMessage }) => { } try { let result = await Promise.resolve(fn(...(data.args || []))); - self.postMessage({ - type: "result", - id: data.id, - result: result, - } as ControllerMessage); + postMessage( + { + type: "result", + id: data.id, + result: result, + } as ControllerMessage, + "*" + ); } catch (e: any) { - self.postMessage({ - type: "error", - id: data.id, - reason: e.message, - } as ControllerMessage); + postMessage( + { + type: "result", + id: data.id, + error: e.message, + } as ControllerMessage, + "*" + ); throw e; } @@ -73,7 +100,11 @@ self.addEventListener("message", (event: { data: WorkerMessage }) => { throw Error("Invalid request id"); } pendingRequests.delete(syscallId); - lookup(data.data); + if (data.error) { + lookup.reject(new Error(data.error)); + } else { + lookup.resolve(data.result); + } break; } }); diff --git a/plugbox/src/types.ts b/plugbox/src/types.ts index 56ed53d..f50b3ec 100644 --- a/plugbox/src/types.ts +++ b/plugbox/src/types.ts @@ -10,18 +10,19 @@ export type WorkerMessage = { name?: string; code?: string; args?: any[]; - data?: any; + result?: any; + error?: any; }; -export type ControllerMessageType = "inited" | "result" | "error" | "syscall"; +export type ControllerMessageType = "inited" | "result" | "syscall"; export type ControllerMessage = { type: ControllerMessageType; id?: number; name?: string; - reason?: string; args?: any[]; - result: any; + error?: string; + result?: any; }; export interface Manifest { @@ -35,3 +36,9 @@ export interface FunctionDef { path?: string; code?: string; } + +export interface WorkerLike { + onMessage?: (message: any) => Promise; + postMessage(message: any): void; + terminate(): void; +} diff --git a/plugbox/src/webworker_sandbox.ts b/plugbox/src/webworker_sandbox.ts new file mode 100644 index 0000000..72ae57a --- /dev/null +++ b/plugbox/src/webworker_sandbox.ts @@ -0,0 +1,34 @@ +import { ControllerMessage, WorkerLike, WorkerMessage } from "./types"; +import { Sandbox, System } from "./runtime"; +import { safeRun } from "./util"; + +class WebWorkerWrapper implements WorkerLike { + private worker: Worker; + onMessage?: (message: any) => Promise; + + constructor(worker: Worker) { + this.worker = worker; + this.worker.addEventListener("message", (evt: any) => { + let data = evt.data; + if (!data) return; + safeRun(async () => { + await this.onMessage!(data); + }); + }); + } + postMessage(message: any): void { + this.worker.postMessage(message); + } + + terminate() { + return this.worker.terminate(); + } +} + +export function createSandbox(system: System) { + // ParcelJS will build this file into a worker. + let worker = new Worker(new URL("sandbox_worker.ts", import.meta.url), { + type: "module", + }); + return new Sandbox(system, new WebWorkerWrapper(worker)); +} diff --git a/plugbox/src/worker_sandbox.ts b/plugbox/src/worker_sandbox.ts deleted file mode 100644 index cce8ac4..0000000 --- a/plugbox/src/worker_sandbox.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { ControllerMessage, WorkerMessage } from "./types"; -import { Sandbox, System } from "./runtime"; - -export class WebworkerSandbox implements Sandbox { - private worker: Worker; - private reqId = 0; - - private outstandingInits = new Map void>(); - private outstandingInvocations = new Map< - number, - { resolve: (result: any) => void; reject: (e: any) => void } - >(); - private loadedFunctions = new Set(); - - constructor(readonly system: System) { - this.worker = new Worker(new URL("sandbox_worker.ts", import.meta.url), { - type: "module", - }); - - this.worker.onmessage = this.onmessage.bind(this); - } - - isLoaded(name: string) { - return this.loadedFunctions.has(name); - } - - async load(name: string, code: string): Promise { - this.worker.postMessage({ - type: "load", - name: name, - code: code, - } as WorkerMessage); - return new Promise((resolve) => { - this.loadedFunctions.add(name); - this.outstandingInits.set(name, resolve); - }); - } - - async onmessage(evt: { data: ControllerMessage }) { - let data = evt.data; - if (!data) return; - switch (data.type) { - case "inited": - let initCb = this.outstandingInits.get(data.name!); - initCb && initCb(); - this.outstandingInits.delete(data.name!); - break; - case "syscall": - let result = await this.system.syscall(data.name!, data.args!); - - this.worker.postMessage({ - type: "syscall-response", - id: data.id, - data: result, - } as WorkerMessage); - break; - case "result": - let resultCb = this.outstandingInvocations.get(data.id!); - this.outstandingInvocations.delete(data.id!); - resultCb && resultCb.resolve(data.result); - break; - case "error": - let errCb = this.outstandingInvocations.get(data.result.id!); - this.outstandingInvocations.delete(data.id!); - errCb && errCb.reject(data.reason); - break; - default: - console.error("Unknown message type", data); - } - } - - async invoke(name: string, args: any[]): Promise { - this.reqId++; - this.worker.postMessage({ - type: "invoke", - id: this.reqId, - name, - args, - }); - return new Promise((resolve, reject) => { - this.outstandingInvocations.set(this.reqId, { resolve, reject }); - }); - } - - stop() { - this.worker.terminate(); - } -} diff --git a/plugs/core/core.plug.json b/plugs/core/core.plug.json index 29f87bc..fadf7d8 100644 --- a/plugs/core/core.plug.json +++ b/plugs/core/core.plug.json @@ -75,6 +75,9 @@ }, "toggle_h2": { "path": "./markup.ts:toggleH2" + }, + "server_test": { + "path": "./server.ts:test" } } } diff --git a/plugs/core/server.ts b/plugs/core/server.ts new file mode 100644 index 0000000..b3f8f9a --- /dev/null +++ b/plugs/core/server.ts @@ -0,0 +1,5 @@ +import { syscall } from "./lib/syscall"; +export function test() { + console.log("I'm running on the server!"); + return 5; +} diff --git a/server/package.json b/server/package.json index dcdb71e..942b3f9 100644 --- a/server/package.json +++ b/server/package.json @@ -24,9 +24,12 @@ "socket.io": "^4.4.1", "socket.io-client": "^4.4.1", "typescript": "^4.6.2", + "vm2": "^3.9.9", "yargs": "^17.3.1" }, "devDependencies": { + "@parcel/optimizer-data-url": "2.3.2", + "@parcel/transformer-inline-string": "2.3.2", "@types/cors": "^2.8.12", "@types/express": "^4.17.13", "jest": "^27.5.1", diff --git a/server/src/api.test.ts b/server/src/api.test.ts index f142c1f..39dbe29 100644 --- a/server/src/api.test.ts +++ b/server/src/api.test.ts @@ -3,7 +3,7 @@ import { test, expect, beforeAll, afterAll, describe } from "@jest/globals"; import { createServer } from "http"; import { io as Client } from "socket.io-client"; import { Server } from "socket.io"; -import { SocketServer } from "./api"; +import { SocketServer } from "./api_server"; import * as path from "path"; import * as fs from "fs"; diff --git a/server/src/api.ts b/server/src/api_server.ts similarity index 73% rename from server/src/api.ts rename to server/src/api_server.ts index 2aea68b..56501d1 100644 --- a/server/src/api.ts +++ b/server/src/api_server.ts @@ -3,6 +3,11 @@ import { Page } from "./types"; import * as path from "path"; import { IndexApi } from "./index_api"; import { PageApi } from "./page_api"; +import { System } from "../../plugbox/src/runtime"; +import { createSandbox } from "../../plugbox/src/node_sandbox"; +import { NuggetHook } from "../../webapp/src/types"; +import corePlug from "../../webapp/src/generated/core.plug.json"; +import pageIndexSyscalls from "./syscalls/page_index"; export class ClientConnection { openPages = new Set(); @@ -15,27 +20,42 @@ export interface ApiProvider { } export class SocketServer { - rootPath: string; - openPages = new Map(); - connectedSockets = new Set(); - serverSocket: Server; + private openPages = new Map(); + private connectedSockets = new Set(); private apis = new Map(); + readonly rootPath: string; + private serverSocket: Server; + system: System; + + constructor(rootPath: string, serverSocket: Server) { + this.rootPath = path.resolve(rootPath); + this.serverSocket = serverSocket; + this.system = new System(); + } async registerApi(name: string, apiProvider: ApiProvider) { await apiProvider.init(); this.apis.set(name, apiProvider); } - constructor(rootPath: string, serverSocket: Server) { - this.rootPath = path.resolve(rootPath); - this.serverSocket = serverSocket; - } - public async init() { - await this.registerApi("index", new IndexApi(this.rootPath)); + const indexApi = new IndexApi(this.rootPath); + await this.registerApi("index", indexApi); + this.system.registerSyscalls(pageIndexSyscalls(indexApi.db)); await this.registerApi( "page", - new PageApi(this.rootPath, this.connectedSockets) + new PageApi( + this.rootPath, + this.connectedSockets, + this.openPages, + this.system + ) + ); + + let plug = await this.system.load( + "core", + corePlug, + createSandbox(this.system) ); this.serverSocket.on("connection", (socket) => { @@ -51,9 +71,8 @@ export class SocketServer { }); socket.on("closePage", (pageName: string) => { - console.log("Closing page", pageName); - clientConn.openPages.delete(pageName); disconnectPageSocket(pageName); + clientConn.openPages.delete(pageName); }); const onCall = ( diff --git a/server/src/index_api.ts b/server/src/index_api.ts index aacb04d..d54a819 100644 --- a/server/src/index_api.ts +++ b/server/src/index_api.ts @@ -1,6 +1,7 @@ -import { ApiProvider, ClientConnection } from "./api"; +import { ApiProvider, ClientConnection } from "./api_server"; import knex, { Knex } from "knex"; import path from "path"; +import pageIndexSyscalls from "./syscalls/page_index"; type IndexItem = { page: string; @@ -10,6 +11,7 @@ type IndexItem = { export class IndexApi implements ApiProvider { db: Knex; + constructor(rootPath: string) { this.db = knex({ client: "better-sqlite3", @@ -33,12 +35,13 @@ export class IndexApi implements ApiProvider { } api() { + const syscalls = pageIndexSyscalls(this.db); return { clearPageIndexForPage: async ( clientConn: ClientConnection, page: string ) => { - await this.db("page_index").where({ page }).del(); + return syscalls["indexer.clearPageIndexForPage"](page); }, set: async ( clientConn: ClientConnection, @@ -46,77 +49,41 @@ export class IndexApi implements ApiProvider { key: string, value: any ) => { - let changed = await this.db("page_index") - .where({ page, key }) - .update("value", JSON.stringify(value)); - if (changed === 0) { - await this.db("page_index").insert({ - page, - key, - value: JSON.stringify(value), - }); - } + return syscalls["indexer.set"](page, key, value); }, get: async (clientConn: ClientConnection, page: string, key: string) => { - let result = await this.db("page_index") - .where({ page, key }) - .select("value"); - if (result.length) { - return JSON.parse(result[0].value); - } else { - return null; - } + return syscalls["indexer.get"](page, key); }, delete: async ( clientConn: ClientConnection, page: string, key: string ) => { - await this.db("page_index").where({ page, key }).del(); + return syscalls["indexer.delete"](page, key); }, scanPrefixForPage: async ( clientConn: ClientConnection, page: string, prefix: string ) => { - return ( - await this.db("page_index") - .where({ page }) - .andWhereLike("key", `${prefix}%`) - .select("page", "key", "value") - ).map(({ page, key, value }) => ({ - page, - key, - value: JSON.parse(value), - })); + return syscalls["indexer.scanPrefixForPage"](page, prefix); }, scanPrefixGlobal: async ( clientConn: ClientConnection, prefix: string ) => { - return ( - await this.db("page_index") - .andWhereLike("key", `${prefix}%`) - .select("page", "key", "value") - ).map(({ page, key, value }) => ({ - page, - key, - value: JSON.parse(value), - })); + return syscalls["indexer.scanPrefixGlobal"](prefix); }, deletePrefixForPage: async ( clientConn: ClientConnection, page: string, prefix: string ) => { - return this.db("page_index") - .where({ page }) - .andWhereLike("key", `${prefix}%`) - .del(); + return syscalls["indexer.deletePrefixForPage"](page, prefix); }, clearPageIndex: async (clientConn: ClientConnection) => { - return this.db("page_index").del(); + return syscalls["indexer.clearPageIndex"](); }, }; } diff --git a/server/src/page_api.ts b/server/src/page_api.ts index 7ffcff2..04e357f 100644 --- a/server/src/page_api.ts +++ b/server/src/page_api.ts @@ -1,7 +1,7 @@ import { ClientPageState, Page, PageMeta } from "./types"; import { ChangeSet } from "@codemirror/state"; import { Update } from "@codemirror/collab"; -import { ApiProvider, ClientConnection } from "./api"; +import { ApiProvider, ClientConnection } from "./api_server"; import { Socket } from "socket.io"; import { DiskStorage } from "./disk_storage"; import { safeRun } from "./util"; @@ -9,17 +9,27 @@ import fs from "fs"; import path from "path"; import { stat } from "fs/promises"; import { Cursor, cursorEffect } from "../../webapp/src/cursorEffect"; +import { System } from "../../plugbox/src/runtime"; +import { NuggetHook } from "../../webapp/src/types"; export class PageApi implements ApiProvider { - openPages = new Map(); + openPages: Map; pageStore: DiskStorage; rootPath: string; connectedSockets: Set; + private system: System; - constructor(rootPath: string, connectedSockets: Set) { + constructor( + rootPath: string, + connectedSockets: Set, + openPages: Map, + system: System + ) { this.pageStore = new DiskStorage(rootPath); this.rootPath = rootPath; + this.openPages = openPages; this.connectedSockets = connectedSockets; + this.system = system; } async init(): Promise { @@ -45,6 +55,7 @@ export class PageApi implements ApiProvider { } disconnectClient(client: ClientPageState, page: Page) { + console.log("Disconnecting client"); page.clientStates.delete(client); if (page.clientStates.size === 0) { console.log("No more clients for", page.name, "flushing"); @@ -178,6 +189,12 @@ export class PageApi implements ApiProvider { textChanged = true; } } + console.log( + "New version", + page.version, + "Updates buffered:", + page.updates.length + ); if (textChanged) { if (page.saveTimer) { @@ -185,6 +202,11 @@ export class PageApi implements ApiProvider { } page.saveTimer = setTimeout(() => { + console.log("This is the time to index a page"); + this.system.dispatchEvent("page:index", { + name: pageName, + text: page!.text.sliceString(0), + }); this.flushPageToDisk(pageName, page!); }, 1000); } diff --git a/server/src/server.ts b/server/src/server.ts index edd4d68..15a6d90 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -2,7 +2,7 @@ import express from "express"; import { readFile } from "fs/promises"; import http from "http"; import { Server } from "socket.io"; -import { SocketServer } from "./api"; +import { SocketServer } from "./api_server"; import yargs from "yargs"; import { hideBin } from "yargs/helpers"; diff --git a/server/src/syscalls/page_index.ts b/server/src/syscalls/page_index.ts new file mode 100644 index 0000000..27dfa4c --- /dev/null +++ b/server/src/syscalls/page_index.ts @@ -0,0 +1,82 @@ +import { Knex } from "knex"; +type IndexItem = { + page: string; + key: string; + value: any; +}; + +export type KV = { + key: string; + value: any; +}; + +export default function (db: Knex) { + const setter = async (page: string, key: string, value: any) => { + let changed = await db("page_index") + .where({ page, key }) + .update("value", JSON.stringify(value)); + if (changed === 0) { + await db("page_index").insert({ + page, + key, + value: JSON.stringify(value), + }); + } + }; + return { + "indexer.clearPageIndexForPage": async (page: string) => { + await db("page_index").where({ page }).del(); + }, + "indexer.set": setter, + "indexer.batchSet": async (page: string, kvs: KV[]) => { + for (let { key, value } of kvs) { + await setter(page, key, value); + } + }, + "indexer.get": async (page: string, key: string) => { + let result = await db("page_index") + .where({ page, key }) + .select("value"); + if (result.length) { + return JSON.parse(result[0].value); + } else { + return null; + } + }, + "indexer.delete": async (page: string, key: string) => { + await db("page_index").where({ page, key }).del(); + }, + "indexer.scanPrefixForPage": async (page: string, prefix: string) => { + return ( + await db("page_index") + .where({ page }) + .andWhereLike("key", `${prefix}%`) + .select("page", "key", "value") + ).map(({ page, key, value }) => ({ + page, + key, + value: JSON.parse(value), + })); + }, + "indexer.scanPrefixGlobal": async (prefix: string) => { + return ( + await db("page_index") + .andWhereLike("key", `${prefix}%`) + .select("page", "key", "value") + ).map(({ page, key, value }) => ({ + page, + key, + value: JSON.parse(value), + })); + }, + "indexer.deletePrefixForPage": async (page: string, prefix: string) => { + return db("page_index") + .where({ page }) + .andWhereLike("key", `${prefix}%`) + .del(); + }, + "indexer.clearPageIndex": async () => { + return db("page_index").del(); + }, + }; +} diff --git a/server/yarn.lock b/server/yarn.lock index 5bed000..df6cbbe 100644 --- a/server/yarn.lock +++ b/server/yarn.lock @@ -706,6 +706,16 @@ cssnano "^5.0.15" postcss "^8.4.5" +"@parcel/optimizer-data-url@2.3.2": + version "2.3.2" + resolved "https://registry.yarnpkg.com/@parcel/optimizer-data-url/-/optimizer-data-url-2.3.2.tgz#22c2951eb6bda7d7b589c28283d99f9d21dae568" + integrity sha512-q3Y1J3acGPf8yyAhEG+59qey7liB04T/U4i7nmvggDdDVG9S8aYgIeAjsPUKi/9fBoHLn4l8K/sJq3M7FFdcnw== + dependencies: + "@parcel/plugin" "2.3.2" + "@parcel/utils" "2.3.2" + isbinaryfile "^4.0.2" + mime "^2.4.4" + "@parcel/optimizer-htmlnano@2.3.2": version "2.3.2" resolved "https://registry.yarnpkg.com/@parcel/optimizer-htmlnano/-/optimizer-htmlnano-2.3.2.tgz#4086736866621182f5dd1a8abe78e9f5764e1a28" @@ -940,6 +950,13 @@ "@parcel/workers" "2.3.2" nullthrows "^1.1.1" +"@parcel/transformer-inline-string@2.3.2": + version "2.3.2" + resolved "https://registry.yarnpkg.com/@parcel/transformer-inline-string/-/transformer-inline-string-2.3.2.tgz#bec5d376d00b5c41abf11c8cf8e3d917036c0646" + integrity sha512-nitgU+YHnJpJjdUEyRqXD3DjIAstcdHDwwKgloTfpt7EDe2VspVuWhA054kH75Kn/Tvn4s0G4VGCTpDw5KxzSw== + dependencies: + "@parcel/plugin" "2.3.2" + "@parcel/transformer-js@2.3.2": version "2.3.2" resolved "https://registry.yarnpkg.com/@parcel/transformer-js/-/transformer-js-2.3.2.tgz#24bcb488d5f82678343a5630fe4bbe822789ac33" @@ -1324,12 +1341,17 @@ acorn-walk@^7.1.1: resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.2.0.tgz#0de889a601203909b0fbe07b8938dc21d2e967bc" integrity sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA== +acorn-walk@^8.2.0: + version "8.2.0" + resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.2.0.tgz#741210f2e2426454508853a2f44d0ab83b7f69c1" + integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA== + acorn@^7.1.1: version "7.4.1" resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A== -acorn@^8.2.4, acorn@^8.5.0: +acorn@^8.2.4, acorn@^8.5.0, acorn@^8.7.0: version "8.7.0" resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.7.0.tgz#90951fde0f8f09df93549481e5fc141445b791cf" integrity sha512-V/LGr1APy+PXIwKebEWrkZPwoeoF+w1jiOBUmuxuiUIaOHtob8Qc9BTrYo7VuI5fR8tqsy+buA2WFooR5olqvQ== @@ -2958,6 +2980,11 @@ isarray@~1.0.0: resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE= +isbinaryfile@^4.0.2: + version "4.0.8" + resolved "https://registry.yarnpkg.com/isbinaryfile/-/isbinaryfile-4.0.8.tgz#5d34b94865bd4946633ecc78a026fc76c5b11fcf" + integrity sha512-53h6XFniq77YdW+spoRrebh0mnmTxRPTlcuIArO57lmMdq4uBKFKaeTjnb92oYWrSn/LVL+LT+Hap2tFQj8V+w== + isexe@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" @@ -3675,6 +3702,11 @@ mime@1.6.0: resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== +mime@^2.4.4: + version "2.6.0" + resolved "https://registry.yarnpkg.com/mime/-/mime-2.6.0.tgz#a2a682a95cd4d0cb1d6257e28f83da7e35800367" + integrity sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg== + mimic-fn@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" @@ -5163,6 +5195,14 @@ vary@^1, vary@~1.1.2: resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc" integrity sha1-IpnwLG3tMNSllhsLn3RSShj2NPw= +vm2@^3.9.9: + version "3.9.9" + resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.9.9.tgz#c0507bc5fbb99388fad837d228badaaeb499ddc5" + integrity sha512-xwTm7NLh/uOjARRBs8/95H0e8fT3Ukw5D/JJWhxMbhKzNh1Nu981jQKvkep9iKYNxzlVrdzD0mlBGkDKZWprlw== + dependencies: + acorn "^8.7.0" + acorn-walk "^8.2.0" + w3c-hr-time@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz#0a89cdf5cc15822df9c360543676963e0cc308cd" diff --git a/tsconfig.json b/tsconfig.json index d1d8f14..7da98c3 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -5,6 +5,7 @@ "moduleResolution": "node", "module": "esnext", "esModuleInterop": true, - "allowSyntheticDefaultImports": true + "allowSyntheticDefaultImports": true, + "resolveJsonModule": true } } diff --git a/webapp/package.json b/webapp/package.json index 339be75..c8716a7 100644 --- a/webapp/package.json +++ b/webapp/package.json @@ -7,12 +7,14 @@ "license": "MIT", "browserslist": "> 0.5%, last 2 versions, not dead", "scripts": { - "start": "parcel serve --no-cache", + "start": "rm -rf .parcel-cache && parcel serve --no-cache", + "start-no-hmr": "rm -rf .parcel-cache && parcel serve --no-cache --no-hmr", "build": "parcel build", "clean": "rm -rf dist" }, "devDependencies": { "@parcel/packager-raw-url": "2.3.2", + "@parcel/transformer-inline-string": "2.3.2", "@parcel/transformer-sass": "2.3.2", "@parcel/transformer-webmanifest": "2.3.2", "@parcel/validator-typescript": "^2.3.2", diff --git a/webapp/src/editor.tsx b/webapp/src/editor.tsx index 41680ec..066ef42 100644 --- a/webapp/src/editor.tsx +++ b/webapp/src/editor.tsx @@ -29,7 +29,8 @@ import { import React, { useEffect, useReducer } from "react"; import ReactDOM from "react-dom"; import { Plug, System } from "../../plugbox/src/runtime"; -import { WebworkerSandbox } from "../../plugbox/src/worker_sandbox"; +import { createSandbox } from "../../plugbox/src/webworker_sandbox"; +import { createSandbox as createIFrameSandbox } from "../../plugbox/src/iframe_sandbox"; import { AppEvent, AppEventDispatcher, ClickEvent } from "./app_event"; import { collabExtension, CollabDocument } from "./collab"; import * as commands from "./commands"; @@ -182,7 +183,7 @@ export class Editor implements AppEventDispatcher { let mainPlug = await system.load( "core", coreManifest, - new WebworkerSandbox(system) + createIFrameSandbox(system) ); this.plugs.push(mainPlug); this.editorCommands = new Map(); diff --git a/webapp/src/indexer.ts b/webapp/src/indexer.ts index e6936cf..ed996a8 100644 --- a/webapp/src/indexer.ts +++ b/webapp/src/indexer.ts @@ -21,7 +21,8 @@ export class Indexer { name: pageName, text, }; - await appEventDispatcher.dispatchAppEvent("page:index", indexEvent); + + // await appEventDispatcher.dispatchAppEvent("page:index", indexEvent); // await this.setPageIndexPageMeta(pageMeta.name, pageMeta); } diff --git a/webapp/yarn.lock b/webapp/yarn.lock index acc4c38..a036c6a 100644 --- a/webapp/yarn.lock +++ b/webapp/yarn.lock @@ -790,6 +790,13 @@ "@parcel/workers" "2.3.2" nullthrows "^1.1.1" +"@parcel/transformer-inline-string@2.3.2": + version "2.3.2" + resolved "https://registry.yarnpkg.com/@parcel/transformer-inline-string/-/transformer-inline-string-2.3.2.tgz#bec5d376d00b5c41abf11c8cf8e3d917036c0646" + integrity sha512-nitgU+YHnJpJjdUEyRqXD3DjIAstcdHDwwKgloTfpt7EDe2VspVuWhA054kH75Kn/Tvn4s0G4VGCTpDw5KxzSw== + dependencies: + "@parcel/plugin" "2.3.2" + "@parcel/transformer-js@2.3.2": version "2.3.2" resolved "https://registry.yarnpkg.com/@parcel/transformer-js/-/transformer-js-2.3.2.tgz#24bcb488d5f82678343a5630fe4bbe822789ac33"