neil/silverbullet
1
0
Fork 0
Code

Enabled back-end running of functions, moved indexing to server.

Browse Source
This commit is contained in:
Zef Hemel 2022-03-18 14:59:04 +01:00
parent 97b9d3a3e0
commit 8bff6d98e1
27 changed files with 538 additions and 367 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
plugbox/bin/plugbox-bundle.mjs
View File

@ -40,7 +40,8 @@ export default ${functionName};`
if (inFile !== filePath) {
await unlink(inFile);
}
return jsCode;
// Strip final ';'
return jsCode.substring(0, jsCode.length - 2);
}
async function bundle(manifestPath, sourceMaps) {

4
plugbox/package.json
View File

@ -8,8 +8,7 @@
},
"scripts": {
"check": "tsc --noEmit",
"test": "jest",
"build-worker": "tsc src/node_worker.ts --outDir dist --module nodenext"
"test": "jest"
},
"dependencies": {
"esbuild": "^0.14.24",
@ -27,7 +26,6 @@
"events": "^3.3.0",
"jest": "^27.5.1",
"parcel": "^2.3.2",
"parceljs": "^0.0.1",
"path-browserify": "^1.0.1",
"ts-jest": "^27.1.3",
"util": "^0.12.4",

3
plugbox/src/sandbox.html → plugbox/src/iframe_sandbox.html
View File

@ -1,7 +1,8 @@
<html>
<body>
<script type="module">
import "./function_worker";
// Sup yo!
import "./sandbox_worker";
</script>
</body>
</html>

43
plugbox/src/iframe_sandbox.ts Normal file
View File

@ -0,0 +1,43 @@
import { ControllerMessage, WorkerLike, WorkerMessage } from "./types";
import { Sandbox, System } from "./runtime";
import { safeRun } from "./util";
// @ts-ignore
import sandboxHtml from "bundle-text:./iframe_sandbox.html";
class IFrameWrapper implements WorkerLike {
private iframe: HTMLIFrameElement;
onMessage?: (message: any) => Promise<void>;
constructor() {
const iframe = document.createElement("iframe", {});
this.iframe = iframe;
iframe.style.display = "none";
// Let's lock this down significantly
iframe.setAttribute("sandbox", "allow-scripts");
iframe.srcdoc = sandboxHtml;
window.addEventListener("message", (evt: any) => {
if (evt.source !== iframe.contentWindow) {
return;
}
let data = evt.data;
if (!data) return;
safeRun(async () => {
await this.onMessage!(data);
});
});
document.body.appendChild(iframe);
}
postMessage(message: any): void {
this.iframe.contentWindow!.postMessage(message, "*");
}
terminate() {
return this.iframe.remove();
}
}
export function createSandbox(system: System<any>) {
return new Sandbox(system, new IFrameWrapper());
}

105
plugbox/src/node_sandbox.ts
View File

@ -1,91 +1,42 @@
import { ControllerMessage, WorkerMessage } from "./types";
import { ControllerMessage, WorkerLike, WorkerMessage } from "./types";
import { System, Sandbox } from "./runtime";
import { Worker } from "worker_threads";
import * as fs from "fs";
import { safeRun } from "./util";
function wrapScript(code: string): string {
return `${code}["default"]`;
}
// ParcelJS will simply inline this into the bundle.
const workerCode = fs.readFileSync(__dirname + "/node_worker.js", "utf-8");
export class NodeSandbox implements Sandbox {
worker: Worker;
private reqId = 0;
class NodeWorkerWrapper implements WorkerLike {
onMessage?: (message: any) => Promise<void>;
private worker: Worker;
outstandingInits = new Map<string, () => void>();
outstandingInvocations = new Map<
number,
{ resolve: (result: any) => void; reject: (e: any) => void }
>();
loadedFunctions = new Set<string>();
constructor(readonly system: System<any>, workerScript: string) {
this.worker = new Worker(workerScript);
this.worker.on("message", this.onmessage.bind(this));
}
isLoaded(name: string): boolean {
return this.loadedFunctions.has(name);
}
async load(name: string, code: string): Promise<void> {
this.worker.postMessage({
type: "load",
name: name,
code: code,
} as WorkerMessage);
return new Promise((resolve) => {
this.loadedFunctions.add(name);
this.outstandingInits.set(name, resolve);
constructor(worker: Worker) {
this.worker = worker;
worker.on("message", (message: any) => {
safeRun(async () => {
await this.onMessage!(message);
});
});
}
async onmessage(data: ControllerMessage) {
// let data = evt.data;
// let data = JSON.parse(msg) as ControllerMessage;
switch (data.type) {
case "inited":
let initCb = this.outstandingInits.get(data.name!);
initCb && initCb();
this.outstandingInits.delete(data.name!);
break;
case "syscall":
let result = await this.system.syscall(data.name!, data.args!);
this.worker.postMessage({
type: "syscall-response",
id: data.id,
data: result,
} as WorkerMessage);
break;
case "result":
let resultCb = this.outstandingInvocations.get(data.id!);
this.outstandingInvocations.delete(data.id!);
resultCb && resultCb.resolve(data.result);
break;
case "error":
let errCb = this.outstandingInvocations.get(data.result.id!);
this.outstandingInvocations.delete(data.id!);
errCb && errCb.reject(data.reason);
break;
default:
console.error("Unknown message type", data);
}
postMessage(message: any): void {
this.worker.postMessage(message);
}
async invoke(name: string, args: any[]): Promise<any> {
this.reqId++;
this.worker.postMessage({
type: "invoke",
id: this.reqId,
name,
args,
});
return new Promise((resolve, reject) => {
this.outstandingInvocations.set(this.reqId, { resolve, reject });
});
}
stop() {
terminate(): void {
this.worker.terminate();
}
}
export function createSandbox(system: System<any>) {
return new Sandbox(
system,
new NodeWorkerWrapper(
new Worker(workerCode, {
eval: true,
})
)
);
}

88
plugbox/src/node_worker.ts
View File

@ -1,88 +0,0 @@
import { VM, VMScript } from "vm2";
import { parentPort } from "worker_threads";
let loadedFunctions = new Map();
let pendingRequests = new Map();
let reqId = 0; // Syscall request ID
let vm = new VM({
sandbox: {
console: console,
syscall: (name: string, args: any[]) => {
return new Promise((resolve, reject) => {
reqId++;
pendingRequests.set(reqId, resolve);
parentPort!.postMessage({
type: "syscall",
id: reqId,
name,
// TODO: Figure out why this is necessary (to avoide a CloneError)
args: JSON.parse(JSON.stringify(args)),
});
});
},
},
});
function wrapScript(code: string) {
return `${code}["default"]`;
}
function safeRun(fn: () => Promise<any>) {
fn().catch((e) => {
console.error(e);
});
}
parentPort!.on("message", (data) => {
safeRun(async () => {
switch (data.type) {
case "load":
console.log("Booting", data.name);
loadedFunctions.set(data.name, new VMScript(wrapScript(data.code)));
parentPort!.postMessage({
type: "inited",
name: data.name,
});
break;
case "invoke":
let fn = loadedFunctions.get(data.name);
if (!fn) {
throw new Error(`Function not loaded: ${data.name}`);
}
try {
let r = vm.run(fn);
let result = await Promise.resolve(r(...data.args));
parentPort!.postMessage({
type: "result",
id: data.id,
result: result,
});
} catch (e: any) {
parentPort!.postMessage({
type: "error",
id: data.id,
reason: e.message,
});
throw e;
}
break;
case "syscall-response":
let syscallId = data.id;
const lookup = pendingRequests.get(syscallId);
if (!lookup) {
console.log(
"Current outstanding requests",
pendingRequests,
"looking up",
syscallId
);
throw Error("Invalid request id");
}
pendingRequests.delete(syscallId);
lookup(data.data);
break;
}
});
});

40
plugbox/src/runtime.test.ts
View File

@ -1,4 +1,4 @@
import { NodeSandbox } from "./node_sandbox";
import { createSandbox } from "./node_sandbox";
import { System } from "./runtime";
import { test, expect } from "@jest/globals";
@ -8,6 +8,9 @@ test("Run a Node sandbox", async () => {
addNumbers: (a, b) => {
return a + b;
},
failingSyscall: () => {
throw new Error("#fail");
},
});
let plug = await system.load(
"test",
@ -26,7 +29,25 @@ test("Run a Node sandbox", async () => {
code: `(() => {
return {
default: async (a, b) => {
return await(syscall("addNumbers", [a, b]));
return await self.syscall(1, "addNumbers", [a, b]);
}
};
})()`,
},
errorOut: {
code: `(() => {
return {
default: () => {
throw Error("BOOM");
}
};
})()`,
},
errorOutSys: {
code: `(() => {
return {
default: async () => {
await self.syscall(2, "failingSyscall", []);
}
};
})()`,
@ -36,12 +57,23 @@ test("Run a Node sandbox", async () => {
events: {},
},
},
new NodeSandbox(system, __dirname + "/../dist/node_worker.js")
createSandbox(system)
);
expect(await plug.invoke("addTen", [10])).toBe(20);
for (let i = 0; i < 100; i++) {
expect(await plug.invoke("addNumbersSyscall", [10, i])).toBe(10 + i);
}
// console.log(plug.sandbox);
try {
await plug.invoke("errorOut", []);
expect(true).toBe(false);
} catch (e: any) {
expect(e.message).toBe("BOOM");
}
try {
await plug.invoke("errorOutSys", []);
expect(true).toBe(false);
} catch (e: any) {
expect(e.message).toBe("#fail");
}
await system.stop();
});

114
plugbox/src/runtime.ts
View File

@ -1,15 +1,101 @@
import { Manifest } from "./types";
// import { WebworkerSandbox } from "./worker_sandbox";
import {
ControllerMessage,
Manifest,
WorkerLike,
WorkerMessage,
} from "./types";
interface SysCallMapping {
[key: string]: (...args: any) => Promise<any> | any;
}
export interface Sandbox {
isLoaded(name: string): boolean;
load(name: string, code: string): Promise<void>;
invoke(name: string, args: any[]): Promise<any>;
stop(): void;
export class Sandbox {
protected worker: WorkerLike;
protected reqId = 0;
protected outstandingInits = new Map<string, () => void>();
protected outstandingInvocations = new Map<
number,
{ resolve: (result: any) => void; reject: (e: any) => void }
>();
protected loadedFunctions = new Set<string>();
protected system: System<any>;
constructor(system: System<any>, worker: WorkerLike) {
worker.onMessage = this.onMessage.bind(this);
this.worker = worker;
this.system = system;
}
isLoaded(name: string) {
return this.loadedFunctions.has(name);
}
async load(name: string, code: string): Promise<void> {
this.worker.postMessage({
type: "load",
name: name,
code: code,
} as WorkerMessage);
return new Promise((resolve) => {
this.loadedFunctions.add(name);
this.outstandingInits.set(name, resolve);
});
}
async onMessage(data: ControllerMessage) {
switch (data.type) {
case "inited":
let initCb = this.outstandingInits.get(data.name!);
initCb && initCb();
this.outstandingInits.delete(data.name!);
break;
case "syscall":
try {
let result = await this.system.syscall(data.name!, data.args!);
this.worker.postMessage({
type: "syscall-response",
id: data.id,
result: result,
} as WorkerMessage);
} catch (e: any) {
this.worker.postMessage({
type: "syscall-response",
id: data.id,
error: e.message,
} as WorkerMessage);
}
break;
case "result":
let resultCbs = this.outstandingInvocations.get(data.id!);
this.outstandingInvocations.delete(data.id!);
if (data.error) {
resultCbs && resultCbs.reject(new Error(data.error));
} else {
resultCbs && resultCbs.resolve(data.result);
}
break;
default:
console.error("Unknown message type", data);
}
}
async invoke(name: string, args: any[]): Promise<any> {
this.reqId++;
this.worker.postMessage({
type: "invoke",
id: this.reqId,
name,
args,
});
return new Promise((resolve, reject) => {
this.outstandingInvocations.set(this.reqId, { resolve, reject });
});
}
stop() {
this.worker.terminate();
}
}
export class Plug<HookT> {
@ -29,6 +115,10 @@ export class Plug<HookT> {
async invoke(name: string, args: Array<any>): Promise<any> {
if (!this.sandbox.isLoaded(name)) {
const funDef = this.manifest!.functions[name];
if (!funDef) {
throw new Error(`Function ${name} not found in manifest`);
}
await this.sandbox.load(name, this.manifest!.functions[name].code!);
}
return await this.sandbox.invoke(name, args);
@ -87,11 +177,17 @@ export class System<HookT> {
return plug;
}
async dispatchEvent(name: string, data?: any): Promise<any[]> {
let promises = [];
for (let plug of this.plugs.values()) {
promises.push(plug.dispatchEvent(name, data));
}
return await Promise.all(promises);
}
async stop(): Promise<void[]> {
return Promise.all(
Array.from(this.plugs.values()).map((plug) => plug.stop())
);
}
}
console.log("Starting");

57
plugbox/src/sandbox_worker.ts
View File

@ -2,21 +2,38 @@ import { ControllerMessage, WorkerMessage, WorkerMessageType } from "./types";
import { safeRun } from "./util";
let loadedFunctions = new Map<string, Function>();
let pendingRequests = new Map<number, (result: unknown) => void>();
let pendingRequests = new Map<
number,
{
resolve: (result: unknown) => void;
reject: (e: any) => void;
}
>();
declare global {
function syscall(id: number, name: string, args: any[]): Promise<any>;
}
let postMessage = self.postMessage.bind(self);
if (window.parent !== window) {
console.log("running in an iframe");
postMessage = window.parent.postMessage.bind(window.parent);
// postMessage({ type: "test" }, "*");
}
self.syscall = async (id: number, name: string, args: any[]) => {
return await new Promise((resolve, reject) => {
pendingRequests.set(id, resolve);
self.postMessage({
pendingRequests.set(id, { resolve, reject });
postMessage(
{
type: "syscall",
id,
name,
args,
});
},
"*"
);
});
};
@ -26,6 +43,7 @@ return fn["default"].apply(null, arguments);`;
}
self.addEventListener("message", (event: { data: WorkerMessage }) => {
// console.log("Got a message", event.data);
safeRun(async () => {
let messageEvent = event;
let data = messageEvent.data;
@ -33,10 +51,13 @@ self.addEventListener("message", (event: { data: WorkerMessage }) => {
case "load":
console.log("Booting", data.name);
loadedFunctions.set(data.name!, new Function(wrapScript(data.code!)));
self.postMessage({
postMessage(
{
type: "inited",
name: data.name,
} as ControllerMessage);
} as ControllerMessage,
"*"
);
break;
case "invoke":
let fn = loadedFunctions.get(data.name!);
@ -45,17 +66,23 @@ self.addEventListener("message", (event: { data: WorkerMessage }) => {
}
try {
let result = await Promise.resolve(fn(...(data.args || [])));
self.postMessage({
postMessage(
{
type: "result",
id: data.id,
result: result,
} as ControllerMessage);
} as ControllerMessage,
"*"
);
} catch (e: any) {
self.postMessage({
type: "error",
postMessage(
{
type: "result",
id: data.id,
reason: e.message,
} as ControllerMessage);
error: e.message,
} as ControllerMessage,
"*"
);
throw e;
}
@ -73,7 +100,11 @@ self.addEventListener("message", (event: { data: WorkerMessage }) => {
throw Error("Invalid request id");
}
pendingRequests.delete(syscallId);
lookup(data.data);
if (data.error) {
lookup.reject(new Error(data.error));
} else {
lookup.resolve(data.result);
}
break;
}
});

15
plugbox/src/types.ts
View File

@ -10,18 +10,19 @@ export type WorkerMessage = {
name?: string;
code?: string;
args?: any[];
data?: any;
result?: any;
error?: any;
};
export type ControllerMessageType = "inited" | "result" | "error" | "syscall";
export type ControllerMessageType = "inited" | "result" | "syscall";
export type ControllerMessage = {
type: ControllerMessageType;
id?: number;
name?: string;
reason?: string;
args?: any[];
result: any;
error?: string;
result?: any;
};
export interface Manifest<HookT> {
@ -35,3 +36,9 @@ export interface FunctionDef {
path?: string;
code?: string;
}
export interface WorkerLike {
onMessage?: (message: any) => Promise<void>;
postMessage(message: any): void;
terminate(): void;
}

34
plugbox/src/webworker_sandbox.ts Normal file
View File

@ -0,0 +1,34 @@
import { ControllerMessage, WorkerLike, WorkerMessage } from "./types";
import { Sandbox, System } from "./runtime";
import { safeRun } from "./util";
class WebWorkerWrapper implements WorkerLike {
private worker: Worker;
onMessage?: (message: any) => Promise<void>;
constructor(worker: Worker) {
this.worker = worker;
this.worker.addEventListener("message", (evt: any) => {
let data = evt.data;
if (!data) return;
safeRun(async () => {
await this.onMessage!(data);
});
});
}
postMessage(message: any): void {
this.worker.postMessage(message);
}
terminate() {
return this.worker.terminate();
}
}
export function createSandbox(system: System<any>) {
// ParcelJS will build this file into a worker.
let worker = new Worker(new URL("sandbox_worker.ts", import.meta.url), {
type: "module",
});
return new Sandbox(system, new WebWorkerWrapper(worker));
}

88
plugbox/src/worker_sandbox.ts
View File

@ -1,88 +0,0 @@
import { ControllerMessage, WorkerMessage } from "./types";
import { Sandbox, System } from "./runtime";
export class WebworkerSandbox implements Sandbox {
private worker: Worker;
private reqId = 0;
private outstandingInits = new Map<string, () => void>();
private outstandingInvocations = new Map<
number,
{ resolve: (result: any) => void; reject: (e: any) => void }
>();
private loadedFunctions = new Set<string>();
constructor(readonly system: System<any>) {
this.worker = new Worker(new URL("sandbox_worker.ts", import.meta.url), {
type: "module",
});
this.worker.onmessage = this.onmessage.bind(this);
}
isLoaded(name: string) {
return this.loadedFunctions.has(name);
}
async load(name: string, code: string): Promise<void> {
this.worker.postMessage({
type: "load",
name: name,
code: code,
} as WorkerMessage);
return new Promise((resolve) => {
this.loadedFunctions.add(name);
this.outstandingInits.set(name, resolve);
});
}
async onmessage(evt: { data: ControllerMessage }) {
let data = evt.data;
if (!data) return;
switch (data.type) {
case "inited":
let initCb = this.outstandingInits.get(data.name!);
initCb && initCb();
this.outstandingInits.delete(data.name!);
break;
case "syscall":
let result = await this.system.syscall(data.name!, data.args!);
this.worker.postMessage({
type: "syscall-response",
id: data.id,
data: result,
} as WorkerMessage);
break;
case "result":
let resultCb = this.outstandingInvocations.get(data.id!);
this.outstandingInvocations.delete(data.id!);
resultCb && resultCb.resolve(data.result);
break;
case "error":
let errCb = this.outstandingInvocations.get(data.result.id!);
this.outstandingInvocations.delete(data.id!);
errCb && errCb.reject(data.reason);
break;
default:
console.error("Unknown message type", data);
}
}
async invoke(name: string, args: any[]): Promise<any> {
this.reqId++;
this.worker.postMessage({
type: "invoke",
id: this.reqId,
name,
args,
});
return new Promise((resolve, reject) => {
this.outstandingInvocations.set(this.reqId, { resolve, reject });
});
}
stop() {
this.worker.terminate();
}
}

3
plugs/core/core.plug.json
View File

@ -75,6 +75,9 @@
},
"toggle_h2": {
"path": "./markup.ts:toggleH2"
},
"server_test": {
"path": "./server.ts:test"
}
}
}

5
plugs/core/server.ts Normal file
View File

@ -0,0 +1,5 @@
import { syscall } from "./lib/syscall";
export function test() {
console.log("I'm running on the server!");
return 5;
}

3
server/package.json
View File

@ -24,9 +24,12 @@
"socket.io": "^4.4.1",
"socket.io-client": "^4.4.1",
"typescript": "^4.6.2",
"vm2": "^3.9.9",
"yargs": "^17.3.1"
},
"devDependencies": {
"@parcel/optimizer-data-url": "2.3.2",
"@parcel/transformer-inline-string": "2.3.2",
"@types/cors": "^2.8.12",
"@types/express": "^4.17.13",
"jest": "^27.5.1",

2
server/src/api.test.ts
View File

@ -3,7 +3,7 @@ import { test, expect, beforeAll, afterAll, describe } from "@jest/globals";
import { createServer } from "http";
import { io as Client } from "socket.io-client";
import { Server } from "socket.io";
import { SocketServer } from "./api";
import { SocketServer } from "./api_server";
import * as path from "path";
import * as fs from "fs";

45
server/src/api.ts → server/src/api_server.ts
View File

@ -3,6 +3,11 @@ import { Page } from "./types";
import * as path from "path";
import { IndexApi } from "./index_api";
import { PageApi } from "./page_api";
import { System } from "../../plugbox/src/runtime";
import { createSandbox } from "../../plugbox/src/node_sandbox";
import { NuggetHook } from "../../webapp/src/types";
import corePlug from "../../webapp/src/generated/core.plug.json";
import pageIndexSyscalls from "./syscalls/page_index";
export class ClientConnection {
openPages = new Set<string>();
@ -15,27 +20,42 @@ export interface ApiProvider {
}
export class SocketServer {
rootPath: string;
openPages = new Map<string, Page>();
connectedSockets = new Set<Socket>();
serverSocket: Server;
private openPages = new Map<string, Page>();
private connectedSockets = new Set<Socket>();
private apis = new Map<string, ApiProvider>();
readonly rootPath: string;
private serverSocket: Server;
system: System<NuggetHook>;
constructor(rootPath: string, serverSocket: Server) {
this.rootPath = path.resolve(rootPath);
this.serverSocket = serverSocket;
this.system = new System<NuggetHook>();
}
async registerApi(name: string, apiProvider: ApiProvider) {
await apiProvider.init();
this.apis.set(name, apiProvider);
}
constructor(rootPath: string, serverSocket: Server) {
this.rootPath = path.resolve(rootPath);
this.serverSocket = serverSocket;
}
public async init() {
await this.registerApi("index", new IndexApi(this.rootPath));
const indexApi = new IndexApi(this.rootPath);
await this.registerApi("index", indexApi);
this.system.registerSyscalls(pageIndexSyscalls(indexApi.db));
await this.registerApi(
"page",
new PageApi(this.rootPath, this.connectedSockets)
new PageApi(
this.rootPath,
this.connectedSockets,
this.openPages,
this.system
)
);
let plug = await this.system.load(
"core",
corePlug,
createSandbox(this.system)
);
this.serverSocket.on("connection", (socket) => {
@ -51,9 +71,8 @@ export class SocketServer {
});
socket.on("closePage", (pageName: string) => {
console.log("Closing page", pageName);
clientConn.openPages.delete(pageName);
disconnectPageSocket(pageName);
clientConn.openPages.delete(pageName);
});
const onCall = (

57
server/src/index_api.ts
View File

@ -1,6 +1,7 @@
import { ApiProvider, ClientConnection } from "./api";
import { ApiProvider, ClientConnection } from "./api_server";
import knex, { Knex } from "knex";
import path from "path";
import pageIndexSyscalls from "./syscalls/page_index";
type IndexItem = {
page: string;
@ -10,6 +11,7 @@ type IndexItem = {
export class IndexApi implements ApiProvider {
db: Knex;
constructor(rootPath: string) {
this.db = knex({
client: "better-sqlite3",
@ -33,12 +35,13 @@ export class IndexApi implements ApiProvider {
}
api() {
const syscalls = pageIndexSyscalls(this.db);
return {
clearPageIndexForPage: async (
clientConn: ClientConnection,
page: string
) => {
await this.db<IndexItem>("page_index").where({ page }).del();
return syscalls["indexer.clearPageIndexForPage"](page);
},
set: async (
clientConn: ClientConnection,
@ -46,77 +49,41 @@ export class IndexApi implements ApiProvider {
key: string,
value: any
) => {
let changed = await this.db<IndexItem>("page_index")
.where({ page, key })
.update("value", JSON.stringify(value));
if (changed === 0) {
await this.db<IndexItem>("page_index").insert({
page,
key,
value: JSON.stringify(value),
});
}
return syscalls["indexer.set"](page, key, value);
},
get: async (clientConn: ClientConnection, page: string, key: string) => {
let result = await this.db<IndexItem>("page_index")
.where({ page, key })
.select("value");
if (result.length) {
return JSON.parse(result[0].value);
} else {
return null;
}
return syscalls["indexer.get"](page, key);
},
delete: async (
clientConn: ClientConnection,
page: string,
key: string
) => {
await this.db<IndexItem>("page_index").where({ page, key }).del();
return syscalls["indexer.delete"](page, key);
},
scanPrefixForPage: async (
clientConn: ClientConnection,
page: string,
prefix: string
) => {
return (
await this.db<IndexItem>("page_index")
.where({ page })
.andWhereLike("key", `${prefix}%`)
.select("page", "key", "value")
).map(({ page, key, value }) => ({
page,
key,
value: JSON.parse(value),
}));
return syscalls["indexer.scanPrefixForPage"](page, prefix);
},
scanPrefixGlobal: async (
clientConn: ClientConnection,
prefix: string
) => {
return (
await this.db<IndexItem>("page_index")
.andWhereLike("key", `${prefix}%`)
.select("page", "key", "value")
).map(({ page, key, value }) => ({
page,
key,
value: JSON.parse(value),
}));
return syscalls["indexer.scanPrefixGlobal"](prefix);
},
deletePrefixForPage: async (
clientConn: ClientConnection,
page: string,
prefix: string
) => {
return this.db<IndexItem>("page_index")
.where({ page })
.andWhereLike("key", `${prefix}%`)
.del();
return syscalls["indexer.deletePrefixForPage"](page, prefix);
},
clearPageIndex: async (clientConn: ClientConnection) => {
return this.db<IndexItem>("page_index").del();
return syscalls["indexer.clearPageIndex"]();
},
};
}

28
server/src/page_api.ts
View File

@ -1,7 +1,7 @@
import { ClientPageState, Page, PageMeta } from "./types";
import { ChangeSet } from "@codemirror/state";
import { Update } from "@codemirror/collab";
import { ApiProvider, ClientConnection } from "./api";
import { ApiProvider, ClientConnection } from "./api_server";
import { Socket } from "socket.io";
import { DiskStorage } from "./disk_storage";
import { safeRun } from "./util";
@ -9,17 +9,27 @@ import fs from "fs";
import path from "path";
import { stat } from "fs/promises";
import { Cursor, cursorEffect } from "../../webapp/src/cursorEffect";
import { System } from "../../plugbox/src/runtime";
import { NuggetHook } from "../../webapp/src/types";
export class PageApi implements ApiProvider {
openPages = new Map<string, Page>();
openPages: Map<string, Page>;
pageStore: DiskStorage;
rootPath: string;
connectedSockets: Set<Socket>;
private system: System<NuggetHook>;
constructor(rootPath: string, connectedSockets: Set<Socket>) {
constructor(
rootPath: string,
connectedSockets: Set<Socket>,
openPages: Map<string, Page>,
system: System<NuggetHook>
) {
this.pageStore = new DiskStorage(rootPath);
this.rootPath = rootPath;
this.openPages = openPages;
this.connectedSockets = connectedSockets;
this.system = system;
}
async init(): Promise<void> {
@ -45,6 +55,7 @@ export class PageApi implements ApiProvider {
}
disconnectClient(client: ClientPageState, page: Page) {
console.log("Disconnecting client");
page.clientStates.delete(client);
if (page.clientStates.size === 0) {
console.log("No more clients for", page.name, "flushing");
@ -178,6 +189,12 @@ export class PageApi implements ApiProvider {
textChanged = true;
}
}
console.log(
"New version",
page.version,
"Updates buffered:",
page.updates.length
);
if (textChanged) {
if (page.saveTimer) {
@ -185,6 +202,11 @@ export class PageApi implements ApiProvider {
}
page.saveTimer = setTimeout(() => {
console.log("This is the time to index a page");
this.system.dispatchEvent("page:index", {
name: pageName,
text: page!.text.sliceString(0),
});
this.flushPageToDisk(pageName, page!);
}, 1000);
}

2
server/src/server.ts
View File

@ -2,7 +2,7 @@ import express from "express";
import { readFile } from "fs/promises";
import http from "http";
import { Server } from "socket.io";
import { SocketServer } from "./api";
import { SocketServer } from "./api_server";
import yargs from "yargs";
import { hideBin } from "yargs/helpers";

82
server/src/syscalls/page_index.ts Normal file
View File

@ -0,0 +1,82 @@
import { Knex } from "knex";
type IndexItem = {
page: string;
key: string;
value: any;
};
export type KV = {
key: string;
value: any;
};
export default function (db: Knex) {
const setter = async (page: string, key: string, value: any) => {
let changed = await db<IndexItem>("page_index")
.where({ page, key })
.update("value", JSON.stringify(value));
if (changed === 0) {
await db<IndexItem>("page_index").insert({
page,
key,
value: JSON.stringify(value),
});
}
};
return {
"indexer.clearPageIndexForPage": async (page: string) => {
await db<IndexItem>("page_index").where({ page }).del();
},
"indexer.set": setter,
"indexer.batchSet": async (page: string, kvs: KV[]) => {
for (let { key, value } of kvs) {
await setter(page, key, value);
}
},
"indexer.get": async (page: string, key: string) => {
let result = await db<IndexItem>("page_index")
.where({ page, key })
.select("value");
if (result.length) {
return JSON.parse(result[0].value);
} else {
return null;
}
},
"indexer.delete": async (page: string, key: string) => {
await db<IndexItem>("page_index").where({ page, key }).del();
},
"indexer.scanPrefixForPage": async (page: string, prefix: string) => {
return (
await db<IndexItem>("page_index")
.where({ page })
.andWhereLike("key", `${prefix}%`)
.select("page", "key", "value")
).map(({ page, key, value }) => ({
page,
key,
value: JSON.parse(value),
}));
},
"indexer.scanPrefixGlobal": async (prefix: string) => {
return (
await db<IndexItem>("page_index")
.andWhereLike("key", `${prefix}%`)
.select("page", "key", "value")
).map(({ page, key, value }) => ({
page,
key,
value: JSON.parse(value),
}));
},
"indexer.deletePrefixForPage": async (page: string, prefix: string) => {
return db<IndexItem>("page_index")
.where({ page })
.andWhereLike("key", `${prefix}%`)
.del();
},
"indexer.clearPageIndex": async () => {
return db<IndexItem>("page_index").del();
},
};
}

42
server/yarn.lock
View File

@ -706,6 +706,16 @@
cssnano "^5.0.15"
postcss "^8.4.5"
"@parcel/optimizer-data-url@2.3.2":
version "2.3.2"
resolved "https://registry.yarnpkg.com/@parcel/optimizer-data-url/-/optimizer-data-url-2.3.2.tgz#22c2951eb6bda7d7b589c28283d99f9d21dae568"
integrity sha512-q3Y1J3acGPf8yyAhEG+59qey7liB04T/U4i7nmvggDdDVG9S8aYgIeAjsPUKi/9fBoHLn4l8K/sJq3M7FFdcnw==
dependencies:
"@parcel/plugin" "2.3.2"
"@parcel/utils" "2.3.2"
isbinaryfile "^4.0.2"
mime "^2.4.4"
"@parcel/optimizer-htmlnano@2.3.2":
version "2.3.2"
resolved "https://registry.yarnpkg.com/@parcel/optimizer-htmlnano/-/optimizer-htmlnano-2.3.2.tgz#4086736866621182f5dd1a8abe78e9f5764e1a28"
@ -940,6 +950,13 @@
"@parcel/workers" "2.3.2"
nullthrows "^1.1.1"
"@parcel/transformer-inline-string@2.3.2":
version "2.3.2"
resolved "https://registry.yarnpkg.com/@parcel/transformer-inline-string/-/transformer-inline-string-2.3.2.tgz#bec5d376d00b5c41abf11c8cf8e3d917036c0646"
integrity sha512-nitgU+YHnJpJjdUEyRqXD3DjIAstcdHDwwKgloTfpt7EDe2VspVuWhA054kH75Kn/Tvn4s0G4VGCTpDw5KxzSw==
dependencies:
"@parcel/plugin" "2.3.2"
"@parcel/transformer-js@2.3.2":
version "2.3.2"
resolved "https://registry.yarnpkg.com/@parcel/transformer-js/-/transformer-js-2.3.2.tgz#24bcb488d5f82678343a5630fe4bbe822789ac33"
@ -1324,12 +1341,17 @@ acorn-walk@^7.1.1:
resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.2.0.tgz#0de889a601203909b0fbe07b8938dc21d2e967bc"
integrity sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA==
acorn-walk@^8.2.0:
version "8.2.0"
resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.2.0.tgz#741210f2e2426454508853a2f44d0ab83b7f69c1"
integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==
acorn@^7.1.1:
version "7.4.1"
resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa"
integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==
acorn@^8.2.4, acorn@^8.5.0:
acorn@^8.2.4, acorn@^8.5.0, acorn@^8.7.0:
version "8.7.0"
resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.7.0.tgz#90951fde0f8f09df93549481e5fc141445b791cf"
integrity sha512-V/LGr1APy+PXIwKebEWrkZPwoeoF+w1jiOBUmuxuiUIaOHtob8Qc9BTrYo7VuI5fR8tqsy+buA2WFooR5olqvQ==
@ -2958,6 +2980,11 @@ isarray@~1.0.0:
resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11"
integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=
isbinaryfile@^4.0.2:
version "4.0.8"
resolved "https://registry.yarnpkg.com/isbinaryfile/-/isbinaryfile-4.0.8.tgz#5d34b94865bd4946633ecc78a026fc76c5b11fcf"
integrity sha512-53h6XFniq77YdW+spoRrebh0mnmTxRPTlcuIArO57lmMdq4uBKFKaeTjnb92oYWrSn/LVL+LT+Hap2tFQj8V+w==
isexe@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10"
@ -3675,6 +3702,11 @@ mime@1.6.0:
resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1"
integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==
mime@^2.4.4:
version "2.6.0"
resolved "https://registry.yarnpkg.com/mime/-/mime-2.6.0.tgz#a2a682a95cd4d0cb1d6257e28f83da7e35800367"
integrity sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg==
mimic-fn@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b"
@ -5163,6 +5195,14 @@ vary@^1, vary@~1.1.2:
resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc"
integrity sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=
vm2@^3.9.9:
version "3.9.9"
resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.9.9.tgz#c0507bc5fbb99388fad837d228badaaeb499ddc5"
integrity sha512-xwTm7NLh/uOjARRBs8/95H0e8fT3Ukw5D/JJWhxMbhKzNh1Nu981jQKvkep9iKYNxzlVrdzD0mlBGkDKZWprlw==
dependencies:
acorn "^8.7.0"
acorn-walk "^8.2.0"
w3c-hr-time@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz#0a89cdf5cc15822df9c360543676963e0cc308cd"

3
tsconfig.json
View File

@ -5,6 +5,7 @@
"moduleResolution": "node",
"module": "esnext",
"esModuleInterop": true,
"allowSyntheticDefaultImports": true
"allowSyntheticDefaultImports": true,
"resolveJsonModule": true
}
}

4
webapp/package.json
View File

@ -7,12 +7,14 @@
"license": "MIT",
"browserslist": "> 0.5%, last 2 versions, not dead",
"scripts": {
"start": "parcel serve --no-cache",
"start": "rm -rf .parcel-cache && parcel serve --no-cache",
"start-no-hmr": "rm -rf .parcel-cache && parcel serve --no-cache --no-hmr",
"build": "parcel build",
"clean": "rm -rf dist"
},
"devDependencies": {
"@parcel/packager-raw-url": "2.3.2",
"@parcel/transformer-inline-string": "2.3.2",
"@parcel/transformer-sass": "2.3.2",
"@parcel/transformer-webmanifest": "2.3.2",
"@parcel/validator-typescript": "^2.3.2",

5
webapp/src/editor.tsx
View File

@ -29,7 +29,8 @@ import {
import React, { useEffect, useReducer } from "react";
import ReactDOM from "react-dom";
import { Plug, System } from "../../plugbox/src/runtime";
import { WebworkerSandbox } from "../../plugbox/src/worker_sandbox";
import { createSandbox } from "../../plugbox/src/webworker_sandbox";
import { createSandbox as createIFrameSandbox } from "../../plugbox/src/iframe_sandbox";
import { AppEvent, AppEventDispatcher, ClickEvent } from "./app_event";
import { collabExtension, CollabDocument } from "./collab";
import * as commands from "./commands";
@ -182,7 +183,7 @@ export class Editor implements AppEventDispatcher {
let mainPlug = await system.load(
"core",
coreManifest,
new WebworkerSandbox(system)
createIFrameSandbox(system)
);
this.plugs.push(mainPlug);
this.editorCommands = new Map<string, AppCommand>();

3
webapp/src/indexer.ts
View File

@ -21,7 +21,8 @@ export class Indexer {
name: pageName,
text,
};
await appEventDispatcher.dispatchAppEvent("page:index", indexEvent);
// await appEventDispatcher.dispatchAppEvent("page:index", indexEvent);
// await this.setPageIndexPageMeta(pageMeta.name, pageMeta);
}

7
webapp/yarn.lock
View File

@ -790,6 +790,13 @@
"@parcel/workers" "2.3.2"
nullthrows "^1.1.1"
"@parcel/transformer-inline-string@2.3.2":
version "2.3.2"
resolved "https://registry.yarnpkg.com/@parcel/transformer-inline-string/-/transformer-inline-string-2.3.2.tgz#bec5d376d00b5c41abf11c8cf8e3d917036c0646"
integrity sha512-nitgU+YHnJpJjdUEyRqXD3DjIAstcdHDwwKgloTfpt7EDe2VspVuWhA054kH75Kn/Tvn4s0G4VGCTpDw5KxzSw==
dependencies:
"@parcel/plugin" "2.3.2"
"@parcel/transformer-js@2.3.2":
version "2.3.2"
resolved "https://registry.yarnpkg.com/@parcel/transformer-js/-/transformer-js-2.3.2.tgz#24bcb488d5f82678343a5630fe4bbe822789ac33"