From a9eb252658d15438f0ea03f41949058464cb0947 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Sun, 14 Jan 2024 13:38:39 +0100 Subject: [PATCH] Plug sandbox rework --- plugos/compile.ts | 13 ++-- plugos/environments/webworker_sandbox.ts | 6 -- plugos/hooks/endpoint.test.ts | 2 +- plugos/manifest_cache.ts | 5 +- plugos/plug.ts | 6 +- plugos/runtime.test.ts | 22 +++++- .../deno_worker_sandbox.ts} | 5 +- plugos/sandboxes/no_sandbox.ts | 68 +++++++++++++++++++ plugos/sandboxes/sandbox.ts | 11 +++ plugos/sandboxes/web_worker_sandbox.ts | 7 ++ .../worker_sandbox.ts} | 15 ++-- plugos/system.ts | 41 ++++++++++- plugos/test_func.test.ts | 10 +-- plugos/worker_runtime.ts | 31 ++++++--- plugs/markdown/markdown_render.test.ts | 2 +- server/server_system.ts | 2 +- web/client_system.ts | 2 +- 17 files changed, 199 insertions(+), 49 deletions(-) delete mode 100644 plugos/environments/webworker_sandbox.ts rename plugos/{environments/deno_sandbox.ts => sandboxes/deno_worker_sandbox.ts} (84%) create mode 100644 plugos/sandboxes/no_sandbox.ts create mode 100644 plugos/sandboxes/sandbox.ts create mode 100644 plugos/sandboxes/web_worker_sandbox.ts rename plugos/{sandbox.ts => sandboxes/worker_sandbox.ts} (89%) diff --git a/plugos/compile.ts b/plugos/compile.ts index b1f2ba8..0155cd7 100644 --- a/plugos/compile.ts +++ b/plugos/compile.ts @@ -4,9 +4,9 @@ import { bundleAssets } from "./asset_bundle/builder.ts"; import { Manifest } from "./types.ts"; import { version } from "../version.ts"; -// const workerRuntimeUrl = new URL("./worker_runtime.ts", import.meta.url); -const workerRuntimeUrl = - `https://deno.land/x/silverbullet@${version}/plugos/worker_runtime.ts`; +const workerRuntimeUrl = new URL("./worker_runtime.ts", import.meta.url); +// const workerRuntimeUrl = +// `https://deno.land/x/silverbullet@${version}/plugos/worker_runtime.ts`; export type CompileOptions = { debug?: boolean; @@ -64,7 +64,7 @@ ${ } // Function mapping -export const functionMapping = { +const functionMapping = { ${ Object.entries(manifest.functions).map(([funcName, def]) => { if (!def.path) { @@ -75,8 +75,11 @@ ${ } }; +// Manifest const manifest = ${JSON.stringify(manifest, null, 2)}; +export const plug = {manifest, functionMapping}; + setupMessageListener(functionMapping, manifest); `; @@ -89,7 +92,7 @@ setupMessageListener(functionMapping, manifest); const result = await esbuild.build({ entryPoints: [path.basename(inFile)], bundle: true, - format: "iife", + format: "esm", globalName: "mod", platform: "browser", sourcemap: options.debug ? "linked" : false, diff --git a/plugos/environments/webworker_sandbox.ts b/plugos/environments/webworker_sandbox.ts deleted file mode 100644 index 0171720..0000000 --- a/plugos/environments/webworker_sandbox.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { Sandbox } from "../sandbox.ts"; -import type { Plug } from "../plug.ts"; - -export function createSandbox(plug: Plug): Sandbox { - return new Sandbox(plug); -} diff --git a/plugos/hooks/endpoint.test.ts b/plugos/hooks/endpoint.test.ts index b69b249..a93df6a 100644 --- a/plugos/hooks/endpoint.test.ts +++ b/plugos/hooks/endpoint.test.ts @@ -1,4 +1,4 @@ -import { createSandbox } from "../environments/deno_sandbox.ts"; +import { createSandbox } from "../sandboxes/deno_worker_sandbox.ts"; import { EndpointHook, EndpointHookT } from "./endpoint.ts"; import { System } from "../system.ts"; diff --git a/plugos/manifest_cache.ts b/plugos/manifest_cache.ts index fa810b9..27e716d 100644 --- a/plugos/manifest_cache.ts +++ b/plugos/manifest_cache.ts @@ -37,15 +37,16 @@ export class InMemoryManifestCache implements ManifestCache { }>(); async getManifest(plug: Plug, hash: number): Promise> { - const cached = this.cache.get(plug.workerUrl.href); + const cached = this.cache.get(plug.name); if (cached && cached.hash === hash) { // console.log("Using memory cached manifest for", plug.name); return cached.manifest; } await plug.sandbox.init(); const manifest = plug.sandbox.manifest!; + // Deliverately removing the assets from the manifest to preserve space, will be re-added upon load of actual worker - this.cache.set(plug.name!, { + this.cache.set(plug.name, { manifest: { ...manifest, assets: undefined }, hash, }); diff --git a/plugos/plug.ts b/plugos/plug.ts index f7e71ea..2a272c1 100644 --- a/plugos/plug.ts +++ b/plugos/plug.ts @@ -1,7 +1,7 @@ import { Manifest } from "./types.ts"; -import { Sandbox } from "./sandbox.ts"; import { System } from "./system.ts"; import { AssetBundle } from "./asset_bundle/bundle.ts"; +import { Sandbox, SandboxFactory } from "./sandboxes/sandbox.ts"; export class Plug { readonly runtimeEnv?: string; @@ -21,10 +21,10 @@ export class Plug { constructor( private system: System, - public workerUrl: URL, + public workerUrl: URL | undefined, readonly name: string, private hash: number, - private sandboxFactory: (plug: Plug) => Sandbox, + private sandboxFactory: SandboxFactory, ) { this.runtimeEnv = system.env; diff --git a/plugos/runtime.test.ts b/plugos/runtime.test.ts index d1a2c7f..6561c76 100644 --- a/plugos/runtime.test.ts +++ b/plugos/runtime.test.ts @@ -1,4 +1,4 @@ -import { createSandbox } from "./environments/deno_sandbox.ts"; +import { createSandbox } from "./sandboxes/deno_worker_sandbox.ts"; import { System } from "./system.ts"; import { assertEquals } from "../test_deps.ts"; import { compileManifest } from "./compile.ts"; @@ -8,6 +8,7 @@ Deno.test("Run a deno sandbox", async () => { const system = new System("server"); system.registerSyscalls([], { addNumbers: (_ctx, a, b) => { + console.log("This is the context", _ctx.plug.name); return a + b; }, failingSyscall: () => { @@ -39,9 +40,24 @@ Deno.test("Run a deno sandbox", async () => { createSandbox, ); - console.log("Plug", plug.manifest); + assertEquals({ + addedNumbers: 3, + yamlMessage: "hello: world\n", + }, await plug.invoke("boot", [])); - assertEquals("hello", await plug.invoke("boot", [])); + await system.unloadAll(); + + // Now load directly from module + const { plug: plugExport } = await import( + `file://${workerPath}` + ); + + const plug2 = await system.loadNoSandbox("test", plugExport); + + assertEquals({ + addedNumbers: 3, + yamlMessage: "hello: world\n", + }, await plug2.invoke("boot", [])); await system.unloadAll(); diff --git a/plugos/environments/deno_sandbox.ts b/plugos/sandboxes/deno_worker_sandbox.ts similarity index 84% rename from plugos/environments/deno_sandbox.ts rename to plugos/sandboxes/deno_worker_sandbox.ts index 25d2925..b30b122 100644 --- a/plugos/environments/deno_sandbox.ts +++ b/plugos/sandboxes/deno_worker_sandbox.ts @@ -1,9 +1,10 @@ -import { Sandbox } from "../sandbox.ts"; +import { WorkerSandbox } from "./worker_sandbox.ts"; import { Plug } from "../plug.ts"; +import { Sandbox } from "./sandbox.ts"; // Uses Deno's permissions to lock the worker down significantly export function createSandbox(plug: Plug): Sandbox { - return new Sandbox(plug, { + return new WorkerSandbox(plug, { deno: { permissions: { // Allow network access diff --git a/plugos/sandboxes/no_sandbox.ts b/plugos/sandboxes/no_sandbox.ts new file mode 100644 index 0000000..b2e5f5a --- /dev/null +++ b/plugos/sandboxes/no_sandbox.ts @@ -0,0 +1,68 @@ +import { PromiseQueue } from "$sb/lib/async.ts"; +import { Plug } from "../plug.ts"; +import { Sandbox } from "./sandbox.ts"; +import { Manifest } from "../types.ts"; + +// We need to hard inject the syscall function into the global scope +declare global { + interface globalThis { + syscall(name: string, ...args: any[]): Promise; + } +} + +export type PlugExport = { + manifest: Manifest; + functionMapping: Record; +}; + +const functionQueue = new PromiseQueue(); + +let activePlug: Plug | undefined; + +// @ts-ignore: globalThis +globalThis.syscall = (name: string, ...args: any[]): Promise => { + if (!activePlug) { + throw new Error("No active plug"); + } + console.log("Calling syscall", name, args); + return activePlug.syscall(name, args); +}; + +export class NoSandbox implements Sandbox { + manifest?: Manifest | undefined; + constructor( + private plug: Plug, + private plugExport: PlugExport, + ) { + this.manifest = plugExport.manifest; + plug.manifest = this.manifest; + } + + init(): Promise { + return Promise.resolve(); + } + + invoke(name: string, args: any[]): Promise { + activePlug = this.plug; + return functionQueue.runInQueue(async () => { + try { + const fn = this.plugExport.functionMapping[name]; + if (!fn) { + throw new Error(`Function not loaded: ${name}`); + } + return await fn(...args); + } finally { + activePlug = undefined; + } + }); + } + + stop() { + } +} + +export function noSandboxFactory( + plugExport: PlugExport, +): (plug: Plug) => Sandbox { + return (plug: Plug) => new NoSandbox(plug, plugExport); +} diff --git a/plugos/sandboxes/sandbox.ts b/plugos/sandboxes/sandbox.ts new file mode 100644 index 0000000..9ae92f4 --- /dev/null +++ b/plugos/sandboxes/sandbox.ts @@ -0,0 +1,11 @@ +import { Plug } from "../plug.ts"; +import { Manifest } from "../types.ts"; + +export type SandboxFactory = (plug: Plug) => Sandbox; + +export interface Sandbox { + manifest?: Manifest; + init(): Promise; + invoke(name: string, args: any[]): Promise; + stop(): void; +} diff --git a/plugos/sandboxes/web_worker_sandbox.ts b/plugos/sandboxes/web_worker_sandbox.ts new file mode 100644 index 0000000..8c5a8e7 --- /dev/null +++ b/plugos/sandboxes/web_worker_sandbox.ts @@ -0,0 +1,7 @@ +import { WorkerSandbox } from "./worker_sandbox.ts"; +import type { Plug } from "../plug.ts"; +import { Sandbox } from "./sandbox.ts"; + +export function createSandbox(plug: Plug): Sandbox { + return new WorkerSandbox(plug); +} diff --git a/plugos/sandbox.ts b/plugos/sandboxes/worker_sandbox.ts similarity index 89% rename from plugos/sandbox.ts rename to plugos/sandboxes/worker_sandbox.ts index c5c41ce..40da509 100644 --- a/plugos/sandbox.ts +++ b/plugos/sandboxes/worker_sandbox.ts @@ -1,15 +1,14 @@ -import { Manifest } from "./types.ts"; -import { ControllerMessage, WorkerMessage } from "./protocol.ts"; -import { Plug } from "./plug.ts"; -import { AssetBundle, AssetJson } from "./asset_bundle/bundle.ts"; - -export type SandboxFactory = (plug: Plug) => Sandbox; +import { Manifest } from "../types.ts"; +import { ControllerMessage, WorkerMessage } from "../protocol.ts"; +import { Plug } from "../plug.ts"; +import { AssetBundle, AssetJson } from "../asset_bundle/bundle.ts"; +import { Sandbox } from "./sandbox.ts"; /** * Represents a "safe" execution environment for plug code * Effectively this wraps a web worker, the reason to have this split from Plugs is to allow plugs to manage multiple sandboxes, e.g. for performance in the future */ -export class Sandbox { +export class WorkerSandbox implements Sandbox { private worker?: Worker; private reqId = 0; private outstandingInvocations = new Map< @@ -36,7 +35,7 @@ export class Sandbox { console.warn("Double init of sandbox, ignoring"); return Promise.resolve(); } - this.worker = new Worker(this.plug.workerUrl, { + this.worker = new Worker(this.plug.workerUrl!, { ...this.workerOptions, type: "module", }); diff --git a/plugos/system.ts b/plugos/system.ts index 46fcaf3..c91c506 100644 --- a/plugos/system.ts +++ b/plugos/system.ts @@ -1,8 +1,9 @@ import { Hook } from "./types.ts"; import { EventEmitter } from "./event.ts"; -import type { SandboxFactory } from "./sandbox.ts"; +import type { SandboxFactory } from "./sandboxes/sandbox.ts"; import { Plug } from "./plug.ts"; import { InMemoryManifestCache, ManifestCache } from "./manifest_cache.ts"; +import { noSandboxFactory, PlugExport } from "./sandboxes/no_sandbox.ts"; export interface SysCallMapping { [key: string]: (ctx: SyscallContext, ...args: any) => Promise | any; @@ -138,6 +139,44 @@ export class System extends EventEmitter> { return plug; } + /** + * Loads a plug without a sandbox, which means it will run in the same context as the caller + * @param name + * @param plugExport extracted via e.g. `import { plug } from "./some.plug.js` + * @returns Plug instance + */ + async loadNoSandbox( + name: string, + plugExport: PlugExport, + ): Promise> { + const plug = new Plug( + this, + undefined, + name, + -1, + noSandboxFactory(plugExport), + ); + + const manifest = plugExport.manifest; + + // Validate the manifest + let errors: string[] = []; + for (const feature of this.enabledHooks) { + errors = [...errors, ...feature.validateManifest(plug.manifest!)]; + } + if (errors.length > 0) { + throw new Error(`Invalid manifest: ${errors.join(", ")}`); + } + if (this.plugs.has(manifest.name)) { + this.unload(manifest.name); + } + console.log("Activated plug without sandbox", manifest.name); + this.plugs.set(manifest.name, plug); + + await this.emit("plugLoaded", plug); + return plug; + } + unload(name: string) { const plug = this.plugs.get(name); if (!plug) { diff --git a/plugos/test_func.test.ts b/plugos/test_func.test.ts index 6471a19..2f6d141 100644 --- a/plugos/test_func.test.ts +++ b/plugos/test_func.test.ts @@ -1,10 +1,12 @@ import * as YAML from "https://deno.land/std@0.184.0/yaml/mod.ts"; import { EndpointRequest, EndpointResponse } from "./hooks/endpoint.ts"; -export function hello() { - console.log(YAML.stringify({ hello: "world" })); - - return "hello"; +export async function hello() { + const numbers = await syscall("addNumbers", 1, 2); + return { + yamlMessage: YAML.stringify({ hello: "world" }), + addedNumbers: numbers, + }; } export function endpoint(req: EndpointRequest): EndpointResponse { diff --git a/plugos/worker_runtime.ts b/plugos/worker_runtime.ts index 7045d43..a398bba 100644 --- a/plugos/worker_runtime.ts +++ b/plugos/worker_runtime.ts @@ -35,28 +35,37 @@ const pendingRequests = new Map< let syscallReqId = 0; +const workerMode = typeof window === "undefined"; + function workerPostMessage(msg: ControllerMessage) { self.postMessage(msg); } -self.syscall = async (name: string, ...args: any[]) => { - return await new Promise((resolve, reject) => { - syscallReqId++; - pendingRequests.set(syscallReqId, { resolve, reject }); - workerPostMessage({ - type: "sys", - id: syscallReqId, - name, - args, +if (workerMode) { + globalThis.syscall = async (name: string, ...args: any[]) => { + return await new Promise((resolve, reject) => { + syscallReqId++; + pendingRequests.set(syscallReqId, { resolve, reject }); + workerPostMessage({ + type: "sys", + id: syscallReqId, + name, + args, + }); }); - }); -}; + }; +} export function setupMessageListener( // deno-lint-ignore ban-types functionMapping: Record, manifest: any, ) { + if (!workerMode) { + // Don't do any of this stuff if this is not a web worker + // This caters to the NoSandbox run mode + return; + } self.addEventListener("message", (event: { data: WorkerMessage }) => { (async () => { const data = event.data; diff --git a/plugs/markdown/markdown_render.test.ts b/plugs/markdown/markdown_render.test.ts index 220ca71..ead97a5 100644 --- a/plugs/markdown/markdown_render.test.ts +++ b/plugs/markdown/markdown_render.test.ts @@ -2,7 +2,7 @@ import buildMarkdown from "../../common/markdown_parser/parser.ts"; import { parse } from "../../common/markdown_parser/parse_tree.ts"; import { System } from "../../plugos/system.ts"; -import { createSandbox } from "../../plugos/environments/deno_sandbox.ts"; +import { createSandbox } from "../../plugos/sandboxes/deno_worker_sandbox.ts"; import { loadMarkdownExtensions } from "../../common/markdown_parser/markdown_ext.ts"; import { renderMarkdownToHtml } from "./markdown_render.ts"; diff --git a/server/server_system.ts b/server/server_system.ts index 7905a36..0a48ce4 100644 --- a/server/server_system.ts +++ b/server/server_system.ts @@ -4,7 +4,7 @@ import { loadMarkdownExtensions } from "../common/markdown_parser/markdown_ext.t import buildMarkdown from "../common/markdown_parser/parser.ts"; import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives.ts"; import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts"; -import { createSandbox } from "../plugos/environments/webworker_sandbox.ts"; +import { createSandbox } from "../plugos/sandboxes/web_worker_sandbox.ts"; import { CronHook } from "../plugos/hooks/cron.ts"; import { EventHook } from "../plugos/hooks/event.ts"; import { MQHook } from "../plugos/hooks/mq.ts"; diff --git a/web/client_system.ts b/web/client_system.ts index b111fca..ebd200b 100644 --- a/web/client_system.ts +++ b/web/client_system.ts @@ -3,7 +3,7 @@ import { Manifest, SilverBulletHooks } from "../common/manifest.ts"; import buildMarkdown from "../common/markdown_parser/parser.ts"; import { CronHook } from "../plugos/hooks/cron.ts"; import { EventHook } from "../plugos/hooks/event.ts"; -import { createSandbox } from "../plugos/environments/webworker_sandbox.ts"; +import { createSandbox } from "../plugos/sandboxes/web_worker_sandbox.ts"; import assetSyscalls from "../plugos/syscalls/asset.ts"; import { eventSyscalls } from "../plugos/syscalls/event.ts";