From a2dbf7b3dbd52130f9269f47458f24ec4f4d4fdd Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Mon, 15 Jan 2024 16:43:12 +0100 Subject: [PATCH] PlugOS refactor and other tweaks (#631) * Prep for in-process plug loading (e.g. for CF workers, Deno Deploy) * Prototype of fixed in-process loading plugs * Fix: buttons not to scroll with content * Better positioning of modal especially on mobile * Move query caching outside query * Fix annoying mouse behavior when filter box appears * Page navigator search tweaks --- cli/plug_run.test.ts | 4 +- cmd/plug_run.ts | 2 +- cmd/server.ts | 4 +- common/space_index.ts | 2 +- common/syscalls/language.ts | 4 +- plug-api/lib/async.ts | 14 +- plug-api/lib/frontmatter.ts | 3 + {common => plug-api/lib}/limited_map.test.ts | 2 +- {common => plug-api/lib}/limited_map.ts | 15 +- plug-api/lib/memory_cache.test.ts | 17 ++ plug-api/lib/memory_cache.ts | 21 ++ plug-api/plugos-syscall/asset.ts | 3 +- plug-api/types.ts | 5 - plugos/hooks/endpoint.test.ts | 7 +- plugos/hooks/event.ts | 2 +- plugos/hooks/mq.ts | 3 +- plugos/lib/datastore.test.ts | 2 +- plugos/lib/datastore.ts | 27 +-- plugos/lib/mq_util.ts | 7 - plugos/plug.ts | 3 +- plugos/runtime.test.ts | 44 +++- plugos/sandboxes/deno_worker_sandbox.ts | 42 ++-- plugos/sandboxes/no_sandbox.ts | 114 +++++++--- plugos/sandboxes/web_worker_sandbox.ts | 6 +- plugos/sandboxes/worker_sandbox.ts | 3 +- plugos/syscalls/asset.ts | 7 +- plugos/syscalls/datastore.ts | 64 ++---- plugos/syscalls/fs.deno.test.ts | 4 +- plugos/syscalls/mq.ts | 21 +- plugos/syscalls/transport.ts | 20 -- plugos/system.ts | 83 ++------ plugos/worker_runtime.ts | 28 ++- plugs/editor/complete.ts | 4 +- plugs/editor/editor.ts | 2 +- plugs/index/anchor.ts | 5 +- plugs/index/api.ts | 16 +- plugs/index/attributes.ts | 3 +- plugs/index/command.ts | 2 +- plugs/index/plug_api.ts | 8 +- plugs/index/tags.ts | 3 +- plugs/markdown/markdown_render.test.ts | 12 +- plugs/markdown/preview.ts | 4 +- plugs/tasks/complete.ts | 4 +- plugs/template/complete.ts | 3 +- server/crypto.ts | 4 + server/http_server.ts | 24 ++- server/server_system.ts | 67 ++++-- server/syscalls/shell.ts | 1 - server/syscalls/space.ts | 10 +- web/client.ts | 13 +- web/client_system.ts | 13 +- web/cm_plugins/markdown_widget.ts | 2 +- web/components/basic_modals.tsx | 72 ++++--- web/components/filter.tsx | 206 +++++++++---------- web/components/fuse_search.ts | 7 +- web/components/top_bar.tsx | 2 +- web/space.ts | 3 - web/styles/editor.scss | 9 +- web/styles/modals.scss | 26 +-- web/syscalls/clientStore.ts | 12 +- web/syscalls/datastore.proxy.ts | 37 +--- web/syscalls/editor.ts | 5 +- web/syscalls/space.ts | 10 +- web/syscalls/system.ts | 31 +-- web/syscalls/util.ts | 5 +- 65 files changed, 591 insertions(+), 617 deletions(-) rename {common => plug-api/lib}/limited_map.test.ts (92%) rename {common => plug-api/lib}/limited_map.ts (78%) create mode 100644 plug-api/lib/memory_cache.test.ts create mode 100644 plug-api/lib/memory_cache.ts delete mode 100644 plugos/lib/mq_util.ts delete mode 100644 plugos/syscalls/transport.ts diff --git a/cli/plug_run.test.ts b/cli/plug_run.test.ts index 1d14ef9..c70b689 100644 --- a/cli/plug_run.test.ts +++ b/cli/plug_run.test.ts @@ -2,9 +2,7 @@ import { AssetBundle } from "../plugos/asset_bundle/bundle.ts"; import { compileManifest } from "../plugos/compile.ts"; import { esbuild } from "../plugos/deps.ts"; import { runPlug } from "./plug_run.ts"; -import assets from "../dist/plug_asset_bundle.json" assert { - type: "json", -}; +import assets from "../dist/plug_asset_bundle.json" with { type: "json" }; import { assertEquals } from "../test_deps.ts"; import { path } from "../common/deps.ts"; diff --git a/cmd/plug_run.ts b/cmd/plug_run.ts index 2bb191d..f12c72a 100644 --- a/cmd/plug_run.ts +++ b/cmd/plug_run.ts @@ -1,6 +1,6 @@ import { runPlug } from "../cli/plug_run.ts"; import { path } from "../common/deps.ts"; -import assets from "../dist/plug_asset_bundle.json" assert { +import assets from "../dist/plug_asset_bundle.json" with { type: "json", }; import { AssetBundle } from "../plugos/asset_bundle/bundle.ts"; diff --git a/cmd/server.ts b/cmd/server.ts index d5fb0aa..1209343 100644 --- a/cmd/server.ts +++ b/cmd/server.ts @@ -1,8 +1,8 @@ import { HttpServer } from "../server/http_server.ts"; -import clientAssetBundle from "../dist/client_asset_bundle.json" assert { +import clientAssetBundle from "../dist/client_asset_bundle.json" with { type: "json", }; -import plugAssetBundle from "../dist/plug_asset_bundle.json" assert { +import plugAssetBundle from "../dist/plug_asset_bundle.json" with { type: "json", }; import { AssetBundle, AssetJson } from "../plugos/asset_bundle/bundle.ts"; diff --git a/common/space_index.ts b/common/space_index.ts index 233d9d2..0d40e44 100644 --- a/common/space_index.ts +++ b/common/space_index.ts @@ -4,7 +4,7 @@ import { System } from "../plugos/system.ts"; const indexVersionKey = ["$indexVersion"]; // Bump this one every time a full reinxex is needed -const desiredIndexVersion = 2; +const desiredIndexVersion = 3; let indexOngoing = false; diff --git a/common/syscalls/language.ts b/common/syscalls/language.ts index 8d7c5f8..3524ea9 100644 --- a/common/syscalls/language.ts +++ b/common/syscalls/language.ts @@ -16,9 +16,7 @@ export function languageSyscalls(): SysCallMapping { } return parse(lang, code); }, - "language.listLanguages": ( - _ctx, - ): string[] => { + "language.listLanguages": (): string[] => { return Object.keys(builtinLanguages); }, }; diff --git a/plug-api/lib/async.ts b/plug-api/lib/async.ts index e8b5ab4..243447a 100644 --- a/plug-api/lib/async.ts +++ b/plug-api/lib/async.ts @@ -37,24 +37,24 @@ export class PromiseQueue { resolve: (value: any) => void; reject: (error: any) => void; }[] = []; - private running = false; + private processing = false; runInQueue(fn: () => Promise): Promise { return new Promise((resolve, reject) => { this.queue.push({ fn, resolve, reject }); - if (!this.running) { - this.run(); + if (!this.processing) { + this.process(); } }); } - private async run(): Promise { + private async process(): Promise { if (this.queue.length === 0) { - this.running = false; + this.processing = false; return; } - this.running = true; + this.processing = true; const { fn, resolve, reject } = this.queue.shift()!; try { @@ -64,7 +64,7 @@ export class PromiseQueue { reject(error); } - this.run(); // Continue processing the next promise in the queue + this.process(); // Continue processing the next promise in the queue } } diff --git a/plug-api/lib/frontmatter.ts b/plug-api/lib/frontmatter.ts index 722464b..c9738a0 100644 --- a/plug-api/lib/frontmatter.ts +++ b/plug-api/lib/frontmatter.ts @@ -7,6 +7,7 @@ import { replaceNodesMatchingAsync, traverseTreeAsync, } from "$sb/lib/tree.ts"; +import { expandPropertyNames } from "$sb/lib/json.ts"; export type FrontMatter = { tags?: string[] } & Record; @@ -116,6 +117,8 @@ export async function extractFrontmatter( data.tags = [...new Set([...tags.map((t) => t.replace(/^#/, ""))])]; // console.log("Extracted tags", data.tags); + // Expand property names (e.g. "foo.bar" => { foo: { bar: true } }) + data = expandPropertyNames(data); return data; } diff --git a/common/limited_map.test.ts b/plug-api/lib/limited_map.test.ts similarity index 92% rename from common/limited_map.test.ts rename to plug-api/lib/limited_map.test.ts index e70f1ba..a62859d 100644 --- a/common/limited_map.test.ts +++ b/plug-api/lib/limited_map.test.ts @@ -1,5 +1,5 @@ import { sleep } from "$sb/lib/async.ts"; -import { assertEquals } from "../test_deps.ts"; +import { assertEquals } from "../../test_deps.ts"; import { LimitedMap } from "./limited_map.ts"; Deno.test("limited map", async () => { diff --git a/common/limited_map.ts b/plug-api/lib/limited_map.ts similarity index 78% rename from common/limited_map.ts rename to plug-api/lib/limited_map.ts index 3ba2259..e422da0 100644 --- a/common/limited_map.ts +++ b/plug-api/lib/limited_map.ts @@ -1,4 +1,8 @@ -type LimitedMapRecord = { value: V; la: number }; +type LimitedMapRecord = { + value: V; + la: number; + expTimer?: number; +}; export class LimitedMap { private map: Map>; @@ -16,8 +20,13 @@ export class LimitedMap { * @param ttl time to live (in ms) */ set(key: string, value: V, ttl?: number) { + const entry: LimitedMapRecord = { value, la: Date.now() }; if (ttl) { - setTimeout(() => { + const existingEntry = this.map.get(key); + if (existingEntry?.expTimer) { + clearTimeout(existingEntry.expTimer); + } + entry.expTimer = setTimeout(() => { this.map.delete(key); }, ttl); } @@ -26,7 +35,7 @@ export class LimitedMap { const oldestKey = this.getOldestKey(); this.map.delete(oldestKey!); } - this.map.set(key, { value, la: Date.now() }); + this.map.set(key, entry); } get(key: string): V | undefined { diff --git a/plug-api/lib/memory_cache.test.ts b/plug-api/lib/memory_cache.test.ts new file mode 100644 index 0000000..f3e7149 --- /dev/null +++ b/plug-api/lib/memory_cache.test.ts @@ -0,0 +1,17 @@ +import { sleep } from "$sb/lib/async.ts"; +import { ttlCache } from "$sb/lib/memory_cache.ts"; +import { assertEquals } from "../../test_deps.ts"; + +Deno.test("Memory cache", async () => { + let calls = 0; + async function expensiveFunction(key: string) { + calls++; + await sleep(1); + return key; + } + assertEquals("key", await ttlCache("key", expensiveFunction, 0.01)); + assertEquals(1, calls); + assertEquals("key", await ttlCache("key", expensiveFunction, 0.01)); + assertEquals(1, calls); + await sleep(10); +}); diff --git a/plug-api/lib/memory_cache.ts b/plug-api/lib/memory_cache.ts new file mode 100644 index 0000000..a9e3046 --- /dev/null +++ b/plug-api/lib/memory_cache.ts @@ -0,0 +1,21 @@ +import { LimitedMap } from "$sb/lib/limited_map.ts"; + +const cache = new LimitedMap(50); + +export async function ttlCache( + key: K, + fn: (key: K) => Promise, + ttlSecs?: number, +): Promise { + if (!ttlSecs) { + return fn(key); + } + const serializedKey = JSON.stringify(key); + const cached = cache.get(serializedKey); + if (cached) { + return cached; + } + const result = await fn(key); + cache.set(serializedKey, result, ttlSecs * 1000); + return result; +} diff --git a/plug-api/plugos-syscall/asset.ts b/plug-api/plugos-syscall/asset.ts index fe15b56..8e93d7c 100644 --- a/plug-api/plugos-syscall/asset.ts +++ b/plug-api/plugos-syscall/asset.ts @@ -2,10 +2,11 @@ import { base64DecodeDataUrl } from "../../plugos/asset_bundle/base64.ts"; import { syscall } from "./syscall.ts"; export async function readAsset( + plugName: string, name: string, encoding: "utf8" | "dataurl" = "utf8", ): Promise { - const dataUrl = await syscall("asset.readAsset", name) as string; + const dataUrl = await syscall("asset.readAsset", plugName, name) as string; switch (encoding) { case "utf8": return new TextDecoder().decode(base64DecodeDataUrl(dataUrl)); diff --git a/plug-api/types.ts b/plug-api/types.ts index 3d6e815..0786cde 100644 --- a/plug-api/types.ts +++ b/plug-api/types.ts @@ -73,11 +73,6 @@ export type Query = { render?: string; renderAll?: boolean; distinct?: boolean; - - /** - * When set, the DS implementation _may_ cache the result for the given number of seconds. - */ - cacheSecs?: number; }; export type KvQuery = Omit & { diff --git a/plugos/hooks/endpoint.test.ts b/plugos/hooks/endpoint.test.ts index a93df6a..5e7a535 100644 --- a/plugos/hooks/endpoint.test.ts +++ b/plugos/hooks/endpoint.test.ts @@ -16,12 +16,7 @@ Deno.test("Run a plugos endpoint server", async () => { tempDir, ); - await system.load( - new URL(`file://${workerPath}`), - "test", - 0, - createSandbox, - ); + await system.load("test", createSandbox(new URL(`file://${workerPath}`))); const app = new Hono(); const port = 3123; diff --git a/plugos/hooks/event.ts b/plugos/hooks/event.ts index 4bfcf8e..f1053c7 100644 --- a/plugos/hooks/event.ts +++ b/plugos/hooks/event.ts @@ -70,7 +70,7 @@ export class EventHook implements Hook { } } catch (e: any) { console.error( - `Error dispatching event ${eventName} to plug ${plug.name}: ${e.message}`, + `Error dispatching event ${eventName} to ${plug.name}.${name}: ${e.message}`, ); } })()); diff --git a/plugos/hooks/mq.ts b/plugos/hooks/mq.ts index 02564a0..59f57b4 100644 --- a/plugos/hooks/mq.ts +++ b/plugos/hooks/mq.ts @@ -1,6 +1,5 @@ import { Hook, Manifest } from "../types.ts"; import { System } from "../system.ts"; -import { fullQueueName } from "../lib/mq_util.ts"; import { MQMessage } from "$sb/types.ts"; import { MessageQueue } from "../lib/mq.ts"; import { throttle } from "$sb/lib/async.ts"; @@ -61,7 +60,7 @@ export class MQHook implements Hook { } const subscriptions = functionDef.mqSubscriptions; for (const subscriptionDef of subscriptions) { - const queue = fullQueueName(plug.name!, subscriptionDef.queue); + const queue = subscriptionDef.queue; // console.log("Subscribing to queue", queue); this.subscriptions.push( this.mq.subscribe( diff --git a/plugos/lib/datastore.test.ts b/plugos/lib/datastore.test.ts index 934c95d..15ca0e7 100644 --- a/plugos/lib/datastore.test.ts +++ b/plugos/lib/datastore.test.ts @@ -7,7 +7,7 @@ import { assertEquals } from "https://deno.land/std@0.165.0/testing/asserts.ts"; import { PrefixedKvPrimitives } from "./prefixed_kv_primitives.ts"; async function test(db: KvPrimitives) { - const datastore = new DataStore(new PrefixedKvPrimitives(db, ["ds"]), false, { + const datastore = new DataStore(new PrefixedKvPrimitives(db, ["ds"]), { count: (arr: any[]) => arr.length, }); await datastore.set(["user", "peter"], { name: "Peter" }); diff --git a/plugos/lib/datastore.ts b/plugos/lib/datastore.ts index 076f4eb..9753216 100644 --- a/plugos/lib/datastore.ts +++ b/plugos/lib/datastore.ts @@ -2,18 +2,13 @@ import { applyQueryNoFilterKV, evalQueryExpression } from "$sb/lib/query.ts"; import { FunctionMap, KV, KvKey, KvQuery } from "$sb/types.ts"; import { builtinFunctions } from "$sb/lib/builtin_query_functions.ts"; import { KvPrimitives } from "./kv_primitives.ts"; -import { LimitedMap } from "../../common/limited_map.ts"; - /** * This is the data store class you'll actually want to use, wrapping the primitives * in a more user-friendly way */ export class DataStore { - private cache = new LimitedMap(20); - constructor( readonly kv: KvPrimitives, - private enableCache = false, private functionMap: FunctionMap = builtinFunctions, ) { } @@ -63,21 +58,6 @@ export class DataStore { } async query(query: KvQuery): Promise[]> { - let cacheKey: string | undefined; - const cacheSecs = query.cacheSecs; - // Should we do caching? - if (cacheSecs && this.enableCache) { - // Remove the cacheSecs from the query - query = { ...query, cacheSecs: undefined }; - console.log("Going to cache query", query); - cacheKey = JSON.stringify(query); - const cachedResult = this.cache.get(cacheKey); - if (cachedResult) { - // Let's use the cached result - return cachedResult; - } - } - const results: KV[] = []; let itemCount = 0; // Accumulate results @@ -104,12 +84,7 @@ export class DataStore { } } // Apply order by, limit, and select - const finalResult = applyQueryNoFilterKV(query, results, this.functionMap); - if (cacheKey) { - // Store in the cache - this.cache.set(cacheKey, finalResult, cacheSecs! * 1000); - } - return finalResult; + return applyQueryNoFilterKV(query, results, this.functionMap); } async queryDelete(query: KvQuery): Promise { diff --git a/plugos/lib/mq_util.ts b/plugos/lib/mq_util.ts deleted file mode 100644 index 6fbe3b7..0000000 --- a/plugos/lib/mq_util.ts +++ /dev/null @@ -1,7 +0,0 @@ -// Adds a plug name to a queue name if it doesn't already have one. -export function fullQueueName(plugName: string, queueName: string) { - if (queueName.includes(".")) { - return queueName; - } - return plugName + "." + queueName; -} diff --git a/plugos/plug.ts b/plugos/plug.ts index 2a272c1..6c093be 100644 --- a/plugos/plug.ts +++ b/plugos/plug.ts @@ -21,7 +21,6 @@ export class Plug { constructor( private system: System, - public workerUrl: URL | undefined, readonly name: string, private hash: number, private sandboxFactory: SandboxFactory, @@ -44,7 +43,7 @@ export class Plug { // Invoke a syscall syscall(name: string, args: any[]): Promise { - return this.system.syscallWithContext({ plug: this }, name, args); + return this.system.syscall({ plug: this.name }, name, args); } /** diff --git a/plugos/runtime.test.ts b/plugos/runtime.test.ts index 6561c76..b907f8d 100644 --- a/plugos/runtime.test.ts +++ b/plugos/runtime.test.ts @@ -1,20 +1,25 @@ import { createSandbox } from "./sandboxes/deno_worker_sandbox.ts"; import { System } from "./system.ts"; -import { assertEquals } from "../test_deps.ts"; +import { assert, assertEquals } from "../test_deps.ts"; import { compileManifest } from "./compile.ts"; import { esbuild } from "./deps.ts"; +import { + createSandbox as createNoSandbox, + runWithSystemLock, +} from "./sandboxes/no_sandbox.ts"; +import { sleep } from "$sb/lib/async.ts"; +import { SysCallMapping } from "./system.ts"; Deno.test("Run a deno sandbox", async () => { const system = new System("server"); system.registerSyscalls([], { addNumbers: (_ctx, a, b) => { - console.log("This is the context", _ctx.plug.name); return a + b; }, failingSyscall: () => { throw new Error("#fail"); }, - }); + } as SysCallMapping); system.registerSyscalls(["restricted"], { restrictedSyscall: () => { return "restricted"; @@ -34,10 +39,8 @@ Deno.test("Run a deno sandbox", async () => { ); const plug = await system.load( - new URL(`file://${workerPath}`), "test", - 0, - createSandbox, + createSandbox(new URL(`file://${workerPath}`)), ); assertEquals({ @@ -52,12 +55,31 @@ Deno.test("Run a deno sandbox", async () => { `file://${workerPath}` ); - const plug2 = await system.loadNoSandbox("test", plugExport); + const plug2 = await system.load("test", createNoSandbox(plugExport)); - assertEquals({ - addedNumbers: 3, - yamlMessage: "hello: world\n", - }, await plug2.invoke("boot", [])); + let running = false; + await Promise.all([ + runWithSystemLock(system, async () => { + console.log("Starting first run"); + running = true; + await sleep(5); + assertEquals({ + addedNumbers: 3, + yamlMessage: "hello: world\n", + }, await plug2.invoke("boot", [])); + console.log("Done first run"); + running = false; + }), + runWithSystemLock(system, async () => { + assert(!running); + console.log("Starting second run"); + assertEquals({ + addedNumbers: 3, + yamlMessage: "hello: world\n", + }, await plug2.invoke("boot", [])); + console.log("Done second run"); + }), + ]); await system.unloadAll(); diff --git a/plugos/sandboxes/deno_worker_sandbox.ts b/plugos/sandboxes/deno_worker_sandbox.ts index b30b122..a8adb93 100644 --- a/plugos/sandboxes/deno_worker_sandbox.ts +++ b/plugos/sandboxes/deno_worker_sandbox.ts @@ -1,26 +1,26 @@ import { WorkerSandbox } from "./worker_sandbox.ts"; -import { Plug } from "../plug.ts"; -import { Sandbox } from "./sandbox.ts"; +import type { SandboxFactory } from "./sandbox.ts"; // Uses Deno's permissions to lock the worker down significantly -export function createSandbox(plug: Plug): Sandbox { - return new WorkerSandbox(plug, { - deno: { - permissions: { - // Allow network access - net: true, - // This is required for console logging to work, apparently? - env: true, - // No talking to native code - ffi: false, - // No invocation of shell commands - run: false, - // No read access to the file system - read: false, - // No write access to the file system - write: false, +export function createSandbox(workerUrl: URL): SandboxFactory { + return (plug) => + new WorkerSandbox(plug, workerUrl, { + deno: { + permissions: { + // Allow network access + net: true, + // This is required for console logging to work, apparently? + env: true, + // No talking to native code + ffi: false, + // No invocation of shell commands + run: false, + // No read access to the file system + read: false, + // No write access to the file system + write: false, + }, }, - }, - // Have to do this because the "deno" option is not standard and doesn't typecheck yet - } as any); + // Have to do this because the "deno" option is not standard and doesn't typecheck yet + }); } diff --git a/plugos/sandboxes/no_sandbox.ts b/plugos/sandboxes/no_sandbox.ts index b2e5f5a..e700946 100644 --- a/plugos/sandboxes/no_sandbox.ts +++ b/plugos/sandboxes/no_sandbox.ts @@ -2,6 +2,38 @@ import { PromiseQueue } from "$sb/lib/async.ts"; import { Plug } from "../plug.ts"; import { Sandbox } from "./sandbox.ts"; import { Manifest } from "../types.ts"; +import { System } from "../system.ts"; +import { SandboxFactory } from "./sandbox.ts"; + +/** + * This implements a "no sandbox" sandbox that actually runs code the main thread, without any isolation. + * This is useful for (often serverless) environments like CloudFlare workers and Deno Deploy that do not support workers. + * Since these environments often also don't allow dynamic loading (or even eval'ing) of code, plug code needs to be + * imported as a regular ESM module (which is possible). + * + * To make this work, a global `syscall` function needs to be injected into the global scope. + * Since a syscall relies on a System, we need to track the active System in a global variable. + * The issue with this is that it means that only a single System can be active at a given time per JS process. + * To enforce this, we have a runWithSystemLock function that can be used to run code in a System-locked context, effectively queuing the execution of tasks sequentially. + * This isn't great, but it's the best we can do. + * + * Luckily, in the only contexts in which you need to run plugs this way are serverless, where code will be + * run in a bunch of isolates with hopefully low parallelism of requests per isolate. + */ + +/** + * A type representing the `plug` export of a plug, used via e.g. `import { plug } from "./some.plug.js` + * Values of this type are passed into the `noSandboxFactory` function when called on a system.load + */ +export type PlugExport = { + manifest: Manifest; + functionMapping: Record any>; +}; + +// The global variable tracking the currently active system (if any) +let activeSystem: + | System + | undefined; // We need to hard inject the syscall function into the global scope declare global { @@ -9,60 +41,76 @@ declare global { syscall(name: string, ...args: any[]): Promise; } } - -export type PlugExport = { - manifest: Manifest; - functionMapping: Record; -}; - -const functionQueue = new PromiseQueue(); - -let activePlug: Plug | undefined; - // @ts-ignore: globalThis globalThis.syscall = (name: string, ...args: any[]): Promise => { - if (!activePlug) { - throw new Error("No active plug"); + if (!activeSystem) { + throw new Error(`No currently active system, can't invoke syscall ${name}`); } - console.log("Calling syscall", name, args); - return activePlug.syscall(name, args); + // Invoke syscall with no active plug set (because we don't know which plug is invoking the syscall) + return activeSystem.syscall({}, name, args); }; +// Global sequential task queue for running tasks in a System-locked context +const taskQueue = new PromiseQueue(); + +/** + * Schedules a task to run in a System-locked context + * in effect this will ensure only one such context is active at a given time allowing for no parallelism + * @param system to activate while running the task + * @param task callback to run + * @returns the result of the task once it completes + */ +export function runWithSystemLock( + system: System, + task: () => Promise, +): Promise { + return taskQueue.runInQueue(async () => { + // Set the global active system, which is used by the syscall function + activeSystem = system; + try { + // Run the logic, note putting the await here is crucial to make sure the `finally` block runs at the right time + return await task(); + } finally { + // And then reset the global active system whether the thing blew up or not + activeSystem = undefined; + } + }); +} + +/** + * Implements a no-sandbox sandbox that runs code in the main thread + */ export class NoSandbox implements Sandbox { - manifest?: Manifest | undefined; + manifest: Manifest; + constructor( - private plug: Plug, - private plugExport: PlugExport, + readonly plug: Plug, + readonly plugExport: PlugExport, ) { this.manifest = plugExport.manifest; plug.manifest = this.manifest; } init(): Promise { + // Nothing to do return Promise.resolve(); } invoke(name: string, args: any[]): Promise { - activePlug = this.plug; - return functionQueue.runInQueue(async () => { - try { - const fn = this.plugExport.functionMapping[name]; - if (!fn) { - throw new Error(`Function not loaded: ${name}`); - } - return await fn(...args); - } finally { - activePlug = undefined; - } - }); + const fn = this.plugExport.functionMapping[name]; + if (!fn) { + throw new Error(`Function not defined: ${name}`); + } + return Promise.resolve(fn(...args)); } stop() { + // Nothing to do } } -export function noSandboxFactory( - plugExport: PlugExport, -): (plug: Plug) => Sandbox { - return (plug: Plug) => new NoSandbox(plug, plugExport); +export function createSandbox( + plugExport: PlugExport, +): SandboxFactory { + return (plug: Plug) => new NoSandbox(plug, plugExport); } diff --git a/plugos/sandboxes/web_worker_sandbox.ts b/plugos/sandboxes/web_worker_sandbox.ts index 8c5a8e7..dc9c585 100644 --- a/plugos/sandboxes/web_worker_sandbox.ts +++ b/plugos/sandboxes/web_worker_sandbox.ts @@ -1,7 +1,7 @@ import { WorkerSandbox } from "./worker_sandbox.ts"; import type { Plug } from "../plug.ts"; -import { Sandbox } from "./sandbox.ts"; +import type { SandboxFactory } from "./sandbox.ts"; -export function createSandbox(plug: Plug): Sandbox { - return new WorkerSandbox(plug); +export function createSandbox(workerUrl: URL): SandboxFactory { + return (plug: Plug) => new WorkerSandbox(plug, workerUrl); } diff --git a/plugos/sandboxes/worker_sandbox.ts b/plugos/sandboxes/worker_sandbox.ts index 40da509..2032025 100644 --- a/plugos/sandboxes/worker_sandbox.ts +++ b/plugos/sandboxes/worker_sandbox.ts @@ -21,6 +21,7 @@ export class WorkerSandbox implements Sandbox { constructor( readonly plug: Plug, + public workerUrl: URL, private workerOptions = {}, ) { } @@ -35,7 +36,7 @@ export class WorkerSandbox implements Sandbox { console.warn("Double init of sandbox, ignoring"); return Promise.resolve(); } - this.worker = new Worker(this.plug.workerUrl!, { + this.worker = new Worker(this.workerUrl, { ...this.workerOptions, type: "module", }); diff --git a/plugos/syscalls/asset.ts b/plugos/syscalls/asset.ts index 80b5d33..67b0e43 100644 --- a/plugos/syscalls/asset.ts +++ b/plugos/syscalls/asset.ts @@ -2,11 +2,8 @@ import { SysCallMapping, System } from "../system.ts"; export default function assetSyscalls(system: System): SysCallMapping { return { - "asset.readAsset": ( - ctx, - name: string, - ): string => { - return system.loadedPlugs.get(ctx.plug.name!)!.assets!.readFileAsDataUrl( + "asset.readAsset": (_ctx, plugName: string, name: string): string => { + return system.loadedPlugs.get(plugName)!.assets!.readFileAsDataUrl( name, ); }, diff --git a/plugos/syscalls/datastore.ts b/plugos/syscalls/datastore.ts index 4345579..95ca71b 100644 --- a/plugos/syscalls/datastore.ts +++ b/plugos/syscalls/datastore.ts @@ -1,75 +1,47 @@ import { KV, KvKey, KvQuery } from "$sb/types.ts"; import type { DataStore } from "../lib/datastore.ts"; -import type { SyscallContext, SysCallMapping } from "../system.ts"; +import type { SysCallMapping } from "../system.ts"; /** * Exposes the datastore API to plugs, but scoping everything to a prefix based on the plug's name * @param ds the datastore to wrap * @param prefix prefix to scope all keys to to which the plug name will be appended */ -export function dataStoreSyscalls( - ds: DataStore, - prefix: KvKey = ["ds"], -): SysCallMapping { +export function dataStoreSyscalls(ds: DataStore): SysCallMapping { return { - "datastore.delete": (ctx, key: KvKey) => { - return ds.delete(applyPrefix(ctx, key)); + "datastore.delete": (_ctx, key: KvKey) => { + return ds.delete(key); }, - "datastore.set": (ctx, key: KvKey, value: any) => { - return ds.set(applyPrefix(ctx, key), value); + "datastore.set": (_ctx, key: KvKey, value: any) => { + return ds.set(key, value); }, - "datastore.batchSet": (ctx, kvs: KV[]) => { - return ds.batchSet( - kvs.map((kv) => ({ key: applyPrefix(ctx, kv.key), value: kv.value })), - ); + "datastore.batchSet": (_ctx, kvs: KV[]) => { + return ds.batchSet(kvs); }, - "datastore.batchDelete": (ctx, keys: KvKey[]) => { - return ds.batchDelete(keys.map((k) => applyPrefix(ctx, k))); + "datastore.batchDelete": (_ctx, keys: KvKey[]) => { + return ds.batchDelete(keys); }, "datastore.batchGet": ( - ctx, + _ctx, keys: KvKey[], ): Promise<(any | undefined)[]> => { - return ds.batchGet(keys.map((k) => applyPrefix(ctx, k))); + return ds.batchGet(keys); }, - "datastore.get": (ctx, key: KvKey): Promise => { - return ds.get(applyPrefix(ctx, key)); + "datastore.get": (_ctx, key: KvKey): Promise => { + return ds.get(key); }, - "datastore.query": async ( - ctx, - query: KvQuery, - ): Promise => { - return (await ds.query({ - ...query, - prefix: applyPrefix(ctx, query.prefix), - })).map((kv) => ({ - key: stripPrefix(kv.key), - value: kv.value, - })); + "datastore.query": async (_ctx, query: KvQuery): Promise => { + return (await ds.query(query)); }, - "datastore.queryDelete": ( - ctx, - query: KvQuery, - ): Promise => { - return ds.queryDelete({ - ...query, - prefix: applyPrefix(ctx, query.prefix), - }); + "datastore.queryDelete": (_ctx, query: KvQuery): Promise => { + return ds.queryDelete(query); }, }; - - function applyPrefix(ctx: SyscallContext, key?: KvKey): KvKey { - return [...prefix, ctx.plug.name!, ...(key ? key : [])]; - } - - function stripPrefix(key: KvKey): KvKey { - return key.slice(prefix.length + 1); - } } diff --git a/plugos/syscalls/fs.deno.test.ts b/plugos/syscalls/fs.deno.test.ts index b7f728c..cf4f65a 100644 --- a/plugos/syscalls/fs.deno.test.ts +++ b/plugos/syscalls/fs.deno.test.ts @@ -3,15 +3,13 @@ import { assert } from "../../test_deps.ts"; import { path } from "../deps.ts"; import fileSystemSyscalls from "./fs.deno.ts"; -const fakeCtx = {} as any; - Deno.test("Test FS operations", async () => { const thisFolder = path.resolve( path.dirname(new URL(import.meta.url).pathname), ); const syscalls = fileSystemSyscalls(thisFolder); const allFiles: FileMeta[] = await syscalls["fs.listFiles"]( - fakeCtx, + {}, thisFolder, true, ); diff --git a/plugos/syscalls/mq.ts b/plugos/syscalls/mq.ts index b4af2eb..93ee29b 100644 --- a/plugos/syscalls/mq.ts +++ b/plugos/syscalls/mq.ts @@ -1,25 +1,24 @@ import { SysCallMapping } from "../system.ts"; -import { fullQueueName } from "../lib/mq_util.ts"; import { MessageQueue } from "../lib/mq.ts"; export function mqSyscalls( mq: MessageQueue, ): SysCallMapping { return { - "mq.send": (ctx, queue: string, body: any) => { - return mq.send(fullQueueName(ctx.plug.name!, queue), body); + "mq.send": (_ctx, queue: string, body: any) => { + return mq.send(queue, body); }, - "mq.batchSend": (ctx, queue: string, bodies: any[]) => { - return mq.batchSend(fullQueueName(ctx.plug.name!, queue), bodies); + "mq.batchSend": (_ctx, queue: string, bodies: any[]) => { + return mq.batchSend(queue, bodies); }, - "mq.ack": (ctx, queue: string, id: string) => { - return mq.ack(fullQueueName(ctx.plug.name!, queue), id); + "mq.ack": (_ctx, queue: string, id: string) => { + return mq.ack(queue, id); }, - "mq.batchAck": (ctx, queue: string, ids: string[]) => { - return mq.batchAck(fullQueueName(ctx.plug.name!, queue), ids); + "mq.batchAck": (_ctx, queue: string, ids: string[]) => { + return mq.batchAck(queue, ids); }, - "mq.getQueueStats": (ctx, queue: string) => { - return mq.getQueueStats(fullQueueName(ctx.plug.name!, queue)); + "mq.getQueueStats": (_ctx, queue: string) => { + return mq.getQueueStats(queue); }, }; } diff --git a/plugos/syscalls/transport.ts b/plugos/syscalls/transport.ts deleted file mode 100644 index eaf525b..0000000 --- a/plugos/syscalls/transport.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { SyscallContext, SysCallMapping } from "../system.ts"; - -export function proxySyscalls( - names: string[], - transportCall: ( - ctx: SyscallContext, - name: string, - ...args: any[] - ) => Promise, -): SysCallMapping { - const syscalls: SysCallMapping = {}; - - for (const name of names) { - syscalls[name] = (ctx, ...args: any[]) => { - return transportCall(ctx, name, ...args); - }; - } - - return syscalls; -} diff --git a/plugos/system.ts b/plugos/system.ts index c91c506..a4221cf 100644 --- a/plugos/system.ts +++ b/plugos/system.ts @@ -3,7 +3,6 @@ import { EventEmitter } from "./event.ts"; import type { SandboxFactory } from "./sandboxes/sandbox.ts"; import { Plug } from "./plug.ts"; import { InMemoryManifestCache, ManifestCache } from "./manifest_cache.ts"; -import { noSandboxFactory, PlugExport } from "./sandboxes/no_sandbox.ts"; export interface SysCallMapping { [key: string]: (ctx: SyscallContext, ...args: any) => Promise | any; @@ -16,11 +15,12 @@ export type SystemEvents = { // Passed to every syscall, allows to pass in additional context that the syscall may use export type SyscallContext = { - plug: Plug; + // This is the plug that is invoking the syscall, + // which may be undefined where this cannot be determined (e.g. when running in a NoSandbox) + plug?: string; }; type SyscallSignature = ( - ctx: SyscallContext, ...args: any[] ) => Promise | any; @@ -75,7 +75,11 @@ export class System extends EventEmitter> { } } - syscallWithContext( + localSyscall(name: string, args: any): Promise { + return this.syscall({}, name, args); + } + + syscall( ctx: SyscallContext, name: string, args: any[], @@ -84,36 +88,29 @@ export class System extends EventEmitter> { if (!syscall) { throw Error(`Unregistered syscall ${name}`); } - for (const permission of syscall.requiredPermissions) { - if (!ctx.plug) { - throw Error(`Syscall ${name} requires permission and no plug is set`); + if (ctx.plug) { + // Only when running in a plug context do we check permissions + const plug = this.loadedPlugs.get(ctx.plug!); + if (!plug) { + throw new Error( + `Plug ${ctx.plug} not found while attempting to invoke ${name}}`, + ); } - if (!ctx.plug.grantedPermissions.includes(permission)) { - throw Error(`Missing permission '${permission}' for syscall ${name}`); + for (const permission of syscall.requiredPermissions) { + if (!plug.grantedPermissions.includes(permission)) { + throw Error(`Missing permission '${permission}' for syscall ${name}`); + } } } return Promise.resolve(syscall.callback(ctx, ...args)); } - localSyscall( - contextPlugName: string, - syscallName: string, - args: any[], - ): Promise { - return this.syscallWithContext( - { plug: this.plugs.get(contextPlugName)! }, - syscallName, - args, - ); - } - async load( - workerUrl: URL, name: string, - hash: number, sandboxFactory: SandboxFactory, + hash = -1, ): Promise> { - const plug = new Plug(this, workerUrl, name, hash, sandboxFactory); + const plug = new Plug(this, name, hash, sandboxFactory); // Wait for worker to boot, and pass back its manifest await plug.ready; @@ -139,44 +136,6 @@ export class System extends EventEmitter> { return plug; } - /** - * Loads a plug without a sandbox, which means it will run in the same context as the caller - * @param name - * @param plugExport extracted via e.g. `import { plug } from "./some.plug.js` - * @returns Plug instance - */ - async loadNoSandbox( - name: string, - plugExport: PlugExport, - ): Promise> { - const plug = new Plug( - this, - undefined, - name, - -1, - noSandboxFactory(plugExport), - ); - - const manifest = plugExport.manifest; - - // Validate the manifest - let errors: string[] = []; - for (const feature of this.enabledHooks) { - errors = [...errors, ...feature.validateManifest(plug.manifest!)]; - } - if (errors.length > 0) { - throw new Error(`Invalid manifest: ${errors.join(", ")}`); - } - if (this.plugs.has(manifest.name)) { - this.unload(manifest.name); - } - console.log("Activated plug without sandbox", manifest.name); - this.plugs.set(manifest.name, plug); - - await this.emit("plugLoaded", plug); - return plug; - } - unload(name: string) { const plug = this.plugs.get(name); if (!plug) { diff --git a/plugos/worker_runtime.ts b/plugos/worker_runtime.ts index a398bba..8ec8453 100644 --- a/plugos/worker_runtime.ts +++ b/plugos/worker_runtime.ts @@ -9,6 +9,20 @@ declare global { function syscall(name: string, ...args: any[]): Promise; } +// Are we running in a (web) worker? + +// Determines if we're running in a web worker environment (Deno or browser) +// - in a browser's main threads, typeof window is "object" +// - in a browser's worker threads, typeof window === "undefined" +// - in Deno's main thread typeof window === "object" +// - in Deno's workers typeof window === "undefined +// - in Cloudflare workers typeof window === "undefined", but typeof globalThis.WebSocketPair is defined +const runningAsWebWorker = typeof window === "undefined" && + // @ts-ignore: globalThis + typeof globalThis.WebSocketPair === "undefined"; + +// console.log("Running as web worker:", runningAsWebWorker); + if (typeof Deno === "undefined") { // @ts-ignore: Deno hack self.Deno = { @@ -35,13 +49,11 @@ const pendingRequests = new Map< let syscallReqId = 0; -const workerMode = typeof window === "undefined"; - function workerPostMessage(msg: ControllerMessage) { self.postMessage(msg); } -if (workerMode) { +if (runningAsWebWorker) { globalThis.syscall = async (name: string, ...args: any[]) => { return await new Promise((resolve, reject) => { syscallReqId++; @@ -61,7 +73,7 @@ export function setupMessageListener( functionMapping: Record, manifest: any, ) { - if (!workerMode) { + if (!runningAsWebWorker) { // Don't do any of this stuff if this is not a web worker // This caters to the NoSandbox run mode return; @@ -163,10 +175,10 @@ export async function sandboxFetch( return syscall("sandboxFetch.fetch", reqInfo, options); } +// @ts-ignore: monkey patching fetch +globalThis.nativeFetch = globalThis.fetch; // Monkey patch fetch() - export function monkeyPatchFetch() { - globalThis.nativeFetch = globalThis.fetch; // @ts-ignore: monkey patching fetch globalThis.fetch = async function ( reqInfo: RequestInfo, @@ -192,4 +204,6 @@ export function monkeyPatchFetch() { }; } -monkeyPatchFetch(); +if (runningAsWebWorker) { + monkeyPatchFetch(); +} diff --git a/plugs/editor/complete.ts b/plugs/editor/complete.ts index 5fd0e03..ca23d92 100644 --- a/plugs/editor/complete.ts +++ b/plugs/editor/complete.ts @@ -18,9 +18,7 @@ export async function pageComplete(completeEvent: CompleteEvent) { completeEvent.linePrefix, ); const tagToQuery = isInTemplateContext ? "template" : "page"; - let allPages: PageMeta[] = await queryObjects(tagToQuery, { - cacheSecs: 5, - }); + let allPages: PageMeta[] = await queryObjects(tagToQuery, {}, 5); const prefix = match[1]; if (prefix.startsWith("!")) { // Federation prefix, let's first see if we're matching anything from federation that is locally synced diff --git a/plugs/editor/editor.ts b/plugs/editor/editor.ts index 26557cb..472268f 100644 --- a/plugs/editor/editor.ts +++ b/plugs/editor/editor.ts @@ -31,6 +31,6 @@ export async function moveToPosCommand() { await editor.moveCursor(pos); } -export async function customFlashMessage(_ctx: any, message: string) { +export async function customFlashMessage(_def: any, message: string) { await editor.flashNotification(message); } diff --git a/plugs/index/anchor.ts b/plugs/index/anchor.ts index d70909e..2f46dbf 100644 --- a/plugs/index/anchor.ts +++ b/plugs/index/anchor.ts @@ -43,10 +43,7 @@ export async function anchorComplete(completeEvent: CompleteEvent) { // "bare" anchor, match any page for completion purposes filter = undefined; } - const allAnchors = await queryObjects("anchor", { - filter, - cacheSecs: 5, - }); + const allAnchors = await queryObjects("anchor", { filter }, 5); return { from: completeEvent.pos - match[1].length, options: allAnchors.map((a) => ({ diff --git a/plugs/index/api.ts b/plugs/index/api.ts index af8441d..e8a0a4a 100644 --- a/plugs/index/api.ts +++ b/plugs/index/api.ts @@ -3,6 +3,7 @@ import { KV, KvKey, KvQuery, ObjectQuery, ObjectValue } from "$sb/types.ts"; import { QueryProviderEvent } from "$sb/app_event.ts"; import { builtins } from "./builtins.ts"; import { AttributeObject, determineType } from "./attributes.ts"; +import { ttlCache } from "$sb/lib/memory_cache.ts"; const indexKey = "idx"; const pageKey = "ridx"; @@ -159,15 +160,18 @@ function cleanKey(ref: string, page: string) { } } -export async function queryObjects( +export function queryObjects( tag: string, query: ObjectQuery, + ttlSecs?: number, ): Promise[]> { - return (await datastore.query({ - ...query, - prefix: [indexKey, tag], - distinct: true, - })).map(({ value }) => value); + return ttlCache(query, async () => { + return (await datastore.query({ + ...query, + prefix: [indexKey, tag], + distinct: true, + })).map(({ value }) => value); + }, ttlSecs); } export async function query( diff --git a/plugs/index/attributes.ts b/plugs/index/attributes.ts index 5313bec..8df715b 100644 --- a/plugs/index/attributes.ts +++ b/plugs/index/attributes.ts @@ -52,8 +52,7 @@ export async function objectAttributeCompleter( select: [{ name: "name" }, { name: "attributeType" }, { name: "tag" }, { name: "readOnly", }], - cacheSecs: 5, - }); + }, 5); return allAttributes.map((value) => { return { name: value.name, diff --git a/plugs/index/command.ts b/plugs/index/command.ts index 4fd8d98..8fba4bd 100644 --- a/plugs/index/command.ts +++ b/plugs/index/command.ts @@ -6,7 +6,7 @@ import { isTemplate } from "$sb/lib/cheap_yaml.ts"; export async function reindexCommand() { await editor.flashNotification("Performing full page reindex..."); - await system.invokeFunction("reindexSpace"); + await system.invokeFunction("index.reindexSpace"); await editor.flashNotification("Done with page index!"); } diff --git a/plugs/index/plug_api.ts b/plugs/index/plug_api.ts index c7adcfd..86a43bc 100644 --- a/plugs/index/plug_api.ts +++ b/plugs/index/plug_api.ts @@ -1,5 +1,6 @@ import { KV, KvQuery, ObjectQuery, ObjectValue } from "$sb/types.ts"; import { invokeFunction } from "$sb/silverbullet-syscall/system.ts"; +import { ttlCache } from "$sb/lib/memory_cache.ts"; export function indexObjects( page: string, @@ -21,8 +22,13 @@ export function query( export function queryObjects( tag: string, query: ObjectQuery, + ttlSecs?: number, ): Promise[]> { - return invokeFunction("index.queryObjects", tag, query); + return ttlCache( + query, + () => invokeFunction("index.queryObjects", tag, query), + ttlSecs, // no-op when undefined + ); } export function getObjectByRef( diff --git a/plugs/index/tags.ts b/plugs/index/tags.ts index 5062f7b..3f22d69 100644 --- a/plugs/index/tags.ts +++ b/plugs/index/tags.ts @@ -73,8 +73,7 @@ export async function tagComplete(completeEvent: CompleteEvent) { filter: ["=", ["attr", "parent"], ["string", parent]], select: [{ name: "name" }], distinct: true, - cacheSecs: 5, - }); + }, 5); if (parent === "page") { // Also add template, even though that would otherwise not appear because has "builtin" as a parent diff --git a/plugs/markdown/markdown_render.test.ts b/plugs/markdown/markdown_render.test.ts index ead97a5..4866ed8 100644 --- a/plugs/markdown/markdown_render.test.ts +++ b/plugs/markdown/markdown_render.test.ts @@ -9,16 +9,16 @@ import { renderMarkdownToHtml } from "./markdown_render.ts"; Deno.test("Markdown render", async () => { const system = new System("server"); await system.load( - new URL("../../dist_plug_bundle/_plug/editor.plug.js", import.meta.url), "editor", - 0, - createSandbox, + createSandbox( + new URL("../../dist_plug_bundle/_plug/editor.plug.js", import.meta.url), + ), ); await system.load( - new URL("../../dist_plug_bundle/_plug/tasks.plug.js", import.meta.url), "tasks", - 0, - createSandbox, + createSandbox( + new URL("../../dist_plug_bundle/_plug/tasks.plug.js", import.meta.url), + ), ); const lang = buildMarkdown(loadMarkdownExtensions(system)); const testFile = Deno.readTextFileSync( diff --git a/plugs/markdown/preview.ts b/plugs/markdown/preview.ts index 556b6a9..1452b3a 100644 --- a/plugs/markdown/preview.ts +++ b/plugs/markdown/preview.ts @@ -11,8 +11,8 @@ export async function updateMarkdownPreview() { const text = await editor.getText(); const mdTree = await markdown.parseMarkdown(text); // const cleanMd = await cleanMarkdown(text); - const css = await asset.readAsset("assets/preview.css"); - const js = await asset.readAsset("assets/preview.js"); + const css = await asset.readAsset("markdown", "assets/preview.css"); + const js = await asset.readAsset("markdown", "assets/preview.js"); await expandCodeWidgets(mdTree, currentPage); const html = renderMarkdownToHtml(mdTree, { diff --git a/plugs/tasks/complete.ts b/plugs/tasks/complete.ts index fd47d8f..08c6076 100644 --- a/plugs/tasks/complete.ts +++ b/plugs/tasks/complete.ts @@ -9,9 +9,7 @@ export async function completeTaskState(completeEvent: CompleteEvent) { if (!taskMatch) { return null; } - const allStates = await queryObjects("taskstate", { - cacheSecs: 5, - }); + const allStates = await queryObjects("taskstate", {}, 5); const states = [...new Set(allStates.map((s) => s.state))]; return { diff --git a/plugs/template/complete.ts b/plugs/template/complete.ts index 571795d..fca95e6 100644 --- a/plugs/template/complete.ts +++ b/plugs/template/complete.ts @@ -56,8 +56,7 @@ export async function templateSlashComplete( "boolean", false, ]]], - cacheSecs: 5, - }); + }, 5); return allTemplates.map((template) => ({ label: template.trigger!, detail: "template", diff --git a/server/crypto.ts b/server/crypto.ts index 4c370f5..51f07d5 100644 --- a/server/crypto.ts +++ b/server/crypto.ts @@ -17,6 +17,7 @@ export class JWTIssuer { async init(authString: string) { const [secret] = await this.kv.batchGet([[jwtSecretKey]]); if (!secret) { + console.log("Generating new JWT secret key"); return this.generateNewKey(); } else { this.key = await crypto.subtle.importKey( @@ -34,6 +35,9 @@ export class JWTIssuer { ]]); const newAuthHash = await this.hashSHA256(authString); if (currentAuthHash && currentAuthHash !== newAuthHash) { + console.log( + "Authentication has changed since last run, so invalidating all existing tokens", + ); // It has, so we need to generate a new key to invalidate all existing tokens await this.generateNewKey(); } diff --git a/server/http_server.ts b/server/http_server.ts index 80e8e56..81a0195 100644 --- a/server/http_server.ts +++ b/server/http_server.ts @@ -344,6 +344,16 @@ export class HttpServer { }), ); + // For when the day comes... + // this.app.use("*", async (c, next) => { + // // if (["POST", "PUT", "DELETE"].includes(c.req.method)) { + // const spaceServer = await this.ensureSpaceServer(c.req); + // return runWithSystemLock(spaceServer.system!, async () => { + // await next(); + // }); + // // } + // }); + // File list this.app.get( "/index.json", @@ -382,10 +392,10 @@ export class HttpServer { }); // RPC syscall - this.app.post("/.rpc/:plug/:syscall", async (c) => { + this.app.post("/.rpc/:plugName/:syscall", async (c) => { const req = c.req; - const plugName = req.param("plug")!; const syscall = req.param("syscall")!; + const plugName = req.param("plugName")!; const spaceServer = await this.ensureSpaceServer(req); const body = await req.json(); try { @@ -394,11 +404,11 @@ export class HttpServer { } const args: string[] = body; try { - const plug = spaceServer.system!.loadedPlugs.get(plugName); - if (!plug) { - throw new Error(`Plug ${plugName} not found`); - } - const result = await plug.syscall(syscall, args); + const result = await spaceServer.system!.syscall( + { plug: plugName }, + syscall, + args, + ); return c.json({ result: result, }); diff --git a/server/server_system.ts b/server/server_system.ts index 0a48ce4..28d97d1 100644 --- a/server/server_system.ts +++ b/server/server_system.ts @@ -35,6 +35,20 @@ import { KvPrimitives } from "../plugos/lib/kv_primitives.ts"; import { ShellBackend } from "./shell_backend.ts"; import { ensureSpaceIndex } from "../common/space_index.ts"; +// // Important: load this before the actual plugs +// import { +// createSandbox as noSandboxFactory, +// runWithSystemLock, +// } from "../plugos/sandboxes/no_sandbox.ts"; + +// // Load list of builtin plugs +// import { plug as plugIndex } from "../dist_plug_bundle/_plug/index.plug.js"; +// import { plug as plugFederation } from "../dist_plug_bundle/_plug/federation.plug.js"; +// import { plug as plugQuery } from "../dist_plug_bundle/_plug/query.plug.js"; +// import { plug as plugSearch } from "../dist_plug_bundle/_plug/search.plug.js"; +// import { plug as plugTasks } from "../dist_plug_bundle/_plug/tasks.plug.js"; +// import { plug as plugTemplate } from "../dist_plug_bundle/_plug/template.plug.js"; + const fileListInterval = 30 * 1000; // 30s const plugNameExtractRegex = /([^/]+)\.plug\.js$/; @@ -138,28 +152,29 @@ export class ServerSystem { ); this.listInterval = setInterval(() => { + // runWithSystemLock(this.system, async () => { + // await space.updatePageList(); + // }); space.updatePageList().catch(console.error); }, fileListInterval); - eventHook.addLocalListener("file:changed", (path, localChange) => { - (async () => { - if (!localChange && path.endsWith(".md")) { - const pageName = path.slice(0, -3); - const data = await this.spacePrimitives.readFile(path); - console.log("Outside page change: reindexing", pageName); - // Change made outside of editor, trigger reindex - await eventHook.dispatchEvent("page:index_text", { - name: pageName, - text: new TextDecoder().decode(data.data), - }); - } + eventHook.addLocalListener("file:changed", async (path, localChange) => { + if (!localChange && path.endsWith(".md")) { + const pageName = path.slice(0, -3); + const data = await this.spacePrimitives.readFile(path); + console.log("Outside page change: reindexing", pageName); + // Change made outside of editor, trigger reindex + await eventHook.dispatchEvent("page:index_text", { + name: pageName, + text: new TextDecoder().decode(data.data), + }); + } - if (path.startsWith("_plug/") && path.endsWith(".plug.js")) { - console.log("Plug updated, reloading:", path); - this.system.unload(path); - await this.loadPlugFromSpace(path); - } - })().catch(console.error); + if (path.startsWith("_plug/") && path.endsWith(".plug.js")) { + console.log("Plug updated, reloading:", path); + this.system.unload(path); + await this.loadPlugFromSpace(path); + } }); // Ensure a valid index @@ -168,10 +183,19 @@ export class ServerSystem { await indexPromise; } + // await runWithSystemLock(this.system, async () => { await eventHook.dispatchEvent("system:ready"); + // }); } async loadPlugs() { + // await this.system.load("index", noSandboxFactory(plugIndex)); + // await this.system.load("federation", noSandboxFactory(plugFederation)); + // await this.system.load("query", noSandboxFactory(plugQuery)); + // await this.system.load("search", noSandboxFactory(plugSearch)); + // await this.system.load("tasks", noSandboxFactory(plugTasks)); + // await this.system.load("template", noSandboxFactory(plugTemplate)); + for (const { name } of await this.spacePrimitives.fetchFileList()) { if (plugNameExtractRegex.test(name)) { await this.loadPlugFromSpace(name); @@ -183,11 +207,12 @@ export class ServerSystem { const { meta, data } = await this.spacePrimitives.readFile(path); const plugName = path.match(plugNameExtractRegex)![1]; return this.system.load( - // Base64 encoding this to support `deno compile` mode - new URL(base64EncodedDataUrl("application/javascript", data)), plugName, + createSandbox( + // Base64 encoding this to support `deno compile` mode + new URL(base64EncodedDataUrl("application/javascript", data)), + ), meta.lastModified, - createSandbox, ); } diff --git a/server/syscalls/shell.ts b/server/syscalls/shell.ts index e504db1..7fe8c1b 100644 --- a/server/syscalls/shell.ts +++ b/server/syscalls/shell.ts @@ -1,4 +1,3 @@ -import { shell } from "$sb/syscalls.ts"; import { SysCallMapping } from "../../plugos/system.ts"; import { ShellResponse } from "../../server/rpc.ts"; import { ShellBackend } from "../shell_backend.ts"; diff --git a/server/syscalls/space.ts b/server/syscalls/space.ts index 17660f5..9781809 100644 --- a/server/syscalls/space.ts +++ b/server/syscalls/space.ts @@ -10,10 +10,7 @@ export function spaceSyscalls(space: Space): SysCallMapping { "space.listPages": (): Promise => { return space.fetchPageList(); }, - "space.readPage": async ( - _ctx, - name: string, - ): Promise => { + "space.readPage": async (_ctx, name: string): Promise => { return (await space.readPage(name)).text; }, "space.getPageMeta": (_ctx, name: string): Promise => { @@ -35,10 +32,7 @@ export function spaceSyscalls(space: Space): SysCallMapping { "space.listAttachments": async (): Promise => { return await space.fetchAttachmentList(); }, - "space.readAttachment": async ( - _ctx, - name: string, - ): Promise => { + "space.readAttachment": async (_ctx, name: string): Promise => { return (await space.readAttachment(name)).data; }, "space.getAttachmentMeta": async ( diff --git a/web/client.ts b/web/client.ts index 318175b..a2b062e 100644 --- a/web/client.ts +++ b/web/client.ts @@ -46,11 +46,12 @@ import { DataStoreSpacePrimitives } from "../common/spaces/datastore_space_primi import { EncryptedSpacePrimitives, } from "../common/spaces/encrypted_space_primitives.ts"; -import { LimitedMap } from "../common/limited_map.ts"; + import { ensureSpaceIndex, markFullSpaceIndexComplete, } from "../common/space_index.ts"; +import { LimitedMap } from "$sb/lib/limited_map.ts"; const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/; const autoSaveInterval = 1000; @@ -135,7 +136,7 @@ export class Client { `${this.dbPrefix}_state`, ); await stateKvPrimitives.init(); - this.stateDataStore = new DataStore(stateKvPrimitives, true); + this.stateDataStore = new DataStore(stateKvPrimitives); // Setup message queue this.mq = new DataStoreMQ(this.stateDataStore); @@ -316,9 +317,13 @@ export class Client { // We're going to look up the anchor through a API invocation const matchingAnchor = await this.system.system.localSyscall( - "index", "system.invokeFunction", - ["getObjectByRef", pageName, "anchor", `${pageName}$${pos}`], + [ + "index.getObjectByRef", + pageName, + "anchor", + `${pageName}$${pos}`, + ], ); if (!matchingAnchor) { diff --git a/web/client_system.ts b/web/client_system.ts index ebd200b..9b22f6d 100644 --- a/web/client_system.ts +++ b/web/client_system.ts @@ -133,10 +133,9 @@ export class ClientSystem { console.log("Plug updated, reloading", plugName, "from", path); this.system.unload(path); const plug = await this.system.load( - new URL(`/${path}`, location.href), plugName, + createSandbox(new URL(`/${path}`, location.href)), newHash, - createSandbox, ); if ((plug.manifest! as Manifest).syntax) { // If there are syntax extensions, rebuild the markdown parser immediately @@ -201,10 +200,9 @@ export class ClientSystem { try { const plugName = plugNameExtractRegex.exec(plugMeta.name)![1]; await this.system.load( - new URL(plugMeta.name, location.origin), plugName, + createSandbox(new URL(plugMeta.name, location.origin)), plugMeta.lastModified, - createSandbox, ); } catch (e: any) { console.error( @@ -228,14 +226,13 @@ export class ClientSystem { } localSyscall(name: string, args: any[]) { - return this.system.localSyscall("editor", name, args); + return this.system.localSyscall(name, args); } queryObjects(tag: string, query: Query): Promise { - return this.system.localSyscall( - "index", + return this.localSyscall( "system.invokeFunction", - ["queryObjects", tag, query], + ["index.queryObjects", tag, query], ); } } diff --git a/web/cm_plugins/markdown_widget.ts b/web/cm_plugins/markdown_widget.ts index 3f579a5..0831a06 100644 --- a/web/cm_plugins/markdown_widget.ts +++ b/web/cm_plugins/markdown_widget.ts @@ -132,7 +132,7 @@ export class MarkdownWidget extends WidgetType { buttons.filter((button) => !button.widgetTarget).map((button, idx) => ` ` ).join("") - }${html}`; + }
${html}
`; } private attachListeners(div: HTMLElement, buttons?: CodeWidgetButton[]) { diff --git a/web/components/basic_modals.tsx b/web/components/basic_modals.tsx index b33c0e3..bd4aea3 100644 --- a/web/components/basic_modals.tsx +++ b/web/components/basic_modals.tsx @@ -19,43 +19,41 @@ export function Prompt({ }) { const [text, setText] = useState(defaultValue || ""); const returnEl = ( -
-
-
- - { - callback(text); - return true; - }} - onEscape={() => { - callback(); - }} - onChange={(text) => { - setText(text); - }} - /> - - -
+
+
+ + { + callback(text); + return true; + }} + onEscape={() => { + callback(); + }} + onChange={(text) => { + setText(text); + }} + /> + +
); diff --git a/web/components/filter.tsx b/web/components/filter.tsx index 7eee85e..4394068 100644 --- a/web/components/filter.tsx +++ b/web/components/filter.tsx @@ -95,115 +95,115 @@ export function FilterList({ }, []); const returnEl = ( -
-
-
{ - // Allow tapping/clicking the header without closing it - e.stopPropagation(); +
+
{ + // Allow tapping/clicking the header without closing it + e.stopPropagation(); + }} + > + + { + onSelect( + shiftDown ? { name: text } : matchingOptions[selectedOption], + ); + return true; }} - > - - { - onSelect( - shiftDown ? { name: text } : matchingOptions[selectedOption], - ); - return true; - }} - onEscape={() => { - onSelect(undefined); - }} - onChange={(text) => { - setText(text); - }} - onKeyUp={(view, e) => { - // This event is triggered after the key has been processed by CM already - if (onKeyPress) { - onKeyPress(e.key, view.state.sliceDoc()); - } - return false; - }} - onKeyDown={(view, e) => { - switch (e.key) { - case "ArrowUp": - setSelectionOption(Math.max(0, selectedOption - 1)); + onEscape={() => { + onSelect(undefined); + }} + onChange={(text) => { + setText(text); + }} + onKeyUp={(view, e) => { + // This event is triggered after the key has been processed by CM already + if (onKeyPress) { + onKeyPress(e.key, view.state.sliceDoc()); + } + return false; + }} + onKeyDown={(view, e) => { + switch (e.key) { + case "ArrowUp": + setSelectionOption(Math.max(0, selectedOption - 1)); + return true; + case "ArrowDown": + setSelectionOption( + Math.min(matchingOptions.length - 1, selectedOption + 1), + ); + return true; + case "PageUp": + setSelectionOption(Math.max(0, selectedOption - 5)); + return true; + case "PageDown": + setSelectionOption(Math.max(0, selectedOption + 5)); + return true; + case "Home": + setSelectionOption(0); + return true; + case "End": + setSelectionOption(matchingOptions.length - 1); + return true; + case " ": { + const text = view.state.sliceDoc(); + if (completePrefix && text === "") { + setText(completePrefix); + // updateFilter(completePrefix); return true; - case "ArrowDown": - setSelectionOption( - Math.min(matchingOptions.length - 1, selectedOption + 1), - ); - return true; - case "PageUp": - setSelectionOption(Math.max(0, selectedOption - 5)); - return true; - case "PageDown": - setSelectionOption(Math.max(0, selectedOption + 5)); - return true; - case "Home": - setSelectionOption(0); - return true; - case "End": - setSelectionOption(matchingOptions.length - 1); - return true; - case " ": { - const text = view.state.sliceDoc(); - if (completePrefix && text === "") { - setText(completePrefix); - // updateFilter(completePrefix); - return true; - } - break; } + break; } - return false; - }} - /> -
-
-
-
- {matchingOptions && matchingOptions.length > 0 - ? matchingOptions.map((option, idx) => ( -
{ + } + return false; + }} + /> +
+
+
+
+ {matchingOptions && matchingOptions.length > 0 + ? matchingOptions.map((option, idx) => ( +
{ + if (selectedOption !== idx) { setSelectionOption(idx); - }} - onClick={(e) => { - e.stopPropagation(); - onSelect(option); - }} - > - {Icon && ( - - - - )} - - {option.name} + } + }} + onClick={(e) => { + e.stopPropagation(); + onSelect(option); + }} + > + {Icon && ( + + - {option.hint && {option.hint}} -
{option.description}
-
- )) - : null} -
+ )} + + {option.name} + + {option.hint && {option.hint}} +
{option.description}
+
+ )) + : null}
); diff --git a/web/components/fuse_search.ts b/web/components/fuse_search.ts index 14dca55..6a79627 100644 --- a/web/components/fuse_search.ts +++ b/web/components/fuse_search.ts @@ -28,17 +28,18 @@ export const fuzzySearchAndSort = ( weight: 0.3, }, { name: "baseName", - weight: 0.7, + weight: 1, }, { name: "displayName", - weight: 0.3, + weight: 0.7, }, { name: "aliases", - weight: 0.7, + weight: 0.5, }], includeScore: true, shouldSort: true, isCaseSensitive: false, + ignoreLocation: true, threshold: 0.6, sortFn: (a, b): number => { if (a.score === b.score) { diff --git a/web/components/top_bar.tsx b/web/components/top_bar.tsx index 83b353b..ea33641 100644 --- a/web/components/top_bar.tsx +++ b/web/components/top_bar.tsx @@ -142,8 +142,8 @@ export function TopBar({