From 9ee9008bf24f9c50f15e50fa9d4fc86423caa387 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Sat, 26 Aug 2023 08:31:51 +0200 Subject: [PATCH] Work on #508 (thin client) --- cli/plug_run.ts | 159 +++--------------- cli/syscalls/index.test.ts | 1 - cli/syscalls/index.ts | 28 +++- cmd/server.ts | 51 +++++- deno.jsonc | 14 +- plug-api/plugos-syscall/fetch.ts | 65 ++++++-- plug-api/silverbullet-syscall/clientStore.ts | 13 ++ plug-api/silverbullet-syscall/mod.ts | 3 +- plugos/compile.ts | 6 +- plugos/lib/kv_store.deno_kv.test.ts | 27 ++++ plugos/lib/kv_store.deno_kv.ts | 102 +++++++----- plugs/core/editor.ts | 11 +- plugs/core/page.ts | 3 +- plugs/markdown/markdown.plug.yaml | 3 +- plugs/markdown/markdown.ts | 6 +- plugs/markdown/preview.ts | 6 +- plugs/search/search.plug.yaml | 8 +- server/http_server.ts | 82 ++++++---- server/rpc.ts | 21 +++ server/server_system.ts | 160 +++++++++++++++++++ silverbullet.ts | 12 +- web/boot.ts | 14 +- web/client.ts | 77 +++++---- web/client_system.ts | 48 +++--- web/index.html | 2 + web/sync_service.ts | 44 +++++ web/syscalls/clientStore.ts | 11 +- web/syscalls/index.proxy.ts | 16 ++ web/syscalls/store.proxy.ts | 18 +++ web/syscalls/util.ts | 33 ++++ 30 files changed, 717 insertions(+), 327 deletions(-) create mode 100644 plug-api/silverbullet-syscall/clientStore.ts create mode 100644 server/rpc.ts create mode 100644 server/server_system.ts create mode 100644 web/syscalls/index.proxy.ts create mode 100644 web/syscalls/store.proxy.ts create mode 100644 web/syscalls/util.ts diff --git a/cli/plug_run.ts b/cli/plug_run.ts index 7154328..36cf69a 100644 --- a/cli/plug_run.ts +++ b/cli/plug_run.ts @@ -1,38 +1,11 @@ import { path } from "../common/deps.ts"; -import { PlugNamespaceHook } from "../common/hooks/plug_namespace.ts"; -import { SilverBulletHooks } from "../common/manifest.ts"; -import { loadMarkdownExtensions } from "../common/markdown_parser/markdown_ext.ts"; -import buildMarkdown from "../common/markdown_parser/parser.ts"; import { DiskSpacePrimitives } from "../common/spaces/disk_space_primitives.ts"; -import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives.ts"; -import { FileMetaSpacePrimitives } from "../common/spaces/file_meta_space_primitives.ts"; -import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts"; import { AssetBundle } from "../plugos/asset_bundle/bundle.ts"; -import { createSandbox } from "../plugos/environments/deno_sandbox.ts"; -import { CronHook } from "../plugos/hooks/cron.ts"; -import { EventHook } from "../plugos/hooks/event.ts"; -import { MQHook } from "../plugos/hooks/mq.ts"; -import { DenoKVStore } from "../plugos/lib/kv_store.deno_kv.ts"; -import { DexieMQ } from "../plugos/lib/mq.dexie.ts"; -import assetSyscalls from "../plugos/syscalls/asset.ts"; -import { eventSyscalls } from "../plugos/syscalls/event.ts"; -import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts"; -import { mqSyscalls } from "../plugos/syscalls/mq.dexie.ts"; -import { shellSyscalls } from "../plugos/syscalls/shell.deno.ts"; -import { storeSyscalls } from "../plugos/syscalls/store.ts"; -import { System } from "../plugos/system.ts"; -import { Space } from "../web/space.ts"; -import { debugSyscalls } from "../web/syscalls/debug.ts"; -import { pageIndexSyscalls } from "./syscalls/index.ts"; -import { markdownSyscalls } from "../web/syscalls/markdown.ts"; -import { systemSyscalls } from "../web/syscalls/system.ts"; -import { yamlSyscalls } from "../web/syscalls/yaml.ts"; -import { spaceSyscalls } from "./syscalls/space.ts"; -import { IDBKeyRange, indexedDB } from "https://esm.sh/fake-indexeddb@4.0.2"; import { Application } from "../server/deps.ts"; -import { EndpointHook } from "../plugos/hooks/endpoint.ts"; import { sleep } from "../common/async_util.ts"; +import { ServerSystem } from "../server/server_system.ts"; +import { AssetBundlePlugSpacePrimitives } from "../common/spaces/asset_bundle_space_primitives.ts"; export async function runPlug( spacePath: string, @@ -44,108 +17,44 @@ export async function runPlug( httpHostname = "127.0.0.1", ) { spacePath = path.resolve(spacePath); - const system = new System("cli"); - - // Event hook - const eventHook = new EventHook(); - system.addHook(eventHook); - - // Cron hook - const cronHook = new CronHook(system); - system.addHook(cronHook); - - const kvStore = new DenoKVStore(); const tempFile = Deno.makeTempFileSync({ suffix: ".db" }); - await kvStore.init(tempFile); - - // Endpoint hook - const app = new Application(); - system.addHook(new EndpointHook(app, "/_")); + console.log("Tempt db file", tempFile); const serverController = new AbortController(); + const app = new Application(); + + const serverSystem = new ServerSystem( + new AssetBundlePlugSpacePrimitives( + new DiskSpacePrimitives(spacePath), + builtinAssetBundle, + ), + tempFile, + app, + ); + await serverSystem.init(); app.listen({ hostname: httpHostname, port: httpServerPort, signal: serverController.signal, }); - // Use DexieMQ for this, in memory - const mq = new DexieMQ("mq", indexedDB, IDBKeyRange); - - const pageIndexCalls = pageIndexSyscalls(kvStore); - - const plugNamespaceHook = new PlugNamespaceHook(); - system.addHook(plugNamespaceHook); - - system.addHook(new MQHook(system, mq)); - - const spacePrimitives = new FileMetaSpacePrimitives( - new EventedSpacePrimitives( - new PlugSpacePrimitives( - new DiskSpacePrimitives(spacePath), - plugNamespaceHook, - ), - eventHook, - ), - pageIndexCalls, - ); - const space = new Space(spacePrimitives, kvStore); - - // Add syscalls - system.registerSyscalls( - [], - eventSyscalls(eventHook), - spaceSyscalls(space), - assetSyscalls(system), - yamlSyscalls(), - storeSyscalls(kvStore), - systemSyscalls(undefined as any, system), - mqSyscalls(mq), - pageIndexCalls, - debugSyscalls(), - markdownSyscalls(buildMarkdown([])), // Will later be replaced with markdown extensions - ); - - // Syscalls that require some additional permissions - system.registerSyscalls( - ["fetch"], - sandboxFetchSyscalls(), - ); - - system.registerSyscalls( - ["shell"], - shellSyscalls("."), - ); - - await loadPlugsFromAssetBundle(system, builtinAssetBundle); - - for (let plugPath of await space.listPlugs()) { - plugPath = path.resolve(spacePath, plugPath); - await system.load( - new URL(`file://${plugPath}`), - createSandbox, - ); - } - - // Load markdown syscalls based on all new syntax (if any) - system.registerSyscalls( - [], - markdownSyscalls(buildMarkdown(loadMarkdownExtensions(system))), - ); - if (indexFirst) { - await system.loadedPlugs.get("core")!.invoke("reindexSpace", []); + await serverSystem.system.loadedPlugs.get("core")!.invoke( + "reindexSpace", + [], + ); } if (functionName) { const [plugName, funcName] = functionName.split("."); - const plug = system.loadedPlugs.get(plugName); + const plug = serverSystem.system.loadedPlugs.get(plugName); if (!plug) { throw new Error(`Plug ${plugName} not found`); } const result = await plug.invoke(funcName, args); - await system.unloadAll(); - await kvStore.delete(); + await serverSystem.close(); + await serverSystem.kvStore?.delete(); + // await Deno.remove(tempFile); serverController.abort(); return result; } else { @@ -155,27 +64,3 @@ export async function runPlug( } } } - -async function loadPlugsFromAssetBundle( - system: System, - assetBundle: AssetBundle, -) { - const tempDir = await Deno.makeTempDir(); - try { - for (const filePath of assetBundle.listFiles()) { - if ( - filePath.endsWith(".plug.js") // && !filePath.includes("search.plug.js") - ) { - const plugPath = path.join(tempDir, filePath); - await Deno.mkdir(path.dirname(plugPath), { recursive: true }); - await Deno.writeFile(plugPath, assetBundle.readFileSync(filePath)); - await system.load( - new URL(`file://${plugPath}`), - createSandbox, - ); - } - } - } finally { - await Deno.remove(tempDir, { recursive: true }); - } -} diff --git a/cli/syscalls/index.test.ts b/cli/syscalls/index.test.ts index f26c92f..8d3e7bd 100644 --- a/cli/syscalls/index.test.ts +++ b/cli/syscalls/index.test.ts @@ -27,7 +27,6 @@ Deno.test("Test KV index", async () => { }, { key: "random", value: "value3" }]); let results = await calls["index.queryPrefix"](ctx, "attr:"); assertEquals(results.length, 4); - console.log("here"); await calls["index.clearPageIndexForPage"](ctx, "page"); results = await calls["index.queryPrefix"](ctx, "attr:"); assertEquals(results.length, 2); diff --git a/cli/syscalls/index.ts b/cli/syscalls/index.ts index 945fe97..2cb9839 100644 --- a/cli/syscalls/index.ts +++ b/cli/syscalls/index.ts @@ -30,10 +30,18 @@ export function pageIndexSyscalls(kv: KVStore): SysCallMapping { }], ); }, - "index.batchSet": async (_ctx, page: string, kvs: KV[]) => { + "index.batchSet": (_ctx, page: string, kvs: KV[]) => { + const batch: KV[] = []; for (const { key, value } of kvs) { - await apiObj["index.set"](_ctx, page, key, value); + batch.push({ + key: `index${sep}${page}${sep}${key}`, + value, + }, { + key: `indexByKey${sep}${key}${sep}${page}`, + value, + }); } + return kv.batchSet(batch); }, "index.delete": (_ctx, page: string, key: string) => { return kv.batchDelete([ @@ -62,20 +70,30 @@ export function pageIndexSyscalls(kv: KVStore): SysCallMapping { await apiObj["index.deletePrefixForPage"](ctx, page, ""); }, "index.deletePrefixForPage": async (_ctx, page: string, prefix: string) => { + const allKeys: string[] = []; for ( const result of await kv.queryPrefix( `index${sep}${page}${sep}${prefix}`, ) ) { const [_ns, page, key] = result.key.split(sep); - await apiObj["index.delete"](_ctx, page, key); + allKeys.push( + `index${sep}${page}${sep}${key}`, + `indexByKey${sep}${key}${sep}${page}`, + ); } + return kv.batchDelete(allKeys); }, - "index.clearPageIndex": async (ctx) => { + "index.clearPageIndex": async () => { + const allKeys: string[] = []; for (const result of await kv.queryPrefix(`index${sep}`)) { const [_ns, page, key] = result.key.split(sep); - await apiObj["index.delete"](ctx, page, key); + allKeys.push( + `index${sep}${page}${sep}${key}`, + `indexByKey${sep}${key}${sep}${page}`, + ); } + return kv.batchDelete(allKeys); }, }; return apiObj; diff --git a/cmd/server.ts b/cmd/server.ts index 0771e0f..0019c06 100644 --- a/cmd/server.ts +++ b/cmd/server.ts @@ -1,4 +1,4 @@ -import { path } from "../server/deps.ts"; +import { Application, path } from "../server/deps.ts"; import { HttpServer } from "../server/http_server.ts"; import clientAssetBundle from "../dist/client_asset_bundle.json" assert { type: "json", @@ -14,16 +14,34 @@ import { S3SpacePrimitives } from "../server/spaces/s3_space_primitives.ts"; import { Authenticator } from "../server/auth.ts"; import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts"; import { sleep } from "../common/async_util.ts"; +import { ServerSystem } from "../server/server_system.ts"; +import { SilverBulletHooks } from "../common/manifest.ts"; +import { System } from "../plugos/system.ts"; export async function serveCommand( - options: any, + options: { + hostname?: string; + port?: number; + user?: string; + auth?: string; + cert?: string; + key?: string; + // Thin client mode + thinClient?: boolean; + reindex?: boolean; + db?: string; + }, folder?: string, ) { const hostname = options.hostname || Deno.env.get("SB_HOSTNAME") || "127.0.0.1"; const port = options.port || (Deno.env.get("SB_PORT") && +Deno.env.get("SB_PORT")!) || 3000; - const maxFileSizeMB = options.maxFileSizeMB || 20; + + const thinClientMode = options.thinClient || Deno.env.has("SB_THIN_CLIENT"); + let dbFile = options.db || Deno.env.get("SB_DB_FILE") || ".silverbullet.db"; + + const app = new Application(); if (!folder) { folder = Deno.env.get("SB_FOLDER"); @@ -55,16 +73,35 @@ To allow outside connections, pass -L 0.0.0.0 as a flag, and put a TLS terminato bucket: Deno.env.get("AWS_BUCKET")!, }); console.log("Running in S3 mode"); + folder = Deno.cwd(); } else { // Regular disk mode folder = path.resolve(Deno.cwd(), folder); spacePrimitives = new DiskSpacePrimitives(folder); } + spacePrimitives = new AssetBundlePlugSpacePrimitives( spacePrimitives, new AssetBundle(plugAssetBundle as AssetJson), ); + let system: System | undefined; + if (thinClientMode) { + dbFile = path.resolve(folder, dbFile); + console.log(`Running in thin client mode, keeping state in ${dbFile}`); + const serverSystem = new ServerSystem(spacePrimitives, dbFile, app); + await serverSystem.init(); + spacePrimitives = serverSystem.spacePrimitives; + system = serverSystem.system; + if (options.reindex) { + console.log("Reindexing space (requested via --reindex flag)"); + await serverSystem.system.loadedPlugs.get("core")!.invoke( + "reindexSpace", + [], + ); + } + } + const authStore = new JSONKVStore(); const authenticator = new Authenticator(authStore); @@ -82,7 +119,7 @@ To allow outside connections, pass -L 0.0.0.0 as a flag, and put a TLS terminato await authStore.load(authFile); (async () => { // Asynchronously kick off file watcher - for await (const _event of Deno.watchFs(options.auth)) { + for await (const _event of Deno.watchFs(options.auth!)) { console.log("Authentication file changed, reloading..."); await authStore.load(authFile); } @@ -95,7 +132,7 @@ To allow outside connections, pass -L 0.0.0.0 as a flag, and put a TLS terminato authStore.loadString(envAuth); } - const httpServer = new HttpServer(spacePrimitives!, { + const httpServer = new HttpServer(spacePrimitives!, app, system, { hostname, port: port, pagesPath: folder!, @@ -103,11 +140,11 @@ To allow outside connections, pass -L 0.0.0.0 as a flag, and put a TLS terminato authenticator, keyFile: options.key, certFile: options.cert, - maxFileSizeMB: +maxFileSizeMB, }); await httpServer.start(); + // Wait in an infinite loop (to keep the HTTP server running, only cancelable via Ctrl+C or other signal) while (true) { - await sleep(1000); + await sleep(10000); } } diff --git a/deno.jsonc b/deno.jsonc index 53dd354..b54a255 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -7,10 +7,10 @@ "test": "deno test -A --unstable", "build": "deno run -A build_plugs.ts && deno run -A --unstable build_web.ts", "plugs": "deno run -A build_plugs.ts", - "server": "deno run -A --check silverbullet.ts", + "server": "deno run -A --unstable --check silverbullet.ts", "watch-web": "deno run -A --check build_web.ts --watch", - "watch-server": "deno run -A --check --watch silverbullet.ts", + "watch-server": "deno run -A --unstable --check --watch silverbullet.ts", "watch-plugs": "deno run -A --check build_plugs.ts -w", "bundle": "deno run -A build_bundle.ts", @@ -19,11 +19,11 @@ "generate": "lezer-generator common/markdown_parser/query.grammar -o common/markdown_parser/parse-query.js", // Compile - "compile": "deno task bundle && deno compile -A -o silverbullet dist/silverbullet.js", - "server:dist:linux-x86_64": "deno task bundle && deno compile -A --target x86_64-unknown-linux-gnu dist/silverbullet.js -o silverbullet && zip silverbullet-server-linux-x86_64.zip silverbullet", - "server:dist:darwin-x86_64": "deno task bundle && deno compile -A --target x86_64-apple-darwin dist/silverbullet.js -o silverbullet && zip silverbullet-server-darwin-x86_64.zip silverbullet", - "server:dist:darwin-aarch64": "deno task bundle && deno task bundle && deno compile -A --target aarch64-apple-darwin dist/silverbullet.js -o silverbullet && zip silverbullet-server-darwin-aarch64.zip silverbullet", - "server:dist:windows-x86_64": "deno compile -A --target x86_64-pc-windows-msvc dist/silverbullet.js -o silverbullet.exe && zip silverbullet-server-windows-x86_64.zip silverbullet.exe" + "compile": "deno task bundle && deno compile -A --unstable -o silverbullet dist/silverbullet.js", + "server:dist:linux-x86_64": "deno task bundle && deno compile -A --unstable --target x86_64-unknown-linux-gnu dist/silverbullet.js -o silverbullet && zip silverbullet-server-linux-x86_64.zip silverbullet", + "server:dist:darwin-x86_64": "deno task bundle && deno compile -A --unstable --target x86_64-apple-darwin dist/silverbullet.js -o silverbullet && zip silverbullet-server-darwin-x86_64.zip silverbullet", + "server:dist:darwin-aarch64": "deno task bundle && deno task bundle && deno compile -A --unstable --target aarch64-apple-darwin dist/silverbullet.js -o silverbullet && zip silverbullet-server-darwin-aarch64.zip silverbullet", + "server:dist:windows-x86_64": "deno task bundle && deno compile -A --unstable --target x86_64-pc-windows-msvc dist/silverbullet.js -o silverbullet.exe && zip silverbullet-server-windows-x86_64.zip silverbullet.exe" }, "compilerOptions": { diff --git a/plug-api/plugos-syscall/fetch.ts b/plug-api/plugos-syscall/fetch.ts index 6718f40..a53c936 100644 --- a/plug-api/plugos-syscall/fetch.ts +++ b/plug-api/plugos-syscall/fetch.ts @@ -1,4 +1,3 @@ -import { init } from "https://esm.sh/v131/node_events.js"; import type { ProxyFetchRequest, ProxyFetchResponse, @@ -8,21 +7,44 @@ import { base64Encode, } from "../../plugos/asset_bundle/base64.ts"; +async function readStream( + stream: ReadableStream, +): Promise { + const arrays: Uint8Array[] = []; + let totalRead = 0; + const reader = stream.getReader(); + while (true) { + // The `read()` method returns a promise that + // resolves when a value has been received. + const { done, value } = await reader.read(); + // Result objects contain two properties: + // `done` - `true` if the stream has already given you all its data. + // `value` - Some data. Always `undefined` when `done` is `true`. + if (done) { + const resultArray = new Uint8Array(totalRead); + let offset = 0; + for (const array of arrays) { + resultArray.set(array, offset); + offset += array.length; + } + return resultArray; + } + arrays.push(value); + totalRead += value.length; + } +} + export async function sandboxFetch( reqInfo: RequestInfo, options?: ProxyFetchRequest, ): Promise { if (typeof reqInfo !== "string") { - // Request as first argument, let's deconstruct it - // console.log("fetch", reqInfo); + const body = new Uint8Array(await reqInfo.arrayBuffer()); + const encodedBody = body.length > 0 ? base64Encode(body) : undefined; options = { method: reqInfo.method, headers: Object.fromEntries(reqInfo.headers.entries()), - base64Body: reqInfo.body - ? base64Encode( - new Uint8Array(await (new Response(reqInfo.body)).arrayBuffer()), - ) - : undefined, + base64Body: encodedBody, }; reqInfo = reqInfo.url; } @@ -30,6 +52,21 @@ export async function sandboxFetch( return syscall("sandboxFetch.fetch", reqInfo, options); } +async function bodyInitToUint8Array(init: BodyInit): Promise { + if (init instanceof Blob) { + const buffer = await init.arrayBuffer(); + return new Uint8Array(buffer); + } else if (init instanceof ArrayBuffer) { + return new Uint8Array(init); + } else if (init instanceof ReadableStream) { + return readStream(init); + } else if (typeof init === "string") { + return new TextEncoder().encode(init); + } else { + throw new Error("Unknown body init type"); + } +} + export function monkeyPatchFetch() { // @ts-ignore: monkey patching fetch globalThis.nativeFetch = globalThis.fetch; @@ -38,16 +75,18 @@ export function monkeyPatchFetch() { reqInfo: RequestInfo, init?: RequestInit, ): Promise { + const encodedBody = init && init.body + ? base64Encode( + new Uint8Array(await (new Response(init.body)).arrayBuffer()), + ) + : undefined; + // console.log("Encoded this body", encodedBody); const r = await sandboxFetch( reqInfo, init && { method: init.method, headers: init.headers as Record, - base64Body: init.body - ? base64Encode( - new Uint8Array(await (new Response(init.body)).arrayBuffer()), - ) - : undefined, + base64Body: encodedBody, }, ); return new Response(r.base64Body ? base64Decode(r.base64Body) : null, { diff --git a/plug-api/silverbullet-syscall/clientStore.ts b/plug-api/silverbullet-syscall/clientStore.ts new file mode 100644 index 0000000..eb26773 --- /dev/null +++ b/plug-api/silverbullet-syscall/clientStore.ts @@ -0,0 +1,13 @@ +import { syscall } from "./syscall.ts"; + +export function set(key: string, value: any): Promise { + return syscall("clientStore.set", key, value); +} + +export function get(key: string): Promise { + return syscall("clientStore.get", key); +} + +export function del(key: string): Promise { + return syscall("clientStore.delete", key); +} diff --git a/plug-api/silverbullet-syscall/mod.ts b/plug-api/silverbullet-syscall/mod.ts index 3d763a6..3520387 100644 --- a/plug-api/silverbullet-syscall/mod.ts +++ b/plug-api/silverbullet-syscall/mod.ts @@ -3,7 +3,6 @@ export * as index from "./index.ts"; export * as markdown from "./markdown.ts"; export * as space from "./space.ts"; export * as system from "./system.ts"; -// Legacy redirect, use "store" in $sb/plugos-syscall/mod.ts instead -export * as clientStore from "./store.ts"; +export * as clientStore from "./clientStore.ts"; export * as sync from "./sync.ts"; export * as debug from "./debug.ts"; diff --git a/plugos/compile.ts b/plugos/compile.ts index 6bd90a3..0a9acf3 100644 --- a/plugos/compile.ts +++ b/plugos/compile.ts @@ -4,9 +4,9 @@ import { bundleAssets } from "./asset_bundle/builder.ts"; import { Manifest } from "./types.ts"; import { version } from "../version.ts"; -// const workerRuntimeUrl = new URL("./worker_runtime.ts", import.meta.url); -const workerRuntimeUrl = - `https://deno.land/x/silverbullet@${version}/plugos/worker_runtime.ts`; +const workerRuntimeUrl = new URL("./worker_runtime.ts", import.meta.url); +// const workerRuntimeUrl = +// `https://deno.land/x/silverbullet@${version}/plugos/worker_runtime.ts`; export type CompileOptions = { debug?: boolean; diff --git a/plugos/lib/kv_store.deno_kv.test.ts b/plugos/lib/kv_store.deno_kv.test.ts index 7c00b9c..dc24326 100644 --- a/plugos/lib/kv_store.deno_kv.test.ts +++ b/plugos/lib/kv_store.deno_kv.test.ts @@ -15,6 +15,25 @@ Deno.test("Test KV index", async () => { { key: "page:hello2", value: "Hello 2" }, { key: "page:hello3", value: "Hello 3" }, { key: "something", value: "Something" }, + { key: "something1", value: "Something" }, + { key: "something2", value: "Something" }, + { key: "something3", value: "Something" }, + { key: "something4", value: "Something" }, + { key: "something5", value: "Something" }, + { key: "something6", value: "Something" }, + { key: "something7", value: "Something" }, + { key: "something8", value: "Something" }, + { key: "something9", value: "Something" }, + { key: "something10", value: "Something" }, + { key: "something11", value: "Something" }, + { key: "something12", value: "Something" }, + { key: "something13", value: "Something" }, + { key: "something14", value: "Something" }, + { key: "something15", value: "Something" }, + { key: "something16", value: "Something" }, + { key: "something17", value: "Something" }, + { key: "something18", value: "Something" }, + { key: "something19", value: "Something" }, ]); const results = await kv.queryPrefix("page:"); @@ -25,5 +44,13 @@ Deno.test("Test KV index", async () => { "Hello 3", ]); + await kv.deletePrefix("page:"); + + assertEquals(await kv.queryPrefix("page:"), []); + assertEquals((await kv.queryPrefix("")).length, 20); + + await kv.deletePrefix(""); + assertEquals(await kv.queryPrefix(""), []); + await kv.delete(); }); diff --git a/plugos/lib/kv_store.deno_kv.ts b/plugos/lib/kv_store.deno_kv.ts index ec1e882..58cc51e 100644 --- a/plugos/lib/kv_store.deno_kv.ts +++ b/plugos/lib/kv_store.deno_kv.ts @@ -2,6 +2,8 @@ import { KV, KVStore } from "./kv_store.ts"; +const kvBatchSize = 10; + export class DenoKVStore implements KVStore { kv!: Deno.Kv; path: string | undefined; @@ -22,55 +24,75 @@ export class DenoKVStore implements KVStore { } } - async del(key: string): Promise { - const res = await this.kv.atomic() - .delete([key]) - .commit(); - if (!res.ok) { - throw res; - } + del(key: string): Promise { + return this.batchDelete([key]); } async deletePrefix(prefix: string): Promise { + const allKeys: string[] = []; for await ( - const result of this.kv.list({ - start: [prefix], - end: [endRange(prefix)], - }) + const result of this.kv.list( + prefix + ? { + start: [prefix], + end: [endRange(prefix)], + } + : { prefix: [] }, + ) ) { - await this.del(result.key[0] as string); + allKeys.push(result.key[0] as string); } + return this.batchDelete(allKeys); } - async deleteAll(): Promise { - for await ( - const result of this.kv.list({ prefix: [] }) - ) { - await this.del(result.key[0] as string); - } + deleteAll(): Promise { + return this.deletePrefix(""); } - async set(key: string, value: any): Promise { - const res = await this.kv.atomic() - .set([key], value) - .commit(); - if (!res.ok) { - throw res; - } + set(key: string, value: any): Promise { + return this.batchSet([{ key, value }]); } async batchSet(kvs: KV[]): Promise { - for (const { key, value } of kvs) { - await this.set(key, value); + // Split into batches of kvBatchSize + const batches: KV[][] = []; + for (let i = 0; i < kvs.length; i += kvBatchSize) { + batches.push(kvs.slice(i, i + kvBatchSize)); + } + for (const batch of batches) { + let batchOp = this.kv.atomic(); + for (const { key, value } of batch) { + batchOp = batchOp.set([key], value); + } + const res = await batchOp.commit(); + if (!res.ok) { + throw res; + } } } async batchDelete(keys: string[]): Promise { - for (const key of keys) { - await this.del(key); + const batches: string[][] = []; + for (let i = 0; i < keys.length; i += kvBatchSize) { + batches.push(keys.slice(i, i + kvBatchSize)); + } + for (const batch of batches) { + let batchOp = this.kv.atomic(); + for (const key of batch) { + batchOp = batchOp.delete([key]); + } + const res = await batchOp.commit(); + if (!res.ok) { + throw res; + } } } - batchGet(keys: string[]): Promise { - const results: Promise[] = []; - for (const key of keys) { - results.push(this.get(key)); + async batchGet(keys: string[]): Promise { + const results: any[] = []; + const batches: Deno.KvKey[][] = []; + for (let i = 0; i < keys.length; i += kvBatchSize) { + batches.push(keys.slice(i, i + kvBatchSize).map((k) => [k])); } - return Promise.all(results); + for (const batch of batches) { + const res = await this.kv.getMany(batch); + results.push(...res.map((r) => r.value)); + } + return results; } async get(key: string): Promise { return (await this.kv.get([key])).value; @@ -81,10 +103,14 @@ export class DenoKVStore implements KVStore { async queryPrefix(keyPrefix: string): Promise<{ key: string; value: any }[]> { const results: { key: string; value: any }[] = []; for await ( - const result of (this.kv).list({ - start: [keyPrefix], - end: [endRange(keyPrefix)], - }) + const result of this.kv.list( + keyPrefix + ? { + start: [keyPrefix], + end: [endRange(keyPrefix)], + } + : { prefix: [] }, + ) ) { results.push({ key: result.key[0] as string, diff --git a/plugs/core/editor.ts b/plugs/core/editor.ts index 00ac5c4..865a2e1 100644 --- a/plugs/core/editor.ts +++ b/plugs/core/editor.ts @@ -1,21 +1,20 @@ -import { editor } from "$sb/silverbullet-syscall/mod.ts"; -import { store } from "$sb/plugos-syscall/mod.ts"; +import { clientStore, editor } from "$sb/silverbullet-syscall/mod.ts"; // Run on "editor:init" export async function setEditorMode() { - if (await store.get("vimMode")) { + if (await clientStore.get("vimMode")) { await editor.setUiOption("vimMode", true); } - if (await store.get("darkMode")) { + if (await clientStore.get("darkMode")) { await editor.setUiOption("darkMode", true); } } export async function toggleDarkMode() { - let darkMode = await store.get("darkMode"); + let darkMode = await clientStore.get("darkMode"); darkMode = !darkMode; await editor.setUiOption("darkMode", darkMode); - await store.set("darkMode", darkMode); + await clientStore.set("darkMode", darkMode); } export async function foldCommand() { diff --git a/plugs/core/page.ts b/plugs/core/page.ts index 3e2dcec..8504e8a 100644 --- a/plugs/core/page.ts +++ b/plugs/core/page.ts @@ -152,12 +152,13 @@ export async function reindexSpace() { } export async function processIndexQueue(messages: Message[]) { - // console.log("Processing batch of", messages.length, "pages to index"); for (const message of messages) { const name: string = message.body; console.log(`Indexing page ${name}`); const text = await space.readPage(name); + // console.log("Going to parse markdown"); const parsed = await markdown.parseMarkdown(text); + // console.log("Dispatching ;age:index"); await events.dispatchEvent("page:index", { name, tree: parsed, diff --git a/plugs/markdown/markdown.plug.yaml b/plugs/markdown/markdown.plug.yaml index 306a0a5..f24bf84 100644 --- a/plugs/markdown/markdown.plug.yaml +++ b/plugs/markdown/markdown.plug.yaml @@ -1,8 +1,6 @@ name: markdown assets: - "assets/*" -requiredPermissions: - - fs functions: toggle: path: "./markdown.ts:togglePreview" @@ -13,6 +11,7 @@ functions: preview: path: "./preview.ts:updateMarkdownPreview" + env: client events: - plug:load - editor:updated diff --git a/plugs/markdown/markdown.ts b/plugs/markdown/markdown.ts index a43e571..4151ae0 100644 --- a/plugs/markdown/markdown.ts +++ b/plugs/markdown/markdown.ts @@ -1,11 +1,11 @@ import { editor } from "$sb/silverbullet-syscall/mod.ts"; import { readSettings } from "$sb/lib/settings_page.ts"; import { updateMarkdownPreview } from "./preview.ts"; -import { store } from "$sb/plugos-syscall/mod.ts"; +import { clientStore } from "$sb/silverbullet-syscall/mod.ts"; export async function togglePreview() { - const currentValue = !!(await store.get("enableMarkdownPreview")); - await store.set("enableMarkdownPreview", !currentValue); + const currentValue = !!(await clientStore.get("enableMarkdownPreview")); + await clientStore.set("enableMarkdownPreview", !currentValue); if (!currentValue) { await updateMarkdownPreview(); } else { diff --git a/plugs/markdown/preview.ts b/plugs/markdown/preview.ts index 20b67dd..da81118 100644 --- a/plugs/markdown/preview.ts +++ b/plugs/markdown/preview.ts @@ -1,11 +1,11 @@ -import { editor, system } from "$sb/silverbullet-syscall/mod.ts"; -import { asset, store } from "$sb/plugos-syscall/mod.ts"; +import { clientStore, editor, system } from "$sb/silverbullet-syscall/mod.ts"; +import { asset } from "$sb/plugos-syscall/mod.ts"; import { parseMarkdown } from "$sb/silverbullet-syscall/markdown.ts"; import { renderMarkdownToHtml } from "./markdown_render.ts"; import { resolvePath } from "$sb/lib/resolve.ts"; export async function updateMarkdownPreview() { - if (!(await store.get("enableMarkdownPreview"))) { + if (!(await clientStore.get("enableMarkdownPreview"))) { return; } const currentPage = await editor.getCurrentPage(); diff --git a/plugs/search/search.plug.yaml b/plugs/search/search.plug.yaml index a5bdeb6..cc12f0e 100644 --- a/plugs/search/search.plug.yaml +++ b/plugs/search/search.plug.yaml @@ -1,9 +1,11 @@ name: search functions: indexPage: - path: search.ts:indexPage - events: - - page:index + path: search.ts:indexPage + # Only enable in client for now + env: client + events: + - page:index clearIndex: path: search.ts:clearIndex diff --git a/server/http_server.ts b/server/http_server.ts index b323005..b567784 100644 --- a/server/http_server.ts +++ b/server/http_server.ts @@ -2,12 +2,19 @@ import { Application, Context, Next, oakCors, Router } from "./deps.ts"; import { SpacePrimitives } from "../common/spaces/space_primitives.ts"; import { AssetBundle } from "../plugos/asset_bundle/bundle.ts"; import { ensureSettingsAndIndex } from "../common/util.ts"; -import { performLocalFetch } from "../common/proxy_fetch.ts"; import { BuiltinSettings } from "../web/types.ts"; import { gitIgnoreCompiler } from "./deps.ts"; import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts"; import { Authenticator } from "./auth.ts"; import { FileMeta } from "$sb/types.ts"; +import { + ShellRequest, + ShellResponse, + SyscallRequest, + SyscallResponse, +} from "./rpc.ts"; +import { SilverBulletHooks } from "../common/manifest.ts"; +import { System } from "../plugos/system.ts"; export type ServerOptions = { hostname: string; @@ -18,11 +25,9 @@ export type ServerOptions = { pass?: string; certFile?: string; keyFile?: string; - maxFileSizeMB?: number; }; export class HttpServer { - app: Application; private hostname: string; private port: number; abortController?: AbortController; @@ -33,27 +38,19 @@ export class HttpServer { constructor( spacePrimitives: SpacePrimitives, + private app: Application, + private system: System | undefined, private options: ServerOptions, ) { this.hostname = options.hostname; this.port = options.port; - this.app = new Application(); this.authenticator = options.authenticator; this.clientAssetBundle = options.clientAssetBundle; let fileFilterFn: (s: string) => boolean = () => true; this.spacePrimitives = new FilteredSpacePrimitives( spacePrimitives, - (meta) => { - // Don't list file exceeding the maximum file size - if ( - options.maxFileSizeMB && - meta.size / (1024 * 1024) > options.maxFileSizeMB - ) { - return false; - } - return fileFilterFn(meta.name); - }, + (meta) => fileFilterFn(meta.name), async () => { await this.reloadSettings(); if (typeof this.settings?.spaceIgnore === "string") { @@ -71,7 +68,7 @@ export class HttpServer { .replaceAll( "{{SPACE_PATH}}", this.options.pagesPath.replaceAll("\\", "\\\\"), - ); + ).replaceAll("{{THIN_CLIENT_MODE}}", this.system ? "on" : "off"); } async start() { @@ -282,13 +279,6 @@ export class HttpServer { const body = await request.body({ type: "json" }).value; try { switch (body.operation) { - // case "fetch": { - // const result = await performLocalFetch(body.url, body.options); - // console.log("Proxying fetch request to", body.url); - // response.headers.set("Content-Type", "application/json"); - // response.body = JSON.stringify(result); - // return; - // } case "shell": { // TODO: Have a nicer way to do this if (this.options.pagesPath.startsWith("s3://")) { @@ -300,9 +290,14 @@ export class HttpServer { }); return; } - console.log("Running shell command:", body.cmd, body.args); - const p = new Deno.Command(body.cmd, { - args: body.args, + const shellCommand: ShellRequest = body; + console.log( + "Running shell command:", + shellCommand.cmd, + shellCommand.args, + ); + const p = new Deno.Command(shellCommand.cmd, { + args: shellCommand.args, cwd: this.options.pagesPath, stdout: "piped", stderr: "piped", @@ -316,12 +311,40 @@ export class HttpServer { stdout, stderr, code: output.code, - }); + } as ShellResponse); if (output.code !== 0) { console.error("Error running shell command", stdout, stderr); } return; } + case "syscall": { + if (!this.system) { + response.headers.set("Content-Type", "text/plain"); + response.status = 400; + response.body = "Unknown operation"; + return; + } + const syscallCommand: SyscallRequest = body; + try { + const result = await this.system.localSyscall( + syscallCommand.ctx, + syscallCommand.name, + syscallCommand.args, + ); + response.headers.set("Content-type", "application/json"); + response.status = 200; + response.body = JSON.stringify({ + result: result, + } as SyscallResponse); + } catch (e: any) { + response.headers.set("Content-type", "application/json"); + response.status = 500; + response.body = JSON.stringify({ + error: e.message, + } as SyscallResponse); + } + return; + } default: response.headers.set("Content-Type", "text/plain"); response.status = 400; @@ -530,10 +553,3 @@ function utcDateString(mtime: number): string { function authCookieName(host: string) { return `auth:${host}`; } - -function copyHeader(fromHeaders: Headers, toHeaders: Headers, header: string) { - const value = fromHeaders.get(header); - if (value) { - toHeaders.set(header, value); - } -} diff --git a/server/rpc.ts b/server/rpc.ts new file mode 100644 index 0000000..c52321b --- /dev/null +++ b/server/rpc.ts @@ -0,0 +1,21 @@ +export type ShellRequest = { + cmd: string; + args: string[]; +}; + +export type ShellResponse = { + stdout: string; + stderr: string; + code: number; +}; + +export type SyscallRequest = { + ctx: string; // Plug name requesting + name: string; + args: any[]; +}; + +export type SyscallResponse = { + result?: any; + error?: string; +}; diff --git a/server/server_system.ts b/server/server_system.ts new file mode 100644 index 0000000..1ac6640 --- /dev/null +++ b/server/server_system.ts @@ -0,0 +1,160 @@ +import { PlugNamespaceHook } from "../common/hooks/plug_namespace.ts"; +import { SilverBulletHooks } from "../common/manifest.ts"; +import { loadMarkdownExtensions } from "../common/markdown_parser/markdown_ext.ts"; +import buildMarkdown from "../common/markdown_parser/parser.ts"; +import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives.ts"; +import { FileMetaSpacePrimitives } from "../common/spaces/file_meta_space_primitives.ts"; +import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts"; +import { createSandbox } from "../plugos/environments/webworker_sandbox.ts"; +import { CronHook } from "../plugos/hooks/cron.ts"; +import { EndpointHook } from "../plugos/hooks/endpoint.ts"; +import { EventHook } from "../plugos/hooks/event.ts"; +import { MQHook } from "../plugos/hooks/mq.ts"; +import { DenoKVStore } from "../plugos/lib/kv_store.deno_kv.ts"; +import { DexieMQ } from "../plugos/lib/mq.dexie.ts"; +import assetSyscalls from "../plugos/syscalls/asset.ts"; +import { eventSyscalls } from "../plugos/syscalls/event.ts"; +import { mqSyscalls } from "../plugos/syscalls/mq.dexie.ts"; +import { storeSyscalls } from "../plugos/syscalls/store.ts"; +import { System } from "../plugos/system.ts"; +import { Space } from "../web/space.ts"; +import { debugSyscalls } from "../web/syscalls/debug.ts"; +import { pageIndexSyscalls } from "../cli/syscalls/index.ts"; +import { markdownSyscalls } from "../web/syscalls/markdown.ts"; +import { spaceSyscalls } from "../cli/syscalls/space.ts"; +import { systemSyscalls } from "../web/syscalls/system.ts"; +import { yamlSyscalls } from "../web/syscalls/yaml.ts"; +import { Application, path } from "./deps.ts"; +import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts"; +import { shellSyscalls } from "../plugos/syscalls/shell.deno.ts"; +import { IDBKeyRange, indexedDB } from "https://esm.sh/fake-indexeddb@4.0.2"; +import { SpacePrimitives } from "../common/spaces/space_primitives.ts"; + +export class ServerSystem { + system: System = new System("server"); + spacePrimitives!: SpacePrimitives; + requeueInterval?: number; + kvStore?: DenoKVStore; + + constructor( + private baseSpacePrimitives: SpacePrimitives, + private dbPath: string, + private app: Application, + ) { + } + + // Always needs to be invoked right after construction + async init() { + // Event hook + const eventHook = new EventHook(); + this.system.addHook(eventHook); + + // Cron hook + const cronHook = new CronHook(this.system); + this.system.addHook(cronHook); + + this.kvStore = new DenoKVStore(); + await this.kvStore.init(this.dbPath); + + // Endpoint hook + this.system.addHook(new EndpointHook(this.app, "/_/")); + + // Use DexieMQ for this, in memory + const mq = new DexieMQ("mq", indexedDB, IDBKeyRange); + + this.requeueInterval = setInterval(() => { + // Timeout after 5s, retries 3 times, otherwise drops the message (no DLQ) + mq.requeueTimeouts(5000, 3, true).catch(console.error); + }, 20000); // Look to requeue every 20s + + const pageIndexCalls = pageIndexSyscalls(this.kvStore); + + const plugNamespaceHook = new PlugNamespaceHook(); + this.system.addHook(plugNamespaceHook); + + this.system.addHook(new MQHook(this.system, mq)); + + this.spacePrimitives = new FileMetaSpacePrimitives( + new EventedSpacePrimitives( + new PlugSpacePrimitives( + this.baseSpacePrimitives, + plugNamespaceHook, + ), + eventHook, + ), + pageIndexCalls, + ); + const space = new Space(this.spacePrimitives, this.kvStore); + + // Add syscalls + this.system.registerSyscalls( + [], + eventSyscalls(eventHook), + spaceSyscalls(space), + assetSyscalls(this.system), + yamlSyscalls(), + storeSyscalls(this.kvStore), + systemSyscalls(undefined as any, this.system), + mqSyscalls(mq), + pageIndexCalls, + debugSyscalls(), + markdownSyscalls(buildMarkdown([])), // Will later be replaced with markdown extensions + ); + + // Syscalls that require some additional permissions + this.system.registerSyscalls( + ["fetch"], + sandboxFetchSyscalls(), + ); + + this.system.registerSyscalls( + ["shell"], + shellSyscalls("."), + ); + + await this.loadPlugs(); + + // for (let plugPath of await space.listPlugs()) { + // plugPath = path.resolve(this.spacePath, plugPath); + // await this.system.load( + // new URL(`file://${plugPath}`), + // createSandbox, + // ); + // } + + // Load markdown syscalls based on all new syntax (if any) + this.system.registerSyscalls( + [], + markdownSyscalls(buildMarkdown(loadMarkdownExtensions(this.system))), + ); + } + + async loadPlugs() { + const tempDir = await Deno.makeTempDir(); + try { + for (const { name } of await this.spacePrimitives.fetchFileList()) { + if ( + name.endsWith(".plug.js") // && !filePath.includes("search.plug.js") + ) { + const plugPath = path.join(tempDir, name); + await Deno.mkdir(path.dirname(plugPath), { recursive: true }); + await Deno.writeFile( + plugPath, + (await this.spacePrimitives.readFile(name)).data, + ); + await this.system.load( + new URL(`file://${plugPath}`), + createSandbox, + ); + } + } + } finally { + await Deno.remove(tempDir, { recursive: true }); + } + } + + async close() { + clearInterval(this.requeueInterval); + await this.system.unloadAll(); + } +} diff --git a/silverbullet.ts b/silverbullet.ts index 2c46a6d..1de185c 100755 --- a/silverbullet.ts +++ b/silverbullet.ts @@ -45,8 +45,16 @@ await new Command() "Path to TLS key", ) .option( - "--maxFileSize [type:number]", - "Do not sync/expose files larger than this (in MB)", + "-t [type:boolean], --thin-client [type:boolean]", + "Enable thin-client mode", + ) + .option( + "--reindex [type:boolean]", + "Reindex space on startup (applies to thin-mode only)", + ) + .option( + "--db ", + "Path to database file (applies to thin-mode only)", ) .action(serveCommand) // plug:compile diff --git a/web/boot.ts b/web/boot.ts index d3144e8..c3f4cf2 100644 --- a/web/boot.ts +++ b/web/boot.ts @@ -1,11 +1,13 @@ import { safeRun } from "../common/util.ts"; import { Client } from "./client.ts"; +const thinClientMode = window.silverBulletConfig.thinClientMode === "on"; safeRun(async () => { console.log("Booting SilverBullet..."); const client = new Client( document.getElementById("sb-root")!, + thinClientMode, ); await client.init(); window.client = client; @@ -19,12 +21,14 @@ if (navigator.serviceWorker) { .then(() => { console.log("Service worker registered..."); }); - navigator.serviceWorker.ready.then((registration) => { - registration.active!.postMessage({ - type: "config", - config: window.silverBulletConfig, + if (!thinClientMode) { + navigator.serviceWorker.ready.then((registration) => { + registration.active!.postMessage({ + type: "config", + config: window.silverBulletConfig, + }); }); - }); + } } else { console.warn( "Not launching service worker, likely because not running from localhost or over HTTPs. This means SilverBullet will not be available offline.", diff --git a/web/client.ts b/web/client.ts index 6a4164d..0269293 100644 --- a/web/client.ts +++ b/web/client.ts @@ -36,6 +36,7 @@ import { MainUI } from "./editor_ui.tsx"; import { DexieMQ } from "../plugos/lib/mq.dexie.ts"; import { cleanPageRef } from "$sb/lib/resolve.ts"; import { expandPropertyNames } from "$sb/lib/json.ts"; +import { SpacePrimitives } from "../common/spaces/space_primitives.ts"; const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/; const autoSaveInterval = 1000; @@ -45,6 +46,7 @@ declare global { // Injected via index.html silverBulletConfig: { spaceFolderPath: string; + thinClientMode: "on" | "off"; }; client: Client; } @@ -53,15 +55,13 @@ declare global { // TODO: Oh my god, need to refactor this export class Client { system: ClientSystem; - editorView: EditorView; - private pageNavigator!: PathPageNavigator; private dbPrefix: string; plugSpaceRemotePrimitives!: PlugSpacePrimitives; - localSpacePrimitives!: FilteredSpacePrimitives; + // localSpacePrimitives!: FilteredSpacePrimitives; remoteSpacePrimitives!: HttpSpacePrimitives; space!: Space; @@ -88,6 +88,7 @@ export class Client { constructor( parent: Element, + private thinClientMode = false, ) { // Generate a semi-unique prefix for the database so not to reuse databases for different space paths this.dbPrefix = "" + simpleHash(window.silverBulletConfig.spaceFolderPath); @@ -116,12 +117,13 @@ export class Client { this.mq, this.dbPrefix, this.eventHook, + this.thinClientMode, ); - this.initSpace(); + const localSpacePrimitives = this.initSpace(); this.syncService = new SyncService( - this.localSpacePrimitives, + localSpacePrimitives, this.plugSpaceRemotePrimitives, this.kvStore, this.eventHook, @@ -133,6 +135,7 @@ export class Client { // Except federated ones path.startsWith("!"); }, + !this.thinClientMode, ); this.ui = new MainUI(this); @@ -319,7 +322,7 @@ export class Client { } } - initSpace() { + initSpace(): SpacePrimitives { this.remoteSpacePrimitives = new HttpSpacePrimitives( location.origin, window.silverBulletConfig.spaceFolderPath, @@ -332,34 +335,40 @@ export class Client { let fileFilterFn: (s: string) => boolean = () => true; - this.localSpacePrimitives = new FilteredSpacePrimitives( - new FileMetaSpacePrimitives( - new EventedSpacePrimitives( - // Using fallback space primitives here to allow (by default) local reads to "fall through" to HTTP when files aren't synced yet - new FallbackSpacePrimitives( - new IndexedDBSpacePrimitives( - `${this.dbPrefix}_space`, - globalThis.indexedDB, - ), - this.plugSpaceRemotePrimitives, - ), - this.eventHook, - ), - this.system.indexSyscalls, - ), - (meta) => fileFilterFn(meta.name), - // Run when a list of files has been retrieved - async () => { - await this.loadSettings(); - if (typeof this.settings?.spaceIgnore === "string") { - fileFilterFn = gitIgnoreCompiler(this.settings.spaceIgnore).accepts; - } else { - fileFilterFn = () => true; - } - }, - ); + let localSpacePrimitives: SpacePrimitives | undefined; - this.space = new Space(this.localSpacePrimitives, this.kvStore); + if (!this.thinClientMode) { + localSpacePrimitives = new FilteredSpacePrimitives( + new FileMetaSpacePrimitives( + new EventedSpacePrimitives( + // Using fallback space primitives here to allow (by default) local reads to "fall through" to HTTP when files aren't synced yet + new FallbackSpacePrimitives( + new IndexedDBSpacePrimitives( + `${this.dbPrefix}_space`, + globalThis.indexedDB, + ), + this.plugSpaceRemotePrimitives, + ), + this.eventHook, + ), + this.system.indexSyscalls, + ), + (meta) => fileFilterFn(meta.name), + // Run when a list of files has been retrieved + async () => { + await this.loadSettings(); + if (typeof this.settings?.spaceIgnore === "string") { + fileFilterFn = gitIgnoreCompiler(this.settings.spaceIgnore).accepts; + } else { + fileFilterFn = () => true; + } + }, + ); + } else { + localSpacePrimitives = this.plugSpaceRemotePrimitives; + } + + this.space = new Space(localSpacePrimitives, this.kvStore); this.space.on({ pageChanged: (meta) => { @@ -379,6 +388,8 @@ export class Client { }); this.space.watch(); + + return localSpacePrimitives; } async loadSettings(): Promise { diff --git a/web/client_system.ts b/web/client_system.ts index dc46501..a2d06a4 100644 --- a/web/client_system.ts +++ b/web/client_system.ts @@ -33,6 +33,8 @@ import { import { DexieMQ } from "../plugos/lib/mq.dexie.ts"; import { MQHook } from "../plugos/hooks/mq.ts"; import { mqSyscalls } from "../plugos/syscalls/mq.dexie.ts"; +import { indexProxySyscalls } from "./syscalls/index.proxy.ts"; +import { storeProxySyscalls } from "./syscalls/store.proxy.ts"; export class ClientSystem { system: System = new System("client"); @@ -45,11 +47,12 @@ export class ClientSystem { mdExtensions: MDExt[] = []; constructor( - private editor: Client, + private client: Client, private kvStore: DexieKVStore, private mq: DexieMQ, private dbPrefix: string, private eventHook: EventHook, + private thinClientMode: boolean, ) { this.system.addHook(this.eventHook); @@ -61,11 +64,15 @@ export class ClientSystem { const cronHook = new CronHook(this.system); this.system.addHook(cronHook); - this.indexSyscalls = pageIndexSyscalls( - `${dbPrefix}_page_index`, - globalThis.indexedDB, - globalThis.IDBKeyRange, - ); + if (thinClientMode) { + this.indexSyscalls = indexProxySyscalls(client); + } else { + this.indexSyscalls = pageIndexSyscalls( + `${dbPrefix}_page_index`, + globalThis.indexedDB, + globalThis.IDBKeyRange, + ); + } // Code widget hook this.codeWidgetHook = new CodeWidgetHook(); @@ -78,7 +85,7 @@ export class ClientSystem { this.commandHook = new CommandHook(); this.commandHook.on({ commandsUpdated: (commandMap) => { - this.editor.ui.viewDispatch({ + this.client.ui.viewDispatch({ type: "update-commands", commands: commandMap, }); @@ -87,7 +94,7 @@ export class ClientSystem { this.system.addHook(this.commandHook); // Slash command hook - this.slashCommandHook = new SlashCommandHook(this.editor); + this.slashCommandHook = new SlashCommandHook(this.client); this.system.addHook(this.slashCommandHook); this.eventHook.addLocalListener("plug:changed", async (fileName) => { @@ -96,7 +103,7 @@ export class ClientSystem { const plug = await this.system.load( new URL(`/${fileName}`, location.href), createSandbox, - this.editor.settings.plugOverrides, + this.client.settings.plugOverrides, ); if ((plug.manifest! as Manifest).syntax) { // If there are syntax extensions, rebuild the markdown parser immediately @@ -108,19 +115,21 @@ export class ClientSystem { } registerSyscalls() { - const storeCalls = storeSyscalls(this.kvStore); + const storeCalls = this.thinClientMode + ? storeProxySyscalls(this.client) + : storeSyscalls(this.kvStore); // Slash command hook - this.slashCommandHook = new SlashCommandHook(this.editor); + this.slashCommandHook = new SlashCommandHook(this.client); this.system.addHook(this.slashCommandHook); // Syscalls available to all plugs this.system.registerSyscalls( [], eventSyscalls(this.eventHook), - editorSyscalls(this.editor), - spaceSyscalls(this.editor), - systemSyscalls(this.editor, this.system), + editorSyscalls(this.client), + spaceSyscalls(this.client), + systemSyscalls(this.client, this.system), markdownSyscalls(buildMarkdown(this.mdExtensions)), assetSyscalls(this.system), yamlSyscalls(), @@ -128,20 +137,19 @@ export class ClientSystem { storeCalls, this.indexSyscalls, debugSyscalls(), - syncSyscalls(this.editor), - // LEGACY - clientStoreSyscalls(storeCalls), + syncSyscalls(this.client), + clientStoreSyscalls(this.kvStore), ); // Syscalls that require some additional permissions this.system.registerSyscalls( ["fetch"], - sandboxFetchSyscalls(this.editor), + sandboxFetchSyscalls(this.client), ); this.system.registerSyscalls( ["shell"], - shellSyscalls(this.editor), + shellSyscalls(this.client), ); } @@ -155,7 +163,7 @@ export class ClientSystem { await this.system.load( new URL(plugName, location.origin), createSandbox, - this.editor.settings.plugOverrides, + this.client.settings.plugOverrides, ); } catch (e: any) { console.error("Could not load plug", plugName, "error:", e.message); diff --git a/web/index.html b/web/index.html index 39cd983..62177c7 100644 --- a/web/index.html +++ b/web/index.html @@ -35,11 +35,13 @@ window.silverBulletConfig = { // These {{VARIABLES}} are replaced by http_server.ts spaceFolderPath: "{{SPACE_PATH}}", + thinClientMode: "{{THIN_CLIENT_MODE}}", }; // But in case these variables aren't replaced by the server, fall back fully static mode (no sync) if (window.silverBulletConfig.spaceFolderPath.includes("{{")) { window.silverBulletConfig = { spaceFolderPath: "", + thinClientMode: "off", }; } diff --git a/web/sync_service.ts b/web/sync_service.ts index 97650b9..09de015 100644 --- a/web/sync_service.ts +++ b/web/sync_service.ts @@ -45,6 +45,7 @@ export class SyncService { private kvStore: KVStore, private eventHook: EventHook, private isSyncCandidate: (path: string) => boolean, + private enabled: boolean, ) { this.spaceSync = new SpaceSync( this.localSpacePrimitives, @@ -74,6 +75,9 @@ export class SyncService { } async isSyncing(): Promise { + if (!this.enabled) { + return false; + } const startTime = await this.kvStore.get(syncStartTimeKey); if (!startTime) { return false; @@ -91,11 +95,19 @@ export class SyncService { } hasInitialSyncCompleted(): Promise { + if (!this.enabled) { + return Promise.resolve(true); + } + // Initial sync has happened when sync progress has been reported at least once, but the syncStartTime has been reset (which happens after sync finishes) return this.kvStore.has(syncInitialFullSyncCompletedKey); } async registerSyncStart(fullSync: boolean): Promise { + if (!this.enabled) { + return; + } + // Assumption: this is called after an isSyncing() check await this.kvStore.batchSet([ { @@ -116,6 +128,10 @@ export class SyncService { } async registerSyncProgress(status?: SyncStatus): Promise { + if (!this.enabled) { + return; + } + // Emit a sync event at most every 2s if (status && this.lastReportedSyncStatus < Date.now() - 2000) { this.eventHook.dispatchEvent("sync:progress", status); @@ -126,6 +142,10 @@ export class SyncService { } async registerSyncStop(isFullSync: boolean): Promise { + if (!this.enabled) { + return; + } + await this.registerSyncProgress(); await this.kvStore.del(syncStartTimeKey); if (isFullSync) { @@ -142,6 +162,10 @@ export class SyncService { // Await a moment when the sync is no longer running async noOngoingSync(timeout: number): Promise { + if (!this.enabled) { + return; + } + // Not completely safe, could have race condition on setting the syncStartTimeKey const startTime = Date.now(); while (await this.isSyncing()) { @@ -155,6 +179,10 @@ export class SyncService { filesScheduledForSync = new Set(); async scheduleFileSync(path: string): Promise { + if (!this.enabled) { + return; + } + if (this.filesScheduledForSync.has(path)) { // Already scheduled, no need to duplicate console.info(`File ${path} already scheduled for sync`); @@ -167,11 +195,19 @@ export class SyncService { } async scheduleSpaceSync(): Promise { + if (!this.enabled) { + return; + } + await this.noOngoingSync(5000); await this.syncSpace(); } start() { + if (!this.enabled) { + return; + } + this.syncSpace().catch(console.error); setInterval(async () => { @@ -191,6 +227,10 @@ export class SyncService { } async syncSpace(): Promise { + if (!this.enabled) { + return 0; + } + if (await this.isSyncing()) { console.log("Aborting space sync: already syncing"); return 0; @@ -218,6 +258,10 @@ export class SyncService { // Syncs a single file async syncFile(name: string) { + if (!this.enabled) { + return; + } + // console.log("Checking if we can sync file", name); if (!this.isSyncCandidate(name)) { console.info("Requested sync, but not a sync candidate", name); diff --git a/web/syscalls/clientStore.ts b/web/syscalls/clientStore.ts index 64830d1..1914a6e 100644 --- a/web/syscalls/clientStore.ts +++ b/web/syscalls/clientStore.ts @@ -1,14 +1,19 @@ +import { KVStore } from "../../plugos/lib/kv_store.ts"; +import { storeSyscalls } from "../../plugos/syscalls/store.ts"; import { proxySyscalls } from "../../plugos/syscalls/transport.ts"; import { SysCallMapping } from "../../plugos/system.ts"; -// DEPRECATED, use store directly export function clientStoreSyscalls( - storeCalls: SysCallMapping, + db: KVStore, ): SysCallMapping { + const localStoreCalls = storeSyscalls(db); return proxySyscalls( ["clientStore.get", "clientStore.set", "clientStore.delete"], (ctx, name, ...args) => { - return storeCalls[name.replace("clientStore.", "store.")](ctx, ...args); + return localStoreCalls[name.replace("clientStore.", "store.")]( + ctx, + ...args, + ); }, ); } diff --git a/web/syscalls/index.proxy.ts b/web/syscalls/index.proxy.ts new file mode 100644 index 0000000..c701e3c --- /dev/null +++ b/web/syscalls/index.proxy.ts @@ -0,0 +1,16 @@ +import { SysCallMapping } from "../../plugos/system.ts"; +import { Client } from "../client.ts"; +import { proxySyscalls } from "./util.ts"; + +export function indexProxySyscalls(client: Client): SysCallMapping { + return proxySyscalls(client, [ + "index.set", + "index.batchSet", + "index.delete", + "index.get", + "index.queryPrefix", + "index.clearPageIndexForPage", + "index.deletePrefixForPage", + "index.clearPageIndex", + ]); +} diff --git a/web/syscalls/store.proxy.ts b/web/syscalls/store.proxy.ts new file mode 100644 index 0000000..fc5b880 --- /dev/null +++ b/web/syscalls/store.proxy.ts @@ -0,0 +1,18 @@ +import type { SysCallMapping } from "../../plugos/system.ts"; +import type { Client } from "../client.ts"; +import { proxySyscalls } from "./util.ts"; + +export function storeProxySyscalls(client: Client): SysCallMapping { + return proxySyscalls(client, [ + "store.delete", + "store.deletePrefix", + "store.deleteAll", + "store.set", + "store.batchSet", + "store.batchDelete", + "store.batchGet", + "store.get", + "store.has", + "store.queryPrefix", + ]); +} diff --git a/web/syscalls/util.ts b/web/syscalls/util.ts new file mode 100644 index 0000000..42b6631 --- /dev/null +++ b/web/syscalls/util.ts @@ -0,0 +1,33 @@ +import { SysCallMapping } from "../../plugos/system.ts"; +import { SyscallResponse } from "../../server/rpc.ts"; +import { Client } from "../client.ts"; + +export function proxySyscalls(client: Client, names: string[]): SysCallMapping { + const syscalls: SysCallMapping = {}; + for (const name of names) { + syscalls[name] = async (_ctx, ...args: any[]) => { + if (!client.remoteSpacePrimitives) { + throw new Error("Not supported"); + } + const resp = await client.remoteSpacePrimitives.authenticatedFetch( + `${client.remoteSpacePrimitives.url}/.rpc`, + { + method: "POST", + body: JSON.stringify({ + operation: "syscall", + name, + args, + }), + }, + ); + const result: SyscallResponse = await resp.json(); + if (result.error) { + console.error("Remote syscall error", result.error); + throw new Error(result.error); + } else { + return result.result; + } + }; + } + return syscalls; +}