diff --git a/common/spaces/evented_space_primitives.ts b/common/spaces/evented_space_primitives.ts index d449b8f..4b73433 100644 --- a/common/spaces/evented_space_primitives.ts +++ b/common/spaces/evented_space_primitives.ts @@ -5,64 +5,59 @@ import type { SpacePrimitives } from "./space_primitives.ts"; /** * Events exposed: - * - file:changed (FileMeta) + * - file:changed (string, localUpdate: boolean) * - file:deleted (string) * - file:listed (FileMeta[]) * - page:saved (string, FileMeta) * - page:deleted (string) */ - export class EventedSpacePrimitives implements SpacePrimitives { - private fileMetaCache = new Map(); + alreadyFetching = false; initialFileListLoad = true; + spaceSnapshot: Record = {}; constructor( private wrapped: SpacePrimitives, private eventHook: EventHook, - private eventsToDispatch = [ - "file:changed", - "file:deleted", - "file:listed", - "page:saved", - "page:deleted", - ], ) {} dispatchEvent(name: string, ...args: any[]): Promise { - if (this.eventsToDispatch.includes(name)) { - return this.eventHook.dispatchEvent(name, ...args); - } else { - return Promise.resolve([]); - } + return this.eventHook.dispatchEvent(name, ...args); } async fetchFileList(): Promise { const newFileList = await this.wrapped.fetchFileList(); - const deletedFiles = new Set(this.fileMetaCache.keys()); + if (this.alreadyFetching) { + // Avoid race conditions + return newFileList; + } + // console.log("HEREEREEEREEREE"); + this.alreadyFetching = true; + const deletedFiles = new Set(Object.keys(this.spaceSnapshot)); for (const meta of newFileList) { - const oldFileMeta = this.fileMetaCache.get(meta.name); - const newFileMeta: FileMeta = { ...meta }; + const oldHash = this.spaceSnapshot[meta.name]; + const newHash = meta.lastModified; if ( ( // New file scenario - !oldFileMeta && !this.initialFileListLoad + !oldHash && !this.initialFileListLoad ) || ( // Changed file scenario - oldFileMeta && - oldFileMeta.lastModified !== newFileMeta.lastModified + oldHash && + oldHash !== newHash ) ) { - this.dispatchEvent("file:changed", newFileMeta); + this.dispatchEvent("file:changed", meta.name); } // Page found, not deleted deletedFiles.delete(meta.name); - // Update in cache - this.fileMetaCache.set(meta.name, newFileMeta); + // Update in snapshot + this.spaceSnapshot[meta.name] = newHash; } for (const deletedFile of deletedFiles) { - this.fileMetaCache.delete(deletedFile); + delete this.spaceSnapshot[deletedFile]; this.dispatchEvent("file:deleted", deletedFile); if (deletedFile.endsWith(".md")) { @@ -71,28 +66,18 @@ export class EventedSpacePrimitives implements SpacePrimitives { } } - const fileList = [...new Set(this.fileMetaCache.values())]; - this.dispatchEvent("file:listed", fileList); + this.dispatchEvent("file:listed", newFileList); + this.alreadyFetching = false; this.initialFileListLoad = false; - return fileList; + return newFileList; } async readFile( name: string, ): Promise<{ data: Uint8Array; meta: FileMeta }> { const data = await this.wrapped.readFile(name); - const previousMeta = this.fileMetaCache.get(name); - const newMeta = data.meta; - if (previousMeta) { - if (previousMeta.lastModified !== newMeta.lastModified) { - // Page changed since last cached metadata, trigger event - this.dispatchEvent("file:changed", newMeta); - } - } - return { - data: data.data, - meta: this.metaCacher(name, newMeta), - }; + this.triggerEventsAndCache(name, data.meta.lastModified); + return data; } async writeFile( @@ -108,9 +93,9 @@ export class EventedSpacePrimitives implements SpacePrimitives { meta, ); if (!selfUpdate) { - this.dispatchEvent("file:changed", newMeta); + this.dispatchEvent("file:changed", name, true); } - this.metaCacher(name, newMeta); + this.spaceSnapshot[name] = newMeta.lastModified; // This can happen async if (name.endsWith(".md")) { @@ -136,17 +121,21 @@ export class EventedSpacePrimitives implements SpacePrimitives { return newMeta; } + triggerEventsAndCache(name: string, newHash: number) { + const oldHash = this.spaceSnapshot[name]; + if (oldHash && oldHash !== newHash) { + // Page changed since last cached metadata, trigger event + this.dispatchEvent("file:changed", name); + } + this.spaceSnapshot[name] = newHash; + return; + } + async getFileMeta(name: string): Promise { try { - const oldMeta = this.fileMetaCache.get(name); const newMeta = await this.wrapped.getFileMeta(name); - if (oldMeta) { - if (oldMeta.lastModified !== newMeta.lastModified) { - // Changed on disk, trigger event - this.dispatchEvent("file:changed", newMeta); - } - } - return this.metaCacher(name, newMeta); + this.triggerEventsAndCache(name, newMeta.lastModified); + return newMeta; } catch (e: any) { console.log("Checking error", e, name); if (e.message === "Not found") { @@ -167,15 +156,7 @@ export class EventedSpacePrimitives implements SpacePrimitives { } // await this.getPageMeta(name); // Check if page exists, if not throws Error await this.wrapped.deleteFile(name); - this.fileMetaCache.delete(name); + delete this.spaceSnapshot[name]; this.dispatchEvent("file:deleted", name); } - - private metaCacher(name: string, meta: FileMeta): FileMeta { - if (meta.lastModified !== 0) { - // Don't cache metadata for pages with a 0 lastModified timestamp (usualy dynamically generated pages) - this.fileMetaCache.set(name, meta); - } - return meta; - } } diff --git a/plug-api/silverbullet-syscall/system.ts b/plug-api/silverbullet-syscall/system.ts index be37e40..9c87ca2 100644 --- a/plug-api/silverbullet-syscall/system.ts +++ b/plug-api/silverbullet-syscall/system.ts @@ -1,6 +1,14 @@ import type { CommandDef } from "../../web/hooks/command.ts"; import { syscall } from "./syscall.ts"; +export function invoke( + name: string, + ...args: any[] +): Promise { + return syscall("system.invoke", name, ...args); +} + +// @deprecated use invoke instead export function invokeFunction( env: string, name: string, diff --git a/plugos/types.ts b/plugos/types.ts index e324457..95f3c3f 100644 --- a/plugos/types.ts +++ b/plugos/types.ts @@ -9,7 +9,7 @@ export interface Manifest { name: string; /** A list of syscall permissions required for this plug to function. - * + * * Possible values: * - `fetch`: enables `fetch` function. (see: plug-api/plugos-syscall/fetch.ts, and plug-api/lib/fetch.ts) * - `shell`: enables the `shell.run` syscall. (see: plug-api/plugos-syscall/shell.ts) @@ -17,28 +17,26 @@ export interface Manifest { requiredPermissions?: string[]; /** A list of files or glob patterns that should be bundled with the plug. - * + * * These files will be accessible through the `asset.readAsset` function. - * + * * see: plug-api/plugos-syscall/asset.ts#readAsset */ assets?: string[] | AssetJson; /** A map of function names to definitions. Declared functions are public, and may be associated with various hooks - * + * * see: common/manifest.ts#SilverBulletHooks */ - functions: { - [key: string]: FunctionDef; - }; + functions: Record>; } /** Associates hooks with a function. This is the generic base structure, that identifies the function. Hooks are defined by the type parameter. */ export type FunctionDef = { /** A function path, in the form `${relativeFilename}:${functionName}`. - * + * * During compilation (see `../build_plugs.ts`) the function is read from the file and inlined into the plug bundle. - * + * * This field and `FunctionDef.redirect` are mutually exclusive. */ path?: string; diff --git a/plugs/core/core.plug.yaml b/plugs/core/core.plug.yaml index 275d82a..751fc55 100644 --- a/plugs/core/core.plug.yaml +++ b/plugs/core/core.plug.yaml @@ -37,6 +37,7 @@ functions: name: "Editor: Move Cursor to Position" clearPageIndex: path: "./page.ts:clearPageIndex" + env: server events: - page:saved - page:deleted @@ -46,6 +47,7 @@ functions: - query:page parseIndexTextRepublish: path: "./page.ts:parseIndexTextRepublish" + env: server events: - page:index_text reindexSpaceCommand: diff --git a/plugs/core/page.ts b/plugs/core/page.ts index 8504e8a..44d75ab 100644 --- a/plugs/core/page.ts +++ b/plugs/core/page.ts @@ -13,7 +13,7 @@ import { import { events, mq } from "$sb/plugos-syscall/mod.ts"; import { applyQuery } from "$sb/lib/query.ts"; -import { invokeFunction } from "$sb/silverbullet-syscall/system.ts"; +import { invoke } from "$sb/silverbullet-syscall/system.ts"; import type { Message } from "$sb/types.ts"; import { sleep } from "../../common/async_util.ts"; import { cacheFileListing } from "../federation/federation.ts"; @@ -135,7 +135,7 @@ export async function reindexSpace() { console.log("Clearing page index..."); await index.clearPageIndex(); // Executed this way to not have to embed the search plug code here - await invokeFunction("client", "search.clearIndex"); + await invoke("search.clearIndex"); const pages = await space.listPages(); // Queue all page names to be indexed @@ -172,7 +172,7 @@ export async function clearPageIndex(page: string) { } export async function parseIndexTextRepublish({ name, text }: IndexEvent) { - // console.log("Reindexing", name); + console.log("Reindexing", name); await events.dispatchEvent("page:index", { name, tree: await markdown.parseMarkdown(text), diff --git a/plugs/directive/eval_directive.ts b/plugs/directive/eval_directive.ts index a9d6eea..8a60753 100644 --- a/plugs/directive/eval_directive.ts +++ b/plugs/directive/eval_directive.ts @@ -44,7 +44,7 @@ export async function evalDirectiveRenderer( const result = await (0, eval)( `(async () => { function invokeFunction(name, ...args) { - return syscall("system.invokeFunction", "server", name, ...args); + return syscall("system.invoke", name, ...args); } return ${replaceTemplateVars(translateJs(expression), pageMeta)}; })()`, diff --git a/plugs/search/search.plug.yaml b/plugs/search/search.plug.yaml index cc12f0e..d70100e 100644 --- a/plugs/search/search.plug.yaml +++ b/plugs/search/search.plug.yaml @@ -12,6 +12,7 @@ functions: searchUnindex: path: "./search.ts:pageUnindex" + env: client events: - page:deleted searchQueryProvider: diff --git a/server/server_system.ts b/server/server_system.ts index 62325f5..2b2abbf 100644 --- a/server/server_system.ts +++ b/server/server_system.ts @@ -30,10 +30,12 @@ import { shellSyscalls } from "../plugos/syscalls/shell.deno.ts"; import { IDBKeyRange, indexedDB } from "https://esm.sh/fake-indexeddb@4.0.2"; import { SpacePrimitives } from "../common/spaces/space_primitives.ts"; +const fileListInterval = 30 * 1000; // 30s + export class ServerSystem { system: System = new System("server"); spacePrimitives!: SpacePrimitives; - requeueInterval?: number; + private requeueInterval?: number; kvStore?: DenoKVStore; constructor( @@ -114,19 +116,31 @@ export class ServerSystem { await this.loadPlugs(); - // for (let plugPath of await space.listPlugs()) { - // plugPath = path.resolve(this.spacePath, plugPath); - // await this.system.load( - // new URL(`file://${plugPath}`), - // createSandbox, - // ); - // } - // Load markdown syscalls based on all new syntax (if any) this.system.registerSyscalls( [], markdownSyscalls(buildMarkdown(loadMarkdownExtensions(this.system))), ); + + setInterval(() => { + space.updatePageList().catch(console.error); + }, fileListInterval); + + eventHook.addLocalListener("file:changed", (path, localChange) => { + (async () => { + // console.log("!!!!! FILE CHANGED", path, localChange); + if (!localChange && path.endsWith(".md")) { + const pageName = path.slice(0, -3); + const data = await this.spacePrimitives.readFile(path); + console.log("Outside page change: reindexing", pageName); + // Change made outside of editor, trigger reindex + await eventHook.dispatchEvent("page:index_text", { + name: pageName, + text: new TextDecoder().decode(data.data), + }); + } + })().catch(console.error); + }); } async loadPlugs() { diff --git a/web/client.ts b/web/client.ts index 2d114ca..7185aaf 100644 --- a/web/client.ts +++ b/web/client.ts @@ -321,12 +321,6 @@ export class Client { } })().catch(console.error); } - - // this.eventHook.addLocalListener("page:deleted", (pageName) => { - // if (pageName === this.currentPage) { - // this.flashNotification("Page does exist, creating as a new page"); - // } - // }); } initSpace(): SpacePrimitives { @@ -375,20 +369,15 @@ export class Client { localSpacePrimitives = new EventedSpacePrimitives( this.plugSpaceRemotePrimitives, this.eventHook, - [ - "file:changed", - "file:listed", - "page:deleted", - ], ); } this.space = new Space(localSpacePrimitives, this.kvStore, this.eventHook); - this.eventHook.addLocalListener("file:changed", (fileMeta: FileMeta) => { + this.eventHook.addLocalListener("file:changed", (path: string) => { // Only reload when watching the current page (to avoid reloading when switching pages) if ( - this.space.watchInterval && `${this.currentPage}.md` === fileMeta.name + this.space.watchInterval && `${this.currentPage}.md` === path ) { console.log("Page changed elsewhere, reloading"); this.flashNotification("Page changed elsewhere, reloading"); diff --git a/web/client_system.ts b/web/client_system.ts index 299232c..b17bd6a 100644 --- a/web/client_system.ts +++ b/web/client_system.ts @@ -37,7 +37,6 @@ import { indexProxySyscalls } from "./syscalls/index.proxy.ts"; import { storeProxySyscalls } from "./syscalls/store.proxy.ts"; export class ClientSystem { - system: System = new System("client"); commandHook: CommandHook; slashCommandHook: SlashCommandHook; namespaceHook: PlugNamespaceHook; @@ -45,15 +44,19 @@ export class ClientSystem { codeWidgetHook: CodeWidgetHook; plugsUpdated = false; mdExtensions: MDExt[] = []; + system: System; constructor( private client: Client, private kvStore: DexieKVStore, private mq: DexieMQ, - private dbPrefix: string, + dbPrefix: string, private eventHook: EventHook, private thinClientMode: boolean, ) { + // Only set environment to "client" when running in thin client mode, otherwise we run everything locally (hybrid) + this.system = new System(thinClientMode ? "client" : undefined); + this.system.addHook(this.eventHook); // Plug page namespace hook diff --git a/web/space.ts b/web/space.ts index 4ec5c3c..9261dbd 100644 --- a/web/space.ts +++ b/web/space.ts @@ -65,36 +65,6 @@ export class Space { public async updatePageList() { // This will trigger appropriate events automatically await this.fetchPageList(); - // const deletedPages = new Set(this.pageMetaCache.keys()); - // newPageList.forEach((meta) => { - // const pageName = meta.name; - // const oldPageMeta = this.pageMetaCache.get(pageName); - // const newPageMeta: PageMeta = { ...meta }; - // if ( - // !oldPageMeta && - // (pageName.startsWith(plugPrefix) || !this.initialPageListLoad) - // ) { - // this.emit("pageCreated", newPageMeta); - // } else if ( - // oldPageMeta && - // oldPageMeta.lastModified !== newPageMeta.lastModified - // ) { - // this.emit("pageChanged", newPageMeta); - // } - // // Page found, not deleted - // deletedPages.delete(pageName); - - // // Update in cache - // this.pageMetaCache.set(pageName, newPageMeta); - // }); - - // for (const deletedPage of deletedPages) { - // this.pageMetaCache.delete(deletedPage); - // this.emit("pageDeleted", deletedPage); - // } - - // this.emit("pageListUpdated", this.listPages()); - // this.initialPageListLoad = false; } async deletePage(name: string): Promise { diff --git a/web/syscalls/system.ts b/web/syscalls/system.ts index 3f3da88..8508e39 100644 --- a/web/syscalls/system.ts +++ b/web/syscalls/system.ts @@ -2,17 +2,27 @@ import type { Plug } from "../../plugos/plug.ts"; import { SysCallMapping, System } from "../../plugos/system.ts"; import type { Client } from "../client.ts"; import { CommandDef } from "../hooks/command.ts"; +import { proxySyscall } from "./util.ts"; export function systemSyscalls( editor: Client, system: System, ): SysCallMapping { - return { + const api: SysCallMapping = { "system.invokeFunction": ( ctx, _env: string, name: string, ...args: any[] + ) => { + // For backwards compatibility + // TODO: Remove at some point + return api["system.invoke"](ctx, name, ...args); + }, + "system.invoke": ( + ctx, + name: string, + ...args: any[] ) => { if (!ctx.plug) { throw Error("No plug associated with context"); @@ -28,6 +38,14 @@ export function systemSyscalls( } name = functionName; } + const functionDef = plug.manifest!.functions[name]; + if (!functionDef) { + throw Error(`Function ${name} not found`); + } + if (functionDef.env && system.env && functionDef.env !== system.env) { + // Proxy to another environment + return proxySyscall(editor.remoteSpacePrimitives, name, args); + } return plug.invoke(name, args); }, "system.invokeCommand": (_ctx, name: string) => { @@ -47,4 +65,5 @@ export function systemSyscalls( return system.env; }, }; + return api; } diff --git a/web/syscalls/util.ts b/web/syscalls/util.ts index 42b6631..ff36969 100644 --- a/web/syscalls/util.ts +++ b/web/syscalls/util.ts @@ -1,3 +1,4 @@ +import { HttpSpacePrimitives } from "../../common/spaces/http_space_primitives.ts"; import { SysCallMapping } from "../../plugos/system.ts"; import { SyscallResponse } from "../../server/rpc.ts"; import { Client } from "../client.ts"; @@ -5,29 +6,34 @@ import { Client } from "../client.ts"; export function proxySyscalls(client: Client, names: string[]): SysCallMapping { const syscalls: SysCallMapping = {}; for (const name of names) { - syscalls[name] = async (_ctx, ...args: any[]) => { - if (!client.remoteSpacePrimitives) { - throw new Error("Not supported"); - } - const resp = await client.remoteSpacePrimitives.authenticatedFetch( - `${client.remoteSpacePrimitives.url}/.rpc`, - { - method: "POST", - body: JSON.stringify({ - operation: "syscall", - name, - args, - }), - }, - ); - const result: SyscallResponse = await resp.json(); - if (result.error) { - console.error("Remote syscall error", result.error); - throw new Error(result.error); - } else { - return result.result; - } + syscalls[name] = (_ctx, ...args: any[]) => { + return proxySyscall(client.remoteSpacePrimitives, name, args); }; } return syscalls; } + +export async function proxySyscall( + httpSpacePrimitives: HttpSpacePrimitives, + name: string, + args: any[], +): Promise { + const resp = await httpSpacePrimitives.authenticatedFetch( + `${httpSpacePrimitives.url}/.rpc`, + { + method: "POST", + body: JSON.stringify({ + operation: "syscall", + name, + args, + }), + }, + ); + const result: SyscallResponse = await resp.json(); + if (result.error) { + console.error("Remote syscall error", result.error); + throw new Error(result.error); + } else { + return result.result; + } +}