neil/silverbullet
1
0
Fork 0
Code

More event refactoring work

Browse Source
This commit is contained in:
Zef Hemel 2023-08-27 14:13:18 +02:00
parent 593597454a
commit c3d384330d
13 changed files with 140 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
common/spaces/evented_space_primitives.ts
View File

@ -5,64 +5,59 @@ import type { SpacePrimitives } from "./space_primitives.ts";
/**
* Events exposed:
* - file:changed (FileMeta)
* - file:changed (string, localUpdate: boolean)
* - file:deleted (string)
* - file:listed (FileMeta[])
* - page:saved (string, FileMeta)
* - page:deleted (string)
*/
export class EventedSpacePrimitives implements SpacePrimitives {
private fileMetaCache = new Map<string, FileMeta>();
alreadyFetching = false;
initialFileListLoad = true;
spaceSnapshot: Record<string, number> = {};
constructor(
private wrapped: SpacePrimitives,
private eventHook: EventHook,
private eventsToDispatch = [
"file:changed",
"file:deleted",
"file:listed",
"page:saved",
"page:deleted",
],
) {}
dispatchEvent(name: string, ...args: any[]): Promise<any[]> {
if (this.eventsToDispatch.includes(name)) {
return this.eventHook.dispatchEvent(name, ...args);
} else {
return Promise.resolve([]);
}
return this.eventHook.dispatchEvent(name, ...args);
}
async fetchFileList(): Promise<FileMeta[]> {
const newFileList = await this.wrapped.fetchFileList();
const deletedFiles = new Set<string>(this.fileMetaCache.keys());
if (this.alreadyFetching) {
// Avoid race conditions
return newFileList;
}
// console.log("HEREEREEEREEREE");
this.alreadyFetching = true;
const deletedFiles = new Set<string>(Object.keys(this.spaceSnapshot));
for (const meta of newFileList) {
const oldFileMeta = this.fileMetaCache.get(meta.name);
const newFileMeta: FileMeta = { ...meta };
const oldHash = this.spaceSnapshot[meta.name];
const newHash = meta.lastModified;
if (
(
// New file scenario
!oldFileMeta && !this.initialFileListLoad
!oldHash && !this.initialFileListLoad
) || (
// Changed file scenario
oldFileMeta &&
oldFileMeta.lastModified !== newFileMeta.lastModified
oldHash &&
oldHash !== newHash
)
) {
this.dispatchEvent("file:changed", newFileMeta);
this.dispatchEvent("file:changed", meta.name);
}
// Page found, not deleted
deletedFiles.delete(meta.name);
// Update in cache
this.fileMetaCache.set(meta.name, newFileMeta);
// Update in snapshot
this.spaceSnapshot[meta.name] = newHash;
}
for (const deletedFile of deletedFiles) {
this.fileMetaCache.delete(deletedFile);
delete this.spaceSnapshot[deletedFile];
this.dispatchEvent("file:deleted", deletedFile);
if (deletedFile.endsWith(".md")) {
@ -71,28 +66,18 @@ export class EventedSpacePrimitives implements SpacePrimitives {
}
}
const fileList = [...new Set(this.fileMetaCache.values())];
this.dispatchEvent("file:listed", fileList);
this.dispatchEvent("file:listed", newFileList);
this.alreadyFetching = false;
this.initialFileListLoad = false;
return fileList;
return newFileList;
}
async readFile(
name: string,
): Promise<{ data: Uint8Array; meta: FileMeta }> {
const data = await this.wrapped.readFile(name);
const previousMeta = this.fileMetaCache.get(name);
const newMeta = data.meta;
if (previousMeta) {
if (previousMeta.lastModified !== newMeta.lastModified) {
// Page changed since last cached metadata, trigger event
this.dispatchEvent("file:changed", newMeta);
}
}
return {
data: data.data,
meta: this.metaCacher(name, newMeta),
};
this.triggerEventsAndCache(name, data.meta.lastModified);
return data;
}
async writeFile(
@ -108,9 +93,9 @@ export class EventedSpacePrimitives implements SpacePrimitives {
meta,
);
if (!selfUpdate) {
this.dispatchEvent("file:changed", newMeta);
this.dispatchEvent("file:changed", name, true);
}
this.metaCacher(name, newMeta);
this.spaceSnapshot[name] = newMeta.lastModified;
// This can happen async
if (name.endsWith(".md")) {
@ -136,17 +121,21 @@ export class EventedSpacePrimitives implements SpacePrimitives {
return newMeta;
}
triggerEventsAndCache(name: string, newHash: number) {
const oldHash = this.spaceSnapshot[name];
if (oldHash && oldHash !== newHash) {
// Page changed since last cached metadata, trigger event
this.dispatchEvent("file:changed", name);
}
this.spaceSnapshot[name] = newHash;
return;
}
async getFileMeta(name: string): Promise<FileMeta> {
try {
const oldMeta = this.fileMetaCache.get(name);
const newMeta = await this.wrapped.getFileMeta(name);
if (oldMeta) {
if (oldMeta.lastModified !== newMeta.lastModified) {
// Changed on disk, trigger event
this.dispatchEvent("file:changed", newMeta);
}
}
return this.metaCacher(name, newMeta);
this.triggerEventsAndCache(name, newMeta.lastModified);
return newMeta;
} catch (e: any) {
console.log("Checking error", e, name);
if (e.message === "Not found") {
@ -167,15 +156,7 @@ export class EventedSpacePrimitives implements SpacePrimitives {
}
// await this.getPageMeta(name); // Check if page exists, if not throws Error
await this.wrapped.deleteFile(name);
this.fileMetaCache.delete(name);
delete this.spaceSnapshot[name];
this.dispatchEvent("file:deleted", name);
}
private metaCacher(name: string, meta: FileMeta): FileMeta {
if (meta.lastModified !== 0) {
// Don't cache metadata for pages with a 0 lastModified timestamp (usualy dynamically generated pages)
this.fileMetaCache.set(name, meta);
}
return meta;
}
}

8
plug-api/silverbullet-syscall/system.ts
View File

@ -1,6 +1,14 @@
import type { CommandDef } from "../../web/hooks/command.ts";
import { syscall } from "./syscall.ts";
export function invoke(
name: string,
...args: any[]
): Promise<any> {
return syscall("system.invoke", name, ...args);
}
// @deprecated use invoke instead
export function invokeFunction(
env: string,
name: string,

16
plugos/types.ts
View File

@ -9,7 +9,7 @@ export interface Manifest<HookT> {
name: string;
/** A list of syscall permissions required for this plug to function.
*
*
* Possible values:
* - `fetch`: enables `fetch` function. (see: plug-api/plugos-syscall/fetch.ts, and plug-api/lib/fetch.ts)
* - `shell`: enables the `shell.run` syscall. (see: plug-api/plugos-syscall/shell.ts)
@ -17,28 +17,26 @@ export interface Manifest<HookT> {
requiredPermissions?: string[];
/** A list of files or glob patterns that should be bundled with the plug.
*
*
* These files will be accessible through the `asset.readAsset` function.
*
*
* see: plug-api/plugos-syscall/asset.ts#readAsset
*/
assets?: string[] | AssetJson;
/** A map of function names to definitions. Declared functions are public, and may be associated with various hooks
*
*
* see: common/manifest.ts#SilverBulletHooks
*/
functions: {
[key: string]: FunctionDef<HookT>;
};
functions: Record<string, FunctionDef<HookT>>;
}
/** Associates hooks with a function. This is the generic base structure, that identifies the function. Hooks are defined by the type parameter. */
export type FunctionDef<HookT> = {
/** A function path, in the form `${relativeFilename}:${functionName}`.
*
*
* During compilation (see `../build_plugs.ts`) the function is read from the file and inlined into the plug bundle.
*
*
* This field and `FunctionDef.redirect` are mutually exclusive.
*/
path?: string;

2
plugs/core/core.plug.yaml
View File

@ -37,6 +37,7 @@ functions:
name: "Editor: Move Cursor to Position"
clearPageIndex:
path: "./page.ts:clearPageIndex"
env: server
events:
- page:saved
- page:deleted
@ -46,6 +47,7 @@ functions:
- query:page
parseIndexTextRepublish:
path: "./page.ts:parseIndexTextRepublish"
env: server
events:
- page:index_text
reindexSpaceCommand:

6
plugs/core/page.ts
View File

@ -13,7 +13,7 @@ import {
import { events, mq } from "$sb/plugos-syscall/mod.ts";
import { applyQuery } from "$sb/lib/query.ts";
import { invokeFunction } from "$sb/silverbullet-syscall/system.ts";
import { invoke } from "$sb/silverbullet-syscall/system.ts";
import type { Message } from "$sb/types.ts";
import { sleep } from "../../common/async_util.ts";
import { cacheFileListing } from "../federation/federation.ts";
@ -135,7 +135,7 @@ export async function reindexSpace() {
console.log("Clearing page index...");
await index.clearPageIndex();
// Executed this way to not have to embed the search plug code here
await invokeFunction("client", "search.clearIndex");
await invoke("search.clearIndex");
const pages = await space.listPages();
// Queue all page names to be indexed
@ -172,7 +172,7 @@ export async function clearPageIndex(page: string) {
}
export async function parseIndexTextRepublish({ name, text }: IndexEvent) {
// console.log("Reindexing", name);
console.log("Reindexing", name);
await events.dispatchEvent("page:index", {
name,
tree: await markdown.parseMarkdown(text),

2
plugs/directive/eval_directive.ts
View File

@ -44,7 +44,7 @@ export async function evalDirectiveRenderer(
const result = await (0, eval)(
`(async () => {
function invokeFunction(name, ...args) {
return syscall("system.invokeFunction", "server", name, ...args);
return syscall("system.invoke", name, ...args);
}
return ${replaceTemplateVars(translateJs(expression), pageMeta)};
})()`,

1
plugs/search/search.plug.yaml
View File

@ -12,6 +12,7 @@ functions:
searchUnindex:
path: "./search.ts:pageUnindex"
env: client
events:
- page:deleted
searchQueryProvider:

32
server/server_system.ts
View File

@ -30,10 +30,12 @@ import { shellSyscalls } from "../plugos/syscalls/shell.deno.ts";
import { IDBKeyRange, indexedDB } from "https://esm.sh/fake-indexeddb@4.0.2";
import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
const fileListInterval = 30 * 1000; // 30s
export class ServerSystem {
system: System<SilverBulletHooks> = new System("server");
spacePrimitives!: SpacePrimitives;
requeueInterval?: number;
private requeueInterval?: number;
kvStore?: DenoKVStore;
constructor(
@ -114,19 +116,31 @@ export class ServerSystem {
await this.loadPlugs();
// for (let plugPath of await space.listPlugs()) {
// plugPath = path.resolve(this.spacePath, plugPath);
// await this.system.load(
// new URL(`file://${plugPath}`),
// createSandbox,
// );
// }
// Load markdown syscalls based on all new syntax (if any)
this.system.registerSyscalls(
[],
markdownSyscalls(buildMarkdown(loadMarkdownExtensions(this.system))),
);
setInterval(() => {
space.updatePageList().catch(console.error);
}, fileListInterval);
eventHook.addLocalListener("file:changed", (path, localChange) => {
(async () => {
// console.log("!!!!! FILE CHANGED", path, localChange);
if (!localChange && path.endsWith(".md")) {
const pageName = path.slice(0, -3);
const data = await this.spacePrimitives.readFile(path);
console.log("Outside page change: reindexing", pageName);
// Change made outside of editor, trigger reindex
await eventHook.dispatchEvent("page:index_text", {
name: pageName,
text: new TextDecoder().decode(data.data),
});
}
})().catch(console.error);
});
}
async loadPlugs() {

15
web/client.ts
View File

@ -321,12 +321,6 @@ export class Client {
}
})().catch(console.error);
}
// this.eventHook.addLocalListener("page:deleted", (pageName) => {
// if (pageName === this.currentPage) {
// this.flashNotification("Page does exist, creating as a new page");
// }
// });
}
initSpace(): SpacePrimitives {
@ -375,20 +369,15 @@ export class Client {
localSpacePrimitives = new EventedSpacePrimitives(
this.plugSpaceRemotePrimitives,
this.eventHook,
[
"file:changed",
"file:listed",
"page:deleted",
],
);
}
this.space = new Space(localSpacePrimitives, this.kvStore, this.eventHook);
this.eventHook.addLocalListener("file:changed", (fileMeta: FileMeta) => {
this.eventHook.addLocalListener("file:changed", (path: string) => {
// Only reload when watching the current page (to avoid reloading when switching pages)
if (
this.space.watchInterval && `${this.currentPage}.md` === fileMeta.name
this.space.watchInterval && `${this.currentPage}.md` === path
) {
console.log("Page changed elsewhere, reloading");
this.flashNotification("Page changed elsewhere, reloading");

7
web/client_system.ts
View File

@ -37,7 +37,6 @@ import { indexProxySyscalls } from "./syscalls/index.proxy.ts";
import { storeProxySyscalls } from "./syscalls/store.proxy.ts";
export class ClientSystem {
system: System<SilverBulletHooks> = new System("client");
commandHook: CommandHook;
slashCommandHook: SlashCommandHook;
namespaceHook: PlugNamespaceHook;
@ -45,15 +44,19 @@ export class ClientSystem {
codeWidgetHook: CodeWidgetHook;
plugsUpdated = false;
mdExtensions: MDExt[] = [];
system: System<SilverBulletHooks>;
constructor(
private client: Client,
private kvStore: DexieKVStore,
private mq: DexieMQ,
private dbPrefix: string,
dbPrefix: string,
private eventHook: EventHook,
private thinClientMode: boolean,
) {
// Only set environment to "client" when running in thin client mode, otherwise we run everything locally (hybrid)
this.system = new System(thinClientMode ? "client" : undefined);
this.system.addHook(this.eventHook);
// Plug page namespace hook

30
web/space.ts
View File

@ -65,36 +65,6 @@ export class Space {
public async updatePageList() {
// This will trigger appropriate events automatically
await this.fetchPageList();
// const deletedPages = new Set<string>(this.pageMetaCache.keys());
// newPageList.forEach((meta) => {
// const pageName = meta.name;
// const oldPageMeta = this.pageMetaCache.get(pageName);
// const newPageMeta: PageMeta = { ...meta };
// if (
// !oldPageMeta &&
// (pageName.startsWith(plugPrefix) || !this.initialPageListLoad)
// ) {
// this.emit("pageCreated", newPageMeta);
// } else if (
// oldPageMeta &&
// oldPageMeta.lastModified !== newPageMeta.lastModified
// ) {
// this.emit("pageChanged", newPageMeta);
// }
// // Page found, not deleted
// deletedPages.delete(pageName);
// // Update in cache
// this.pageMetaCache.set(pageName, newPageMeta);
// });
// for (const deletedPage of deletedPages) {
// this.pageMetaCache.delete(deletedPage);
// this.emit("pageDeleted", deletedPage);
// }
// this.emit("pageListUpdated", this.listPages());
// this.initialPageListLoad = false;
}
async deletePage(name: string): Promise<void> {

21
web/syscalls/system.ts
View File

@ -2,17 +2,27 @@ import type { Plug } from "../../plugos/plug.ts";
import { SysCallMapping, System } from "../../plugos/system.ts";
import type { Client } from "../client.ts";
import { CommandDef } from "../hooks/command.ts";
import { proxySyscall } from "./util.ts";
export function systemSyscalls(
editor: Client,
system: System<any>,
): SysCallMapping {
return {
const api: SysCallMapping = {
"system.invokeFunction": (
ctx,
_env: string,
name: string,
...args: any[]
) => {
// For backwards compatibility
// TODO: Remove at some point
return api["system.invoke"](ctx, name, ...args);
},
"system.invoke": (
ctx,
name: string,
...args: any[]
) => {
if (!ctx.plug) {
throw Error("No plug associated with context");
@ -28,6 +38,14 @@ export function systemSyscalls(
}
name = functionName;
}
const functionDef = plug.manifest!.functions[name];
if (!functionDef) {
throw Error(`Function ${name} not found`);
}
if (functionDef.env && system.env && functionDef.env !== system.env) {
// Proxy to another environment
return proxySyscall(editor.remoteSpacePrimitives, name, args);
}
return plug.invoke(name, args);
},
"system.invokeCommand": (_ctx, name: string) => {
@ -47,4 +65,5 @@ export function systemSyscalls(
return system.env;
},
};
return api;
}

50
web/syscalls/util.ts
View File

@ -1,3 +1,4 @@
import { HttpSpacePrimitives } from "../../common/spaces/http_space_primitives.ts";
import { SysCallMapping } from "../../plugos/system.ts";
import { SyscallResponse } from "../../server/rpc.ts";
import { Client } from "../client.ts";
@ -5,29 +6,34 @@ import { Client } from "../client.ts";
export function proxySyscalls(client: Client, names: string[]): SysCallMapping {
const syscalls: SysCallMapping = {};
for (const name of names) {
syscalls[name] = async (_ctx, ...args: any[]) => {
if (!client.remoteSpacePrimitives) {
throw new Error("Not supported");
}
const resp = await client.remoteSpacePrimitives.authenticatedFetch(
`${client.remoteSpacePrimitives.url}/.rpc`,
{
method: "POST",
body: JSON.stringify({
operation: "syscall",
name,
args,
}),
},
);
const result: SyscallResponse = await resp.json();
if (result.error) {
console.error("Remote syscall error", result.error);
throw new Error(result.error);
} else {
return result.result;
}
syscalls[name] = (_ctx, ...args: any[]) => {
return proxySyscall(client.remoteSpacePrimitives, name, args);
};
}
return syscalls;
}
export async function proxySyscall(
httpSpacePrimitives: HttpSpacePrimitives,
name: string,
args: any[],
): Promise<any> {
const resp = await httpSpacePrimitives.authenticatedFetch(
`${httpSpacePrimitives.url}/.rpc`,
{
method: "POST",
body: JSON.stringify({
operation: "syscall",
name,
args,
}),
},
);
const result: SyscallResponse = await resp.json();
if (result.error) {
console.error("Remote syscall error", result.error);
throw new Error(result.error);
} else {
return result.result;
}
}