From 76636dd9b1b25aeae24def2ef4088860189eedff Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Tue, 26 Apr 2022 19:04:36 +0200 Subject: [PATCH] Work --- .DS_Store | Bin 6148 -> 6148 bytes .../common/spaces/evented_space_primitives.ts | 4 +- packages/common/spaces/space.ts | 117 +++++------- .../plugos-silverbullet-syscall/editor.ts | 8 + packages/plugos-silverbullet-syscall/space.ts | 4 +- .../plugos-silverbullet-syscall/system.ts | 4 + packages/plugos/bin/plugos-bundle.ts | 4 + packages/plugos/hooks/endpoint.test.ts | 2 +- packages/plugos/hooks/event.ts | 23 ++- packages/plugos/hooks/node_cron.ts | 4 +- packages/plugos/plug_loader.ts | 19 +- packages/plugos/runtime.test.ts | 2 +- packages/plugos/syscalls/esbuild.ts | 2 - .../syscalls/store.dexie_browser.test.ts | 2 +- .../plugos/syscalls/store.knex_node.test.ts | 2 +- packages/plugos/system.ts | 24 +-- packages/plugos/types.ts | 1 + packages/plugs/core/core.plug.yaml | 1 + packages/plugs/emoji/emoji.plug.yaml | 1 + packages/plugs/ghost/ghost.plug.yaml | 1 + packages/plugs/git/git.plug.yaml | 1 + packages/plugs/markdown/markdown.plug.yaml | 1 + .../plugs/mattermost/mattermost.plug.yaml | 1 + packages/plugs/plugger/plugger.plug.yaml | 8 - packages/plugs/plugger/plugger.ts | 85 --------- packages/plugs/plugmd/plugmd.plug.yaml | 25 +++ packages/plugs/plugmd/plugmd.ts | 174 ++++++++++++++++++ packages/plugs/query/query.plug.yaml | 1 + packages/plugs/tasks/tasks.plug.yaml | 1 + packages/server/api_server.ts | 67 ++++--- packages/server/server.ts | 5 + packages/server/syscalls/space.ts | 4 +- packages/server/syscalls/system.ts | 21 +++ packages/web/components/status_bar.tsx | 2 +- packages/web/editor.tsx | 104 ++++++----- packages/web/reducer.ts | 12 ++ packages/web/styles/main.scss | 19 +- packages/web/syscalls/editor.ts | 12 ++ packages/web/syscalls/space.ts | 4 +- packages/web/syscalls/system.ts | 9 +- packages/web/types.ts | 6 + 41 files changed, 500 insertions(+), 287 deletions(-) delete mode 100644 packages/plugs/plugger/plugger.plug.yaml delete mode 100644 packages/plugs/plugger/plugger.ts create mode 100644 packages/plugs/plugmd/plugmd.plug.yaml create mode 100644 packages/plugs/plugmd/plugmd.ts create mode 100644 packages/server/syscalls/system.ts diff --git a/.DS_Store b/.DS_Store index 3e078dbdff5f062fbd9f49ce7c839161f4511cd3..5cc86e8093806fd13adba02ff7a8feeaaf282db1 100644 GIT binary patch delta 131 zcmZoMXfc=|#>B)qu~2NHo+2a5#DLw41(+BaStj!^ei!6mC}2orNM^_elIaYo48@b> z7#ldu%`J5lObrbuFJrWq#ZXcfT$GoSpO?nqhN*$QEV*-0U2H delta 69 zcmZoMXfc=|#>B`mu~2NHo+2aD#DLwC4MbQb^D}+f{F|AFWwQXw55~>x9Q+(WMVlE} XzB5ne7qR4E00Kq^2ByscB3qaNm>&@X diff --git a/packages/common/spaces/evented_space_primitives.ts b/packages/common/spaces/evented_space_primitives.ts index 0b41cc8..79278ce 100644 --- a/packages/common/spaces/evented_space_primitives.ts +++ b/packages/common/spaces/evented_space_primitives.ts @@ -2,7 +2,7 @@ import { SpacePrimitives } from "./space_primitives"; import { EventHook } from "@plugos/plugos/hooks/event"; import { PageMeta } from "../types"; import { Plug } from "@plugos/plugos/plug"; -import { trashPrefix } from "./constants"; +import { plugPrefix, trashPrefix } from "./constants"; export class EventedSpacePrimitives implements SpacePrimitives { constructor(private wrapped: SpacePrimitives, private eventHook: EventHook) {} @@ -41,7 +41,7 @@ export class EventedSpacePrimitives implements SpacePrimitives { lastModified ); // This can happen async - if (!pageName.startsWith(trashPrefix)) { + if (!pageName.startsWith(trashPrefix) && !pageName.startsWith(plugPrefix)) { this.eventHook .dispatchEvent("page:saved", pageName) .then(() => { diff --git a/packages/common/spaces/space.ts b/packages/common/spaces/space.ts index 02118b7..a7e340c 100644 --- a/packages/common/spaces/space.ts +++ b/packages/common/spaces/space.ts @@ -13,8 +13,6 @@ export type SpaceEvents = { pageChanged: (meta: PageMeta) => void; pageDeleted: (name: string) => void; pageListUpdated: (pages: Set) => void; - plugLoaded: (plugName: string, plug: Manifest) => void; - plugUnloaded: (plugName: string) => void; }; export class Space extends EventEmitter { @@ -25,70 +23,45 @@ export class Space extends EventEmitter { constructor(private space: SpacePrimitives, private trashEnabled = true) { super(); - this.on({ - pageCreated: async (pageMeta) => { - if (pageMeta.name.startsWith(plugPrefix)) { - let pageData = await this.readPage(pageMeta.name); - this.emit( - "plugLoaded", - pageMeta.name.substring(plugPrefix.length), - JSON.parse(pageData.text) - ); - this.watchPage(pageMeta.name); - } - }, - pageChanged: async (pageMeta) => { - if (pageMeta.name.startsWith(plugPrefix)) { - let pageData = await this.readPage(pageMeta.name); - this.emit( - "plugLoaded", - pageMeta.name.substring(plugPrefix.length), - JSON.parse(pageData.text) - ); - } - }, - }); } - public updatePageListAsync() { - safeRun(async () => { - let newPageList = await this.space.fetchPageList(); - let deletedPages = new Set(this.pageMetaCache.keys()); - newPageList.pages.forEach((meta) => { - const pageName = meta.name; - const oldPageMeta = this.pageMetaCache.get(pageName); - const newPageMeta = { - name: pageName, - lastModified: meta.lastModified, - }; - if ( - !oldPageMeta && - (pageName.startsWith(plugPrefix) || !this.initialPageListLoad) - ) { - this.emit("pageCreated", newPageMeta); - } else if ( - oldPageMeta && - oldPageMeta.lastModified !== newPageMeta.lastModified && - (!this.trashEnabled || - (this.trashEnabled && !pageName.startsWith(trashPrefix))) - ) { - this.emit("pageChanged", newPageMeta); - } - // Page found, not deleted - deletedPages.delete(pageName); - - // Update in cache - this.pageMetaCache.set(pageName, newPageMeta); - }); - - for (const deletedPage of deletedPages) { - this.pageMetaCache.delete(deletedPage); - this.emit("pageDeleted", deletedPage); + public async updatePageList() { + let newPageList = await this.space.fetchPageList(); + let deletedPages = new Set(this.pageMetaCache.keys()); + newPageList.pages.forEach((meta) => { + const pageName = meta.name; + const oldPageMeta = this.pageMetaCache.get(pageName); + const newPageMeta = { + name: pageName, + lastModified: meta.lastModified, + }; + if ( + !oldPageMeta && + (pageName.startsWith(plugPrefix) || !this.initialPageListLoad) + ) { + this.emit("pageCreated", newPageMeta); + } else if ( + oldPageMeta && + oldPageMeta.lastModified !== newPageMeta.lastModified && + (!this.trashEnabled || + (this.trashEnabled && !pageName.startsWith(trashPrefix))) + ) { + this.emit("pageChanged", newPageMeta); } + // Page found, not deleted + deletedPages.delete(pageName); - this.emit("pageListUpdated", this.listPages()); - this.initialPageListLoad = false; + // Update in cache + this.pageMetaCache.set(pageName, newPageMeta); }); + + for (const deletedPage of deletedPages) { + this.pageMetaCache.delete(deletedPage); + this.emit("pageDeleted", deletedPage); + } + + this.emit("pageListUpdated", this.listPages()); + this.initialPageListLoad = false; } watch() { @@ -109,7 +82,7 @@ export class Space extends EventEmitter { } }); }, pageWatchInterval); - this.updatePageListAsync(); + this.updatePageList().catch(console.error); } async deletePage(name: string, deleteDate?: number): Promise { @@ -152,14 +125,18 @@ export class Space extends EventEmitter { return this.space.invokeFunction(plug, env, name, args); } - listPages(): Set { - return new Set( - [...this.pageMetaCache.values()].filter( - (pageMeta) => - !pageMeta.name.startsWith(trashPrefix) && - !pageMeta.name.startsWith(plugPrefix) - ) - ); + listPages(unfiltered = false): Set { + if (unfiltered) { + return new Set(this.pageMetaCache.values()); + } else { + return new Set( + [...this.pageMetaCache.values()].filter( + (pageMeta) => + !pageMeta.name.startsWith(trashPrefix) && + !pageMeta.name.startsWith(plugPrefix) + ) + ); + } } listTrash(): Set { diff --git a/packages/plugos-silverbullet-syscall/editor.ts b/packages/plugos-silverbullet-syscall/editor.ts index e408daf..a9fa788 100644 --- a/packages/plugos-silverbullet-syscall/editor.ts +++ b/packages/plugos-silverbullet-syscall/editor.ts @@ -62,6 +62,14 @@ export function hideLhs(): Promise { return syscall("editor.hideLhs"); } +export function showBhs(html: string, flex = 1): Promise { + return syscall("editor.showBhs", html, flex); +} + +export function hideBhs(): Promise { + return syscall("editor.hideBhs"); +} + export function insertAtPos(text: string, pos: number): Promise { return syscall("editor.insertAtPos", text, pos); } diff --git a/packages/plugos-silverbullet-syscall/space.ts b/packages/plugos-silverbullet-syscall/space.ts index f2de08e..41f23b3 100644 --- a/packages/plugos-silverbullet-syscall/space.ts +++ b/packages/plugos-silverbullet-syscall/space.ts @@ -1,8 +1,8 @@ import { syscall } from "./syscall"; import { PageMeta } from "../common/types"; -export async function listPages(): Promise { - return syscall("space.listPages"); +export async function listPages(unfiltered = false): Promise { + return syscall("space.listPages", unfiltered); } export async function readPage( diff --git a/packages/plugos-silverbullet-syscall/system.ts b/packages/plugos-silverbullet-syscall/system.ts index 104347d..af2220e 100644 --- a/packages/plugos-silverbullet-syscall/system.ts +++ b/packages/plugos-silverbullet-syscall/system.ts @@ -7,3 +7,7 @@ export async function invokeFunction( ): Promise { return syscall("system.invokeFunction", env, name, ...args); } + +export async function reloadPlugs() { + return syscall("system.reloadPlugs"); +} diff --git a/packages/plugos/bin/plugos-bundle.ts b/packages/plugos/bin/plugos-bundle.ts index 9a80b04..ccacbeb 100755 --- a/packages/plugos/bin/plugos-bundle.ts +++ b/packages/plugos/bin/plugos-bundle.ts @@ -20,6 +20,10 @@ async function bundle( (await readFile(manifestPath)).toString() ) as Manifest; + if (!manifest.name) { + throw new Error(`Missing 'name' in ${manifestPath}`); + } + for (let [name, def] of Object.entries(manifest.functions)) { let jsFunctionName = "default", filePath = path.join(rootPath, def.path!); diff --git a/packages/plugos/hooks/endpoint.test.ts b/packages/plugos/hooks/endpoint.test.ts index 39d674f..40b4334 100644 --- a/packages/plugos/hooks/endpoint.test.ts +++ b/packages/plugos/hooks/endpoint.test.ts @@ -9,8 +9,8 @@ import { System } from "../system"; test("Run a plugos endpoint server", async () => { let system = new System("server"); let plug = await system.load( - "test", { + name: "test", functions: { testhandler: { http: { diff --git a/packages/plugos/hooks/event.ts b/packages/plugos/hooks/event.ts index d90be3b..3d1e032 100644 --- a/packages/plugos/hooks/event.ts +++ b/packages/plugos/hooks/event.ts @@ -1,6 +1,7 @@ import { Hook, Manifest } from "../types"; import { System } from "../system"; import { safeRun } from "../util"; +import { EventEmitter } from "events"; // System events: // - plug:load (plugName: string) @@ -11,6 +12,14 @@ export type EventHookT = { export class EventHook implements Hook { private system?: System; + public localListeners: Map any)[]> = new Map(); + + addLocalListener(eventName: string, callback: (data: any) => any) { + if (!this.localListeners.has(eventName)) { + this.localListeners.set(eventName, []); + } + this.localListeners.get(eventName)!.push(callback); + } async dispatchEvent(eventName: string, data?: any): Promise { if (!this.system) { @@ -32,15 +41,25 @@ export class EventHook implements Hook { } } } + let localListeners = this.localListeners.get(eventName); + if (localListeners) { + for (let localListener of localListeners) { + let result = await Promise.resolve(localListener(data)); + if (result) { + responses.push(result); + } + } + } + return responses; } apply(system: System): void { this.system = system; this.system.on({ - plugLoaded: (name) => { + plugLoaded: (plug) => { safeRun(async () => { - await this.dispatchEvent("plug:load", name); + await this.dispatchEvent("plug:load", plug.name); }); }, }); diff --git a/packages/plugos/hooks/node_cron.ts b/packages/plugos/hooks/node_cron.ts index ddf4b97..f08589a 100644 --- a/packages/plugos/hooks/node_cron.ts +++ b/packages/plugos/hooks/node_cron.ts @@ -11,10 +11,10 @@ export class NodeCronHook implements Hook { apply(system: System): void { let tasks: ScheduledTask[] = []; system.on({ - plugLoaded: (name, plug) => { + plugLoaded: () => { reloadCrons(); }, - plugUnloaded(name, plug) { + plugUnloaded() { reloadCrons(); }, }); diff --git a/packages/plugos/plug_loader.ts b/packages/plugos/plug_loader.ts index 8c456cd..b121ed2 100644 --- a/packages/plugos/plug_loader.ts +++ b/packages/plugos/plug_loader.ts @@ -3,11 +3,7 @@ import watch from "node-watch"; import path from "path"; import { createSandbox } from "./environments/node_sandbox"; import { System } from "./system"; - -function extractPlugName(localPath: string): string { - const baseName = path.basename(localPath); - return baseName.substring(0, baseName.length - ".plug.json".length); -} +import { Manifest } from "./types"; export class DiskPlugLoader { private system: System; @@ -27,13 +23,13 @@ export class DiskPlugLoader { .then(async () => { try { // let localPath = path.join(this.plugPath, filename); - const plugName = extractPlugName(localPath); - console.log("Change detected for", plugName); + console.log("Change detected for", localPath); try { await fs.stat(localPath); } catch (e) { // Likely removed - await this.system.unload(plugName); + console.log("Plug removed, TODO: Unload"); + return; } const plugDef = await this.loadPlugFromFile(localPath); } catch (e) { @@ -47,12 +43,11 @@ export class DiskPlugLoader { private async loadPlugFromFile(localPath: string) { const plug = await fs.readFile(localPath, "utf8"); - const plugName = extractPlugName(localPath); - console.log("Now loading plug", plugName); try { - const plugDef = JSON.parse(plug); - await this.system.load(plugName, plugDef, createSandbox); + const plugDef: Manifest = JSON.parse(plug); + console.log("Now loading plug", plugDef.name); + await this.system.load(plugDef, createSandbox); return plugDef; } catch (e) { console.error("Could not parse plugin file", e); diff --git a/packages/plugos/runtime.test.ts b/packages/plugos/runtime.test.ts index 2fa2528..52ffd42 100644 --- a/packages/plugos/runtime.test.ts +++ b/packages/plugos/runtime.test.ts @@ -23,8 +23,8 @@ test("Run a Node sandbox", async () => { }, }); let plug = await system.load( - "test", { + name: "test", requiredPermissions: ["dangerous"], functions: { addTen: { diff --git a/packages/plugos/syscalls/esbuild.ts b/packages/plugos/syscalls/esbuild.ts index 786b966..f0f0888 100644 --- a/packages/plugos/syscalls/esbuild.ts +++ b/packages/plugos/syscalls/esbuild.ts @@ -34,9 +34,7 @@ export function esbuildSyscalls(): SysCallMapping { } await writeFile(`${tmpDir}/${filename}`, code); - console.log("Dir", tmpDir); let jsCode = await compile(`${tmpDir}/${filename}`, "", false, ["yaml"]); - // console.log("JS code", jsCode); await rm(tmpDir, { recursive: true }); return jsCode; }, diff --git a/packages/plugos/syscalls/store.dexie_browser.test.ts b/packages/plugos/syscalls/store.dexie_browser.test.ts index 124ecd3..9480077 100644 --- a/packages/plugos/syscalls/store.dexie_browser.test.ts +++ b/packages/plugos/syscalls/store.dexie_browser.test.ts @@ -10,8 +10,8 @@ test("Test store", async () => { let system = new System("server"); system.registerSyscalls([], storeSyscalls("test", "test")); let plug = await system.load( - "test", { + name: "test", functions: { test1: { code: `(() => { diff --git a/packages/plugos/syscalls/store.knex_node.test.ts b/packages/plugos/syscalls/store.knex_node.test.ts index 51af124..0015930 100644 --- a/packages/plugos/syscalls/store.knex_node.test.ts +++ b/packages/plugos/syscalls/store.knex_node.test.ts @@ -17,8 +17,8 @@ test("Test store", async () => { let system = new System("server"); system.registerSyscalls([], storeSyscalls(db, "test_table")); let plug = await system.load( - "test", { + name: "test", functions: { test1: { code: `(() => { diff --git a/packages/plugos/system.ts b/packages/plugos/system.ts index 99ef8e4..6f9ad63 100644 --- a/packages/plugos/system.ts +++ b/packages/plugos/system.ts @@ -7,11 +7,11 @@ export interface SysCallMapping { [key: string]: (ctx: SyscallContext, ...args: any) => Promise | any; } -export type SystemJSON = { [key: string]: Manifest }; +export type SystemJSON = Manifest[]; export type SystemEvents = { - plugLoaded: (name: string, plug: Plug) => void; - plugUnloaded: (name: string, plug: Plug) => void; + plugLoaded: (plug: Plug) => void; + plugUnloaded: (name: string) => void; }; export type SyscallContext = { @@ -83,10 +83,10 @@ export class System extends EventEmitter> { } async load( - name: string, manifest: Manifest, sandboxFactory: SandboxFactory ): Promise> { + const name = manifest.name; if (this.plugs.has(name)) { await this.unload(name); } @@ -100,29 +100,31 @@ export class System extends EventEmitter> { } // Ok, let's load this thing! const plug = new Plug(this, name, sandboxFactory); + console.log("Loading", name); await plug.load(manifest); this.plugs.set(name, plug); - this.emit("plugLoaded", name, plug); + this.emit("plugLoaded", plug); return plug; } async unload(name: string) { + console.log("Unloading", name); const plug = this.plugs.get(name); if (!plug) { throw Error(`Plug ${name} not found`); } await plug.stop(); - this.emit("plugUnloaded", name, plug); + this.emit("plugUnloaded", name); this.plugs.delete(name); } toJSON(): SystemJSON { - let plugJSON: { [key: string]: Manifest } = {}; + let plugJSON: Manifest[] = []; for (let [name, plug] of this.plugs) { if (!plug.manifest) { continue; } - plugJSON[name] = plug.manifest; + plugJSON.push(plug.manifest); } return plugJSON; } @@ -132,9 +134,9 @@ export class System extends EventEmitter> { sandboxFactory: SandboxFactory ) { await this.unloadAll(); - for (let [name, manifest] of Object.entries(json)) { - console.log("Loading plug", name); - await this.load(name, manifest, sandboxFactory); + for (let manifest of json) { + console.log("Loading plug", manifest.name); + await this.load(manifest, sandboxFactory); } } diff --git a/packages/plugos/types.ts b/packages/plugos/types.ts index a671b7a..4ec82e9 100644 --- a/packages/plugos/types.ts +++ b/packages/plugos/types.ts @@ -1,6 +1,7 @@ import { System } from "./system"; export interface Manifest { + name: string; requiredPermissions?: string[]; functions: { [key: string]: FunctionDef; diff --git a/packages/plugs/core/core.plug.yaml b/packages/plugs/core/core.plug.yaml index be3ea0c..f75bc88 100644 --- a/packages/plugs/core/core.plug.yaml +++ b/packages/plugs/core/core.plug.yaml @@ -1,3 +1,4 @@ +name: core syntax: HashTag: firstCharacters: diff --git a/packages/plugs/emoji/emoji.plug.yaml b/packages/plugs/emoji/emoji.plug.yaml index dc53541..e6c4a8e 100644 --- a/packages/plugs/emoji/emoji.plug.yaml +++ b/packages/plugs/emoji/emoji.plug.yaml @@ -1,3 +1,4 @@ +name: emoji functions: emojiCompleter: path: "./emoji.ts:emojiCompleter" diff --git a/packages/plugs/ghost/ghost.plug.yaml b/packages/plugs/ghost/ghost.plug.yaml index 0d62c1c..1ab0f60 100644 --- a/packages/plugs/ghost/ghost.plug.yaml +++ b/packages/plugs/ghost/ghost.plug.yaml @@ -1,3 +1,4 @@ +name: ghost functions: downloadAllPostsCommand: path: "./ghost.ts:downloadAllPostsCommand" diff --git a/packages/plugs/git/git.plug.yaml b/packages/plugs/git/git.plug.yaml index 08d6b12..34c7ea2 100644 --- a/packages/plugs/git/git.plug.yaml +++ b/packages/plugs/git/git.plug.yaml @@ -1,3 +1,4 @@ +name: git requiredPermissions: - shell functions: diff --git a/packages/plugs/markdown/markdown.plug.yaml b/packages/plugs/markdown/markdown.plug.yaml index 733fceb..ccad33c 100644 --- a/packages/plugs/markdown/markdown.plug.yaml +++ b/packages/plugs/markdown/markdown.plug.yaml @@ -1,3 +1,4 @@ +name: markdown functions: toggle: path: "./markdown.ts:togglePreview" diff --git a/packages/plugs/mattermost/mattermost.plug.yaml b/packages/plugs/mattermost/mattermost.plug.yaml index 6f333a3..922a6d3 100644 --- a/packages/plugs/mattermost/mattermost.plug.yaml +++ b/packages/plugs/mattermost/mattermost.plug.yaml @@ -1,3 +1,4 @@ +name: mattermost functions: test: path: mattermost.ts:savedPostsQueryProvider diff --git a/packages/plugs/plugger/plugger.plug.yaml b/packages/plugs/plugger/plugger.plug.yaml deleted file mode 100644 index 6f4202c..0000000 --- a/packages/plugs/plugger/plugger.plug.yaml +++ /dev/null @@ -1,8 +0,0 @@ -functions: - compile: - path: "./plugger.ts:compileCommand" - command: - name: "Plugger: Compile" - compileJS: - path: "./plugger.ts:compileJS" - env: server \ No newline at end of file diff --git a/packages/plugs/plugger/plugger.ts b/packages/plugs/plugger/plugger.ts deleted file mode 100644 index ea24ac9..0000000 --- a/packages/plugs/plugger/plugger.ts +++ /dev/null @@ -1,85 +0,0 @@ -import type { Manifest } from "@silverbulletmd/common/manifest"; -import { - addParentPointers, - collectNodesOfType, - findNodeOfType, -} from "@silverbulletmd/common/tree"; -import { - getCurrentPage, - getText, -} from "@silverbulletmd/plugos-silverbullet-syscall/editor"; -import { parseMarkdown } from "@silverbulletmd/plugos-silverbullet-syscall/markdown"; -import { writePage } from "@silverbulletmd/plugos-silverbullet-syscall/space"; -import { invokeFunction } from "@silverbulletmd/plugos-silverbullet-syscall/system"; -import YAML from "yaml"; -import { extractMeta } from "../query/data"; - -export async function compileCommand() { - let text = await getText(); - let tree = await parseMarkdown(text); - addParentPointers(tree); - let allHeaders = collectNodesOfType(tree, "ATXHeading2"); - let manifest: Manifest = { - functions: {}, - }; - for (let t of allHeaders) { - let parent = t.parent!; - let headerIdx = parent.children!.indexOf(t); - let headerTitle = t.children![1].text!.trim(); - if (!headerTitle.startsWith("function ")) { - continue; - } - let functionName = headerTitle - .substring("function ".length) - .replace(/[^\w]/g, "_"); - let meta: any; - let code: string | undefined; - let language = "js"; - for (let i = headerIdx + 1; i < parent.children!.length; i++) { - let child = parent.children![i]; - if (child.type === "FencedCode") { - let codeInfo = findNodeOfType(child, "CodeInfo")!.children![0].text!; - let codeText = findNodeOfType(child, "CodeText")!.children![0].text!; - if (codeInfo === "yaml") { - meta = YAML.parse(codeText); - continue; - } - if (codeInfo === "typescript" || codeInfo === "ts") { - language = "ts"; - } - code = codeText; - } - - if (child.type?.startsWith("ATXHeading")) { - break; - } - } - if (code) { - let compiled = await invokeFunction( - "server", - "compileJS", - `file.${language}`, - code - ); - manifest.functions[functionName] = meta; - manifest.functions[functionName].code = compiled; - } - } - - let pageMeta = extractMeta(tree); - - if (pageMeta.name) { - await writePage( - `_plug/${pageMeta.name}`, - JSON.stringify(manifest, null, 2) - ); - console.log("Wrote this plug", manifest); - } -} - -export async function compileJS( - filename: string, - code: string -): Promise { - return self.syscall("esbuild.compile", filename, code); -} diff --git a/packages/plugs/plugmd/plugmd.plug.yaml b/packages/plugs/plugmd/plugmd.plug.yaml new file mode 100644 index 0000000..2440bc9 --- /dev/null +++ b/packages/plugs/plugmd/plugmd.plug.yaml @@ -0,0 +1,25 @@ +name: plugmd +functions: + updatePlugsCommand: + path: ./plugmd.ts:updatePlugsCommand + command: + name: "Plugs: Update" + key: "Ctrl-Shift-p" + mac: "Cmd-Shift-p" + updatePlugs: + path: ./plugmd.ts:updatePlugs + env: server + compile: + path: "./plugmd.ts:compileCommand" + command: + name: "Plug: Compile" + mac: "Cmd-Shift-c" + key: "Ctrl-Shift-c" + compileJS: + path: "./plugmd.ts:compileJS" + env: server + + getPlugPlugMd: + path: "./plugmd.ts:getPlugPlugMd" + events: + - get-plug:plugmd diff --git a/packages/plugs/plugmd/plugmd.ts b/packages/plugs/plugmd/plugmd.ts new file mode 100644 index 0000000..633ef41 --- /dev/null +++ b/packages/plugs/plugmd/plugmd.ts @@ -0,0 +1,174 @@ +import { dispatch } from "@plugos/plugos-syscall/event"; +import type { Manifest } from "@silverbulletmd/common/manifest"; +import { + addParentPointers, + collectNodesOfType, + findNodeOfType, +} from "@silverbulletmd/common/tree"; +import { + getCurrentPage, + getText, + hideBhs, + showBhs, +} from "@silverbulletmd/plugos-silverbullet-syscall/editor"; +import { parseMarkdown } from "@silverbulletmd/plugos-silverbullet-syscall/markdown"; +import { + deletePage, + listPages, + readPage, + writePage, +} from "@silverbulletmd/plugos-silverbullet-syscall/space"; +import { + invokeFunction, + reloadPlugs, +} from "@silverbulletmd/plugos-silverbullet-syscall/system"; +import YAML from "yaml"; +import { extractMeta } from "../query/data"; + +export async function compileCommand() { + let text = await getText(); + try { + let manifest = await compileDefinition(text); + await writePage( + `_plug/${manifest.name}`, + JSON.stringify(manifest, null, 2) + ); + console.log("Wrote this plug", manifest); + await hideBhs(); + // Important not to await! + reloadPlugs(); + } catch (e: any) { + await showBhs(e.message); + // console.error("Got this error from compiler", e.message); + } +} + +async function compileDefinition(text: string): Promise { + let tree = await parseMarkdown(text); + + let pageMeta = extractMeta(tree); + + if (!pageMeta.name) { + throw new Error("No 'name' specified in page meta"); + } + + addParentPointers(tree); + let allHeaders = collectNodesOfType(tree, "ATXHeading2"); + let manifest: Manifest = { + name: pageMeta.name, + functions: {}, + }; + for (let t of allHeaders) { + let parent = t.parent!; + let headerIdx = parent.children!.indexOf(t); + let headerTitle = t.children![1].text!.trim(); + if (!headerTitle.startsWith("function ")) { + continue; + } + let functionName = headerTitle + .substring("function ".length) + .replace(/[^\w]/g, "_"); + let meta: any; + let code: string | undefined; + let language = "js"; + for (let i = headerIdx + 1; i < parent.children!.length; i++) { + let child = parent.children![i]; + if (child.type === "FencedCode") { + let codeInfo = findNodeOfType(child, "CodeInfo")!.children![0].text!; + let codeText = findNodeOfType(child, "CodeText")!.children![0].text!; + if (codeInfo === "yaml") { + meta = YAML.parse(codeText); + continue; + } + if (codeInfo === "typescript" || codeInfo === "ts") { + language = "ts"; + } + code = codeText; + } + + if (child.type?.startsWith("ATXHeading")) { + break; + } + } + if (code) { + let compiled = await invokeFunction( + "server", + "compileJS", + `file.${language}`, + code + ); + manifest.functions[functionName] = meta; + manifest.functions[functionName].code = compiled; + } + } + return manifest; +} + +export async function compileJS( + filename: string, + code: string +): Promise { + return self.syscall("esbuild.compile", filename, code); +} + +async function listPlugs(): Promise { + let unfilteredPages = await listPages(true); + return unfilteredPages + .filter((p) => p.name.startsWith("_plug/")) + .map((p) => p.name.substring("_plug/".length)); +} + +export async function listCommand() { + console.log(await listPlugs()); +} + +export async function updatePlugsCommand() { + await invokeFunction("server", "updatePlugs"); + await reloadPlugs(); +} + +export async function updatePlugs() { + let { text: plugPageText } = await readPage("PLUGS"); + + let tree = await parseMarkdown(plugPageText); + + let codeTextNode = findNodeOfType(tree, "CodeText"); + if (!codeTextNode) { + console.error("Could not find yaml block in PLUGS"); + return; + } + let plugYaml = codeTextNode.children![0].text; + let plugList = YAML.parse(plugYaml!); + console.log("Plug YAML", plugList); + let allPlugNames: string[] = []; + for (let plugUri of plugList) { + let [protocol, ...rest] = plugUri.split(":"); + let manifests = await dispatch(`get-plug:${protocol}`, rest.join(":")); + if (manifests.length === 0) { + console.error("Could not resolve plug", plugUri); + } + // console.log("Got manifests", plugUri, protocol, manifests); + let manifest = manifests[0]; + allPlugNames.push(manifest.name); + // console.log("Writing", `_plug/${manifest.name}`); + await writePage( + `_plug/${manifest.name}`, + JSON.stringify(manifest, null, 2) + ); + } + + // And delete extra ones + for (let existingPlug of await listPlugs()) { + if (!allPlugNames.includes(existingPlug)) { + console.log("Removing plug", existingPlug); + await deletePage(`_plug/${existingPlug}`); + } + } + // Important not to await! + reloadPlugs(); +} + +export async function getPlugPlugMd(pageName: string): Promise { + let { text } = await readPage(pageName); + return compileDefinition(text); +} diff --git a/packages/plugs/query/query.plug.yaml b/packages/plugs/query/query.plug.yaml index 4829744..515c3f2 100644 --- a/packages/plugs/query/query.plug.yaml +++ b/packages/plugs/query/query.plug.yaml @@ -1,3 +1,4 @@ +name: query functions: updateMaterializedQueriesOnPage: path: ./materialized_queries.ts:updateMaterializedQueriesOnPage diff --git a/packages/plugs/tasks/tasks.plug.yaml b/packages/plugs/tasks/tasks.plug.yaml index 59a6c66..0d675ae 100644 --- a/packages/plugs/tasks/tasks.plug.yaml +++ b/packages/plugs/tasks/tasks.plug.yaml @@ -1,3 +1,4 @@ +name: tasks syntax: DeadlineDate: firstCharacters: diff --git a/packages/server/api_server.ts b/packages/server/api_server.ts index f3e3b3a..a24976e 100644 --- a/packages/server/api_server.ts +++ b/packages/server/api_server.ts @@ -1,5 +1,5 @@ import express, { Express } from "express"; -import { SilverBulletHooks } from "@silverbulletmd/common/manifest"; +import { Manifest, SilverBulletHooks } from "@silverbulletmd/common/manifest"; import { EndpointHook } from "@plugos/plugos/hooks/endpoint"; import { readFile } from "fs/promises"; import { System } from "@plugos/plugos/system"; @@ -24,36 +24,40 @@ import buildMarkdown from "@silverbulletmd/web/parser"; import { loadMarkdownExtensions } from "@silverbulletmd/web/markdown_ext"; import http, { Server } from "http"; import { esbuildSyscalls } from "@plugos/plugos/syscalls/esbuild"; +import { systemSyscalls } from "./syscalls/system"; export class ExpressServer { app: Express; system: System; - private rootPath: string; private space: Space; private distDir: string; private eventHook: EventHook; private db: Knex; private port: number; private server?: Server; + builtinPlugDir: string; + preloadedModules: string[]; constructor( port: number, - rootPath: string, + pagesPath: string, distDir: string, + builtinPlugDir: string, preloadedModules: string[] ) { this.port = port; this.app = express(); - this.rootPath = rootPath; + this.builtinPlugDir = builtinPlugDir; this.distDir = distDir; this.system = new System("server"); + this.preloadedModules = preloadedModules; // Setup system this.eventHook = new EventHook(); this.system.addHook(this.eventHook); this.space = new Space( new EventedSpacePrimitives( - new DiskSpacePrimitives(rootPath), + new DiskSpacePrimitives(pagesPath), this.eventHook ), true @@ -61,12 +65,12 @@ export class ExpressServer { this.db = knex({ client: "better-sqlite3", connection: { - filename: path.join(rootPath, "data.db"), + filename: path.join(pagesPath, "data.db"), }, useNullAsDefault: true, }); - this.system.registerSyscalls(["shell"], shellSyscalls(rootPath)); + this.system.registerSyscalls(["shell"], shellSyscalls(pagesPath)); this.system.addHook(new NodeCronHook()); this.system.registerSyscalls([], pageIndexSyscalls(this.db)); @@ -74,6 +78,7 @@ export class ExpressServer { this.system.registerSyscalls([], eventSyscalls(this.eventHook)); this.system.registerSyscalls([], markdownSyscalls(buildMarkdown([]))); this.system.registerSyscalls([], esbuildSyscalls()); + this.system.registerSyscalls([], systemSyscalls(this)); this.system.registerSyscalls([], jwtSyscalls()); this.system.addHook(new EndpointHook(this.app, "/_/")); @@ -81,29 +86,26 @@ export class ExpressServer { this.rebuildMdExtensions(); }, 100); - this.space.on({ - plugLoaded: (plugName, plug) => { - safeRun(async () => { - console.log("Plug load", plugName); - await this.system.load(plugName, plug, (p) => - createSandbox(p, preloadedModules) + this.eventHook.addLocalListener( + "get-plug:builtin", + async (plugName: string): Promise => { + // console.log("Ok, resovling a plugin", plugName); + try { + let manifestJson = await readFile( + path.join(this.builtinPlugDir, `${plugName}.plug.json`), + "utf8" ); - }); - throttledRebuildMdExtensions(); - }, - plugUnloaded: (plugName) => { - safeRun(async () => { - console.log("Plug unload", plugName); - await this.system.unload(plugName); - }); - throttledRebuildMdExtensions(); - }, - }); + return JSON.parse(manifestJson); + } catch (e) { + throw new Error(`No such builtin: ${plugName}`); + } + } + ); setInterval(() => { - this.space.updatePageListAsync(); + this.space.updatePageList().catch(console.error); }, 5000); - this.space.updatePageListAsync(); + this.reloadPlugs().catch(console.error); } rebuildMdExtensions() { @@ -113,6 +115,19 @@ export class ExpressServer { ); } + async reloadPlugs() { + await this.space.updatePageList(); + await this.system.unloadAll(); + console.log("Reloading plugs"); + for (let pageInfo of this.space.listPlugs()) { + let { text } = await this.space.readPage(pageInfo.name); + await this.system.load(JSON.parse(text), (p) => + createSandbox(p, this.preloadedModules) + ); + } + this.rebuildMdExtensions(); + } + async start() { await ensurePageIndexTable(this.db); console.log("Setting up router"); diff --git a/packages/server/server.ts b/packages/server/server.ts index f05a7be..479f37c 100755 --- a/packages/server/server.ts +++ b/packages/server/server.ts @@ -28,11 +28,16 @@ const webappDistDir = realpathSync( `${nodeModulesDir}/node_modules/@silverbulletmd/web/dist` ); console.log("Webapp dist dir", webappDistDir); +const plugDistDir = realpathSync( + `${nodeModulesDir}/node_modules/@silverbulletmd/plugs/dist` +); +console.log("Builtin plug dist dir", plugDistDir); const expressServer = new ExpressServer( port, pagesPath, webappDistDir, + plugDistDir, preloadModules ); expressServer.start().catch((e) => { diff --git a/packages/server/syscalls/space.ts b/packages/server/syscalls/space.ts index 4f9499f..3d93ba0 100644 --- a/packages/server/syscalls/space.ts +++ b/packages/server/syscalls/space.ts @@ -4,8 +4,8 @@ import { Space } from "@silverbulletmd/common/spaces/space"; export default (space: Space): SysCallMapping => { return { - "space.listPages": async (ctx): Promise => { - return [...space.listPages()]; + "space.listPages": async (ctx, unfiltered = false): Promise => { + return [...space.listPages(unfiltered)]; }, "space.readPage": async ( ctx, diff --git a/packages/server/syscalls/system.ts b/packages/server/syscalls/system.ts new file mode 100644 index 0000000..b72bcb4 --- /dev/null +++ b/packages/server/syscalls/system.ts @@ -0,0 +1,21 @@ +import { SysCallMapping } from "@plugos/plugos/system"; +import type { ExpressServer } from "../api_server"; + +export function systemSyscalls(expressServer: ExpressServer): SysCallMapping { + return { + "system.invokeFunction": async ( + ctx, + env: string, + name: string, + ...args: any[] + ) => { + if (!ctx.plug) { + throw Error("No plug associated with context"); + } + return ctx.plug.invoke(name, args); + }, + "system.reloadPlugs": async () => { + return expressServer.reloadPlugs(); + }, + }; +} diff --git a/packages/web/components/status_bar.tsx b/packages/web/components/status_bar.tsx index 4f561c2..3a471c3 100644 --- a/packages/web/components/status_bar.tsx +++ b/packages/web/components/status_bar.tsx @@ -10,7 +10,7 @@ export function StatusBar({ editorView }: { editorView?: EditorView }) { readingTime = util.readingTime(wordCount); } return ( -
+
{wordCount} words | {readingTime} min
diff --git a/packages/web/editor.tsx b/packages/web/editor.tsx index 151d11d..6970385 100644 --- a/packages/web/editor.tsx +++ b/packages/web/editor.tsx @@ -58,13 +58,10 @@ import { FilterOption } from "@silverbulletmd/common/types"; import { syntaxTree } from "@codemirror/language"; class PageState { - scrollTop: number; - selection: EditorSelection; - - constructor(scrollTop: number, selection: EditorSelection) { - this.scrollTop = scrollTop; - this.selection = selection; - } + constructor( + readonly scrollTop: number, + readonly selection: EditorSelection + ) {} } const saveInterval = 1000; @@ -123,7 +120,7 @@ export class Editor { this.system.registerSyscalls([], editorSyscalls(this)); this.system.registerSyscalls([], spaceSyscalls(this)); this.system.registerSyscalls([], indexerSyscalls(this.space)); - this.system.registerSyscalls([], systemSyscalls(this.space)); + this.system.registerSyscalls([], systemSyscalls(this)); this.system.registerSyscalls( [], markdownSyscalls(buildMarkdown(this.mdExtensions)) @@ -153,10 +150,6 @@ export class Editor { } }); - let throttledRebuildEditorState = throttle(() => { - this.rebuildEditorState(); - }, 100); - this.space.on({ pageChanged: (meta) => { if (this.currentPage === meta.name) { @@ -171,22 +164,10 @@ export class Editor { pages: pages, }); }, - plugLoaded: (plugName, plug) => { - safeRun(async () => { - console.log("Plug load", plugName); - await this.system.load(plugName, plug, createIFrameSandbox); - throttledRebuildEditorState(); - }); - }, - plugUnloaded: (plugName) => { - safeRun(async () => { - console.log("Plug unload", plugName); - await this.system.unload(plugName); - throttledRebuildEditorState(); - }); - }, }); + await this.reloadPlugs(); + if (this.pageNavigator.getCurrentPage() === "") { await this.pageNavigator.navigate("start"); } @@ -359,7 +340,7 @@ export class Editor { mac: "Cmd-k", run: (): boolean => { this.viewDispatch({ type: "start-navigate" }); - this.space.updatePageListAsync(); + this.space.updatePageList(); return true; }, }, @@ -427,6 +408,17 @@ export class Editor { }); } + async reloadPlugs() { + await this.space.updatePageList(); + await this.system.unloadAll(); + console.log("(Re)loading plugs"); + for (let pageInfo of this.space.listPlugs()) { + let { text } = await this.space.readPage(pageInfo.name); + await this.system.load(JSON.parse(text), createIFrameSandbox); + } + this.rebuildEditorState(); + } + rebuildEditorState() { const editorView = this.editorView; if (editorView && this.currentPage) { @@ -438,13 +430,16 @@ export class Editor { markdownSyscalls(buildMarkdown(this.mdExtensions)) ); + this.saveState(); + editorView.setState( this.createEditorState(this.currentPage, editorView.state.sliceDoc()) ); if (editorView.contentDOM) { editorView.contentDOM.spellcheck = true; } - editorView.focus(); + + this.restoreState(this.currentPage); } } @@ -489,12 +484,7 @@ export class Editor { // Persist current page state and nicely close page if (this.currentPage) { - let pageState = this.openPages.get(this.currentPage); - if (pageState) { - pageState.selection = this.editorView!.state.selection; - pageState.scrollTop = this.editorView!.scrollDOM.scrollTop; - // console.log("Saved pageState", this.currentPage, pageState); - } + this.saveState(); this.space.unwatchPage(this.currentPage); await this.save(true); } @@ -513,26 +503,11 @@ export class Editor { } let editorState = this.createEditorState(pageName, doc.text); - let pageState = this.openPages.get(pageName); editorView.setState(editorState); if (editorView.contentDOM) { editorView.contentDOM.spellcheck = true; } - if (!pageState) { - pageState = new PageState(0, editorState.selection); - this.openPages.set(pageName, pageState!); - editorView.dispatch({ - selection: { anchor: 0 }, - }); - } else { - // Restore state - // console.log("Restoring selection state", pageState); - editorView.dispatch({ - selection: pageState.selection, - }); - editorView.scrollDOM.scrollTop = pageState!.scrollTop; - } - + this.restoreState(pageName); this.space.watchPage(pageName); this.viewDispatch({ @@ -543,6 +518,30 @@ export class Editor { await this.eventHook.dispatchEvent("editor:pageSwitched"); } + private restoreState(pageName: string) { + let pageState = this.openPages.get(pageName); + const editorView = this.editorView!; + if (pageState) { + // Restore state + // console.log("Restoring selection state", pageState); + editorView.dispatch({ + selection: pageState.selection, + }); + editorView.scrollDOM.scrollTop = pageState!.scrollTop; + } + editorView.focus(); + } + + private saveState() { + this.openPages.set( + this.currentPage!, + new PageState( + this.editorView!.scrollDOM.scrollTop, + this.editorView!.state.selection + ) + ); + } + ViewComponent(): React.ReactElement { const [viewState, dispatch] = useReducer(reducer, initialViewState); this.viewState = viewState; @@ -625,6 +624,11 @@ export class Editor { )}
+ {!!viewState.showBHS && ( +
+ +
+ )} ); diff --git a/packages/web/reducer.ts b/packages/web/reducer.ts index bcc9e2b..9168870 100644 --- a/packages/web/reducer.ts +++ b/packages/web/reducer.ts @@ -102,6 +102,18 @@ export default function reducer( showLHS: 0, lhsHTML: "", }; + case "show-bhs": + return { + ...state, + showBHS: action.flex, + bhsHTML: action.html, + }; + case "hide-bhs": + return { + ...state, + showBHS: 0, + bhsHTML: "", + }; case "show-filterbox": return { ...state, diff --git a/packages/web/styles/main.scss b/packages/web/styles/main.scss index 2e61188..cf66395 100644 --- a/packages/web/styles/main.scss +++ b/packages/web/styles/main.scss @@ -110,13 +110,26 @@ body { height: 100%; } -#bottom { +#bhs { + height: 200px; + width: 100%; + + .panel { + iframe { + border: 0; + width: 100%; + height: 100%; + padding: 0; + margin: 0; + } + } +} + +#status-bar { height: 40px; line-height: 40px; padding: 0 10px; text-align: right; background-color: rgb(213, 213, 213); border-top: rgb(193, 193, 193) 1px solid; - .inner { - } } diff --git a/packages/web/syscalls/editor.ts b/packages/web/syscalls/editor.ts index f413218..00e6afd 100644 --- a/packages/web/syscalls/editor.ts +++ b/packages/web/syscalls/editor.ts @@ -92,6 +92,18 @@ export function editorSyscalls(editor: Editor): SysCallMapping { type: "hide-lhs", }); }, + "editor.showBhs": (ctx, html: string, flex: number) => { + editor.viewDispatch({ + type: "show-bhs", + flex, + html, + }); + }, + "editor.hideBhs": (ctx) => { + editor.viewDispatch({ + type: "hide-bhs", + }); + }, "editor.insertAtPos": (ctx, text: string, pos: number) => { editor.editorView!.dispatch({ changes: { diff --git a/packages/web/syscalls/space.ts b/packages/web/syscalls/space.ts index dcae710..3c9d852 100644 --- a/packages/web/syscalls/space.ts +++ b/packages/web/syscalls/space.ts @@ -4,8 +4,8 @@ import { PageMeta } from "@silverbulletmd/common/types"; export function spaceSyscalls(editor: Editor): SysCallMapping { return { - "space.listPages": async (): Promise => { - return [...(await editor.space.listPages())]; + "space.listPages": async (ctx, unfiltered = false): Promise => { + return [...(await editor.space.listPages(unfiltered))]; }, "space.readPage": async ( ctx, diff --git a/packages/web/syscalls/system.ts b/packages/web/syscalls/system.ts index fe402f7..2a82c37 100644 --- a/packages/web/syscalls/system.ts +++ b/packages/web/syscalls/system.ts @@ -1,7 +1,7 @@ import { SysCallMapping } from "@plugos/plugos/system"; -import { Space } from "@silverbulletmd/common/spaces/space"; +import type { Editor } from "../editor"; -export function systemSyscalls(space: Space): SysCallMapping { +export function systemSyscalls(editor: Editor): SysCallMapping { return { "system.invokeFunction": async ( ctx, @@ -17,7 +17,10 @@ export function systemSyscalls(space: Space): SysCallMapping { return ctx.plug.invoke(name, args); } - return space.invokeFunction(ctx.plug, env, name, args); + return editor.space.invokeFunction(ctx.plug, env, name, args); + }, + "system.reloadPlugs": async () => { + return editor.reloadPlugs(); }, }; } diff --git a/packages/web/types.ts b/packages/web/types.ts index 06fab3b..bfd7568 100644 --- a/packages/web/types.ts +++ b/packages/web/types.ts @@ -16,8 +16,10 @@ export type AppViewState = { unsavedChanges: boolean; showLHS: number; // 0 = hide, > 0 = flex showRHS: number; // 0 = hide, > 0 = flex + showBHS: number; rhsHTML: string; lhsHTML: string; + bhsHTML: string; allPages: Set; commands: Map; notifications: Notification[]; @@ -36,8 +38,10 @@ export const initialViewState: AppViewState = { unsavedChanges: false, showLHS: 0, showRHS: 0, + showBHS: 0, rhsHTML: "", lhsHTML: "", + bhsHTML: "", allPages: new Set(), commands: new Map(), notifications: [], @@ -65,6 +69,8 @@ export type Action = | { type: "hide-rhs" } | { type: "show-lhs"; html: string; flex: number } | { type: "hide-lhs" } + | { type: "show-bhs"; html: string; flex: number } + | { type: "hide-bhs" } | { type: "show-filterbox"; options: FilterOption[];