diff --git a/cli/plug_run.test.ts b/cli/plug_run.test.ts new file mode 100644 index 0000000..07f4d94 --- /dev/null +++ b/cli/plug_run.test.ts @@ -0,0 +1,38 @@ +import { AssetBundle } from "../plugos/asset_bundle/bundle.ts"; +import { compileManifest } from "../plugos/compile.ts"; +import { esbuild } from "../plugos/deps.ts"; +import { runPlug } from "./plug_run.ts"; +import assets from "../dist/plug_asset_bundle.json" assert { + type: "json", +}; +import { assertEquals } from "../test_deps.ts"; +import { path } from "../common/deps.ts"; + +Deno.test("Test plug run", async () => { + // const tempDir = await Deno.makeTempDir(); + + const assetBundle = new AssetBundle(assets); + + const testFolder = path.dirname(new URL(import.meta.url).pathname); + const testSpaceFolder = path.join(testFolder, "test_space"); + + const plugFolder = path.join(testSpaceFolder, "_plug"); + await Deno.mkdir(plugFolder, { recursive: true }); + + await compileManifest( + path.join(testFolder, "test.plug.yaml"), + plugFolder, + ); + assertEquals( + await runPlug( + testSpaceFolder, + "test.run", + [], + assetBundle, + ), + "Hello", + ); + + // await Deno.remove(tempDir, { recursive: true }); + esbuild.stop(); +}); diff --git a/cli/plug_run.ts b/cli/plug_run.ts new file mode 100644 index 0000000..e6883bd --- /dev/null +++ b/cli/plug_run.ts @@ -0,0 +1,145 @@ +import { path } from "../common/deps.ts"; +import { PlugNamespaceHook } from "../common/hooks/plug_namespace.ts"; +import { SilverBulletHooks } from "../common/manifest.ts"; +import { loadMarkdownExtensions } from "../common/markdown_parser/markdown_ext.ts"; +import buildMarkdown from "../common/markdown_parser/parser.ts"; +import { DiskSpacePrimitives } from "../common/spaces/disk_space_primitives.ts"; +import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives.ts"; +import { FileMetaSpacePrimitives } from "../common/spaces/file_meta_space_primitives.ts"; +import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts"; +import { AssetBundle } from "../plugos/asset_bundle/bundle.ts"; +import { createSandbox } from "../plugos/environments/deno_sandbox.ts"; +import { CronHook } from "../plugos/hooks/cron.ts"; +import { EventHook } from "../plugos/hooks/event.ts"; +import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts"; +import assetSyscalls from "../plugos/syscalls/asset.ts"; +import { eventSyscalls } from "../plugos/syscalls/event.ts"; +import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts"; +import { shellSyscalls } from "../plugos/syscalls/shell.deno.ts"; +import { storeSyscalls } from "../plugos/syscalls/store.ts"; +import { System } from "../plugos/system.ts"; +import { Space } from "../web/space.ts"; +import { debugSyscalls } from "../web/syscalls/debug.ts"; +import { markdownSyscalls } from "../web/syscalls/markdown.ts"; +import { systemSyscalls } from "../web/syscalls/system.ts"; +import { yamlSyscalls } from "../web/syscalls/yaml.ts"; +import { pageIndexSyscalls } from "./syscalls/index.ts"; +import { spaceSyscalls } from "./syscalls/space.ts"; + +export async function runPlug( + spacePath: string, + functionName: string, + args: string[] = [], + builtinAssetBundle: AssetBundle, + indexFirst = false, +) { + spacePath = path.resolve(spacePath); + const system = new System("cli"); + + // Event hook + const eventHook = new EventHook(); + system.addHook(eventHook); + + // Cron hook + const cronHook = new CronHook(system); + system.addHook(cronHook); + + const pageIndexCalls = pageIndexSyscalls("run.db"); + + // TODO: Add endpoint + + const plugNamespaceHook = new PlugNamespaceHook(); + system.addHook(plugNamespaceHook); + + const spacePrimitives = new FileMetaSpacePrimitives( + new EventedSpacePrimitives( + new PlugSpacePrimitives( + new DiskSpacePrimitives(spacePath), + plugNamespaceHook, + ), + eventHook, + ), + pageIndexCalls, + ); + const kvStore = new JSONKVStore(); + const space = new Space(spacePrimitives, kvStore); + + // Add syscalls + system.registerSyscalls( + [], + eventSyscalls(eventHook), + spaceSyscalls(space), + assetSyscalls(system), + yamlSyscalls(), + storeSyscalls(kvStore), + systemSyscalls(undefined as any, system), + pageIndexCalls, + debugSyscalls(), + markdownSyscalls(buildMarkdown([])), // Will later be replaced with markdown extensions + ); + + // Syscalls that require some additional permissions + system.registerSyscalls( + ["fetch"], + sandboxFetchSyscalls(), + ); + + system.registerSyscalls( + ["shell"], + shellSyscalls("."), + ); + + await loadPlugsFromAssetBundle(system, builtinAssetBundle); + + for (let plugPath of await space.listPlugs()) { + plugPath = path.resolve(spacePath, plugPath); + await system.load( + new URL(`file://${plugPath}`), + createSandbox, + ); + } + + // Load markdown syscalls based on all new syntax (if any) + system.registerSyscalls( + [], + markdownSyscalls(buildMarkdown(loadMarkdownExtensions(system))), + ); + + if (indexFirst) { + await system.loadedPlugs.get("core")!.invoke("reindexSpace", []); + } + + const [plugName, funcName] = functionName.split("."); + + const plug = system.loadedPlugs.get(plugName); + if (!plug) { + throw new Error(`Plug ${plugName} not found`); + } + const result = await plug.invoke(funcName, args); + + await system.unloadAll(); + await pageIndexCalls["index.close"]({} as any); + return result; +} + +async function loadPlugsFromAssetBundle( + system: System, + assetBundle: AssetBundle, +) { + const tempDir = await Deno.makeTempDir(); + try { + for (const filePath of assetBundle.listFiles()) { + if (filePath.endsWith(".plug.js")) { + const plugPath = path.join(tempDir, filePath); + await Deno.mkdir(path.dirname(plugPath), { recursive: true }); + await Deno.writeFile(plugPath, assetBundle.readFileSync(filePath)); + await system.load( + new URL(`file://${plugPath}`), + createSandbox, + ); + } + } + } finally { + await Deno.remove(tempDir, { recursive: true }); + } +} diff --git a/cli/plug_test.ts b/cli/plug_test.ts new file mode 100644 index 0000000..4816925 --- /dev/null +++ b/cli/plug_test.ts @@ -0,0 +1,7 @@ +import { index } from "$sb/silverbullet-syscall/mod.ts"; + +export async function run() { + console.log("Hello from plug_test.ts"); + console.log(await index.queryPrefix(`tag:`)); + return "Hello"; +} diff --git a/cli/syscalls/index.test.ts b/cli/syscalls/index.test.ts new file mode 100644 index 0000000..c478df8 --- /dev/null +++ b/cli/syscalls/index.test.ts @@ -0,0 +1,34 @@ +import { assertEquals } from "../../test_deps.ts"; +import { pageIndexSyscalls } from "./index.ts"; + +Deno.test("Test KV index", async () => { + const ctx: any = {}; + const calls = pageIndexSyscalls(); + await calls["index.set"](ctx, "page", "test", "value"); + assertEquals(await calls["index.get"](ctx, "page", "test"), "value"); + await calls["index.delete"](ctx, "page", "test"); + assertEquals(await calls["index.get"](ctx, "page", "test"), null); + await calls["index.batchSet"](ctx, "page", [{ + key: "attr:test", + value: "value", + }, { + key: "attr:test2", + value: "value2", + }, { key: "random", value: "value3" }]); + await calls["index.batchSet"](ctx, "page2", [{ + key: "attr:test", + value: "value", + }, { + key: "attr:test2", + value: "value2", + }, { key: "random", value: "value3" }]); + let results = await calls["index.queryPrefix"](ctx, "attr:"); + assertEquals(results.length, 4); + await calls["index.clearPageIndexForPage"](ctx, "page"); + results = await calls["index.queryPrefix"](ctx, "attr:"); + assertEquals(results.length, 2); + await calls["index.clearPageIndex"](ctx); + results = await calls["index.queryPrefix"](ctx, ""); + assertEquals(results.length, 0); + await calls["index.close"](ctx); +}); diff --git a/cli/syscalls/index.ts b/cli/syscalls/index.ts new file mode 100644 index 0000000..3074d02 --- /dev/null +++ b/cli/syscalls/index.ts @@ -0,0 +1,103 @@ +/// + +import type { SysCallMapping } from "../../plugos/system.ts"; + +type Item = { + page: string; + key: string; + value: any; +}; + +export type KV = { + key: string; + value: any; +}; + +// Keyspace: +// ["index", page, key] -> value +// ["indexByKey", key, page] -> value + +/** + * Implements the index syscalls using Deno's KV store. + * @param dbFile + * @returns + */ +export function pageIndexSyscalls(dbFile?: string): SysCallMapping { + const kv = Deno.openKv(dbFile); + const apiObj: SysCallMapping = { + "index.set": async (_ctx, page: string, key: string, value: any) => { + const res = await (await kv).atomic() + .set(["index", page, key], value) + .set(["indexByKey", key, page], value) + .commit(); + if (!res.ok) { + throw res; + } + }, + "index.batchSet": async (_ctx, page: string, kvs: KV[]) => { + // await items.bulkPut(kvs); + for (const { key, value } of kvs) { + await apiObj["index.set"](_ctx, page, key, value); + } + }, + "index.delete": async (_ctx, page: string, key: string) => { + const res = await (await kv).atomic() + .delete(["index", page, key]) + .delete(["indexByKey", key, page]) + .commit(); + if (!res.ok) { + throw res; + } + }, + "index.get": async (_ctx, page: string, key: string) => { + return (await (await kv).get(["index", page, key])).value; + }, + "index.queryPrefix": async (_ctx, prefix: string) => { + const results: { key: string; page: string; value: any }[] = []; + for await ( + const result of (await kv).list({ + start: ["indexByKey", prefix], + end: [ + "indexByKey", + prefix.slice(0, -1) + + // This is a hack to get the next character in the ASCII table (e.g. "a" -> "b") + String.fromCharCode(prefix.charCodeAt(prefix.length - 1) + 1), + ], + }) + ) { + results.push({ + key: result.key[1] as string, + page: result.key[2] as string, + value: result.value, + }); + } + return results; + }, + "index.clearPageIndexForPage": async (ctx, page: string) => { + await apiObj["index.deletePrefixForPage"](ctx, page, ""); + }, + "index.deletePrefixForPage": async (_ctx, page: string, prefix: string) => { + for await ( + const result of (await kv).list({ + start: ["index", page, prefix], + end: ["index", page, prefix + "~"], + }) + ) { + await apiObj["index.delete"](_ctx, page, result.key[2]); + } + }, + "index.clearPageIndex": async (ctx) => { + for await ( + const result of (await kv).list({ + prefix: ["index"], + }) + ) { + await apiObj["index.delete"](ctx, result.key[1], result.key[2]); + } + }, + "index.close": async () => { + (await kv).close(); + }, + }; + return apiObj; +} diff --git a/cli/syscalls/space.ts b/cli/syscalls/space.ts new file mode 100644 index 0000000..d6a7006 --- /dev/null +++ b/cli/syscalls/space.ts @@ -0,0 +1,61 @@ +import { SysCallMapping } from "../../plugos/system.ts"; +import type { Space } from "../../web/space.ts"; +import { AttachmentMeta, PageMeta } from "../../web/types.ts"; + +/** + * Almost the same as web/syscalls/space.ts except leaving out client-specific stuff + */ +export function spaceSyscalls(space: Space): SysCallMapping { + return { + "space.listPages": (): Promise => { + return space.fetchPageList(); + }, + "space.readPage": async ( + _ctx, + name: string, + ): Promise => { + return (await space.readPage(name)).text; + }, + "space.getPageMeta": (_ctx, name: string): Promise => { + return space.getPageMeta(name); + }, + "space.writePage": ( + _ctx, + name: string, + text: string, + ): Promise => { + return space.writePage(name, text); + }, + "space.deletePage": async (_ctx, name: string) => { + await space.deletePage(name); + }, + "space.listPlugs": (): Promise => { + return space.listPlugs(); + }, + "space.listAttachments": async (): Promise => { + return await space.fetchAttachmentList(); + }, + "space.readAttachment": async ( + _ctx, + name: string, + ): Promise => { + return (await space.readAttachment(name)).data; + }, + "space.getAttachmentMeta": async ( + _ctx, + name: string, + ): Promise => { + return await space.getAttachmentMeta(name); + }, + "space.writeAttachment": ( + _ctx, + name: string, + data: Uint8Array, + ): Promise => { + return space.writeAttachment(name, data); + }, + "space.deleteAttachment": async (_ctx, name: string) => { + await space.deleteAttachment(name); + }, + }; +} diff --git a/cli/test.plug.yaml b/cli/test.plug.yaml new file mode 100644 index 0000000..1f7a068 --- /dev/null +++ b/cli/test.plug.yaml @@ -0,0 +1,6 @@ +name: test +requiredPermissions: +- shell +functions: + run: + path: plug_test.ts:run diff --git a/cmd/plug_run.ts b/cmd/plug_run.ts new file mode 100644 index 0000000..b14598a --- /dev/null +++ b/cmd/plug_run.ts @@ -0,0 +1,34 @@ +import { runPlug } from "../cli/plug_run.ts"; +import { path } from "../common/deps.ts"; +import assets from "../dist/plug_asset_bundle.json" assert { + type: "json", +}; +import { AssetBundle } from "../plugos/asset_bundle/bundle.ts"; + +export async function plugRunCommand( + { + noIndex, + }: { + noIndex: boolean; + }, + spacePath: string, + functionName: string, + ...args: string[] +) { + spacePath = path.resolve(spacePath); + console.log("Space path", spacePath); + console.log("Function to run:", functionName, "with arguments", args); + try { + const result = await runPlug( + spacePath, + functionName, + args, + new AssetBundle(assets), + !noIndex, + ); + console.log("Output", result); + } catch (e: any) { + console.error(e.message); + Deno.exit(1); + } +} diff --git a/deno.jsonc b/deno.jsonc index de1cdad..53dd354 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -2,7 +2,7 @@ "tasks": { "clean": "rm -rf dist dist_client_bundle dist_plug_bundle website_build", "deep-clean-mac": "rm -f deno.lock && rm -rf $HOME/Library/Caches/deno && deno task clean", - "install": "deno install -f -A --importmap import_map.json silverbullet.ts", + "install": "deno install -f --unstable -A --importmap import_map.json silverbullet.ts", "check": "find . -name '*.ts*' | xargs deno check", "test": "deno test -A --unstable", "build": "deno run -A build_plugs.ts && deno run -A --unstable build_web.ts", diff --git a/plugos/environments/deno_sandbox.ts b/plugos/environments/deno_sandbox.ts index 020712c..25d2925 100644 --- a/plugos/environments/deno_sandbox.ts +++ b/plugos/environments/deno_sandbox.ts @@ -6,8 +6,8 @@ export function createSandbox(plug: Plug): Sandbox { return new Sandbox(plug, { deno: { permissions: { - // Disallow network access - net: false, + // Allow network access + net: true, // This is required for console logging to work, apparently? env: true, // No talking to native code diff --git a/plugos/lib/kv_store.deno_kv.test.ts b/plugos/lib/kv_store.deno_kv.test.ts new file mode 100644 index 0000000..7c00b9c --- /dev/null +++ b/plugos/lib/kv_store.deno_kv.test.ts @@ -0,0 +1,29 @@ +import { assertEquals } from "../../test_deps.ts"; +import { DenoKVStore } from "./kv_store.deno_kv.ts"; + +Deno.test("Test KV index", async () => { + const kv = new DenoKVStore(); + await kv.init("test.db"); + + await kv.set("name", "Peter"); + assertEquals(await kv.get("name"), "Peter"); + await kv.del("name"); + assertEquals(await kv.has("name"), false); + + await kv.batchSet([ + { key: "page:hello", value: "Hello" }, + { key: "page:hello2", value: "Hello 2" }, + { key: "page:hello3", value: "Hello 3" }, + { key: "something", value: "Something" }, + ]); + + const results = await kv.queryPrefix("page:"); + assertEquals(results.length, 3); + + assertEquals(await kv.batchGet(["page:hello", "page:hello3"]), [ + "Hello", + "Hello 3", + ]); + + await kv.delete(); +}); diff --git a/plugos/lib/kv_store.deno_kv.ts b/plugos/lib/kv_store.deno_kv.ts new file mode 100644 index 0000000..ec1e882 --- /dev/null +++ b/plugos/lib/kv_store.deno_kv.ts @@ -0,0 +1,102 @@ +/// + +import { KV, KVStore } from "./kv_store.ts"; + +export class DenoKVStore implements KVStore { + kv!: Deno.Kv; + path: string | undefined; + + async init(path?: string) { + this.path = path; + this.kv = await Deno.openKv(path); + } + + close() { + this.kv.close(); + } + + async delete() { + this.kv.close(); + if (this.path) { + await Deno.remove(this.path); + } + } + + async del(key: string): Promise { + const res = await this.kv.atomic() + .delete([key]) + .commit(); + if (!res.ok) { + throw res; + } + } + async deletePrefix(prefix: string): Promise { + for await ( + const result of this.kv.list({ + start: [prefix], + end: [endRange(prefix)], + }) + ) { + await this.del(result.key[0] as string); + } + } + async deleteAll(): Promise { + for await ( + const result of this.kv.list({ prefix: [] }) + ) { + await this.del(result.key[0] as string); + } + } + async set(key: string, value: any): Promise { + const res = await this.kv.atomic() + .set([key], value) + .commit(); + if (!res.ok) { + throw res; + } + } + async batchSet(kvs: KV[]): Promise { + for (const { key, value } of kvs) { + await this.set(key, value); + } + } + async batchDelete(keys: string[]): Promise { + for (const key of keys) { + await this.del(key); + } + } + batchGet(keys: string[]): Promise { + const results: Promise[] = []; + for (const key of keys) { + results.push(this.get(key)); + } + return Promise.all(results); + } + async get(key: string): Promise { + return (await this.kv.get([key])).value; + } + async has(key: string): Promise { + return (await this.kv.get([key])).value !== null; + } + async queryPrefix(keyPrefix: string): Promise<{ key: string; value: any }[]> { + const results: { key: string; value: any }[] = []; + for await ( + const result of (this.kv).list({ + start: [keyPrefix], + end: [endRange(keyPrefix)], + }) + ) { + results.push({ + key: result.key[0] as string, + value: result.value as any, + }); + } + return results; + } +} + +function endRange(prefix: string) { + const lastChar = prefix[prefix.length - 1]; + const nextLastChar = String.fromCharCode(lastChar.charCodeAt(0) + 1); + return prefix.slice(0, -1) + nextLastChar; +} diff --git a/plugos/plug.ts b/plugos/plug.ts index 7748983..c292729 100644 --- a/plugos/plug.ts +++ b/plugos/plug.ts @@ -1,10 +1,10 @@ -import { Manifest, RuntimeEnvironment } from "./types.ts"; +import { Manifest } from "./types.ts"; import { Sandbox } from "./sandbox.ts"; import { System } from "./system.ts"; import { AssetBundle, AssetJson } from "./asset_bundle/bundle.ts"; export class Plug { - readonly runtimeEnv?: RuntimeEnvironment; + readonly runtimeEnv?: string; public grantedPermissions: string[] = []; public sandbox: Sandbox; diff --git a/plugos/syscalls/fetch.ts b/plugos/syscalls/fetch.ts new file mode 100644 index 0000000..cd279d5 --- /dev/null +++ b/plugos/syscalls/fetch.ts @@ -0,0 +1,25 @@ +import type { SysCallMapping } from "../../plugos/system.ts"; +import { + ProxyFetchRequest, + ProxyFetchResponse, +} from "../../common/proxy_fetch.ts"; +import { base64Encode } from "../asset_bundle/base64.ts"; + +export function sandboxFetchSyscalls(): SysCallMapping { + return { + "sandboxFetch.fetch": async ( + _ctx, + url: string, + options: ProxyFetchRequest, + ): Promise => { + // console.log("Got sandbox fetch ", url); + const resp = await fetch(url, options); + return { + status: resp.status, + ok: resp.ok, + headers: Object.fromEntries(resp.headers.entries()), + base64Body: base64Encode(new Uint8Array(await resp.arrayBuffer())), + }; + }, + }; +} diff --git a/plugos/syscalls/shell.deno.ts b/plugos/syscalls/shell.deno.ts index c17fea3..a83596b 100644 --- a/plugos/syscalls/shell.deno.ts +++ b/plugos/syscalls/shell.deno.ts @@ -1,21 +1,21 @@ import type { SysCallMapping } from "../system.ts"; -export default function (cwd: string): SysCallMapping { +export function shellSyscalls(cwd: string): SysCallMapping { return { "shell.run": async ( _ctx, cmd: string, args: string[], ): Promise<{ stdout: string; stderr: string }> => { - const p = Deno.run({ - cmd: [cmd, ...args], - cwd: cwd, + const p = new Deno.Command(cmd, { + args: args, + cwd, stdout: "piped", stderr: "piped", }); - await p.status(); - const stdout = new TextDecoder().decode(await p.output()); - const stderr = new TextDecoder().decode(await p.stderrOutput()); + const output = await p.output(); + const stdout = new TextDecoder().decode(output.stdout); + const stderr = new TextDecoder().decode(output.stderr); return { stdout, stderr }; }, diff --git a/plugos/syscalls/store.dexie_browser.ts b/plugos/syscalls/store.ts similarity index 90% rename from plugos/syscalls/store.dexie_browser.ts rename to plugos/syscalls/store.ts index e0113f7..cae1b6f 100644 --- a/plugos/syscalls/store.dexie_browser.ts +++ b/plugos/syscalls/store.ts @@ -1,9 +1,8 @@ import { SysCallMapping } from "../system.ts"; -import { DexieKVStore } from "../lib/kv_store.dexie.ts"; -import { KV } from "../lib/kv_store.ts"; +import { KV, KVStore } from "../lib/kv_store.ts"; export function storeSyscalls( - db: DexieKVStore, + db: KVStore, ): SysCallMapping { return { "store.delete": (_ctx, key: string) => { diff --git a/plugos/system.ts b/plugos/system.ts index e06d1df..cd2bf52 100644 --- a/plugos/system.ts +++ b/plugos/system.ts @@ -1,4 +1,4 @@ -import { Hook, RuntimeEnvironment } from "./types.ts"; +import { Hook } from "./types.ts"; import { EventEmitter } from "./event.ts"; import type { SandboxFactory } from "./sandbox.ts"; import { Plug } from "./plug.ts"; @@ -32,7 +32,7 @@ export class System extends EventEmitter> { protected registeredSyscalls = new Map(); protected enabledHooks = new Set>(); - constructor(readonly env?: RuntimeEnvironment) { + constructor(readonly env?: string) { super(); } diff --git a/plugos/types.ts b/plugos/types.ts index 1a70a43..146c595 100644 --- a/plugos/types.ts +++ b/plugos/types.ts @@ -17,11 +17,9 @@ export type FunctionDef = { // Reuse an // Format: plugName.functionName redirect?: string; - env?: RuntimeEnvironment; + env?: string; } & HookT; -export type RuntimeEnvironment = "client" | "server"; - export interface Hook { validateManifest(manifest: Manifest): string[]; apply(system: System): void; diff --git a/plugs/core/core.plug.yaml b/plugs/core/core.plug.yaml index f85ae16..374fae9 100644 --- a/plugs/core/core.plug.yaml +++ b/plugs/core/core.plug.yaml @@ -52,6 +52,8 @@ functions: path: "./page.ts:reindexCommand" command: name: "Space: Reindex" + reindexSpace: + path: "./page.ts:reindexSpace" deletePage: path: "./page.ts:deletePage" command: diff --git a/plugs/directive/command.ts b/plugs/directive/command.ts index 65e15fc..a78f540 100644 --- a/plugs/directive/command.ts +++ b/plugs/directive/command.ts @@ -1,5 +1,6 @@ import { editor, markdown, space, sync } from "$sb/silverbullet-syscall/mod.ts"; import { + ParseTree, removeParentPointers, renderToText, traverseTree, @@ -36,42 +37,7 @@ export async function updateDirectivesOnPageCommand() { await editor.save(); - // Collect all directives and their body replacements - const replacements: { fullMatch: string; textPromise: Promise }[] = - []; - - // Convenience array to wait for all promises to resolve - const allPromises: Promise[] = []; - - removeParentPointers(tree); - - traverseTree(tree, (tree) => { - if (tree.type !== "Directive") { - return false; - } - const fullMatch = text.substring(tree.from!, tree.to!); - try { - const promise = renderDirectives(pageMeta, tree); - replacements.push({ - textPromise: promise, - fullMatch, - }); - allPromises.push(promise); - } catch (e: any) { - replacements.push({ - fullMatch, - textPromise: Promise.resolve( - `${renderToText(tree.children![0])}\n**ERROR:** ${e.message}\n${ - renderToText(tree.children![tree.children!.length - 1]) - }`, - ), - }); - } - return true; - }); - - // Wait for all to have processed - await Promise.all(allPromises); + const replacements = await findReplacements(tree, text, pageMeta); // Iterate again and replace the bodies. Iterating again (not using previous positions) // because text may have changed in the mean time (directive processing may take some time) @@ -111,27 +77,27 @@ export async function updateDirectivesOnPageCommand() { } } -export async function updateDirectives( - pageMeta: PageMeta, +async function findReplacements( + tree: ParseTree, text: string, + pageMeta: PageMeta, ) { - const tree = await markdown.parseMarkdown(text); // Collect all directives and their body replacements const replacements: { fullMatch: string; textPromise: Promise }[] = []; + // Convenience array to wait for all promises to resolve const allPromises: Promise[] = []; + removeParentPointers(tree); + traverseTree(tree, (tree) => { if (tree.type !== "Directive") { return false; } const fullMatch = text.substring(tree.from!, tree.to!); try { - const promise = renderDirectives( - pageMeta, - tree, - ); + const promise = renderDirectives(pageMeta, tree); replacements.push({ textPromise: promise, fullMatch, @@ -153,9 +119,51 @@ export async function updateDirectives( // Wait for all to have processed await Promise.all(allPromises); + return replacements; +} + +export async function updateDirectivesInSpace() { + const allPages = await space.listPages(); + let counter = 0; + for (const page of allPages) { + counter++; + console.log( + `Updating directives in page [${counter}/${allPages.length}]`, + page.name, + ); + try { + await updateDirectivesForPage(page.name); + } catch (e: any) { + console.error("Error while updating directives on page", page.name, e); + } + } +} + +async function updateDirectivesForPage( + pageName: string, +) { + const pageMeta = await space.getPageMeta(pageName); + const currentText = await space.readPage(pageName); + const newText = await updateDirectives(pageMeta, currentText); + if (newText !== currentText) { + console.info("Content of page changed, saving."); + await space.writePage(pageName, newText); + } +} + +export async function updateDirectives( + pageMeta: PageMeta, + text: string, +) { + const tree = await markdown.parseMarkdown(text); + const replacements = await findReplacements(tree, text, pageMeta); + // Iterate again and replace the bodies. for (const replacement of replacements) { - text = text.replace(replacement.fullMatch, await replacement.textPromise); + text = text.replace( + replacement.fullMatch, + await replacement.textPromise, + ); } return text; } diff --git a/plugs/directive/directive.plug.yaml b/plugs/directive/directive.plug.yaml index 1f5b86e..a7591df 100644 --- a/plugs/directive/directive.plug.yaml +++ b/plugs/directive/directive.plug.yaml @@ -9,6 +9,8 @@ functions: key: "Alt-q" events: - editor:pageLoaded + updateDirectivesInSpace: + path: ./command.ts:updateDirectivesInSpace indexData: path: ./data.ts:indexData events: diff --git a/plugs/directive/directives.ts b/plugs/directive/directives.ts index 4dd3b8e..5e518ec 100644 --- a/plugs/directive/directives.ts +++ b/plugs/directive/directives.ts @@ -1,5 +1,4 @@ import { ParseTree, renderToText } from "$sb/lib/tree.ts"; -import { sync } from "../../plug-api/silverbullet-syscall/mod.ts"; import { PageMeta } from "../../web/types.ts"; import { evalDirectiveRenderer } from "./eval_directive.ts"; diff --git a/silverbullet.ts b/silverbullet.ts index 0267e94..3432c94 100755 --- a/silverbullet.ts +++ b/silverbullet.ts @@ -11,6 +11,7 @@ import { userAdd } from "./cmd/user_add.ts"; import { userPasswd } from "./cmd/user_passwd.ts"; import { userDelete } from "./cmd/user_delete.ts"; import { userChgrp } from "./cmd/user_chgrp.ts"; +import { plugRunCommand } from "./cmd/plug_run.ts"; await new Command() .name("silverbullet") @@ -66,6 +67,13 @@ await new Command() .option("--importmap ", "Path to import map file to use") .option("--runtimeUrl ", "URL to worker_runtime.ts to use") .action(plugCompileCommand) + // plug:run + .command("plug:run", "Run a PlugOS function from the CLI") + .arguments(" [...args:string]") + .option("--noIndex [type:boolean]", "Do not run a full space index first", { + default: false, + }) + .action(plugRunCommand) .command("user:add", "Add a new user to an authentication file") .arguments("[username:string]") .option( diff --git a/web/client_system.ts b/web/client_system.ts index 4765c9b..ae2211d 100644 --- a/web/client_system.ts +++ b/web/client_system.ts @@ -8,7 +8,7 @@ import { createSandbox } from "../plugos/environments/webworker_sandbox.ts"; import assetSyscalls from "../plugos/syscalls/asset.ts"; import { eventSyscalls } from "../plugos/syscalls/event.ts"; -import { storeSyscalls } from "../plugos/syscalls/store.dexie_browser.ts"; +import { storeSyscalls } from "../plugos/syscalls/store.ts"; import { SysCallMapping, System } from "../plugos/system.ts"; import type { Client } from "./client.ts"; import { CodeWidgetHook } from "./hooks/code_widget.ts"; diff --git a/web/space.ts b/web/space.ts index 39b736b..6186b90 100644 --- a/web/space.ts +++ b/web/space.ts @@ -4,8 +4,8 @@ import { EventEmitter } from "../plugos/event.ts"; import { plugPrefix } from "../common/spaces/constants.ts"; import { safeRun } from "../common/util.ts"; import { AttachmentMeta, PageMeta } from "./types.ts"; -import { DexieKVStore } from "../plugos/lib/kv_store.dexie.ts"; import { throttle } from "../common/async_util.ts"; +import { KVStore } from "../plugos/lib/kv_store.ts"; export type SpaceEvents = { pageCreated: (meta: PageMeta) => void; @@ -46,7 +46,7 @@ export class Space extends EventEmitter { constructor( readonly spacePrimitives: SpacePrimitives, - private kvStore: DexieKVStore, + private kvStore: KVStore, ) { super(); this.kvStore.get("imageHeightCache").then((cache) => { diff --git a/website/Attributes.md b/website/Attributes.md index a6ed7a3..e2938cc 100644 --- a/website/Attributes.md +++ b/website/Attributes.md @@ -33,7 +33,7 @@ Example query: |name |lastModified |contentType |size|perm|pageAttribute| |----------|-------------|-------------|----|--|-----| -|Attributes|1690384301337|text/markdown|1591|rw|hello| +|Attributes|1691165890257|text/markdown|1609|rw|hello| This attaches an attribute to an item: @@ -57,5 +57,5 @@ Example query: |name|done |taskAttribute|page |pos | |----|-----|-----|----------|----| -|Task|false|hello|Attributes|1352| +|Task|false|hello|Attributes|1355| diff --git a/website/Getting Started.md b/website/Getting Started.md index 338fda2..0ca64d6 100644 --- a/website/Getting Started.md +++ b/website/Getting Started.md @@ -23,18 +23,18 @@ You will notice this whole page section is wrapped in a strange type of block. T Don’t believe me, check this out, here’s a list of (max 10) pages in your space ordered by last modified date, it updates (somewhat) dynamically 🀯. Create some new pages and come back here to see that it works: -|name | -|-------------------------| -|CHANGELOG | -|πŸ”¨ Development | -|Server | -|Raspberry Pi Installation| -|STYLES | -|Getting Started | -|Sandbox | -|SETTINGS | -|SilverBullet | -|πŸ”Œ Core/Templates | +|name | +|------------------| +|πŸ”Œ Directive/Query| +|Attributes | +|Getting Started | +|πŸ”Œ Core/Tags | +|πŸ”Œ Github | +|πŸ”Œ Mattermost | +|πŸ”Œ Git | +|πŸ”Œ Ghost | +|πŸ”Œ Share | +|Install | That said, the directive used wrapping this page section is `#use` which uses the content of another page as a template and inlines it. Directives recalculate their bodies in two scenarios: diff --git a/website/Install.md b/website/Install.md index 8f9106f..73288cc 100644 --- a/website/Install.md +++ b/website/Install.md @@ -20,13 +20,13 @@ This consists of two steps (unless Deno is already installed β€” in which case w With Deno installed, run: ```shell -deno install -f --name silverbullet -A https://get.silverbullet.md +deno install -f --name silverbullet --unstable -A https://get.silverbullet.md ``` This will give you (and when you use `silverbullet upgrade`) the latest stable release. If you prefer to live on the bleeding edge, you can install using the following command instead: ```shell -deno install -f --name silverbullet -A https://silverbullet.md/silverbullet.js +deno install -f --name silverbullet --unstable -A https://silverbullet.md/silverbullet.js ``` This will install `silverbullet` into your `~/.deno/bin` folder (which should already be in your `$PATH` if you followed the Deno install instructions). diff --git a/website/πŸ”Œ Core/Tags.md b/website/πŸ”Œ Core/Tags.md index 2358954..231ea1f 100644 --- a/website/πŸ”Œ Core/Tags.md +++ b/website/πŸ”Œ Core/Tags.md @@ -18,7 +18,7 @@ and be queried: |name |tags |page |pos| |-------------------------------|--------|------------|---| -|This is a tagged item #core-tag|core-tag|πŸ”Œ Core/Tags|486| +|This is a tagged item #core-tag|core-tag|πŸ”Œ Core/Tags|493| and **tags**: @@ -28,5 +28,5 @@ and **tags**: And they can be queried this way: -* [ ] [[πŸ”Œ Core/Tags@783]] This is a tagged task #core-tag +* [ ] [[πŸ”Œ Core/Tags@804]] This is a tagged task #core-tag diff --git a/website/πŸ”Œ Directive/Query.md b/website/πŸ”Œ Directive/Query.md index 06bc058..8fa8979 100644 --- a/website/πŸ”Œ Directive/Query.md +++ b/website/πŸ”Œ Directive/Query.md @@ -88,8 +88,8 @@ Example: |name|age|city |country|page |pos | |----|--|-----|-----|------------------|----| -|John|50|Milan|Italy|πŸ”Œ Directive/Query|3293| -|Jane|53|Rome |Italy|πŸ”Œ Directive/Query|3294| +|John|50|Milan|Italy|πŸ”Œ Directive/Query|2933| +|Jane|53|Rome |Italy|πŸ”Œ Directive/Query|2934| #### 4.2 Plugs’ data sources @@ -156,11 +156,11 @@ For the sake of simplicity, we will use the `page` data source and limit the res **Result:** Look at the data. This is more than we need. The query even gives us template pages. Let's try to limit it in the next step. -|name |lastModified |contentType |size|perm| -|--------------|-------------|-------------|----|--| -|API |1688987324351|text/markdown|1405|rw| -|Authelia |1688482500313|text/markdown|866 |rw| -|Authentication|1686682290943|text/markdown|1730|rw| +|name |lastModified |contentType |size |perm| +|--|--|--|--|--| +|Authentication |1686682290943|text/markdown|1730 |rw| +|Guide/Deployment/Cloudflare and Portainer|1690298800145|text/markdown|12899|rw| +|Markdown |1676121406520|text/markdown|1178 |rw| @@ -171,13 +171,13 @@ For the sake of simplicity, we will use the `page` data source and limit the res **Result:** Okay, this is what we wanted but there is also information such as `perm`, `type` and `lastModified` that we don't need. -|name |lastModified |contentType |size|perm|type|repo |uri |author | -|--|--|--|--|--|--|--|--|--| -|πŸ”Œ Directive|1688987324365|text/markdown|2607|rw|plug|https://github.com/silverbulletmd/silverbullet | | | -|πŸ”Œ KaTeX |1687099068396|text/markdown|1342|rw|plug|https://github.com/silverbulletmd/silverbullet-katex |github:silverbulletmd/silverbullet-katex/katex.plug.js |Zef Hemel | -|πŸ”Œ Core |1687094809367|text/markdown|402 |rw|plug|https://github.com/silverbulletmd/silverbullet | | | -|πŸ”Œ Twitter |1685105433212|text/markdown|1266|rw|plug|https://github.com/silverbulletmd/silverbullet-twitter|github:silverbulletmd/silverbullet-twitter/twitter.plug.js|SilverBullet Authors| -|πŸ”Œ Mermaid |1685105423879|text/markdown|1096|rw|plug|https://github.com/silverbulletmd/silverbullet-mermaid|github:silverbulletmd/silverbullet-mermaid/mermaid.plug.js|Zef Hemel | +|name |lastModified |contentType |size|perm|type|uri |repo |author |share-support| +|--|--|--|--|--|--|--|--|--|--| +|πŸ”Œ Github |1691137925014|text/markdown|2206|rw|plug|github:silverbulletmd/silverbullet-github/github.plug.js |https://github.com/silverbulletmd/silverbullet-github |Zef Hemel|true| +|πŸ”Œ Mattermost|1691137924741|text/markdown|3535|rw|plug|github:silverbulletmd/silverbullet-mattermost/mattermost.plug.json|https://github.com/silverbulletmd/silverbullet-mattermost|Zef Hemel|true| +|πŸ”Œ Git |1691137924435|text/markdown|1112|rw|plug|github:silverbulletmd/silverbullet-git/git.plug.js |https://github.com/silverbulletmd/silverbullet-git |Zef Hemel| | +|πŸ”Œ Ghost |1691137922296|text/markdown|1733|rw|plug|github:silverbulletmd/silverbullet-ghost/ghost.plug.js |https://github.com/silverbulletmd/silverbullet-ghost |Zef Hemel|true| +|πŸ”Œ Share |1691137921643|text/markdown|693 |rw|plug| |https://github.com/silverbulletmd/silverbullet | | | #### 6.3 Query to select only certain fields @@ -189,13 +189,13 @@ and `repo` columns and then sort by last modified time. from a visual perspective. -|name |author |repo | +|name |author |repo | |--|--|--| -|πŸ”Œ Directive| |https://github.com/silverbulletmd/silverbullet | -|πŸ”Œ KaTeX |Zef Hemel |https://github.com/silverbulletmd/silverbullet-katex | -|πŸ”Œ Core | |https://github.com/silverbulletmd/silverbullet | -|πŸ”Œ Twitter |SilverBullet Authors|https://github.com/silverbulletmd/silverbullet-twitter| -|πŸ”Œ Mermaid |Zef Hemel |https://github.com/silverbulletmd/silverbullet-mermaid| +|πŸ”Œ Github |Zef Hemel|https://github.com/silverbulletmd/silverbullet-github | +|πŸ”Œ Mattermost|Zef Hemel|https://github.com/silverbulletmd/silverbullet-mattermost| +|πŸ”Œ Git |Zef Hemel|https://github.com/silverbulletmd/silverbullet-git | +|πŸ”Œ Ghost |Zef Hemel|https://github.com/silverbulletmd/silverbullet-ghost | +|πŸ”Œ Share | |https://github.com/silverbulletmd/silverbullet | #### 6.4 Display the data in a format defined by a template @@ -205,11 +205,11 @@ from a visual perspective. **Result:** Here you go. This is the result we would like to achieve πŸŽ‰. Did you see how I used `render` and `template/plug` in a query? πŸš€ -* [[πŸ”Œ Directive]] -* [[πŸ”Œ KaTeX]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-katex)) -* [[πŸ”Œ Core]] -* [[πŸ”Œ Twitter]] by **SilverBullet Authors** ([repo](https://github.com/silverbulletmd/silverbullet-twitter)) -* [[πŸ”Œ Mermaid]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-mermaid)) +* [[πŸ”Œ Github]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-github)) +* [[πŸ”Œ Mattermost]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-mattermost)) +* [[πŸ”Œ Git]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-git)) +* [[πŸ”Œ Ghost]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-ghost)) +* [[πŸ”Œ Share]] PS: You don't need to select only certain fields to use templates. Templates are diff --git a/website/πŸ”Œ Ghost.md b/website/πŸ”Œ Ghost.md index 91df059..73a89cd 100644 --- a/website/πŸ”Œ Ghost.md +++ b/website/πŸ”Œ Ghost.md @@ -9,8 +9,7 @@ share-support: true # Ghost plug for Silver Bullet -This allows you to publish your pages as [Ghost](https://ghost.org/) pages or -posts. I use it to publish [Zef+](https://zef.plus). +This allows you to publish your pages as [Ghost](https://ghost.org/) pages or posts. I use it to publish [Zef+](https://zef.plus). ## Configuration @@ -22,9 +21,7 @@ In your `SETTINGS` specify the following settings: url: https://your-ghost-blog.ghost.io ``` -Then, create a Custom Integration (in your Ghost control panel under Settings > -Advanced > Integrations > Add Custom Integration). Enter a name (whatever you -want), then copy the full Admin API Key in your `SECRETS` file, mirroring the +Then, create a Custom Integration (in your Ghost control panel under Settings > Advanced > Integrations > Add Custom Integration). Enter a name (whatever you want), then copy the full Admin API Key in your `SECRETS` file, mirroring the structure of SETTINGS: ```yaml @@ -34,10 +31,8 @@ structure of SETTINGS: ## Usage -The plugin hooks into Silver Bullet's -[Share infrastructure](https://silverbullet.md/%F0%9F%94%8C_Share). Therefore to -share a page as either a Ghost page or post, add a `$share` front matter key. -For posts this should take the shape of: +The plugin hooks into Silver Bullet's [Share infrastructure](https://silverbullet.md/%F0%9F%94%8C_Share). Therefore to +share a page as either a Ghost page or post, add a `$share` front matter key. For posts this should take the shape of: --- $share: @@ -51,8 +46,7 @@ And for pages: - ghost:myblog:page:my-page-slug --- -Now, when you {[Share: Publish]} (Cmd-s/Ctrl-s) your post will automatically be -created (as a draft) or updated if it already exists. +Now, when you {[Share: Publish]} (Cmd-s/Ctrl-s) your post will automatically be created (as a draft) or updated if it already exists. Enjoy! diff --git a/website/πŸ”Œ Git.md b/website/πŸ”Œ Git.md index fdae78a..7e15d86 100644 --- a/website/πŸ”Œ Git.md +++ b/website/πŸ”Œ Git.md @@ -7,28 +7,37 @@ author: Zef Hemel # SilverBullet plug for Git + Very basic in functionality, it assumes you have git configured for push and pull in your space. What it does, roughly speaking: -`Git : Sync`: -* Adds all *.md files in your folder to git -* It commits them with a "Snapshot" commit message -* It `git pull`s changes from the remote server -* It `git push`es changes to the remote server +{[Git: Sync]}: -`Git: Snapshot`: -* Asks you for a commit message -* Commits +- Adds all files in your folder to git +- It commits them with a "Snapshot" commit message +- It `git pull`s changes from the remote server +- It `git push`es changes to the remote server + +{[Git: Snapshot]}: + +- Asks you for a commit message +- Commits + +{[Github: Clone]}: + +Clones into your space from a Github repository. This will do authentication based on a [personal access token](https://github.com/settings/tokens). ## Installation + Open your `PLUGS` note in SilverBullet and add this plug to the list: ``` -- github:silverbulletmd/silverbullet-git/git.plug.json +- github:silverbulletmd/silverbullet-git/git.plug.js ``` Then run the `Plugs: Update` command and off you go! ## To Build + ```shell deno task build ``` diff --git a/website/πŸ”Œ Github.md b/website/πŸ”Œ Github.md index 1042f4c..6f9f1b5 100644 --- a/website/πŸ”Œ Github.md +++ b/website/πŸ”Œ Github.md @@ -17,7 +17,7 @@ Provides various integrations with Github: Open your `PLUGS` note in SilverBullet and add this plug to the list: ``` -- github:silverbulletmd/silverbullet-github/github.plug.json +- github:silverbulletmd/silverbullet-github/github.plug.js ``` Then run the `Plugs: Update` command and off you go! diff --git a/website/πŸ”Œ Mattermost.md b/website/πŸ”Œ Mattermost.md index 1438375..3cb2d04 100644 --- a/website/πŸ”Œ Mattermost.md +++ b/website/πŸ”Œ Mattermost.md @@ -7,12 +7,12 @@ share-support: true --- -# Mattermost for SilverBullet +# Mattermost for Silver Bullet This plug provides various integrations with the [Mattermost suite](https://www.mattermost.com) of products. Please follow the installation, configuration sections, and have a look at the example. Features: -* Integration with [SilverBullet Share](https://silverbullet.md/%F0%9F%94%8C_Share), allowing you to publish and update a page as a post on Mattermost, as well as load existing posts into SB as a page using the {[Share: Mattermost Post: Publish]} (to publish an existing page as a Mattermost post) and {[Share: Mattermost Post: Load]} (to load an existing post into SB) commands. +* Integration with [Silver Bullet Share](https://silverbullet.md/%F0%9F%94%8C_Share), allowing you to publish and update a page as a post on Mattermost, as well as load existing posts into SB as a page using the {[Share: Mattermost Post: Publish]} (to publish an existing page as a Mattermost post) and {[Share: Mattermost Post: Load]} (to load an existing post into SB) commands. * Access your saved posts via the `mm-saved` query provider * Unfurl support for posts (after dumping a permalink URL to a post in a page, use the {[Link: Unfurl]} command). * Boards support is WIP @@ -46,7 +46,7 @@ In `SECRETS` provide a Mattermost personal access token (or hijack one from your * `mm-saved` fetches (by default 15) saved posts in Mattermost, you need to add a `where server = "community"` (with server name) clause to your query to select the mattermost server to query. -To make the `mm-saved` query results look good, it's recommended you render your query results with a template. Here is one to start with: you can keep it in e.g., `templates/mm-saved`: +To make the `mm-saved` query results look good, it's recommended you render your query results a template. Here is one to start with, you can keep it in e.g. `templates/mm-saved`: [{{username}}]({{url}}) in {{#if channelName}}**{{channelName}}**{{else}}a DM{{/if}} at _{{updatedAt}}_ {[Unsave]}: diff --git a/website/πŸ”Œ Share.md b/website/πŸ”Œ Share.md index 4867ba1..2b475b1 100644 --- a/website/πŸ”Œ Share.md +++ b/website/πŸ”Œ Share.md @@ -11,7 +11,6 @@ Specific implementations for sharing are implemented in other plugs, specificall * [[πŸ”Œ Ghost]] * [[πŸ”Œ Markdown]] -* [[πŸ”Œ Collab]] * [[πŸ”Œ Mattermost]] * [[πŸ”Œ Github]]