diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e310971..647b23e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -33,7 +33,7 @@ jobs: - name: Setup Deno uses: denoland/setup-deno@v1 with: - deno-version: v1.34 + deno-version: v1.37 - name: Run bundle build run: | @@ -70,7 +70,6 @@ jobs: type=semver,pattern=latest,enable=true # When pushing to main branch, release as :edge type=edge,branch=main - - name: Build and push main docker images uses: docker/build-push-action@v4.0.0 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 81c8204..cf47ecf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,7 @@ jobs: - name: Setup Deno uses: denoland/setup-deno@v1 with: - deno-version: v1.34 + deno-version: v1.37 - name: Run build run: deno task build - name: Bundle diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 38c1a02..5a365dd 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -16,7 +16,7 @@ jobs: - name: Setup Deno uses: denoland/setup-deno@v1 with: - deno-version: v1.34 + deno-version: v1.37 - name: Build bundles run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dc01aed..265552f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,7 +20,7 @@ jobs: - name: Setup Deno uses: denoland/setup-deno@v1 with: - deno-version: v1.35 + deno-version: v1.37 - name: Run build run: deno task build diff --git a/Dockerfile b/Dockerfile index c63eaf3..6fb657f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM lukechannings/deno:v1.36.3 +FROM lukechannings/deno:v1.37.1 # The volume that will keep the space data # Create a volume first: # docker volume create myspace diff --git a/Dockerfile.s3 b/Dockerfile.s3 deleted file mode 100644 index 7462c33..0000000 --- a/Dockerfile.s3 +++ /dev/null @@ -1,24 +0,0 @@ -# This Dockerfile is used to build a Docker image that runs silverbullet with an S3 bucket as a backend -# Configure it with the following environment variables - -# AWS_ACCESS_KEY_ID=XXXX -# AWS_SECRET_ACCESS_KEY=XXXX -# AWS_ENDPOINT=s3.eu-central-1.amazonaws.com -# AWS_REGION=eu-central-1 -# AWS_BUCKET=my-sb-bucket - -FROM denoland/deno:alpine-1.33.2 - -# Copy the bundled version of silverbullet into the container -ADD ./dist/silverbullet.js /silverbullet.js - -# deno user id is 1000 in alpine image -USER deno - -# Expose port 3000 -# Port map this when running, e.g. with -p 3002:3000 (where 3002 is the host port) -EXPOSE 3000 - -# Run the server, allowing to pass in additional argument at run time, e.g. -# docker run -p 3002:3000 -v myspace:/space -it zefhemel/silverbullet --user me:letmein -ENTRYPOINT deno run -A /silverbullet.js -L 0.0.0.0 s3:// diff --git a/cli/plug_run.test.ts b/cli/plug_run.test.ts index e8c1039..70b165c 100644 --- a/cli/plug_run.test.ts +++ b/cli/plug_run.test.ts @@ -8,7 +8,10 @@ import assets from "../dist/plug_asset_bundle.json" assert { import { assertEquals } from "../test_deps.ts"; import { path } from "../common/deps.ts"; -Deno.test("Test plug run", async () => { +Deno.test("Test plug run", { + sanitizeResources: false, + sanitizeOps: false, +}, async () => { // const tempDir = await Deno.makeTempDir(); const tempDbFile = await Deno.makeTempFile({ suffix: ".db" }); diff --git a/cli/plug_run.ts b/cli/plug_run.ts index 099f311..9d11920 100644 --- a/cli/plug_run.ts +++ b/cli/plug_run.ts @@ -1,4 +1,3 @@ -import { path } from "../common/deps.ts"; import { DiskSpacePrimitives } from "../common/spaces/disk_space_primitives.ts"; import { AssetBundle } from "../plugos/asset_bundle/bundle.ts"; diff --git a/cli/plug_test.ts b/cli/plug_test.ts index 4816925..f5de998 100644 --- a/cli/plug_test.ts +++ b/cli/plug_test.ts @@ -1,7 +1,7 @@ -import { index } from "$sb/silverbullet-syscall/mod.ts"; +import { datastore } from "$sb/syscalls.ts"; export async function run() { console.log("Hello from plug_test.ts"); - console.log(await index.queryPrefix(`tag:`)); + await datastore.set(["plug_test"], "Hello"); return "Hello"; } diff --git a/common/languages.ts b/common/languages.ts new file mode 100644 index 0000000..082eaa2 --- /dev/null +++ b/common/languages.ts @@ -0,0 +1,134 @@ +import { LRLanguage } from "@codemirror/language"; +import { + cLanguage, + cppLanguage, + csharpLanguage, + dartLanguage, + htmlLanguage, + javaLanguage, + javascriptLanguage, + jsonLanguage, + kotlinLanguage, + Language, + objectiveCLanguage, + objectiveCppLanguage, + postgresqlLanguage, + protobufLanguage, + pythonLanguage, + rustLanguage, + scalaLanguage, + shellLanguage, + sqlLanguage, + StreamLanguage, + tomlLanguage, + typescriptLanguage, + xmlLanguage, + yamlLanguage, +} from "./deps.ts"; +import { highlightingDirectiveParser } from "./markdown_parser/parser.ts"; + +const languageCache = new Map(); + +export function languageFor(name: string): Language | null { + if (languageCache.has(name)) { + return languageCache.get(name)!; + } + const language = languageLookup(name); + if (!language) { + return null; + } + languageCache.set(name, language); + return language; +} + +function languageLookup(name: string): Language | null { + switch (name) { + case "meta": + case "yaml": + case "template": + case "embed": + case "data": + return StreamLanguage.define(yamlLanguage); + + case "javascript": + case "js": + return javascriptLanguage; + case "typescript": + case "ts": + return typescriptLanguage; + case "sql": + return StreamLanguage.define(sqlLanguage); + case "postgresql": + case "pgsql": + case "postgres": + return StreamLanguage.define(postgresqlLanguage); + case "rust": + case "rs": + return StreamLanguage.define(rustLanguage); + case "css": + return StreamLanguage.define(sqlLanguage); + case "html": + return htmlLanguage; + case "python": + case "py": + return StreamLanguage.define(pythonLanguage); + case "protobuf": + case "proto": + return StreamLanguage.define(protobufLanguage); + case "shell": + case "sh": + case "bash": + case "zsh": + case "fish": + return StreamLanguage.define(shellLanguage); + case "swift": + return StreamLanguage.define(rustLanguage); + case "toml": + return StreamLanguage.define(tomlLanguage); + case "json": + return StreamLanguage.define(jsonLanguage); + case "xml": + return StreamLanguage.define(xmlLanguage); + case "c": + return StreamLanguage.define(cLanguage); + case "cpp": + case "c++": + case "cxx": + return StreamLanguage.define(cppLanguage); + case "java": + return StreamLanguage.define(javaLanguage); + case "csharp": + case "cs": + case "c#": + return StreamLanguage.define(csharpLanguage); + case "scala": + return StreamLanguage.define(scalaLanguage); + case "kotlin": + return StreamLanguage.define(kotlinLanguage); + case "objc": + case "objective-c": + case "objectivec": + return StreamLanguage.define(objectiveCLanguage); + case "objcpp": + case "objective-cpp": + case "objectivecpp": + case "objective-c++": + case "objectivec++": + return StreamLanguage.define(objectiveCppLanguage); + + case "dart": + return StreamLanguage.define(dartLanguage); + + case "query": + return LRLanguage.define({ + name: "query", + parser: highlightingDirectiveParser, + }); + + default: + if (name.startsWith("#")) { + return StreamLanguage.define(yamlLanguage); + } + } + return null; +} diff --git a/common/limited_map.test.ts b/common/limited_map.test.ts new file mode 100644 index 0000000..2014d35 --- /dev/null +++ b/common/limited_map.test.ts @@ -0,0 +1,20 @@ +import { sleep } from "$sb/lib/async.ts"; +import { assertEquals } from "../test_deps.ts"; +import { LimitedMap } from "./limited_map.ts"; + +Deno.test("limited map", async () => { + const mp = new LimitedMap(3); + mp.set("a", "a"); + mp.set("b", "b"); + mp.set("c", "c"); + await sleep(2); + assertEquals(mp.get("a"), "a"); + await sleep(2); + assertEquals(mp.get("b"), "b"); + await sleep(2); + assertEquals(mp.get("c"), "c"); + // Drops the first key + mp.set("d", "d"); + await sleep(2); + assertEquals(mp.get("a"), undefined); +}); diff --git a/common/limited_map.ts b/common/limited_map.ts new file mode 100644 index 0000000..4561cef --- /dev/null +++ b/common/limited_map.ts @@ -0,0 +1,50 @@ +type LimitedMapRecord = Record; + +export class LimitedMap { + constructor(private maxSize: number, private map: LimitedMapRecord = {}) { + } + + set(key: string, value: V) { + if (Object.keys(this.map).length >= this.maxSize) { + // Remove the oldest key before adding a new one + const oldestKey = this.getOldestKey(); + delete this.map[oldestKey!]; + } + this.map[key] = { value, la: Date.now() }; + } + + get(key: string): V | undefined { + const entry = this.map[key]; + if (entry) { + // Update the last accessed timestamp + entry.la = Date.now(); + return entry.value; + } + return undefined; + } + + remove(key: string) { + delete this.map[key]; + } + + toJSON() { + return this.map; + } + + private getOldestKey(): string | undefined { + let oldestKey: string | undefined; + let oldestTimestamp: number | undefined; + + for (const key in this.map) { + if (Object.prototype.hasOwnProperty.call(this.map, key)) { + const entry = this.map[key]; + if (!oldestTimestamp || entry.la < oldestTimestamp) { + oldestKey = key; + oldestTimestamp = entry.la; + } + } + } + + return oldestKey; + } +} diff --git a/common/markdown_parser/customtags.ts b/common/markdown_parser/customtags.ts index 5d93e4e..bdc97e0 100644 --- a/common/markdown_parser/customtags.ts +++ b/common/markdown_parser/customtags.ts @@ -22,5 +22,5 @@ export const AttributeNameTag = Tag.define(); export const AttributeValueTag = Tag.define(); export const TaskTag = Tag.define(); -export const TaskMarkerTag = Tag.define(); +export const TaskMarkTag = Tag.define(); export const TaskStateTag = Tag.define(); diff --git a/common/markdown_parser/expression.grammar b/common/markdown_parser/expression.grammar new file mode 100644 index 0000000..049d38e --- /dev/null +++ b/common/markdown_parser/expression.grammar @@ -0,0 +1,119 @@ +@top Program { Expression } + +@precedence { + mulop @left + addop @left + binop @left + and @left + or @left +} + +@skip { + space +} + +commaSep { content ("," content)* } + +kw { @specialize[@name={term}] } + + +Query { + TagIdentifier ( WhereClause | LimitClause | OrderClause | SelectClause | RenderClause )* +} + +WhereClause { kw<"where"> Expression } +LimitClause { kw<"limit"> Expression } +OrderClause { Order commaSep } +OrderBy { Expression OrderDirection? } +SelectClause { kw<"select"> commaSep } +RenderClause { kw<"render"> PageRef } + +Select { Identifier | Expression kw<"as"> Identifier } OrderDirection { - "desc" | "asc" + OrderKW } -Value { Number | String | Bool | Regex | Null | List } +Value { Number | String | Bool | Regex | kw<"null"> | List } -LogicalExpr { FilterExpr (And FilterExpr)* } - -FilterExpr { - Name "<" Value -| Name "<=" Value -| Name "=" Value -| Name "!=" Value -| Name ">=" Value -| Name ">" Value -| Name "=~" Value -| Name "!=~" Value -| Name "in" Value +Attribute { + LVal "." Identifier } -List { "[" commaSep "]" } +Call { + Identifier "(" commaSep ")" | Identifier "(" ")" +} + +LVal { + Identifier +| Attribute +} + +ParenthesizedExpression { "(" Expression ")" } + +LogicalExpression { +Expression !and kw<"and"> Expression +| Expression !or kw<"or"> Expression +} + +Expression { + Value +| LVal +| ParenthesizedExpression +| LogicalExpression +| BinExpression +| Call +} + +BinExpression { + Expression !binop "<" Expression +| Expression !binop "<=" Expression +| Expression !binop "=" Expression +| Expression !binop "!=" Expression +| Expression !binop ">=" Expression +| Expression !binop ">" Expression +| Expression !binop "=~" Expression +| Expression !binop "!=~" Expression +| Expression !binop InKW Expression + +| Expression !mulop "*" Expression +| Expression !mulop "/" Expression +| Expression !mulop "%" Expression +| Expression !addop "+" Expression +| Expression !addop "-" Expression +} + +List { "[" commaSep "]" } Bool { - "true" | "false" + BooleanKW } @tokens { space { std.whitespace+ } - Name { (std.asciiLetter | "-" | "_")+ } - Where { "where" } - Order { "order by" } - Select { "select" } - Render { "render" } - Limit { "limit" } - And { "and" } - Null { "null" } + TagIdentifier { @asciiLetter (@asciiLetter | @digit | "-" | "_" | "/" )* } + + Identifier { @asciiLetter (@asciiLetter | @digit | "-" | "_")* } String { ("\"" | "“" | "”") ![\"”“]* ("\"" | "“" | "”") @@ -60,9 +104,16 @@ Bool { PageRef { "[" "[" ![\]]* "]" "]" } + Order { "order by" } Regex { "/" ( ![/\\\n\r] | "\\" _ )* "/"? } Number { std.digit+ } - // @precedence { Where, Sort, Select, Render, Limit, And, Null, Name } + BooleanKW { "true" | "false" } + + InKW { "in" } + + OrderKW { "asc" | "desc" } + + @precedence { Order, BooleanKW, InKW, OrderKW, Identifier, Number } } diff --git a/common/spaces/indexeddb_space_primitives.test.ts b/common/spaces/datastore_space_primitives.test.ts similarity index 65% rename from common/spaces/indexeddb_space_primitives.test.ts rename to common/spaces/datastore_space_primitives.test.ts index b09e4a4..fd91a8a 100644 --- a/common/spaces/indexeddb_space_primitives.test.ts +++ b/common/spaces/datastore_space_primitives.test.ts @@ -1,9 +1,17 @@ -import { indexedDB } from "https://deno.land/x/indexeddb@1.3.5/ponyfill_memory.ts"; -import { IndexedDBSpacePrimitives } from "./indexeddb_space_primitives.ts"; +import "https://esm.sh/fake-indexeddb@4.0.2/auto"; import { assertEquals } from "../../test_deps.ts"; +import { DataStore } from "../../plugos/lib/datastore.ts"; +import { IndexedDBKvPrimitives } from "../../plugos/lib/indexeddb_kv_primitives.ts"; +import { DataStoreSpacePrimitives } from "./datastore_space_primitives.ts"; -Deno.test("IndexedDBSpacePrimitives", async () => { - const space = new IndexedDBSpacePrimitives("test", indexedDB); +Deno.test("DataStoreSpacePrimitives", { + sanitizeResources: false, + sanitizeOps: false, +}, async () => { + const db = new IndexedDBKvPrimitives("test"); + await db.init(); + + const space = new DataStoreSpacePrimitives(new DataStore(db)); const files = await space.fetchFileList(); assertEquals(files, []); // Write text file @@ -28,6 +36,8 @@ Deno.test("IndexedDBSpacePrimitives", async () => { await space.deleteFile("test.bin"); assertEquals(await space.fetchFileList(), [fileMeta]); + + db.close(); }); function stringToBytes(str: string): Uint8Array { diff --git a/common/spaces/indexeddb_space_primitives.ts b/common/spaces/datastore_space_primitives.ts similarity index 53% rename from common/spaces/indexeddb_space_primitives.ts rename to common/spaces/datastore_space_primitives.ts index 7dcaae2..046de0d 100644 --- a/common/spaces/indexeddb_space_primitives.ts +++ b/common/spaces/datastore_space_primitives.ts @@ -1,7 +1,7 @@ import type { SpacePrimitives } from "./space_primitives.ts"; -import Dexie, { Table } from "dexie"; import { mime } from "../deps.ts"; import { FileMeta } from "$sb/types.ts"; +import { DataStore } from "../../plugos/lib/datastore.ts"; export type FileContent = { name: string; @@ -9,34 +9,27 @@ export type FileContent = { data: Uint8Array; }; -export class IndexedDBSpacePrimitives implements SpacePrimitives { - private db: Dexie; - filesMetaTable: Table; - filesContentTable: Table; +const filesMetaPrefix = ["file", "meta"]; +const filesContentPrefix = ["file", "content"]; +export class DataStoreSpacePrimitives implements SpacePrimitives { constructor( - dbName: string, - indexedDB?: any, + private ds: DataStore, ) { - this.db = new Dexie(dbName, { - indexedDB, - }); - this.db.version(1).stores({ - fileMeta: "name", - fileContent: "name", - }); - this.filesMetaTable = this.db.table("fileMeta"); - this.filesContentTable = this.db.table("fileContent"); } - fetchFileList(): Promise { - return this.filesMetaTable.toArray(); + async fetchFileList(): Promise { + return (await this.ds.query({ prefix: filesMetaPrefix })) + .map((kv) => kv.value); } async readFile( name: string, ): Promise<{ data: Uint8Array; meta: FileMeta }> { - const fileContent = await this.filesContentTable.get(name); + const fileContent = await this.ds.get([ + ...filesContentPrefix, + name, + ]); if (!fileContent) { throw new Error("Not found"); } @@ -60,22 +53,35 @@ export class IndexedDBSpacePrimitives implements SpacePrimitives { size: data.byteLength, perm: suggestedMeta?.perm || "rw", }; - await this.filesContentTable.put({ name, data, meta }); - await this.filesMetaTable.put(meta); + await this.ds.batchSet([ + { + key: [...filesContentPrefix, name], + value: { name, data, meta }, + }, + { + key: [...filesMetaPrefix, name], + value: meta, + }, + ]); return meta; } async deleteFile(name: string): Promise { - const fileMeta = await this.filesMetaTable.get(name); + const fileMeta = await this.ds.get([ + ...filesMetaPrefix, + name, + ]); if (!fileMeta) { throw new Error("Not found"); } - await this.filesMetaTable.delete(name); - await this.filesContentTable.delete(name); + return this.ds.batchDelete([ + [...filesMetaPrefix, name], + [...filesContentPrefix, name], + ]); } async getFileMeta(name: string): Promise { - const fileMeta = await this.filesMetaTable.get(name); + const fileMeta = await this.ds.get([...filesMetaPrefix, name]); if (!fileMeta) { throw new Error("Not found"); } diff --git a/common/spaces/disk_space_primitives.ts b/common/spaces/disk_space_primitives.ts index 6ce8fea..1b2463a 100644 --- a/common/spaces/disk_space_primitives.ts +++ b/common/spaces/disk_space_primitives.ts @@ -93,14 +93,7 @@ export class DiskSpacePrimitives implements SpacePrimitives { file.close(); // Fetch new metadata - const s = await Deno.stat(localPath); - return { - name: name, - size: s.size, - contentType: lookupContentType(name), - lastModified: s.mtime!.getTime(), - perm: "rw", - }; + return this.getFileMeta(name); } catch (e) { console.error("Error while writing file", name, e); throw Error(`Could not write ${name}`); diff --git a/common/spaces/evented_space_primitives.ts b/common/spaces/evented_space_primitives.ts index 0792f28..4926308 100644 --- a/common/spaces/evented_space_primitives.ts +++ b/common/spaces/evented_space_primitives.ts @@ -47,7 +47,7 @@ export class EventedSpacePrimitives implements SpacePrimitives { oldHash !== newHash ) ) { - this.dispatchEvent("file:changed", meta.name); + await this.dispatchEvent("file:changed", meta.name); } // Page found, not deleted deletedFiles.delete(meta.name); @@ -58,7 +58,7 @@ export class EventedSpacePrimitives implements SpacePrimitives { for (const deletedFile of deletedFiles) { delete this.spaceSnapshot[deletedFile]; - this.dispatchEvent("file:deleted", deletedFile); + await this.dispatchEvent("file:deleted", deletedFile); if (deletedFile.endsWith(".md")) { const pageName = deletedFile.substring(0, deletedFile.length - 3); @@ -66,7 +66,7 @@ export class EventedSpacePrimitives implements SpacePrimitives { } } - this.dispatchEvent("file:listed", newFileList); + await this.dispatchEvent("file:listed", newFileList); this.alreadyFetching = false; this.initialFileListLoad = false; return newFileList; @@ -93,7 +93,7 @@ export class EventedSpacePrimitives implements SpacePrimitives { meta, ); if (!selfUpdate) { - this.dispatchEvent("file:changed", name, true); + await this.dispatchEvent("file:changed", name, true); } this.spaceSnapshot[name] = newMeta.lastModified; @@ -104,16 +104,11 @@ export class EventedSpacePrimitives implements SpacePrimitives { const decoder = new TextDecoder("utf-8"); text = decoder.decode(data); - this.dispatchEvent("page:saved", pageName, newMeta) - .then(() => { - return this.dispatchEvent("page:index_text", { - name: pageName, - text, - }); - }) - .catch((e) => { - console.error("Error dispatching page:saved event", e); - }); + await this.dispatchEvent("page:saved", pageName, newMeta); + await this.dispatchEvent("page:index_text", { + name: pageName, + text, + }); } return newMeta; } @@ -134,9 +129,9 @@ export class EventedSpacePrimitives implements SpacePrimitives { this.triggerEventsAndCache(name, newMeta.lastModified); return newMeta; } catch (e: any) { - console.log("Checking error", e, name); + // console.log("Checking error", e, name); if (e.message === "Not found") { - this.dispatchEvent("file:deleted", name); + await this.dispatchEvent("file:deleted", name); if (name.endsWith(".md")) { const pageName = name.substring(0, name.length - 3); await this.dispatchEvent("page:deleted", pageName); @@ -154,6 +149,6 @@ export class EventedSpacePrimitives implements SpacePrimitives { // await this.getPageMeta(name); // Check if page exists, if not throws Error await this.wrapped.deleteFile(name); delete this.spaceSnapshot[name]; - this.dispatchEvent("file:deleted", name); + await this.dispatchEvent("file:deleted", name); } } diff --git a/common/spaces/file_meta_space_primitives.ts b/common/spaces/file_meta_space_primitives.ts deleted file mode 100644 index 7fa4c00..0000000 --- a/common/spaces/file_meta_space_primitives.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { SpacePrimitives } from "./space_primitives.ts"; -import type { SysCallMapping } from "../../plugos/system.ts"; -import { FileMeta } from "$sb/types.ts"; - -// Enriches the file list listing with custom metadata from the page index -export class FileMetaSpacePrimitives implements SpacePrimitives { - constructor( - private wrapped: SpacePrimitives, - private indexSyscalls: SysCallMapping, - ) { - } - - async fetchFileList(): Promise { - const files = await this.wrapped.fetchFileList(); - // Enrich the file list with custom meta data (for pages) - const allFilesMap: Map = new Map( - files.map((fm) => [fm.name, fm]), - ); - for ( - const { page, value } of await this.indexSyscalls["index.queryPrefix"]( - {} as any, - "meta:", - ) - ) { - const p = allFilesMap.get(`${page}.md`); - if (p) { - for (const [k, v] of Object.entries(value)) { - if ( - ["name", "lastModified", "size", "perm", "contentType"].includes(k) - ) { - continue; - } - p[k] = v; - } - } - } - return [...allFilesMap.values()]; - } - - readFile( - name: string, - ): Promise<{ data: Uint8Array; meta: FileMeta }> { - return this.wrapped.readFile(name); - } - - async getFileMeta(name: string): Promise { - const meta = await this.wrapped.getFileMeta(name); - if (name.endsWith(".md")) { - const pageName = name.slice(0, -3); - const additionalMeta = await this.indexSyscalls["index.get"]( - {} as any, - pageName, - "meta:", - ); - if (additionalMeta) { - for (const [k, v] of Object.entries(additionalMeta)) { - if ( - ["name", "lastModified", "size", "perm", "contentType"].includes(k) - ) { - continue; - } - meta[k] = v; - } - } - } - return meta; - } - - writeFile( - name: string, - data: Uint8Array, - selfUpdate?: boolean, - meta?: FileMeta, - ): Promise { - return this.wrapped.writeFile( - name, - data, - selfUpdate, - meta, - ); - } - - deleteFile(name: string): Promise { - return this.wrapped.deleteFile(name); - } -} diff --git a/plugs/directive/handlebar_helpers.ts b/common/syscalls/handlebar_helpers.ts similarity index 95% rename from plugs/directive/handlebar_helpers.ts rename to common/syscalls/handlebar_helpers.ts index 38b2248..2b2380b 100644 --- a/plugs/directive/handlebar_helpers.ts +++ b/common/syscalls/handlebar_helpers.ts @@ -1,6 +1,6 @@ import { niceDate } from "$sb/lib/dates.ts"; -export function handlebarHelpers(_pageName: string) { +export function handlebarHelpers() { return { json: (v: any) => JSON.stringify(v), niceDate: (ts: any) => niceDate(new Date(ts)), diff --git a/common/syscalls/handlebars.ts b/common/syscalls/handlebars.ts new file mode 100644 index 0000000..e6485d8 --- /dev/null +++ b/common/syscalls/handlebars.ts @@ -0,0 +1,23 @@ +import { SysCallMapping } from "../../plugos/system.ts"; +import { handlebarHelpers } from "./handlebar_helpers.ts"; +import Handlebars from "handlebars"; + +export function handlebarsSyscalls(): SysCallMapping { + return { + "handlebars.renderTemplate": ( + _ctx, + template: string, + obj: any, + globals: Record = {}, + ): string => { + const templateFn = Handlebars.compile( + template, + { noEscape: true }, + ); + return templateFn(obj, { + helpers: handlebarHelpers(), + data: globals, + }); + }, + }; +} diff --git a/common/syscalls/language.ts b/common/syscalls/language.ts new file mode 100644 index 0000000..73605bf --- /dev/null +++ b/common/syscalls/language.ts @@ -0,0 +1,20 @@ +import { SysCallMapping } from "../../plugos/system.ts"; +import { parse } from "../markdown_parser/parse_tree.ts"; +import type { ParseTree } from "$sb/lib/tree.ts"; +import { languageFor } from "../languages.ts"; + +export function languageSyscalls(): SysCallMapping { + return { + "language.parseLanguage": ( + _ctx, + language: string, + code: string, + ): ParseTree => { + const lang = languageFor(language); + if (!lang) { + throw new Error(`Unknown language ${language}`); + } + return parse(lang, code); + }, + }; +} diff --git a/web/syscalls/markdown.ts b/common/syscalls/markdown.ts similarity index 73% rename from web/syscalls/markdown.ts rename to common/syscalls/markdown.ts index c2c5ec6..d9ee61e 100644 --- a/web/syscalls/markdown.ts +++ b/common/syscalls/markdown.ts @@ -1,6 +1,6 @@ import { SysCallMapping } from "../../plugos/system.ts"; -import { parse } from "../../common/markdown_parser/parse_tree.ts"; -import { Language } from "../deps.ts"; +import { parse } from "../markdown_parser/parse_tree.ts"; +import { Language } from "../../web/deps.ts"; import type { ParseTree } from "$sb/lib/tree.ts"; export function markdownSyscalls(lang: Language): SysCallMapping { diff --git a/web/syscalls/yaml.ts b/common/syscalls/yaml.ts similarity index 89% rename from web/syscalls/yaml.ts rename to common/syscalls/yaml.ts index 862882b..636f7a1 100644 --- a/web/syscalls/yaml.ts +++ b/common/syscalls/yaml.ts @@ -1,5 +1,5 @@ import { SysCallMapping } from "../../plugos/system.ts"; -import { YAML } from "../deps.ts"; +import { YAML } from "../../web/deps.ts"; export function yamlSyscalls(): SysCallMapping { return { diff --git a/deno.jsonc b/deno.jsonc index b54a255..bfb2ab3 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -15,8 +15,7 @@ "bundle": "deno run -A build_bundle.ts", // Regenerates some bundle files (checked into the repo) - // Install lezer-generator with "npm install -g @lezer/generator" - "generate": "lezer-generator common/markdown_parser/query.grammar -o common/markdown_parser/parse-query.js", + "generate": "./scripts/generate.sh", // Compile "compile": "deno task bundle && deno compile -A --unstable -o silverbullet dist/silverbullet.js", diff --git a/import_map.json b/import_map.json index c1ecd6a..f49ebf0 100644 --- a/import_map.json +++ b/import_map.json @@ -19,7 +19,6 @@ "preact": "https://esm.sh/preact@10.11.1", "$sb/": "./plug-api/", - "handlebars": "https://esm.sh/handlebars@4.7.7?target=es2022", - "dexie": "https://esm.sh/dexie@3.2.2?target=es2022" + "handlebars": "https://esm.sh/handlebars@4.7.7?target=es2022" } } diff --git a/plug-api/app_event.ts b/plug-api/app_event.ts index 3e3639c..3767b05 100644 --- a/plug-api/app_event.ts +++ b/plug-api/app_event.ts @@ -1,6 +1,6 @@ import type { ParseTree } from "$sb/lib/tree.ts"; -import { ParsedQuery } from "$sb/lib/query.ts"; import { TextChange } from "$sb/lib/change.ts"; +import { Query } from "$sb/types.ts"; export type AppEvent = | "page:click" @@ -16,7 +16,7 @@ export type AppEvent = | "editor:pageModified"; export type QueryProviderEvent = { - query: ParsedQuery; + query: Query; pageName: string; }; diff --git a/plug-api/lib/builtin_query_functions.ts b/plug-api/lib/builtin_query_functions.ts new file mode 100644 index 0000000..12ad1a6 --- /dev/null +++ b/plug-api/lib/builtin_query_functions.ts @@ -0,0 +1,39 @@ +import type { FunctionMap } from "$sb/types.ts"; +import { niceDate } from "$sb/lib/dates.ts"; + +export const builtinFunctions: FunctionMap = { + today() { + return niceDate(new Date()); + }, + max(...args: number[]) { + return Math.max(...args); + }, + min(...args: number[]) { + return Math.min(...args); + }, + toJSON(obj: any) { + return JSON.stringify(obj); + }, + // Note: these assume Monday as the first day of the week + firstDayOfWeek(dateString: string): string { + const date = new Date(dateString); + const dayOfWeek = date.getDay(); + const daysToSubtract = (dayOfWeek + 7 - 1) % 7; + const firstDayOfWeek = new Date(date); + firstDayOfWeek.setDate(date.getDate() - daysToSubtract); + return niceDate(firstDayOfWeek); + }, + lastDayOfWeek(dateString: string): string { + const date = new Date(dateString); + const dayOfWeek = date.getDay(); + const daysToAdd = (7 - dayOfWeek) % 7; + const lastDayOfWeek = new Date(date); + lastDayOfWeek.setDate(date.getDate() + daysToAdd); + return niceDate(lastDayOfWeek); + }, + addDays(dateString: string, daysToAdd: number): string { + const date = new Date(dateString); + date.setDate(date.getDate() + daysToAdd); + return niceDate(date); + }, +}; diff --git a/plug-api/lib/frontmatter.ts b/plug-api/lib/frontmatter.ts index f569510..ad1d302 100644 --- a/plug-api/lib/frontmatter.ts +++ b/plug-api/lib/frontmatter.ts @@ -2,7 +2,7 @@ import { YAML } from "$sb/plugos-syscall/mod.ts"; import { addParentPointers, - findNodeOfType, + collectNodesOfType, ParseTree, renderToText, replaceNodesMatchingAsync, @@ -18,21 +18,24 @@ export async function extractFrontmatter( ): Promise { let data: any = {}; addParentPointers(tree); + let paragraphCounter = 0; await replaceNodesMatchingAsync(tree, async (t) => { - // Find top-level hash tags - if (t.type === "Hashtag") { - // Check if if nested directly into a Paragraph - if (t.parent && t.parent.type === "Paragraph") { - const tagname = t.children![0].text!.substring(1); + if (t.type === "Paragraph") { + paragraphCounter++; + // Only attach hashtags in the first paragraph to the page + if (paragraphCounter !== 1) { + return; + } + collectNodesOfType(t, "Hashtag").forEach((h) => { if (!data.tags) { data.tags = []; } + const tagname = h.children![0].text!.substring(1); if (Array.isArray(data.tags) && !data.tags.includes(tagname)) { data.tags.push(tagname); } - } - return; + }); } // Find FrontMatter and parse it if (t.type === "FrontMatter") { @@ -64,43 +67,6 @@ export async function extractFrontmatter( } } - // Find a fenced code block with `meta` as the language type - if (t.type !== "FencedCode") { - return; - } - const codeInfoNode = findNodeOfType(t, "CodeInfo"); - if (!codeInfoNode) { - return; - } - if (codeInfoNode.children![0].text !== "meta") { - return; - } - const codeTextNode = findNodeOfType(t, "CodeText"); - if (!codeTextNode) { - // Honestly, this shouldn't happen - return; - } - const codeText = codeTextNode.children![0].text!; - const parsedData: any = YAML.parse(codeText); - const newData = { ...parsedData }; - data = { ...data, ...parsedData }; - if (removeKeys.length > 0) { - let removedOne = false; - for (const key of removeKeys) { - if (key in newData) { - delete newData[key]; - removedOne = true; - } - } - if (removedOne) { - codeTextNode.children![0].text = (await YAML.stringify(newData)).trim(); - } - } - // If nothing is left, let's just delete this whole block - if (Object.keys(newData).length === 0) { - return null; - } - return undefined; }); diff --git a/plug-api/lib/parse-query.ts b/plug-api/lib/parse-query.ts new file mode 100644 index 0000000..080b171 --- /dev/null +++ b/plug-api/lib/parse-query.ts @@ -0,0 +1,180 @@ +import type { AST } from "$sb/lib/tree.ts"; +import type { Query, QueryExpression } from "$sb/types.ts"; + +export function astToKvQuery( + node: AST, +): Query { + const query: Query = { + querySource: "", + }; + const [queryType, querySource, ...clauses] = node; + if (queryType !== "Query") { + throw new Error(`Expected query type, got ${queryType}`); + } + query.querySource = querySource[1] as string; + for (const clause of clauses) { + const [clauseType] = clause; + switch (clauseType) { + case "WhereClause": { + if (query.filter) { + query.filter = [ + "and", + query.filter, + expressionToKvQueryFilter(clause[2]), + ]; + } else { + query.filter = expressionToKvQueryFilter(clause[2]); + } + break; + } + case "OrderClause": { + if (!query.orderBy) { + query.orderBy = []; + } + for (const orderBy of clause.slice(2)) { + if (orderBy[0] === "OrderBy") { + // console.log("orderBy", orderBy); + const expr = orderBy[1][1]; + if (orderBy[2]) { + query.orderBy.push({ + expr: expressionToKvQueryExpression(expr), + desc: orderBy[2][1][1] === "desc", + }); + } else { + query.orderBy.push({ + expr: expressionToKvQueryExpression(expr), + desc: false, + }); + } + } + } + + break; + } + case "LimitClause": { + query.limit = expressionToKvQueryExpression(clause[2][1]); + break; + } + case "SelectClause": { + for (const select of clause.slice(2)) { + if (select[0] === "Select") { + if (!query.select) { + query.select = []; + } + if (select.length === 2) { + query.select.push({ name: select[1][1] as string }); + } else { + query.select.push({ + name: select[3][1] as string, + expr: expressionToKvQueryExpression(select[1]), + }); + } + } + } + break; + } + case "RenderClause": { + query.render = (clause[2][1] as string).slice(2, -2); + break; + } + default: + throw new Error(`Unknown clause type: ${clauseType}`); + } + } + return query; +} + +export function expressionToKvQueryExpression(node: AST): QueryExpression { + if (["LVal", "Expression", "Value"].includes(node[0])) { + return expressionToKvQueryExpression(node[1]); + } + // console.log("Got expression", node); + switch (node[0]) { + case "Attribute": { + return [ + "attr", + expressionToKvQueryExpression(node[1]), + node[3][1] as string, + ]; + } + case "Identifier": + return ["attr", node[1] as string]; + case "String": + return ["string", (node[1] as string).slice(1, -1)]; + case "Number": + return ["number", +(node[1])]; + case "Bool": + return ["boolean", node[1][1] === "true"]; + case "null": + return ["null"]; + case "Regex": + return ["regexp", (node[1] as string).slice(1, -1), "i"]; + case "List": { + const exprs: AST[] = []; + for (const expr of node.slice(2)) { + if (expr[0] === "Expression") { + exprs.push(expr); + } + } + return ["array", exprs.map(expressionToKvQueryExpression)]; + } + case "BinExpression": { + const lval = expressionToKvQueryExpression(node[1]); + const binOp = (node[2] as string).trim(); + const val = expressionToKvQueryExpression(node[3]); + return [binOp as any, lval, val]; + } + case "LogicalExpression": { + const op1 = expressionToKvQueryFilter(node[1]); + const op = node[2]; + const op2 = expressionToKvQueryFilter(node[3]); + return [op[1] as any, op1, op2]; + } + case "ParenthesizedExpression": { + return expressionToKvQueryFilter(node[2]); + } + case "Call": { + // console.log("Call", node); + const fn = node[1][1] as string; + const args: AST[] = []; + for (const expr of node.slice(2)) { + if (expr[0] === "Expression") { + args.push(expr); + } + } + return ["call", fn, args.map(expressionToKvQueryExpression)]; + } + default: + throw new Error(`Not supported: ${node[0]}`); + } +} + +function expressionToKvQueryFilter( + node: AST, +): QueryExpression { + const [expressionType] = node; + if (expressionType === "Expression") { + return expressionToKvQueryFilter(node[1]); + } + switch (expressionType) { + case "BinExpression": { + const lval = expressionToKvQueryExpression(node[1]); + const binOp = node[2][0] === "InKW" ? "in" : (node[2] as string).trim(); + const val = expressionToKvQueryExpression(node[3]); + return [binOp as any, lval, val]; + } + case "LogicalExpression": { + // console.log("Logical expression", node); + // 0 = first operand, 1 = whitespace, 2 = operator, 3 = whitespace, 4 = second operand + const op1 = expressionToKvQueryFilter(node[1]); + const op = node[2]; // 1 is whitespace + const op2 = expressionToKvQueryFilter(node[3]); + return [op[1] as any, op1, op2]; + } + case "ParenthesizedExpression": { + return expressionToKvQueryFilter(node[2]); + } + default: + throw new Error(`Unknown expression type: ${expressionType}`); + } +} diff --git a/plug-api/lib/parser-query.test.ts b/plug-api/lib/parser-query.test.ts new file mode 100644 index 0000000..3c609e7 --- /dev/null +++ b/plug-api/lib/parser-query.test.ts @@ -0,0 +1,193 @@ +import { parse } from "../../common/markdown_parser/parse_tree.ts"; +import buildMarkdown from "../../common/markdown_parser/parser.ts"; +import { AST, findNodeOfType, parseTreeToAST } from "$sb/lib/tree.ts"; +import { assertEquals } from "../../test_deps.ts"; +import { astToKvQuery } from "$sb/lib/parse-query.ts"; + +const lang = buildMarkdown([]); + +function wrapQueryParse(query: string): AST | null { + const tree = parse(lang, `\n$\n`); + return parseTreeToAST(findNodeOfType(tree, "Query")!); +} + +Deno.test("Test directive parser", () => { + // const query = ; + // console.log("query", query); + assertEquals( + astToKvQuery(wrapQueryParse(`page where name = "test"`)!), + { + querySource: "page", + filter: ["=", ["attr", "name"], ["string", "test"]], + }, + ); + + assertEquals( + astToKvQuery(wrapQueryParse(`page where name =~ /test/`)!), + { + querySource: "page", + filter: ["=~", ["attr", "name"], ["regexp", "test", "i"]], + }, + ); + + assertEquals( + astToKvQuery(wrapQueryParse(`page where parent.name = "test"`)!), + { + querySource: "page", + filter: ["=", ["attr", ["attr", "parent"], "name"], ["string", "test"]], + }, + ); + + assertEquals( + astToKvQuery( + wrapQueryParse(`page where name = "test" and age > 20`)!, + ), + { + querySource: "page", + filter: ["and", ["=", ["attr", "name"], ["string", "test"]], [">", [ + "attr", + "age", + ], ["number", 20]]], + }, + ); + + assertEquals( + astToKvQuery( + wrapQueryParse(`page where name = "test" and age > 20 or done = true`)!, + ), + { + querySource: "page", + filter: ["or", ["and", ["=", ["attr", "name"], ["string", "test"]], [ + ">", + [ + "attr", + "age", + ], + ["number", 20], + ]], ["=", ["attr", "done"], ["boolean", true]]], + }, + ); + + assertEquals( + astToKvQuery( + wrapQueryParse(`page where (age <= 20) or task.done = null`)!, + ), + { + querySource: "page", + filter: ["or", ["<=", ["attr", "age"], ["number", 20]], [ + "=", + [ + "attr", + [ + "attr", + "task", + ], + "done", + ], + ["null"], + ]], + }, + ); + + assertEquals( + astToKvQuery( + wrapQueryParse(`task order by lastModified asc`)!, + ), + { + querySource: "task", + orderBy: [{ expr: ["attr", "lastModified"], desc: false }], + }, + ); + assertEquals( + astToKvQuery( + wrapQueryParse(`task order by lastModified`)!, + ), + { + querySource: "task", + orderBy: [{ expr: ["attr", "lastModified"], desc: false }], + }, + ); + assertEquals( + astToKvQuery( + wrapQueryParse(`task order by lastModified desc, name, age asc`)!, + ), + { + querySource: "task", + orderBy: [{ expr: ["attr", "lastModified"], desc: true }, { + expr: ["attr", "name"], + desc: false, + }, { expr: ["attr", "age"], desc: false }], + }, + ); + assertEquals( + astToKvQuery( + wrapQueryParse(`task order by lastModified desc limit 5`)!, + ), + { + querySource: "task", + orderBy: [{ expr: ["attr", "lastModified"], desc: true }], + limit: ["number", 5], + }, + ); + assertEquals( + astToKvQuery( + wrapQueryParse(`task select name, lastModified + 20 as modified`)!, + ), + { + querySource: "task", + select: [{ name: "name" }, { + name: "modified", + expr: ["+", ["attr", "lastModified"], ["number", 20]], + }], + }, + ); + + assertEquals( + astToKvQuery( + wrapQueryParse(`task render [[my/page]]`)!, + ), + { + querySource: "task", + render: "my/page", + }, + ); + + assertEquals( + astToKvQuery( + wrapQueryParse(`task where name in ["hello", 1]`)!, + ), + { + querySource: "task", + filter: ["in", ["attr", "name"], ["array", [["string", "hello"], [ + "number", + 1, + ]]]], + }, + ); + + assertEquals( + astToKvQuery( + wrapQueryParse(`task select today() as today2`)!, + ), + { + querySource: "task", + select: [{ + name: "today2", + expr: ["call", "today", []], + }], + }, + ); + + assertEquals( + astToKvQuery( + wrapQueryParse(`task select today(1, 2, 3) as today`)!, + ), + { + querySource: "task", + select: [{ + name: "today", + expr: ["call", "today", [["number", 1], ["number", 2], ["number", 3]]], + }], + }, + ); +}); diff --git a/plug-api/lib/query.ts b/plug-api/lib/query.ts index d0a3824..0110c1f 100644 --- a/plug-api/lib/query.ts +++ b/plug-api/lib/query.ts @@ -1,145 +1,227 @@ import { ParseTree, renderToText, replaceNodesMatching } from "$sb/lib/tree.ts"; +import { FunctionMap, KV, Query, QueryExpression } from "$sb/types.ts"; export const queryRegex = /()(.+?)()/gs; export const directiveStartRegex = //s; - export const directiveEndRegex = //s; -export type QueryFilter = { - op: string; - prop: string; - value: any; -}; +export function evalQueryExpression( + val: QueryExpression, + obj: any, + functionMap: FunctionMap = {}, +): any { + const [type, op1] = val; -export type QueryOrdering = { - orderBy: string; - orderDesc: boolean; -}; - -export type ParsedQuery = { - table: string; - limit?: number; - ordering: QueryOrdering[]; - /** @deprecated Please use ordering. - * Deprecated due to PR #387 - * Currently holds ordering[0] if exists - */ - orderBy?: string; - /** @deprecated Please use ordering. - * Deprecated due to PR #387 - * Currently holds ordering[0] if exists - */ - orderDesc?: boolean; - filter: QueryFilter[]; - select?: string[]; - render?: string; -}; - -export function applyQuery(parsedQuery: ParsedQuery, records: T[]): T[] { - let resultRecords: any[] = []; - if (parsedQuery.filter.length === 0) { - resultRecords = records.slice(); - } else { - recordLoop: - for (const record of records) { - const recordAny: any = record; - for (const { op, prop, value } of parsedQuery.filter) { - switch (op) { - case "=": { - const recordPropVal = recordAny[prop]; - if (Array.isArray(recordPropVal) && !Array.isArray(value)) { - // Record property is an array, and value is a scalar: find the value in the array - if (!recordPropVal.includes(value)) { - continue recordLoop; - } - } else if (Array.isArray(recordPropVal) && Array.isArray(value)) { - // Record property is an array, and value is an array: find the value in the array - if (!recordPropVal.some((v) => value.includes(v))) { - continue recordLoop; - } - } else if (!(recordPropVal == value)) { - // Both are scalars: exact value - continue recordLoop; - } - break; - } - case "!=": - if (!(recordAny[prop] != value)) { - continue recordLoop; - } - break; - case "<": - if (!(recordAny[prop] < value)) { - continue recordLoop; - } - break; - case "<=": - if (!(recordAny[prop] <= value)) { - continue recordLoop; - } - break; - case ">": - if (!(recordAny[prop] > value)) { - continue recordLoop; - } - break; - case ">=": - if (!(recordAny[prop] >= value)) { - continue recordLoop; - } - break; - case "=~": - // TODO: Cache regexps somehow - if (!new RegExp(value).exec(recordAny[prop])) { - continue recordLoop; - } - break; - case "!=~": - if (new RegExp(value).exec(recordAny[prop])) { - continue recordLoop; - } - break; - case "in": - if (!value.includes(recordAny[prop])) { - continue recordLoop; - } - break; + switch (type) { + // Logical operators + case "and": + return evalQueryExpression(op1, obj, functionMap) && + evalQueryExpression(val[2], obj, functionMap); + case "or": + return evalQueryExpression(op1, obj, functionMap) || + evalQueryExpression(val[2], obj, functionMap); + // Value types + case "null": + return null; + case "number": + case "string": + case "boolean": + return op1; + case "regexp": + return [op1, val[2]]; + case "attr": { + let attributeVal = obj; + if (val.length === 3) { + attributeVal = evalQueryExpression(val[1], obj, functionMap); + if (attributeVal) { + return attributeVal[val[2]]; + } else { + return null; } + } else if (!val[1]) { + return obj; + } else { + return attributeVal[val[1]]; } - resultRecords.push(recordAny); + } + case "array": { + return op1.map((v) => evalQueryExpression(v, obj, functionMap)); + } + case "object": + return obj; + case "call": { + const fn = functionMap[op1]; + if (!fn) { + throw new Error(`Unknown function: ${op1}`); + } + return fn( + ...val[2].map((v) => evalQueryExpression(v, obj, functionMap)), + ); } } - if (parsedQuery.ordering.length > 0) { - resultRecords = resultRecords.sort((a: any, b: any) => { - for (const { orderBy, orderDesc } of parsedQuery.ordering) { - if (a[orderBy] < b[orderBy] || a[orderBy] === undefined) { - return orderDesc ? 1 : -1; + // Binary operators, here we can pre-calculate the two operand values + const val1 = evalQueryExpression(op1, obj, functionMap); + const val2 = evalQueryExpression(val[2], obj, functionMap); + + switch (type) { + case "+": + return val1 + val2; + case "-": + return val1 - val2; + case "*": + return val1 * val2; + case "/": + return val1 / val2; + case "%": + return val1 % val2; + case "=": { + if (Array.isArray(val1) && !Array.isArray(val2)) { + // Record property is an array, and value is a scalar: find the value in the array + if (val1.includes(val2)) { + return true; } - if (a[orderBy] > b[orderBy] || b[orderBy] === undefined) { - return orderDesc ? -1 : 1; + } else if (Array.isArray(val1) && Array.isArray(val2)) { + // Record property is an array, and value is an array: find the value in the array + if (val1.some((v) => val2.includes(v))) { + return true; } - // Consider them equal. This way helps with comparing arrays (like tags) } + return val1 == val2; + } + case "!=": + return val1 != val2; + case "=~": { + if (!Array.isArray(val2)) { + throw new Error(`Invalid regexp: ${val2}`); + } + const r = new RegExp(val2[0], val2[1]); + return r.test(val1); + } + case "!=~": { + if (!Array.isArray(val2)) { + throw new Error(`Invalid regexp: ${val2}`); + } + const r = new RegExp(val2[0], val2[1]); + return !r.test(val1); + } + case "<": + return val1 < val2; + case "<=": + return val1 <= val2; + case ">": + return val1 > val2; + case ">=": + return val1 >= val2; + case "in": + return val2.includes(val1); + default: + throw new Error(`Unupported operator: ${type}`); + } +} + +/** + * Looks for an attribute assignment in the expression, and returns the expression assigned to the attribute or throws an error when not found + * Side effect: effectively removes the attribute assignment from the expression (by replacing it with true = true) + */ +export function liftAttributeFilter( + expression: QueryExpression | undefined, + attributeName: string, +): QueryExpression { + if (!expression) { + throw new Error(`Cannot find attribute assignment for ${attributeName}`); + } + switch (expression[0]) { + case "=": { + if (expression[1][0] === "attr" && expression[1][1] === attributeName) { + const val = expression[2]; + // Remove the filter by changing it to true = true + expression[1] = ["boolean", true]; + expression[2] = ["boolean", true]; + return val; + } + break; + } + case "and": + case "or": { + const newOp1 = liftAttributeFilter(expression[1], attributeName); + if (newOp1) { + return newOp1; + } + const newOp2 = liftAttributeFilter(expression[2], attributeName); + if (newOp2) { + return newOp2; + } + throw new Error(`Cannot find attribute assignment for ${attributeName}`); + } + } + throw new Error(`Cannot find attribute assignment for ${attributeName}`); +} + +export function applyQuery(query: Query, allItems: T[]): T[] { + // Filter + if (query.filter) { + allItems = allItems.filter((item) => + evalQueryExpression(query.filter!, item) + ); + } + // Add dummy keys, then remove them + return applyQueryNoFilterKV( + query, + allItems.map((v) => ({ key: [], value: v })), + ).map((v) => v.value); +} + +export function applyQueryNoFilterKV( + query: Query, + allItems: KV[], + functionMap: FunctionMap = {}, // TODO: Figure this out later +): KV[] { + // Order by + if (query.orderBy) { + allItems.sort((a, b) => { + const aVal = a.value; + const bVal = b.value; + for (const { expr, desc } of query.orderBy!) { + const evalA = evalQueryExpression(expr, aVal, functionMap); + const evalB = evalQueryExpression(expr, bVal, functionMap); + if ( + evalA < evalB || evalA === undefined + ) { + return desc ? 1 : -1; + } + if ( + evalA > evalB || evalB === undefined + ) { + return desc ? -1 : 1; + } + } + // Consider them equal. This helps with comparing arrays (like tags) return 0; }); } - if (parsedQuery.limit) { - resultRecords = resultRecords.slice(0, parsedQuery.limit); - } - if (parsedQuery.select) { - resultRecords = resultRecords.map((rec) => { + if (query.select) { + for (let i = 0; i < allItems.length; i++) { + const rec = allItems[i].value; const newRec: any = {}; - for (const k of parsedQuery.select!) { - newRec[k] = rec[k]; + for (const { name, expr } of query.select) { + newRec[name] = expr + ? evalQueryExpression(expr, rec, functionMap) + : rec[name]; } - return newRec; - }); + allItems[i].value = newRec; + } } - return resultRecords; + if (query.limit) { + const limit = evalQueryExpression(query.limit, {}, functionMap); + if (allItems.length > limit) { + allItems = allItems.slice(0, limit); + } + } + return allItems; } export function removeQueries(pt: ParseTree) { diff --git a/plug-api/lib/tree.test.ts b/plug-api/lib/tree.test.ts index c7cd5e1..a08e455 100644 --- a/plug-api/lib/tree.test.ts +++ b/plug-api/lib/tree.test.ts @@ -4,6 +4,7 @@ import { collectNodesMatching, findParentMatching, nodeAtPos, + parseTreeToAST, removeParentPointers, renderToText, replaceNodesMatching, @@ -77,3 +78,9 @@ Deno.test("Test parsing", () => { let mdTree3 = parse(lang, mdTest3); // console.log(JSON.stringify(mdTree3, null, 2)); }); + +Deno.test("AST functions", () => { + const lang = wikiMarkdownLang([]); + const mdTree = parse(lang, mdTest1); + console.log(JSON.stringify(parseTreeToAST(mdTree), null, 2)); +}); diff --git a/plug-api/lib/tree.ts b/plug-api/lib/tree.ts index 1b3c54c..ab4af12 100644 --- a/plug-api/lib/tree.ts +++ b/plug-api/lib/tree.ts @@ -8,6 +8,8 @@ export type ParseTree = { parent?: ParseTree; }; +export type AST = [string, ...AST[]] | string; + export function addParentPointers(tree: ParseTree) { if (!tree.children) { return; @@ -208,3 +210,19 @@ export function cloneTree(tree: ParseTree): ParseTree { delete newTree.parent; return newTree; } + +export function parseTreeToAST(tree: ParseTree): AST { + if (tree.text !== undefined) { + return tree.text; + } + const ast: AST = [tree.type!]; + for (const node of tree.children!) { + if (node.type && !node.type.endsWith("Mark")) { + ast.push(parseTreeToAST(node)); + } + if (node.text && node.text.trim()) { + ast.push(node.text); + } + } + return ast; +} diff --git a/plug-api/plugos-syscall/datastore.ts b/plug-api/plugos-syscall/datastore.ts new file mode 100644 index 0000000..6fd3f2f --- /dev/null +++ b/plug-api/plugos-syscall/datastore.ts @@ -0,0 +1,38 @@ +import { syscall } from "$sb/plugos-syscall/syscall.ts"; +import { KV, KvKey, KvQuery } from "$sb/types.ts"; + +export function set(key: KvKey, value: any): Promise { + return syscall("datastore.set", key, value); +} + +export function batchSet(kvs: KV[]): Promise { + return syscall("datastore.batchSet", kvs); +} + +export function get(key: KvKey): Promise { + return syscall("datastore.get", key); +} + +export function batchGet(keys: KvKey[]): Promise<(any | undefined)[]> { + return syscall("datastore.batchGet", keys); +} + +export function del(key: KvKey): Promise { + return syscall("datastore.delete", key); +} + +export function batchDel(keys: KvKey[]): Promise { + return syscall("datastore.batchDelete", keys); +} + +export function query( + query: KvQuery, +): Promise { + return syscall("datastore.query", query); +} + +export function queryDelete( + query: KvQuery, +): Promise { + return syscall("datastore.queryDelete", query); +} diff --git a/plug-api/plugos-syscall/mod.ts b/plug-api/plugos-syscall/mod.ts index afd5252..952dc3c 100644 --- a/plug-api/plugos-syscall/mod.ts +++ b/plug-api/plugos-syscall/mod.ts @@ -1,7 +1,7 @@ export * as asset from "./asset.ts"; export * as events from "./event.ts"; export * as shell from "./shell.ts"; -export * as store from "./store.ts"; export * as YAML from "./yaml.ts"; export * as mq from "./mq.ts"; export * from "./syscall.ts"; +export * as datastore from "./datastore.ts"; diff --git a/plug-api/plugos-syscall/store.ts b/plug-api/plugos-syscall/store.ts deleted file mode 100644 index cf6578f..0000000 --- a/plug-api/plugos-syscall/store.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { syscall } from "./syscall.ts"; - -export type KV = { - key: string; - value: any; -}; - -export type Query = { - filter?: Filter[]; - orderBy?: string; - orderDesc?: boolean; - limit?: number; - select?: string[]; -}; - -export type Filter = { - op: string; - prop: string; - value: any; -}; - -export function set(key: string, value: any): Promise { - return syscall("store.set", key, value); -} - -export function batchSet(kvs: KV[]): Promise { - return syscall("store.batchSet", kvs); -} - -export function get(key: string): Promise { - return syscall("store.get", key); -} - -export function batchGet(keys: string[]): Promise<(any | undefined)[]> { - return syscall("store.batchGet", keys); -} - -export function has(key: string): Promise { - return syscall("store.has", key); -} - -export function del(key: string): Promise { - return syscall("store.delete", key); -} - -export function batchDel(keys: string[]): Promise { - return syscall("store.batchDelete", keys); -} - -export function queryPrefix( - prefix: string, -): Promise<{ key: string; value: any }[]> { - return syscall("store.queryPrefix", prefix); -} - -export function deletePrefix(prefix: string): Promise { - return syscall("store.deletePrefix", prefix); -} - -export function deleteAll(): Promise { - return syscall("store.deleteAll"); -} diff --git a/plug-api/silverbullet-syscall/clientStore.ts b/plug-api/silverbullet-syscall/clientStore.ts index eb26773..b48e4c2 100644 --- a/plug-api/silverbullet-syscall/clientStore.ts +++ b/plug-api/silverbullet-syscall/clientStore.ts @@ -1,5 +1,10 @@ import { syscall } from "./syscall.ts"; +/** + * Implements a very simple (string) key value store for the client. + * Generally should only be used to set some client-specific states, such as preferences. + */ + export function set(key: string, value: any): Promise { return syscall("clientStore.set", key, value); } diff --git a/plug-api/silverbullet-syscall/editor.ts b/plug-api/silverbullet-syscall/editor.ts index f31c60a..4068c76 100644 --- a/plug-api/silverbullet-syscall/editor.ts +++ b/plug-api/silverbullet-syscall/editor.ts @@ -72,7 +72,7 @@ export function filterBox( } export function showPanel( - id: "lhs" | "rhs" | "bhs" | "modal", + id: "lhs" | "rhs" | "bhs" | "modal" | "ps", mode: number, html: string, script = "", @@ -80,7 +80,9 @@ export function showPanel( return syscall("editor.showPanel", id, mode, html, script); } -export function hidePanel(id: "lhs" | "rhs" | "bhs" | "modal"): Promise { +export function hidePanel( + id: "lhs" | "rhs" | "bhs" | "modal" | "ps", +): Promise { return syscall("editor.hidePanel", id); } diff --git a/plug-api/silverbullet-syscall/handlebars.ts b/plug-api/silverbullet-syscall/handlebars.ts new file mode 100644 index 0000000..9b5dbee --- /dev/null +++ b/plug-api/silverbullet-syscall/handlebars.ts @@ -0,0 +1,16 @@ +import { syscall } from "$sb/silverbullet-syscall/syscall.ts"; + +/** + * Renders + * @param template + * @param obj + * @param globals + * @returns + */ +export function renderTemplate( + template: string, + obj: any, + globals: Record = {}, +): Promise { + return syscall("handlebars.renderTemplate", template, obj, globals); +} diff --git a/plug-api/silverbullet-syscall/index.ts b/plug-api/silverbullet-syscall/index.ts deleted file mode 100644 index 596dbef..0000000 --- a/plug-api/silverbullet-syscall/index.ts +++ /dev/null @@ -1,54 +0,0 @@ -import type { Query } from "../plugos-syscall/store.ts"; -import { syscall } from "./syscall.ts"; - -export type KV = { - key: string; - value: any; -}; - -export function set( - page: string, - key: string, - value: any, -): Promise { - return syscall("index.set", page, key, value); -} - -export function batchSet(page: string, kvs: KV[]): Promise { - return syscall("index.batchSet", page, kvs); -} - -export function get(page: string, key: string): Promise { - return syscall("index.get", page, key); -} - -export function del(page: string, key: string): Promise { - return syscall("index.delete", page, key); -} - -export function queryPrefix( - prefix: string, -): Promise<{ key: string; page: string; value: any }[]> { - return syscall("index.queryPrefix", prefix); -} - -export function query( - query: Query, -): Promise<{ key: string; page: string; value: any }[]> { - return syscall("index.query", query); -} - -export function clearPageIndexForPage(page: string): Promise { - return syscall("index.clearPageIndexForPage", page); -} - -export function deletePrefixForPage( - page: string, - prefix: string, -): Promise { - return syscall("index.deletePrefixForPage", page, prefix); -} - -export function clearPageIndex(): Promise { - return syscall("index.clearPageIndex"); -} diff --git a/plug-api/silverbullet-syscall/language.ts b/plug-api/silverbullet-syscall/language.ts new file mode 100644 index 0000000..aec1a5e --- /dev/null +++ b/plug-api/silverbullet-syscall/language.ts @@ -0,0 +1,16 @@ +import { syscall } from "$sb/silverbullet-syscall/syscall.ts"; + +import type { ParseTree } from "$sb/lib/tree.ts"; + +/** + * Parses a piece of code using any of the supported SB languages, see `common/languages.ts` for a list + * @param language the language to parse + * @param code the code to parse + * @returns a ParseTree representation of the code + */ +export function parseLanguage( + language: string, + code: string, +): Promise { + return syscall("language.parseLanguage", language, code); +} diff --git a/plug-api/silverbullet-syscall/mod.ts b/plug-api/silverbullet-syscall/mod.ts index 3520387..5f74088 100644 --- a/plug-api/silverbullet-syscall/mod.ts +++ b/plug-api/silverbullet-syscall/mod.ts @@ -1,8 +1,9 @@ export * as editor from "./editor.ts"; -export * as index from "./index.ts"; export * as markdown from "./markdown.ts"; export * as space from "./space.ts"; export * as system from "./system.ts"; export * as clientStore from "./clientStore.ts"; export * as sync from "./sync.ts"; export * as debug from "./debug.ts"; +export * as language from "./language.ts"; +export * as handlebars from "./handlebars.ts"; diff --git a/plug-api/silverbullet-syscall/space.ts b/plug-api/silverbullet-syscall/space.ts index a5662ce..9a0675f 100644 --- a/plug-api/silverbullet-syscall/space.ts +++ b/plug-api/silverbullet-syscall/space.ts @@ -1,6 +1,5 @@ import { syscall } from "./syscall.ts"; -import type { AttachmentMeta, PageMeta } from "../../web/types.ts"; -import { FileMeta } from "$sb/types.ts"; +import { AttachmentMeta, FileMeta, PageMeta } from "$sb/types.ts"; export function listPages(unfiltered = false): Promise { return syscall("space.listPages", unfiltered); diff --git a/plug-api/silverbullet-syscall/store.ts b/plug-api/silverbullet-syscall/store.ts deleted file mode 100644 index 9d526b9..0000000 --- a/plug-api/silverbullet-syscall/store.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { syscall } from "./syscall.ts"; - -export function set(key: string, value: any): Promise { - return syscall("store.set", key, value); -} - -export function get(key: string): Promise { - return syscall("store.get", key); -} - -export function del(key: string): Promise { - return syscall("store.delete", key); -} diff --git a/plug-api/types.ts b/plug-api/types.ts index 310dd16..6f30c6e 100644 --- a/plug-api/types.ts +++ b/plug-api/types.ts @@ -1,3 +1,28 @@ +export type FileMeta = { + name: string; + lastModified: number; + contentType: string; + size: number; + perm: "ro" | "rw"; + noSync?: boolean; +}; + +export type PageMeta = { + name: string; + lastModified: number; + lastOpened?: number; + perm: "ro" | "rw"; +}; + +export type AttachmentMeta = { + name: string; + contentType: string; + lastModified: number; + size: number; + perm: "ro" | "rw"; +}; + +// Message Queue related types export type MQMessage = { id: string; queue: string; @@ -16,11 +41,76 @@ export type MQSubscribeOptions = { pollInterval?: number; }; -export type FileMeta = { +// Key-Value Store related types +export type KvKey = string[]; + +export type KV = { + key: KvKey; + value: T; +}; + +export type OrderBy = { + expr: QueryExpression; + desc: boolean; +}; + +export type Select = { name: string; - lastModified: number; - contentType: string; - size: number; - perm: "ro" | "rw"; - noSync?: boolean; -} & Record; + expr?: QueryExpression; +}; + +export type Query = { + querySource?: string; + filter?: QueryExpression; + orderBy?: OrderBy[]; + select?: Select[]; + limit?: QueryExpression; + render?: string; +}; + +export type KvQuery = Omit & { + prefix?: KvKey; +}; + +export type QueryExpression = + | ["and", QueryExpression, QueryExpression] + | ["or", QueryExpression, QueryExpression] + | ["=", QueryExpression, QueryExpression] + | ["!=", QueryExpression, QueryExpression] + | ["=~", QueryExpression, QueryExpression] + | ["!=~", QueryExpression, QueryExpression] + | ["<", QueryExpression, QueryExpression] + | ["<=", QueryExpression, QueryExpression] + | [">", QueryExpression, QueryExpression] + | [">=", QueryExpression, QueryExpression] + | ["in", QueryExpression, QueryExpression] + | ["attr", QueryExpression, string] + | ["attr", string] + | ["number", number] + | ["string", string] + | ["boolean", boolean] + | ["null"] + | ["array", QueryExpression[]] + | ["object", Record] + | ["regexp", string, string] // regex, modifier + | ["+", QueryExpression, QueryExpression] + | ["-", QueryExpression, QueryExpression] + | ["*", QueryExpression, QueryExpression] + | ["%", QueryExpression, QueryExpression] + | ["/", QueryExpression, QueryExpression] + | ["call", string, QueryExpression[]]; + +export type FunctionMap = Record any>; + +/** + * An ObjectValue that can be indexed by the `index` plug, needs to have a minimum of + * of two fields: + * - ref: a unique reference (id) for the object, ideally a page reference + * - tags: a list of tags that the object belongs to + */ +export type ObjectValue = { + ref: string; + tags: string[]; +} & T; + +export type ObjectQuery = Omit; diff --git a/plugos/compile.ts b/plugos/compile.ts index 0a9acf3..f240bf0 100644 --- a/plugos/compile.ts +++ b/plugos/compile.ts @@ -172,7 +172,7 @@ export async function compileManifests( } } console.log("Change detected, rebuilding..."); - buildAll(); + await buildAll(); } } } diff --git a/plugos/hooks/endpoint.ts b/plugos/hooks/endpoint.ts index 071c57e..381fb60 100644 --- a/plugos/hooks/endpoint.ts +++ b/plugos/hooks/endpoint.ts @@ -105,7 +105,7 @@ export class EndpointHook implements Hook { } } // console.log("Shouldn't get here"); - next(); + await next(); }); } diff --git a/plugos/hooks/event.ts b/plugos/hooks/event.ts index 85b87d2..3a4ff42 100644 --- a/plugos/hooks/event.ts +++ b/plugos/hooks/event.ts @@ -53,12 +53,18 @@ export class EventHook implements Hook { manifest!.functions, ) ) { - if (functionDef.events && functionDef.events.includes(eventName)) { - // Only dispatch functions that can run in this environment - if (await plug.canInvoke(name)) { - const result = await plug.invoke(name, args); - if (result !== undefined) { - responses.push(result); + if (functionDef.events) { + for (const event of functionDef.events) { + if ( + event === eventName || eventNameToRegex(event).test(eventName) + ) { + // Only dispatch functions that can run in this environment + if (await plug.canInvoke(name)) { + const result = await plug.invoke(name, args); + if (result !== undefined) { + responses.push(result); + } + } } } } @@ -100,3 +106,9 @@ export class EventHook implements Hook { return errors; } } + +function eventNameToRegex(eventName: string): RegExp { + return new RegExp( + `^${eventName.replace(/\*/g, ".*").replace(/\//g, "\\/")}$`, + ); +} diff --git a/plugos/lib/datastore.test.ts b/plugos/lib/datastore.test.ts index 26e46ff..6089aa8 100644 --- a/plugos/lib/datastore.test.ts +++ b/plugos/lib/datastore.test.ts @@ -6,15 +6,17 @@ import { KvPrimitives } from "./kv_primitives.ts"; import { assertEquals } from "https://deno.land/std@0.165.0/testing/asserts.ts"; async function test(db: KvPrimitives) { - const dataStore = new DataStore(db); - await dataStore.set(["user", "peter"], { name: "Peter" }); - await dataStore.set(["user", "hank"], { name: "Hank" }); - let results = await dataStore.query({ + const datastore = new DataStore(db, ["ds"], { + count: (arr: any[]) => arr.length, + }); + await datastore.set(["user", "peter"], { name: "Peter" }); + await datastore.set(["user", "hank"], { name: "Hank" }); + let results = await datastore.query({ prefix: ["user"], - filter: ["=", "name", "Peter"], + filter: ["=", ["attr", "name"], ["string", "Peter"]], }); assertEquals(results, [{ key: ["user", "peter"], value: { name: "Peter" } }]); - await dataStore.batchSet([ + await datastore.batchSet([ { key: ["kv", "name"], value: "Zef" }, { key: ["kv", "data"], value: new Uint8Array([1, 2, 3]) }, { @@ -29,32 +31,42 @@ async function test(db: KvPrimitives) { }, }, ]); - assertEquals(await dataStore.get(["kv", "name"]), "Zef"); - assertEquals(await dataStore.get(["kv", "data"]), new Uint8Array([1, 2, 3])); - results = await dataStore.query({ + assertEquals(await datastore.get(["kv", "name"]), "Zef"); + assertEquals(await datastore.get(["kv", "data"]), new Uint8Array([1, 2, 3])); + results = await datastore.query({ prefix: ["kv"], - filter: ["=", "", "Zef"], + filter: ["=~", ["attr", ""], ["regexp", "Z.f", "i"]], }); assertEquals(results, [{ key: ["kv", "name"], value: "Zef" }]); - results = await dataStore.query({ + results = await datastore.query({ prefix: ["kv"], - filter: ["and", ["=", "parents", "John"], [ + filter: ["and", ["=", ["attr", "parents"], ["string", "John"]], [ "=", - "address.city", - "San Francisco", + ["attr", ["attr", "address"], "city"], + ["string", "San Francisco"], ]], - select: ["name"], + select: [ + { name: "parents" }, + { + name: "name", + expr: ["+", ["attr", "name"], ["string", "!"]], + }, + { + name: "parentCount", + expr: ["call", "count", [["attr", "parents"]]], + }, + ], }); + assertEquals(results.length, 1); assertEquals(results[0], { key: ["kv", "complicated"], - value: { name: "Frank" }, + value: { name: "Frank!", parentCount: 2, parents: ["John", "Jane"] }, }); } Deno.test("Test Deno KV DataStore", async () => { const tmpFile = await Deno.makeTempFile(); - const db = new DenoKvPrimitives(tmpFile); - await db.init(); + const db = new DenoKvPrimitives(await Deno.openKv(tmpFile)); await test(db); db.close(); await Deno.remove(tmpFile); diff --git a/plugos/lib/datastore.ts b/plugos/lib/datastore.ts index 1dc3a7a..5e53e86 100644 --- a/plugos/lib/datastore.ts +++ b/plugos/lib/datastore.ts @@ -1,183 +1,116 @@ -import { KvKey, KvPrimitives } from "./kv_primitives.ts"; - -export type { KvKey }; - -export type KvValue = any; - -export type KV = { - key: KvKey; - value: KvValue; -}; - -export type KvOrderBy = { - attribute: string; - desc: boolean; -}; - -export type KvQuery = { - prefix: KvKey; - filter?: KvQueryFilter; - orderBy?: KvOrderBy[]; - limit?: number; - select?: string[]; -}; - -export type KvQueryFilter = - | ["=", string, any] - | ["!=", string, any] - | ["=~", string, RegExp] - | ["!=~", string, RegExp] - | ["prefix", string, string] - | ["<", string, any] - | ["<=", string, any] - | [">", string, any] - | [">=", string, any] - | ["in", string, any[]] - | ["and", KvQueryFilter, KvQueryFilter] - | ["or", KvQueryFilter, KvQueryFilter]; - -function filterKvQuery(kvQuery: KvQueryFilter, obj: KvValue): boolean { - const [op, op1, op2] = kvQuery; - - if (op === "and") { - return filterKvQuery(op1, obj) && - filterKvQuery(op2, obj); - } else if (op === "or") { - return filterKvQuery(op1, obj) || filterKvQuery(op2, obj); - } - - // Look up the value of the attribute, supporting nested attributes via `attr.attr2.attr3`, and empty attribute value signifies the root object - let attributeVal = obj; - for (const part of op1.split(".")) { - if (!part) { - continue; - } - if (attributeVal === undefined) { - return false; - } - attributeVal = attributeVal[part]; - } - - // And apply the operator - switch (op) { - case "=": { - if (Array.isArray(attributeVal) && !Array.isArray(op2)) { - // Record property is an array, and value is a scalar: find the value in the array - if (attributeVal.includes(op2)) { - return true; - } - } else if (Array.isArray(attributeVal) && Array.isArray(obj)) { - // Record property is an array, and value is an array: find the value in the array - if (attributeVal.some((v) => obj.includes(v))) { - return true; - } - } - - return attributeVal === op2; - } - case "!=": - return attributeVal !== op2; - case "=~": - return op2.test(attributeVal); - case "!=~": - return !op2.test(attributeVal); - case "prefix": - return attributeVal.startsWith(op2); - case "<": - return attributeVal < op2; - case "<=": - return attributeVal <= op2; - case ">": - return attributeVal > op2; - case ">=": - return attributeVal >= op2; - case "in": - return op2.includes(attributeVal); - default: - throw new Error(`Unupported operator: ${op}`); - } -} +import { applyQueryNoFilterKV, evalQueryExpression } from "$sb/lib/query.ts"; +import { FunctionMap, KV, KvKey, KvQuery } from "$sb/types.ts"; +import { builtinFunctions } from "$sb/lib/builtin_query_functions.ts"; +import { KvPrimitives } from "./kv_primitives.ts"; /** * This is the data store class you'll actually want to use, wrapping the primitives * in a more user-friendly way */ export class DataStore { - constructor(private kv: KvPrimitives) { + constructor( + private kv: KvPrimitives, + private prefix: KvKey = [], + private functionMap: FunctionMap = builtinFunctions, + ) { } - async get(key: KvKey): Promise { - return (await this.kv.batchGet([key]))[0]; + prefixed(prefix: KvKey): DataStore { + return new DataStore( + this.kv, + [...this.prefix, ...prefix], + this.functionMap, + ); } - batchGet(keys: KvKey[]): Promise { - return this.kv.batchGet(keys); + async get(key: KvKey): Promise { + return (await this.batchGet([key]))[0]; } - set(key: KvKey, value: KvValue): Promise { - return this.kv.batchSet([{ key, value }]); + batchGet(keys: KvKey[]): Promise<(T | null)[]> { + return this.kv.batchGet(keys.map((key) => this.applyPrefix(key))); } - batchSet(entries: KV[]): Promise { - return this.kv.batchSet(entries); + set(key: KvKey, value: any): Promise { + return this.batchSet([{ key, value }]); + } + + batchSet(entries: KV[]): Promise { + const allKeyStrings = new Set(); + const uniqueEntries: KV[] = []; + for (const { key, value } of entries) { + const keyString = JSON.stringify(key); + if (allKeyStrings.has(keyString)) { + console.warn(`Duplicate key ${keyString} in batchSet, skipping`); + } else { + allKeyStrings.add(keyString); + uniqueEntries.push({ key: this.applyPrefix(key), value }); + } + } + return this.kv.batchSet(uniqueEntries); } delete(key: KvKey): Promise { - return this.kv.batchDelete([key]); + return this.batchDelete([key]); } batchDelete(keys: KvKey[]): Promise { - return this.kv.batchDelete(keys); + return this.kv.batchDelete(keys.map((key) => this.applyPrefix(key))); } - async query(query: KvQuery): Promise { - const results: KV[] = []; + async query(query: KvQuery): Promise[]> { + const results: KV[] = []; let itemCount = 0; - // Accumuliate results - for await (const entry of this.kv.query({ prefix: query.prefix })) { + // Accumulate results + let limit = Infinity; + const prefixedQuery: KvQuery = { + ...query, + prefix: query.prefix ? this.applyPrefix(query.prefix) : undefined, + }; + if (query.limit) { + limit = evalQueryExpression(query.limit, {}, this.functionMap); + } + for await ( + const entry of this.kv.query(prefixedQuery) + ) { // Filter - if (query.filter && !filterKvQuery(query.filter, entry.value)) { + if ( + query.filter && + !evalQueryExpression(query.filter, entry.value, this.functionMap) + ) { continue; } results.push(entry); itemCount++; // Stop when the limit has been reached - if (itemCount === query.limit) { + if (itemCount === limit) { break; } } - // Order by - if (query.orderBy) { - results.sort((a, b) => { - const aVal = a.value; - const bVal = b.value; - for (const { attribute, desc } of query.orderBy!) { - if ( - aVal[attribute] < bVal[attribute] || aVal[attribute] === undefined - ) { - return desc ? 1 : -1; - } - if ( - aVal[attribute] > bVal[attribute] || bVal[attribute] === undefined - ) { - return desc ? -1 : 1; - } - } - // Consider them equal. This helps with comparing arrays (like tags) - return 0; - }); - } + // Apply order by, limit, and select + return applyQueryNoFilterKV(prefixedQuery, results, this.functionMap).map(( + { key, value }, + ) => ({ key: this.stripPrefix(key), value })); + } - if (query.select) { - for (let i = 0; i < results.length; i++) { - const rec = results[i].value; - const newRec: any = {}; - for (const k of query.select) { - newRec[k] = rec[k]; - } - results[i].value = newRec; - } + async queryDelete(query: KvQuery): Promise { + const keys: KvKey[] = []; + for ( + const { key } of await this.query({ + ...query, + prefix: query.prefix ? this.applyPrefix(query.prefix) : undefined, + }) + ) { + keys.push(key); } - return results; + return this.batchDelete(keys); + } + + private applyPrefix(key: KvKey): KvKey { + return [...this.prefix, ...(key ? key : [])]; + } + + private stripPrefix(key: KvKey): KvKey { + return key.slice(this.prefix.length); } } diff --git a/plugos/lib/deno_kv_primitives.test.ts b/plugos/lib/deno_kv_primitives.test.ts index 3f44753..ca0727d 100644 --- a/plugos/lib/deno_kv_primitives.test.ts +++ b/plugos/lib/deno_kv_primitives.test.ts @@ -3,8 +3,7 @@ import { allTests } from "./kv_primitives.test.ts"; Deno.test("Test Deno KV Primitives", async () => { const tmpFile = await Deno.makeTempFile(); - const db = new DenoKvPrimitives(tmpFile); - await db.init(); + const db = new DenoKvPrimitives(await Deno.openKv(tmpFile)); await allTests(db); db.close(); await Deno.remove(tmpFile); diff --git a/plugos/lib/deno_kv_primitives.ts b/plugos/lib/deno_kv_primitives.ts index 818f848..94cb14c 100644 --- a/plugos/lib/deno_kv_primitives.ts +++ b/plugos/lib/deno_kv_primitives.ts @@ -1,15 +1,12 @@ /// -import { KV, KvKey, KvPrimitives, KvQueryOptions } from "./kv_primitives.ts"; -const kvBatchSize = 10; +import { KV, KvKey } from "$sb/types.ts"; +import { KvPrimitives, KvQueryOptions } from "./kv_primitives.ts"; + +const kvBatchSize = 100; export class DenoKvPrimitives implements KvPrimitives { - db!: Deno.Kv; - constructor(private path?: string) { - } - - async init() { - this.db = await Deno.openKv(this.path); + constructor(private db: Deno.Kv) { } async batchGet(keys: KvKey[]): Promise { diff --git a/plugos/lib/indexeddb_kv_primitives.ts b/plugos/lib/indexeddb_kv_primitives.ts index 32c6f34..aaa310e 100644 --- a/plugos/lib/indexeddb_kv_primitives.ts +++ b/plugos/lib/indexeddb_kv_primitives.ts @@ -1,32 +1,33 @@ -import { KV, KvKey, KvPrimitives, KvQueryOptions } from "./kv_primitives.ts"; +import { KV, KvKey } from "$sb/types.ts"; +import { KvPrimitives, KvQueryOptions } from "./kv_primitives.ts"; import { IDBPDatabase, openDB } from "https://esm.sh/idb@7.1.1/with-async-ittr"; const sep = "\0"; +const objectStoreName = "data"; export class IndexedDBKvPrimitives implements KvPrimitives { db!: IDBPDatabase; constructor( private dbName: string, - private objectStoreName: string = "data", ) { } async init() { this.db = await openDB(this.dbName, 1, { upgrade: (db) => { - db.createObjectStore(this.objectStoreName); + db.createObjectStore(objectStoreName); }, }); } batchGet(keys: KvKey[]): Promise { - const tx = this.db.transaction(this.objectStoreName, "readonly"); + const tx = this.db.transaction(objectStoreName, "readonly"); return Promise.all(keys.map((key) => tx.store.get(this.buildKey(key)))); } async batchSet(entries: KV[]): Promise { - const tx = this.db.transaction(this.objectStoreName, "readwrite"); + const tx = this.db.transaction(objectStoreName, "readwrite"); await Promise.all([ ...entries.map(({ key, value }) => tx.store.put(value, this.buildKey(key)) @@ -36,7 +37,7 @@ export class IndexedDBKvPrimitives implements KvPrimitives { } async batchDelete(keys: KvKey[]): Promise { - const tx = this.db.transaction(this.objectStoreName, "readwrite"); + const tx = this.db.transaction(objectStoreName, "readwrite"); await Promise.all([ ...keys.map((key) => tx.store.delete(this.buildKey(key))), tx.done, @@ -44,12 +45,12 @@ export class IndexedDBKvPrimitives implements KvPrimitives { } async *query({ prefix }: KvQueryOptions): AsyncIterableIterator { - const tx = this.db.transaction(this.objectStoreName, "readonly"); + const tx = this.db.transaction(objectStoreName, "readonly"); prefix = prefix || []; for await ( const entry of tx.store.iterate(IDBKeyRange.bound( this.buildKey([...prefix, ""]), - this.buildKey([...prefix, "\ufffe"]), + this.buildKey([...prefix, "\uffff"]), )) ) { yield { key: this.extractKey(entry.key), value: entry.value }; diff --git a/plugos/lib/kv_primitives.test.ts b/plugos/lib/kv_primitives.test.ts index 3a565ec..8343a0c 100644 --- a/plugos/lib/kv_primitives.test.ts +++ b/plugos/lib/kv_primitives.test.ts @@ -1,5 +1,6 @@ -import { KV, KvPrimitives } from "./kv_primitives.ts"; +import { KvPrimitives } from "./kv_primitives.ts"; import { assertEquals } from "../../test_deps.ts"; +import { KV } from "$sb/types.ts"; export async function allTests(db: KvPrimitives) { await db.batchSet([ diff --git a/plugos/lib/kv_primitives.ts b/plugos/lib/kv_primitives.ts index ce9709b..983e503 100644 --- a/plugos/lib/kv_primitives.ts +++ b/plugos/lib/kv_primitives.ts @@ -1,17 +1,11 @@ -export type KvKey = string[]; -export type KvValue = any; - -export type KV = { - key: KvKey; - value: KvValue; -}; +import { KV, KvKey } from "$sb/types.ts"; export type KvQueryOptions = { prefix?: KvKey; }; export interface KvPrimitives { - batchGet(keys: KvKey[]): Promise<(KvValue | undefined)[]>; + batchGet(keys: KvKey[]): Promise<(any | undefined)[]>; batchSet(entries: KV[]): Promise; batchDelete(keys: KvKey[]): Promise; query(options: KvQueryOptions): AsyncIterableIterator; diff --git a/plugos/lib/kv_store.deno_kv.test.ts b/plugos/lib/kv_store.deno_kv.test.ts deleted file mode 100644 index a286996..0000000 --- a/plugos/lib/kv_store.deno_kv.test.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { assertEquals } from "../../test_deps.ts"; -import { DenoKVStore } from "./kv_store.deno_kv.ts"; - -Deno.test("Test KV index", async () => { - const tmpFile = await Deno.makeTempFile(); - const denoKv = await Deno.openKv(tmpFile); - const kv = new DenoKVStore(denoKv); - - await kv.set("name", "Peter"); - assertEquals(await kv.get("name"), "Peter"); - await kv.del("name"); - assertEquals(await kv.has("name"), false); - - await kv.batchSet([ - { key: "page:hello", value: "Hello" }, - { key: "page:hello2", value: "Hello 2" }, - { key: "page:hello3", value: "Hello 3" }, - { key: "something", value: "Something" }, - { key: "something1", value: "Something" }, - { key: "something2", value: "Something" }, - { key: "something3", value: "Something" }, - { key: "something4", value: "Something" }, - { key: "something5", value: "Something" }, - { key: "something6", value: "Something" }, - { key: "something7", value: "Something" }, - { key: "something8", value: "Something" }, - { key: "something9", value: "Something" }, - { key: "something10", value: "Something" }, - { key: "something11", value: "Something" }, - { key: "something12", value: "Something" }, - { key: "something13", value: "Something" }, - { key: "something14", value: "Something" }, - { key: "something15", value: "Something" }, - { key: "something16", value: "Something" }, - { key: "something17", value: "Something" }, - { key: "something18", value: "Something" }, - { key: "something19", value: "Something" }, - ]); - - const results = await kv.queryPrefix("page:"); - assertEquals(results.length, 3); - - assertEquals(await kv.batchGet(["page:hello", "page:hello3"]), [ - "Hello", - "Hello 3", - ]); - - await kv.deletePrefix("page:"); - - assertEquals(await kv.queryPrefix("page:"), []); - assertEquals((await kv.queryPrefix("")).length, 20); - - await kv.deletePrefix(""); - assertEquals(await kv.queryPrefix(""), []); - - denoKv.close(); - await Deno.remove(tmpFile); -}); diff --git a/plugos/lib/kv_store.deno_kv.ts b/plugos/lib/kv_store.deno_kv.ts deleted file mode 100644 index 60933bb..0000000 --- a/plugos/lib/kv_store.deno_kv.ts +++ /dev/null @@ -1,112 +0,0 @@ -/// - -import { KV, KVStore } from "./kv_store.ts"; - -const kvBatchSize = 10; - -export class DenoKVStore implements KVStore { - constructor(private kv: Deno.Kv) { - } - - del(key: string): Promise { - return this.batchDelete([key]); - } - async deletePrefix(prefix: string): Promise { - const allKeys: string[] = []; - for await ( - const result of this.kv.list( - prefix - ? { - start: [prefix], - end: [endRange(prefix)], - } - : { prefix: [] }, - ) - ) { - allKeys.push(result.key[0] as string); - } - return this.batchDelete(allKeys); - } - deleteAll(): Promise { - return this.deletePrefix(""); - } - set(key: string, value: any): Promise { - return this.batchSet([{ key, value }]); - } - async batchSet(kvs: KV[]): Promise { - // Split into batches of kvBatchSize - const batches: KV[][] = []; - for (let i = 0; i < kvs.length; i += kvBatchSize) { - batches.push(kvs.slice(i, i + kvBatchSize)); - } - for (const batch of batches) { - let batchOp = this.kv.atomic(); - for (const { key, value } of batch) { - batchOp = batchOp.set([key], value); - } - const res = await batchOp.commit(); - if (!res.ok) { - throw res; - } - } - } - async batchDelete(keys: string[]): Promise { - const batches: string[][] = []; - for (let i = 0; i < keys.length; i += kvBatchSize) { - batches.push(keys.slice(i, i + kvBatchSize)); - } - for (const batch of batches) { - let batchOp = this.kv.atomic(); - for (const key of batch) { - batchOp = batchOp.delete([key]); - } - const res = await batchOp.commit(); - if (!res.ok) { - throw res; - } - } - } - async batchGet(keys: string[]): Promise { - const results: any[] = []; - const batches: Deno.KvKey[][] = []; - for (let i = 0; i < keys.length; i += kvBatchSize) { - batches.push(keys.slice(i, i + kvBatchSize).map((k) => [k])); - } - for (const batch of batches) { - const res = await this.kv.getMany(batch); - results.push(...res.map((r) => r.value)); - } - return results; - } - async get(key: string): Promise { - return (await this.kv.get([key])).value; - } - async has(key: string): Promise { - return (await this.kv.get([key])).value !== null; - } - async queryPrefix(keyPrefix: string): Promise<{ key: string; value: any }[]> { - const results: { key: string; value: any }[] = []; - for await ( - const result of this.kv.list( - keyPrefix - ? { - start: [keyPrefix], - end: [endRange(keyPrefix)], - } - : { prefix: [] }, - ) - ) { - results.push({ - key: result.key[0] as string, - value: result.value as any, - }); - } - return results; - } -} - -function endRange(prefix: string) { - const lastChar = prefix[prefix.length - 1]; - const nextLastChar = String.fromCharCode(lastChar.charCodeAt(0) + 1); - return prefix.slice(0, -1) + nextLastChar; -} diff --git a/plugos/lib/kv_store.dexie.ts b/plugos/lib/kv_store.dexie.ts deleted file mode 100644 index cf62a2d..0000000 --- a/plugos/lib/kv_store.dexie.ts +++ /dev/null @@ -1,82 +0,0 @@ -import Dexie, { Table } from "dexie"; -import type { KV, KVStore } from "./kv_store.ts"; - -export class DexieKVStore implements KVStore { - db: Dexie; - items: Table; - constructor( - dbName: string, - tableName: string, - indexedDB?: any, - IDBKeyRange?: any, - ) { - this.db = new Dexie(dbName, { - indexedDB, - IDBKeyRange, - }); - this.db.version(1).stores({ - [tableName]: "key", - }); - this.items = this.db.table(tableName); - } - - async del(key: string) { - await this.items.delete(key); - } - - async deletePrefix(prefix: string) { - await this.items.where("key").startsWith(prefix).delete(); - } - - async deleteAll() { - await this.items.clear(); - } - - async set(key: string, value: any) { - await this.items.put({ - key, - value, - }); - } - - async batchSet(kvs: KV[]) { - await this.items.bulkPut( - kvs.map(({ key, value }) => ({ - key, - value, - })), - ); - } - - async batchDelete(keys: string[]) { - await this.items.bulkDelete(keys); - } - - async batchGet( - keys: string[], - ): Promise<(any | undefined)[]> { - return (await this.items.bulkGet(keys)).map((result) => result?.value); - } - - async get(key: string): Promise { - const result = await this.items.get({ key }); - return result ? result.value : null; - } - - async has(key: string): Promise { - return await this.items.get({ - key, - }) !== undefined; - } - - async queryPrefix( - keyPrefix: string, - ): Promise<{ key: string; value: any }[]> { - const results = await this.items.where("key").startsWith(keyPrefix) - .toArray(); - return results.map((result) => ({ - key: result.key, - value: result.value, - })); - } -} diff --git a/plugos/lib/kv_store.json_file.ts b/plugos/lib/kv_store.json_file.ts index 782d703..ce4f54f 100644 --- a/plugos/lib/kv_store.json_file.ts +++ b/plugos/lib/kv_store.json_file.ts @@ -1,6 +1,6 @@ -import { KV, KVStore } from "./kv_store.ts"; +import { KV } from "$sb/types.ts"; -export class JSONKVStore implements KVStore { +export class JSONKVStore { private data: { [key: string]: any } = {}; async load(path: string) { @@ -38,21 +38,6 @@ export class JSONKVStore implements KVStore { this.data[key] = value; return Promise.resolve(); } - batchSet(kvs: KV[]): Promise { - for (const kv of kvs) { - this.data[kv.key] = kv.value; - } - return Promise.resolve(); - } - batchDelete(keys: string[]): Promise { - for (const key of keys) { - delete this.data[key]; - } - return Promise.resolve(); - } - batchGet(keys: string[]): Promise { - return Promise.resolve(keys.map((key) => this.data[key])); - } get(key: string): Promise { return Promise.resolve(this.data[key]); } diff --git a/plugos/lib/kv_store.ts b/plugos/lib/kv_store.ts deleted file mode 100644 index 44ecd2e..0000000 --- a/plugos/lib/kv_store.ts +++ /dev/null @@ -1,59 +0,0 @@ -export type KV = { - key: string; - value: any; -}; - -/** - * An interface to any simple key-value store. - */ -export interface KVStore { - /** - * Deletes the value associated with a given key. - */ - del(key: string): Promise; - - /** - * Deletes all keys that start with a specific prefix. - */ - deletePrefix(prefix: string): Promise; - - /** - * Deletes all keys in the store. - */ - deleteAll(): Promise; - - /** - * Sets the value for a given key. - */ - set(key: string, value: any): Promise; - - /** - * Sets the values for a list of key-value pairs. - */ - batchSet(kvs: KV[]): Promise; - - /** - * Deletes a list of keys. - */ - batchDelete(keys: string[]): Promise; - - /** - * Gets the values for a list of keys. - */ - batchGet(keys: string[]): Promise<(any | undefined)[]>; - - /** - * Gets the value for a given key. - */ - get(key: string): Promise; - - /** - * Checks whether a given key exists in the store. - */ - has(key: string): Promise; - - /** - * Gets all key-value pairs where the key starts with a specific prefix. - */ - queryPrefix(keyPrefix: string): Promise<{ key: string; value: any }[]>; -} diff --git a/plugos/lib/mq.dexie.test.ts b/plugos/lib/mq.datastore.test.ts similarity index 74% rename from plugos/lib/mq.dexie.test.ts rename to plugos/lib/mq.datastore.test.ts index 03900e4..2efff4f 100644 --- a/plugos/lib/mq.dexie.test.ts +++ b/plugos/lib/mq.datastore.test.ts @@ -1,10 +1,14 @@ -import { IDBKeyRange, indexedDB } from "https://esm.sh/fake-indexeddb@4.0.2"; -import { DexieMQ } from "./mq.dexie.ts"; +import { DataStoreMQ } from "./mq.datastore.ts"; import { assertEquals } from "../../test_deps.ts"; import { sleep } from "$sb/lib/async.ts"; +import { DenoKvPrimitives } from "./deno_kv_primitives.ts"; +import { DataStore } from "./datastore.ts"; -Deno.test("Dexie MQ", async () => { - const mq = new DexieMQ("test", indexedDB, IDBKeyRange); +Deno.test("DataStore MQ", async () => { + const tmpFile = await Deno.makeTempFile(); + const db = new DenoKvPrimitives(await Deno.openKv(tmpFile)); + + const mq = new DataStoreMQ(new DataStore(db, ["mq"])); await mq.send("test", "Hello World"); let messages = await mq.poll("test", 10); assertEquals(messages.length, 1); @@ -28,12 +32,15 @@ Deno.test("Dexie MQ", async () => { let receivedMessage = false; const unsubscribe = mq.subscribe("test123", {}, async (messages) => { assertEquals(messages.length, 1); - await mq.ack("test123", messages[0].id); receivedMessage = true; + console.log("RECEIVED TEH EMSSSAGE"); + await mq.ack("test123", messages[0].id); }); - mq.send("test123", "Hello World"); + await mq.send("test123", "Hello World"); + console.log("After send"); // Give time to process the message - await sleep(1); + await sleep(10); + console.log("After sleep"); assertEquals(receivedMessage, true); unsubscribe(); @@ -50,4 +57,7 @@ Deno.test("Dexie MQ", async () => { assertEquals(await mq.fetchProcessingMessages(), []); // Give time to close the db await sleep(20); + + db.close(); + await Deno.remove(tmpFile); }); diff --git a/plugos/lib/mq.datastore.ts b/plugos/lib/mq.datastore.ts new file mode 100644 index 0000000..225bb7f --- /dev/null +++ b/plugos/lib/mq.datastore.ts @@ -0,0 +1,276 @@ +import { KV, MQMessage, MQStats, MQSubscribeOptions } from "$sb/types.ts"; +import { MessageQueue } from "./mq.ts"; +import { DataStore } from "./datastore.ts"; + +export type ProcessingMessage = MQMessage & { + ts: number; +}; + +const queuedPrefix = ["mq", "queued"]; +const processingPrefix = ["mq", "processing"]; +const dlqPrefix = ["mq", "dlq"]; + +export class DataStoreMQ implements MessageQueue { + // queue -> set of run() functions + localSubscriptions = new Map void>>(); + + constructor( + private ds: DataStore, + ) { + } + + // Internal sequencer for messages, only really necessary when batch sending tons of messages within a millisecond + seq = 0; + + async batchSend(queue: string, bodies: any[]): Promise { + const messages: KV[] = bodies.map((body) => { + const id = `${Date.now()}-${String(++this.seq).padStart(6, "0")}`; + const key = [...queuedPrefix, queue, id]; + return { + key, + value: { id, queue, body }, + }; + }); + + await this.ds.batchSet(messages); + + // See if we can immediately process the message with a local subscription + const localSubscriptions = this.localSubscriptions.get(queue); + if (localSubscriptions) { + for (const run of localSubscriptions) { + run(); + } + } + } + + send(queue: string, body: any): Promise { + return this.batchSend(queue, [body]); + } + + async poll(queue: string, maxItems: number): Promise { + // Note: this is not happening in a transactional way, so we may get duplicate message delivery + // Retrieve a batch of messages + const messages = await this.ds.query({ + prefix: [...queuedPrefix, queue], + limit: ["number", maxItems], + }); + // Put them in the processing queue + await this.ds.batchSet( + messages.map((m) => ({ + key: [...processingPrefix, queue, m.value.id], + value: { + ...m.value, + ts: Date.now(), + }, + })), + ); + // Delete them from the queued queue + await this.ds.batchDelete(messages.map((m) => m.key)); + + // Return them + return messages.map((m) => m.value); + } + + /** + * @param queue + * @param batchSize + * @param callback + * @returns a function to be called to unsubscribe + */ + subscribe( + queue: string, + options: MQSubscribeOptions, + callback: (messages: MQMessage[]) => Promise | void, + ): () => void { + let running = true; + let timeout: number | undefined; + const batchSize = options.batchSize || 1; + const run = async () => { + try { + if (!running) { + return; + } + const messages = await this.poll(queue, batchSize); + if (messages.length > 0) { + await callback(messages); + } + // If we got exactly the batch size, there might be more messages + if (messages.length === batchSize) { + await run(); + } + if (timeout) { + clearTimeout(timeout); + } + timeout = setTimeout(run, options.pollInterval || 5000); + } catch (e: any) { + console.error("Error in MQ subscription handler", e); + } + }; + + // Register as a local subscription handler + const localSubscriptions = this.localSubscriptions.get(queue); + if (!localSubscriptions) { + this.localSubscriptions.set(queue, new Set([run])); + } else { + localSubscriptions.add(run); + } + + // Run the first time (which will schedule subsequent polling intervals) + run(); + + // And return an unsubscribe function + return () => { + running = false; + if (timeout) { + clearTimeout(timeout); + } + // Remove the subscription from localSubscriptions + const queueSubscriptions = this.localSubscriptions.get(queue); + if (queueSubscriptions) { + queueSubscriptions.delete(run); + } + }; + } + + ack(queue: string, id: string) { + return this.batchAck(queue, [id]); + } + + async batchAck(queue: string, ids: string[]) { + await this.ds.batchDelete( + ids.map((id) => [...processingPrefix, queue, id]), + ); + } + + async requeueTimeouts( + timeout: number, + maxRetries?: number, + disableDLQ?: boolean, + ) { + const now = Date.now(); + const messages = await this.ds.query({ + prefix: processingPrefix, + filter: ["<", ["attr", "ts"], ["number", now - timeout]], + }); + await this.ds.batchDelete(messages.map((m) => m.key)); + const newMessages: KV[] = []; + for (const { value: m } of messages) { + const retries = (m.retries || 0) + 1; + if (maxRetries && retries > maxRetries) { + if (disableDLQ) { + console.warn( + "[mq]", + "Message exceeded max retries, flushing message", + m, + ); + } else { + console.warn( + "[mq]", + "Message exceeded max retries, moving to DLQ", + m, + ); + newMessages.push({ + key: [...dlqPrefix, m.queue, m.id], + value: { + queue: m.queue, + id: m.id, + body: m.body, + ts: Date.now(), + retries, + }, + }); + } + } else { + console.info("[mq]", "Message ack timed out, requeueing", m); + newMessages.push({ + key: [...queuedPrefix, m.queue, m.id], + value: { + ...m, + retries, + }, + }); + } + } + await this.ds.batchSet(newMessages); + } + + async fetchDLQMessages(): Promise { + return (await this.ds.query({ prefix: dlqPrefix })).map(( + { value }, + ) => value); + } + + async fetchProcessingMessages(): Promise { + return (await this.ds.query({ + prefix: processingPrefix, + })).map(( + { value }, + ) => value); + } + + flushDLQ(): Promise { + return this.ds.queryDelete({ prefix: dlqPrefix }); + } + + async getQueueStats(queue: string): Promise { + const queued = + (await (this.ds.query({ prefix: [...queuedPrefix, queue] }))).length; + const processing = + (await (this.ds.query({ prefix: [...processingPrefix, queue] }))).length; + const dlq = + (await (this.ds.query({ prefix: [...dlqPrefix, queue] }))).length; + return { + queued, + processing, + dlq, + }; + } + + async getAllQueueStats(): Promise> { + const allStatus: Record = {}; + for ( + const { value: message } of await this.ds.query({ + prefix: queuedPrefix, + }) + ) { + if (!allStatus[message.queue]) { + allStatus[message.queue] = { + queued: 0, + processing: 0, + dlq: 0, + }; + } + allStatus[message.queue].queued++; + } + for ( + const { value: message } of await this.ds.query({ + prefix: processingPrefix, + }) + ) { + if (!allStatus[message.queue]) { + allStatus[message.queue] = { + queued: 0, + processing: 0, + dlq: 0, + }; + } + allStatus[message.queue].processing++; + } + for ( + const { value: message } of await this.ds.query({ + prefix: dlqPrefix, + }) + ) { + if (!allStatus[message.queue]) { + allStatus[message.queue] = { + queued: 0, + processing: 0, + dlq: 0, + }; + } + allStatus[message.queue].dlq++; + } + + return allStatus; + } +} diff --git a/plugos/lib/mq.deno_kv.test.ts b/plugos/lib/mq.deno_kv.test.ts deleted file mode 100644 index 61d2af5..0000000 --- a/plugos/lib/mq.deno_kv.test.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { sleep } from "$sb/lib/async.ts"; -import { DenoKvMQ } from "./mq.deno_kv.ts"; - -Deno.test("Deno MQ", async () => { - const denoKv = await Deno.openKv("test.db"); - const mq = new DenoKvMQ(denoKv); - const unsub = mq.subscribe("test", {}, (messages) => { - console.log("Received on test", messages); - }); - const unsub2 = mq.subscribe("test2", {}, (messages) => { - console.log("Received on test2", messages); - }); - await mq.send("test", "Hello World"); - await mq.batchSend("test2", ["Hello World 2", "Hello World 3"]); - - // Let's avoid a panic here - await sleep(20); - denoKv.close(); - await Deno.remove("test.db"); -}); diff --git a/plugos/lib/mq.deno_kv.ts b/plugos/lib/mq.deno_kv.ts deleted file mode 100644 index 4501370..0000000 --- a/plugos/lib/mq.deno_kv.ts +++ /dev/null @@ -1,93 +0,0 @@ -/// - -import { - MQMessage, - MQStats, - MQSubscribeOptions, -} from "../../plug-api/types.ts"; -import { MessageQueue } from "./mq.ts"; - -type QueuedMessage = [string, MQMessage]; - -export class DenoKvMQ implements MessageQueue { - listeners: Map void | Promise>> = - new Map(); - - constructor(private kv: Deno.Kv) { - kv.listenQueue(async (message: unknown) => { - const [queue, body] = message as QueuedMessage; - const listeners = this.listeners.get(queue); - if (!listeners) { - return; - } - for (const listener of listeners) { - await Promise.resolve(listener([{ id: "_dummyid", queue, body }])); - } - }); - } - - // Dummy implementation - getQueueStats(_queue: string): Promise { - return Promise.resolve({ - queued: 0, - processing: 0, - dlq: 0, - }); - } - - // Dummy implementation - getAllQueueStats(): Promise> { - return Promise.resolve({}); - } - - async batchSend(queue: string, bodies: any[]): Promise { - for (const body of bodies) { - const result = await this.kv.enqueue([queue, body]); - if (!result.ok) { - throw result; - } - } - // const results = await Promise.all( - // bodies.map((body) => this.kv.enqueue([queue, body])), - // ); - // for (const result of results) { - // if (!result.ok) { - // throw result; - // } - // } - } - async send(queue: string, body: any): Promise { - const result = await this.kv.enqueue([queue, body]); - if (!result.ok) { - throw result; - } - } - subscribe( - queue: string, - _options: MQSubscribeOptions, - callback: (messages: MQMessage[]) => void | Promise, - ): () => void { - const listeners = this.listeners.get(queue); - if (!listeners) { - this.listeners.set(queue, new Set([callback])); - } else { - listeners.add(callback); - } - - return () => { - const listeners = this.listeners.get(queue); - if (!listeners) { - return; - } - listeners.delete(callback); - }; - } - ack(_queue: string, _id: string): Promise { - // Doesn't apply to this implementation - return Promise.resolve(); - } - batchAck(_queue: string, _ids: string[]): Promise { - // Doesn't apply to this implementation - return Promise.resolve(); - } -} diff --git a/plugos/lib/mq.dexie.ts b/plugos/lib/mq.dexie.ts deleted file mode 100644 index e224a8a..0000000 --- a/plugos/lib/mq.dexie.ts +++ /dev/null @@ -1,279 +0,0 @@ -import Dexie, { Table } from "dexie"; -import { MQMessage, MQStats, MQSubscribeOptions } from "$sb/types.ts"; -import { MessageQueue } from "./mq.ts"; - -export type ProcessingMessage = MQMessage & { - ts: number; -}; - -export class DexieMQ implements MessageQueue { - db: Dexie; - queued: Table; - processing: Table; - dlq: Table; - - // queue -> set of run() functions - localSubscriptions = new Map void>>(); - - constructor( - dbName: string, - indexedDB?: any, - IDBKeyRange?: any, - ) { - this.db = new Dexie(dbName, { - indexedDB, - IDBKeyRange, - }); - this.db.version(1).stores({ - queued: "[queue+id], queue, id", - processing: "[queue+id], queue, id, ts", - dlq: "[queue+id], queue, id", - }); - this.queued = this.db.table("queued"); - this.processing = this.db.table("processing"); - this.dlq = this.db.table("dlq"); - } - - // Internal sequencer for messages, only really necessary when batch sending tons of messages within a millisecond - seq = 0; - - async batchSend(queue: string, bodies: any[]) { - const messages = bodies.map((body) => ({ - id: `${Date.now()}-${String(++this.seq).padStart(6, "0")}`, - queue, - body, - })); - - await this.queued.bulkAdd(messages); - - // See if we can immediately process the message with a local subscription - const localSubscriptions = this.localSubscriptions.get(queue); - if (localSubscriptions) { - for (const run of localSubscriptions) { - run(); - } - } - } - - send(queue: string, body: any) { - return this.batchSend(queue, [body]); - } - - poll(queue: string, maxItems: number): Promise { - return this.db.transaction( - "rw", - [this.queued, this.processing], - async (tx) => { - const messages = - (await tx.table("queued").where({ - queue, - }) - .sortBy("id")).slice(0, maxItems); - const ids: [string, string][] = messages.map((m) => [queue, m.id]); - await tx.table("queued").bulkDelete(ids); - await tx.table("processing") - .bulkPut( - messages.map((m) => ({ - ...m, - ts: Date.now(), - })), - ); - return messages; - }, - ); - } - - /** - * @param queue - * @param batchSize - * @param callback - * @returns a function to be called to unsubscribe - */ - subscribe( - queue: string, - options: MQSubscribeOptions, - callback: (messages: MQMessage[]) => Promise | void, - ): () => void { - let running = true; - let timeout: number | undefined; - const batchSize = options.batchSize || 1; - const run = async () => { - try { - if (!running) { - return; - } - const messages = await this.poll(queue, batchSize); - if (messages.length > 0) { - await callback(messages); - } - // If we got exactly the batch size, there might be more messages - if (messages.length === batchSize) { - await run(); - } - if (timeout) { - clearTimeout(timeout); - } - timeout = setTimeout(run, options.pollInterval || 5000); - } catch (e: any) { - console.error("Error in MQ subscription handler", e); - } - }; - - // Register as a local subscription handler - const localSubscriptions = this.localSubscriptions.get(queue); - if (!localSubscriptions) { - this.localSubscriptions.set(queue, new Set([run])); - } else { - localSubscriptions.add(run); - } - - // Run the first time (which will schedule subsequent polling intervals) - run(); - - // And return an unsubscribe function - return () => { - running = false; - if (timeout) { - clearTimeout(timeout); - } - // Remove the subscription from localSubscriptions - const queueSubscriptions = this.localSubscriptions.get(queue); - if (queueSubscriptions) { - queueSubscriptions.delete(run); - } - }; - } - - ack(queue: string, id: string) { - return this.batchAck(queue, [id]); - } - - async batchAck(queue: string, ids: string[]) { - await this.processing.bulkDelete(ids.map((id) => [queue, id])); - } - - async requeueTimeouts( - timeout: number, - maxRetries?: number, - disableDLQ?: boolean, - ) { - const now = Date.now(); - const messages = await this.processing.where("ts").below(now - timeout) - .toArray(); - const ids: [string, string][] = messages.map((m) => [m.queue, m.id]); - await this.db.transaction( - "rw", - [this.queued, this.processing, this.dlq], - async (tx) => { - await tx.table("processing").bulkDelete(ids); - const requeuedMessages: ProcessingMessage[] = []; - const dlqMessages: ProcessingMessage[] = []; - for (const m of messages) { - const retries = (m.retries || 0) + 1; - if (maxRetries && retries > maxRetries) { - if (disableDLQ) { - console.warn( - "[mq]", - "Message exceeded max retries, flushing message", - m, - ); - } else { - console.warn( - "[mq]", - "Message exceeded max retries, moving to DLQ", - m, - ); - dlqMessages.push({ - queue: m.queue, - id: m.id, - body: m.body, - ts: Date.now(), - retries, - }); - } - } else { - console.info("[mq]", "Message ack timed out, requeueing", m); - requeuedMessages.push({ - ...m, - retries, - }); - } - } - await tx.table("queued").bulkPut(requeuedMessages); - await tx.table("dlq").bulkPut(dlqMessages); - }, - ); - } - - fetchDLQMessages(): Promise { - return this.dlq.toArray(); - } - - fetchProcessingMessages(): Promise { - return this.processing.toArray(); - } - - flushDLQ(): Promise { - return this.dlq.clear(); - } - - getQueueStats(queue: string): Promise { - return this.db.transaction( - "r", - [this.queued, this.processing, this.dlq], - async (tx) => { - const queued = await tx.table("queued").where({ queue }).count(); - const processing = await tx.table("processing").where({ queue }) - .count(); - const dlq = await tx.table("dlq").where({ queue }).count(); - return { - queued, - processing, - dlq, - }; - }, - ); - } - - async getAllQueueStats(): Promise> { - const allStatus: Record = {}; - await this.db.transaction( - "r", - [this.queued, this.processing, this.dlq], - async (tx) => { - for (const item of await tx.table("queued").toArray()) { - if (!allStatus[item.queue]) { - allStatus[item.queue] = { - queued: 0, - processing: 0, - dlq: 0, - }; - } - allStatus[item.queue].queued++; - } - for (const item of await tx.table("processing").toArray()) { - if (!allStatus[item.queue]) { - allStatus[item.queue] = { - queued: 0, - processing: 0, - dlq: 0, - }; - } - allStatus[item.queue].processing++; - } - for (const item of await tx.table("dlq").toArray()) { - if (!allStatus[item.queue]) { - allStatus[item.queue] = { - queued: 0, - processing: 0, - dlq: 0, - }; - } - allStatus[item.queue].dlq++; - } - }, - ); - - return allStatus; - } -} diff --git a/plugos/plug.ts b/plugos/plug.ts index c292729..fbe8ff2 100644 --- a/plugos/plug.ts +++ b/plugos/plug.ts @@ -86,7 +86,7 @@ export class Plug { `Function ${name} is not available in ${this.runtimeEnv}`, ); } - return await sandbox.invoke(name, args); + return sandbox.invoke(name, args); } stop() { diff --git a/plugos/syscalls/datastore.ts b/plugos/syscalls/datastore.ts new file mode 100644 index 0000000..4345579 --- /dev/null +++ b/plugos/syscalls/datastore.ts @@ -0,0 +1,75 @@ +import { KV, KvKey, KvQuery } from "$sb/types.ts"; +import type { DataStore } from "../lib/datastore.ts"; +import type { SyscallContext, SysCallMapping } from "../system.ts"; + +/** + * Exposes the datastore API to plugs, but scoping everything to a prefix based on the plug's name + * @param ds the datastore to wrap + * @param prefix prefix to scope all keys to to which the plug name will be appended + */ +export function dataStoreSyscalls( + ds: DataStore, + prefix: KvKey = ["ds"], +): SysCallMapping { + return { + "datastore.delete": (ctx, key: KvKey) => { + return ds.delete(applyPrefix(ctx, key)); + }, + + "datastore.set": (ctx, key: KvKey, value: any) => { + return ds.set(applyPrefix(ctx, key), value); + }, + + "datastore.batchSet": (ctx, kvs: KV[]) => { + return ds.batchSet( + kvs.map((kv) => ({ key: applyPrefix(ctx, kv.key), value: kv.value })), + ); + }, + + "datastore.batchDelete": (ctx, keys: KvKey[]) => { + return ds.batchDelete(keys.map((k) => applyPrefix(ctx, k))); + }, + + "datastore.batchGet": ( + ctx, + keys: KvKey[], + ): Promise<(any | undefined)[]> => { + return ds.batchGet(keys.map((k) => applyPrefix(ctx, k))); + }, + + "datastore.get": (ctx, key: KvKey): Promise => { + return ds.get(applyPrefix(ctx, key)); + }, + + "datastore.query": async ( + ctx, + query: KvQuery, + ): Promise => { + return (await ds.query({ + ...query, + prefix: applyPrefix(ctx, query.prefix), + })).map((kv) => ({ + key: stripPrefix(kv.key), + value: kv.value, + })); + }, + + "datastore.queryDelete": ( + ctx, + query: KvQuery, + ): Promise => { + return ds.queryDelete({ + ...query, + prefix: applyPrefix(ctx, query.prefix), + }); + }, + }; + + function applyPrefix(ctx: SyscallContext, key?: KvKey): KvKey { + return [...prefix, ctx.plug.name!, ...(key ? key : [])]; + } + + function stripPrefix(key: KvKey): KvKey { + return key.slice(prefix.length + 1); + } +} diff --git a/plugos/syscalls/mq.dexie.ts b/plugos/syscalls/mq.ts similarity index 100% rename from plugos/syscalls/mq.dexie.ts rename to plugos/syscalls/mq.ts diff --git a/plugos/syscalls/store.ts b/plugos/syscalls/store.ts deleted file mode 100644 index cae1b6f..0000000 --- a/plugos/syscalls/store.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { SysCallMapping } from "../system.ts"; -import { KV, KVStore } from "../lib/kv_store.ts"; - -export function storeSyscalls( - db: KVStore, -): SysCallMapping { - return { - "store.delete": (_ctx, key: string) => { - return db.del(key); - }, - - "store.deletePrefix": (_ctx, prefix: string) => { - return db.deletePrefix(prefix); - }, - - "store.deleteAll": () => { - return db.deleteAll(); - }, - - "store.set": (_ctx, key: string, value: any) => { - return db.set(key, value); - }, - - "store.batchSet": (_ctx, kvs: KV[]) => { - return db.batchSet(kvs); - }, - - "store.batchDelete": (_ctx, keys: string[]) => { - return db.batchDelete(keys); - }, - - "store.batchGet": ( - _ctx, - keys: string[], - ): Promise<(any | undefined)[]> => { - return db.batchGet(keys); - }, - - "store.get": (_ctx, key: string): Promise => { - return db.get(key); - }, - - "store.has": (_ctx, key: string): Promise => { - return db.has(key); - }, - - "store.queryPrefix": ( - _ctx, - keyPrefix: string, - ): Promise<{ key: string; value: any }[]> => { - return db.queryPrefix(keyPrefix); - }, - }; -} diff --git a/plugos/system.ts b/plugos/system.ts index e517ad6..7296f38 100644 --- a/plugos/system.ts +++ b/plugos/system.ts @@ -86,8 +86,7 @@ export class System extends EventEmitter> { args: any[], ): Promise { return this.syscallWithContext( - // Mock the plug - { plug: { name: contextPlugName } as any }, + { plug: this.plugs.get(contextPlugName)! }, syscallName, args, ); diff --git a/plugos/worker_runtime.ts b/plugos/worker_runtime.ts index 5b41f9f..7045d43 100644 --- a/plugos/worker_runtime.ts +++ b/plugos/worker_runtime.ts @@ -75,7 +75,12 @@ export function setupMessageListener( result: result, } as ControllerMessage); } catch (e: any) { - console.error(e); + console.error( + "An exception was thrown as a result of invoking function", + data.name, + "error:", + e, + ); workerPostMessage({ type: "invr", id: data.id!, diff --git a/plugs/builtin_plugs.ts b/plugs/builtin_plugs.ts index 4669640..60f78f1 100644 --- a/plugs/builtin_plugs.ts +++ b/plugs/builtin_plugs.ts @@ -7,6 +7,7 @@ export const builtinPlugNames = [ "plug-manager", "directive", "emoji", + "query", "markdown", "share", "tasks", diff --git a/plugs/directive/command.ts b/plugs/directive/command.ts index 17982eb..eba17f7 100644 --- a/plugs/directive/command.ts +++ b/plugs/directive/command.ts @@ -1,5 +1,8 @@ import { editor, markdown, mq, space, sync } from "$sb/syscalls.ts"; import { + addParentPointers, + findParentMatching, + nodeAtPos, ParseTree, removeParentPointers, renderToText, @@ -7,9 +10,8 @@ import { } from "$sb/lib/tree.ts"; import { renderDirectives } from "./directives.ts"; import { extractFrontmatter } from "$sb/lib/frontmatter.ts"; -import type { PageMeta } from "../../web/types.ts"; import { isFederationPath } from "$sb/lib/resolve.ts"; -import { MQMessage } from "$sb/types.ts"; +import { MQMessage, PageMeta } from "$sb/types.ts"; import { sleep } from "$sb/lib/async.ts"; const directiveUpdateQueueName = "directiveUpdateQueue"; @@ -200,3 +202,30 @@ export async function updateDirectives( } return text; } + +export async function convertToLiveQuery() { + const text = await editor.getText(); + const pos = await editor.getCursor(); + const tree = await markdown.parseMarkdown(text); + addParentPointers(tree); + const currentNode = nodeAtPos(tree, pos); + const directive = findParentMatching( + currentNode!, + (node) => node.type === "Directive", + ); + if (!directive) { + await editor.flashNotification( + "No directive found at cursor position", + "error", + ); + return; + } + const queryText = renderToText(directive!.children![0].children![1]); + await editor.dispatch({ + changes: { + from: directive.from, + to: directive.to, + insert: "```query\n" + queryText + "\n```", + }, + }); +} diff --git a/plugs/directive/complete.ts b/plugs/directive/complete.ts index fa9076e..8b5c980 100644 --- a/plugs/directive/complete.ts +++ b/plugs/directive/complete.ts @@ -1,11 +1,11 @@ import { events } from "$sb/syscalls.ts"; import { CompleteEvent } from "$sb/app_event.ts"; import { buildHandebarOptions } from "./util.ts"; -import type { PageMeta } from "../../web/types.ts"; import type { AttributeCompleteEvent, AttributeCompletion, } from "../index/attributes.ts"; +import { PageMeta } from "$sb/types.ts"; export async function queryComplete(completeEvent: CompleteEvent) { const querySourceMatch = /#query\s+([\w\-_]*)$/.exec( @@ -14,18 +14,31 @@ export async function queryComplete(completeEvent: CompleteEvent) { if (querySourceMatch) { const allEvents = await events.listEvents(); + const completionOptions = allEvents + .filter((eventName) => + eventName.startsWith("query:") && !eventName.includes("*") + ) + .map((source) => ({ + label: source.substring("query:".length), + })); + + const allObjectTypes: string[] = (await events.dispatchEvent("query_", {})) + .flat(); + + for (const type of allObjectTypes) { + completionOptions.push({ + label: type, + }); + } + return { from: completeEvent.pos - querySourceMatch[1].length, - options: allEvents - .filter((eventName) => eventName.startsWith("query:")) - .map((source) => ({ - label: source.substring("query:".length), - })), + options: completionOptions, }; } if (completeEvent.parentNodes.includes("DirectiveStart")) { - const querySourceMatch = /#query\s+([\w\-_]+)/.exec( + const querySourceMatch = /#query\s+([\w\-_\/]+)/.exec( completeEvent.linePrefix, ); const whereMatch = @@ -69,9 +82,9 @@ export async function templateVariableComplete(completeEvent: CompleteEvent) { ); const completions = (await events.dispatchEvent( - `attribute:complete:*`, + `attribute:complete:_`, { - source: "*", + source: "", prefix: match[1], } as AttributeCompleteEvent, )).flat() as AttributeCompletion[]; @@ -92,7 +105,7 @@ export function attributeCompletionsToCMCompletion( return completions.map( (completion) => ({ label: completion.name, - detail: `${completion.type} (${completion.source})`, + detail: `${completion.attributeType} (${completion.source})`, type: "attribute", }), ); diff --git a/plugs/directive/directive.plug.yaml b/plugs/directive/directive.plug.yaml index a54271a..df3e0a2 100644 --- a/plugs/directive/directive.plug.yaml +++ b/plugs/directive/directive.plug.yaml @@ -20,14 +20,6 @@ functions: mqSubscriptions: - queue: directiveUpdateQueue batchSize: 3 - indexData: - path: ./data.ts:indexData - events: - - page:index - dataQueryProvider: - path: ./data.ts:queryProvider - events: - - query:data queryComplete: path: ./complete.ts:queryComplete events: @@ -37,43 +29,13 @@ functions: events: - editor:complete + # Conversion + convertToLiveQuery: + path: command.ts:convertToLiveQuery + command: + name: "Directive: Convert Query to Live Query" + # Templates - insertQuery: - redirect: template.insertTemplateText - slashCommand: - name: query - description: Insert a query - value: | - - - - insertInclude: - redirect: template.insertTemplateText - slashCommand: - name: include - description: Include another page - value: | - - - - insertUseTemplate: - redirect: template.insertTemplateText - slashCommand: - name: use - description: Use a template - value: | - - - - insertUseVerboseTemplate: - redirect: template.insertTemplateText - slashCommand: - name: use-verbose - description: Use a template (verbose mode) - value: | - - - insertEvalTemplate: redirect: template.insertTemplateText slashCommand: diff --git a/plugs/directive/directives.ts b/plugs/directive/directives.ts index 89e1c58..2ad999d 100644 --- a/plugs/directive/directives.ts +++ b/plugs/directive/directives.ts @@ -1,5 +1,11 @@ -import { ParseTree, renderToText } from "$sb/lib/tree.ts"; -import { PageMeta } from "../../web/types.ts"; +import { + addParentPointers, + findParentMatching, + ParseTree, + renderToText, +} from "$sb/lib/tree.ts"; +import { PageMeta } from "$sb/types.ts"; +import { editor, markdown } from "$sb/syscalls.ts"; import { evalDirectiveRenderer } from "./eval_directive.ts"; import { queryDirectiveRenderer } from "./query_directive.ts"; @@ -53,8 +59,29 @@ export async function directiveDispatcher( const directiveStartText = renderToText(directiveStart).trim(); const directiveEndText = renderToText(directiveEnd).trim(); - if (directiveStart.children!.length === 1) { - // Everything not #query + const firstPart = directiveStart.children![0].text!; + if (firstPart?.includes("#query")) { + // #query + const newBody = await directiveRenderers["query"]( + "query", + pageMeta, + directiveStart.children![1].children![0], // The query ParseTree + ); + const result = + `${directiveStartText}\n${newBody.trim()}\n${directiveEndText}`; + return result; + } else if (firstPart?.includes("#eval")) { + console.log("Eval stuff", directiveStart.children![1].children![0]); + const newBody = await directiveRenderers["eval"]( + "eval", + pageMeta, + directiveStart.children![1].children![0], + ); + const result = + `${directiveStartText}\n${newBody.trim()}\n${directiveEndText}`; + return result; + } else { + // Everything not #query and #eval const match = directiveStartRegex.exec(directiveStart.children![0].text!); if (!match) { throw Error("No match"); @@ -70,16 +97,6 @@ export async function directiveDispatcher( } catch (e: any) { return `${directiveStartText}\n**ERROR:** ${e.message}\n${directiveEndText}`; } - } else { - // #query - const newBody = await directiveRenderers["query"]( - "query", - pageMeta, - directiveStart.children![1], // The query ParseTree - ); - const result = - `${directiveStartText}\n${newBody.trim()}\n${directiveEndText}`; - return result; } } diff --git a/plugs/directive/eval_directive.ts b/plugs/directive/eval_directive.ts index 63e0075..51bd2b9 100644 --- a/plugs/directive/eval_directive.ts +++ b/plugs/directive/eval_directive.ts @@ -1,23 +1,9 @@ -// This is some shocking stuff. My profession would kill me for this. - -import { YAML } from "$sb/syscalls.ts"; -import { ParseTree } from "$sb/lib/tree.ts"; -import { jsonToMDTable, renderTemplate } from "./util.ts"; -import type { PageMeta } from "../../web/types.ts"; +import { ParseTree, parseTreeToAST } from "$sb/lib/tree.ts"; import { replaceTemplateVars } from "../template/template.ts"; - -// Enables plugName.functionName(arg1, arg2) syntax in JS expressions -function translateJs(js: string): string { - return js.replaceAll( - /(\w+\.\w+)\s*\(/g, - 'await invokeFunction("$1", ', - ); -} - -// Syntaxes to support: -// - random JS expression -// - random JS expression render [[some/template]] -const expressionRegex = /(.+?)(\s+render\s+\[\[([^\]]+)\]\])?$/; +import { PageMeta } from "$sb/types.ts"; +import { expressionToKvQueryExpression } from "$sb/lib/parse-query.ts"; +import { evalQueryExpression } from "$sb/lib/query.ts"; +import { builtinFunctions } from "$sb/lib/builtin_query_functions.ts"; // This is rather scary and fragile stuff, but it works. export async function evalDirectiveRenderer( @@ -25,42 +11,19 @@ export async function evalDirectiveRenderer( pageMeta: PageMeta, expression: string | ParseTree, ): Promise { - if (typeof expression !== "string") { - throw new Error("Expected a string"); - } - // console.log("Got JS expression", expression); - const match = expressionRegex.exec(expression); - if (!match) { - throw new Error(`Invalid eval directive: ${expression}`); - } - let template = ""; - if (match[3]) { - // This is the template reference - expression = match[1]; - template = match[3]; - } try { - // Why the weird "eval" call? https://esbuild.github.io/content-types/#direct-eval - const result = await (0, eval)( - `(async () => { - function invokeFunction(name, ...args) { - return syscall("system.invokeFunction", name, ...args); - } - return ${replaceTemplateVars(translateJs(expression), pageMeta)}; - })()`, + const result = evalQueryExpression( + expressionToKvQueryExpression(parseTreeToAST( + JSON.parse( + await replaceTemplateVars(JSON.stringify(expression), pageMeta), + ), + )), + {}, + builtinFunctions, ); - if (template) { - return await renderTemplate(pageMeta, template, result); - } - if (typeof result === "string") { - return result; - } else if (typeof result === "number") { - return "" + result; - } else if (Array.isArray(result)) { - return jsonToMDTable(result); - } - return await YAML.stringify(result); + + return Promise.resolve("" + result); } catch (e: any) { - return `**ERROR:** ${e.message}`; + return Promise.resolve(`**ERROR:** ${e.message}`); } } diff --git a/plugs/directive/parser.ts b/plugs/directive/parser.ts deleted file mode 100644 index 6006edb..0000000 --- a/plugs/directive/parser.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { - collectNodesOfType, - findNodeOfType, - ParseTree, - replaceNodesMatching, -} from "$sb/lib/tree.ts"; - -// @ts-ignore auto generated -import { ParsedQuery, QueryFilter } from "$sb/lib/query.ts"; - -export function parseQuery(queryTree: ParseTree): ParsedQuery { - // const n = lezerToParseTree(query, parser.parse(query).topNode); - // Clean the tree a bit - replaceNodesMatching(queryTree, (n) => { - if (!n.type) { - const trimmed = n.text!.trim(); - if (!trimmed) { - return null; - } - n.text = trimmed; - } - }); - - // console.log("Parsed", JSON.stringify(n, null, 2)); - const queryNode = queryTree.children![0]; - const parsedQuery: ParsedQuery = { - table: queryNode.children![0].children![0].text!, - filter: [], - ordering: [], - }; - - const orderByNodes = collectNodesOfType(queryNode, "OrderClause"); - for (const orderByNode of orderByNodes) { - const nameNode = findNodeOfType(orderByNode, "Name"); - const orderBy = nameNode!.children![0].text!; - const orderNode = findNodeOfType(orderByNode, "OrderDirection"); - const orderDesc = orderNode - ? orderNode.children![0].text! === "desc" - : false; - parsedQuery.ordering.push({ orderBy, orderDesc }); - } - /** - * @deprecated due to PR #387 - * We'll take the first ordering and send that as the deprecated - * fields orderBy and orderDesc. This way it will be backward - * Plugs using the old ParsedQuery. - * Remove this block completely when ParsedQuery no longer have - * those two fields - */ - if (parsedQuery.ordering.length > 0) { - parsedQuery.orderBy = parsedQuery.ordering[0].orderBy; - parsedQuery.orderDesc = parsedQuery.ordering[0].orderDesc; - } - /** @end-deprecation due to PR #387 */ - - const limitNode = findNodeOfType(queryNode, "LimitClause"); - if (limitNode) { - const nameNode = findNodeOfType(limitNode, "Number"); - parsedQuery.limit = valueNodeToVal(nameNode!); - } - - const filterNodes = collectNodesOfType(queryNode, "FilterExpr"); - for (const filterNode of filterNodes) { - let val: any = undefined; - const valNode = filterNode.children![2].children![0]; - val = valueNodeToVal(valNode); - const f: QueryFilter = { - prop: filterNode.children![0].children![0].text!, - op: filterNode.children![1].text!, - value: val, - }; - parsedQuery.filter.push(f); - } - const selectNode = findNodeOfType(queryNode, "SelectClause"); - if (selectNode) { - parsedQuery.select = []; - collectNodesOfType(selectNode, "Name").forEach((t) => { - parsedQuery.select!.push(t.children![0].text!); - }); - } - - const renderNode = findNodeOfType(queryNode, "RenderClause"); - if (renderNode) { - let renderNameNode = findNodeOfType(renderNode, "PageRef"); - if (!renderNameNode) { - renderNameNode = findNodeOfType(renderNode, "String"); - } - parsedQuery.render = valueNodeToVal(renderNameNode!); - } - - return parsedQuery; -} - -export function valueNodeToVal(valNode: ParseTree): any { - switch (valNode.type) { - case "Number": - return +valNode.children![0].text!; - case "Bool": - return valNode.children![0].text! === "true"; - case "Null": - return null; - case "Name": - return valNode.children![0].text!; - case "Regex": { - const val = valNode.children![0].text!; - return val.substring(1, val.length - 1); - } - case "String": { - const stringVal = valNode.children![0].text!; - return stringVal.substring(1, stringVal.length - 1); - } - case "PageRef": { - const pageRefVal = valNode.children![0].text!; - return pageRefVal.substring(2, pageRefVal.length - 2); - } - case "List": { - return collectNodesOfType(valNode, "Value").map((t) => - valueNodeToVal(t.children![0]) - ); - } - } -} diff --git a/plugs/directive/query.test.ts b/plugs/directive/query.test.ts deleted file mode 100644 index 622525b..0000000 --- a/plugs/directive/query.test.ts +++ /dev/null @@ -1,187 +0,0 @@ -import { assertEquals } from "../../test_deps.ts"; -import { applyQuery } from "$sb/lib/query.ts"; - -import wikiMarkdownLang from "../../common/markdown_parser/parser.ts"; -import { parse } from "../../common/markdown_parser/parse_tree.ts"; -import { parseQuery as parseQueryQuery } from "./parser.ts"; -import { findNodeOfType, renderToText } from "../../plug-api/lib/tree.ts"; - -function parseQuery(query: string) { - const lang = wikiMarkdownLang([]); - const mdTree = parse( - lang, - ` - - `, - ); - const programNode = findNodeOfType(mdTree, "Program")!; - return parseQueryQuery(programNode); -} - -Deno.test("Test parser", () => { - const parsedBasicQuery = parseQuery(`page`); - assertEquals(parsedBasicQuery.table, "page"); - - const parsedQuery1 = parseQuery( - `task where completed = false and dueDate <= "{{today}}" order by dueDate desc limit 5`, - ); - assertEquals(parsedQuery1.table, "task"); - assertEquals(parsedQuery1.ordering.length, 1); - assertEquals(parsedQuery1.ordering[0].orderBy, "dueDate"); - assertEquals(parsedQuery1.ordering[0].orderDesc, true); - assertEquals(parsedQuery1.limit, 5); - assertEquals(parsedQuery1.filter.length, 2); - assertEquals(parsedQuery1.filter[0], { - op: "=", - prop: "completed", - value: false, - }); - assertEquals(parsedQuery1.filter[1], { - op: "<=", - prop: "dueDate", - value: "{{today}}", - }); - - const parsedQuery2 = parseQuery(`page where name =~ /interview\\/.*/"`); - assertEquals(parsedQuery2.table, "page"); - assertEquals(parsedQuery2.filter.length, 1); - assertEquals(parsedQuery2.filter[0], { - op: "=~", - prop: "name", - value: "interview\\/.*", - }); - - const parsedQuery3 = parseQuery(`page where something != null`); - assertEquals(parsedQuery3.table, "page"); - assertEquals(parsedQuery3.filter.length, 1); - assertEquals(parsedQuery3.filter[0], { - op: "!=", - prop: "something", - value: null, - }); - - assertEquals(parseQuery(`page select name`).select, ["name"]); - assertEquals(parseQuery(`page select name, age`).select, [ - "name", - "age", - ]); - - assertEquals( - parseQuery(`gh-events where type in ["PushEvent", "somethingElse"]`), - { - table: "gh-events", - ordering: [], - filter: [ - { - op: "in", - prop: "type", - value: ["PushEvent", "somethingElse"], - }, - ], - }, - ); - - assertEquals(parseQuery(`something render [[template/table]]`), { - table: "something", - ordering: [], - filter: [], - render: "template/table", - }); - - assertEquals(parseQuery(`something render "template/table"`), { - table: "something", - ordering: [], - filter: [], - render: "template/table", - }); -}); - -Deno.test("Test applyQuery", () => { - const data: any[] = [ - { name: "interview/My Interview", lastModified: 1 }, - { name: "interview/My Interview 2", lastModified: 2 }, - { name: "Pete", age: 38 }, - { name: "Angie", age: 28 }, - ]; - - assertEquals( - applyQuery(parseQuery(`page where name =~ /interview\\/.*/`), data), - [ - { name: "interview/My Interview", lastModified: 1 }, - { name: "interview/My Interview 2", lastModified: 2 }, - ], - ); - assertEquals( - applyQuery( - parseQuery(`page where name =~ /interview\\/.*/ order by lastModified`), - data, - ), - [ - { name: "interview/My Interview", lastModified: 1 }, - { name: "interview/My Interview 2", lastModified: 2 }, - ], - ); - assertEquals( - applyQuery( - parseQuery( - `page where name =~ /interview\\/.*/ order by lastModified desc`, - ), - data, - ), - [ - { name: "interview/My Interview 2", lastModified: 2 }, - { name: "interview/My Interview", lastModified: 1 }, - ], - ); - assertEquals(applyQuery(parseQuery(`page where age > 30`), data), [ - { name: "Pete", age: 38 }, - ]); - assertEquals( - applyQuery(parseQuery(`page where age > 28 and age < 38`), data), - [], - ); - assertEquals( - applyQuery(parseQuery(`page where age > 30 select name`), data), - [{ name: "Pete" }], - ); - - assertEquals( - applyQuery(parseQuery(`page where name in ["Pete"] select name`), data), - [{ name: "Pete" }], - ); -}); - -Deno.test("Test applyQuery with multi value", () => { - const data: any[] = [ - { name: "Pete", children: ["John", "Angie"] }, - { name: "Angie", children: ["Angie"] }, - { name: "Steve" }, - ]; - - assertEquals( - applyQuery(parseQuery(`page where children = "Angie"`), data), - [ - { name: "Pete", children: ["John", "Angie"] }, - { name: "Angie", children: ["Angie"] }, - ], - ); - - assertEquals( - applyQuery(parseQuery(`page where children = ["Angie", "John"]`), data), - [ - { name: "Pete", children: ["John", "Angie"] }, - { name: "Angie", children: ["Angie"] }, - ], - ); -}); - -const testQuery = ` - -`; - -Deno.test("Query parsing and serialization", () => { - const lang = wikiMarkdownLang([]); - const mdTree = parse(lang, testQuery); - // console.log(JSON.stringify(mdTree, null, 2)); - assertEquals(renderToText(mdTree), testQuery); -}); diff --git a/plugs/directive/query_directive.ts b/plugs/directive/query_directive.ts index 1c4c2f8..705ce63 100644 --- a/plugs/directive/query_directive.ts +++ b/plugs/directive/query_directive.ts @@ -2,10 +2,10 @@ import { events } from "$sb/syscalls.ts"; import { replaceTemplateVars } from "../template/template.ts"; import { renderTemplate } from "./util.ts"; -import { parseQuery } from "./parser.ts"; import { jsonToMDTable } from "./util.ts"; -import { ParseTree } from "$sb/lib/tree.ts"; -import type { PageMeta } from "../../web/types.ts"; +import { ParseTree, parseTreeToAST } from "$sb/lib/tree.ts"; +import { astToKvQuery } from "$sb/lib/parse-query.ts"; +import { PageMeta, Query } from "$sb/types.ts"; export async function queryDirectiveRenderer( _directive: string, @@ -15,11 +15,14 @@ export async function queryDirectiveRenderer( if (typeof query === "string") { throw new Error("Argument must be a ParseTree"); } - const parsedQuery = parseQuery( - JSON.parse(replaceTemplateVars(JSON.stringify(query), pageMeta)), + const parsedQuery: Query = astToKvQuery( + parseTreeToAST( + JSON.parse(await replaceTemplateVars(JSON.stringify(query), pageMeta)), + ), ); + // console.log("QUERY", parsedQuery); - const eventName = `query:${parsedQuery.table}`; + const eventName = `query:${parsedQuery.querySource}`; // console.log("Parsed query", parsedQuery); // Let's dispatch an event and see what happens @@ -30,24 +33,23 @@ export async function queryDirectiveRenderer( ); if (results.length === 0) { // This means there was no handler for the event which means it's unsupported - return `**Error:** Unsupported query source '${parsedQuery.table}'`; - } else if (results.length === 1) { + return `**Error:** Unsupported query source '${parsedQuery.querySource}'`; + } else { // console.log("Parsed query", parsedQuery); + const allResults = results.flat(); if (parsedQuery.render) { const rendered = await renderTemplate( pageMeta, parsedQuery.render, - results[0], + allResults, ); return rendered.trim(); } else { - if (results[0].length === 0) { + if (allResults.length === 0) { return "No results"; } else { - return jsonToMDTable(results[0]); + return jsonToMDTable(allResults); } } - } else { - throw new Error(`Too many query results: ${results.length}`); } } diff --git a/plugs/directive/template_directive.ts b/plugs/directive/template_directive.ts index 74ca6f7..e6a53db 100644 --- a/plugs/directive/template_directive.ts +++ b/plugs/directive/template_directive.ts @@ -1,15 +1,13 @@ import { queryRegex } from "$sb/lib/query.ts"; import { ParseTree, renderToText } from "$sb/lib/tree.ts"; -import { markdown, space } from "$sb/syscalls.ts"; -import Handlebars from "handlebars"; +import { handlebars, markdown, space } from "$sb/syscalls.ts"; import { replaceTemplateVars } from "../template/template.ts"; import { extractFrontmatter } from "$sb/lib/frontmatter.ts"; import { directiveRegex } from "./directives.ts"; import { updateDirectives } from "./command.ts"; -import { buildHandebarOptions } from "./util.ts"; -import { PageMeta } from "../../web/types.ts"; import { resolvePath, rewritePageRefs } from "$sb/lib/resolve.ts"; +import { PageMeta } from "$sb/types.ts"; const templateRegex = /\[\[([^\]]+)\]\]\s*(.*)\s*/; @@ -30,7 +28,7 @@ export async function templateDirectiveRenderer( let parsedArgs = {}; if (args) { try { - parsedArgs = JSON.parse(replaceTemplateVars(args, pageMeta)); + parsedArgs = JSON.parse(await replaceTemplateVars(args, pageMeta)); } catch { throw new Error( `Failed to parse template instantiation arg: ${ @@ -65,11 +63,9 @@ export async function templateDirectiveRenderer( // if it's a template injection (not a literal "include") if (directive === "use") { - const templateFn = Handlebars.compile( - newBody, - { noEscape: true }, - ); - newBody = templateFn(parsedArgs, buildHandebarOptions(pageMeta)); + newBody = await handlebars.renderTemplate(newBody, parsedArgs, { + page: pageMeta, + }); // Recursively render directives const tree = await markdown.parseMarkdown(newBody); diff --git a/plugs/directive/util.ts b/plugs/directive/util.ts index 6ecfa51..1a772f1 100644 --- a/plugs/directive/util.ts +++ b/plugs/directive/util.ts @@ -1,8 +1,6 @@ -import Handlebars from "handlebars"; - -import { space } from "$sb/syscalls.ts"; -import type { PageMeta } from "../../web/types.ts"; -import { handlebarHelpers } from "./handlebar_helpers.ts"; +import { handlebars, space } from "$sb/syscalls.ts"; +import { handlebarHelpers } from "../../common/syscalls/handlebar_helpers.ts"; +import { PageMeta } from "$sb/types.ts"; const maxWidth = 70; @@ -10,6 +8,9 @@ export function defaultJsonTransformer(_k: string, v: any) { if (v === undefined) { return ""; } + if (typeof v === "string") { + return v.replaceAll("\n", " ").replaceAll("|", "\\|"); + } return "" + v; } @@ -86,13 +87,12 @@ export async function renderTemplate( ): Promise { let templateText = await space.readPage(renderTemplate); templateText = `{{#each .}}\n${templateText}\n{{/each}}`; - const template = Handlebars.compile(templateText, { noEscape: true }); - return template(data, buildHandebarOptions(pageMeta)); + return handlebars.renderTemplate(templateText, data, { page: pageMeta }); } export function buildHandebarOptions(pageMeta: PageMeta) { return { - helpers: handlebarHelpers(pageMeta.name), + helpers: handlebarHelpers(), data: { page: pageMeta }, }; } diff --git a/plugs/editor/complete.ts b/plugs/editor/complete.ts index 1dbe008..c97d3a9 100644 --- a/plugs/editor/complete.ts +++ b/plugs/editor/complete.ts @@ -1,6 +1,6 @@ import { CompleteEvent } from "$sb/app_event.ts"; import { space } from "$sb/syscalls.ts"; -import { PageMeta } from "../../web/types.ts"; +import { PageMeta } from "$sb/types.ts"; import { cacheFileListing } from "../federation/federation.ts"; // Completion diff --git a/plugs/editor/link.ts b/plugs/editor/link.ts index b3844e1..e1f11de 100644 --- a/plugs/editor/link.ts +++ b/plugs/editor/link.ts @@ -57,7 +57,7 @@ export async function titleUnfurl(url: string): Promise { const response = await fetch(url); if (response.status < 200 || response.status >= 300) { console.error("Unfurl failed", await response.text()); - throw new Error(`Failed to fetch: ${await response.statusText}`); + throw new Error(`Failed to fetch: ${response.statusText}`); } const body = await response.text(); const match = titleRegex.exec(body); diff --git a/plugs/editor/page.ts b/plugs/editor/page.ts index 5e4fe16..4833fef 100644 --- a/plugs/editor/page.ts +++ b/plugs/editor/page.ts @@ -1,9 +1,5 @@ -import type { CompleteEvent } from "$sb/app_event.ts"; import { editor, space } from "$sb/syscalls.ts"; -import { cacheFileListing } from "../federation/federation.ts"; -import type { PageMeta } from "../../web/types.ts"; - export async function deletePage() { const pageName = await editor.getCurrentPage(); if ( diff --git a/plugs/editor/vim.ts b/plugs/editor/vim.ts index 50fa75b..f9021c0 100644 --- a/plugs/editor/vim.ts +++ b/plugs/editor/vim.ts @@ -1,11 +1,11 @@ import { readCodeBlockPage } from "$sb/lib/yaml_page.ts"; -import { editor, store } from "$sb/syscalls.ts"; +import { clientStore, editor } from "$sb/syscalls.ts"; export async function toggleVimMode() { - let vimMode = await store.get("vimMode"); + let vimMode = await clientStore.get("vimMode"); vimMode = !vimMode; await editor.setUiOption("vimMode", vimMode); - await store.set("vimMode", vimMode); + await clientStore.set("vimMode", vimMode); } export async function loadVimRc() { @@ -28,7 +28,7 @@ export async function loadVimRc() { } } } - } catch (e: any) { + } catch { // No VIMRC page found } } diff --git a/plugs/federation/federation.ts b/plugs/federation/federation.ts index 59d9c8e..e4bc6a3 100644 --- a/plugs/federation/federation.ts +++ b/plugs/federation/federation.ts @@ -1,7 +1,7 @@ import "$sb/lib/fetch.ts"; import { federatedPathToUrl } from "$sb/lib/resolve.ts"; import { readFederationConfigs } from "./config.ts"; -import { store } from "$sb/syscalls.ts"; +import { datastore } from "$sb/syscalls.ts"; import type { FileMeta } from "$sb/types.ts"; async function responseToFileMeta( @@ -29,7 +29,7 @@ async function responseToFileMeta( }; } -const fileListingPrefixCacheKey = `federationListCache:`; +const fileListingPrefixCacheKey = `federationListCache`; const listingCacheTimeout = 1000 * 30; const listingFetchTimeout = 2000; @@ -56,8 +56,8 @@ export async function listFiles(): Promise { } export async function cacheFileListing(uri: string): Promise { - const cachedListing = await store.get( - `${fileListingPrefixCacheKey}${uri}`, + const cachedListing = await datastore.get( + [fileListingPrefixCacheKey, uri], ) as FileListingCacheEntry; if ( cachedListing && @@ -99,7 +99,7 @@ export async function cacheFileListing(uri: string): Promise { perm: "ro", name: `${rootUri}/${meta.name}`, })); - await store.set(`${fileListingPrefixCacheKey}${uri}`, { + await datastore.set([fileListingPrefixCacheKey, uri], { items, lastUpdated: Date.now(), } as FileListingCacheEntry); diff --git a/plugs/index/anchor.ts b/plugs/index/anchor.ts index a47bf8c..f374491 100644 --- a/plugs/index/anchor.ts +++ b/plugs/index/anchor.ts @@ -1,24 +1,31 @@ import { collectNodesOfType } from "$sb/lib/tree.ts"; -import { index } from "$sb/syscalls.ts"; import type { CompleteEvent, IndexTreeEvent } from "$sb/app_event.ts"; import { removeQueries } from "$sb/lib/query.ts"; +import { ObjectValue } from "$sb/types.ts"; +import { indexObjects, queryObjects } from "./api.ts"; -// Key space -// a:pageName:anchorName => pos +type AnchorObject = ObjectValue<{ + name: string; + page: string; + pos: number; +}>; export async function indexAnchors({ name: pageName, tree }: IndexTreeEvent) { removeQueries(tree); - const anchors: { key: string; value: string }[] = []; + const anchors: ObjectValue[] = []; collectNodesOfType(tree, "NamedAnchor").forEach((n) => { const aName = n.children![0].text!.substring(1); anchors.push({ - key: `a:${pageName}:${aName}`, - value: "" + n.from, + ref: `${pageName}@${aName}`, + tags: ["anchor"], + name: aName, + page: pageName, + pos: n.from!, }); }); // console.log("Found", anchors.length, "anchors(s)"); - await index.batchSet(pageName, anchors); + await indexObjects(pageName, anchors); } export async function anchorComplete(completeEvent: CompleteEvent) { @@ -31,13 +38,13 @@ export async function anchorComplete(completeEvent: CompleteEvent) { if (!pageRef) { pageRef = completeEvent.pageName; } - const allAnchors = await index.queryPrefix( - `a:${pageRef}:${anchorRef}`, - ); + const allAnchors = await queryObjects("anchor", { + filter: ["=", ["attr", "page"], ["string", pageRef]], + }); return { from: completeEvent.pos - anchorRef.length, options: allAnchors.map((a) => ({ - label: a.key.split(":")[2], + label: a.name, type: "anchor", })), }; diff --git a/plugs/index/api.ts b/plugs/index/api.ts new file mode 100644 index 0000000..744f6c9 --- /dev/null +++ b/plugs/index/api.ts @@ -0,0 +1,156 @@ +import { datastore } from "$sb/syscalls.ts"; +import { KV, KvKey, ObjectQuery, ObjectValue } from "$sb/types.ts"; +import { QueryProviderEvent } from "$sb/app_event.ts"; +import { builtins } from "./builtins.ts"; +import { AttributeObject, determineType } from "./attributes.ts"; + +const indexKey = "idx"; +const pageKey = "ridx"; + +/* + * Key namespace: + * [indexKey, type, ...key, page] -> value + * [pageKey, page, ...key] -> true // for fast page clearing + * ["type", type] -> true // for fast type listing + */ + +export function batchSet(page: string, kvs: KV[]): Promise { + const finalBatch: KV[] = []; + for (const { key, value } of kvs) { + finalBatch.push({ + key: [indexKey, ...key, page], + value, + }, { + key: [pageKey, page, ...key], + value: true, + }); + } + return datastore.batchSet(finalBatch); +} + +/** + * Clears all keys for a given page + * @param page + */ +export async function clearPageIndex(page: string): Promise { + const allKeys: KvKey[] = []; + for ( + const { key } of await datastore.query({ + prefix: [pageKey, page], + }) + ) { + allKeys.push(key); + allKeys.push([indexKey, ...key.slice(2), page]); + } + await datastore.batchDel(allKeys); +} + +/** + * Clears the entire datastore for this indexKey plug + */ +export async function clearIndex(): Promise { + const allKeys: KvKey[] = []; + for ( + const { key } of await datastore.query({ prefix: [] }) + ) { + allKeys.push(key); + } + await datastore.batchDel(allKeys); + console.log("Deleted", allKeys.length, "keys from the index"); +} + +// ENTITIES API + +/** + * Indexes entities in the data store + */ +export async function indexObjects( + page: string, + objects: ObjectValue[], +): Promise { + const kvs: KV[] = []; + const allAttributes = new Map(); // tag:name -> attributeType + for (const obj of objects) { + for (const tag of obj.tags) { + kvs.push({ + key: [tag, cleanKey(obj.ref, page)], + value: obj, + }); + // Index attributes + if (!builtins[tag]) { + // But only for non-builtin tags + for ( + const [attrName, attrValue] of Object.entries( + obj as Record, + ) + ) { + if (attrName.startsWith("$")) { + continue; + } + allAttributes.set(`${tag}:${attrName}`, determineType(attrValue)); + } + } + } + } + if (allAttributes.size > 0) { + await indexObjects( + page, + [...allAttributes].map(([key, value]) => { + const [tag, name] = key.split(":"); + return { + ref: key, + tags: ["attribute"], + tag, + name, + attributeType: value, + page, + }; + }), + ); + } + return batchSet(page, kvs); +} + +function cleanKey(ref: string, page: string) { + if (ref.startsWith(`${page}@`)) { + return ref.substring(page.length + 1); + } else { + return ref; + } +} + +export async function queryObjects( + tag: string, + query: ObjectQuery, +): Promise[]> { + return (await datastore.query({ + ...query, + prefix: [indexKey, tag], + })).map(({ value }) => value); +} + +export async function getObjectByRef( + page: string, + tag: string, + ref: string, +): Promise | undefined> { + console.log("Fetching!!!!!", [indexKey, tag, cleanKey(ref, page), page]); + return (await datastore.get([indexKey, tag, cleanKey(ref, page), page])); +} + +export async function objectSourceProvider({ + query, +}: QueryProviderEvent): Promise { + const tag = query.querySource!; + const results = await datastore.query({ + ...query, + prefix: [indexKey, tag], + }); + return results.map((r) => r.value); +} + +export async function discoverSources() { + return (await datastore.query({ prefix: [indexKey, "tag"] })).map(( + { key }, + ) => key[2]); +} diff --git a/plugs/index/asset/linked_mentions.js b/plugs/index/asset/linked_mentions.js new file mode 100644 index 0000000..cf87892 --- /dev/null +++ b/plugs/index/asset/linked_mentions.js @@ -0,0 +1,17 @@ +function processClick(e) { + const dataEl = e.target.closest("[data-ref]"); + syscall( + "system.invokeFunction", + "index.navigateToMention", + dataEl.getAttribute("data-ref"), + ).catch(console.error); +} + +document.getElementById("link-ul").addEventListener("click", processClick); +document.getElementById("hide-button").addEventListener("click", function () { + console.log("HERE") + syscall( + "system.invokeFunction", + "index.toggleMentions", + ).catch(console.error); +}); diff --git a/plugs/index/asset/style.css b/plugs/index/asset/style.css new file mode 100644 index 0000000..30f3aa0 --- /dev/null +++ b/plugs/index/asset/style.css @@ -0,0 +1,25 @@ +body { + font-family: var(--ui-font); + background-color: var(--root-background-color); + color: var(--root-color); + overflow: scroll; +} + +.sb-line-h2 { + border-top-right-radius: 5px; + border-top-left-radius: 5px; + margin: 0; + padding: 10px !important; + background-color: rgba(233, 233, 233, 0.5); +} + +#hide-button { + position: absolute; + right: 15px; + top: 15px; +} + +li code { + font-size: 80%; + color: #a5a4a4; +} \ No newline at end of file diff --git a/plugs/index/attributes.ts b/plugs/index/attributes.ts index 51d0e20..abee087 100644 --- a/plugs/index/attributes.ts +++ b/plugs/index/attributes.ts @@ -1,12 +1,15 @@ -import { index } from "$sb/silverbullet-syscall/mod.ts"; import type { CompleteEvent } from "$sb/app_event.ts"; import { events } from "$sb/syscalls.ts"; +import { queryObjects } from "./api.ts"; +import { ObjectValue, QueryExpression } from "$sb/types.ts"; +import { builtinPseudoPage } from "./builtins.ts"; -export type AttributeContext = "page" | "item" | "task"; - -type AttributeEntry = { - type: string; -}; +export type AttributeObject = ObjectValue<{ + name: string; + attributeType: string; + tag: string; + page: string; +}>; export type AttributeCompleteEvent = { source: string; @@ -16,41 +19,11 @@ export type AttributeCompleteEvent = { export type AttributeCompletion = { name: string; source: string; - type: string; + attributeType: string; builtin?: boolean; }; -const builtinAttributes: Record> = { - page: { - name: "string", - lastModified: "number", - perm: "rw|ro", - contentType: "string", - size: "number", - tags: "array", - }, - task: { - name: "string", - done: "boolean", - page: "string", - state: "string", - deadline: "string", - pos: "number", - tags: "array", - }, - item: { - name: "string", - page: "string", - pos: "number", - tags: "array", - }, - tag: { - name: "string", - freq: "number", - }, -}; - -function determineType(v: any): string { +export function determineType(v: any): string { const t = typeof v; if (t === "object") { if (Array.isArray(v)) { @@ -60,69 +33,23 @@ function determineType(v: any): string { return t; } -const attributeKeyPrefix = "attr:"; - -export async function indexAttributes( - pageName: string, - attributes: Record, - context: AttributeContext, -) { - await index.batchSet( - pageName, - Object.entries(attributes).map(([k, v]) => { - return { - key: `${attributeKeyPrefix}${context}:${k}`, - value: { - type: determineType(v), - } as AttributeEntry, - }; - }), - ); -} - -export async function customAttributeCompleter( +export async function objectAttributeCompleter( attributeCompleteEvent: AttributeCompleteEvent, ): Promise { - const sourcePrefix = attributeCompleteEvent.source === "*" - ? "" - : `${attributeCompleteEvent.source}:`; - const allAttributes = await index.queryPrefix( - `${attributeKeyPrefix}${sourcePrefix}`, - ); - return allAttributes.map((attr) => { - const [_prefix, context, name] = attr.key.split(":"); - return { - name, - source: context, - type: attr.value.type, - }; + const attributeFilter: QueryExpression | undefined = + attributeCompleteEvent.source === "" + ? undefined + : ["=", ["attr", "tag"], ["string", attributeCompleteEvent.source]]; + const allAttributes = await queryObjects("attribute", { + filter: attributeFilter, }); -} - -export function builtinAttributeCompleter( - attributeCompleteEvent: AttributeCompleteEvent, -): AttributeCompletion[] { - let allAttributes = builtinAttributes[attributeCompleteEvent.source]; - if (attributeCompleteEvent.source === "*") { - allAttributes = {}; - for (const [source, attributes] of Object.entries(builtinAttributes)) { - for (const [name, type] of Object.entries(attributes)) { - allAttributes[name] = `${type}|${source}`; - } - } - } - if (!allAttributes) { - return []; - } - return Object.entries(allAttributes).map(([name, type]) => { + return allAttributes.map((value) => { return { - name, - source: attributeCompleteEvent.source === "*" - ? type.split("|")[1] - : attributeCompleteEvent.source, - type: attributeCompleteEvent.source === "*" ? type.split("|")[0] : type, - builtin: true, - }; + name: value.name, + source: value.tag, + attributeType: value.attributeType, + builtin: value.page === builtinPseudoPage, + } as AttributeCompletion; }); } @@ -184,7 +111,7 @@ export function attributeCompletionsToCMCompletion( (completion) => ({ label: completion.name, apply: `${completion.name}: `, - detail: `${completion.type} (${completion.source})`, + detail: `${completion.attributeType} (${completion.source})`, type: "attribute", }), ); diff --git a/plugs/index/builtins.ts b/plugs/index/builtins.ts new file mode 100644 index 0000000..b6b5cf5 --- /dev/null +++ b/plugs/index/builtins.ts @@ -0,0 +1,79 @@ +import { ObjectValue } from "$sb/types.ts"; +import { indexObjects } from "./api.ts"; +import { AttributeObject } from "./attributes.ts"; +import { TagObject } from "./tags.ts"; + +export const builtinPseudoPage = ":builtin:"; + +export const builtins: Record> = { + page: { + name: "string", + lastModified: "date", + perm: "rw|ro", + contentType: "string", + size: "number", + tags: "array", + }, + task: { + name: "string", + done: "boolean", + page: "string", + state: "string", + deadline: "string", + pos: "number", + tags: "array", + }, + tag: { + name: "string", + page: "string", + context: "string", + }, + attribute: { + name: "string", + attributeType: "string", + type: "string", + page: "string", + }, + anchor: { + name: "string", + page: "string", + pos: "number", + }, + link: { + name: "string", + page: "string", + pos: "number", + alias: "string", + inDirective: "boolean", + asTemplate: "boolean", + }, +}; + +export async function loadBuiltinsIntoIndex() { + console.log("Loading builtins attributes into index"); + const allTags: ObjectValue[] = []; + for (const [tag, attributes] of Object.entries(builtins)) { + allTags.push({ + ref: tag, + tags: ["tag"], + name: tag, + page: builtinPseudoPage, + parent: "builtin", + }); + await indexObjects( + builtinPseudoPage, + Object.entries(attributes).map(([name, attributeType]) => { + return { + ref: `${tag}:${name}`, + tags: ["attribute"], + tag, + name, + attributeType, + builtinPseudoPage, + page: builtinPseudoPage, + }; + }), + ); + } + await indexObjects(builtinPseudoPage, allTags); +} diff --git a/plugs/index/command.ts b/plugs/index/command.ts new file mode 100644 index 0000000..3dd4524 --- /dev/null +++ b/plugs/index/command.ts @@ -0,0 +1,51 @@ +import { editor, events, markdown, mq, space, system } from "$sb/syscalls.ts"; +import { sleep } from "$sb/lib/async.ts"; +import { IndexEvent } from "$sb/app_event.ts"; +import { MQMessage } from "$sb/types.ts"; + +export async function reindexCommand() { + await editor.flashNotification("Performing full page reindex..."); + await system.invokeFunction("reindexSpace"); + await editor.flashNotification("Done with page index!"); +} + +export async function reindexSpace() { + console.log("Clearing page index..."); + // Executed this way to not have to embed the search plug code here + await system.invokeFunction("search.clearIndex"); + await system.invokeFunction("index.clearIndex"); + const pages = await space.listPages(); + + // Queue all page names to be indexed + await mq.batchSend("indexQueue", pages.map((page) => page.name)); + + // Now let's wait for the processing to finish + let queueStats = await mq.getQueueStats("indexQueue"); + while (queueStats.queued > 0 || queueStats.processing > 0) { + sleep(1000); + queueStats = await mq.getQueueStats("indexQueue"); + } + // And notify the user + console.log("Indexing completed!"); +} + +export async function processIndexQueue(messages: MQMessage[]) { + for (const message of messages) { + const name: string = message.body; + console.log(`Indexing page ${name}`); + const text = await space.readPage(name); + const parsed = await markdown.parseMarkdown(text); + await events.dispatchEvent("page:index", { + name, + tree: parsed, + }); + } +} + +export async function parseIndexTextRepublish({ name, text }: IndexEvent) { + // console.log("Reindexing", name); + await events.dispatchEvent("page:index", { + name, + tree: await markdown.parseMarkdown(text), + }); +} diff --git a/plugs/directive/data.ts b/plugs/index/data.ts similarity index 52% rename from plugs/directive/data.ts rename to plugs/index/data.ts index 456912c..c3a09e8 100644 --- a/plugs/directive/data.ts +++ b/plugs/index/data.ts @@ -1,13 +1,20 @@ -// Index key space: -// data:page@pos - -import type { IndexTreeEvent, QueryProviderEvent } from "$sb/app_event.ts"; -import { index, YAML } from "$sb/syscalls.ts"; +import type { IndexTreeEvent } from "$sb/app_event.ts"; +import { YAML } from "$sb/syscalls.ts"; import { collectNodesOfType, findNodeOfType } from "$sb/lib/tree.ts"; -import { applyQuery, removeQueries } from "$sb/lib/query.ts"; +import { removeQueries } from "$sb/lib/query.ts"; +import { ObjectValue } from "$sb/types.ts"; +import { indexObjects } from "./api.ts"; +import { TagObject } from "./tags.ts"; + +type DataObject = ObjectValue< + { + pos: number; + page: string; + } & Record +>; export async function indexData({ name, tree }: IndexTreeEvent) { - const dataObjects: { key: string; value: any }[] = []; + const dataObjects: ObjectValue[] = []; removeQueries(tree); @@ -17,7 +24,8 @@ export async function indexData({ name, tree }: IndexTreeEvent) { if (!codeInfoNode) { return; } - if (codeInfoNode.children![0].text !== "data") { + const fenceType = codeInfoNode.children![0].text!; + if (fenceType !== "data" && !fenceType.startsWith("#")) { return; } const codeTextNode = findNodeOfType(t, "CodeText"); @@ -26,6 +34,7 @@ export async function indexData({ name, tree }: IndexTreeEvent) { return; } const codeText = codeTextNode.children![0].text!; + const dataType = fenceType === "data" ? "data" : fenceType.substring(1); try { const docs = codeText.split("---"); // We support multiple YAML documents in one block @@ -34,12 +43,25 @@ export async function indexData({ name, tree }: IndexTreeEvent) { if (!doc) { continue; } + const pos = t.from! + i; dataObjects.push({ - key: `data:${name}@${t.from! + i}`, - value: doc, + ref: `${name}@${pos}`, + tags: [dataType], + ...doc, + pos, + page: name, }); } // console.log("Parsed data", parsedData); + await indexObjects(name, [ + { + ref: dataType, + tags: ["tag"], + name: dataType, + page: name, + parent: "data", + }, + ]); } catch (e) { console.error("Could not parse data", codeText, "error:", e); return; @@ -47,20 +69,5 @@ export async function indexData({ name, tree }: IndexTreeEvent) { }), ); // console.log("Found", dataObjects.length, "data objects"); - await index.batchSet(name, dataObjects); -} - -export async function queryProvider({ - query, -}: QueryProviderEvent): Promise { - const allData: any[] = []; - for (const { key, page, value } of await index.queryPrefix("data:")) { - const [, pos] = key.split("@"); - allData.push({ - ...value, - page: page, - pos: +pos, - }); - } - return applyQuery(query, allData); + await indexObjects(name, dataObjects); } diff --git a/plugs/index/index.plug.yaml b/plugs/index/index.plug.yaml index ad8d773..4b221d6 100644 --- a/plugs/index/index.plug.yaml +++ b/plugs/index/index.plug.yaml @@ -10,34 +10,67 @@ syntax: - "$" regex: "\\$[a-zA-Z\\.\\-\\/]+[\\w\\.\\-\\/]*" className: sb-named-anchor +assets: + - asset/* functions: - clearPageIndex: - path: "./page.ts:clearPageIndex" + loadBuiltinsIntoIndex: + path: builtins.ts:loadBuiltinsIntoIndex + env: server + events: + - system:ready + + # Public API + batchSet: + path: api.ts:batchSet + env: server + indexObjects: + path: api.ts:indexObjects + env: server + queryObjects: + path: api.ts:queryObjects + env: server + getObjectByRef: + path: api.ts:getObjectByRef + env: server + + objectSourceProvider: + path: api.ts:objectSourceProvider + events: + - query:* + discoverSources: + path: api.ts:discoverSources + events: + - query_ + clearIndex: + path: api.ts:clearIndex + env: server + + clearDSIndex: + path: api.ts:clearPageIndex env: server events: - page:saved - page:deleted - pageQueryProvider: - path: ./page.ts:pageQueryProvider - events: - - query:page + parseIndexTextRepublish: - path: "./page.ts:parseIndexTextRepublish" + path: "./command.ts:parseIndexTextRepublish" env: server events: - page:index_text + reindexSpaceCommand: - path: "./page.ts:reindexCommand" + path: "./command.ts:reindexCommand" command: name: "Space: Reindex" processIndexQueue: - path: ./page.ts:processIndexQueue + path: ./command.ts:processIndexQueue mqSubscriptions: - queue: indexQueue batchSize: 10 autoAck: true reindexSpace: - path: "./page.ts:reindexSpace" + path: "./command.ts:reindexSpace" + env: server # Attachments attachmentQueryProvider: @@ -45,46 +78,37 @@ functions: events: - query:attachment + indexPage: + path: page.ts:indexPage + events: + - page:index + # Backlinks indexLinks: path: "./page_links.ts:indexLinks" events: - page:index - linkQueryProvider: - path: ./page_links.ts:linkQueryProvider - events: - - query:link attributeComplete: path: "./attributes.ts:attributeComplete" events: - editor:complete - customAttributeCompleter: - path: ./attributes.ts:customAttributeCompleter + objectAttributeCompleter: + path: ./attributes.ts:objectAttributeCompleter events: - - attribute:complete:page - - attribute:complete:task - - attribute:complete:item - attribute:complete:* - builtinAttributeCompleter: - path: ./attributes.ts:builtinAttributeCompleter - events: - - attribute:complete:page - - attribute:complete:task - - attribute:complete:item - - attribute:complete:* + # builtinAttributeCompleter: + # path: ./attributes.ts:builtinAttributeCompleter + # events: + # - attribute:complete:* # Item indexing indexItem: path: "./item.ts:indexItems" events: - page:index - itemQueryProvider: - path: "./item.ts:queryProvider" - events: - - query:item # Anchors indexAnchors: @@ -96,19 +120,21 @@ functions: events: - editor:complete + # Data + indexData: + path: data.ts:indexData + events: + - page:index + # Hashtags indexTags: - path: "./tags.ts:indexTags" + path: tags.ts:indexTags events: - page:index tagComplete: - path: "./tags.ts:tagComplete" + path: tags.ts:tagComplete events: - editor:complete - tagProvider: - path: "./tags.ts:tagProvider" - events: - - query:tag renamePageCommand: path: "./refactor.ts:renamePageCommand" @@ -128,4 +154,18 @@ functions: command: name: "Page: Extract" + # Mentions panel (postscript) + toggleMentions: + path: "./mentions_ps.ts:toggleMentions" + command: + name: "Mentions: Toggle" + key: ctrl-alt-m + updateMentions: + path: "./mentions_ps.ts:updateMentions" + env: client + events: + - plug:load + - editor:pageLoaded + navigateToMention: + path: "./mentions_ps.ts:navigate" diff --git a/plugs/index/item.ts b/plugs/index/item.ts index cd47f54..44c93b3 100644 --- a/plugs/index/item.ts +++ b/plugs/index/item.ts @@ -1,31 +1,28 @@ -import type { IndexTreeEvent, QueryProviderEvent } from "$sb/app_event.ts"; +import type { IndexTreeEvent } from "$sb/app_event.ts"; -import { index } from "$sb/syscalls.ts"; import { collectNodesOfType, ParseTree, renderToText } from "$sb/lib/tree.ts"; -import { applyQuery, removeQueries } from "$sb/lib/query.ts"; +import { removeQueries } from "$sb/lib/query.ts"; import { extractAttributes } from "$sb/lib/attribute.ts"; import { rewritePageRefs } from "$sb/lib/resolve.ts"; -import { indexAttributes } from "./attributes.ts"; +import { ObjectValue } from "$sb/types.ts"; +import { indexObjects } from "./api.ts"; -export type Item = { - name: string; - nested?: string; - tags?: string[]; - // Not stored in DB - page?: string; - pos?: number; -} & Record; +export type ItemObject = ObjectValue< + { + name: string; + page: string; + pos: number; + } & Record +>; export async function indexItems({ name, tree }: IndexTreeEvent) { - const items: { key: string; value: Item }[] = []; + const items: ObjectValue[] = []; removeQueries(tree); // console.log("Indexing items", name); const coll = collectNodesOfType(tree, "ListItem"); - const allAttributes: Record = {}; - for (const n of coll) { if (!n.children) { continue; @@ -35,16 +32,24 @@ export async function indexItems({ name, tree }: IndexTreeEvent) { continue; } - const item: Item = { + const item: ItemObject = { + ref: `${name}@${n.from}`, + tags: [], name: "", // to be replaced + page: name, + pos: n.from!, }; const textNodes: ParseTree[] = []; - let nested: string | undefined; + + collectNodesOfType(n, "Hashtag").forEach((h) => { + // Push tag to the list, removing the initial # + item.tags.push(h.children![0].text!.substring(1)); + }); + for (const child of n.children!.slice(1)) { rewritePageRefs(child, name); if (child.type === "OrderedList" || child.type === "BulletList") { - nested = renderToText(child); break; } // Extract attributes and remove from tree @@ -52,44 +57,17 @@ export async function indexItems({ name, tree }: IndexTreeEvent) { for (const [key, value] of Object.entries(extractedAttributes)) { item[key] = value; - allAttributes[key] = value; } textNodes.push(child); } item.name = textNodes.map(renderToText).join("").trim(); - if (nested) { - item.nested = nested; - } - collectNodesOfType(n, "Hashtag").forEach((h) => { - if (!item.tags) { - item.tags = []; - } - // Push tag to the list, removinn the initial # - item.tags.push(h.children![0].text!.substring(1)); - }); - items.push({ - key: `it:${n.from}`, - value: item, - }); + if (item.tags.length > 0) { + // Only index items with tags + items.push(item); + } } // console.log("Found", items, "item(s)"); - await index.batchSet(name, items); - await indexAttributes(name, allAttributes, "item"); -} - -export async function queryProvider({ - query, -}: QueryProviderEvent): Promise { - const allItems: Item[] = []; - for (const { key, page, value } of await index.queryPrefix("it:")) { - const [, pos] = key.split(":"); - allItems.push({ - ...value, - page: page, - pos: +pos, - }); - } - return applyQuery(query, allItems); + await indexObjects(name, items); } diff --git a/plugs/index/mentions_ps.ts b/plugs/index/mentions_ps.ts new file mode 100644 index 0000000..7c4c55e --- /dev/null +++ b/plugs/index/mentions_ps.ts @@ -0,0 +1,79 @@ +import { asset } from "$sb/plugos-syscall/mod.ts"; +import { clientStore, editor } from "$sb/silverbullet-syscall/mod.ts"; +import { queryObjects } from "./api.ts"; +import { LinkObject } from "./page_links.ts"; + +const hideMentionsKey = "hideMentions"; + +export async function toggleMentions() { + let hideMentions = await clientStore.get(hideMentionsKey); + hideMentions = !hideMentions; + await clientStore.set(hideMentionsKey, hideMentions); + if (!hideMentions) { + const name = await editor.getCurrentPage(); + await renderMentions(name); + } else { + await editor.hidePanel("ps"); + } +} + +// Triggered when switching pages or upon first load +export async function updateMentions() { + if (await clientStore.get(hideMentionsKey)) { + return; + } + const name = await editor.getCurrentPage(); + await renderMentions(name); +} + +// use internal navigation via syscall to prevent reloading the full page. +export async function navigate(ref: string) { + const [page, pos] = ref.split("@"); + await editor.navigate(page, +pos); +} + +function escapeHtml(unsafe: string) { + return unsafe.replace(/&/g, "&").replace(//g, + ">", + ); +} + +async function renderMentions(page: string) { + const linksResult = await queryObjects("link", { + // Query all links that point to this page, excluding those that are inside directives and self pointers. + filter: ["and", ["!=", ["attr", "page"], ["string", page]], ["and", ["=", [ + "attr", + "toPage", + ], ["string", page]], ["=", ["attr", "inDirective"], ["boolean", false]]]], + }); + if (linksResult.length === 0) { + // Don't show the panel if there are no links here. + await editor.hidePanel("ps"); + } else { + const css = await asset.readAsset("asset/style.css"); + const js = await asset.readAsset("asset/linked_mentions.js"); + + await editor.showPanel( + "ps", + 1, + ` + +
+ +
Linked Mentions
+ +
+ `, + js, + ); + } +} diff --git a/plugs/index/page.ts b/plugs/index/page.ts index 9f2f3c1..002dbf2 100644 --- a/plugs/index/page.ts +++ b/plugs/index/page.ts @@ -1,75 +1,37 @@ -import type { IndexEvent, QueryProviderEvent } from "$sb/app_event.ts"; -import { - editor, - events, - index, - markdown, - mq, - space, - system, -} from "$sb/syscalls.ts"; +import type { IndexTreeEvent } from "$sb/app_event.ts"; +import { space } from "$sb/syscalls.ts"; -import { applyQuery } from "$sb/lib/query.ts"; -import type { MQMessage } from "$sb/types.ts"; -import { sleep } from "$sb/lib/async.ts"; +import type { ObjectValue, PageMeta } from "$sb/types.ts"; +import { extractFrontmatter } from "$sb/lib/frontmatter.ts"; +import { extractAttributes } from "$sb/lib/attribute.ts"; +import { indexObjects } from "./api.ts"; -// Key space: -// meta: => metaJson +type PageObject = ObjectValue< + // The base is PageMeta, but we override lastModified to be a string + Omit & { + lastModified: string; // indexing it as a string + } & Record +>; -export async function pageQueryProvider({ - query, -}: QueryProviderEvent): Promise { - return applyQuery(query, await space.listPages()); -} - -export async function reindexCommand() { - await editor.flashNotification("Performing full page reindex..."); - await reindexSpace(); - await editor.flashNotification("Done with page index!"); -} - -export async function reindexSpace() { - console.log("Clearing page index..."); - await index.clearPageIndex(); - // Executed this way to not have to embed the search plug code here - await system.invokeFunction("search.clearIndex"); - const pages = await space.listPages(); - - // Queue all page names to be indexed - await mq.batchSend("indexQueue", pages.map((page) => page.name)); - - // Now let's wait for the processing to finish - let queueStats = await mq.getQueueStats("indexQueue"); - while (queueStats.queued > 0 || queueStats.processing > 0) { - sleep(1000); - queueStats = await mq.getQueueStats("indexQueue"); - } - // And notify the user - console.log("Indexing completed!"); -} - -export async function processIndexQueue(messages: MQMessage[]) { - for (const message of messages) { - const name: string = message.body; - console.log(`Indexing page ${name}`); - const text = await space.readPage(name); - const parsed = await markdown.parseMarkdown(text); - await events.dispatchEvent("page:index", { - name, - tree: parsed, - }); - } -} - -export async function clearPageIndex(page: string) { - // console.log("Clearing page index for page", page); - await index.clearPageIndexForPage(page); -} - -export async function parseIndexTextRepublish({ name, text }: IndexEvent) { - // console.log("Reindexing", name); - await events.dispatchEvent("page:index", { - name, - tree: await markdown.parseMarkdown(text), - }); +export async function indexPage({ name, tree }: IndexTreeEvent) { + const pageMeta = await space.getPageMeta(name); + let pageObj: PageObject = { + ref: name, + tags: [], // will be overridden in a bit + ...pageMeta, + lastModified: new Date(pageMeta.lastModified).toISOString(), + }; + + const frontmatter: Record = await extractFrontmatter(tree); + const toplevelAttributes = await extractAttributes(tree, false); + + // Push them all into the page object + pageObj = { ...pageObj, ...frontmatter, ...toplevelAttributes }; + + pageObj.tags = ["page", ...pageObj.tags || []]; + + // console.log("Page object", pageObj); + + // console.log("Extracted page meta data", pageMeta); + await indexObjects(name, [pageObj]); } diff --git a/plugs/index/page_links.ts b/plugs/index/page_links.ts index a7b9c83..4b38d93 100644 --- a/plugs/index/page_links.ts +++ b/plugs/index/page_links.ts @@ -1,48 +1,53 @@ -import { index } from "$sb/syscalls.ts"; -import { findNodeOfType, traverseTree } from "$sb/lib/tree.ts"; -import { extractFrontmatter } from "$sb/lib/frontmatter.ts"; -import { extractAttributes } from "$sb/lib/attribute.ts"; -import { IndexTreeEvent, QueryProviderEvent } from "$sb/app_event.ts"; -import { applyQuery } from "$sb/lib/query.ts"; +import { findNodeOfType, renderToText, traverseTree } from "$sb/lib/tree.ts"; +import { IndexTreeEvent } from "$sb/app_event.ts"; import { resolvePath } from "$sb/lib/resolve.ts"; -import { indexAttributes } from "./attributes.ts"; +import { indexObjects, queryObjects } from "./api.ts"; +import { ObjectValue } from "$sb/types.ts"; -// Key space: -// l:toPage:pos => {name: pageName, inDirective: true, asTemplate: true} - -export const backlinkPrefix = `l:`; - -export type BacklinkEntry = { - name: string; +export type LinkObject = { + ref: string; + tags: string[]; + // The page the link points to + toPage: string; + // The page the link occurs in + page: string; + pos: number; + snippet: string; alias?: string; - inDirective?: boolean; - asTemplate?: boolean; + inDirective: boolean; + asTemplate: boolean; }; +export function extractSnippet(text: string, pos: number): string { + let prefix = ""; + for (let i = pos - 1; i > 0; i--) { + if (text[i] === "\n") { + break; + } + prefix = text[i] + prefix; + if (prefix.length > 25) { + break; + } + } + let suffix = ""; + for (let i = pos; i < text.length; i++) { + if (text[i] === "\n") { + break; + } + suffix += text[i]; + if (suffix.length > 25) { + break; + } + } + return prefix + suffix; +} + export async function indexLinks({ name, tree }: IndexTreeEvent) { - const backLinks: { key: string; value: BacklinkEntry }[] = []; + const links: ObjectValue[] = []; // [[Style Links]] // console.log("Now indexing links for", name); - const pageMeta = await extractFrontmatter(tree); - const toplevelAttributes = await extractAttributes(tree, false); - if ( - Object.keys(pageMeta).length > 0 || - Object.keys(toplevelAttributes).length > 0 - ) { - for (const [k, v] of Object.entries(toplevelAttributes)) { - pageMeta[k] = v; - } - // Don't index meta data starting with $ - for (const key in pageMeta) { - if (key.startsWith("$")) { - delete pageMeta[key]; - } - } - // console.log("Extracted page meta data", pageMeta); - await index.set(name, "meta:", pageMeta); - } - await indexAttributes(name, pageMeta, "page"); + const pageText = renderToText(tree); let directiveDepth = 0; traverseTree(tree, (n): boolean => { @@ -54,9 +59,16 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) { name, pageRef.children![0].text!.slice(2, -2), ); - backLinks.push({ - key: `${backlinkPrefix}${pageRefName}:${pageRef.from! + 2}`, - value: { name, asTemplate: true }, + const pos = pageRef.from! + 2; + links.push({ + ref: `${name}@${pos}`, + tags: ["link"], + toPage: pageRefName, + pos: pos, + snippet: extractSnippet(pageText, pos), + page: name, + asTemplate: true, + inDirective: false, }); } const directiveText = n.children![0].text; @@ -65,18 +77,23 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) { const match = /\[\[(.+)\]\]/.exec(directiveText); if (match) { const pageRefName = resolvePath(name, match[1]); - backLinks.push({ - key: `${backlinkPrefix}${pageRefName}:${ - n.from! + match.index! + 2 - }`, - value: { name, asTemplate: true }, + const pos = n.from! + match.index! + 2; + links.push({ + ref: `${name}@${pos}`, + tags: ["link"], + toPage: pageRefName, + page: name, + snippet: extractSnippet(pageText, pos), + pos: pos, + asTemplate: true, + inDirective: false, }); } } return true; } - if (n.type === "DirectiveStop") { + if (n.type === "DirectiveEnd") { directiveDepth--; return true; } @@ -85,66 +102,39 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) { const wikiLinkPage = findNodeOfType(n, "WikiLinkPage")!; const wikiLinkAlias = findNodeOfType(n, "WikiLinkAlias"); let toPage = resolvePath(name, wikiLinkPage.children![0].text!); + const pos = wikiLinkPage.from!; if (toPage.includes("@")) { toPage = toPage.split("@")[0]; } - const blEntry: BacklinkEntry = { name }; + const link: LinkObject = { + ref: `${name}@${pos}`, + tags: ["link"], + toPage: toPage, + snippet: extractSnippet(pageText, pos), + pos, + page: name, + inDirective: false, + asTemplate: false, + }; if (directiveDepth > 0) { - blEntry.inDirective = true; + link.inDirective = true; } if (wikiLinkAlias) { - blEntry.alias = wikiLinkAlias.children![0].text!; + link.alias = wikiLinkAlias.children![0].text!; } - backLinks.push({ - key: `${backlinkPrefix}${toPage}:${wikiLinkPage.from}`, - value: blEntry, - }); + links.push(link); return true; } return false; }); // console.log("Found", backLinks, "page link(s)"); - await index.batchSet(name, backLinks); + await indexObjects(name, links); } -export async function linkQueryProvider({ - query, - pageName, -}: QueryProviderEvent): Promise { - const links: any[] = []; - for ( - const { value: blEntry, key } of await index.queryPrefix( - `${backlinkPrefix}${pageName}:`, - ) - ) { - const [, , pos] = key.split(":"); // Key: l:page:pos - if (!blEntry.inDirective) { - blEntry.inDirective = false; - } - if (!blEntry.asTemplate) { - blEntry.asTemplate = false; - } - links.push({ ...blEntry, pos }); - } - return applyQuery(query, links); -} - -type BackLinkPage = { - page: string; - pos: number; -}; - -export async function getBackLinks(pageName: string): Promise { - const allBackLinks = await index.queryPrefix( - `${backlinkPrefix}${pageName}:`, - ); - const pagesToUpdate: BackLinkPage[] = []; - for (const { key, value: { name } } of allBackLinks) { - const keyParts = key.split(":"); - pagesToUpdate.push({ - page: name, - pos: +keyParts[keyParts.length - 1], - }); - } - return pagesToUpdate; +export async function getBackLinks( + pageName: string, +): Promise { + return (await queryObjects("link", { + filter: ["=", ["attr", "toPage"], ["string", pageName]], + })); } diff --git a/plugs/index/plug_api.ts b/plugs/index/plug_api.ts new file mode 100644 index 0000000..4789d76 --- /dev/null +++ b/plugs/index/plug_api.ts @@ -0,0 +1,24 @@ +import { ObjectQuery, ObjectValue } from "$sb/types.ts"; +import { invokeFunction } from "$sb/silverbullet-syscall/system.ts"; + +export function indexObjects( + page: string, + objects: ObjectValue[], +): Promise { + return invokeFunction("index.indexObjects", page, objects); +} + +export function queryObjects( + tag: string, + query: ObjectQuery, +): Promise[]> { + return invokeFunction("index.queryObjects", tag, query); +} + +export function getObjectByRef( + page: string, + tag: string, + ref: string, +): Promise[]> { + return invokeFunction("index.getObjectByRef", page, tag, ref); +} diff --git a/plugs/index/snippet.test.ts b/plugs/index/snippet.test.ts new file mode 100644 index 0000000..8cf1df1 --- /dev/null +++ b/plugs/index/snippet.test.ts @@ -0,0 +1,17 @@ +import { assertEquals } from "../../test_deps.ts"; +import { extractSnippet } from "./page_links.ts"; + +Deno.test("Snippet extraction", () => { + const sample1 = `This is a test +and a [[new]] line that runs super duper duper duper duper duper long +[[SETTINGS]] + super`; + assertEquals( + extractSnippet(sample1, sample1.indexOf("[[new]]")), + "and a [[new]] line that runs sup", + ); + assertEquals( + extractSnippet(sample1, sample1.indexOf("[[SETTINGS]]")), + "[[SETTINGS]]", + ); +}); diff --git a/plugs/index/tags.ts b/plugs/index/tags.ts index 8edb6f8..f4491d3 100644 --- a/plugs/index/tags.ts +++ b/plugs/index/tags.ts @@ -1,67 +1,78 @@ -import { collectNodesOfType } from "$sb/lib/tree.ts"; -import { index } from "$sb/syscalls.ts"; -import type { - CompleteEvent, - IndexTreeEvent, - QueryProviderEvent, -} from "$sb/app_event.ts"; -import { applyQuery, removeQueries } from "$sb/lib/query.ts"; +import type { CompleteEvent, IndexTreeEvent } from "$sb/app_event.ts"; +import { removeQueries } from "$sb/lib/query.ts"; import { extractFrontmatter } from "$sb/lib/frontmatter.ts"; +import { indexObjects, queryObjects } from "./api.ts"; +import { + addParentPointers, + collectNodesOfType, + findParentMatching, +} from "$sb/lib/tree.ts"; -// Key space -// tag:TAG => true (for completion) +export type TagObject = { + ref: string; + tags: string[]; + name: string; + page: string; + parent: string; +}; export async function indexTags({ name, tree }: IndexTreeEvent) { removeQueries(tree); - const allTags = new Set(); - const { tags } = await extractFrontmatter(tree); - if (Array.isArray(tags)) { - tags.forEach((t) => allTags.add(t)); + const tags = new Set(); // name:parent + addParentPointers(tree); + const pageTags: string[] = (await extractFrontmatter(tree)).tags || []; + for (const pageTag of pageTags) { + tags.add(`${pageTag}:page`); } - collectNodesOfType(tree, "Hashtag").forEach((n) => { - allTags.add(n.children![0].text!.substring(1)); + collectNodesOfType(tree, "Hashtag").forEach((h) => { + const tagName = h.children![0].text!.substring(1); + // Check if this occurs in the context of a task + if (findParentMatching(h, (n) => n.type === "Task")) { + tags.add(`${tagName}:task`); + } else if (findParentMatching(h, (n) => n.type === "ListItem")) { + // Or an item + tags.add(`${tagName}:item`); + } }); - await index.batchSet( + // console.log("Indexing these tags", tags); + await indexObjects( name, - [...allTags].map((t) => ({ key: `tag:${t}`, value: t })), + [...tags].map((tag) => { + const [tagName, parent] = tag.split(":"); + return { + ref: tagName, + tags: ["tag"], + name: tagName, + page: name, + parent, + }; + }), ); } +const taskPrefixRegex = /^\s*[\-\*]\s+\[([^\]]+)\]/; +const itemPrefixRegex = /^\s*[\-\*]\s+/; + export async function tagComplete(completeEvent: CompleteEvent) { const match = /#[^#\s]+$/.exec(completeEvent.linePrefix); if (!match) { return null; } const tagPrefix = match[0].substring(1); - const allTags = await index.queryPrefix(`tag:${tagPrefix}`); + let parent = "page"; + if (taskPrefixRegex.test(completeEvent.linePrefix)) { + parent = "task"; + } else if (itemPrefixRegex.test(completeEvent.linePrefix)) { + parent = "item"; + } + const allTags = await queryObjects("tag", { + filter: ["=", ["attr", "parent"], ["string", parent]], + }); return { from: completeEvent.pos - tagPrefix.length, options: allTags.map((tag) => ({ - label: tag.value, + label: tag.name, type: "tag", })), }; } - -type Tag = { - name: string; - freq: number; -}; - -export async function tagProvider({ query }: QueryProviderEvent) { - const allTags = new Map(); - for (const { value } of await index.queryPrefix("tag:")) { - let currentFreq = allTags.get(value); - if (!currentFreq) { - currentFreq = 0; - } - allTags.set(value, currentFreq + 1); - } - return applyQuery( - query, - [...allTags.entries()].map(([name, freq]) => ({ - name, - freq, - })), - ); -} diff --git a/plugs/markdown/markdown_render.test.ts b/plugs/markdown/markdown_render.test.ts index a824a5a..5b06ade 100644 --- a/plugs/markdown/markdown_render.test.ts +++ b/plugs/markdown/markdown_render.test.ts @@ -22,7 +22,7 @@ Deno.test("Markdown render", async () => { new URL("test/example.md", import.meta.url).pathname, ); const tree = parse(lang, testFile); - await renderMarkdownToHtml(tree, { + renderMarkdownToHtml(tree, { failOnUnknown: true, }); // console.log("HTML", html); @@ -34,7 +34,7 @@ Deno.test("Smart hard break test", async () => { *world!*`; const lang = buildMarkdown([]); const tree = parse(lang, example); - const html = await renderMarkdownToHtml(tree, { + const html = renderMarkdownToHtml(tree, { failOnUnknown: true, smartHardBreak: true, }); diff --git a/plugs/markdown/markdown_render.ts b/plugs/markdown/markdown_render.ts index b66a138..992d7f7 100644 --- a/plugs/markdown/markdown_render.ts +++ b/plugs/markdown/markdown_render.ts @@ -1,4 +1,5 @@ import { + collectNodesOfType, findNodeOfType, ParseTree, renderToText, @@ -115,9 +116,13 @@ function render( case "FencedCode": case "CodeBlock": { // Clear out top-level indent blocks + const lang = findNodeOfType(t, "CodeInfo"); t.children = t.children!.filter((c) => c.type); return { name: "pre", + attrs: { + "data-lang": lang ? lang.children![0].text : undefined, + }, body: cleanTags(mapRender(t.children!)), }; } @@ -235,6 +240,7 @@ function render( name: "a", attrs: { href: `/${ref.replace("@", "#")}`, + "data-ref": ref, }, body: linkText, }; @@ -255,12 +261,25 @@ function render( body: t.children![0].text!, }; - case "Task": + case "Task": { + let externalTaskRef = ""; + collectNodesOfType(t, "WikiLinkPage").forEach((wikilink) => { + const ref = wikilink.children![0].text!; + if (!externalTaskRef && ref.includes("@")) { + externalTaskRef = ref; + } + }); + return { name: "span", + attrs: externalTaskRef + ? { + "data-external-task-ref": externalTaskRef, + } + : {}, body: cleanTags(mapRender(t.children!)), }; - + } case "TaskState": { // child[0] = marker, child[1] = state, child[2] = marker const stateText = t.children![1].text!; @@ -269,8 +288,8 @@ function render( name: "input", attrs: { type: "checkbox", - checked: t.children![0].text !== "[ ]" ? "checked" : undefined, - "data-onclick": JSON.stringify(["task", t.to]), + checked: stateText !== " " ? "checked" : undefined, + "data-state": stateText, }, body: "", }; diff --git a/plugs/markdown/widget.ts b/plugs/markdown/widget.ts index 1ebfeed..e2b3b10 100644 --- a/plugs/markdown/widget.ts +++ b/plugs/markdown/widget.ts @@ -12,7 +12,7 @@ export async function markdownWidget( }); return Promise.resolve({ html: html, - script: `updateHeight(); + script: ` document.addEventListener("click", () => { api({type: "blur"}); });`, diff --git a/plugs/query/assets/common.js b/plugs/query/assets/common.js new file mode 100644 index 0000000..28743db --- /dev/null +++ b/plugs/query/assets/common.js @@ -0,0 +1,102 @@ +async function init() { + // Make edit button send the "blur" API call so that the MD code is visible + document.getElementById("edit-button").addEventListener("click", () => { + api({ type: "blur" }); + }); + document.getElementById("reload-button").addEventListener("click", () => { + api({ type: "reload" }); + }); + + document.querySelectorAll("a[data-ref]").forEach((el) => { + el.addEventListener("click", (e) => { + e.preventDefault(); + syscall("editor.navigate", el.dataset.ref); + }); + }); + + // Find all fenced code blocks and replace them with iframes (if a code widget is defined for them) + const allWidgets = document.querySelectorAll("pre[data-lang]"); + for (const widget of allWidgets) { + const lang = widget.getAttribute("data-lang"); + const body = widget.innerText; + + try { + const result = await syscall("widget.render", lang, body); + const iframe = document.createElement("iframe"); + iframe.srcdoc = panelHtml; // set as a global + iframe.onload = () => { + iframe.contentWindow.postMessage({ + type: "html", + theme: document.getElementsByTagName("html")[0].getAttribute( + "data-theme", + ), + ...result, + }, "*"); + }; + widget.parentNode.replaceChild(iframe, widget); + + globalThis.addEventListener("message", (e) => { + if (e.source !== iframe.contentWindow) { + return; + } + const messageData = e.data; + switch (messageData.type) { + case "setHeight": + iframe.style.height = messageData.height + "px"; + // Propagate height setting to parent + updateHeight(); + break; + case "syscall": { + // Intercept syscall messages and send them to the parent + const { id, name, args } = messageData; + syscall(name, ...args).then((result) => { + iframe.contentWindow.postMessage( + { id, type: "syscall-response", result }, + "*", + ); + }).catch((error) => { + iframe.contentWindow.postMessage({ + id, + type: "syscall-response", + error, + }, "*"); + }); + break; + } + default: + // Bubble up any other messages to parent iframe + window.parent.postMessage(messageData, "*"); + } + }); + } catch (e) { + if (e.message.includes("not found")) { + // Not a code widget, ignore + } else { + console.error("Error rendering widget", e); + } + } + } + + // Find all task toggles and propagate their state + document.querySelectorAll("span[data-external-task-ref]").forEach((el) => { + const taskRef = el.dataset.externalTaskRef; + el.querySelector("input[type=checkbox]").addEventListener("change", (e) => { + const oldState = e.target.dataset.state; + const newState = oldState === " " ? "x" : " "; + // Update state in DOM as well for future toggles + e.target.dataset.state = newState; + console.log("Toggling task", taskRef); + syscall( + "system.invokeFunction", + "tasks.updateTaskState", + taskRef, + oldState, + newState, + ).catch( + console.error, + ); + }); + }); +} + +init().catch(console.error); diff --git a/plugs/query/assets/style.css b/plugs/query/assets/style.css new file mode 100644 index 0000000..413a3df --- /dev/null +++ b/plugs/query/assets/style.css @@ -0,0 +1,45 @@ +body { + font-family: var(--editor-font); + background-color: var(--root-background-color); + color: var(--root-color); + overflow: scroll; +} + +ul li p { + margin: 0; +} + +body:hover #button-bar, +body:active #button-bar { + display: block; +} + +#button-bar { + position: absolute; + right: 12px; + top: 3px; + display: none; + background: rgb(255 255 255 / 0.9); + padding: 0 3px; +} + +#button-bar button { + border: none; + background: none; + cursor: pointer; + color: var(--root-color); +} + +#edit-button { + margin-left: -10px; +} + +li code { + font-size: 80%; + color: #a5a4a4; +} + +iframe { + border: none; + width: 100%; +} \ No newline at end of file diff --git a/plugs/query/complete.ts b/plugs/query/complete.ts new file mode 100644 index 0000000..cede24c --- /dev/null +++ b/plugs/query/complete.ts @@ -0,0 +1,80 @@ +import { CompleteEvent } from "$sb/app_event.ts"; +import { events } from "$sb/syscalls.ts"; +import { + AttributeCompleteEvent, + AttributeCompletion, +} from "../index/attributes.ts"; + +export async function queryComplete(completeEvent: CompleteEvent) { + const fencedParent = completeEvent.parentNodes.find((node) => + node === "FencedCode:query" + ); + if (!fencedParent) { + return null; + } + let querySourceMatch = /^\s*([\w\-_]*)$/.exec( + completeEvent.linePrefix, + ); + if (querySourceMatch) { + const allEvents = await events.listEvents(); + + const completionOptions = allEvents + .filter((eventName) => + eventName.startsWith("query:") && !eventName.includes("*") + ) + .map((source) => ({ + label: source.substring("query:".length), + })); + + const allObjectTypes: string[] = (await events.dispatchEvent("query_", {})) + .flat(); + + for (const type of allObjectTypes) { + completionOptions.push({ + label: type, + }); + } + + return { + from: completeEvent.pos - querySourceMatch[1].length, + options: completionOptions, + }; + } + + querySourceMatch = /^\s*([\w\-_]*)/.exec( + completeEvent.linePrefix, + ); + const whereMatch = + /(where|order\s+by|and|or|select(\s+[\w\s,]+)?)\s+([\w\-_]*)$/ + .exec( + completeEvent.linePrefix, + ); + if (querySourceMatch && whereMatch) { + const type = querySourceMatch[1]; + const attributePrefix = whereMatch[3]; + const completions = (await events.dispatchEvent( + `attribute:complete:${type}`, + { + source: type, + prefix: attributePrefix, + } as AttributeCompleteEvent, + )).flat() as AttributeCompletion[]; + return { + from: completeEvent.pos - attributePrefix.length, + options: attributeCompletionsToCMCompletion(completions), + }; + } + return null; +} + +function attributeCompletionsToCMCompletion( + completions: AttributeCompletion[], +) { + return completions.map( + (completion) => ({ + label: completion.name, + detail: `${completion.attributeType} (${completion.source})`, + type: "attribute", + }), + ); +} diff --git a/plugs/query/query.plug.yaml b/plugs/query/query.plug.yaml new file mode 100644 index 0000000..b36cae5 --- /dev/null +++ b/plugs/query/query.plug.yaml @@ -0,0 +1,45 @@ +name: query +assets: + - "assets/*" +functions: + queryWidget: + path: query.ts:widget + codeWidget: query + + templateWidget: + path: template.ts:widget + codeWidget: template + + queryComplete: + path: complete.ts:queryComplete + events: + - editor:complete + + # Slash commands + insertQuery: + redirect: template.insertTemplateText + slashCommand: + name: query + description: Insert a query + value: | + ```query + |^| + ``` + insertInclude: + redirect: template.insertTemplateText + slashCommand: + name: include + description: Include another page + value: | + + + + insertUseTemplate: + redirect: template.insertTemplateText + slashCommand: + name: template + description: Use a template + value: | + ```template + page: "[[|^|]]" + ``` diff --git a/plugs/query/query.ts b/plugs/query/query.ts new file mode 100644 index 0000000..7e9bd41 --- /dev/null +++ b/plugs/query/query.ts @@ -0,0 +1,79 @@ +import type { WidgetContent } from "$sb/app_event.ts"; +import { editor, events, language, markdown, space } from "$sb/syscalls.ts"; +import { parseTreeToAST } from "$sb/lib/tree.ts"; +import { astToKvQuery } from "$sb/lib/parse-query.ts"; +import { jsonToMDTable, renderTemplate } from "../directive/util.ts"; +import { renderMarkdownToHtml } from "../markdown/markdown_render.ts"; +import { replaceTemplateVars } from "../template/template.ts"; +import { prepareJS, wrapHTML } from "./util.ts"; + +export async function widget(bodyText: string): Promise { + const pageMeta = await space.getPageMeta(await editor.getCurrentPage()); + + try { + const queryAST = parseTreeToAST( + await language.parseLanguage("query", bodyText), + ); + const parsedQuery = astToKvQuery( + JSON.parse( + await replaceTemplateVars(JSON.stringify(queryAST[1]), pageMeta), + ), + ); + + // console.log("actual query", parsedQuery); + const eventName = `query:${parsedQuery.querySource}`; + + let resultMarkdown = ""; + + // console.log("Parsed query", parsedQuery); + // Let's dispatch an event and see what happens + const results = await events.dispatchEvent( + eventName, + { query: parsedQuery, pageName: pageMeta.name }, + 30 * 1000, + ); + if (results.length === 0) { + // This means there was no handler for the event which means it's unsupported + return { + html: + `**Error:** Unsupported query source '${parsedQuery.querySource}'`, + }; + } else { + const allResults = results.flat(); + if (allResults.length === 0) { + resultMarkdown = "No results"; + } else { + if (parsedQuery.render) { + // Configured a custom rendering template, let's use it! + const rendered = await renderTemplate( + pageMeta, + parsedQuery.render, + allResults, + ); + resultMarkdown = rendered.trim(); + } else { + // TODO: At this point it's a bit pointless to first render a markdown table, and then convert that to HTML + // We should just render the HTML table directly + resultMarkdown = jsonToMDTable(allResults); + } + } + } + + // Parse markdown to a ParseTree + const mdTree = await markdown.parseMarkdown(resultMarkdown); + // And then render it to HTML + const html = renderMarkdownToHtml(mdTree, { smartHardBreak: true }); + return { + html: await wrapHTML(` + ${parsedQuery.render ? "" : `
`} + ${html} + ${parsedQuery.render ? "" : `
`} + `), + script: await prepareJS(), + }; + } catch (e: any) { + return { + html: await wrapHTML(`Error: ${e.message}`), + }; + } +} diff --git a/plugs/query/template.ts b/plugs/query/template.ts new file mode 100644 index 0000000..c29285d --- /dev/null +++ b/plugs/query/template.ts @@ -0,0 +1,58 @@ +import { WidgetContent } from "$sb/app_event.ts"; +import { editor, handlebars, markdown, space, YAML } from "$sb/syscalls.ts"; +import { renderMarkdownToHtml } from "../markdown/markdown_render.ts"; +import { prepareJS, wrapHTML } from "./util.ts"; + +type TemplateConfig = { + // Pull the template from a page + page?: string; + // Or use a string directly + template?: string; + // Optional argument to pass + value?: any; + // If true, don't render the template, just use it as-is + raw?: boolean; +}; + +export async function widget(bodyText: string): Promise { + const pageMeta = await space.getPageMeta(await editor.getCurrentPage()); + + try { + const config: TemplateConfig = await YAML.parse(bodyText); + let templateText = config.template || ""; + if (config.page) { + let page = config.page; + if (!page) { + throw new Error("Missing `page`"); + } + + if (page.startsWith("[[")) { + page = page.slice(2, -2); + } + templateText = await space.readPage(page); + } + + const rendered = config.raw + ? templateText + : await handlebars.renderTemplate( + templateText, + config.value, + { + page: pageMeta, + }, + ); + const parsedMarkdown = await markdown.parseMarkdown(rendered); + const html = renderMarkdownToHtml(parsedMarkdown, { + smartHardBreak: true, + }); + + return { + html: await wrapHTML(html), + script: await prepareJS(), + }; + } catch (e: any) { + return { + html: `Error: ${e.message}`, + }; + } +} diff --git a/plugs/query/util.ts b/plugs/query/util.ts new file mode 100644 index 0000000..0245dd1 --- /dev/null +++ b/plugs/query/util.ts @@ -0,0 +1,31 @@ +import { asset } from "$sb/syscalls.ts"; +import { panelHtml } from "../../web/components/panel_html.ts"; + +export async function prepareJS() { + const iframeJS = await asset.readAsset("assets/common.js"); + + return ` + const panelHtml = \`${panelHtml}\`; + ${iframeJS} + `; +} + +export async function wrapHTML(html: string): Promise { + const css = await asset.readAsset("assets/style.css"); + + return ` + + + + + +
+ +
+ + +
+ ${html} +
+ `; +} diff --git a/plugs/search/engine.test.ts b/plugs/search/engine.test.ts index 5cf7d18..5c494bd 100644 --- a/plugs/search/engine.test.ts +++ b/plugs/search/engine.test.ts @@ -1,44 +1,42 @@ +import { KV, KvKey } from "$sb/types.ts"; import { assertEquals } from "../../test_deps.ts"; import { BatchKVStore, SimpleSearchEngine } from "./engine.ts"; class InMemoryBatchKVStore implements BatchKVStore { private store = new Map(); - get(keys: string[]): Promise<(any | undefined)[]> { - const results: (any | undefined)[] = keys.map((key) => this.store.get(key)); - return Promise.resolve(results); - } - - queryPrefix(prefix: string): Promise<[string, any][]> { - const results: [string, any][] = []; + query({ prefix }: { prefix: KvKey }): Promise { + const results: KV[] = []; + entries: for (const [key, value] of this.store.entries()) { - if (key.startsWith(prefix)) { - results.push([key, value]); + const parsedKey: string[] = JSON.parse(key); + for (let i = 0; i < prefix.length; i++) { + if (prefix[i] !== parsedKey[i]) { + continue entries; + } } + results.push({ key: parsedKey, value }); } return Promise.resolve(results); } - set(entries: Map): Promise { - for (const [key, value] of entries) { - this.store.set(key, value); + batchSet(kvs: KV[]): Promise { + for (const { key, value } of kvs) { + this.store.set(JSON.stringify(key), value); } return Promise.resolve(); } - delete(keys: string[]): Promise { + batchDel(keys: KvKey[]): Promise { for (const key of keys) { - this.store.delete(key); + this.store.delete(JSON.stringify(key)); } return Promise.resolve(); } } Deno.test("Test full text search", async () => { - const engine = new SimpleSearchEngine( - new InMemoryBatchKVStore(), - new InMemoryBatchKVStore(), - ); + const engine = new SimpleSearchEngine(new InMemoryBatchKVStore()); await engine.indexDocument({ id: "1", text: "The quick brown fox" }); await engine.indexDocument({ id: "2", text: "jumps over the lazy dogs" }); diff --git a/plugs/search/engine.ts b/plugs/search/engine.ts index ce1f276..5c294bc 100644 --- a/plugs/search/engine.ts +++ b/plugs/search/engine.ts @@ -1,4 +1,5 @@ import { stemmer } from "https://esm.sh/porter-stemmer@0.9.1"; +import { KV, KvKey } from "$sb/types.ts"; export type Document = { id: string; @@ -6,10 +7,9 @@ export type Document = { }; export interface BatchKVStore { - get(keys: string[]): Promise<(any | undefined)[]>; - set(entries: Map): Promise; - delete(keys: string[]): Promise; - queryPrefix(prefix: string): Promise<[string, any][]>; + batchSet(kvs: KV[]): Promise; + batchDel(keys: KvKey[]): Promise; + query(options: { prefix: KvKey }): Promise; } type ResultObject = { @@ -22,7 +22,7 @@ export class SimpleSearchEngine { constructor( public index: BatchKVStore, - public reverseIndex: BatchKVStore, + // public reverseIndex: BatchKVStore, ) { } @@ -68,8 +68,16 @@ export class SimpleSearchEngine { // console.log("updateIndexMap", updateIndexMap); - await this.index.set(updateIndexMap); - await this.reverseIndex.set(updateReverseIndexMap); + await this.index.batchSet( + [...updateIndexMap.entries()].map(( + [key, value], + ) => ({ key: ["fts", ...key.split("!")], value: value })), + ); + await this.index.batchSet( + [...updateReverseIndexMap.entries()].map(( + [key, value], + ) => ({ key: ["fts_rev", ...key.split("!")], value: value })), + ); } // Search for a phrase and return document ids sorted by match count @@ -82,9 +90,9 @@ export class SimpleSearchEngine { const matchCounts: Map = new Map(); // pageName -> count for (const stemmedWord of stemmedWords) { - const entries = await this.index.queryPrefix(`${stemmedWord}!`); - for (const [key, value] of entries) { - const id = key.split("!").slice(1).join("!"); + const entries = await this.index.query({ prefix: ["fts", stemmedWord] }); + for (const { key, value } of entries) { + const id = key[2]; if (matchCounts.has(id)) { matchCounts.set(id, matchCounts.get(id)! + value); } else { @@ -102,17 +110,15 @@ export class SimpleSearchEngine { // Delete a document from the index public async deleteDocument(documentId: string): Promise { - const words: [string, boolean][] = await this.reverseIndex.queryPrefix( - `${documentId}!`, - ); - const keysToDelete: string[] = []; - const revKeysToDelete: string[] = []; - for (const [wordKey] of words) { - const word = wordKey.split("!").slice(1).join("!"); - keysToDelete.push(`${word}!${documentId}`); - revKeysToDelete.push(wordKey); + const words = await this.index.query({ + prefix: ["fts_rev", documentId], + }); + const keysToDelete: KvKey[] = []; + for (const { key } of words) { + const word = key[2]; + keysToDelete.push(["fts", word, documentId]); + keysToDelete.push(key); } - await this.index.delete(keysToDelete); - await this.reverseIndex.delete(revKeysToDelete); + await this.index.batchDel(keysToDelete); } } diff --git a/plugs/search/search.ts b/plugs/search/search.ts index b604666..c4bff50 100644 --- a/plugs/search/search.ts +++ b/plugs/search/search.ts @@ -1,41 +1,18 @@ import { IndexTreeEvent, QueryProviderEvent } from "$sb/app_event.ts"; import { renderToText } from "$sb/lib/tree.ts"; -import { applyQuery } from "$sb/lib/query.ts"; -import { editor, index, store } from "$sb/syscalls.ts"; -import { BatchKVStore, SimpleSearchEngine } from "./engine.ts"; -import { FileMeta } from "$sb/types.ts"; +import { + applyQuery, + evalQueryExpression, + liftAttributeFilter, +} from "$sb/lib/query.ts"; +import { datastore, editor } from "$sb/syscalls.ts"; +import { SimpleSearchEngine } from "./engine.ts"; +import { FileMeta, KvKey } from "$sb/types.ts"; import { PromiseQueue } from "$sb/lib/async.ts"; const searchPrefix = "🔍 "; -class StoreKVStore implements BatchKVStore { - constructor(private prefix: string) { - } - async queryPrefix(prefix: string): Promise<[string, any][]> { - const results = await store.queryPrefix(this.prefix + prefix); - return results.map(( - { key, value }, - ) => [key.substring(this.prefix.length), value]); - } - get(keys: string[]): Promise<(string[] | undefined)[]> { - return store.batchGet(keys.map((key) => this.prefix + key)); - } - set(entries: Map): Promise { - return store.batchSet( - Array.from(entries.entries()).map(( - [key, value], - ) => ({ key: this.prefix + key, value })), - ); - } - delete(keys: string[]): Promise { - return store.batchDel(keys.map((key) => this.prefix + key)); - } -} - -const ftsKvStore = new StoreKVStore("fts:"); -const ftsRevKvStore = new StoreKVStore("fts_rev:"); - -const engine = new SimpleSearchEngine(ftsKvStore, ftsRevKvStore); +const engine = new SimpleSearchEngine(datastore); // Search indexing is prone to concurrency issues, so we queue all write operations const promiseQueue = new PromiseQueue(); @@ -50,8 +27,14 @@ export function indexPage({ name, tree }: IndexTreeEvent) { } export async function clearIndex() { - await store.deletePrefix("fts:"); - await store.deletePrefix("fts_rev:"); + const keysToDelete: KvKey[] = []; + for (const { key } of await datastore.query({ prefix: ["fts"] })) { + keysToDelete.push(key); + } + for (const { key } of await datastore.query({ prefix: ["fts_rev"] })) { + keysToDelete.push(key); + } + await datastore.batchDel(keysToDelete); } export function pageUnindex(pageName: string) { @@ -63,11 +46,13 @@ export function pageUnindex(pageName: string) { export async function queryProvider({ query, }: QueryProviderEvent): Promise { - const phraseFilter = query.filter.find((f) => f.prop === "phrase"); + const phraseFilter = liftAttributeFilter(query.filter, "phrase"); if (!phraseFilter) { throw Error("No 'phrase' filter specified, this is mandatory"); } - let results: any[] = await engine.search(phraseFilter.value); + const phrase = evalQueryExpression(phraseFilter, {}); + // console.log("Phrase", phrase); + let results: any[] = await engine.search(phrase); // Patch the object to a format that users expect (translate id to name) for (const r of results) { @@ -75,21 +60,6 @@ export async function queryProvider({ delete r.id; } - const allPageMap: Map = new Map( - results.map((r: any) => [r.name, r]), - ); - for (const { page, value } of await index.queryPrefix("meta:")) { - const p = allPageMap.get(page); - if (p) { - for (const [k, v] of Object.entries(value)) { - p[k] = v; - } - } - } - - // Remove the "phrase" filter - query.filter.splice(query.filter.indexOf(phraseFilter), 1); - results = applyQuery(query, results); return results; } diff --git a/plugs/tasks/complete.ts b/plugs/tasks/complete.ts index d83545c..abf2dd2 100644 --- a/plugs/tasks/complete.ts +++ b/plugs/tasks/complete.ts @@ -1,5 +1,6 @@ import { CompleteEvent } from "$sb/app_event.ts"; -import { index } from "$sb/syscalls.ts"; +import { queryObjects } from "../index/plug_api.ts"; +import { TaskStateObject } from "./task.ts"; export async function completeTaskState(completeEvent: CompleteEvent) { const taskMatch = /([\-\*]\s+\[)([^\[\]]+)$/.exec( @@ -8,8 +9,8 @@ export async function completeTaskState(completeEvent: CompleteEvent) { if (!taskMatch) { return null; } - const allStates = await index.queryPrefix("taskState:"); - const states = [...new Set(allStates.map((s) => s.key.split(":")[1]))]; + const allStates = await queryObjects("taskstate", {}); + const states = [...new Set(allStates.map((s) => s.state))]; return { from: completeEvent.pos - taskMatch[2].length, diff --git a/plugs/tasks/task.ts b/plugs/tasks/task.ts index c0bcdd7..d6154dd 100644 --- a/plugs/tasks/task.ts +++ b/plugs/tasks/task.ts @@ -1,10 +1,6 @@ -import type { - ClickEvent, - IndexTreeEvent, - QueryProviderEvent, -} from "$sb/app_event.ts"; +import type { ClickEvent, IndexTreeEvent } from "$sb/app_event.ts"; -import { editor, index, markdown, space, sync } from "$sb/syscalls.ts"; +import { editor, markdown, space, sync } from "$sb/syscalls.ts"; import { addParentPointers, @@ -17,24 +13,32 @@ import { replaceNodesMatching, traverseTreeAsync, } from "$sb/lib/tree.ts"; -import { applyQuery, removeQueries } from "$sb/lib/query.ts"; +import { removeQueries } from "$sb/lib/query.ts"; import { niceDate } from "$sb/lib/dates.ts"; import { extractAttributes } from "$sb/lib/attribute.ts"; import { rewritePageRefs } from "$sb/lib/resolve.ts"; -import { indexAttributes } from "../index/attributes.ts"; +import { ObjectValue } from "$sb/types.ts"; +import { indexObjects, queryObjects } from "../index/plug_api.ts"; -export type Task = { +export type TaskObject = { + ref: string; + tags: string[]; + page: string; + pos: number; name: string; done: boolean; state: string; deadline?: string; - tags?: string[]; - nested?: string; - // Not saved in DB, just added when pulled out (from key) - pos?: number; - page?: string; } & Record; +export type TaskStateObject = { + ref: string; + tags: string[]; + state: string; + count: number; + page: string; +}; + function getDeadline(deadlineNode: ParseTree): string { return deadlineNode.children![0].text!.replace(/📅\s*/, ""); } @@ -43,27 +47,32 @@ const completeStates = ["x", "X"]; const incompleteStates = [" "]; export async function indexTasks({ name, tree }: IndexTreeEvent) { - const tasks: { key: string; value: Task }[] = []; - const taskStates = new Map(); + const tasks: ObjectValue[] = []; + const taskStates = new Map(); removeQueries(tree); addParentPointers(tree); - const allAttributes: Record = {}; + // const allAttributes: AttributeObject[] = []; + // const allTags = new Set(); await traverseTreeAsync(tree, async (n) => { if (n.type !== "Task") { return false; } const state = n.children![0].children![1].text!; if (!incompleteStates.includes(state) && !completeStates.includes(state)) { - if (!taskStates.has(state)) { - taskStates.set(state, 1); - } else { - taskStates.set(state, taskStates.get(state)! + 1); + let currentState = taskStates.get(state); + if (!currentState) { + currentState = { count: 0, firstPos: n.from! }; } + currentState.count++; } const complete = completeStates.includes(state); - const task: Task = { + const task: TaskObject = { + ref: `${name}@${n.from}`, + tags: [], name: "", done: complete, + page: name, + pos: n.from!, state, }; @@ -76,65 +85,60 @@ export async function indexTasks({ name, tree }: IndexTreeEvent) { return null; } if (tree.type === "Hashtag") { - if (!task.tags) { - task.tags = []; - } // Push the tag to the list, removing the initial # - task.tags.push(tree.children![0].text!.substring(1)); - // Remove this node from the tree - // return null; + const tagName = tree.children![0].text!.substring(1); + task.tags.push(tagName); } }); + task.tags = ["task", ...task.tags]; // Extract attributes and remove from tree const extractedAttributes = await extractAttributes(n, true); for (const [key, value] of Object.entries(extractedAttributes)) { task[key] = value; - allAttributes[key] = value; } task.name = n.children!.slice(1).map(renderToText).join("").trim(); - const taskIndex = n.parent!.children!.indexOf(n); - const nestedItems = n.parent!.children!.slice(taskIndex + 1); - if (nestedItems.length > 0) { - task.nested = nestedItems.map(renderToText).join("").trim(); - } - tasks.push({ - key: `task:${n.from}`, - value: task, - }); + tasks.push(task); return true; }); - // console.log("Found", tasks, "task(s)"); - await index.batchSet(name, tasks); - await indexAttributes(name, allAttributes, "task"); - await index.batchSet( - name, - Array.from(taskStates.entries()).map(([state, count]) => ({ - key: `taskState:${state}`, - value: count, - })), - ); + // Index task states + if (taskStates.size > 0) { + await indexObjects( + name, + Array.from(taskStates.entries()).map(([state, { firstPos, count }]) => ({ + ref: `${name}@${firstPos}`, + tags: ["taskstate"], + state, + count, + page: name, + })), + ); + } + + // Index tasks themselves + if (tasks.length > 0) { + await indexObjects(name, tasks); + } } export function taskToggle(event: ClickEvent) { if (event.altKey) { return; } - return taskCycleAtPos(event.page, event.pos); + return taskCycleAtPos(event.pos); } -export async function previewTaskToggle(eventString: string) { +export function previewTaskToggle(eventString: string) { const [eventName, pos] = JSON.parse(eventString); if (eventName === "task") { - return taskCycleAtPos(await editor.getCurrentPage(), +pos); + return taskCycleAtPos(+pos); } } async function cycleTaskState( - pageName: string, node: ParseTree, ) { const stateText = node.children![1].text!; @@ -145,8 +149,8 @@ async function cycleTaskState( changeTo = "x"; } else { // Not a checkbox, but a custom state - const allStates = await index.queryPrefix("taskState:"); - const states = [...new Set(allStates.map((s) => s.key.split(":")[1]))]; + const allStates = await queryObjects("taskstate", {}); + const states = [...new Set(allStates.map((s) => s.state))]; states.sort(); // Select a next state const currentStateIndex = states.indexOf(stateText); @@ -174,64 +178,73 @@ async function cycleTaskState( for (const wikiLink of parentWikiLinks) { const ref = wikiLink.children![0].text!; if (ref.includes("@")) { - const [page, posS] = ref.split("@"); - const pos = +posS; - if (page === pageName) { - // In current page, just update the task marker with dispatch - const editorText = await editor.getText(); - // Check if the task state marker is still there - const targetText = editorText.substring( - pos + 1, - pos + 1 + stateText.length, - ); - if (targetText !== stateText) { - console.error( - "Reference not a task marker, out of date?", - targetText, - ); - return; - } - await editor.dispatch({ - changes: { - from: pos + 1, - to: pos + 1 + stateText.length, - insert: changeTo, - }, - }); - } else { - let text = await space.readPage(page); - - const referenceMdTree = await markdown.parseMarkdown(text); - // Adding +1 to immediately hit the task state node - const taskStateNode = nodeAtPos(referenceMdTree, pos + 1); - if (!taskStateNode || taskStateNode.type !== "TaskState") { - console.error( - "Reference not a task marker, out of date?", - taskStateNode, - ); - return; - } - taskStateNode.children![1].text = changeTo; - text = renderToText(referenceMdTree); - await space.writePage(page, text); - sync.scheduleFileSync(`${page}.md`); - } + await updateTaskState(ref, stateText, changeTo); } } } -export async function taskCycleAtPos(pageName: string, pos: number) { +export async function updateTaskState( + ref: string, + oldState: string, + newState: string, +) { + const currentPage = await editor.getCurrentPage(); + const [page, posS] = ref.split("@"); + const pos = +posS; + if (page === currentPage) { + // In current page, just update the task marker with dispatch + const editorText = await editor.getText(); + // Check if the task state marker is still there + const targetText = editorText.substring( + pos + 1, + pos + 1 + oldState.length, + ); + if (targetText !== oldState) { + console.error( + "Reference not a task marker, out of date?", + targetText, + ); + return; + } + await editor.dispatch({ + changes: { + from: pos + 1, + to: pos + 1 + oldState.length, + insert: newState, + }, + }); + } else { + let text = await space.readPage(page); + + const referenceMdTree = await markdown.parseMarkdown(text); + // Adding +1 to immediately hit the task state node + const taskStateNode = nodeAtPos(referenceMdTree, pos + 1); + if (!taskStateNode || taskStateNode.type !== "TaskState") { + console.error( + "Reference not a task marker, out of date?", + taskStateNode, + ); + return; + } + taskStateNode.children![1].text = newState; + text = renderToText(referenceMdTree); + await space.writePage(page, text); + sync.scheduleFileSync(`${page}.md`); + } +} + +export async function taskCycleAtPos(pos: number) { const text = await editor.getText(); const mdTree = await markdown.parseMarkdown(text); addParentPointers(mdTree); let node = nodeAtPos(mdTree, pos); if (node) { - if (node.type === "TaskMarker") { + if (node.type === "TaskMark") { node = node.parent!; } if (node.type === "TaskState") { - await cycleTaskState(pageName, node); + await cycleTaskState(node); } } } @@ -265,7 +278,7 @@ export async function taskCycleCommand() { } const taskState = findNodeOfType(taskNode!, "TaskState"); if (taskState) { - await cycleTaskState(await editor.getCurrentPage(), taskState); + await cycleTaskState(taskState); } } @@ -318,19 +331,3 @@ export async function postponeCommand() { }, }); } - -export async function queryProvider({ - query, -}: QueryProviderEvent): Promise { - const allTasks: Task[] = []; - - for (const { key, page, value } of await index.queryPrefix("task:")) { - const pos = key.split(":")[1]; - allTasks.push({ - ...value, - page: page, - pos: +pos, - }); - } - return applyQuery(query, allTasks); -} diff --git a/plugs/tasks/tasks.plug.yaml b/plugs/tasks/tasks.plug.yaml index 91f3cf0..99a5ea2 100644 --- a/plugs/tasks/tasks.plug.yaml +++ b/plugs/tasks/tasks.plug.yaml @@ -19,6 +19,11 @@ syntax: styles: backgroundColor: "rgba(22,22,22,0.07)" functions: + # API + updateTaskState: + path: task.ts:updateTaskState + + turnIntoTask: redirect: template.applyLineReplace slashCommand: @@ -35,10 +40,10 @@ functions: path: "./task.ts:taskToggle" events: - page:click - itemQueryProvider: - path: ./task.ts:queryProvider - events: - - query:task + # itemQueryProvider: + # path: ./task.ts:queryProvider + # events: + # - query:task taskToggleCommand: path: ./task.ts:taskCycleCommand command: diff --git a/plugs/template/template.ts b/plugs/template/template.ts index 978a417..19f9593 100644 --- a/plugs/template/template.ts +++ b/plugs/template/template.ts @@ -1,13 +1,10 @@ -import { editor, markdown, space } from "$sb/syscalls.ts"; +import { editor, handlebars, markdown, space } from "$sb/syscalls.ts"; import { extractFrontmatter } from "$sb/lib/frontmatter.ts"; import { renderToText } from "$sb/lib/tree.ts"; import { niceDate } from "$sb/lib/dates.ts"; import { readSettings } from "$sb/lib/settings_page.ts"; -import { PageMeta } from "../../web/types.ts"; -import { buildHandebarOptions } from "../directive/util.ts"; - -import Handlebars from "handlebars"; import { cleanPageRef } from "$sb/lib/resolve.ts"; +import { PageMeta } from "$sb/types.ts"; export async function instantiateTemplateCommand() { const allPages = await space.listPages(); @@ -48,7 +45,7 @@ export async function instantiateTemplateCommand() { }; if (additionalPageMeta.$name) { - additionalPageMeta.$name = replaceTemplateVars( + additionalPageMeta.$name = await replaceTemplateVars( additionalPageMeta.$name, tempPageMeta, ); @@ -79,7 +76,10 @@ export async function instantiateTemplateCommand() { // The preferred scenario, let's keep going } - const pageText = replaceTemplateVars(renderToText(parseTree), tempPageMeta); + const pageText = await replaceTemplateVars( + renderToText(parseTree), + tempPageMeta, + ); await space.writePage(pageName, pageText); await editor.navigate(pageName); } @@ -110,10 +110,10 @@ export async function insertSnippet() { } const text = await space.readPage(`${snippetPrefix}${selectedSnippet.name}`); - let templateText = replaceTemplateVars(text, pageMeta); + let templateText = await replaceTemplateVars(text, pageMeta); const carretPos = templateText.indexOf("|^|"); templateText = templateText.replace("|^|", ""); - templateText = replaceTemplateVars(templateText, pageMeta); + templateText = await replaceTemplateVars(templateText, pageMeta); await editor.insertAtCursor(templateText); if (carretPos !== -1) { await editor.moveCursor(cursorPos + carretPos); @@ -148,20 +148,21 @@ export async function applyPageTemplateCommand() { const text = await space.readPage( `${pageTemplatePrefix}${selectedPage.name}`, ); - let templateText = replaceTemplateVars(text, pageMeta); + let templateText = await replaceTemplateVars(text, pageMeta); const carretPos = templateText.indexOf("|^|"); templateText = templateText.replace("|^|", ""); - templateText = replaceTemplateVars(templateText, pageMeta); + templateText = await replaceTemplateVars(templateText, pageMeta); await editor.insertAtCursor(templateText); if (carretPos !== -1) { await editor.moveCursor(cursorPos + carretPos); } } -// TODO: This should probably be replaced with handlebards somehow? -export function replaceTemplateVars(s: string, pageMeta: PageMeta): string { - const template = Handlebars.compile(s, { noEscape: true }); - return template({}, buildHandebarOptions(pageMeta)); +export function replaceTemplateVars( + s: string, + pageMeta: PageMeta, +): Promise { + return handlebars.renderTemplate(s, {}, { page: pageMeta }); } export async function quickNoteCommand() { @@ -204,7 +205,7 @@ export async function dailyNoteCommand() { await space.writePage( pageName, - replaceTemplateVars(dailyNoteTemplateText, { + await replaceTemplateVars(dailyNoteTemplateText, { name: pageName, lastModified: 0, perm: "rw", @@ -248,7 +249,7 @@ export async function weeklyNoteCommand() { // Doesn't exist, let's create await space.writePage( pageName, - replaceTemplateVars(weeklyNoteTemplateText, { + await replaceTemplateVars(weeklyNoteTemplateText, { name: pageName, lastModified: 0, perm: "rw", @@ -278,7 +279,7 @@ export async function insertTemplateText(cmdDef: any) { let templateText: string = cmdDef.value; const carretPos = templateText.indexOf("|^|"); templateText = templateText.replace("|^|", ""); - templateText = replaceTemplateVars(templateText, pageMeta); + templateText = await replaceTemplateVars(templateText, pageMeta!); await editor.insertAtCursor(templateText); if (carretPos !== -1) { await editor.moveCursor(cursorPos + carretPos); diff --git a/scripts/generate.sh b/scripts/generate.sh new file mode 100755 index 0000000..24c62e9 --- /dev/null +++ b/scripts/generate.sh @@ -0,0 +1,9 @@ +#!/bin/sh -e + +QUERY_GRAMMAR=common/markdown_parser/query.grammar +EXPRESSION_GRAMMAR=common/markdown_parser/expression.grammar +echo "@top Program { Expression }" > $EXPRESSION_GRAMMAR +tail -n +2 $QUERY_GRAMMAR >> $EXPRESSION_GRAMMAR + +npx lezer-generator $QUERY_GRAMMAR -o common/markdown_parser/parse-query.js +npx lezer-generator $EXPRESSION_GRAMMAR -o common/markdown_parser/parse-expression.js \ No newline at end of file diff --git a/server/auth.ts b/server/auth.ts index ef98ca7..5a2e023 100644 --- a/server/auth.ts +++ b/server/auth.ts @@ -1,4 +1,4 @@ -import { KVStore } from "../plugos/lib/kv_store.ts"; +import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts"; export type User = { username: string; @@ -24,7 +24,7 @@ async function createUser( const userPrefix = `u:`; export class Authenticator { - constructor(private store: KVStore) { + constructor(private store: JSONKVStore) { } async register( diff --git a/server/http_server.ts b/server/http_server.ts index 21175f1..5864896 100644 --- a/server/http_server.ts +++ b/server/http_server.ts @@ -330,8 +330,11 @@ export class HttpServer { } const syscallCommand: SyscallRequest = body; try { - const result = await this.system.localSyscall( - syscallCommand.ctx, + const plug = this.system.loadedPlugs.get(syscallCommand.ctx); + if (!plug) { + throw new Error(`Plug ${syscallCommand.ctx} not found`); + } + const result = await plug.syscall( syscallCommand.name, syscallCommand.args, ); diff --git a/server/server_system.ts b/server/server_system.ts index ae4db73..4981ccf 100644 --- a/server/server_system.ts +++ b/server/server_system.ts @@ -3,33 +3,35 @@ import { SilverBulletHooks } from "../common/manifest.ts"; import { loadMarkdownExtensions } from "../common/markdown_parser/markdown_ext.ts"; import buildMarkdown from "../common/markdown_parser/parser.ts"; import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives.ts"; -import { FileMetaSpacePrimitives } from "../common/spaces/file_meta_space_primitives.ts"; import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts"; import { createSandbox } from "../plugos/environments/webworker_sandbox.ts"; import { CronHook } from "../plugos/hooks/cron.ts"; import { EndpointHook } from "../plugos/hooks/endpoint.ts"; import { EventHook } from "../plugos/hooks/event.ts"; import { MQHook } from "../plugos/hooks/mq.ts"; -import { DenoKVStore } from "../plugos/lib/kv_store.deno_kv.ts"; import assetSyscalls from "../plugos/syscalls/asset.ts"; import { eventSyscalls } from "../plugos/syscalls/event.ts"; -import { mqSyscalls } from "../plugos/syscalls/mq.dexie.ts"; -import { storeSyscalls } from "../plugos/syscalls/store.ts"; +import { mqSyscalls } from "../plugos/syscalls/mq.ts"; import { System } from "../plugos/system.ts"; import { Space } from "../web/space.ts"; import { debugSyscalls } from "../web/syscalls/debug.ts"; -import { pageIndexSyscalls } from "./syscalls/index.ts"; -import { markdownSyscalls } from "../web/syscalls/markdown.ts"; +import { markdownSyscalls } from "../common/syscalls/markdown.ts"; import { spaceSyscalls } from "./syscalls/space.ts"; import { systemSyscalls } from "../web/syscalls/system.ts"; -import { yamlSyscalls } from "../web/syscalls/yaml.ts"; +import { yamlSyscalls } from "../common/syscalls/yaml.ts"; import { Application } from "./deps.ts"; import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts"; import { shellSyscalls } from "../plugos/syscalls/shell.deno.ts"; import { SpacePrimitives } from "../common/spaces/space_primitives.ts"; -import { DenoKvMQ } from "../plugos/lib/mq.deno_kv.ts"; import { base64EncodedDataUrl } from "../plugos/asset_bundle/base64.ts"; import { Plug } from "../plugos/plug.ts"; +import { DenoKvPrimitives } from "../plugos/lib/deno_kv_primitives.ts"; +import { DataStore } from "../plugos/lib/datastore.ts"; +import { dataStoreSyscalls } from "../plugos/syscalls/datastore.ts"; +import { DataStoreMQ } from "../plugos/lib/mq.datastore.ts"; +import { language } from "@codemirror/language"; +import { languageSyscalls } from "../common/syscalls/language.ts"; +import { handlebarsSyscalls } from "../common/syscalls/handlebars.ts"; const fileListInterval = 30 * 1000; // 30s @@ -37,8 +39,8 @@ export class ServerSystem { system: System = new System("server"); spacePrimitives!: SpacePrimitives; denoKv!: Deno.Kv; - kvStore!: DenoKVStore; listInterval?: number; + ds!: DataStore; constructor( private baseSpacePrimitives: SpacePrimitives, @@ -58,33 +60,26 @@ export class ServerSystem { this.system.addHook(cronHook); this.denoKv = await Deno.openKv(this.dbPath); - - this.kvStore = new DenoKVStore(this.denoKv); + this.ds = new DataStore(new DenoKvPrimitives(this.denoKv)); // Endpoint hook this.system.addHook(new EndpointHook(this.app, "/_/")); - // Use DexieMQ for this, in memory - const mq = new DenoKvMQ(this.denoKv); - - const pageIndexCalls = pageIndexSyscalls(this.kvStore); + const mq = new DataStoreMQ(this.ds); const plugNamespaceHook = new PlugNamespaceHook(); this.system.addHook(plugNamespaceHook); this.system.addHook(new MQHook(this.system, mq)); - this.spacePrimitives = new FileMetaSpacePrimitives( - new EventedSpacePrimitives( - new PlugSpacePrimitives( - this.baseSpacePrimitives, - plugNamespaceHook, - ), - eventHook, + this.spacePrimitives = new EventedSpacePrimitives( + new PlugSpacePrimitives( + this.baseSpacePrimitives, + plugNamespaceHook, ), - pageIndexCalls, + eventHook, ); - const space = new Space(this.spacePrimitives, this.kvStore, eventHook); + const space = new Space(this.spacePrimitives, this.ds, eventHook); // Add syscalls this.system.registerSyscalls( @@ -93,10 +88,11 @@ export class ServerSystem { spaceSyscalls(space), assetSyscalls(this.system), yamlSyscalls(), - storeSyscalls(this.kvStore), systemSyscalls(this.system), mqSyscalls(mq), - pageIndexCalls, + languageSyscalls(), + handlebarsSyscalls(), + dataStoreSyscalls(this.ds), debugSyscalls(), markdownSyscalls(buildMarkdown([])), // Will later be replaced with markdown extensions ); @@ -147,15 +143,17 @@ export class ServerSystem { }); // Check if this space was ever indexed before - if (!await this.kvStore.has("$initialIndexCompleted")) { + if (!await this.ds.get(["$initialIndexDone"])) { console.log("Indexing space for the first time (in the background)"); this.system.loadedPlugs.get("index")!.invoke( "reindexSpace", [], ).then(() => { - this.kvStore.set("$initialIndexCompleted", true); + this.ds.set(["$initialIndexDone"], true); }).catch(console.error); } + + await eventHook.dispatchEvent("system:ready"); } async loadPlugs() { diff --git a/server/syscalls/index.test.ts b/server/syscalls/index.test.ts deleted file mode 100644 index e59570e..0000000 --- a/server/syscalls/index.test.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { DenoKVStore } from "../../plugos/lib/kv_store.deno_kv.ts"; -import { assertEquals } from "../../test_deps.ts"; -import { pageIndexSyscalls } from "./index.ts"; - -Deno.test("Test KV index", async () => { - const ctx: any = {}; - const denoKv = await Deno.openKv("test.db"); - const kv = new DenoKVStore(denoKv); - const calls = pageIndexSyscalls(kv); - await calls["index.set"](ctx, "page", "test", "value"); - assertEquals(await calls["index.get"](ctx, "page", "test"), "value"); - await calls["index.delete"](ctx, "page", "test"); - assertEquals(await calls["index.get"](ctx, "page", "test"), null); - await calls["index.batchSet"](ctx, "page", [{ - key: "attr:test", - value: "value", - }, { - key: "attr:test2", - value: "value2", - }, { key: "random", value: "value3" }]); - await calls["index.batchSet"](ctx, "page2", [{ - key: "attr:test", - value: "value", - }, { - key: "attr:test2", - value: "value2", - }, { key: "random", value: "value3" }]); - let results = await calls["index.queryPrefix"](ctx, "attr:"); - assertEquals(results.length, 4); - await calls["index.clearPageIndexForPage"](ctx, "page"); - results = await calls["index.queryPrefix"](ctx, "attr:"); - assertEquals(results.length, 2); - await calls["index.clearPageIndex"](ctx); - results = await calls["index.queryPrefix"](ctx, ""); - assertEquals(results.length, 0); - denoKv.close(); - await Deno.remove("test.db"); -}); diff --git a/server/syscalls/index.ts b/server/syscalls/index.ts deleted file mode 100644 index 2cb9839..0000000 --- a/server/syscalls/index.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { KVStore } from "../../plugos/lib/kv_store.ts"; -import type { SysCallMapping } from "../../plugos/system.ts"; - -export type KV = { - key: string; - value: any; -}; - -// Keyspace: -// ["index", page, key] -> value -// ["indexByKey", key, page] -> value - -const sep = "!"; - -/** - * Implements the index syscalls using Deno's KV store. - * @param dbFile - * @returns - */ -export function pageIndexSyscalls(kv: KVStore): SysCallMapping { - const apiObj: SysCallMapping = { - "index.set": (_ctx, page: string, key: string, value: any) => { - return kv.batchSet( - [{ - key: `index${sep}${page}${sep}${key}`, - value, - }, { - key: `indexByKey${sep}${key}${sep}${page}`, - value, - }], - ); - }, - "index.batchSet": (_ctx, page: string, kvs: KV[]) => { - const batch: KV[] = []; - for (const { key, value } of kvs) { - batch.push({ - key: `index${sep}${page}${sep}${key}`, - value, - }, { - key: `indexByKey${sep}${key}${sep}${page}`, - value, - }); - } - return kv.batchSet(batch); - }, - "index.delete": (_ctx, page: string, key: string) => { - return kv.batchDelete([ - `index${sep}${page}${sep}${key}`, - `indexByKey${sep}${key}${sep}${page}`, - ]); - }, - "index.get": (_ctx, page: string, key: string) => { - return kv.get(`index${sep}${page}${sep}${key}`); - }, - "index.queryPrefix": async (_ctx, prefix: string) => { - const results: { key: string; page: string; value: any }[] = []; - for ( - const result of await kv.queryPrefix(`indexByKey!${prefix}`) - ) { - const [_ns, key, page] = result.key.split(sep); - results.push({ - key, - page, - value: result.value, - }); - } - return results; - }, - "index.clearPageIndexForPage": async (ctx, page: string) => { - await apiObj["index.deletePrefixForPage"](ctx, page, ""); - }, - "index.deletePrefixForPage": async (_ctx, page: string, prefix: string) => { - const allKeys: string[] = []; - for ( - const result of await kv.queryPrefix( - `index${sep}${page}${sep}${prefix}`, - ) - ) { - const [_ns, page, key] = result.key.split(sep); - allKeys.push( - `index${sep}${page}${sep}${key}`, - `indexByKey${sep}${key}${sep}${page}`, - ); - } - return kv.batchDelete(allKeys); - }, - "index.clearPageIndex": async () => { - const allKeys: string[] = []; - for (const result of await kv.queryPrefix(`index${sep}`)) { - const [_ns, page, key] = result.key.split(sep); - allKeys.push( - `index${sep}${page}${sep}${key}`, - `indexByKey${sep}${key}${sep}${page}`, - ); - } - return kv.batchDelete(allKeys); - }, - }; - return apiObj; -} diff --git a/server/syscalls/space.ts b/server/syscalls/space.ts index 56eec28..0ad7df6 100644 --- a/server/syscalls/space.ts +++ b/server/syscalls/space.ts @@ -1,7 +1,6 @@ -import { FileMeta } from "$sb/types.ts"; +import { AttachmentMeta, FileMeta, PageMeta } from "$sb/types.ts"; import { SysCallMapping } from "../../plugos/system.ts"; import type { Space } from "../../web/space.ts"; -import { AttachmentMeta, PageMeta } from "../../web/types.ts"; /** * Almost the same as web/syscalls/space.ts except leaving out client-specific stuff diff --git a/web/boot.ts b/web/boot.ts index 721040d..e15f684 100644 --- a/web/boot.ts +++ b/web/boot.ts @@ -1,5 +1,4 @@ import { safeRun } from "../common/util.ts"; -import { IndexedDBKvPrimitives } from "../plugos/lib/indexeddb_kv_primitives.ts"; import { Client } from "./client.ts"; const syncMode = window.silverBulletConfig.syncOnly || diff --git a/web/client.ts b/web/client.ts index 73f2d4f..a1b8f75 100644 --- a/web/client.ts +++ b/web/client.ts @@ -4,10 +4,11 @@ import { CompletionResult, EditorView, gitIgnoreCompiler, + SyntaxNode, syntaxTree, } from "../common/deps.ts"; import { fileMetaToPageMeta, Space } from "./space.ts"; -import { FilterOption, PageMeta } from "./types.ts"; +import { FilterOption } from "./types.ts"; import { parseYamlSettings } from "../common/util.ts"; import { EventHook } from "../plugos/hooks/event.ts"; import { AppCommand } from "./hooks/command.ts"; @@ -18,8 +19,6 @@ import { AppViewState, BuiltinSettings } from "./types.ts"; import type { AppEvent, CompleteEvent } from "../plug-api/app_event.ts"; import { throttle } from "$sb/lib/async.ts"; import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts"; -import { IndexedDBSpacePrimitives } from "../common/spaces/indexeddb_space_primitives.ts"; -import { FileMetaSpacePrimitives } from "../common/spaces/file_meta_space_primitives.ts"; import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives.ts"; import { ISyncService, @@ -28,7 +27,6 @@ import { SyncService, } from "./sync_service.ts"; import { simpleHash } from "../common/crypto.ts"; -import { DexieKVStore } from "../plugos/lib/kv_store.dexie.ts"; import { SyncStatus } from "../common/spaces/sync.ts"; import { HttpSpacePrimitives } from "../common/spaces/http_space_primitives.ts"; import { FallbackSpacePrimitives } from "../common/spaces/fallback_space_primitives.ts"; @@ -38,11 +36,14 @@ import { ClientSystem } from "./client_system.ts"; import { createEditorState } from "./editor_state.ts"; import { OpenPages } from "./open_pages.ts"; import { MainUI } from "./editor_ui.tsx"; -import { DexieMQ } from "../plugos/lib/mq.dexie.ts"; import { cleanPageRef } from "$sb/lib/resolve.ts"; import { expandPropertyNames } from "$sb/lib/json.ts"; import { SpacePrimitives } from "../common/spaces/space_primitives.ts"; -import { FileMeta } from "$sb/types.ts"; +import { FileMeta, PageMeta } from "$sb/types.ts"; +import { DataStore } from "../plugos/lib/datastore.ts"; +import { IndexedDBKvPrimitives } from "../plugos/lib/indexeddb_kv_primitives.ts"; +import { DataStoreMQ } from "../plugos/lib/mq.datastore.ts"; +import { DataStoreSpacePrimitives } from "../common/spaces/datastore_space_primitives.ts"; const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/; const autoSaveInterval = 1000; @@ -60,8 +61,8 @@ declare global { // TODO: Oh my god, need to refactor this export class Client { - system: ClientSystem; - editorView: EditorView; + system!: ClientSystem; + editorView!: EditorView; private pageNavigator!: PathPageNavigator; private dbPrefix: string; @@ -78,22 +79,34 @@ export class Client { .catch((e) => console.error("Error dispatching editor:updated event", e)); }, 1000); + debouncedPlugsUpdatedEvent = throttle(async () => { + // To register new commands, update editor state based on new plugs + this.rebuildEditorState(); + await this.dispatchAppEvent( + "editor:pageLoaded", + this.currentPage, + undefined, + true, + ); + }, 1000); + // Track if plugs have been updated since sync cycle fullSyncCompleted = false; - syncService: ISyncService; + syncService!: ISyncService; settings!: BuiltinSettings; - kvStore: DexieKVStore; - mq: DexieMQ; // Event bus used to communicate between components - eventHook: EventHook; + eventHook!: EventHook; - ui: MainUI; - openPages: OpenPages; + ui!: MainUI; + openPages!: OpenPages; + stateDataStore!: DataStore; + spaceDataStore!: DataStore; + mq!: DataStoreMQ; constructor( - parent: Element, + private parent: Element, public syncMode = false, ) { if (!syncMode) { @@ -101,15 +114,21 @@ export class Client { } // Generate a semi-unique prefix for the database so not to reuse databases for different space paths this.dbPrefix = "" + simpleHash(window.silverBulletConfig.spaceFolderPath); + } - this.kvStore = new DexieKVStore( - `${this.dbPrefix}_store`, - "data", - globalThis.indexedDB, - globalThis.IDBKeyRange, + /** + * Initialize the client + * This is a separated from the constructor to allow for async initialization + */ + async init() { + const stateKvPrimitives = new IndexedDBKvPrimitives( + `${this.dbPrefix}_state`, ); + await stateKvPrimitives.init(); + this.stateDataStore = new DataStore(stateKvPrimitives); - this.mq = new DexieMQ(`${this.dbPrefix}_mq`, indexedDB, IDBKeyRange); + // Setup message queue + this.mq = new DataStoreMQ(this.stateDataStore); setInterval(() => { // Timeout after 5s, retries 3 times, otherwise drops the message (no DLQ) @@ -122,19 +141,18 @@ export class Client { // Instantiate a PlugOS system this.system = new ClientSystem( this, - this.kvStore, this.mq, - this.dbPrefix, + this.stateDataStore, this.eventHook, ); - const localSpacePrimitives = this.initSpace(); + const localSpacePrimitives = await this.initSpace(); this.syncService = this.syncMode ? new SyncService( localSpacePrimitives, this.plugSpaceRemotePrimitives, - this.kvStore, + this.stateDataStore, this.eventHook, (path) => { // TODO: At some point we should remove the data.db exception here @@ -148,7 +166,7 @@ export class Client { : new NoSyncSyncService(this.space); this.ui = new MainUI(this); - this.ui.render(parent); + this.ui.render(this.parent); this.editorView = new EditorView({ state: createEditorState(this, "", "", false), @@ -160,13 +178,8 @@ export class Client { this.focus(); // This constructor will always be followed by an (async) invocatition of init() - } + await this.system.init(); - /** - * Initialize the client - * This is a separated from the constructor to allow for async initialization - */ - async init() { // Load settings this.settings = await this.loadSettings(); @@ -193,7 +206,6 @@ export class Client { await this.dispatchAppEvent("editor:init"); setInterval(() => { - // console.log("Syncing page", this.currentPage, "in background"); try { this.syncService.syncFile(`${this.currentPage!}.md`).catch((e: any) => { console.error("Interval sync error", e); @@ -201,7 +213,6 @@ export class Client { } catch (e: any) { console.error("Interval sync error", e); } - // console.log("End of kick-off of background sync of", this.currentPage); }, pageSyncInterval); } @@ -218,23 +229,12 @@ export class Client { // "sync:success" is called with a number of operations only from syncSpace(), not from syncing individual pages this.fullSyncCompleted = true; } - if (this.system.plugsUpdated) { - // To register new commands, update editor state based on new plugs - this.rebuildEditorState(); - this.dispatchAppEvent( - "editor:pageLoaded", - this.currentPage, - undefined, - true, - ); - if (operations) { - // Likely initial sync so let's show visually that we're synced now - // this.flashNotification(`Synced ${operations} files`, "info"); - this.showProgress(100); - } + // if (this.system.plugsUpdated) { + if (operations) { + // Likely initial sync so let's show visually that we're synced now + this.showProgress(100); } - // Reset for next sync cycle - this.system.plugsUpdated = false; + // } this.ui.viewDispatch({ type: "sync-change", syncSuccess: true }); }); @@ -276,28 +276,27 @@ export class Client { if (typeof pos === "string") { console.log("Navigating to anchor", pos); - // We're going to look up the anchor through a direct page store query... - // TODO: This should be extracted - const posLookup = await this.system.localSyscall( - "index.get", - [ - pageName, - `a:${pageName}:${pos}`, - ], + // We're going to look up the anchor through a API invocation + const matchingAnchor = await this.system.system.localSyscall( + "index", + "system.invokeFunction", + ["getObjectByRef", pageName, "anchor", `${pageName}@${pos}`], ); - if (!posLookup) { + if (!matchingAnchor) { return this.flashNotification( - `Could not find anchor @${pos}`, + `Could not find anchor $${pos}`, "error", ); } else { - pos = +posLookup; + pos = matchingAnchor.pos as number; } } - this.editorView.dispatch({ - selection: { anchor: pos }, - effects: EditorView.scrollIntoView(pos, { y: "start" }), + setTimeout(() => { + this.editorView.dispatch({ + selection: { anchor: pos as number }, + effects: EditorView.scrollIntoView(pos as number, { y: "start" }), + }); }); } else if (!stateRestored) { // Somewhat ad-hoc way to determine if the document contains frontmatter and if so, putting the cursor _after it_. @@ -318,13 +317,16 @@ export class Client { scrollIntoView: true, }); } - await this.kvStore.set("lastOpenedPage", pageName); + await this.stateDataStore.set(["client", "lastOpenedPage"], pageName); }); if (location.hash === "#boot") { (async () => { // Cold start PWA load - const lastPage = await this.kvStore.get("lastOpenedPage"); + const lastPage = await this.stateDataStore.get([ + "client", + "lastOpenedPage", + ]); if (lastPage) { await this.navigate(lastPage); } @@ -332,7 +334,7 @@ export class Client { } } - initSpace(): SpacePrimitives { + async initSpace(): Promise { this.remoteSpacePrimitives = new HttpSpacePrimitives( location.origin, window.silverBulletConfig.spaceFolderPath, @@ -348,20 +350,20 @@ export class Client { let localSpacePrimitives: SpacePrimitives | undefined; if (this.syncMode) { + // We'll store the space files in a separate data store + const spaceKvPrimitives = new IndexedDBKvPrimitives( + `${this.dbPrefix}_synced_space`, + ); + await spaceKvPrimitives.init(); + localSpacePrimitives = new FilteredSpacePrimitives( - new FileMetaSpacePrimitives( - new EventedSpacePrimitives( - // Using fallback space primitives here to allow (by default) local reads to "fall through" to HTTP when files aren't synced yet - new FallbackSpacePrimitives( - new IndexedDBSpacePrimitives( - `${this.dbPrefix}_space`, - globalThis.indexedDB, - ), - this.plugSpaceRemotePrimitives, - ), - this.eventHook, + new EventedSpacePrimitives( + // Using fallback space primitives here to allow (by default) local reads to "fall through" to HTTP when files aren't synced yet + new FallbackSpacePrimitives( + new DataStoreSpacePrimitives(new DataStore(spaceKvPrimitives)), + this.plugSpaceRemotePrimitives, ), - this.system.indexSyscalls, + this.eventHook, ), (meta) => fileFilterFn(meta.name), // Run when a list of files has been retrieved @@ -381,7 +383,11 @@ export class Client { ); } - this.space = new Space(localSpacePrimitives, this.kvStore, this.eventHook); + this.space = new Space( + localSpacePrimitives, + this.stateDataStore, + this.eventHook, + ); this.eventHook.addLocalListener("file:changed", (path: string) => { // Only reload when watching the current page (to avoid reloading when switching pages) @@ -585,6 +591,7 @@ export class Client { async loadPlugs() { await this.system.reloadPlugsFromSpace(this.space); this.rebuildEditorState(); + await this.eventHook.dispatchEvent("system:ready"); await this.dispatchAppEvent("plugs:loaded"); } @@ -627,13 +634,20 @@ export class Client { const linePrefix = line.text.slice(0, selection.from - line.from); const parentNodes: string[] = []; - const currentNode = syntaxTree(editorState).resolveInner(selection.from); + const sTree = syntaxTree(editorState); + const currentNode = sTree.resolveInner(selection.from); if (currentNode) { - let node = currentNode; - while (node.parent) { - parentNodes.push(node.parent.name); + let node: SyntaxNode | null = currentNode; + do { + if (node.name === "FencedCode") { + const code = editorState.sliceDoc(node.from + 3, node.to); + const fencedCodeLanguage = code.split("\n")[0]; + parentNodes.push(`FencedCode:${fencedCodeLanguage}`); + } else { + parentNodes.push(node.name); + } node = node.parent; - } + } while (node); } const results = await this.dispatchAppEvent(eventName, { @@ -742,9 +756,6 @@ export class Client { let doc; try { doc = await this.space.readPage(pageName); - if (doc.meta.contentType.startsWith("text/html")) { - throw new Error("Got HTML page, not markdown"); - } } catch (e: any) { if (e.message.includes("Not found")) { // Not found, new page diff --git a/web/client_system.ts b/web/client_system.ts index fb58be2..aa3f3e7 100644 --- a/web/client_system.ts +++ b/web/client_system.ts @@ -3,13 +3,11 @@ import { Manifest, SilverBulletHooks } from "../common/manifest.ts"; import buildMarkdown from "../common/markdown_parser/parser.ts"; import { CronHook } from "../plugos/hooks/cron.ts"; import { EventHook } from "../plugos/hooks/event.ts"; -import { DexieKVStore } from "../plugos/lib/kv_store.dexie.ts"; import { createSandbox } from "../plugos/environments/webworker_sandbox.ts"; import assetSyscalls from "../plugos/syscalls/asset.ts"; import { eventSyscalls } from "../plugos/syscalls/event.ts"; -import { storeSyscalls } from "../plugos/syscalls/store.ts"; -import { SysCallMapping, System } from "../plugos/system.ts"; +import { System } from "../plugos/system.ts"; import type { Client } from "./client.ts"; import { CodeWidgetHook } from "./hooks/code_widget.ts"; import { CommandHook } from "./hooks/command.ts"; @@ -18,40 +16,41 @@ import { clientStoreSyscalls } from "./syscalls/clientStore.ts"; import { debugSyscalls } from "./syscalls/debug.ts"; import { editorSyscalls } from "./syscalls/editor.ts"; import { sandboxFetchSyscalls } from "./syscalls/fetch.ts"; -import { pageIndexSyscalls } from "./syscalls/index.ts"; -import { markdownSyscalls } from "./syscalls/markdown.ts"; +import { markdownSyscalls } from "../common/syscalls/markdown.ts"; import { shellSyscalls } from "./syscalls/shell.ts"; import { spaceSyscalls } from "./syscalls/space.ts"; import { syncSyscalls } from "./syscalls/sync.ts"; import { systemSyscalls } from "./syscalls/system.ts"; -import { yamlSyscalls } from "./syscalls/yaml.ts"; +import { yamlSyscalls } from "../common/syscalls/yaml.ts"; import { Space } from "./space.ts"; import { loadMarkdownExtensions, MDExt, } from "../common/markdown_parser/markdown_ext.ts"; -import { DexieMQ } from "../plugos/lib/mq.dexie.ts"; import { MQHook } from "../plugos/hooks/mq.ts"; -import { mqSyscalls } from "../plugos/syscalls/mq.dexie.ts"; -import { indexProxySyscalls } from "./syscalls/index.proxy.ts"; -import { storeProxySyscalls } from "./syscalls/store.proxy.ts"; +import { mqSyscalls } from "../plugos/syscalls/mq.ts"; import { mqProxySyscalls } from "./syscalls/mq.proxy.ts"; +import { dataStoreProxySyscalls } from "./syscalls/datastore.proxy.ts"; +import { dataStoreSyscalls } from "../plugos/syscalls/datastore.ts"; +import { DataStore } from "../plugos/lib/datastore.ts"; +import { MessageQueue } from "../plugos/lib/mq.ts"; +import { languageSyscalls } from "../common/syscalls/language.ts"; +import { handlebarsSyscalls } from "../common/syscalls/handlebars.ts"; +import { widgetSyscalls } from "./syscalls/widget.ts"; export class ClientSystem { commandHook: CommandHook; slashCommandHook: SlashCommandHook; namespaceHook: PlugNamespaceHook; - indexSyscalls: SysCallMapping; codeWidgetHook: CodeWidgetHook; - plugsUpdated = false; mdExtensions: MDExt[] = []; system: System; constructor( private client: Client, - private kvStore: DexieKVStore, - private mq: DexieMQ, - dbPrefix: string, + private mq: MessageQueue, + private ds: DataStore, + // private dbPrefix: string, private eventHook: EventHook, ) { // Only set environment to "client" when running in thin client mode, otherwise we run everything locally (hybrid) @@ -67,18 +66,6 @@ export class ClientSystem { const cronHook = new CronHook(this.system); this.system.addHook(cronHook); - if (!client.syncMode) { - // In non-sync mode, proxy these to the server - this.indexSyscalls = indexProxySyscalls(client); - } else { - // In sync mode, run them locally - this.indexSyscalls = pageIndexSyscalls( - `${dbPrefix}_page_index`, - globalThis.indexedDB, - globalThis.IDBKeyRange, - ); - } - // Code widget hook this.codeWidgetHook = new CodeWidgetHook(); this.system.addHook(this.codeWidgetHook); @@ -93,7 +80,7 @@ export class ClientSystem { this.commandHook = new CommandHook(); this.commandHook.on({ commandsUpdated: (commandMap) => { - this.client.ui.viewDispatch({ + this.client.ui?.viewDispatch({ type: "update-commands", commands: commandMap, }); @@ -118,7 +105,7 @@ export class ClientSystem { // If there are syntax extensions, rebuild the markdown parser immediately this.updateMarkdownParser(); } - this.plugsUpdated = true; + this.client.debouncedPlugsUpdatedEvent(); } }); @@ -138,17 +125,9 @@ export class ClientSystem { // this.eventHook.addLocalListener("file:deleted", (file) => { // console.log("File deleted", file); // }); - - this.registerSyscalls(); } - registerSyscalls() { - const storeCalls = this.client.syncMode - // In sync mode handle locally - ? storeSyscalls(this.kvStore) - // In non-sync mode proxy to server - : storeProxySyscalls(this.client); - + async init() { // Slash command hook this.slashCommandHook = new SlashCommandHook(this.client); this.system.addHook(this.slashCommandHook); @@ -163,16 +142,20 @@ export class ClientSystem { markdownSyscalls(buildMarkdown(this.mdExtensions)), assetSyscalls(this.system), yamlSyscalls(), + handlebarsSyscalls(), + widgetSyscalls(this.client), + languageSyscalls(), this.client.syncMode // In sync mode handle locally ? mqSyscalls(this.mq) // In non-sync mode proxy to server : mqProxySyscalls(this.client), - storeCalls, - this.indexSyscalls, + this.client.syncMode + ? dataStoreSyscalls(this.ds) + : dataStoreProxySyscalls(this.client), debugSyscalls(), syncSyscalls(this.client), - clientStoreSyscalls(this.kvStore), + clientStoreSyscalls(this.ds), ); // Syscalls that require some additional permissions diff --git a/web/cm_plugins/directive.ts b/web/cm_plugins/directive.ts index 960dc69..a619cc8 100644 --- a/web/cm_plugins/directive.ts +++ b/web/cm_plugins/directive.ts @@ -19,11 +19,11 @@ export function directivePlugin() { return; } - const cursorInRange = isCursorInRange(state, [from, to]); + const cursorInRange = isCursorInRange(state, [parent.from, parent.to]); if (type.name === "DirectiveStart") { if (cursorInRange) { - // Cursor outside this directive + // Cursor inside this directive widgets.push( Decoration.line({ class: "sb-directive-start" }).range(from), ); diff --git a/web/cm_plugins/fenced_code.ts b/web/cm_plugins/fenced_code.ts index 8fcbba1..fae524b 100644 --- a/web/cm_plugins/fenced_code.ts +++ b/web/cm_plugins/fenced_code.ts @@ -1,5 +1,4 @@ import { WidgetContent } from "../../plug-api/app_event.ts"; -import { panelHtml } from "../components/panel.tsx"; import { Decoration, EditorState, syntaxTree, WidgetType } from "../deps.ts"; import type { Client } from "../client.ts"; import { CodeWidgetCallback } from "../hooks/code_widget.ts"; @@ -8,8 +7,11 @@ import { invisibleDecoration, isCursorInRange, } from "./util.ts"; +import { createWidgetSandboxIFrame } from "../components/widget_sandbox_iframe.ts"; class IFrameWidget extends WidgetType { + iframe?: HTMLIFrameElement; + constructor( readonly from: number, readonly to: number, @@ -21,71 +23,44 @@ class IFrameWidget extends WidgetType { } toDOM(): HTMLElement { - const iframe = document.createElement("iframe"); - iframe.srcdoc = panelHtml; - // iframe.style.height = "0"; - - const messageListener = (evt: any) => { - if (evt.source !== iframe.contentWindow) { - return; - } - const data = evt.data; - if (!data) { - return; - } - switch (data.type) { - case "event": - this.editor.dispatchAppEvent(data.name, ...data.args); - break; - case "setHeight": - iframe.style.height = data.height + "px"; - break; - case "setBody": - this.editor.editorView.dispatch({ - changes: { - from: this.from, - to: this.to, - insert: data.body, - }, - }); - break; - case "blur": - this.editor.editorView.dispatch({ - selection: { anchor: this.from }, - }); - this.editor.focus(); - break; - } - }; - - iframe.onload = () => { - // Subscribe to message event on global object (to receive messages from iframe) - globalThis.addEventListener("message", messageListener); - // Only run this code once - iframe.onload = null; - this.codeWidgetCallback(this.bodyText).then( - (widgetContent: WidgetContent) => { - if (widgetContent.html) { - iframe.contentWindow!.postMessage({ - type: "html", - html: widgetContent.html, - script: widgetContent.script, + const iframe = createWidgetSandboxIFrame( + this.editor, + this.bodyText, + this.codeWidgetCallback(this.bodyText), + (message) => { + switch (message.type) { + case "blur": + this.editor.editorView.dispatch({ + selection: { anchor: this.from }, }); - } else if (widgetContent.url) { - iframe.contentWindow!.location.href = widgetContent.url; - if (widgetContent.height) { - iframe.style.height = widgetContent.height + "px"; - } - if (widgetContent.width) { - iframe.style.width = widgetContent.width + "px"; - } - } - }, - ); - }; + this.editor.focus(); + + break; + case "reload": + this.codeWidgetCallback(this.bodyText).then( + (widgetContent: WidgetContent) => { + iframe.contentWindow!.postMessage({ + type: "html", + html: widgetContent.html, + script: widgetContent.script, + theme: document.getElementsByTagName("html")[0].dataset.theme, + }); + }, + ); + break; + } + }, + ); + return iframe; } + get estimatedHeight(): number { + const cachedHeight = this.editor.space.getCachedWidgetHeight(this.bodyText); + // console.log("Calling estimated height", cachedHeight); + return cachedHeight || 150; + } + eq(other: WidgetType): boolean { return ( other instanceof IFrameWidget && diff --git a/web/cm_plugins/post_script.ts b/web/cm_plugins/post_script.ts new file mode 100644 index 0000000..3ac9d3d --- /dev/null +++ b/web/cm_plugins/post_script.ts @@ -0,0 +1,46 @@ +import { Decoration, EditorState, WidgetType } from "../deps.ts"; +import type { Client } from "../client.ts"; +import { decoratorStateField } from "./util.ts"; +import { PanelConfig } from "../types.ts"; +import { createWidgetSandboxIFrame } from "../components/widget_sandbox_iframe.ts"; + +class IFrameWidget extends WidgetType { + constructor( + readonly editor: Client, + readonly panel: PanelConfig, + ) { + super(); + } + + toDOM(): HTMLElement { + const iframe = createWidgetSandboxIFrame(this.editor, null, this.panel); + iframe.classList.add("sb-ps-iframe"); + return iframe; + } + + eq(other: WidgetType): boolean { + return this.panel.html === + (other as IFrameWidget).panel.html && + this.panel.script === + (other as IFrameWidget).panel.script; + } +} + +export function postScriptPlugin(editor: Client) { + return decoratorStateField((state: EditorState) => { + const widgets: any[] = []; + if (editor.ui.viewState.panels.ps.html) { + widgets.push( + Decoration.widget({ + widget: new IFrameWidget( + editor, + editor.ui.viewState.panels.ps, + ), + side: 1, + block: true, + }).range(state.doc.length), + ); + } + return Decoration.set(widgets); + }); +} diff --git a/web/components/page_navigator.tsx b/web/components/page_navigator.tsx index bb13d08..85d7d5b 100644 --- a/web/components/page_navigator.tsx +++ b/web/components/page_navigator.tsx @@ -1,6 +1,7 @@ import { FilterList } from "./filter.tsx"; -import { FilterOption, PageMeta } from "../types.ts"; +import { FilterOption } from "../types.ts"; import { CompletionContext, CompletionResult } from "../deps.ts"; +import { PageMeta } from "$sb/types.ts"; export function PageNavigator({ allPages, @@ -51,7 +52,7 @@ export function PageNavigator({ darkMode={darkMode} completer={completer} allowNew={true} - helpText="Start typing the page name to filter results, press Return to open." + helpText="Press Enter to open the selected page, or Shift-Enter to create a new page." newHint="Create page" completePrefix={completePrefix} onSelect={(opt) => { diff --git a/web/components/panel.tsx b/web/components/panel.tsx index b3403a2..e1b495e 100644 --- a/web/components/panel.tsx +++ b/web/components/panel.tsx @@ -1,97 +1,7 @@ import { useEffect, useRef } from "../deps.ts"; import { Client } from "../client.ts"; import { PanelConfig } from "../types.ts"; - -export const panelHtml = ` - - - - - - - -Loading... - -`; +import { panelHtml } from "./panel_html.ts"; export function Panel({ config, diff --git a/web/components/panel_html.ts b/web/components/panel_html.ts new file mode 100644 index 0000000..44a32c9 --- /dev/null +++ b/web/components/panel_html.ts @@ -0,0 +1,103 @@ +export const panelHtml = ` + + + + + + + + + +`; diff --git a/web/components/widget_sandbox_iframe.ts b/web/components/widget_sandbox_iframe.ts new file mode 100644 index 0000000..423ffb1 --- /dev/null +++ b/web/components/widget_sandbox_iframe.ts @@ -0,0 +1,96 @@ +import { WidgetContent } from "$sb/app_event.ts"; +import { Client } from "../client.ts"; +import { panelHtml } from "./panel_html.ts"; + +export function createWidgetSandboxIFrame( + client: Client, + widgetHeightCacheKey: string | null, + content: WidgetContent | Promise, + onMessage?: (message: any) => void, +) { + const iframe = document.createElement("iframe"); + iframe.srcdoc = panelHtml; + // iframe.style.height = "150px"; + + const messageListener = (evt: any) => { + (async () => { + if (evt.source !== iframe.contentWindow) { + return; + } + const data = evt.data; + if (!data) { + return; + } + switch (data.type) { + case "syscall": { + const { id, name, args } = data; + try { + const result = await client.system.localSyscall(name, args); + if (!iframe.contentWindow) { + // iFrame already went away + return; + } + iframe.contentWindow!.postMessage({ + type: "syscall-response", + id, + result, + }); + } catch (e: any) { + if (!iframe.contentWindow) { + // iFrame already went away + return; + } + iframe.contentWindow!.postMessage({ + type: "syscall-response", + id, + error: e.message, + }); + } + break; + } + case "setHeight": + iframe.style.height = data.height + "px"; + if (widgetHeightCacheKey) { + client.space.setCachedWidgetHeight( + widgetHeightCacheKey, + data.height, + ); + } + break; + default: + if (onMessage) { + onMessage(data); + } + } + })().catch((e) => { + console.error("Message listener error", e); + }); + }; + + iframe.onload = () => { + // Subscribe to message event on global object (to receive messages from iframe) + globalThis.addEventListener("message", messageListener); + // Only run this code once + iframe.onload = null; + Promise.resolve(content).then((content) => { + if (content.html) { + iframe.contentWindow!.postMessage({ + type: "html", + html: content.html, + script: content.script, + theme: document.getElementsByTagName("html")[0].dataset.theme, + }); + } else if (content.url) { + iframe.contentWindow!.location.href = content.url; + if (content.height) { + iframe.style.height = content.height + "px"; + } + if (content.width) { + iframe.style.width = content.width + "px"; + } + } + }).catch(console.error); + }; + + return iframe; +} diff --git a/web/editor_state.ts b/web/editor_state.ts index 3fcd4a5..9f10de8 100644 --- a/web/editor_state.ts +++ b/web/editor_state.ts @@ -3,14 +3,10 @@ import { readonlyMode } from "./cm_plugins/readonly.ts"; import customMarkdownStyle from "./style.ts"; import { autocompletion, - cLanguage, closeBrackets, closeBracketsKeymap, codeFolding, completionKeymap, - cppLanguage, - csharpLanguage, - dartLanguage, drawSelection, dropCursor, EditorState, @@ -18,37 +14,18 @@ import { highlightSpecialChars, history, historyKeymap, - htmlLanguage, indentOnInput, indentWithTab, - javaLanguage, - javascriptLanguage, - jsonLanguage, KeyBinding, keymap, - kotlinLanguage, LanguageDescription, LanguageSupport, markdown, - objectiveCLanguage, - objectiveCppLanguage, - postgresqlLanguage, - protobufLanguage, - pythonLanguage, - rustLanguage, - scalaLanguage, searchKeymap, - shellLanguage, - sqlLanguage, standardKeymap, - StreamLanguage, syntaxHighlighting, - tomlLanguage, - typescriptLanguage, ViewPlugin, ViewUpdate, - xmlLanguage, - yamlLanguage, } from "../common/deps.ts"; import { Client } from "./client.ts"; import { vim } from "./deps.ts"; @@ -63,6 +40,8 @@ import { pasteLinkExtension, } from "./cm_plugins/editor_paste.ts"; import { TextChange } from "$sb/lib/change.ts"; +import { postScriptPlugin } from "./cm_plugins/post_script.ts"; +import { languageFor } from "../common/languages.ts"; export function createEditorState( editor: Client, @@ -124,143 +103,17 @@ export function createEditorState( // The uber markdown mode markdown({ base: markdownLanguage, - codeLanguages: [ - LanguageDescription.of({ - name: "yaml", - alias: ["meta", "data", "embed"], - support: new LanguageSupport(StreamLanguage.define(yamlLanguage)), - }), - LanguageDescription.of({ - name: "javascript", - alias: ["js"], - support: new LanguageSupport(javascriptLanguage), - }), - LanguageDescription.of({ - name: "typescript", - alias: ["ts"], - support: new LanguageSupport(typescriptLanguage), - }), - LanguageDescription.of({ - name: "sql", - alias: ["sql"], - support: new LanguageSupport(StreamLanguage.define(sqlLanguage)), - }), - LanguageDescription.of({ - name: "postgresql", - alias: ["pgsql", "postgres"], - support: new LanguageSupport( - StreamLanguage.define(postgresqlLanguage), - ), - }), - LanguageDescription.of({ - name: "rust", - alias: ["rs"], - support: new LanguageSupport(StreamLanguage.define(rustLanguage)), - }), - LanguageDescription.of({ - name: "css", - support: new LanguageSupport(StreamLanguage.define(sqlLanguage)), - }), - LanguageDescription.of({ - name: "html", - support: new LanguageSupport(htmlLanguage), - }), - LanguageDescription.of({ - name: "python", - alias: ["py"], - support: new LanguageSupport( - StreamLanguage.define(pythonLanguage), - ), - }), - LanguageDescription.of({ - name: "protobuf", - alias: ["proto"], - support: new LanguageSupport( - StreamLanguage.define(protobufLanguage), - ), - }), - LanguageDescription.of({ - name: "shell", - alias: ["sh", "bash", "zsh", "fish"], - support: new LanguageSupport( - StreamLanguage.define(shellLanguage), - ), - }), - LanguageDescription.of({ - name: "swift", - support: new LanguageSupport(StreamLanguage.define(rustLanguage)), - }), - LanguageDescription.of({ - name: "toml", - support: new LanguageSupport(StreamLanguage.define(tomlLanguage)), - }), - LanguageDescription.of({ - name: "json", - support: new LanguageSupport(StreamLanguage.define(jsonLanguage)), - }), - LanguageDescription.of({ - name: "xml", - support: new LanguageSupport(StreamLanguage.define(xmlLanguage)), - }), - LanguageDescription.of({ - name: "c", - support: new LanguageSupport(StreamLanguage.define(cLanguage)), - }), - LanguageDescription.of({ - name: "cpp", - alias: ["c++", "cxx"], - support: new LanguageSupport(StreamLanguage.define(cppLanguage)), - }), - LanguageDescription.of({ - name: "java", - support: new LanguageSupport(StreamLanguage.define(javaLanguage)), - }), - LanguageDescription.of({ - name: "csharp", - alias: ["c#", "cs"], - support: new LanguageSupport( - StreamLanguage.define(csharpLanguage), - ), - }), - LanguageDescription.of({ - name: "scala", - alias: ["sc"], - support: new LanguageSupport( - StreamLanguage.define(scalaLanguage), - ), - }), - LanguageDescription.of({ - name: "kotlin", - alias: ["kt", "kts"], - support: new LanguageSupport( - StreamLanguage.define(kotlinLanguage), - ), - }), - LanguageDescription.of({ - name: "objc", - alias: ["objective-c", "objectivec"], - support: new LanguageSupport( - StreamLanguage.define(objectiveCLanguage), - ), - }), - LanguageDescription.of({ - name: "objcpp", - alias: [ - "objc++", - "objective-cpp", - "objectivecpp", - "objective-c++", - "objectivec++", - ], - support: new LanguageSupport( - StreamLanguage.define(objectiveCppLanguage), - ), - }), - LanguageDescription.of({ - name: "dart", - support: new LanguageSupport(StreamLanguage.define(dartLanguage)), - }), - ], + codeLanguages: (info) => { + const lang = languageFor(info); + if (lang) { + return LanguageDescription.of({ + name: info, + support: new LanguageSupport(lang), + }); + } + + return null; + }, addKeymap: true, }), markdownLanguage.data.of({ @@ -286,6 +139,7 @@ export function createEditorState( indentOnInput(), ...cleanModePlugins(editor), EditorView.lineWrapping, + postScriptPlugin(editor), lineWrapper([ { selector: "ATXHeading1", class: "sb-line-h1" }, { selector: "ATXHeading2", class: "sb-line-h2" }, diff --git a/web/service_worker.ts b/web/service_worker.ts index c4942bd..1ba636d 100644 --- a/web/service_worker.ts +++ b/web/service_worker.ts @@ -1,8 +1,7 @@ -import Dexie from "https://esm.sh/v120/dexie@3.2.2/dist/dexie.js"; - -import type { FileContent } from "../common/spaces/indexeddb_space_primitives.ts"; +import type { FileContent } from "../common/spaces/datastore_space_primitives.ts"; import { simpleHash } from "../common/crypto.ts"; -import { FileMeta } from "$sb/types.ts"; +import { DataStore } from "../plugos/lib/datastore.ts"; +import { IndexedDBKvPrimitives } from "../plugos/lib/indexeddb_kv_primitives.ts"; const CACHE_NAME = "{{CACHE_NAME}}"; @@ -61,9 +60,9 @@ self.addEventListener("activate", (event: any) => { ); }); -let db: Dexie | undefined; -let fileContentTable: Dexie.Table | undefined; -let fileMetatable: Dexie.Table | undefined; +let ds: DataStore | undefined; +const filesMetaPrefix = ["file", "meta"]; +const filesContentPrefix = ["file", "content"]; self.addEventListener("fetch", (event: any) => { const url = new URL(event.request.url); @@ -89,7 +88,7 @@ self.addEventListener("fetch", (event: any) => { return cachedResponse; } - if (!fileContentTable) { + if (!ds) { // Not initialzed yet, or in thin client mode, let's just proxy return fetch(request); } @@ -124,18 +123,14 @@ async function handleLocalFileRequest( request: Request, pathname: string, ): Promise { - if (!db?.isOpen()) { - console.log("Detected that the DB was closed, reopening"); - await db!.open(); - } + // if (!db?.isOpen()) { + // console.log("Detected that the DB was closed, reopening"); + // await db!.open(); + // } const path = decodeURIComponent(pathname.slice(1)); - const data = await fileContentTable!.get(path); + const data = await ds?.get([...filesContentPrefix, path]); if (data) { // console.log("Serving from space", path); - if (!data.meta) { - // Legacy database not fully synced yet - data.meta = (await fileMetatable!.get(path))!; - } return new Response( data.data, { @@ -177,7 +172,7 @@ self.addEventListener("message", (event: any) => { caches.delete(CACHE_NAME) .then(() => { console.log("[Service worker]", "Cache deleted"); - db?.close(); + // ds?.close(); event.source.postMessage({ type: "cacheFlushed" }); }); } @@ -186,15 +181,10 @@ self.addEventListener("message", (event: any) => { const dbPrefix = "" + simpleHash(spaceFolderPath); // Setup space - db = new Dexie(`${dbPrefix}_space`, { - indexedDB: globalThis.indexedDB, + const kv = new IndexedDBKvPrimitives(`${dbPrefix}_synced_space`); + kv.init().then(() => { + ds = new DataStore(kv); + console.log("Datastore in service worker initialized..."); }); - db.version(1).stores({ - fileMeta: "name", - fileContent: "name", - }); - - fileContentTable = db.table("fileContent"); - fileMetatable = db.table("fileMeta"); } }); diff --git a/web/space.ts b/web/space.ts index 24312bb..c203a4f 100644 --- a/web/space.ts +++ b/web/space.ts @@ -1,33 +1,49 @@ import { SpacePrimitives } from "../common/spaces/space_primitives.ts"; import { plugPrefix } from "../common/spaces/constants.ts"; import { safeRun } from "../common/util.ts"; -import { AttachmentMeta, PageMeta } from "./types.ts"; -import { KVStore } from "../plugos/lib/kv_store.ts"; -import { FileMeta } from "$sb/types.ts"; +import { AttachmentMeta, FileMeta, PageMeta } from "$sb/types.ts"; import { EventHook } from "../plugos/hooks/event.ts"; import { throttle } from "$sb/lib/async.ts"; +import { DataStore } from "../plugos/lib/datastore.ts"; +import { LimitedMap } from "../common/limited_map.ts"; const pageWatchInterval = 5000; export class Space { - imageHeightCache: Record = {}; - // pageMetaCache = new Map(); + imageHeightCache = new LimitedMap(100); // url -> height + widgetHeightCache = new LimitedMap(100); // bodytext -> height cachedPageList: PageMeta[] = []; - debouncedCacheFlush = throttle(() => { - this.kvStore.set("imageHeightCache", this.imageHeightCache).catch( + debouncedImageCacheFlush = throttle(() => { + this.ds.set(["cache", "imageHeight"], this.imageHeightCache).catch( console.error, ); console.log("Flushed image height cache to store"); }, 5000); setCachedImageHeight(url: string, height: number) { - this.imageHeightCache[url] = height; - this.debouncedCacheFlush(); + this.imageHeightCache.set(url, height); + this.debouncedImageCacheFlush(); } getCachedImageHeight(url: string): number { - return this.imageHeightCache[url] ?? -1; + return this.imageHeightCache.get(url) ?? -1; + } + + debouncedWidgetCacheFlush = throttle(() => { + this.ds.set(["cache", "widgetHeight"], this.widgetHeightCache.toJSON()) + .catch( + console.error, + ); + console.log("Flushed widget height cache to store"); + }, 5000); + + setCachedWidgetHeight(bodyText: string, height: number) { + this.widgetHeightCache.set(bodyText, height); + this.debouncedWidgetCacheFlush(); + } + getCachedWidgetHeight(bodyText: string): number { + return this.widgetHeightCache.get(bodyText) ?? -1; } // We do watch files in the background to detect changes @@ -40,16 +56,20 @@ export class Space { constructor( readonly spacePrimitives: SpacePrimitives, - private kvStore: KVStore, + private ds: DataStore, private eventHook: EventHook, ) { // super(); - this.kvStore.get("imageHeightCache").then((cache) => { - if (cache) { - // console.log("Loaded image height cache from KV store", cache); - this.imageHeightCache = cache; - } - }); + this.ds.batchGet([["cache", "imageHeight"], ["cache", "widgetHeight"]]) + .then(([imageCache, widgetCache]) => { + if (imageCache) { + this.imageHeightCache = new LimitedMap(100, imageCache); + } + if (widgetCache) { + // console.log("Loaded widget cache from store", widgetCache); + this.widgetHeightCache = new LimitedMap(100, widgetCache); + } + }); eventHook.addLocalListener("file:listed", (files: FileMeta[]) => { this.cachedPageList = files.filter(this.isListedPage).map( fileMetaToPageMeta, diff --git a/web/style.ts b/web/style.ts index d104828..396d93c 100644 --- a/web/style.ts +++ b/web/style.ts @@ -21,7 +21,7 @@ export default function highlightStyles(mdExtension: MDExt[]) { { tag: ct.AttributeTag, class: "sb-frontmatter" }, { tag: ct.AttributeNameTag, class: "sb-atom" }, { tag: ct.TaskTag, class: "sb-task" }, - { tag: ct.TaskMarkerTag, class: "sb-task-marker" }, + { tag: ct.TaskMarkTag, class: "sb-task-mark" }, { tag: ct.TaskStateTag, class: "sb-task-state" }, { tag: ct.CodeInfoTag, class: "sb-code-info" }, { tag: ct.CommentTag, class: "sb-comment" }, diff --git a/web/styles/colors.scss b/web/styles/colors.scss index 876cac0..385f961 100644 --- a/web/styles/colors.scss +++ b/web/styles/colors.scss @@ -378,7 +378,7 @@ color: var(--editor-wiki-link-page-color); // #8f96c2; } - .sb-task-marker { + .sb-task-mark { color: var(--editor-task-marker-color); } diff --git a/web/styles/editor.scss b/web/styles/editor.scss index 5c7ea45..8e4df62 100644 --- a/web/styles/editor.scss +++ b/web/styles/editor.scss @@ -308,7 +308,7 @@ cursor: pointer; } - .sb-task-marker { + .sb-task-mark { font-size: 91%; } @@ -448,9 +448,11 @@ iframe { border: 0; width: 100%; + max-width: 100%; padding: 0; margin: 0; - max-width: 100%; + border: 1px solid var(--editor-directive-background-color); + border-radius: 5px; } } diff --git a/web/styles/main.scss b/web/styles/main.scss index 5ed6cbe..658cc1d 100644 --- a/web/styles/main.scss +++ b/web/styles/main.scss @@ -74,10 +74,8 @@ body { } .sb-notifications { - position: fixed; - bottom: 0; - left: 5px; - right: 5px; + float: right; + margin-top: 8px; font-size: 15px; z-index: 100; @@ -87,66 +85,74 @@ body { border-radius: 5px; } } + } - #sb-current-page { - flex: 1; + #sb-current-page { + flex: 1; - overflow: hidden; - white-space: nowrap; - text-align: left; - display: block; + overflow: hidden; + white-space: nowrap; + text-align: left; + display: block; - .cm-scroller { - font-family: var(--ui-font); - } + .cm-scroller { + font-family: var(--ui-font); + } - .cm-content { + .cm-content { + padding: 0; + + .cm-line { padding: 0; - - .cm-line { - padding: 0; - } } } } - - .sb-actions { - text-align: right; - position: absolute; - right: 15px; - top: 0; - } - - .progress-wrapper { - display: inline-block; - position: relative; - top: -6px; - padding: 4px; - background-color: var(--top-background-color); - margin-right: -2px; - } - - .progress-bar { - display: flex; - justify-content: center; - align-items: center; - - width: 20px; - height: 20px; - border-radius: 50%; - font-size: 6px; - } - - // .progress-bar::before { - // content: "66%"; - // } } - .sb-panel { - flex: 1; + .sb-actions { + text-align: right; + position: absolute; + right: 15px; + top: 0; } + + .progress-wrapper { + display: inline-block; + position: relative; + top: -6px; + padding: 4px; + background-color: var(--top-background-color); + margin-right: -2px; + } + + .progress-bar { + display: flex; + justify-content: center; + align-items: center; + + width: 20px; + height: 20px; + border-radius: 50%; + font-size: 6px; + } + + // .progress-bar::before { + // content: "66%"; + // } } +.sb-panel { + flex: 1; +} + +.sb-ps-iframe { + width: 100%; + margin-top: 10px; + border: 1px solid var(--editor-directive-background-color); + border-radius: 5px; +} + + #sb-main { display: flex; flex-direction: row; diff --git a/web/sync_service.ts b/web/sync_service.ts index 2425b0e..1749f30 100644 --- a/web/sync_service.ts +++ b/web/sync_service.ts @@ -6,22 +6,22 @@ import { SyncStatusItem, } from "../common/spaces/sync.ts"; import { EventHook } from "../plugos/hooks/event.ts"; -import { KVStore } from "../plugos/lib/kv_store.ts"; +import { DataStore } from "../plugos/lib/datastore.ts"; import { Space } from "./space.ts"; // Keeps the current sync snapshot -const syncSnapshotKey = "syncSnapshot"; +const syncSnapshotKey = ["sync", "snapshot"]; // Keeps the start time of an ongoing sync, is reset once the sync is done -const syncStartTimeKey = "syncStartTime"; +const syncStartTimeKey = ["sync", "startTime"]; // Keeps the start time of the last full sync cycle -const syncLastFullCycleKey = "syncLastFullCycle"; +const syncLastFullCycleKey = ["sync", "lastFullCycle"]; // Keeps the last time an activity was registered, used to detect if a sync is still alive and whether a new one should be started already -const syncLastActivityKey = "syncLastActivity"; +const syncLastActivityKey = ["sync", "lastActivity"]; -const syncInitialFullSyncCompletedKey = "syncInitialFullSyncCompleted"; +const syncInitialFullSyncCompletedKey = ["sync", "initialFullSyncCompleted"]; // maximum time between two activities before we consider a sync crashed const syncMaxIdleTimeout = 1000 * 27; @@ -53,7 +53,7 @@ export class SyncService implements ISyncService { constructor( readonly localSpacePrimitives: SpacePrimitives, readonly remoteSpace: SpacePrimitives, - private kvStore: KVStore, + private ds: DataStore, private eventHook: EventHook, private isSyncCandidate: (path: string) => boolean, ) { @@ -91,30 +91,30 @@ export class SyncService implements ISyncService { } async isSyncing(): Promise { - const startTime = await this.kvStore.get(syncStartTimeKey); + const startTime = await this.ds.get(syncStartTimeKey); if (!startTime) { return false; } // Sync is running, but is it still alive? - const lastActivity = await this.kvStore.get(syncLastActivityKey)!; + const lastActivity = await this.ds.get(syncLastActivityKey)!; if (Date.now() - lastActivity > syncMaxIdleTimeout) { // It's been too long since the last activity, let's consider this one crashed and // reset the sync start state - await this.kvStore.del(syncStartTimeKey); + await this.ds.delete(syncStartTimeKey); console.info("Sync without activity for too long, resetting"); return false; } return true; } - hasInitialSyncCompleted(): Promise { + async hasInitialSyncCompleted(): Promise { // Initial sync has happened when sync progress has been reported at least once, but the syncStartTime has been reset (which happens after sync finishes) - return this.kvStore.has(syncInitialFullSyncCompletedKey); + return !!(await this.ds.get(syncInitialFullSyncCompletedKey)); } async registerSyncStart(fullSync: boolean): Promise { // Assumption: this is called after an isSyncing() check - await this.kvStore.batchSet([ + await this.ds.batchSet([ { key: syncStartTimeKey, value: Date.now(), @@ -135,23 +135,23 @@ export class SyncService implements ISyncService { async registerSyncProgress(status?: SyncStatus): Promise { // Emit a sync event at most every 2s if (status && this.lastReportedSyncStatus < Date.now() - 2000) { - this.eventHook.dispatchEvent("sync:progress", status); + await this.eventHook.dispatchEvent("sync:progress", status); this.lastReportedSyncStatus = Date.now(); await this.saveSnapshot(status.snapshot); } - await this.kvStore.set(syncLastActivityKey, Date.now()); + await this.ds.set(syncLastActivityKey, Date.now()); } async registerSyncStop(isFullSync: boolean): Promise { await this.registerSyncProgress(); - await this.kvStore.del(syncStartTimeKey); + await this.ds.delete(syncStartTimeKey); if (isFullSync) { - await this.kvStore.set(syncInitialFullSyncCompletedKey, true); + await this.ds.set(syncInitialFullSyncCompletedKey, true); } } async getSnapshot(): Promise> { - const snapshot = (await this.kvStore.get(syncSnapshotKey)) || {}; + const snapshot = (await this.ds.get(syncSnapshotKey)) || {}; return new Map( Object.entries(snapshot), ); @@ -194,8 +194,7 @@ export class SyncService implements ISyncService { setInterval(async () => { try { if (!await this.isSyncing()) { - const lastFullCycle = - (await this.kvStore.get(syncLastFullCycleKey)) || 0; + const lastFullCycle = (await this.ds.get(syncLastFullCycleKey)) || 0; if (lastFullCycle && Date.now() - lastFullCycle > spaceSyncInterval) { // It's been a while since the last full cycle, let's sync the whole space await this.syncSpace(); @@ -223,11 +222,11 @@ export class SyncService implements ISyncService { ); await this.saveSnapshot(snapshot); await this.registerSyncStop(true); - this.eventHook.dispatchEvent("sync:success", operations); + await this.eventHook.dispatchEvent("sync:success", operations); } catch (e: any) { await this.saveSnapshot(snapshot); await this.registerSyncStop(false); - this.eventHook.dispatchEvent("sync:error", e.message); + await this.eventHook.dispatchEvent("sync:error", e.message); console.error("Sync error", e.message); } return operations; @@ -302,7 +301,7 @@ export class SyncService implements ISyncService { } async saveSnapshot(snapshot: Map) { - await this.kvStore.set(syncSnapshotKey, Object.fromEntries(snapshot)); + await this.ds.set(syncSnapshotKey, Object.fromEntries(snapshot)); } public async plugAwareConflictResolver( diff --git a/web/syscalls/clientStore.ts b/web/syscalls/clientStore.ts index 1914a6e..2d1300d 100644 --- a/web/syscalls/clientStore.ts +++ b/web/syscalls/clientStore.ts @@ -1,19 +1,20 @@ -import { KVStore } from "../../plugos/lib/kv_store.ts"; -import { storeSyscalls } from "../../plugos/syscalls/store.ts"; -import { proxySyscalls } from "../../plugos/syscalls/transport.ts"; import { SysCallMapping } from "../../plugos/system.ts"; +import { DataStore } from "../../plugos/lib/datastore.ts"; +import { KvKey } from "$sb/types.ts"; export function clientStoreSyscalls( - db: KVStore, + ds: DataStore, + prefix: KvKey = ["client"], ): SysCallMapping { - const localStoreCalls = storeSyscalls(db); - return proxySyscalls( - ["clientStore.get", "clientStore.set", "clientStore.delete"], - (ctx, name, ...args) => { - return localStoreCalls[name.replace("clientStore.", "store.")]( - ctx, - ...args, - ); + return { + "clientStore.get": (ctx, key: string): Promise => { + return ds.get([...prefix, ctx.plug!.name!, key]); }, - ); + "clientStore.set": (ctx, key: string, val: any): Promise => { + return ds.set([...prefix, ctx.plug!.name!, key], val); + }, + "clientStore.delete": (ctx, key: string): Promise => { + return ds.delete([...prefix, ctx.plug!.name!, key]); + }, + }; } diff --git a/web/syscalls/datastore.proxy.ts b/web/syscalls/datastore.proxy.ts new file mode 100644 index 0000000..0e65f6c --- /dev/null +++ b/web/syscalls/datastore.proxy.ts @@ -0,0 +1,15 @@ +import type { SysCallMapping } from "../../plugos/system.ts"; +import type { Client } from "../client.ts"; +import { proxySyscalls } from "./util.ts"; + +export function dataStoreProxySyscalls(client: Client): SysCallMapping { + return proxySyscalls(client, [ + "datastore.delete", + "datastore.set", + "datastore.batchSet", + "datastore.batchDelete", + "datastore.batchGet", + "datastore.get", + "datastore.query", + ]); +} diff --git a/web/syscalls/editor.ts b/web/syscalls/editor.ts index f7fdd2d..ce414c8 100644 --- a/web/syscalls/editor.ts +++ b/web/syscalls/editor.ts @@ -89,12 +89,20 @@ export function editorSyscalls(editor: Client): SysCallMapping { id: id as any, config: { html, script, mode }, }); + setTimeout(() => { + // Dummy dispatch to rerender the editor and toggle the panel + editor.editorView.dispatch({}); + }); }, "editor.hidePanel": (_ctx, id: string) => { editor.ui.viewDispatch({ type: "hide-panel", id: id as any, }); + setTimeout(() => { + // Dummy dispatch to rerender the editor and toggle the panel + editor.editorView.dispatch({}); + }); }, "editor.insertAtPos": (_ctx, text: string, pos: number) => { editor.editorView.dispatch({ diff --git a/web/syscalls/index.proxy.ts b/web/syscalls/index.proxy.ts deleted file mode 100644 index c701e3c..0000000 --- a/web/syscalls/index.proxy.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { SysCallMapping } from "../../plugos/system.ts"; -import { Client } from "../client.ts"; -import { proxySyscalls } from "./util.ts"; - -export function indexProxySyscalls(client: Client): SysCallMapping { - return proxySyscalls(client, [ - "index.set", - "index.batchSet", - "index.delete", - "index.get", - "index.queryPrefix", - "index.clearPageIndexForPage", - "index.deletePrefixForPage", - "index.clearPageIndex", - ]); -} diff --git a/web/syscalls/index.ts b/web/syscalls/index.ts deleted file mode 100644 index 60b293d..0000000 --- a/web/syscalls/index.ts +++ /dev/null @@ -1,65 +0,0 @@ -import type { SysCallMapping } from "../../plugos/system.ts"; -import Dexie from "dexie"; - -type Item = { - page: string; - key: string; - value: any; -}; - -export type KV = { - key: string; - value: any; -}; - -export function pageIndexSyscalls( - dbName: string, - indexedDB?: any, - IDBKeyRange?: any, -): SysCallMapping { - const db = new Dexie(dbName, { - indexedDB, - IDBKeyRange, - }); - db.version(1).stores({ - "index": "[page+key], page, key", - }); - const items = db.table("index"); - const apiObj: SysCallMapping = { - "index.set": (_ctx, page: string, key: string, value: any) => { - return items.put({ page, key, value }); - }, - "index.batchSet": async (_ctx, page: string, kvs: KV[]) => { - // await items.bulkPut(kvs); - if (kvs.length === 0) { - return; - } - const values = kvs.flatMap((kv) => ({ - page, - key: kv.key, - value: kv.value, - })); - await items.bulkPut(values); - }, - "index.delete": (_ctx, page: string, key: string) => { - return items.delete({ page, key }); - }, - "index.get": async (_ctx, page: string, key: string) => { - return (await items.get({ page, key }))?.value; - }, - "index.queryPrefix": (_ctx, prefix: string) => { - return items.where("key").startsWith(prefix).toArray(); - }, - "index.clearPageIndexForPage": async (ctx, page: string) => { - await apiObj["index.deletePrefixForPage"](ctx, page, ""); - }, - "index.deletePrefixForPage": (_ctx, page: string, prefix: string) => { - return items.where({ page }).and((it) => it.key.startsWith(prefix)) - .delete(); - }, - "index.clearPageIndex": () => { - return items.clear(); - }, - }; - return apiObj; -} diff --git a/web/syscalls/space.ts b/web/syscalls/space.ts index 5685032..85f9866 100644 --- a/web/syscalls/space.ts +++ b/web/syscalls/space.ts @@ -1,7 +1,6 @@ import { Client } from "../client.ts"; import { SysCallMapping } from "../../plugos/system.ts"; -import { AttachmentMeta, PageMeta } from "../types.ts"; -import { FileMeta } from "$sb/types.ts"; +import { AttachmentMeta, FileMeta, PageMeta } from "$sb/types.ts"; export function spaceSyscalls(editor: Client): SysCallMapping { return { diff --git a/web/syscalls/store.proxy.ts b/web/syscalls/store.proxy.ts deleted file mode 100644 index fc5b880..0000000 --- a/web/syscalls/store.proxy.ts +++ /dev/null @@ -1,18 +0,0 @@ -import type { SysCallMapping } from "../../plugos/system.ts"; -import type { Client } from "../client.ts"; -import { proxySyscalls } from "./util.ts"; - -export function storeProxySyscalls(client: Client): SysCallMapping { - return proxySyscalls(client, [ - "store.delete", - "store.deletePrefix", - "store.deleteAll", - "store.set", - "store.batchSet", - "store.batchDelete", - "store.batchGet", - "store.get", - "store.has", - "store.queryPrefix", - ]); -} diff --git a/web/syscalls/system.ts b/web/syscalls/system.ts index 57e0c7d..5565d0a 100644 --- a/web/syscalls/system.ts +++ b/web/syscalls/system.ts @@ -14,10 +14,6 @@ export function systemSyscalls( name: string, ...args: any[] ) => { - if (!ctx.plug) { - throw Error("No plug associated with context"); - } - if (name === "server" || name === "client") { // Backwards compatibility mode (previously there was an 'env' argument) name = args[0]; @@ -25,7 +21,9 @@ export function systemSyscalls( } let plug: Plug | undefined = ctx.plug; - if (name.indexOf(".") !== -1) { + const fullName = name; + // console.log("Invoking function", fullName, "on plug", plug); + if (name.includes(".")) { // plug name in the name const [plugName, functionName] = name.split("."); plug = system.loadedPlugs.get(plugName); @@ -34,7 +32,7 @@ export function systemSyscalls( } name = functionName; } - const functionDef = plug.manifest!.functions[name]; + const functionDef = plug?.manifest!.functions[name]; if (!functionDef) { throw Error(`Function ${name} not found`); } @@ -43,7 +41,12 @@ export function systemSyscalls( functionDef.env !== system.env ) { // Proxy to another environment - return proxySyscall(ctx, client.remoteSpacePrimitives, name, args); + return proxySyscall( + ctx, + client.remoteSpacePrimitives, + "system.invokeFunction", + [fullName, ...args], + ); } return plug.invoke(name, args); }, diff --git a/web/syscalls/util.ts b/web/syscalls/util.ts index eb52ca5..38b5f26 100644 --- a/web/syscalls/util.ts +++ b/web/syscalls/util.ts @@ -1,4 +1,3 @@ -import { plugCompileCommand } from "../../cmd/plug_compile.ts"; import { HttpSpacePrimitives } from "../../common/spaces/http_space_primitives.ts"; import { SyscallContext, SysCallMapping } from "../../plugos/system.ts"; import { SyscallResponse } from "../../server/rpc.ts"; diff --git a/web/syscalls/widget.ts b/web/syscalls/widget.ts new file mode 100644 index 0000000..c6e8805 --- /dev/null +++ b/web/syscalls/widget.ts @@ -0,0 +1,22 @@ +import { SysCallMapping } from "../../plugos/system.ts"; +import { Client } from "../client.ts"; + +export function widgetSyscalls( + client: Client, +): SysCallMapping { + return { + "widget.render": ( + _ctx, + lang: string, + body: string, + ): Promise<{ html: string; script: string }> => { + const langCallback = client.system.codeWidgetHook.codeWidgetCallbacks.get( + lang, + ); + if (!langCallback) { + throw new Error(`Code widget ${lang} not found`); + } + return langCallback(body); + }, + }; +} diff --git a/web/types.ts b/web/types.ts index 04ffc5d..5b7e469 100644 --- a/web/types.ts +++ b/web/types.ts @@ -1,21 +1,7 @@ import { Manifest } from "../common/manifest.ts"; +import { PageMeta } from "$sb/types.ts"; import { AppCommand } from "./hooks/command.ts"; -export type PageMeta = { - name: string; - lastModified: number; - lastOpened?: number; - perm: "ro" | "rw"; -} & Record; - -export type AttachmentMeta = { - name: string; - contentType: string; - lastModified: number; - size: number; - perm: "ro" | "rw"; -}; - // Used by FilterBox export type FilterOption = { name: string; @@ -104,6 +90,7 @@ export const initialViewState: AppViewState = { rhs: {}, bhs: {}, modal: {}, + ps: {}, }, allPages: [], commands: new Map(), diff --git a/website/Anchors.md b/website/Anchors.md new file mode 100644 index 0000000..d7c1e3a --- /dev/null +++ b/website/Anchors.md @@ -0,0 +1 @@ +Anchor represent named locations within a page and are defined using the $anchor syntax. They can then be referenced withing the page using [[@anchor]], or cross-page via [[Anchors@anchor]]. \ No newline at end of file diff --git a/website/Attributes.md b/website/Attributes.md index 7ca1e7d..f248e52 100644 --- a/website/Attributes.md +++ b/website/Attributes.md @@ -1,4 +1,4 @@ -Attributes can contribute additional [[Metadata]] to various entities: +Attribute syntax can contribute additional [[Metadata]] to various [[Objects]] including: * Pages * Items @@ -13,7 +13,7 @@ The syntax is as follows: For Obsidian/LogSeq compatibility, you can also double the colon like this: `[attributeName:: value]` -Attribute names need to be alphanumeric. Values are interpreted as [[YAML]] values. So here are some examples of valid attribute definitions: +Attribute names need to be alpha numeric. Values are interpreted as [[YAML]] values. So here are some examples of valid attribute definitions: * string: [attribute1: sup] * number: [attribute2: 10] @@ -26,36 +26,32 @@ Multiple attributes can be attached to a single entity, e.g. like so: ## Scope Depending on where these attributes appear, they attach to different things. For instance, this attaches an attribute to a page: -[pageAttribute:: hello] +[pageAttribute: hello] + +However, usually [[Frontmatter]] is be used for this purpose instead. Example query: - -|name |pageAttribute| -|----------|-----| -|Attributes|hello| - +```query +page where name = "{{@page.name}}" select name, pageAttribute +``` This attaches an attribute to an item: -* Item [itemAttribute:: hello] +* Item [itemAttribute: hello] #specialitem Example query: - -|name|itemAttribute| -|----|-----| -|Item|hello| - +```query +specialitem where itemAttribute = "hello" select name, itemAttribute +``` This attaches an attribute to a task: -* [ ] Task [taskAttribute:: hello] +* [ ] Task [taskAttribute: hello] Example query: - -|name|taskAttribute| -|----|-----| -|Task|hello| - +```query +task where page = "Attributes" and taskAttribute = "hello" select name, taskAttribute +``` diff --git a/website/CHANGELOG.md b/website/CHANGELOG.md index 651c824..d1817b0 100644 --- a/website/CHANGELOG.md +++ b/website/CHANGELOG.md @@ -3,6 +3,19 @@ release. --- +## 0.5.0 +Oh boy, this is a big one. This release brings you the following: + +* [[Objects]]: a more generic system to indexing and querying content in your space, including the ability to define your own custom object “types” (dubbed [[Tags]]). See the referenced pages for examples. +* [[Live Queries]] and [[Live Templates]]: ultimately will replace [[🔌 Directive]] in future versions and **[[🔌 Directive]] is now deprecated.** They differ from directives in that they don’t materialize their output into the page itself, but rather render them on the fly so only the query/template instantiation is kept on disk. All previous directive examples on this website how now been replaced with [[Live Templates]] and [[Live Queries]]. To ease the conversion there is {[Directive: Convert Query to Live Query]} command: just put your cursor inside of an existing (query) directive and run it to auto-convert. +* The query syntax used in [[Live Queries]] (but also used in [[🔌 Directive]]) has been significantly expanded, although there may still be bugs. There’s still more value to be unlocked here in future releases. +* The previous “backlinks” plug is now built into SilverBullet as [[Linked Mentions]] and appears at the bottom of every page (if there are incoming links). You can toggle linked mentions via {[Mentions: Toggle]}. +* A whole bunch of [[PlugOS]] syscalls have been updated, I’ll do my best update known existing plugs, but if you built existing ones some things may have broken. Please report anything broken in [Github issues](https://github.com/silverbulletmd/silverbullet/issues). + +Due to significant changes in how data is stored, likely your space will be resynced to all your clients once you upgrade. Just in case you may also want to {[Space: Reindex]} your space. If things are really broken, try the {[Debug: Reset Client]} command. + +--- + ## 0.4.0 The big change in this release is that SilverBullet now supports two [[Client Modes|client modes]]: _online_ mode and _sync_ mode. Read more about them here: [[Client Modes]]. @@ -144,7 +157,7 @@ Besides these architectural changes, a few other breaking changes were made to s ## 0.2.13 -* Support for multiple `order by` clauses in [[🔌 Directive/Query]] by [Siddhant Sanyam](https://github.com/silverbulletmd/silverbullet/pull/387) +* Support for multiple `order by` clauses in [[Live Queries]] by [Siddhant Sanyam](https://github.com/silverbulletmd/silverbullet/pull/387) * Tags included in `tags` [[Frontmatter]] now included in hash tag auto complete * Regression fix: when instantiating a page from a template it would always claim the page already existed (even if it didn't) diff --git a/website/Getting Started.md b/website/Getting Started.md index c89a43c..a2aa90d 100644 --- a/website/Getting Started.md +++ b/website/Getting Started.md @@ -22,20 +22,9 @@ You will notice this whole page section is wrapped in a strange type of block. T Don’t believe me, check this out, here’s a list of (max 10) pages in your space ordered by name, it updates (somewhat) dynamically 🤯. Create some new pages and come back here to see that it works: - -|name | -|---------------| -|API | -|Attributes | -|Authelia | -|Authentication | -|CHANGELOG | -|Cloud Links | -|Deployments | -|Federation | -|Frontmatter | -|Getting Started| - +```query +page select name order by name limit 10 +``` That said, the directive used wrapping this page section is `#use` which uses the content of another page as a template and inlines it. Directives recalculate their bodies in two scenarios: diff --git a/website/Linked Mentions.md b/website/Linked Mentions.md new file mode 100644 index 0000000..0fce43c --- /dev/null +++ b/website/Linked Mentions.md @@ -0,0 +1 @@ +Linked mentions \ No newline at end of file diff --git a/website/Live Preview.md b/website/Live Preview.md index d2da0b9..b5c4289 100644 --- a/website/Live Preview.md +++ b/website/Live Preview.md @@ -1,3 +1,5 @@ SilverBullet uses a “live preview” markdown editor. This mechanism is heavily inspired by [Obsidian’s live preview mode](https://help.obsidian.md/Live+preview+update). -It reduces visual noise by not constantly showing [[Markdown]] codes such as `[SilverBullet website](https://silverbullet.md)`, only showing the underlying Markdown when the cursor is placed inside. \ No newline at end of file +It reduces visual noise by not constantly showing [[Markdown]] codes such as `[SilverBullet website](https://silverbullet.md)`, only showing the underlying Markdown when the cursor is placed inside. + +In addition, live preview is also heavily leveraged to implement [[Live Queries]] and [[Live Templates]]. diff --git a/website/Live Queries.md b/website/Live Queries.md new file mode 100644 index 0000000..e38cbb7 --- /dev/null +++ b/website/Live Queries.md @@ -0,0 +1,101 @@ +Live Queries enable a (quasi) live view on various data sources, usually [[Objects]], and renders their results inline via [[Live Preview]] either as a template, or using [[Templates]]. + +## Syntax +The syntax of live queries are inspired by [SQL](https://en.wikipedia.org/wiki/SQL). Below is a query that demonstrates some of the supported clauses, hover over the result and click the edit icon to shows the code that generates the view: +```query +page +order by lastModified desc +where size > 100 +select name +limit 10 +render [[template/page]] +``` +It’s most convenient to use `/query` [[Slash Commands]] to insert a query in a page. + +For those comfortable reading such things [here you can find the full query grammar](https://github.com/silverbulletmd/silverbullet/blob/main/common/markdown_parser/query.grammar). + +The general syntax is to specify a `querySource` followed by a number of clauses that modify or restrict. If you haven’t already, check out how [[Objects]] work in SilverBullet. + +## Clauses +## `where` [[@expression]] +A `where` clause filters out all objects that do not match a certain condition. You can have multiple `where` clauses if you like, which will have the same effect as combining them with the `and` keyword. + +Here is a simple example based on a custom tag `#person` (see [[Objects]] on how this works): + +```#person +name: John +age: 7 +--- +name: Pete +age: 25 +``` + +To query all `person`s that are above age 21, we can use the following `where` clause: + +```query +person where page = "{{@page.name}}" and age > 21 +``` +## `order by` [[@expression]] +In order to sort results, a `order by` clause can be used, optionally with `desc` to order in descending order (ascending is the default): + +```query +person where page = "{{@page.name}}" order by age desc +``` +## `limit` [[@expression]] +To limit the number of results, you can use a `limit` clause: + +```query +person where page = "{{@page.name}}" limit 1 +``` +### `select` +To select only specific attributes from the result set, you can use the `select` clause. You can use it either simply as `select attribute1, attribute2` but also select the value of certain expressions and give them a name via the `select age + 1 as nextYear` syntax: + +```query +person +where page = "{{@page.name}}" +select name, age, age + 1 as nextYear +``` +### `render [[template]]` +By default results are rendered as a table, to instead render each result item using [[Templates|a template]], use the `render` clause: + +```query +person +where page = "{{@page.name}}" +render [[template/person]] +``` +## Expressions +$expression + +Primitives: + +* strings: `"a string"` +* numbers: `10` +* booleans: `true` or `false` +* regular expressions: `/[a-z]+/` +* null: `null` +* lists: `["value 1", 10, false]` + +Attributes can be accessed via the `attribute` syntax, and nested attributes via `attribute.subattribute.subsubattribute`. + +Logical expressions: + +* and: `name = "this" and age > 10` +* or: `name = "this" or age > 10` + +Binary expressions: +- `=` equals, e.g. `name = "Pete"` +- `!=` not equals, e.g. `name != "Pete"` +- `<` less than, e.g. `age < 10` +- `<=` less than or equals, e.g. `age <= 10` +- `>` greater than, e.g. `age > 10` +- `>=` greater than or equals, e.g. `age >= 10` +- `=~` to match against a regular expression, e.g. `name =~ /^template\//` +- `!=~` to not match a regular expression, e.g. `name !=~ /^template\//` +- `in` member of a list (e.g. `prop in ["foo", "bar"]`) +* `+` addition (can also concatenate strings), e.g. `10 + 12` or `name + "!!!"` +* `-` subtraction, e.g. `10 - 12` +* `/` addition, e.g. `10 / 12` +* `*` multiplication, e.g. `10 * 12` +* `%` modulo, e.g. `10 % 12` + +Operator precedence follows standard rules, use parentheses when in doubt, e.g. `(age > 10) or (name = "Pete")` diff --git a/website/Live Templates.md b/website/Live Templates.md new file mode 100644 index 0000000..527045c --- /dev/null +++ b/website/Live Templates.md @@ -0,0 +1,34 @@ +Live templates rendering [[Templates]] inline in a page. + +## Syntax +Live Templates are specified using [[Markdown]]‘s fenced code block notation using `template` as a language. The body of the code block specifies the template to use, as well as any arguments to pass to it. + +Generally you’d use it in one of two ways, either using a `page` template reference, or an inline `template`: + +Here’s an example using `page`: + +```template +page: "[[template/today]]" +``` +And here’s an example using `template`: + +```template +template: | + Today is {{today}}! +``` +To pass in a value to the template, you can specify the optional `value` attribute: + +```template +template: | + Hello, {{name}}! Today is _{{today}}_ +value: + name: Pete +``` +If you just want to render the raw markdown without handling it as a handlebars template, set `raw` to true: + +```template +template: | + This is not going to be {{processed}} by Handlebars +raw: true +``` + diff --git a/website/Manual.md b/website/Manual.md index edc0f9a..c6ac444 100644 --- a/website/Manual.md +++ b/website/Manual.md @@ -14,9 +14,9 @@ A full manual is still missing, but this is an attempt to give pointers on topic * [[Markdown]] * [[Markdown/Syntax Highlighting]] * [[Markdown/Code Widgets]] -* [[Metadata]] + * [[Live Queries]] + * [[Live Templates]] +* [[Objects]] * [[Frontmatter]] * [[Attributes]] -* [[🔌 Directive|Directives]] - * [[🔌 Directive/Query]] * [[SETTINGS]]: A few settings you can tweak diff --git a/website/Markdown.md b/website/Markdown.md index 11d36d7..ec38981 100644 --- a/website/Markdown.md +++ b/website/Markdown.md @@ -9,6 +9,7 @@ We mentioned markdown _extensions_, here are the ones currently supported: * Hashtags, e.g. `#mytag`. * Command link syntax: `{[Stats: Show]}` rendered into a clickable button {[Stats: Show]}. * [[Markdown/Code Widgets]] +* [[Anchors]] * [Tables](https://www.markdownguide.org/extended-syntax/#tables) * [Fenced code blocks](https://www.markdownguide.org/extended-syntax/#fenced-code-blocks) * [Task lists](https://www.markdownguide.org/extended-syntax/#task-lists) diff --git a/website/Markdown/Syntax Highlighting.md b/website/Markdown/Syntax Highlighting.md index 89c69b5..feb5708 100644 --- a/website/Markdown/Syntax Highlighting.md +++ b/website/Markdown/Syntax Highlighting.md @@ -60,4 +60,11 @@ class MyClass { void main() { print('Hello, World!'); } -``` \ No newline at end of file +``` + +```c +void main() { + +} +``` + diff --git a/website/Metadata.md b/website/Metadata.md index e813347..3cd5e3e 100644 --- a/website/Metadata.md +++ b/website/Metadata.md @@ -1,29 +1,5 @@ -Metadata is data about data. There are a few entities you can add meta data to: +Metadata is data about data. Most [[Objects]] have a set of default attributes that can be augmented in a few additional ways: -* **page**: Pages have a default set of meta data built-in, but this can be expanded through mechanisms described below. The available metadata for a page is: - * `name`: The full name of the page - * `lastModified`: a timestamp (in ms since 1970-01-01) of when the page was last modified - * `perm`: either `ro` (read-only) or `rw`: this determines whether the editor opens in read-write or read-only mode. - * `contentType`: for pages always `text/markdown` - * `size`: the size of the file in bytes - * `tags`: A list of tags used in the top-level of the page (if any) -* **item**: Every list item appearing in a numbered, or unordered list is indexed and contains the following default set of metadata: - * `name`: The full content of the item minus attributes (see later) - * `page`: The page the item appears in - * `pos`: The offset (number of characters from the beginning of the page) where the item starts - * `tags`: A list of tags used in the item (if any) -* **task**: Every task defined in the space using the `* [ ] Task name` syntax - * `name`: The full task name/description - * `done`: Whether the task has been marked as done - * `page`: The page where the task appears - * `pos`: The offset (number of characters from the beginning of the page) where the item starts - * `tags`: A list of tags used in the task (if any) -* **tag**: Every tag used in the space - * `name`: The name of the tag (without `#`) - * `freq`: The frequency of the use of the tag - -In addition, this metadata can be augmented in a few additional ways: - -* [[Tags]]: adds to the `tags` attribute +* [[Tags]]: to tag the object (and add to the `tags` attribute directly) * [[Frontmatter]]: at the top of pages, a [[YAML]] encoded block can be used to define additional attributes to a page -* [[Attributes]] \ No newline at end of file +* [[Attributes]] syntax diff --git a/website/Objects.md b/website/Objects.md new file mode 100644 index 0000000..e46b67b --- /dev/null +++ b/website/Objects.md @@ -0,0 +1,123 @@ +#meta-tag + +SilverBullet automatically builds and maintains an index of _objects_ extracted from all markdown pages in your space. It subsequently allows you to [[Live Queries]] this database in (potentially) useful ways. + +Some examples of things you can query for: +* Give me a list of all books that I have marked as _want to read_ +* Give me a list of all tasks not yet completed that have today as a due date +* Give me a list of items tagged with `#quote` +* Give me a list of not-completed tasks that reference the current page + +By design, the truth remains in the markdown: all data indexed into objects will have a representation in markdown text as well. The index can be flushed at any time and be rebuilt from markdown files. + +# Object representation +Every object has a set of [[Attributes]]. + +At the very least: +* `ref`: a unique _identifier_ (unique to the page, at least), often represented as a pointer to the place (page, position) in your space where the object is defined. For instance, a _page_ object will use the page name as its `ref` attribute, and a `task` will use `page@pos` (where `pos` is the location the task appears in `page`). +* `tags`: an array of type(s) of an object, see [[@tags]]. + +In addition, any number of additional tag-specific and custom [[Attributes]] can be defined (see below). + +# Tags +$tags +Every object has one or more tags, defining the types of an object. Some tags are built-in (as described below), but you can easily define new tags by simply using the #hashtag notation in strategic locations (more on these locations later). + +Here are the currently built-in tags: + +## page +Every page in your space is available via the `page` tag. You can attach _additional tags_ to a page, by either specifying them in the `tags` attribute [[Frontmatter]], or by putting additional [[Tags]] in the _first paragraph of your page_, as is done in this particular page with a #meta-tag. + +In addition to `ref` and `tags`, the `page` tag defines a bunch of additional attributes as can be seen in this example query: + +```query +page where name = "{{@page.name}}" +``` + +## task +Every task in your space is tagged with the `task` tag by default. You tag it with additional tags by using [[Tags]] in the task name, e.g. + +* [ ] My task #upnext + +And can then be queried via either `task` or `upnext`. + +The following query shows all attributes available for tasks: + +```query +upnext +``` +Although you may want to render it using a template such as [[template/task]] instead: + +```query +upnext render [[template/task]] +``` + +## item +List items are not currently indexed unless explicitly tagged (for performance reasons). Like other things, an an item can be tagged using [[Tags]]. + +Here is an example of a #quote item using a custom [[Attributes|attribute]]: + +* “If you don’t know where you’re going you may not get there.” [by: Yogi Berra] #quote + +And then queried via the #quote tag: + +```query +quote select by, name +``` + +## data +You can also embed arbitrary YAML data blocks in pages via fenced code blocks and use a tag as a coding language, e.g. + +```#person +name: Pete +age: 55 +``` + +Which then becomes queriable via the `person` tag: + +```query +person +``` + +## link +All page _links_ are tagged with `link`. You cannot attach additional tags to links. The main two attributes of a link are: + +* `toPage` the page the link is linking _to_ +* `page` the page the link appears on + +In addition, the `snippet` attribute attempts to capture a little bit of context on where the link appears. + +_Note_: this is the data source used for the {[Mentions: Toggle]} feature as well page {[Page: Rename]}. + +Here is an query that shows all links that appear in this particular page: + +```query +link where page = "{{@page.name}}" and inDirective = false +``` + +## anchor +$myanchor + +[[Anchors]] use the `$myanchor` notation to allow deeplinking into a page and are also indexed and queryable. It is not possible to attach additional tags to an anchor. + +Here is an example query: + +```query +anchor where page = "{{@page.name}}" +``` + +## tag +The ultimate meta tag is _tag_ itself, which indexes for all tags used, in which page they appear and what their “parent tag” is (the context of the tag: either `page`, `item` or `task`). + +Here are the tags used/defined in this page: + +```query +tag where page = "{{@page.name}}" +``` + +## attribute +This is another meta tag, which is used to index all [[Attributes]] used in your space. This is used by e.g. attribute completion in various contexts. You likely don’t need to use this tag directly, but it’s there. + +```query +attribute where page = "{{@page.name}}" limit 1 +``` diff --git a/website/SilverBullet.md b/website/SilverBullet.md index 7aed935..d0c7b8f 100644 --- a/website/SilverBullet.md +++ b/website/SilverBullet.md @@ -9,12 +9,12 @@ Now that we got that out of the way let’s have a look at some of SilverBullet ## Features * Runs in any modern browser (including on mobile) as a [[PWA]] in two potential [[Client Modes]] (_online_ and _synced_ mode), where the _synced mode_ enables **100% offline operation**, keeping a copy of content in the browser, syncing back to the server when a network connection is available. * Provides an enjoyable [[Markdown]] writing experience with a clean UI, rendering text using [[Live Preview|live preview]], further **reducing visual noise** while still providing direct access to the underlying markdown syntax. -* Supports wiki-style **page linking** using the `[[page link]]` syntax, even keeping links up-to-date when pages are renamed. +* Supports wiki-style **page linking** using the `[[page link]]` syntax. Incoming links are indexed and appear as “Linked Mentions” at the bottom of the pages linked to thereby providing _bi-directional linking_. * Optimized for **keyboard-based operation**: * Quickly navigate between pages using the **page switcher** (triggered with `Cmd-k` on Mac or `Ctrl-k` on Linux and Windows). * Run commands via their keyboard shortcuts or the **command palette** (triggered with `Cmd-/` or `Ctrl-/` on Linux and Windows). - * Use [[🔌 Core/Slash Commands|slash commands]] to perform common text editing operations. -* Provides a platform for [end-user programming](https://www.inkandswitch.com/end-user-programming/) through its support for annotating pages with [[Frontmatter]] and [[🔌 Directive|directives]] (such as [[🔌 Directive/Query|#query]]), making parts of pages _dynamic_. + * Use [[Slash Commands]] to perform common text editing operations. +* Provides a platform for [end-user programming](https://www.inkandswitch.com/end-user-programming/) through its support for [[Objects]], [[Live Queries]] and [[Live Templates]]. * Robust extension mechanism using [[🔌 Plugs]]. * **Self-hosted**: you own your data. All content is stored as plain files in a folder on disk. Back up, sync, edit, publish, script with any additional tools you like. * SilverBullet is [open source, MIT licensed](https://github.com/silverbulletmd/silverbullet) software. @@ -80,4 +80,4 @@ Have a lock at our work-in-progress [[Manual]]. ## Support If you (hypothetically) find bugs or have feature requests, post them in [our issue tracker](https://github.com/silverbulletmd/silverbullet/issues). Want to contribute? [Check out the code](https://github.com/silverbulletmd/silverbullet). -Want to chat with us? [Join our Discord](https://discord.gg/EvXbFucTxn)! +Want to chat with us? [Join our Discord](https://discord.gg/EvXbFucTxn)! \ No newline at end of file diff --git a/website/🔌 Core/Slash Commands.md b/website/Slash Commands.md similarity index 100% rename from website/🔌 Core/Slash Commands.md rename to website/Slash Commands.md diff --git a/website/Tags.md b/website/Tags.md index 3d0f84e..7466024 100644 --- a/website/Tags.md +++ b/website/Tags.md @@ -1,32 +1,4 @@ -Tags in SilverBullet can be added in two ways: +Tags in SilverBullet are used to encode types of [[Objects]]. -1. Through the `tags` attribute in [[Frontmatter]] -2. By putting a `#tag` at the top level (to tag a page), or at the task or item level to tag those blocks specifically. +See [[Objects@tags]] for more information. -For instance, by using the #core-tag in this page, it has been tagged and can be used in a [[🔌 Directive/Query]]: - - -* [[Tags]] - - -Similarly, tags can be applied to list **items**: - -* This is a tagged item #core-tag - -and be queried: - - -|name |tags |page|pos| -|-------------------------------|--------|----|---| -|This is a tagged item #core-tag|core-tag|Tags|494| - - -and **tags**: - -* [ ] This is a tagged task #core-tag - -And they can be queried this way: - - -* [ ] [[Tags@808]] This is a tagged task #core-tag - diff --git a/website/Templates.md b/website/Templates.md new file mode 100644 index 0000000..2e03761 --- /dev/null +++ b/website/Templates.md @@ -0,0 +1,28 @@ +For various use cases, SilverBullet uses [Handlebars templates](https://handlebarsjs.com/). + +Generally templates are stored in your space as regular pages, which allows for reuse. Some examples include [[template/task]] and [[template/page]]. +As a convention, we often name templates with a `template/` prefix, although this is purely a convention. + +[[Live Templates]] allow templates to be define inline, for instance: +```template +template: | + Hello, {{name}}! Today is _{{today}}_ +value: + name: Pete +``` +### Template helpers +There are a number of built-in handlebars helpers you can use + +- `{{today}}`: Today’s date in the usual YYYY-MM-DD format +- `{{tomorrow}}`: Tomorrow’s date in the usual YYY-MM-DD format +- `{{yesterday}}`: Yesterday’s date in the usual YYY-MM-DD format +- `{{lastWeek}}`: Current date - 7 days +- `{{nextWeek}}`: Current date + 7 days +- `{{escapeRegexp "hello/there"}}` to escape a regexp, useful when injecting e.g. a page name into a query — think `name =~ /{{escapeRegexp @page.name}}/ +`* `{{replaceRegexp string regexp replacement}}`: replace a regular expression in a string, example use: `{{replaceRegexp name "#[^#\d\s\[\]]+\w+" ""}}` to remove hashtags from a task name +- `{{json @page}}` translate any (object) value to JSON, mostly useful for debugging +- `{{relativePath @page.name}}` translate a path to a relative one (to the current page), useful when injecting page names, e.g. `{{relativePath name}}`. +- `{{substring "my string" 0 3}}` performs a substring operation on the first argument, which in this example would result in `my ` +- `{{prefixLines "my string\nanother" " "}}` prefixes each line (except the first) with the given prefix. +- `{{niceDate @page.lastModified}}` translates any timestamp into a “nice” format (e.g. `2023-06-20`). +- The `@page` variable contains all page meta data (`name`, `lastModified`, `contentType`, as well as any custom [[Frontmatter]] attributes). You can address it like so: `{{@page.name}}` diff --git a/website/template/person.md b/website/template/person.md new file mode 100644 index 0000000..b182b7a --- /dev/null +++ b/website/template/person.md @@ -0,0 +1 @@ +* Person **{{name}}** has {{age}} \ No newline at end of file diff --git a/website/template/task.md b/website/template/task.md index 35d6a39..6d21030 100644 --- a/website/template/task.md +++ b/website/template/task.md @@ -1 +1 @@ -* [{{state}}] [[{{page}}@{{pos}}]] {{name}} \ No newline at end of file +* [{{state}}] [[{{ref}}]] {{name}} \ No newline at end of file diff --git a/website/template/today.md b/website/template/today.md new file mode 100644 index 0000000..7c97914 --- /dev/null +++ b/website/template/today.md @@ -0,0 +1 @@ +Today is {{today}}! \ No newline at end of file diff --git a/website/🔌 Backlinks.md b/website/🔌 Backlinks.md deleted file mode 100644 index 9093aa6..0000000 --- a/website/🔌 Backlinks.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -type: plug -uri: github:silverbulletmd/silverbullet-backlinks/backlinks.plug.js -repo: https://github.com/silverbulletmd/silverbullet-backlinks -author: Guillermo Vayá ---- - - -# SilverBullet plug for Backlinks - -Provides access to pages that link to the one currently being edited. - -## Wait, SilverBullet? - -If you don't know what it is, check its [webpage](https://silverbullet.md), but if you want me to spoil the fun: it is an extensible note taking app with markdown and plain files at its core (well... there is a bit of magic in there too, but what good it would be without a little magic?) - -## Installation - -Open (`cmd+k`) your `PLUGS` note in SilverBullet and add this plug to the list: - -```yaml -- github:silverbulletmd/silverbullet-backlinks/backlinks.plug.js -``` - -Then run the `Plugs: Update` command and off you go! - diff --git a/website/🔌 Directive.md b/website/🔌 Directive.md index ad302a2..b202043 100644 --- a/website/🔌 Directive.md +++ b/website/🔌 Directive.md @@ -3,9 +3,12 @@ type: plug repo: https://github.com/silverbulletmd/silverbullet --- +> **Warning** Deprecated +> Directives are now deprecated and will likely soon be removed, use [[Live Templates]] and [[Live Queries]] instead. + The directive plug is a built-in plug implementing various so-called “directive” that all take the form of `` and close with ``. Currently the following directives are supported: -* `#query` to perform queries: [[🔌 Directive/Query]] +* `#query` to perform queries: [[Live Queries]] * `#include` to inline the content of another page verbatim: [[@include]] * `#use` to use the content of another as a [handlebars](https://handlebarsjs.com/) template: [[@use]] * `#eval` to evaluate an arbitrary JavaScript expression and inline the result: [[@eval]] @@ -63,7 +66,7 @@ However, you can also invoke arbitrary plug functions, e.g. the `titleUnfurlOpti |title-unfurl|Extract title| -Optionally, you can use a `render` clause to render the result as a template, similar to [[🔌 Directive/Query]]: +Optionally, you can use a `render` clause to render the result as a template, similar to [[Live Queries]]: id: title-unfurl diff --git a/website/🔌 Directive/Query.md b/website/🔌 Directive/Query.md deleted file mode 100644 index 559bd4f..0000000 --- a/website/🔌 Directive/Query.md +++ /dev/null @@ -1,224 +0,0 @@ -## Query -The `#query` is the most widely used directive. It can be used to query various data sources and render results in various ways. - -### Syntax -1. _start with_: `` -2. _end with_: `` -3. _write your query_: replace `[QUERY GOES HERE]` with any query you want using the options below. -4. _available query options_: Usage of options is similar to SQL except for the special `render` option. The `render` option is used to display the data in a format that you created in a separate template. - * `where` - * `order by` - * `limit` - * `select` - * `render` - -P.S.: If you are a developer or have a technical knowledge to read a code and would like to know more about syntax, please check out -[query grammar](https://github.com/silverbulletmd/silverbullet/blob/main/common/markdown_parser/query.grammar). - -#### 2.1. Available query operators: - -- `=` equals -- `!=` not equals -- `<` less than -- `<=` less than or equals -- `>` greater than -- `>=` greater than or equals -- `=~` to match against a regular expression -- `!=~` does not match this regular expression -- `in` member of a list (e.g. `prop in ["foo", "bar"]`) - -Further, you can combine multiple of these with `and`. Example -`prop =~ /something/ and prop != “something”`. - -### 3. How to run a query? -After writing the query, there are three options: - -1. Open the **command palette** and run {[Directives: Update]} -2. Use shortcut: hit **Alt-q** (Windows, Linux) or **Option-q** (Mac) -3. Go to another page and come back to the page where the query is located, it always updates when a page is loaded - -After using one of the options, the “body” of the query is replaced with the new results of the query data will be displayed. - -### 4. Data sources - -Available data sources can be categorized as: - -1. Builtin data sources -2. Data that can be inserted by users -3. Plug’s data sources - -The best part about data sources: there is auto-completion. 🎉 - -Start writing `` - -#### 4.1. Available data sources - -- `page`: list of all pages -- `attachment`: list of all attachments -- `task`: list of all tasks (created with `[ ]`) across all pages -- `full-text`: use it with `where phrase = "SOME_TEXT"`. List of all pages where `SOME_TEXT` is mentioned -- `item`: list of ordered and unordered items such as bulleted lists across all pages -- `tag`: list of all hashtags used in all pages -- `link`: list of all pages giving a link to the page where query is written -- `data`: You can insert data using the syntax below. You can query the data using the `data` source. - -```data -name: John -age: 50 -city: Milan -country: Italy ---- -name: Jane -age: 53 -city: Rome -country: Italy ---- -name: Francesco -age: 28 -city: Berlin -country: Germany -``` - -Example: - -|name|age|city |country|page |pos | -|----|--|-----|-----|------------------|----| -|John|50|Milan|Italy|🔌 Directive/Query|2933| -|Jane|53|Rome |Italy|🔌 Directive/Query|2934| - - -#### 4.2 Plugs’ data sources - -Certain plugs can also provide special data sources to query specific data. Some examples are: - -- [[🔌 Github]] provides `gh-pull` to query PRs for selected repo -- [[🔌 Mattermost]] provides `mm-saved` to fetch (by default 15) saved posts in - Mattermost - -For a complete list of data sources, please check plugs’ own pages. - -### 5. Templates - -Templates are predefined formats to render the body of the query. - -#### 5.1 How to create a template? - -It is pretty easy. You just need to create a new page. However, it is -recommended to create your templates using `template/[TEMPLATE_NAME]` -convention. For this guide, we will create `template/plug` to display list of Plugs available in SilverBullet. We will use this template in the Examples section below. - -#### 5.2 What is the syntax? - -We are using Handlebars which is a simple templating language. It is using double curly braces and the name of the parameter to be injected. For our `template/plug`, we are using simple template like below. - - * [[{{name}}]] by **{{author}}** ([repo]({{repo}})) - -Let me break it down for you - -- `*` is creating a bullet point for each item in SilverBullet -- `[[{{name}}]]` is injecting the name of Plug and creating an internal link to - the page of the Plug -- `**{{author}}**` is injecting the author of the Plug and making it bold -- `([repo]({{repo}}))` is injecting the name of the Plug and creating an - external link to the GitHub page of the Plug - -For more information on the Handlebars syntax, you can read the -[official documentation](https://handlebarsjs.com/). - -#### 5.3 How to use the template? - -You just need to add the `render` keyword followed by the link of the template to the query like below: - - - -`#query page where type = "plug" render [[template/plug]]` - -You can see the usage of our template in example 6.4 below. - -### 6. Examples - -We will walk you through a set of examples starting from a very basic one -through one formatting the data using templates. - -Our goal in this exercise is to (i) get all plug pages (ii) ordered by last modified time and (iii) display in a nice format. - -For the sake of simplicity, we will use the `page` data source and limit the results not to spoil the page. - -#### 6.1 Simple query without any condition - -**Goal:** We would like to get the list of all pages. - -**Result:** Look at the data. This is more than we need. The query even gives us template pages. Let's try to limit it in the next step. - - -|name |lastModified |contentType |size|perm|pageAttribute| -|----------|-------------|-------------|----|--|-----| -|API |1692191260028|text/markdown|2200|rw| | -|Attributes|1691176701257|text/markdown|1466|rw|hello| -|Authelia |1688482500313|text/markdown|866 |rw| | - - - -#### 6.2 Simple query with a condition - -**Goal:** We would like to get all plug pages sorted by last modified time. - -**Result:** Okay, this is what we wanted but there is also information such as `perm`, `type` and `lastModified` that we don't need. - - -|name |lastModified |contentType |size|perm|type|uri |repo |author |share-support| -|--|--|--|--|--|--|--|--|--|--| -|🔌 Twitter |1692810059854|text/markdown|1266|rw|plug|github:silverbulletmd/silverbullet-twitter/twitter.plug.js |https://github.com/silverbulletmd/silverbullet-twitter |SilverBullet Authors| | -|🔌 Share |1691177844386|text/markdown|693 |rw|plug| |https://github.com/silverbulletmd/silverbullet | | | -|🔌 Github |1691137925014|text/markdown|2206|rw|plug|github:silverbulletmd/silverbullet-github/github.plug.js |https://github.com/silverbulletmd/silverbullet-github |Zef Hemel |true| -|🔌 Mattermost|1691137924741|text/markdown|3535|rw|plug|github:silverbulletmd/silverbullet-mattermost/mattermost.plug.json|https://github.com/silverbulletmd/silverbullet-mattermost|Zef Hemel |true| -|🔌 Git |1691137924435|text/markdown|1112|rw|plug|github:silverbulletmd/silverbullet-git/git.plug.js |https://github.com/silverbulletmd/silverbullet-git |Zef Hemel | | - - -#### 6.3 Query to select only certain fields - -**Goal:** We would like to get all plug pages, selecting only `name`, `author` -and `repo` columns and then sort by last modified time. - -**Result:** Okay, this is much better. However, I believe this needs a touch -from a visual perspective. - - -|name |author |repo | -|--|--|--| -|🔌 Twitter |SilverBullet Authors|https://github.com/silverbulletmd/silverbullet-twitter | -|🔌 Share | |https://github.com/silverbulletmd/silverbullet | -|🔌 Github |Zef Hemel |https://github.com/silverbulletmd/silverbullet-github | -|🔌 Mattermost|Zef Hemel |https://github.com/silverbulletmd/silverbullet-mattermost| -|🔌 Git |Zef Hemel |https://github.com/silverbulletmd/silverbullet-git | - - -#### 6.4 Display the data in a format defined by a template - -**Goal:** We would like to display the data from step 5.3 in a nice format using bullet points with links to Plug pages, with the author name and a link to their GitHub repo. - -**Result:** Here you go. This is the result we would like to achieve 🎉. Did you see how I used `render` and `template/plug` in a query? 🚀 - - -* [[🔌 Twitter]] by **SilverBullet Authors** ([repo](https://github.com/silverbulletmd/silverbullet-twitter)) -* [[🔌 Share]] -* [[🔌 Github]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-github)) -* [[🔌 Mattermost]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-mattermost)) -* [[🔌 Git]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-git)) - - -PS: You don't need to select only certain fields to use templates. Templates are -smart enough to get only the information needed to render the data. Therefore, -the following queries are the same in terms of end result when using the -templates. - - - -and: - - diff --git a/website/🔌 Ghost.md b/website/🔌 Ghost.md index 73a89cd..9a7ecb2 100644 --- a/website/🔌 Ghost.md +++ b/website/🔌 Ghost.md @@ -6,57 +6,7 @@ author: Zef Hemel share-support: true --- - -# Ghost plug for Silver Bullet - -This allows you to publish your pages as [Ghost](https://ghost.org/) pages or posts. I use it to publish [Zef+](https://zef.plus). - -## Configuration - -In your `SETTINGS` specify the following settings: - - ```yaml - ghost: - myblog: - url: https://your-ghost-blog.ghost.io - ``` - -Then, create a Custom Integration (in your Ghost control panel under Settings > Advanced > Integrations > Add Custom Integration). Enter a name (whatever you want), then copy the full Admin API Key in your `SECRETS` file, mirroring the -structure of SETTINGS: - - ```yaml - ghost: - myblog: your:adminkey - ``` - -## Usage - -The plugin hooks into Silver Bullet's [Share infrastructure](https://silverbullet.md/%F0%9F%94%8C_Share). Therefore to -share a page as either a Ghost page or post, add a `$share` front matter key. For posts this should take the shape of: - - --- - $share: - - ghost:myblog:post:my-post-slug - --- - -And for pages: - - --- - $share: - - ghost:myblog:page:my-page-slug - --- - -Now, when you {[Share: Publish]} (Cmd-s/Ctrl-s) your post will automatically be created (as a draft) or updated if it already exists. - -Enjoy! - -## Installation - -Open your `PLUGS` note in SilverBullet and add this plug to the list, or simply use the `Plugs: Add` command: - -``` -- github:silverbulletmd/silverbullet-ghost/ghost.plug.js -``` - -Then run the `Plugs: Update` command and off you go! - +```template +page: "[[!raw.githubusercontent.com/silverbulletmd/silverbullet-ghost/main/README]]" +raw: true +``` \ No newline at end of file diff --git a/website/🔌 Git.md b/website/🔌 Git.md index 7e15d86..20fe7d1 100644 --- a/website/🔌 Git.md +++ b/website/🔌 Git.md @@ -5,40 +5,7 @@ repo: https://github.com/silverbulletmd/silverbullet-git author: Zef Hemel --- - -# SilverBullet plug for Git - -Very basic in functionality, it assumes you have git configured for push and pull in your space. What it does, roughly speaking: - -{[Git: Sync]}: - -- Adds all files in your folder to git -- It commits them with a "Snapshot" commit message -- It `git pull`s changes from the remote server -- It `git push`es changes to the remote server - -{[Git: Snapshot]}: - -- Asks you for a commit message -- Commits - -{[Github: Clone]}: - -Clones into your space from a Github repository. This will do authentication based on a [personal access token](https://github.com/settings/tokens). - -## Installation - -Open your `PLUGS` note in SilverBullet and add this plug to the list: - -``` -- github:silverbulletmd/silverbullet-git/git.plug.js -``` - -Then run the `Plugs: Update` command and off you go! - -## To Build - -```shell -deno task build -``` - +```template +page: "[[!raw.githubusercontent.com/silverbulletmd/silverbullet-git/main/README]]" +raw: true +``` \ No newline at end of file diff --git a/website/🔌 Github.md b/website/🔌 Github.md index 6f9f1b5..4ae7ada 100644 --- a/website/🔌 Github.md +++ b/website/🔌 Github.md @@ -6,59 +6,7 @@ author: Zef Hemel share-support: true --- - -# SilverBullet plug for Github -Provides various integrations with Github: - -* Query sources for events, notifications and pull requests -* Ability to load and share pages as Gists - -## Installation -Open your `PLUGS` note in SilverBullet and add this plug to the list: - -``` -- github:silverbulletmd/silverbullet-github/github.plug.js -``` - -Then run the `Plugs: Update` command and off you go! - -## Configuration -To configure, add a `githubToken` key to your `SECRETS` page, this should be a [personal access token](https://github.com/settings/tokens): - - ```yaml - githubToken: your-github-token - ``` - -## Query sources - -* `gh-event` List events of a user - * `username`: the user whose events to query -* `gh-pull`: List pull requests in a repository - * `repo`: the repo to query PRs for -* `gh-search-issue`: Search for issues and pull requests - * `query`: [the search query](https://docs.github.com/en/rest/search#search-issues-and-pull-requests) -* `gh-notification` requires a `githubToken` to be configured in `SECRETS`. - -## Share as Gist support - -To use: navigate to a page, and run the {[Share: Gist: Public Gist]} command, this will perform an initial publish, and add a `$share` attribute to your page's front matter. Subsequent updates can be performed via {[Share: Publish]}. - -To pull an *existing* gist into your space, use the {[Share: Gist: Load]} command and paste the URL to the gist. -## Example - -Example uses of the query providers: - - ## Recent pushes - - - - - ## Recent PRs - - - - -Where the `template/gh-pull` looks as follows: - - * ({{state}}) [{{title}}]({{html_url}}) - +```template +page: "[[!raw.githubusercontent.com/silverbulletmd/silverbullet-github/main/README]]" +raw: true +``` \ No newline at end of file diff --git a/website/🔌 Graph View.md b/website/🔌 Graph View.md index b7f6ed4..8854030 100644 --- a/website/🔌 Graph View.md +++ b/website/🔌 Graph View.md @@ -5,71 +5,7 @@ repo: https://github.com/silverbulletmd/silverbullet-graphview author: Bertjan Broeksema --- - -# SilverBullet plug for showing a graph view of the documents - -This plug aims to bring similar functionality as the Obsidian Graph view to -Silver Bullet. - -This repo is a fork of -[Bertjan Broeksema's original repo](https://github.com/bbroeksema/silverbullet-graphview) - -## Installation - -Open (`cmd+k`) your `PLUGS` note in SilverBullet and add this plug to the list: - -```yaml -- github:silverbulletmd/silverbullet-graphview/graphview.plug.js -``` - -Then run the `Plugs: Update` command and off you go! - -## Usage - -Run the `Show Global Graph` command to open up the graph view. Zoom and pan is -supported by scroll and pinch gestures with the mouse(pad). - -### Tags & Paths - -Set tags on the pages to customize their appearance in the graph - -- `#node_color=ff0000` → Change node color to red -- `#.graphignore` → Hide the page from the graph - -You can also use other custom tags to define node colors: Create a colormap with -HEX-colorcodes in `SETTINGS.md`. In this example, a node of a page where the tag -`#garden` is set will be rendered as green: - -```yaml -# Graphview -graphview: - default_color: "000000" - colormap: - path: - ⚙ services: "01017a" - 📓 notes: "02bdb6" - 🚧 projects: "ffc533" - 🧰 how-to: "96020e" - tag: - garden: "0bbd02" -``` - -## Links - -Click on the node labels to directly navigate to pages in your space - -## Label-shortening - -Long labels are shortened for readability. E.g. -`notesarecool/somethingverylong/subsubsubsub/foo` → `notes./somet./subsu./foo` - -## For offline development - -To ease development of the visual part, the offline folder contains a copy of -the html and custom javascript. As well as a simple graph model. - -```bash -$ cd offline -$ python -m http.server -``` - +```template +page: "[[!raw.githubusercontent.com/silverbulletmd/silverbullet-graphview/main/README]]" +raw: true +``` \ No newline at end of file diff --git a/website/🔌 KaTeX.md b/website/🔌 KaTeX.md index 39cbad6..d23bab1 100644 --- a/website/🔌 KaTeX.md +++ b/website/🔌 KaTeX.md @@ -5,44 +5,7 @@ repo: https://github.com/silverbulletmd/silverbullet-katex author: Zef Hemel --- - -# Silver Bullet KaTeX plug - -## Installation -Run the {[Plugs: Add]} command and paste in: `github:silverbulletmd/silverbullet-katex/katex.plug.js` - -That's all! - -## Use - -Put a latex block in your markdown: - - ```latex - c = \pm\sqrt{a^2 + b^2} - ``` - -And move your cursor outside of the block to live preview it! - -**Note:** [KaTeX](https://katex.org) itself is not bundled with this plug, it pulls the JavaScript, CSS and fonts from the JSDelivr CDN. This means _this plug will not work without an Internet connection_. The reason for this limitation is that it is not yet possible to distribute font files via plugs, and KaTeX depends on specific web fonts. - -## Build -Assuming you have Deno and Silver Bullet installed, simply build using: - -```shell -deno task build -``` - -Or to watch for changes and rebuild automatically - -```shell -deno task watch -``` - -Then, load the locally built plug, add it to your `PLUGS` note with an absolute path, for instance: - -``` -- file:/Users/you/path/to/katex.plug.json -``` - -And run the `Plugs: Update` command in SilverBullet. - +```template +page: "[[!raw.githubusercontent.com/silverbulletmd/silverbullet-katex/main/README]]" +raw: true +``` \ No newline at end of file diff --git a/website/🔌 Mattermost.md b/website/🔌 Mattermost.md index 3cb2d04..6577234 100644 --- a/website/🔌 Mattermost.md +++ b/website/🔌 Mattermost.md @@ -6,67 +6,7 @@ author: Zef Hemel share-support: true --- - -# Mattermost for Silver Bullet -This plug provides various integrations with the [Mattermost suite](https://www.mattermost.com) of products. Please follow the installation, configuration sections, and have a look at the example. - -Features: - -* Integration with [Silver Bullet Share](https://silverbullet.md/%F0%9F%94%8C_Share), allowing you to publish and update a page as a post on Mattermost, as well as load existing posts into SB as a page using the {[Share: Mattermost Post: Publish]} (to publish an existing page as a Mattermost post) and {[Share: Mattermost Post: Load]} (to load an existing post into SB) commands. -* Access your saved posts via the `mm-saved` query provider -* Unfurl support for posts (after dumping a permalink URL to a post in a page, use the {[Link: Unfurl]} command). -* Boards support is WIP - -## Installation -Run the {[Plugs: Add]} command and paste in the following URI: `github:silverbulletmd/silverbullet-mattermost/mattermost.plug.json` - -## Configuration -In `SETTINGS` provide the `mattermost` key with a `url` and `defaultTeam` for each server (you can name them arbitrarily): - - ```yaml - mattermost: - community: - url: https://community.mattermost.com - defaultTeam: core - silverbullet: - url: https://silverbullet.cloud.mattermost.com - defaultTeam: main - ``` - -In `SECRETS` provide a Mattermost personal access token (or hijack one from your current session) for each server: - - ```yaml - mattermost: - community: 1234 - silverbullet: 1234 - ``` - - -## Query sources - -* `mm-saved` fetches (by default 15) saved posts in Mattermost, you need to add a `where server = "community"` (with server name) clause to your query to select the mattermost server to query. - -To make the `mm-saved` query results look good, it's recommended you render your query results a template. Here is one to start with, you can keep it in e.g. `templates/mm-saved`: - - [{{username}}]({{url}}) in {{#if channelName}}**{{channelName}}**{{else}}a DM{{/if}} at _{{updatedAt}}_ {[Unsave]}: - - {{prefixLines (substring message 0 300 " ... (More)") "> "}} - - --- - -Note that the `{[Unsave]}` button when clicked, will unsave the post automatically 😎 - -Example use of `mm-saved` (using the `template/mm-saved` template above): - - - - - -## Posting to a channel - -You can use the {[Share: Mattermost Post: Publish]} command to publish the current page to a channel. You will be prompted to select the server and channel to post to. A `$share` key will be injected into frontmatter after the initial post. Subsequent post edits can be published via the standard {[Share: Publish]} command. - -## Loading a post into SB - -Using the {[Share: Mattermost Post: Load]} command you can load an existing post into your space. All you need for this is to have the Mattermost authentication configured as described above. You will be prompted for a post permalink and a page to save it to. If you are the author of the post, the `$share` frontmatter will also be set up so you can change the page and push changes back into Mattermost. - +```template +page: "[[!raw.githubusercontent.com/silverbulletmd/silverbullet-mattermost/main/README]]" +raw: true +``` \ No newline at end of file diff --git a/website/🔌 Mermaid.md b/website/🔌 Mermaid.md index 3b67c3a..cae01f2 100644 --- a/website/🔌 Mermaid.md +++ b/website/🔌 Mermaid.md @@ -5,25 +5,13 @@ repo: https://github.com/silverbulletmd/silverbullet-mermaid author: Zef Hemel --- - -# Silver Bullet plug for Mermaid diagrams -This plug adds basic [Mermaid](https://mermaid.js.org/) support to Silver Bullet. +Example use: +```mermaid +flowchart TD + Start --> Stop +``` -**Note:** The Mermaid library itself is not bundled with this plug, it pulls the JavaScript from the JSDelivr CDN. This means _this plug will not work without an Internet connection_. The reason for this is primarily plug size (bundling the library would amount to 1.1MB). This way Mermaid is only loaded on pages with actual Mermaid diagrams rather than on every SB load. - -## Installation -Run the {[Plugs: Add]} command and paste in: `github:silverbulletmd/silverbullet-mermaid/mermaid.plug.js` - -That's all! - -## Use - -Put a mermaid block in your markdown: - - ```mermaid - flowchart TD - Start --> Stop - ``` - -And move your cursor outside of the block to live preview it! - +```template +page: "[[!raw.githubusercontent.com/silverbulletmd/silverbullet-mermaid/main/README]]" +raw: true +``` \ No newline at end of file diff --git a/website/🔌 Plugs.md b/website/🔌 Plugs.md index 990a903..8591774 100644 --- a/website/🔌 Plugs.md +++ b/website/🔌 Plugs.md @@ -16,31 +16,14 @@ Plugs are distributed as self-contained JavaScript bundles (ending with `.plug.j ## Core plugs These plugs are distributed with SilverBullet and are automatically enabled: - -* [[🔌 Directive]] -* [[🔌 Editor]] -* [[🔌 Emoji]] -* [[🔌 Index]] -* [[🔌 Markdown]] -* [[🔌 Share]] -* [[🔌 Tasks]] -* [[🔌 Template]] - - +```query +page where type = "plug" and uri = null order by name render [[template/plug]] +``` ## Third-party plugs These plugs are written either by third parties or distributed separately from the main SB distribution: - -* [[🔌 Backlinks]] by **Guillermo Vayá** ([repo](https://github.com/silverbulletmd/silverbullet-backlinks)) -* [[🔌 Ghost]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-ghost)) -* [[🔌 Git]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-git)) -* [[🔌 Github]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-github)) -* [[🔌 Graph View]] by **Bertjan Broeksema** ([repo](https://github.com/silverbulletmd/silverbullet-graphview)) -* [[🔌 KaTeX]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-katex)) -* [[🔌 Mattermost]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-mattermost)) -* [[🔌 Mermaid]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-mermaid)) -* [[🔌 Serendipity]] by **Pantelis Vratsalis** ([repo](https://github.com/m1lt0n/silverbullet-serendipity)) -* [[🔌 Twitter]] by **SilverBullet Authors** ([repo](https://github.com/silverbulletmd/silverbullet-twitter)) - +```query +page where type = "plug" and uri != null order by name render [[template/plug]] +``` ## How to develop your own plug The easiest way to get started is to click the “Use this template” on the [silverbullet-plug-template](https://github.com/silverbulletmd/silverbullet-plug-template) repo. diff --git a/website/🔌 Serendipity.md b/website/🔌 Serendipity.md index 4157572..aded5b9 100644 --- a/website/🔌 Serendipity.md +++ b/website/🔌 Serendipity.md @@ -5,27 +5,7 @@ repo: https://github.com/m1lt0n/silverbullet-serendipity author: Pantelis Vratsalis --- - -# Serendipity plug for SilverBullet - -Serendipity introduces randomness in your navigation to your pages and notes in [silverbullet](https://silverbullet.md/). - -The plug includes 3 commands: - -* Open a random page: navigates to a totally random page -* Open a random page that contains a tag (e.g. `#hobbies`): narrows down the random pages to only those that have a specific tag -* Open a random page based on a search term (e.g. `performance management`): narrows down the random pages to only those that match the search term. - -In order to easily access the commands, all of their names are prefixed with `Serendipity:`. - - -## Installation - -Open (`cmd+k` in Mac and `ctrl+k` in other systems) your `PLUGS` note in SilverBullet and add this plug to the list: - -```yaml -- github:m1lt0n/silverbullet-serendipity/serendipity.plug.json -``` - -Then run the `Plugs: Update` command and you're ready! - +```template +page: "[[!raw.githubusercontent.com/m1lt0n/silverbullet-serendipity/main/README]]" +raw: true +``` \ No newline at end of file diff --git a/website/🔌 Share.md b/website/🔌 Share.md index 6ba50f6..5f5cd84 100644 --- a/website/🔌 Share.md +++ b/website/🔌 Share.md @@ -8,9 +8,6 @@ The Share plug provides infrastructure for sharing pages outside of your space. See the [original RFC](https://github.com/silverbulletmd/silverbullet/discussions/117) for implementation details. Specific implementations for sharing are implemented in other plugs, specifically: - -* [[🔌 Ghost]] -* [[🔌 Github]] -* [[🔌 Markdown]] -* [[🔌 Mattermost]] - +```query +page where share-support = true render [[template/page]] +``` diff --git a/website/🔌 Tasks.md b/website/🔌 Tasks.md index 1c0a685..d6e5c7c 100644 --- a/website/🔌 Tasks.md +++ b/website/🔌 Tasks.md @@ -38,30 +38,19 @@ Tasks can specify deadlines: When the cursor is positioned inside of a due date, the {[Task: Postpone]} command can be used to postpone the task for a certain period. ## Querying -All meta data (`done` status, `state`, `tags`, `deadline` and custom attributes) is extracted and available via the `task` query source to [[🔌 Directive/Query]]: +All meta data (`done` status, `state`, `tags`, `deadline` and custom attributes) is extracted and available via the `task` query source to [[Live Queries]]: - -|name |done |state |page |pos |tags |taskAttribute|deadline | -|--|--|--|--|--|--|--|--| -|Remote toggle me |false| |🔌 Tasks|3056| | | | -|This is a task (toggle me!) |false| |🔌 Tasks|321 | | | | -|This task is still to do |false|TODO |🔌 Tasks|420 | | | | -|In progress task |false|IN PROGRESS|🔌 Tasks|454 | | | | -|A resolved task |false|RESOLVED |🔌 Tasks|487 | | | | -|Whatever this state means |false|- |🔌 Tasks|516 | | | | -|Or this one |false|/ |🔌 Tasks|548 | | | | -|This is a tagged task #my-tag |false| |🔌 Tasks|824 |my-tag| | | -|This is a task with attributes|false| |🔌 Tasks|889 | |true| | -|This is due |false| |🔌 Tasks|993 | | |2022-11-26| - +```query +task where page = "{{@page.name}}" +``` ## Rendering There is a [[!silverbullet.md/template/task]] template you can use to render tasks nicely rather than using the default table (as demonstrated above). When you use this template, you can even cycle through the states of the task by click on its state _inside_ the rendered query, and it will update the state of the _original_ task automatically (although not yet in reverse) — this works across pages. Try it (by clicking on the checkbox inside of the directive): - -* [ ] [[🔌 Tasks@3056]] Remote toggle me - +```query +task where page = "{{@page.name}}" and name = "Remote toggle me" render [[template/task]] +``` * [ ] Remote toggle me diff --git a/website/🔌 Twitter.md b/website/🔌 Twitter.md index d85660d..b908c3c 100644 --- a/website/🔌 Twitter.md +++ b/website/🔌 Twitter.md @@ -5,24 +5,7 @@ repo: https://github.com/silverbulletmd/silverbullet-twitter author: SilverBullet Authors --- - -# SilverBullet for Twitter -Currently the only thing this plug offers is unfurling links to tweets. To use, paste in a link to a Tweet like `https://twitter.com/zef/status/1547943418403295232`, then run the `Link: Unfurl` command and select `Tweet content` to "enrich" the tweet URL with the content of the linked tweet, e.g. - - https://twitter.com/zef/status/1547687321679511552 - -Turns into: - - [Zef Hemel](https://twitter.com/zef/status/1547687321679511552): - > For those who missed my earlier posts on Silver Bullet: it’s my new powerful note taking/PKM app. Demo video from a user’s perspective: https://t.co/MKauSTcUG3 How it works technically (plugins all the way down): https://t.co/sqCkAa0pem Repo: https://t.co/rrxQdyxze1 - -## Installation - -Open (`cmd+k`) your `PLUGS` note in SilverBullet and add this plug to the list: - -```yaml -- github:silverbulletmd/silverbullet-twitter/twitter.plug.js -``` - -Then run the `Plugs: Update` command and off you go! - +```template +page: "[[!raw.githubusercontent.com/silverbulletmd/silverbullet-twitter/main/README]]" +raw: true +``` \ No newline at end of file