From 7b8d8af2c12ed25d28dbc0a68acf27b50c8fe773 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Wed, 26 Jul 2023 17:12:56 +0200 Subject: [PATCH] Attributes now have YAML values --- common/markdown_parser/parser.test.ts | 24 ++++++++++-- common/markdown_parser/parser.ts | 43 ++++++++++++++++++---- plug-api/lib/attribute.test.ts | 14 ++++--- plug-api/lib/attribute.ts | 22 ++++++----- plug-api/lib/syscall_mock.ts | 10 +++++ plug-api/lib/tree.ts | 2 +- plug-api/plugos-syscall/syscall.ts | 2 +- plugs/core/item.ts | 10 ++--- plugs/core/page.ts | 2 +- plugs/tasks/task.ts | 10 ++++- web/cm_plugins/smart_quotes.ts | 1 + website/Attributes.md | 53 ++++++++++++++++++--------- 12 files changed, 139 insertions(+), 54 deletions(-) create mode 100644 plug-api/lib/syscall_mock.ts diff --git a/common/markdown_parser/parser.test.ts b/common/markdown_parser/parser.test.ts index 18461cf..85f1010 100644 --- a/common/markdown_parser/parser.test.ts +++ b/common/markdown_parser/parser.test.ts @@ -91,14 +91,32 @@ Deno.test("Test directive parser", () => { }); const inlineAttributeSample = ` -Hello there [a link](http://zef.plus) and [age:: 100] +Hello there [a link](http://zef.plus) +[age: 100] +[age:: 200] + +Here's a more [ambiguous: case](http://zef.plus) + +And one with nested brackets: [array: [1, 2, 3]] `; Deno.test("Test inline attribute syntax", () => { const lang = buildMarkdown([]); const tree = parse(lang, inlineAttributeSample); - const nameNode = findNodeOfType(tree, "AttributeName"); + console.log("Attribute parsed", JSON.stringify(tree, null, 2)); + const attributes = collectNodesOfType(tree, "Attribute"); + let nameNode = findNodeOfType(attributes[0], "AttributeName"); assertEquals(nameNode?.children![0].text, "age"); - const valueNode = findNodeOfType(tree, "AttributeValue"); + let valueNode = findNodeOfType(attributes[0], "AttributeValue"); assertEquals(valueNode?.children![0].text, "100"); + + nameNode = findNodeOfType(attributes[1], "AttributeName"); + assertEquals(nameNode?.children![0].text, "age"); + valueNode = findNodeOfType(attributes[1], "AttributeValue"); + assertEquals(valueNode?.children![0].text, "200"); + + nameNode = findNodeOfType(attributes[2], "AttributeName"); + assertEquals(nameNode?.children![0].text, "array"); + valueNode = findNodeOfType(attributes[2], "AttributeValue"); + assertEquals(valueNode?.children![0].text, "[1, 2, 3]"); }); diff --git a/common/markdown_parser/parser.ts b/common/markdown_parser/parser.ts index a5058f5..53d5f8f 100644 --- a/common/markdown_parser/parser.ts +++ b/common/markdown_parser/parser.ts @@ -142,7 +142,7 @@ export const Highlight: MarkdownConfig = { ], }; -export const attributeRegex = /^\[(\w+)(::\s*)([^\]]+)\]/; +export const attributeStartRegex = /^\[(\w+)(::?\s*)/; export const Attribute: MarkdownConfig = { defineNodes: [ @@ -157,19 +157,46 @@ export const Attribute: MarkdownConfig = { name: "Attribute", parse(cx, next, pos) { let match: RegExpMatchArray | null; + const textFromPos = cx.slice(pos, cx.end); if ( next != 91 /* '[' */ || // and match the whole thing - !(match = attributeRegex.exec(cx.slice(pos, cx.end))) + !(match = attributeStartRegex.exec(textFromPos)) ) { return -1; } - const [fullMatch, attributeName, attributeColon, _attributeValue] = - match; - const endPos = pos + fullMatch.length; + const [fullMatch, attributeName, attributeColon] = match; + const attributeValueStart = pos + fullMatch.length; + let bracketNestingDepth = 1; + let valueLength = fullMatch.length; + loopLabel: + for (; valueLength < textFromPos.length; valueLength++) { + switch (textFromPos[valueLength]) { + case "[": + bracketNestingDepth++; + break; + case "]": + bracketNestingDepth--; + if (bracketNestingDepth === 0) { + // Done! + break loopLabel; + } + break; + } + } + if (bracketNestingDepth !== 0) { + console.log("Failed to parse attribute", fullMatch, textFromPos); + return -1; + } + + if (textFromPos[valueLength + 1] === "(") { + console.log("Link", fullMatch, textFromPos); + // This turns out to be a link, back out! + return -1; + } return cx.addElement( - cx.elt("Attribute", pos, endPos, [ + cx.elt("Attribute", pos, pos + valueLength + 1, [ cx.elt("AttributeMark", pos, pos + 1), // [ cx.elt("AttributeName", pos + 1, pos + 1 + attributeName.length), cx.elt( @@ -180,9 +207,9 @@ export const Attribute: MarkdownConfig = { cx.elt( "AttributeValue", pos + 1 + attributeName.length + attributeColon.length, - endPos - 1, + pos + valueLength, ), - cx.elt("AttributeMark", endPos - 1, endPos), // [ + cx.elt("AttributeMark", pos + valueLength, pos + valueLength + 1), // [ ]), ); }, diff --git a/plug-api/lib/attribute.test.ts b/plug-api/lib/attribute.test.ts index c998a2b..72131cd 100644 --- a/plug-api/lib/attribute.test.ts +++ b/plug-api/lib/attribute.test.ts @@ -1,3 +1,4 @@ +import "$sb/lib/syscall_mock.ts"; import { parse } from "../../common/markdown_parser/parse_tree.ts"; import buildMarkdown from "../../common/markdown_parser/parser.ts"; import { extractAttributes } from "$sb/lib/attribute.ts"; @@ -6,7 +7,7 @@ import { renderToText } from "$sb/lib/tree.ts"; const inlineAttributeSample = ` # My document -Top level attributes: [name:: sup] [age:: 42] +Top level attributes: [name:: sup] [age:: 42] [children: [pete, "john", mary]] * [ ] Attribute in a task [tag:: foo] * Regular item [tag:: bar] @@ -16,7 +17,7 @@ Top level attributes: [name:: sup] [age:: 42] const cleanedInlineAttributeSample = ` # My document -Top level attributes: +Top level attributes: * [ ] Attribute in a task [tag:: foo] * Regular item [tag:: bar] @@ -24,16 +25,17 @@ Top level attributes: 1. Itemized list [tag:: baz] `; -Deno.test("Test attribute extraction", () => { +Deno.test("Test attribute extraction", async () => { const lang = buildMarkdown([]); const tree = parse(lang, inlineAttributeSample); - const toplevelAttributes = extractAttributes(tree, false); - assertEquals(Object.keys(toplevelAttributes).length, 2); + const toplevelAttributes = await extractAttributes(tree, false); + // console.log("All attributes", toplevelAttributes); assertEquals(toplevelAttributes.name, "sup"); assertEquals(toplevelAttributes.age, 42); + assertEquals(toplevelAttributes.children, ["pete", "john", "mary"]); // Check if the attributes are still there assertEquals(renderToText(tree), inlineAttributeSample); // Now once again with cleaning - extractAttributes(tree, true); + await extractAttributes(tree, true); assertEquals(renderToText(tree), cleanedInlineAttributeSample); }); diff --git a/plug-api/lib/attribute.ts b/plug-api/lib/attribute.ts index b5d6dfe..097c9ca 100644 --- a/plug-api/lib/attribute.ts +++ b/plug-api/lib/attribute.ts @@ -1,28 +1,28 @@ import { findNodeOfType, ParseTree, - replaceNodesMatching, + replaceNodesMatchingAsync, } from "$sb/lib/tree.ts"; +import { YAML } from "$sb/plugos-syscall/mod.ts"; + export type Attribute = { name: string; value: string; }; -const numberRegex = /^-?\d+(\.\d+)?$/; - /** * Extracts attributes from a tree, optionally cleaning them out of the tree. * @param tree tree to extract attributes from * @param clean whether or not to clean out the attributes from the tree * @returns mapping from attribute name to attribute value */ -export function extractAttributes( +export async function extractAttributes( tree: ParseTree, clean: boolean, -): Record { +): Promise> { const attributes: Record = {}; - replaceNodesMatching(tree, (n) => { + await replaceNodesMatchingAsync(tree, async (n) => { if (n.type === "ListItem") { // Find top-level only, no nested lists return n; @@ -31,11 +31,13 @@ export function extractAttributes( const nameNode = findNodeOfType(n, "AttributeName"); const valueNode = findNodeOfType(n, "AttributeValue"); if (nameNode && valueNode) { - let val: any = valueNode.children![0].text!; - if (numberRegex.test(val)) { - val = +val; + const name = nameNode.children![0].text!; + const val = valueNode.children![0].text!; + try { + attributes[name] = await YAML.parse(val); + } catch (e: any) { + console.error("Error parsing attribute value as YAML", val, e); } - attributes[nameNode.children![0].text!] = val; } // Remove from tree if (clean) { diff --git a/plug-api/lib/syscall_mock.ts b/plug-api/lib/syscall_mock.ts new file mode 100644 index 0000000..3729095 --- /dev/null +++ b/plug-api/lib/syscall_mock.ts @@ -0,0 +1,10 @@ +import { YAML } from "../../common/deps.ts"; + +globalThis.syscall = (name: string, ...args: readonly any[]) => { + switch (name) { + case "yaml.parse": + return Promise.resolve(YAML.parse(args[0])); + default: + throw Error(`Not implemented in tests: ${name}`); + } +}; diff --git a/plug-api/lib/tree.ts b/plug-api/lib/tree.ts index 9c2954a..1b3c54c 100644 --- a/plug-api/lib/tree.ts +++ b/plug-api/lib/tree.ts @@ -129,7 +129,7 @@ export async function replaceNodesMatchingAsync( tree.children.splice(pos, 1); } } else { - replaceNodesMatchingAsync(child, substituteFn); + await replaceNodesMatchingAsync(child, substituteFn); } } } diff --git a/plug-api/plugos-syscall/syscall.ts b/plug-api/plugos-syscall/syscall.ts index b4d4e5e..18908c4 100644 --- a/plug-api/plugos-syscall/syscall.ts +++ b/plug-api/plugos-syscall/syscall.ts @@ -2,4 +2,4 @@ declare global { function syscall(name: string, ...args: any[]): Promise; } -export const syscall = self.syscall; +export const syscall = globalThis.syscall; diff --git a/plugs/core/item.ts b/plugs/core/item.ts index 18ff0de..79313a3 100644 --- a/plugs/core/item.ts +++ b/plugs/core/item.ts @@ -22,13 +22,13 @@ export async function indexItems({ name, tree }: IndexTreeEvent) { const coll = collectNodesOfType(tree, "ListItem"); - coll.forEach((n) => { + for (const n of coll) { if (!n.children) { - return; + continue; } if (collectNodesOfType(n, "Task").length > 0) { // This is a task item, skip it - return; + continue; } const item: Item = { @@ -43,7 +43,7 @@ export async function indexItems({ name, tree }: IndexTreeEvent) { break; } // Extract attributes and remove from tree - const extractedAttributes = extractAttributes(child, true); + const extractedAttributes = await extractAttributes(child, true); for (const [key, value] of Object.entries(extractedAttributes)) { item[key] = value; } @@ -66,7 +66,7 @@ export async function indexItems({ name, tree }: IndexTreeEvent) { key: `it:${n.from}`, value: item, }); - }); + } // console.log("Found", items.length, "item(s)"); await index.batchSet(name, items); } diff --git a/plugs/core/page.ts b/plugs/core/page.ts index c760e23..5bf4815 100644 --- a/plugs/core/page.ts +++ b/plugs/core/page.ts @@ -43,7 +43,7 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) { // [[Style Links]] // console.log("Now indexing links for", name); const pageMeta = await extractFrontmatter(tree); - const toplevelAttributes = extractAttributes(tree, false); + const toplevelAttributes = await extractAttributes(tree, false); if ( Object.keys(pageMeta).length > 0 || Object.keys(toplevelAttributes).length > 0 diff --git a/plugs/tasks/task.ts b/plugs/tasks/task.ts index 7d0113f..ccec3f9 100644 --- a/plugs/tasks/task.ts +++ b/plugs/tasks/task.ts @@ -14,12 +14,14 @@ import { import { addParentPointers, collectNodesMatching, + collectNodesMatchingAsync, collectNodesOfType, findNodeOfType, nodeAtPos, ParseTree, renderToText, replaceNodesMatching, + traverseTreeAsync, } from "$sb/lib/tree.ts"; import { applyQuery, removeQueries } from "$sb/lib/query.ts"; import { niceDate } from "$sb/lib/dates.ts"; @@ -44,7 +46,10 @@ export async function indexTasks({ name, tree }: IndexTreeEvent) { const tasks: { key: string; value: Task }[] = []; removeQueries(tree); addParentPointers(tree); - collectNodesOfType(tree, "Task").forEach((n) => { + await traverseTreeAsync(tree, async (n) => { + if (n.type !== "Task") { + return false; + } const complete = n.children![0].children![0].text! !== "[ ]"; const task: Task = { name: "", @@ -69,7 +74,7 @@ export async function indexTasks({ name, tree }: IndexTreeEvent) { }); // Extract attributes and remove from tree - const extractedAttributes = extractAttributes(n, true); + const extractedAttributes = await extractAttributes(n, true); for (const [key, value] of Object.entries(extractedAttributes)) { task[key] = value; } @@ -85,6 +90,7 @@ export async function indexTasks({ name, tree }: IndexTreeEvent) { key: `task:${n.from}`, value: task, }); + return true; }); // console.log("Found", tasks.length, "task(s)"); diff --git a/web/cm_plugins/smart_quotes.ts b/web/cm_plugins/smart_quotes.ts index b637c42..9c5a5a8 100644 --- a/web/cm_plugins/smart_quotes.ts +++ b/web/cm_plugins/smart_quotes.ts @@ -6,6 +6,7 @@ const straightQuoteContexts = [ "InlineCode", "FrontMatterCode", "DirectiveStart", + "Attribute", ]; // TODO: Add support for selection (put quotes around or create blockquote block?) diff --git a/website/Attributes.md b/website/Attributes.md index 9bffc33..a6ed7a3 100644 --- a/website/Attributes.md +++ b/website/Attributes.md @@ -5,38 +5,57 @@ Attributes can contribute additional [[Metadata]] to various entities: * Tasks ## Syntax -The syntax for attributes in inspired by [Obsidian’s Dataview](https://blacksmithgu.github.io/obsidian-dataview/annotation/add-metadata/) plugin, as well as [LogSeq](https://logseq.com/)‘s: +The syntax is as follows: ``` -[attributeName:: value] +[attributeName: value] ``` -Attribute names need to be alphanumeric. Values are interpreted as text by default, unless they take the shape of a number, in which case they’re parsed as a number. +For Obsidian/LogSeq compatibility, you can also double the colon like this: `[attributeName:: value]` + +Attribute names need to be alphanumeric. Values are interpreted as [[YAML]] values. So here are some examples of valid attribute definitions: + +* string: [attribute1: sup] +* number: [attribute2: 10] +* array: [attribute3: [sup, yo]] Multiple attributes can be attached to a single entity, e.g. like so: -* Some item [attribute1:: sup][attribute2:: 22] - -And queried like so: - - -|name |attribute1|attribute2|page |pos| -|---------|---|--|----------|---| -|Some item|sup|22|Attributes|569| - - +* Some item [attribute1: sup][attribute2: 22] ## Scope -Depending on where these attributes appear, they attach to different things. For instance here: +Depending on where these attributes appear, they attach to different things. For instance, this attaches an attribute to a page: [pageAttribute:: hello] -The attribute attaches to a page, whereas +Example query: + + +|name |lastModified |contentType |size|perm|pageAttribute| +|----------|-------------|-------------|----|--|-----| +|Attributes|1690384301337|text/markdown|1591|rw|hello| + + +This attaches an attribute to an item: * Item [itemAttribute:: hello] -it attaches to an item, and finally: +Example query: + + +|name|itemAttribute|page |pos | +|----|-----|----------|----| +|Item|hello|Attributes|1079| + + +This attaches an attribute to a task: * [ ] Task [taskAttribute:: hello] -Here it attaches to a task. \ No newline at end of file +Example query: + + +|name|done |taskAttribute|page |pos | +|----|-----|-----|----------|----| +|Task|false|hello|Attributes|1352| +