Attributes now have YAML values

common/markdown_parser/parser.test.ts

@@ -91,14 +91,32 @@ Deno.test("Test directive parser", () => {
 });
 
 const inlineAttributeSample = `
-Hello there [a link](http://zef.plus) and [age:: 100]
+Hello there [a link](http://zef.plus)
+[age: 100]
+[age:: 200]
+
+Here's a more [ambiguous: case](http://zef.plus)
+
+And one with nested brackets: [array: [1, 2, 3]]
 `;
 
 Deno.test("Test inline attribute syntax", () => {
   const lang = buildMarkdown([]);
   const tree = parse(lang, inlineAttributeSample);
-  const nameNode = findNodeOfType(tree, "AttributeName");
+  console.log("Attribute parsed", JSON.stringify(tree, null, 2));
+  const attributes = collectNodesOfType(tree, "Attribute");
+  let nameNode = findNodeOfType(attributes[0], "AttributeName");
   assertEquals(nameNode?.children![0].text, "age");
-  const valueNode = findNodeOfType(tree, "AttributeValue");
+  let valueNode = findNodeOfType(attributes[0], "AttributeValue");
   assertEquals(valueNode?.children![0].text, "100");
+
+  nameNode = findNodeOfType(attributes[1], "AttributeName");
+  assertEquals(nameNode?.children![0].text, "age");
+  valueNode = findNodeOfType(attributes[1], "AttributeValue");
+  assertEquals(valueNode?.children![0].text, "200");
+
+  nameNode = findNodeOfType(attributes[2], "AttributeName");
+  assertEquals(nameNode?.children![0].text, "array");
+  valueNode = findNodeOfType(attributes[2], "AttributeValue");
+  assertEquals(valueNode?.children![0].text, "[1, 2, 3]");
 });

common/markdown_parser/parser.ts

@@ -142,7 +142,7 @@ export const Highlight: MarkdownConfig = {
   ],
 };
 
-export const attributeRegex = /^\[(\w+)(::\s*)([^\]]+)\]/;
+export const attributeStartRegex = /^\[(\w+)(::?\s*)/;
 
 export const Attribute: MarkdownConfig = {
   defineNodes: [
@@ -157,19 +157,46 @@ export const Attribute: MarkdownConfig = {
       name: "Attribute",
       parse(cx, next, pos) {
         let match: RegExpMatchArray | null;
+        const textFromPos = cx.slice(pos, cx.end);
         if (
           next != 91 /* '[' */ ||
           // and match the whole thing
-          !(match = attributeRegex.exec(cx.slice(pos, cx.end)))
+          !(match = attributeStartRegex.exec(textFromPos))
         ) {
           return -1;
         }
-        const [fullMatch, attributeName, attributeColon, _attributeValue] =
-          match;
-        const endPos = pos + fullMatch.length;
+        const [fullMatch, attributeName, attributeColon] = match;
+        const attributeValueStart = pos + fullMatch.length;
+        let bracketNestingDepth = 1;
+        let valueLength = fullMatch.length;
+        loopLabel:
+        for (; valueLength < textFromPos.length; valueLength++) {
+          switch (textFromPos[valueLength]) {
+            case "[":
+              bracketNestingDepth++;
+              break;
+            case "]":
+              bracketNestingDepth--;
+              if (bracketNestingDepth === 0) {
+                // Done!
+                break loopLabel;
+              }
+              break;
+          }
+        }
+        if (bracketNestingDepth !== 0) {
+          console.log("Failed to parse attribute", fullMatch, textFromPos);
+          return -1;
+        }
+
+        if (textFromPos[valueLength + 1] === "(") {
+          console.log("Link", fullMatch, textFromPos);
+          // This turns out to be a link, back out!
+          return -1;
+        }
 
         return cx.addElement(
-          cx.elt("Attribute", pos, endPos, [
+          cx.elt("Attribute", pos, pos + valueLength + 1, [
             cx.elt("AttributeMark", pos, pos + 1), // [
             cx.elt("AttributeName", pos + 1, pos + 1 + attributeName.length),
             cx.elt(
@@ -180,9 +207,9 @@ export const Attribute: MarkdownConfig = {
             cx.elt(
               "AttributeValue",
               pos + 1 + attributeName.length + attributeColon.length,
-              endPos - 1,
+              pos + valueLength,
             ),
-            cx.elt("AttributeMark", endPos - 1, endPos), // [
+            cx.elt("AttributeMark", pos + valueLength, pos + valueLength + 1), // [
           ]),
         );
       },

plug-api/lib/attribute.test.ts

@@ -1,3 +1,4 @@
+import "$sb/lib/syscall_mock.ts";
 import { parse } from "../../common/markdown_parser/parse_tree.ts";
 import buildMarkdown from "../../common/markdown_parser/parser.ts";
 import { extractAttributes } from "$sb/lib/attribute.ts";
@@ -6,7 +7,7 @@ import { renderToText } from "$sb/lib/tree.ts";
 
 const inlineAttributeSample = `
 # My document
-Top level attributes: [name:: sup] [age:: 42]
+Top level attributes: [name:: sup] [age:: 42] [children: [pete, "john", mary]]
 
 * [ ] Attribute in a task [tag:: foo]
 * Regular item [tag:: bar]
@@ -24,16 +25,17 @@ Top level attributes:
 1. Itemized list [tag:: baz]
 `;
 
-Deno.test("Test attribute extraction", () => {
+Deno.test("Test attribute extraction", async () => {
   const lang = buildMarkdown([]);
   const tree = parse(lang, inlineAttributeSample);
-  const toplevelAttributes = extractAttributes(tree, false);
-  assertEquals(Object.keys(toplevelAttributes).length, 2);
+  const toplevelAttributes = await extractAttributes(tree, false);
+  // console.log("All attributes", toplevelAttributes);
   assertEquals(toplevelAttributes.name, "sup");
   assertEquals(toplevelAttributes.age, 42);
+  assertEquals(toplevelAttributes.children, ["pete", "john", "mary"]);
   // Check if the attributes are still there
   assertEquals(renderToText(tree), inlineAttributeSample);
   // Now once again with cleaning
-  extractAttributes(tree, true);
+  await extractAttributes(tree, true);
   assertEquals(renderToText(tree), cleanedInlineAttributeSample);
 });

plug-api/lib/attribute.ts

@@ -1,28 +1,28 @@
 import {
   findNodeOfType,
   ParseTree,
-  replaceNodesMatching,
+  replaceNodesMatchingAsync,
 } from "$sb/lib/tree.ts";
 
+import { YAML } from "$sb/plugos-syscall/mod.ts";
+
 export type Attribute = {
   name: string;
   value: string;
 };
 
-const numberRegex = /^-?\d+(\.\d+)?$/;
-
 /**
  * Extracts attributes from a tree, optionally cleaning them out of the tree.
  * @param tree tree to extract attributes from
  * @param clean whether or not to clean out the attributes from the tree
  * @returns mapping from attribute name to attribute value
  */
-export function extractAttributes(
+export async function extractAttributes(
   tree: ParseTree,
   clean: boolean,
-): Record<string, any> {
+): Promise<Record<string, any>> {
   const attributes: Record<string, any> = {};
-  replaceNodesMatching(tree, (n) => {
+  await replaceNodesMatchingAsync(tree, async (n) => {
     if (n.type === "ListItem") {
       // Find top-level only, no nested lists
       return n;
@@ -31,11 +31,13 @@ export function extractAttributes(
       const nameNode = findNodeOfType(n, "AttributeName");
       const valueNode = findNodeOfType(n, "AttributeValue");
       if (nameNode && valueNode) {
-        let val: any = valueNode.children![0].text!;
-        if (numberRegex.test(val)) {
-          val = +val;
+        const name = nameNode.children![0].text!;
+        const val = valueNode.children![0].text!;
+        try {
+          attributes[name] = await YAML.parse(val);
+        } catch (e: any) {
+          console.error("Error parsing attribute value as YAML", val, e);
         }
-        attributes[nameNode.children![0].text!] = val;
       }
       // Remove from tree
       if (clean) {

plug-api/lib/syscall_mock.ts

@@ -0,0 +1,10 @@
+import { YAML } from "../../common/deps.ts";
+
+globalThis.syscall = (name: string, ...args: readonly any[]) => {
+  switch (name) {
+    case "yaml.parse":
+      return Promise.resolve(YAML.parse(args[0]));
+    default:
+      throw Error(`Not implemented in tests: ${name}`);
+  }
+};

plug-api/lib/tree.ts

@@ -129,7 +129,7 @@ export async function replaceNodesMatchingAsync(
           tree.children.splice(pos, 1);
         }
       } else {
-        replaceNodesMatchingAsync(child, substituteFn);
+        await replaceNodesMatchingAsync(child, substituteFn);
       }
     }
   }

plug-api/plugos-syscall/syscall.ts

@@ -2,4 +2,4 @@ declare global {
   function syscall(name: string, ...args: any[]): Promise<any>;
 }
 
-export const syscall = self.syscall;
+export const syscall = globalThis.syscall;

plugs/core/item.ts

@@ -22,13 +22,13 @@ export async function indexItems({ name, tree }: IndexTreeEvent) {
 
   const coll = collectNodesOfType(tree, "ListItem");
 
-  coll.forEach((n) => {
+  for (const n of coll) {
     if (!n.children) {
-      return;
+      continue;
     }
     if (collectNodesOfType(n, "Task").length > 0) {
       // This is a task item, skip it
-      return;
+      continue;
     }
 
     const item: Item = {
@@ -43,7 +43,7 @@ export async function indexItems({ name, tree }: IndexTreeEvent) {
         break;
       }
       // Extract attributes and remove from tree
-      const extractedAttributes = extractAttributes(child, true);
+      const extractedAttributes = await extractAttributes(child, true);
       for (const [key, value] of Object.entries(extractedAttributes)) {
         item[key] = value;
       }
@@ -66,7 +66,7 @@ export async function indexItems({ name, tree }: IndexTreeEvent) {
       key: `it:${n.from}`,
       value: item,
     });
-  });
+  }
   // console.log("Found", items.length, "item(s)");
   await index.batchSet(name, items);
 }

plugs/core/page.ts

@@ -43,7 +43,7 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) {
   // [[Style Links]]
   // console.log("Now indexing links for", name);
   const pageMeta = await extractFrontmatter(tree);
-  const toplevelAttributes = extractAttributes(tree, false);
+  const toplevelAttributes = await extractAttributes(tree, false);
   if (
     Object.keys(pageMeta).length > 0 ||
     Object.keys(toplevelAttributes).length > 0

plugs/tasks/task.ts

@@ -14,12 +14,14 @@ import {
 import {
   addParentPointers,
   collectNodesMatching,
+  collectNodesMatchingAsync,
   collectNodesOfType,
   findNodeOfType,
   nodeAtPos,
   ParseTree,
   renderToText,
   replaceNodesMatching,
+  traverseTreeAsync,
 } from "$sb/lib/tree.ts";
 import { applyQuery, removeQueries } from "$sb/lib/query.ts";
 import { niceDate } from "$sb/lib/dates.ts";
@@ -44,7 +46,10 @@ export async function indexTasks({ name, tree }: IndexTreeEvent) {
   const tasks: { key: string; value: Task }[] = [];
   removeQueries(tree);
   addParentPointers(tree);
-  collectNodesOfType(tree, "Task").forEach((n) => {
+  await traverseTreeAsync(tree, async (n) => {
+    if (n.type !== "Task") {
+      return false;
+    }
     const complete = n.children![0].children![0].text! !== "[ ]";
     const task: Task = {
       name: "",
@@ -69,7 +74,7 @@ export async function indexTasks({ name, tree }: IndexTreeEvent) {
     });
 
     // Extract attributes and remove from tree
-    const extractedAttributes = extractAttributes(n, true);
+    const extractedAttributes = await extractAttributes(n, true);
     for (const [key, value] of Object.entries(extractedAttributes)) {
       task[key] = value;
     }
@@ -85,6 +90,7 @@ export async function indexTasks({ name, tree }: IndexTreeEvent) {
       key: `task:${n.from}`,
       value: task,
     });
+    return true;
   });
 
   // console.log("Found", tasks.length, "task(s)");

web/cm_plugins/smart_quotes.ts

@@ -6,6 +6,7 @@ const straightQuoteContexts = [
   "InlineCode",
   "FrontMatterCode",
   "DirectiveStart",
+  "Attribute",
 ];
 
 // TODO: Add support for selection (put quotes around or create blockquote block?)

website/Attributes.md

@@ -5,38 +5,57 @@ Attributes can contribute additional [[Metadata]] to various entities:
 * Tasks
 
 ## Syntax
-The syntax for attributes in inspired by [Obsidian’s Dataview](https://blacksmithgu.github.io/obsidian-dataview/annotation/add-metadata/) plugin, as well as [LogSeq](https://logseq.com/)‘s:
+The syntax is as follows:
 
 ```
-[attributeName:: value]
+[attributeName: value]
 ```
 
-Attribute names need to be alphanumeric. Values are interpreted as text by default, unless they take the shape of a number, in which case they’re parsed as a number.
+For Obsidian/LogSeq compatibility, you can also double the colon like this: `[attributeName:: value]`
+ 
+Attribute names need to be alphanumeric. Values are interpreted as [[YAML]] values. So here are some examples of valid attribute definitions:
+
+* string: [attribute1: sup]
+* number: [attribute2: 10]
+* array: [attribute3: [sup, yo]]
 
 Multiple attributes can be attached to a single entity, e.g. like so:
 
-* Some item [attribute1:: sup][attribute2:: 22]
-
-And queried like so:
-
-<!-- #query item where page = "Attributes" and name =~ /Some item/ -->
-|name     |attribute1|attribute2|page      |pos|
-|---------|---|--|----------|---|
-|Some item|sup|22|Attributes|569|
-<!-- /query -->
-
+* Some item [attribute1: sup][attribute2: 22]
 
 ## Scope
-Depending on where these attributes appear, they attach to different things. For instance here:
+Depending on where these attributes appear, they attach to different things. For instance, this attaches an attribute to a page:
 
 [pageAttribute:: hello]
 
-The attribute attaches to a page, whereas
+Example query:
+
+<!-- #query page where name = "Attributes" -->
+|name      |lastModified |contentType  |size|perm|pageAttribute|
+|----------|-------------|-------------|----|--|-----|
+|Attributes|1690384301337|text/markdown|1591|rw|hello|
+<!-- /query -->
+
+This attaches an attribute to an item:
 
 * Item [itemAttribute:: hello]
 
-it attaches to an item, and finally:
+Example query:
+
+<!-- #query item where page = "Attributes" and itemAttribute = "hello" -->
+|name|itemAttribute|page      |pos |
+|----|-----|----------|----|
+|Item|hello|Attributes|1079|
+<!-- /query -->
+
+This attaches an attribute to a task:
 
 * [ ] Task [taskAttribute:: hello]
 
-Here it attaches to a task.
+Example query:
+
+<!-- #query task where page = "Attributes" and taskAttribute = "hello" -->
+|name|done |taskAttribute|page      |pos |
+|----|-----|-----|----------|----|
+|Task|false|hello|Attributes|1352|
+<!-- /query -->