Fixes #529 by removing directives (#613)

* Fixes #529 by removing directives
* Load builtin tags on space reindex

common/languages.ts

@@ -25,7 +25,7 @@ import {
   xmlLanguage,
   yamlLanguage,
 } from "./deps.ts";
-import { highlightingDirectiveParser } from "./markdown_parser/parser.ts";
+import { highlightingQueryParser } from "./markdown_parser/parser.ts";
 
 export const builtinLanguages: Record<string, Language> = {
   "meta": StreamLanguage.define(yamlLanguage),
@@ -79,7 +79,7 @@ export const builtinLanguages: Record<string, Language> = {
   "dart": StreamLanguage.define(dartLanguage),
   "query": LRLanguage.define({
     name: "query",
-    parser: highlightingDirectiveParser,
+    parser: highlightingQueryParser,
   }),
 };
 

common/markdown_parser/customtags.ts

@@ -13,11 +13,6 @@ export const OrderedList = Tag.define();
 export const Highlight = Tag.define();
 export const HorizontalRuleTag = Tag.define();
 
-export const DirectiveTag = Tag.define();
-export const DirectiveStartTag = Tag.define();
-export const DirectiveEndTag = Tag.define();
-export const DirectiveProgramTag = Tag.define();
-
 export const AttributeTag = Tag.define();
 export const AttributeNameTag = Tag.define();
 export const AttributeValueTag = Tag.define();

common/markdown_parser/parser.test.ts

@@ -51,28 +51,6 @@ Deno.test("Test parser", () => {
   assertEquals(node, undefined);
 });
 
-const directiveSample = `
-Before
-<!-- #query page -->
-Body line 1
-
-Body line 2
-<!-- /query -->
-End
-`;
-
-const nestedDirectiveExample = `
-Before
-<!-- #query page -->
-1
-<!-- #eval 10 * 10 -->
-100
-<!-- /eval -->
-3
-<!-- /query -->
-End
-`;
-
 const inlineAttributeSample = `
 Hello there [a link](http://zef.plus)
 [age: 100]
@@ -152,9 +130,9 @@ Deno.test("Test command link arguments", () => {
   const commands = collectNodesOfType(tree, "CommandLink");
   assertEquals(commands.length, 2);
 
-  const args1 = findNodeOfType(commands[0], "CommandLinkArgs")
+  const args1 = findNodeOfType(commands[0], "CommandLinkArgs");
   assertEquals(args1!.children![0].text, '"with", "args"');
 
-  const args2 = findNodeOfType(commands[1], "CommandLinkArgs")
+  const args2 = findNodeOfType(commands[1], "CommandLinkArgs");
   assertEquals(args2!.children![0].text, '"other", "args", 123');
-});
+});

common/markdown_parser/parser.ts

@@ -160,6 +160,21 @@ export const Highlight: MarkdownConfig = {
   ],
 };
 
+import { parser as queryParser } from "./parse-query.js";
+
+export const highlightingQueryParser = queryParser.configure({
+  props: [
+    styleTags({
+      "Name": t.variableName,
+      "String": t.string,
+      "Number": t.number,
+      "PageRef": ct.WikiLinkTag,
+      "where limit select render Order OrderKW and or as InKW each all":
+        t.keyword,
+    }),
+  ],
+});
+
 export const attributeStartRegex = /^\[([\w\$]+)(::?\s*)/;
 
 export const Attribute: MarkdownConfig = {
@@ -264,121 +279,8 @@ export const Comment: MarkdownConfig = {
   ],
 };
 
-// Directive parser
-
-const directiveStart = /^\s*<!--\s*#([a-z]+)\s*(.*?)-->\s*/;
-const directiveEnd = /^\s*<!--\s*\/(.*?)-->\s*/;
-
-import { parser as directiveParser } from "./parse-query.js";
-import { parser as expressionParser } from "./parse-expression.js";
 import { Table } from "./table_parser.ts";
 import { foldNodeProp } from "@codemirror/language";
-import { lezerToParseTree } from "./parse_tree.ts";
-
-export const highlightingDirectiveParser = directiveParser.configure({
-  props: [
-    styleTags({
-      "Name": t.variableName,
-      "String": t.string,
-      "Number": t.number,
-      "PageRef": ct.WikiLinkTag,
-      "where limit select render Order OrderKW and or as InKW each all":
-        t.keyword,
-    }),
-  ],
-});
-
-export const Directive: MarkdownConfig = {
-  defineNodes: [
-    { name: "Directive", block: true, style: ct.DirectiveTag },
-    { name: "DirectiveStart", style: ct.DirectiveStartTag, block: true },
-    { name: "DirectiveEnd", style: ct.DirectiveEndTag },
-    { name: "DirectiveBody", block: true },
-  ],
-  parseBlock: [{
-    name: "Directive",
-    parse: (cx, line: Line) => {
-      const match = directiveStart.exec(line.text);
-      if (!match) {
-        return false;
-      }
-
-      // console.log("Parsing directive", line.text);
-
-      const frontStart = cx.parsedPos;
-      const [fullMatch, directive, arg] = match;
-      const elts = [];
-      if (directive === "query") {
-        const queryParseTree = highlightingDirectiveParser.parse(arg);
-        elts.push(cx.elt(
-          "DirectiveStart",
-          cx.parsedPos,
-          cx.parsedPos + line.text.length + 1,
-          [cx.elt(queryParseTree, frontStart + fullMatch.indexOf(arg))],
-        ));
-      } else if (directive === "eval") {
-        const expressionParseTree = expressionParser.parse(arg);
-        elts.push(cx.elt(
-          "DirectiveStart",
-          cx.parsedPos,
-          cx.parsedPos + line.text.length + 1,
-          [cx.elt(expressionParseTree, frontStart + fullMatch.indexOf(arg))],
-        ));
-      } else {
-        elts.push(cx.elt(
-          "DirectiveStart",
-          cx.parsedPos,
-          cx.parsedPos + line.text.length + 1,
-        ));
-      }
-
-      // console.log("Query parse tree", queryParseTree.topNode);
-
-      cx.nextLine();
-      const startPos = cx.parsedPos;
-      let endPos = startPos;
-      let text = "";
-      let lastPos = cx.parsedPos;
-      let nesting = 0;
-      while (true) {
-        if (directiveEnd.exec(line.text) && nesting === 0) {
-          break;
-        }
-        text += line.text + "\n";
-        endPos += line.text.length + 1;
-        if (directiveStart.exec(line.text)) {
-          nesting++;
-        }
-        if (directiveEnd.exec(line.text)) {
-          nesting--;
-        }
-        cx.nextLine();
-        if (cx.parsedPos === lastPos) {
-          // End of file, no progress made, there may be a better way to do this but :shrug:
-          return false;
-        }
-        lastPos = cx.parsedPos;
-      }
-      const directiveBodyTree = cx.parser.parse(text);
-
-      elts.push(
-        cx.elt("DirectiveBody", startPos, endPos, [
-          cx.elt(directiveBodyTree, startPos),
-        ]),
-      );
-      endPos = cx.parsedPos + line.text.length;
-      elts.push(cx.elt(
-        "DirectiveEnd",
-        cx.parsedPos,
-        cx.parsedPos + line.text.length,
-      ));
-      cx.nextLine();
-      cx.addElement(cx.elt("Directive", frontStart, endPos, elts));
-      return true;
-    },
-    before: "HTMLBlock",
-  }],
-};
 
 // FrontMatter parser
 
@@ -450,7 +352,6 @@ export default function buildMarkdown(mdExtensions: MDExt[]): Language {
       CommandLink,
       Attribute,
       FrontMatter,
-      Directive,
       TaskList,
       Comment,
       Highlight,

common/spaces/sync.test.ts

@@ -1,4 +1,4 @@
-import { removeDirectiveBody, SpaceSync, SyncStatusItem } from "./sync.ts";
+import { SpaceSync, SyncStatusItem } from "./sync.ts";
 import { DiskSpacePrimitives } from "./disk_space_primitives.ts";
 import { assertEquals } from "../../test_deps.ts";
 
@@ -117,23 +117,11 @@ Deno.test("Test store", async () => {
   await primary.writeFile("index", stringToBytes("Hello 1"));
   await secondary.writeFile("index", stringToBytes("Hello 1"));
 
-  // And two more files with different bodies, but only within a query directive — shouldn't conflict
-  await primary.writeFile(
-    "index.md",
-    stringToBytes(
-      "Hello\n<!-- #query page -->\nHello 1\n<!-- /query -->",
-    ),
-  );
-  await secondary.writeFile(
-    "index.md",
-    stringToBytes("Hello\n<!-- #query page -->\nHello 2\n<!-- /query -->"),
-  );
-
   await doSync();
   await doSync();
 
   // test + index + index.md + previous index.conflicting copy but nothing more
-  assertEquals((await primary.fetchFileList()).length, 4);
+  assertEquals((await primary.fetchFileList()).length, 3);
 
   console.log("Bringing a third device in the mix");
 
@@ -191,26 +179,6 @@ function sleep(ms = 10): Promise<void> {
   });
 }
 
-Deno.test("Remove directive bodies", () => {
-  assertEquals(
-    removeDirectiveBody(`<!-- #query page -->
-This is a body
-bla bla bla
-<!-- /query -->
-Hello
-<!-- #include [[test]] -->
-This is a body
-<!-- /include -->
-`),
-    `<!-- #query page -->
-<!-- /query -->
-Hello
-<!-- #include [[test]] -->
-<!-- /include -->
-`,
-  );
-});
-
 function stringToBytes(s: string): Uint8Array {
   return new TextEncoder().encode(s);
 }

common/spaces/sync.ts

@@ -1,6 +1,3 @@
-import { renderToText, replaceNodesMatching } from "../../plug-api/lib/tree.ts";
-import buildMarkdown from "../markdown_parser/parser.ts";
-import { parse } from "../markdown_parser/parse_tree.ts";
 import { SpacePrimitives } from "./space_primitives.ts";
 import { EventEmitter } from "../../plugos/event.ts";
 import { FileMeta } from "$sb/types.ts";
@@ -305,56 +302,32 @@ export class SpaceSync extends EventEmitter<SyncEvents> {
     const pageData1 = await primary.readFile(name);
     const pageData2 = await secondary.readFile(name);
 
-    if (name.endsWith(".md")) {
-      console.log(
-        "[sync]",
-        "File is markdown, using smart conflict resolution",
-      );
-      // Let's use a smartert check for markdown files, ignoring directive bodies
-      const pageText1 = removeDirectiveBody(
-        new TextDecoder().decode(pageData1.data),
-      );
-      const pageText2 = removeDirectiveBody(
-        new TextDecoder().decode(pageData2.data),
-      );
-      if (pageText1 === pageText2) {
-        console.log(
-          "[sync]",
-          "Files are the same (eliminating the directive bodies), no conflict",
-        );
+    let byteWiseMatch = true;
+    const arrayBuffer1 = pageData1.data;
+    const arrayBuffer2 = pageData2.data;
+    if (arrayBuffer1.byteLength !== arrayBuffer2.byteLength) {
+      byteWiseMatch = false;
+    }
+    if (byteWiseMatch) {
+      // Byte-wise comparison
+      for (let i = 0; i < arrayBuffer1.byteLength; i++) {
+        if (arrayBuffer1[i] !== arrayBuffer2[i]) {
+          byteWiseMatch = false;
+          break;
+        }
+      }
+      // Byte wise they're still the same, so no confict
+      if (byteWiseMatch) {
+        console.log("[sync]", "Files are the same, no conflict");
+
         snapshot.set(name, [
           pageData1.meta.lastModified,
           pageData2.meta.lastModified,
         ]);
         return 0;
       }
-    } else {
-      let byteWiseMatch = true;
-      const arrayBuffer1 = pageData1.data;
-      const arrayBuffer2 = pageData2.data;
-      if (arrayBuffer1.byteLength !== arrayBuffer2.byteLength) {
-        byteWiseMatch = false;
-      }
-      if (byteWiseMatch) {
-        // Byte-wise comparison
-        for (let i = 0; i < arrayBuffer1.byteLength; i++) {
-          if (arrayBuffer1[i] !== arrayBuffer2[i]) {
-            byteWiseMatch = false;
-            break;
-          }
-        }
-        // Byte wise they're still the same, so no confict
-        if (byteWiseMatch) {
-          console.log("[sync]", "Files are the same, no conflict");
-
-          snapshot.set(name, [
-            pageData1.meta.lastModified,
-            pageData2.meta.lastModified,
-          ]);
-          return 0;
-        }
-      }
     }
+
     let operations = 0;
     const revisionFileName = filePieces.length === 1
       ? `${name}.conflicted.${pageData2.meta.lastModified}`
@@ -403,18 +376,3 @@ export class SpaceSync extends EventEmitter<SyncEvents> {
     }
   }
 }
-
-const markdownLanguage = buildMarkdown([]);
-
-export function removeDirectiveBody(text: string): string {
-  // Parse
-  const tree = parse(markdownLanguage, text);
-  // Remove bodies
-  replaceNodesMatching(tree, (node) => {
-    if (node.type === "DirectiveBody") {
-      return null;
-    }
-  });
-  // Turn back into text
-  return renderToText(tree);
-}