Major directive refactor (#195)

Fixes #188 #144 #76: major refactor of directive parsing, rendering, styling
2022-12-14 20:04:20 +01:00
parent 79e1151ee6
commit aaebea5e54
44 changed files with 656 additions and 347 deletions
@@ -42,11 +42,9 @@ export {
  TaskList,
 } from "@lezer/markdown";

-export { parseMixed } from "@lezer/common";
-
 export type { NodeType, SyntaxNode, SyntaxNodeRef, Tree } from "@lezer/common";

-export { searchKeymap } from "https://esm.sh/@codemirror/search@6.2.2?external=@codemirror/state,@codemirror/view";
+export { searchKeymap } from "https://esm.sh/@codemirror/search@6.2.3?external=@codemirror/state,@codemirror/view";
 export {
  Decoration,
  drawSelection,
@@ -61,7 +59,7 @@ export {
 } from "@codemirror/view";
 export type { DecorationSet, KeyBinding } from "@codemirror/view";

-export { markdown } from "https://esm.sh/@codemirror/lang-markdown@6.0.4?external=@codemirror/state,@lezer/common,@codemirror/language,@lezer/markdown,@codemirror/view,@lezer/highlight";
+export { markdown } from "https://esm.sh/@codemirror/lang-markdown@6.0.5?external=@codemirror/state,@lezer/common,@codemirror/language,@lezer/markdown,@codemirror/view,@lezer/highlight,@@codemirror/lang-html";

 export {
  EditorSelection,
@@ -96,4 +94,4 @@ export { yaml as yamlLanguage } from "https://esm.sh/@codemirror/legacy-modes@6.
 export {
  javascriptLanguage,
  typescriptLanguage,
-} from "https://esm.sh/@codemirror/lang-javascript@6.1.1?external=@codemirror/language,@codemirror/autocomplete,@codemirror/view,@codemirror/state,@codemirror/lint,@lezer/common,@lezer/lr,@lezer/javascript,@codemirror/commands";
+} from "https://esm.sh/@codemirror/lang-javascript@6.1.2?external=@codemirror/language,@codemirror/autocomplete,@codemirror/view,@codemirror/state,@codemirror/lint,@lezer/common,@lezer/lr,@lezer/javascript,@codemirror/commands";
@@ -1,4 +1,4 @@
-import { Tag } from "./deps.ts";
+import { Tag } from "../deps.ts";

 export const CommandLinkTag = Tag.define();
 export const CommandLinkNameTag = Tag.define();
@@ -13,3 +13,8 @@ export const BulletList = Tag.define();
 export const OrderedList = Tag.define();
 export const Highlight = Tag.define();
 export const HorizontalRuleTag = Tag.define();
+
+export const DirectiveTag = Tag.define();
+export const DirectiveStartTag = Tag.define();
+export const DirectiveEndTag = Tag.define();
+export const DirectiveProgramTag = Tag.define();
@@ -1,7 +1,7 @@
-import { Tag } from "./deps.ts";
-import type { MarkdownConfig } from "./deps.ts";
-import { System } from "../plugos/system.ts";
-import { Manifest } from "./manifest.ts";
+import { Tag } from "../deps.ts";
+import type { MarkdownConfig } from "../deps.ts";
+import { System } from "../../plugos/system.ts";
+import { Manifest } from "../manifest.ts";

 export type MDExt = {
  // unicode char code for efficiency .charCodeAt(0)
@@ -0,0 +1,16 @@
+// This file was generated by lezer-generator. You probably shouldn't edit it.
+import {LRParser} from "@lezer/lr"
+export const parser = LRParser.deserialize({
+  version: 14,
+  states: "&`OVQPOOOmQQO'#C^QOQPOOOtQPO'#C`OyQPO'#ClO!OQPO'#CnO!TQPO'#CqO!YQPO'#CsOOQO'#Cv'#CvO!bQQO,58xO!iQQO'#CcO#WQQO'#CbOOQO,58z,58zOOQO,59W,59WO#oQQO,59YO$ZQQO'#D`OOQO,59],59]OOQO,59_,59_OOQO-E6t-E6tO$rQQO,58}OtQPO'#CxO%ZQQO,58|OOQO'#Cp'#CpOOQO1G.t1G.tO%rQPO'#CyO%wQQO,59zOOQO'#Cg'#CgO$rQQO'#CjOOQO'#Cd'#CdOOQO1G.i1G.iOOQO,59d,59dOOQO-E6v-E6vOOQO,59e,59eOOQO-E6w-E6wO&`QPO'#DRO&hQPO,59UO$rQQO'#CwO&mQPO,59mOOQO1G.p1G.pOOQO,59c,59cOOQO-E6u-E6u",
+  stateData: "&u~OpOS~ORPO~OTROaSOcTOfUOhVO~OnQX~P[ORYO~OX]O~OR^O~OR_O~OYaOiaO~OnQa~P[OqcOxcOycOzcO{cO|cO}cO!OcO!PcO~O_dOTUXaUXcUXfUXhUXnUX~O!QfO!RfOTbaabacbafbahbanba~OvhOT!SXa!SXc!SXf!SXh!SXn!SX~OXlOYlO[lO]lOrjOsjOtkO~O_dOTUaaUacUafUahUanUa~ORpO~OvhOT!Saa!Sac!Saf!Sah!San!Sa~OvtOwuX~OwvO~OvtOwua~O",
+  goto: "#g!TPP!UP!XP!]!`!fPP!oPP!oP!XP!XP!t!XP!XPP!w!}#T#ZPPPPPPP#aPPPPPPPPPPPP#dRQOTWPXR[RQZRRndQmcQrkRwtVlcktRg^QXPRbXQurRxuQeZRoeQi_RqiRskR`U",
+  nodeNames: "⚠ Program Query Name WhereClause Where LogicalExpr FilterExpr Value Number String Bool Regex Null List And LimitClause Limit OrderClause Order OrderDirection SelectClause Select RenderClause Render PageRef",
+  maxTerm: 50,
+  skippedNodes: [0],
+  repeatNodeCount: 4,
+  tokenData: "Ao~R|X^#{pq#{qr$prs%T|}%o}!O%t!P!Q&V!Q![&|!^!_'U!_!`'c!`!a'p!c!}%t!}#O'}#P#Q(n#R#S%t#T#U(s#U#W%t#W#X+Y#X#Y%t#Y#Z-U#Z#]%t#]#^/f#^#`%t#`#a0b#a#b%t#b#c2u#c#d4q#d#f%t#f#g7h#g#h:d#h#i=`#i#k%t#k#l?[#l#o%t#y#z#{$f$g#{#BY#BZ#{$IS$I_#{$Ip$Iq%T$Iq$Ir%T$I|$JO#{$JT$JU#{$KV$KW#{&FU&FV#{~$QYp~X^#{pq#{#y#z#{$f$g#{#BY#BZ#{$IS$I_#{$I|$JO#{$JT$JU#{$KV$KW#{&FU&FV#{~$sP!_!`$v~${Pz~#r#s%O~%TO!O~~%WUOr%Trs%js$Ip%T$Ip$Iq%j$Iq$Ir%j$Ir~%T~%oOY~~%tOv~P%ySRP}!O%t!c!}%t#R#S%t#T#o%t~&[V[~OY&VZ]&V^!P&V!P!Q&q!Q#O&V#O#P&v#P~&V~&vO[~~&yPO~&V~'RPX~!Q![&|~'ZPq~!_!`'^~'cOx~~'hPy~#r#s'k~'pO}~~'uP|~!_!`'x~'}O{~R(SPtQ!}#O(VP(YRO#P(V#P#Q(c#Q~(VP(fP#P#Q(iP(nOiP~(sOw~R(xWRP}!O%t!c!}%t#R#S%t#T#b%t#b#c)b#c#g%t#g#h*^#h#o%tR)gURP}!O%t!c!}%t#R#S%t#T#W%t#W#X)y#X#o%tR*QS_QRP}!O%t!c!}%t#R#S%t#T#o%tR*cURP}!O%t!c!}%t#R#S%t#T#V%t#V#W*u#W#o%tR*|S!RQRP}!O%t!c!}%t#R#S%t#T#o%tR+_URP}!O%t!c!}%t#R#S%t#T#X%t#X#Y+q#Y#o%tR+vURP}!O%t!c!}%t#R#S%t#T#g%t#g#h,Y#h#o%tR,_URP}!O%t!c!}%t#R#S%t#T#V%t#V#W,q#W#o%tR,xS!QQRP}!O%t!c!}%t#R#S%t#T#o%tR-ZTRP}!O%t!c!}%t#R#S%t#T#U-j#U#o%tR-oURP}!O%t!c!}%t#R#S%t#T#`%t#`#a.R#a#o%tR.WURP}!O%t!c!}%t#R#S%t#T#g%t#g#h.j#h#o%tR.oURP}!O%t!c!}%t#R#S%t#T#X%t#X#Y/R#Y#o%tR/YSsQRP}!O%t!c!}%t#R#S%t#T#o%tR/kURP}!O%t!c!}%t#R#S%t#T#b%t#b#c/}#c#o%tR0US!PQRP}!O%t!c!}%t#R#S%t#T#o%tR0gURP}!O%t!c!}%t#R#S%t#T#]%t#]#^0y#^#o%tR1OURP}!O%t!c!}%t#R#S%t#T#a%t#a#b1b#b#o%tR1gURP}!O%t!c!}%t#R#S%t#T#]%t#]#^1y#^#o%tR2OURP}!O%t!c!}%t#R#S%t#T#h%t#h#i2b#i#o%tR2iSaQRP}!O%t!c!}%t#R#S%t#T#o%tR2zURP}!O%t!c!}%t#R#S%t#T#i%t#i#j3^#j#o%tR3cURP}!O%t!c!}%t#R#S%t#T#`%t#`#a3u#a#o%tR3zURP}!O%t!c!}%t#R#S%t#T#`%t#`#a4^#a#o%tR4eSRP]Q}!O%t!c!}%t#R#S%t#T#o%tR4vURP}!O%t!c!}%t#R#S%t#T#f%t#f#g5Y#g#o%tR5_URP}!O%t!c!}%t#R#S%t#T#W%t#W#X5q#X#o%tR5vURP}!O%t!c!}%t#R#S%t#T#X%t#X#Y6Y#Y#o%tR6_URP}!O%t!c!}%t#R#S%t#T#f%t#f#g6q#g#o%tR6vTRPpq7V}!O%t!c!}%t#R#S%t#T#o%tQ7YP#U#V7]Q7`P#m#n7cQ7hOcQR7mURP}!O%t!c!}%t#R#S%t#T#X%t#X#Y8P#Y#o%tR8UURP}!O%t!c!}%t#R#S%t#T#b%t#b#c8h#c#o%tR8mURP}!O%t!c!}%t#R#S%t#T#W%t#W#X9P#X#o%tR9UURP}!O%t!c!}%t#R#S%t#T#X%t#X#Y9h#Y#o%tR9mURP}!O%t!c!}%t#R#S%t#T#f%t#f#g:P#g#o%tR:WSRPhQ}!O%t!c!}%t#R#S%t#T#o%tR:iURP}!O%t!c!}%t#R#S%t#T#X%t#X#Y:{#Y#o%tR;QURP}!O%t!c!}%t#R#S%t#T#`%t#`#a;d#a#o%tR;iURP}!O%t!c!}%t#R#S%t#T#X%t#X#Y;{#Y#o%tR<QURP}!O%t!c!}%t#R#S%t#T#V%t#V#W<d#W#o%tR<iURP}!O%t!c!}%t#R#S%t#T#h%t#h#i<{#i#o%tR=SSRPfQ}!O%t!c!}%t#R#S%t#T#o%tR=eURP}!O%t!c!}%t#R#S%t#T#f%t#f#g=w#g#o%tR=|URP}!O%t!c!}%t#R#S%t#T#i%t#i#j>`#j#o%tR>eURP}!O%t!c!}%t#R#S%t#T#X%t#X#Y>w#Y#o%tR?OSrQRP}!O%t!c!}%t#R#S%t#T#o%tR?aURP}!O%t!c!}%t#R#S%t#T#[%t#[#]?s#]#o%tR?xURP}!O%t!c!}%t#R#S%t#T#X%t#X#Y@[#Y#o%tR@aURP}!O%t!c!}%t#R#S%t#T#f%t#f#g@s#g#o%tR@xURP}!O%t!c!}%t#R#S%t#T#X%t#X#YA[#Y#o%tRAcSRPTQ}!O%t!c!}%t#R#S%t#T#o%t",
+  tokenizers: [0, 1],
+  topRules: {"Program":[0,1]},
+  tokenPrec: 0
+})
@@ -0,0 +1,27 @@
+// This file was generated by lezer-generator. You probably shouldn't edit it.
+export const
+  Program = 1,
+  Query = 2,
+  Name = 3,
+  WhereClause = 4,
+  Where = 5,
+  LogicalExpr = 6,
+  FilterExpr = 7,
+  Value = 8,
+  Number = 9,
+  String = 10,
+  Bool = 11,
+  Regex = 12,
+  Null = 13,
+  List = 14,
+  And = 15,
+  LimitClause = 16,
+  Limit = 17,
+  OrderClause = 18,
+  Order = 19,
+  OrderDirection = 20,
+  SelectClause = 21,
+  Select = 22,
+  RenderClause = 23,
+  Render = 24,
+  PageRef = 25
@@ -1,5 +1,5 @@
 import type { ParseTree } from "$sb/lib/tree.ts";
-import type { Language, SyntaxNode } from "./deps.ts";
+import type { Language, SyntaxNode } from "../deps.ts";

 export function lezerToParseTree(
  text: string,
@@ -60,24 +60,5 @@ export function lezerToParseTree(

 export function parse(language: Language, text: string): ParseTree {
  const tree = lezerToParseTree(text, language.parser.parse(text).topNode);
-  // replaceNodesMatching(tree, (n): MarkdownTree | undefined | null => {
-  //   if (n.type === "FencedCode") {
-  //     let infoN = findNodeMatching(n, (n) => n.type === "CodeInfo");
-  //     let language = infoN!.children![0].text;
-  //     let textN = findNodeMatching(n, (n) => n.type === "CodeText");
-  //     let text = textN!.children![0].text!;
-  //
-  //     console.log(language, text);
-  //     switch (language) {
-  //       case "yaml":
-  //         let parsed = StreamLanguage.define(yaml).parser.parse(text);
-  //         let subTree = treeToAST(text, parsed.topNode, n.from);
-  //         // console.log(JSON.stringify(subTree, null, 2));
-  //         subTree.type = "yaml";
-  //         return subTree;
-  //     }
-  //   }
-  //   return;
-  // });
  return tree;
 }
@@ -4,8 +4,8 @@ import {
  collectNodesOfType,
  findNodeOfType,
  renderToText,
-} from "../plug-api/lib/tree.ts";
-import { assertEquals, assertNotEquals } from "../test_deps.ts";
+} from "../../plug-api/lib/tree.ts";
+import { assertEquals, assertNotEquals } from "../../test_deps.ts";

 const sample1 = `---
 type: page
@@ -27,10 +27,7 @@ Supper`;

 Deno.test("Test parser", () => {
  const lang = buildMarkdown([]);
-  let tree = parse(
-    lang,
-    sample1,
-  );
+  let tree = parse(lang, sample1);
  // console.log("tree", JSON.stringify(tree, null, 2));
  // Check if rendering back to text works
  assertEquals(renderToText(tree), sample1);
@@ -53,3 +50,42 @@ Deno.test("Test parser", () => {
  // console.log("Invalid node", node);
  assertEquals(node, undefined);
 });
+
+const directiveSample = `
+Before
+<!-- #query page -->
+Body line 1
+
+Body line 2
+<!-- /query -->
+End
+`;
+
+const nestedDirectiveExample = `
+Before
+<!-- #query page -->
+1
+<!-- #eval 10 * 10 -->
+100
+<!-- /eval -->
+3
+<!-- /query -->
+End
+`;
+
+Deno.test("Test directive parser", () => {
+  const lang = buildMarkdown([]);
+  let tree = parse(lang, directiveSample);
+  // console.log("tree", JSON.stringify(tree, null, 2));
+  assertEquals(renderToText(tree), directiveSample);
+
+  tree = parse(lang, nestedDirectiveExample);
+  // console.log("tree", JSON.stringify(tree, null, 2));
+  assertEquals(renderToText(tree), nestedDirectiveExample);
+
+  const orderByExample = `<!-- #query page order by lastModified -->
+  
+  <!-- /query -->`;
+  tree = parse(lang, orderByExample);
+  console.log("Tree", JSON.stringify(tree, null, 2));
+});
@@ -13,7 +13,7 @@ import {
  tags as t,
  TaskList,
  yamlLanguage,
-} from "./deps.ts";
+} from "../deps.ts";
 import * as ct from "./customtags.ts";
 import {
  MDExt,
@@ -112,14 +112,6 @@ const CommandLink: MarkdownConfig = {
            cx.elt("CommandLinkMark", endPos - 2, endPos),
          ]),
        );
-
-        // return cx.addElement(
-        //   cx.elt("CommandLink", pos, endPos, [
-        //     cx.elt("CommandLinkMark", pos, pos + 2),
-        //     cx.elt("CommandLinkName", pos + 2, endPos - 2),
-        //     cx.elt("CommandLinkMark", endPos - 2, endPos),
-        //   ]),
-        // );
      },
      after: "Emphasis",
    },
@@ -180,6 +172,108 @@ export const Comment: MarkdownConfig = {
  ],
 };

+// Directive parser
+
+const directiveStart = /^\s*<!--\s*#([a-z]+)\s*(.*?)-->\s*/;
+const directiveEnd = /^\s*<!--\s*\/(.*?)-->\s*/;
+
+import { parser as directiveParser } from "./parse-query.js";
+
+const highlightingDirectiveParser = directiveParser.configure({
+  props: [
+    styleTags({
+      "Name": t.variableName,
+      "String PageRef": t.string,
+      "Number": t.number,
+      "Where Limit Select Render Order OrderDirection And": t.keyword,
+    }),
+  ],
+});
+
+export const Directive: MarkdownConfig = {
+  defineNodes: [
+    { name: "Directive", block: true, style: ct.DirectiveTag },
+    { name: "DirectiveStart", style: ct.DirectiveStartTag, block: true },
+    { name: "DirectiveEnd", style: ct.DirectiveEndTag },
+    { name: "DirectiveBody", block: true },
+  ],
+  parseBlock: [{
+    name: "Directive",
+    parse: (cx, line: Line) => {
+      const match = directiveStart.exec(line.text);
+      if (!match) {
+        return false;
+      }
+
+      // console.log("Parsing directive", line.text);
+
+      const frontStart = cx.parsedPos;
+      const [fullMatch, directive, arg] = match;
+      const elts = [];
+      if (directive === "query") {
+        const queryParseTree = highlightingDirectiveParser.parse(arg);
+        elts.push(cx.elt(
+          "DirectiveStart",
+          cx.parsedPos,
+          cx.parsedPos + line.text.length + 1,
+          [cx.elt(queryParseTree, frontStart + fullMatch.indexOf(arg))],
+        ));
+      } else {
+        elts.push(cx.elt(
+          "DirectiveStart",
+          cx.parsedPos,
+          cx.parsedPos + line.text.length + 1,
+        ));
+      }
+
+      // console.log("Query parse tree", queryParseTree.topNode);
+
+      cx.nextLine();
+      const startPos = cx.parsedPos;
+      let endPos = startPos;
+      let text = "";
+      let lastPos = cx.parsedPos;
+      let nesting = 0;
+      while (true) {
+        if (directiveEnd.exec(line.text) && nesting === 0) {
+          break;
+        }
+        text += line.text + "\n";
+        endPos += line.text.length + 1;
+        if (directiveStart.exec(line.text)) {
+          nesting++;
+        }
+        if (directiveEnd.exec(line.text)) {
+          nesting--;
+        }
+        cx.nextLine();
+        if (cx.parsedPos === lastPos) {
+          // End of file, no progress made, there may be a better way to do this but :shrug:
+          return false;
+        }
+        lastPos = cx.parsedPos;
+      }
+      const directiveBodyTree = cx.parser.parse(text);
+
+      elts.push(
+        cx.elt("DirectiveBody", startPos, endPos, [
+          cx.elt(directiveBodyTree, startPos),
+        ]),
+      );
+      endPos = cx.parsedPos + line.text.length;
+      elts.push(cx.elt(
+        "DirectiveEnd",
+        cx.parsedPos,
+        cx.parsedPos + line.text.length,
+      ));
+      cx.nextLine();
+      cx.addElement(cx.elt("Directive", frontStart, endPos, elts));
+      return true;
+    },
+    before: "HTMLBlock",
+  }],
+};
+
 // FrontMatter parser

 const yamlLang = StreamLanguage.define(yamlLanguage);
@@ -249,6 +343,7 @@ export default function buildMarkdown(mdExtensions: MDExt[]): Language {
      WikiLink,
      CommandLink,
      FrontMatter,
+      Directive,
      TaskList,
      Comment,
      Highlight,
@@ -0,0 +1,68 @@
+@precedence { logic @left }
+@top Program { Query }
+@skip { space }
+
+Query {
+    Name ( WhereClause | LimitClause | OrderClause | SelectClause | RenderClause )*
+}
+
+commaSep<content> { content ("," content)* }
+
+WhereClause { Where LogicalExpr }
+LimitClause { Limit Number }
+OrderClause { Order Name OrderDirection? }
+SelectClause { Select commaSep<Name> }
+RenderClause { Render (PageRef | String) }
+
+OrderDirection {
+  "desc" | "asc"
+}
+
+Value { Number | String | Bool | Regex | Null | List }
+
+LogicalExpr { FilterExpr (And FilterExpr)* }
+
+FilterExpr {
+  Name "<" Value
+| Name "<=" Value
+| Name "=" Value
+| Name "!=" Value
+| Name ">=" Value
+| Name ">" Value
+| Name "=~" Value
+| Name "!=~" Value
+| Name "in" Value
+}
+
+List { "[" commaSep<Value> "]" }
+
+
+Bool {
+  "true" | "false"
+}
+
+
+@tokens {
+  space { std.whitespace+ }
+  Name { (std.asciiLetter | "-" | "_")+ }
+
+  Where { "where" }
+  Order { "order by" }
+  Select { "select" }
+  Render { "render" }
+  Limit { "limit" }
+  And { "and" }
+  Null { "null" }
+
+  String {
+    ("\"" | "“" | "”") ![\"”“]* ("\"" | "“" | "”")
+  }
+  PageRef {
+    "[" "[" ![\]]* "]" "]"
+  }
+  Regex { "/" ( ![/\\\n\r] | "\\" _ )* "/"? }
+
+  Number { std.digit+ }
+
+  // @precedence { Where, Sort, Select, Render, Limit, And, Null, Name }
+}
@@ -1,5 +1,5 @@
 import { SysCallMapping } from "../../plugos/system.ts";
-import { parse } from "../parse_tree.ts";
+import { parse } from "../markdown_parser/parse_tree.ts";
 import { Language } from "../deps.ts";
 import type { ParseTree } from "$sb/lib/tree.ts";