Materialized query parser redo
This commit is contained in:
Zef Hemel
+11
-1
@@ -13,4 +13,14 @@ export type SilverBulletHooks = CommandHookT &
|
||||
CronHookT &
|
||||
EventHookT;
|
||||
|
||||
export type Manifest = plugos.Manifest<SilverBulletHooks>;
|
||||
export type SyntaxExtensions = {
|
||||
syntax?: { [key: string]: NodeDef };
|
||||
};
|
||||
|
||||
export type NodeDef = {
|
||||
firstCharacters: string[];
|
||||
regex: string;
|
||||
styles: { [key: string]: string };
|
||||
};
|
||||
|
||||
export type Manifest = plugos.Manifest<SilverBulletHooks> & SyntaxExtensions;
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
import type { SyntaxNode } from "@lezer/common";
|
||||
import type { Language } from "@codemirror/language";
|
||||
import { ParseTree } from "./tree";
|
||||
|
||||
export function lezerToParseTree(
|
||||
text: string,
|
||||
n: SyntaxNode,
|
||||
offset = 0
|
||||
): ParseTree {
|
||||
let children: ParseTree[] = [];
|
||||
let nodeText: string | undefined;
|
||||
let child = n.firstChild;
|
||||
while (child) {
|
||||
children.push(lezerToParseTree(text, child));
|
||||
child = child.nextSibling;
|
||||
}
|
||||
|
||||
if (children.length === 0) {
|
||||
children = [
|
||||
{
|
||||
from: n.from + offset,
|
||||
to: n.to + offset,
|
||||
text: text.substring(n.from, n.to),
|
||||
},
|
||||
];
|
||||
} else {
|
||||
let newChildren: ParseTree[] = [];
|
||||
let index = n.from;
|
||||
for (let child of children) {
|
||||
let s = text.substring(index, child.from);
|
||||
if (s) {
|
||||
newChildren.push({
|
||||
from: index + offset,
|
||||
to: child.from! + offset,
|
||||
text: s,
|
||||
});
|
||||
}
|
||||
newChildren.push(child);
|
||||
index = child.to!;
|
||||
}
|
||||
let s = text.substring(index, n.to);
|
||||
if (s) {
|
||||
newChildren.push({ from: index + offset, to: n.to + offset, text: s });
|
||||
}
|
||||
children = newChildren;
|
||||
}
|
||||
|
||||
let result: ParseTree = {
|
||||
type: n.name,
|
||||
from: n.from + offset,
|
||||
to: n.to + offset,
|
||||
};
|
||||
if (children.length > 0) {
|
||||
result.children = children;
|
||||
}
|
||||
if (nodeText) {
|
||||
result.text = nodeText;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function parse(language: Language, text: string): ParseTree {
|
||||
let tree = lezerToParseTree(text, language.parser.parse(text).topNode);
|
||||
// replaceNodesMatching(tree, (n): MarkdownTree | undefined | null => {
|
||||
// if (n.type === "FencedCode") {
|
||||
// let infoN = findNodeMatching(n, (n) => n.type === "CodeInfo");
|
||||
// let language = infoN!.children![0].text;
|
||||
// let textN = findNodeMatching(n, (n) => n.type === "CodeText");
|
||||
// let text = textN!.children![0].text!;
|
||||
//
|
||||
// console.log(language, text);
|
||||
// switch (language) {
|
||||
// case "yaml":
|
||||
// let parsed = StreamLanguage.define(yaml).parser.parse(text);
|
||||
// let subTree = treeToAST(text, parsed.topNode, n.from);
|
||||
// // console.log(JSON.stringify(subTree, null, 2));
|
||||
// subTree.type = "yaml";
|
||||
// return subTree;
|
||||
// }
|
||||
// }
|
||||
// return;
|
||||
// });
|
||||
return tree;
|
||||
}
|
||||
@@ -1,10 +1,12 @@
|
||||
import { SysCallMapping } from "../../plugos/system";
|
||||
import { MarkdownTree, parse } from "../tree";
|
||||
import { parse } from "../parse_tree";
|
||||
import { Language } from "@codemirror/language";
|
||||
import type { ParseTree } from "../tree";
|
||||
|
||||
export function markdownSyscalls(): SysCallMapping {
|
||||
export function markdownSyscalls(lang: Language): SysCallMapping {
|
||||
return {
|
||||
"markdown.parseMarkdown": (ctx, text: string): MarkdownTree => {
|
||||
return parse(text);
|
||||
"markdown.parseMarkdown": (ctx, text: string): ParseTree => {
|
||||
return parse(lang, text);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import { expect, test } from "@jest/globals";
|
||||
import { parse } from "./parse_tree";
|
||||
import {
|
||||
addParentPointers,
|
||||
collectNodesMatching,
|
||||
findParentMatching,
|
||||
nodeAtPos,
|
||||
removeParentPointers,
|
||||
renderToText,
|
||||
replaceNodesMatching
|
||||
} from "./tree";
|
||||
import wikiMarkdownLang from "../webapp/parser";
|
||||
|
||||
const mdTest1 = `
|
||||
# Heading
|
||||
## Sub _heading_ cool
|
||||
|
||||
Hello, this is some **bold** text and *italic*. And [a link](http://zef.me).
|
||||
|
||||
%% My comment here
|
||||
%% And second line
|
||||
|
||||
And an @mention
|
||||
|
||||
http://zef.plus
|
||||
|
||||
- This is a list [[PageLink]]
|
||||
- With another item
|
||||
- TODOs:
|
||||
- [ ] A task that's not yet done
|
||||
- [x] Hello
|
||||
- And a _third_ one [[Wiki Page]] yo
|
||||
`;
|
||||
|
||||
const mdTest2 = `
|
||||
Hello
|
||||
|
||||
* Item 1
|
||||
*
|
||||
|
||||
Sup`;
|
||||
|
||||
const mdTest3 = `
|
||||
\`\`\`yaml
|
||||
name: something
|
||||
\`\`\`
|
||||
`;
|
||||
|
||||
test("Run a Node sandbox", async () => {
|
||||
const lang = wikiMarkdownLang([]);
|
||||
let mdTree = parse(lang, mdTest1);
|
||||
addParentPointers(mdTree);
|
||||
// console.log(JSON.stringify(mdTree, null, 2));
|
||||
let wikiLink = nodeAtPos(mdTree, mdTest1.indexOf("Wiki Page"))!;
|
||||
expect(wikiLink.type).toBe("WikiLink");
|
||||
expect(
|
||||
findParentMatching(wikiLink, (n) => n.type === "BulletList")
|
||||
).toBeDefined();
|
||||
|
||||
let allTodos = collectNodesMatching(mdTree, (n) => n.type === "Task");
|
||||
expect(allTodos.length).toBe(2);
|
||||
|
||||
// Render back into markdown should be equivalent
|
||||
expect(renderToText(mdTree)).toBe(mdTest1);
|
||||
|
||||
removeParentPointers(mdTree);
|
||||
replaceNodesMatching(mdTree, (n) => {
|
||||
if (n.type === "Task") {
|
||||
return {
|
||||
type: "Tosk",
|
||||
};
|
||||
}
|
||||
});
|
||||
console.log(JSON.stringify(mdTree, null, 2));
|
||||
let mdTree3 = parse(lang, mdTest3);
|
||||
console.log(JSON.stringify(mdTree3, null, 2));
|
||||
});
|
||||
+122
-72
@@ -1,87 +1,137 @@
|
||||
import { SyntaxNode } from "@lezer/common";
|
||||
import wikiMarkdownLang from "../webapp/parser";
|
||||
|
||||
export type MarkdownTree = {
|
||||
export type ParseTree = {
|
||||
type?: string; // undefined === text node
|
||||
from?: number;
|
||||
to?: number;
|
||||
text?: string;
|
||||
children?: MarkdownTree[];
|
||||
children?: ParseTree[];
|
||||
// Only present after running addParentPointers
|
||||
parent?: ParseTree;
|
||||
};
|
||||
|
||||
function treeToAST(text: string, n: SyntaxNode, offset = 0): MarkdownTree {
|
||||
let children: MarkdownTree[] = [];
|
||||
let nodeText: string | undefined;
|
||||
let child = n.firstChild;
|
||||
while (child) {
|
||||
children.push(treeToAST(text, child));
|
||||
child = child.nextSibling;
|
||||
export function addParentPointers(tree: ParseTree) {
|
||||
if (!tree.children) {
|
||||
return;
|
||||
}
|
||||
for (let child of tree.children) {
|
||||
child.parent = tree;
|
||||
addParentPointers(child);
|
||||
}
|
||||
}
|
||||
|
||||
if (children.length === 0) {
|
||||
children = [
|
||||
{
|
||||
from: n.from + offset,
|
||||
to: n.to + offset,
|
||||
text: text.substring(n.from, n.to),
|
||||
},
|
||||
];
|
||||
} else {
|
||||
let newChildren: MarkdownTree[] | string = [];
|
||||
let index = n.from;
|
||||
export function removeParentPointers(tree: ParseTree) {
|
||||
delete tree.parent;
|
||||
if (!tree.children) {
|
||||
return;
|
||||
}
|
||||
for (let child of tree.children) {
|
||||
removeParentPointers(child);
|
||||
}
|
||||
}
|
||||
|
||||
export function findParentMatching(
|
||||
tree: ParseTree,
|
||||
matchFn: (tree: ParseTree) => boolean
|
||||
): ParseTree | null {
|
||||
let node = tree.parent;
|
||||
while (node) {
|
||||
if (matchFn(node)) {
|
||||
return node;
|
||||
}
|
||||
node = node.parent;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function collectNodesOfType(
|
||||
tree: ParseTree,
|
||||
nodeType: string
|
||||
): ParseTree[] {
|
||||
return collectNodesMatching(tree, (n) => n.type === nodeType);
|
||||
}
|
||||
|
||||
export function collectNodesMatching(
|
||||
tree: ParseTree,
|
||||
matchFn: (tree: ParseTree) => boolean
|
||||
): ParseTree[] {
|
||||
if (matchFn(tree)) {
|
||||
return [tree];
|
||||
}
|
||||
let results: ParseTree[] = [];
|
||||
if (tree.children) {
|
||||
for (let child of tree.children) {
|
||||
results = [...results, ...collectNodesMatching(child, matchFn)];
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
// return value: returning undefined = not matched, continue, null = delete, new node = replace
|
||||
export function replaceNodesMatching(
|
||||
tree: ParseTree,
|
||||
substituteFn: (tree: ParseTree) => ParseTree | null | undefined
|
||||
) {
|
||||
if (tree.children) {
|
||||
let children = tree.children.slice();
|
||||
for (let child of children) {
|
||||
let s = text.substring(index, child.from);
|
||||
if (s) {
|
||||
newChildren.push({
|
||||
from: index + offset,
|
||||
to: child.from! + offset,
|
||||
text: s,
|
||||
});
|
||||
let subst = substituteFn(child);
|
||||
if (subst !== undefined) {
|
||||
let pos = tree.children.indexOf(child);
|
||||
if (subst) {
|
||||
tree.children.splice(pos, 1, subst);
|
||||
} else {
|
||||
// null = delete
|
||||
tree.children.splice(pos, 1);
|
||||
}
|
||||
} else {
|
||||
replaceNodesMatching(child, substituteFn);
|
||||
}
|
||||
newChildren.push(child);
|
||||
index = child.to!;
|
||||
}
|
||||
let s = text.substring(index, n.to);
|
||||
if (s) {
|
||||
newChildren.push({ from: index + offset, to: n.to + offset, text: s });
|
||||
}
|
||||
children = newChildren;
|
||||
}
|
||||
|
||||
let result: MarkdownTree = {
|
||||
type: n.name,
|
||||
from: n.from + offset,
|
||||
to: n.to + offset,
|
||||
};
|
||||
if (children.length > 0) {
|
||||
result.children = children;
|
||||
}
|
||||
if (nodeText) {
|
||||
result.text = nodeText;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function parse(text: string): MarkdownTree {
|
||||
let tree = treeToAST(text, wikiMarkdownLang.parser.parse(text).topNode);
|
||||
// replaceNodesMatching(tree, (n): MarkdownTree | undefined | null => {
|
||||
// if (n.type === "FencedCode") {
|
||||
// let infoN = findNodeMatching(n, (n) => n.type === "CodeInfo");
|
||||
// let language = infoN!.children![0].text;
|
||||
// let textN = findNodeMatching(n, (n) => n.type === "CodeText");
|
||||
// let text = textN!.children![0].text!;
|
||||
//
|
||||
// console.log(language, text);
|
||||
// switch (language) {
|
||||
// case "yaml":
|
||||
// let parsed = StreamLanguage.define(yaml).parser.parse(text);
|
||||
// let subTree = treeToAST(text, parsed.topNode, n.from);
|
||||
// // console.log(JSON.stringify(subTree, null, 2));
|
||||
// subTree.type = "yaml";
|
||||
// return subTree;
|
||||
// }
|
||||
// }
|
||||
// return;
|
||||
// });
|
||||
return tree;
|
||||
export function findNodeMatching(
|
||||
tree: ParseTree,
|
||||
matchFn: (tree: ParseTree) => boolean
|
||||
): ParseTree | null {
|
||||
return collectNodesMatching(tree, matchFn)[0];
|
||||
}
|
||||
|
||||
export function findNodeOfType(
|
||||
tree: ParseTree,
|
||||
nodeType: string
|
||||
): ParseTree | null {
|
||||
return collectNodesMatching(tree, (n) => n.type === nodeType)[0];
|
||||
}
|
||||
|
||||
// Finds non-text node at position
|
||||
export function nodeAtPos(tree: ParseTree, pos: number): ParseTree | null {
|
||||
if (pos < tree.from! || pos > tree.to!) {
|
||||
return null;
|
||||
}
|
||||
if (!tree.children) {
|
||||
return tree;
|
||||
}
|
||||
for (let child of tree.children) {
|
||||
let n = nodeAtPos(child, pos);
|
||||
if (n && n.text !== undefined) {
|
||||
// Got a text node, let's return its parent
|
||||
return tree;
|
||||
} else if (n) {
|
||||
// Got it
|
||||
return n;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Turn ParseTree back into text
|
||||
export function renderToText(tree: ParseTree): string {
|
||||
let pieces: string[] = [];
|
||||
if (tree.text !== undefined) {
|
||||
return tree.text;
|
||||
}
|
||||
for (let child of tree.children!) {
|
||||
pieces.push(renderToText(child));
|
||||
}
|
||||
return pieces.join("");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user