Changed all indexes to use (pre) parsed trees.
This commit is contained in:
Zef Hemel
+3
-9
@@ -1,23 +1,17 @@
|
||||
// Index key space:
|
||||
// data:page@pos
|
||||
|
||||
import { IndexEvent } from "../../webapp/app_event";
|
||||
import { IndexTreeEvent } from "../../webapp/app_event";
|
||||
import { batchSet, scanPrefixGlobal } from "plugos-silverbullet-syscall";
|
||||
import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
|
||||
import { collectNodesOfType, findNodeOfType, ParseTree, replaceNodesMatching } from "../../common/tree";
|
||||
import YAML, { parse as parseYaml, parseAllDocuments } from "yaml";
|
||||
import { whiteOutQueries } from "./util";
|
||||
import type { QueryProviderEvent } from "./engine";
|
||||
import { applyQuery } from "./engine";
|
||||
|
||||
export async function indexData({ name, text }: IndexEvent) {
|
||||
text = whiteOutQueries(text);
|
||||
// console.log("Now data indexing", name);
|
||||
let mdTree = await parseMarkdown(text);
|
||||
|
||||
export async function indexData({ name, tree }: IndexTreeEvent) {
|
||||
let dataObjects: { key: string; value: Object }[] = [];
|
||||
|
||||
collectNodesOfType(mdTree, "FencedCode").forEach((t) => {
|
||||
collectNodesOfType(tree, "FencedCode").forEach((t) => {
|
||||
let codeInfoNode = findNodeOfType(t, "CodeInfo");
|
||||
if (!codeInfoNode) {
|
||||
return;
|
||||
|
||||
@@ -1,30 +1,18 @@
|
||||
import { flashNotification, getCurrentPage, reloadPage, save } from "plugos-silverbullet-syscall/editor";
|
||||
import { flashNotification, getCurrentPage, getText, reloadPage, save } from "plugos-silverbullet-syscall/editor";
|
||||
|
||||
import { readPage, writePage } from "plugos-silverbullet-syscall/space";
|
||||
import { invokeFunction } from "plugos-silverbullet-syscall/system";
|
||||
import { parseQuery } from "./engine";
|
||||
import { replaceTemplateVars } from "../core/template";
|
||||
import { queryRegex } from "./util";
|
||||
import { queryRegex, removeQueries } from "./util";
|
||||
import { dispatch } from "plugos-syscall/event";
|
||||
|
||||
async function replaceAsync(
|
||||
str: string,
|
||||
regex: RegExp,
|
||||
asyncFn: (match: string, ...args: any[]) => Promise<string>
|
||||
) {
|
||||
const promises: Promise<string>[] = [];
|
||||
str.replace(regex, (match: string, ...args: any[]): string => {
|
||||
const promise = asyncFn(match, ...args);
|
||||
promises.push(promise);
|
||||
return "";
|
||||
});
|
||||
const data = await Promise.all(promises);
|
||||
return str.replace(regex, () => data.shift()!);
|
||||
}
|
||||
import { replaceAsync } from "../lib/util";
|
||||
import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
|
||||
|
||||
export async function updateMaterializedQueriesCommand() {
|
||||
const currentPage = await getCurrentPage();
|
||||
await save();
|
||||
await flashNotification("Updating materialized queries...");
|
||||
await invokeFunction(
|
||||
"server",
|
||||
"updateMaterializedQueriesOnPage",
|
||||
@@ -34,6 +22,12 @@ export async function updateMaterializedQueriesCommand() {
|
||||
await flashNotification("Updated materialized queries");
|
||||
}
|
||||
|
||||
export async function whiteOutQueriesCommand() {
|
||||
const text = await getText();
|
||||
const parsed = await parseMarkdown(text);
|
||||
console.log(removeQueries(parsed));
|
||||
}
|
||||
|
||||
// Called from client, running on server
|
||||
export async function updateMaterializedQueriesOnPage(pageName: string) {
|
||||
let { text } = await readPage(pageName);
|
||||
@@ -49,7 +43,7 @@ export async function updateMaterializedQueriesOnPage(pageName: string) {
|
||||
let results = await dispatch(
|
||||
`query:${parsedQuery.table}`,
|
||||
{ query: parsedQuery, pageName: pageName },
|
||||
5000
|
||||
10 * 1000
|
||||
);
|
||||
if (results.length === 0) {
|
||||
return `${startQuery}\n${endQuery}`;
|
||||
|
||||
@@ -3,15 +3,19 @@ import { LRParser } from "@lezer/lr";
|
||||
|
||||
export const parser = LRParser.deserialize({
|
||||
version: 13,
|
||||
states: "$[OVQPOOO[QQO'#C^QOQPOOOjQPO'#C`OoQQO'#CjOtQPO'#ClOOQO'#Cm'#CmOyQQO,58xO!XQPO'#CcO!sQQO'#CaOOQO'#Ca'#CaOOQO,58z,58zO#UQPO,59UOOQO,59W,59WOOQO-E6k-E6kO#ZQQO,58}OjQPO,58|O#oQQO1G.pOOQO'#Cg'#CgOOQO'#Ci'#CiOOQO'#Cd'#CdOOQO1G.i1G.iOOQO1G.h1G.hOOQO'#Ck'#CkOOQO7+$[7+$[",
|
||||
stateData: "$T~OdOS~ORPO~OeROrSOvTObQX~ORWO~Os[O~OX]O~OeROrSOvTObQa~Of_Oj_Ok_Ol_Om_On_Oo_Op_O~Oq`ObTXeTXrTXvTX~ORaO~OXdOYdO[dOgbOhbOicO~OtgOugOb^ie^ir^iv^i~O",
|
||||
states:
|
||||
"$[OVQPOOO[QQO'#C^QOQPOOOjQPO'#C`OoQQO'#CjOtQPO'#ClOOQO'#Cm'#CmOyQQO,58xO!XQPO'#CcO!sQQO'#CaOOQO'#Ca'#CaOOQO,58z,58zO#UQPO,59UOOQO,59W,59WOOQO-E6k-E6kO#ZQQO,58}OjQPO,58|O#oQQO1G.pOOQO'#Cg'#CgOOQO'#Ci'#CiOOQO'#Cd'#CdOOQO1G.i1G.iOOQO1G.h1G.hOOQO'#Ck'#CkOOQO7+$[7+$[",
|
||||
stateData:
|
||||
"$T~OdOS~ORPO~OeROrSOvTObQX~ORWO~Os[O~OX]O~OeROrSOvTObQa~Of_Oj_Ok_Ol_Om_On_Oo_Op_O~Oq`ObTXeTXrTXvTX~ORaO~OXdOYdO[dOgbOhbOicO~OtgOugOb^ie^ir^iv^i~O",
|
||||
goto: "!VbPPcPfjmpvPPyPyf|f!PRQOTUPVRZRRYRQXRRf`Re_Rd_RhaQVPR^V",
|
||||
nodeNames: "⚠ Program Query Name WhereClause LogicalExpr AndExpr FilterExpr Value Number String Bool Regex Null OrderClause Order LimitClause",
|
||||
nodeNames:
|
||||
"⚠ Program Query Name WhereClause LogicalExpr AndExpr FilterExpr Value Number String Bool Regex Null OrderClause Order LimitClause",
|
||||
maxTerm: 38,
|
||||
skippedNodes: [0],
|
||||
repeatNodeCount: 1,
|
||||
tokenData: ":W~RvX^#ipq#iqr$^rs$q}!O%]!P!Q%n!Q![&e!^!_&m!_!`&z!`!a'X!c!}%]#R#S%]#T#U'f#U#V){#V#W%]#W#X*w#X#Y%]#Y#Z,s#Z#`%]#`#a/T#a#b%]#b#c1h#c#d3d#d#h%]#h#i5w#i#k%]#k#l7s#l#o%]#y#z#i$f$g#i#BY#BZ#i$IS$I_#i$Ip$Iq$q$Iq$Ir$q$I|$JO#i$JT$JU#i$KV$KW#i&FU&FV#i~#nYd~X^#ipq#i#y#z#i$f$g#i#BY#BZ#i$IS$I_#i$I|$JO#i$JT$JU#i$KV$KW#i&FU&FV#i~$aP!_!`$d~$iPl~#r#s$l~$qOp~~$tUOr$qrs%Ws$Ip$q$Ip$Iq%W$Iq$Ir%W$Ir~$q~%]OY~P%bSRP}!O%]!c!}%]#R#S%]#T#o%]~%sV[~OY%nZ]%n^!P%n!P!Q&Y!Q#O%n#O#P&_#P~%n~&_O[~~&bPO~%n~&jPX~!Q![&e~&rPf~!_!`&u~&zOj~~'PPk~#r#s'S~'XOo~~'^Pn~!_!`'a~'fOm~R'kWRP}!O%]!c!}%]#R#S%]#T#b%]#b#c(T#c#g%]#g#h)P#h#o%]R(YURP}!O%]!c!}%]#R#S%]#T#W%]#W#X(l#X#o%]R(sSqQRP}!O%]!c!}%]#R#S%]#T#o%]R)UURP}!O%]!c!}%]#R#S%]#T#V%]#V#W)h#W#o%]R)oSuQRP}!O%]!c!}%]#R#S%]#T#o%]R*QURP}!O%]!c!}%]#R#S%]#T#m%]#m#n*d#n#o%]R*kSsQRP}!O%]!c!}%]#R#S%]#T#o%]R*|URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y+`#Y#o%]R+eURP}!O%]!c!}%]#R#S%]#T#g%]#g#h+w#h#o%]R+|URP}!O%]!c!}%]#R#S%]#T#V%]#V#W,`#W#o%]R,gStQRP}!O%]!c!}%]#R#S%]#T#o%]R,xTRP}!O%]!c!}%]#R#S%]#T#U-X#U#o%]R-^URP}!O%]!c!}%]#R#S%]#T#`%]#`#a-p#a#o%]R-uURP}!O%]!c!}%]#R#S%]#T#g%]#g#h.X#h#o%]R.^URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y.p#Y#o%]R.wShQRP}!O%]!c!}%]#R#S%]#T#o%]R/YURP}!O%]!c!}%]#R#S%]#T#]%]#]#^/l#^#o%]R/qURP}!O%]!c!}%]#R#S%]#T#a%]#a#b0T#b#o%]R0YURP}!O%]!c!}%]#R#S%]#T#]%]#]#^0l#^#o%]R0qURP}!O%]!c!}%]#R#S%]#T#h%]#h#i1T#i#o%]R1[SvQRP}!O%]!c!}%]#R#S%]#T#o%]R1mURP}!O%]!c!}%]#R#S%]#T#i%]#i#j2P#j#o%]R2UURP}!O%]!c!}%]#R#S%]#T#`%]#`#a2h#a#o%]R2mURP}!O%]!c!}%]#R#S%]#T#`%]#`#a3P#a#o%]R3WSiQRP}!O%]!c!}%]#R#S%]#T#o%]R3iURP}!O%]!c!}%]#R#S%]#T#f%]#f#g3{#g#o%]R4QURP}!O%]!c!}%]#R#S%]#T#W%]#W#X4d#X#o%]R4iURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y4{#Y#o%]R5QURP}!O%]!c!}%]#R#S%]#T#f%]#f#g5d#g#o%]R5kSrQRP}!O%]!c!}%]#R#S%]#T#o%]R5|URP}!O%]!c!}%]#R#S%]#T#f%]#f#g6`#g#o%]R6eURP}!O%]!c!}%]#R#S%]#T#i%]#i#j6w#j#o%]R6|URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y7`#Y#o%]R7gSgQRP}!O%]!c!}%]#R#S%]#T#o%]R7xURP}!O%]!c!}%]#R#S%]#T#[%]#[#]8[#]#o%]R8aURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y8s#Y#o%]R8xURP}!O%]!c!}%]#R#S%]#T#f%]#f#g9[#g#o%]R9aURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y9s#Y#o%]R9zSeQRP}!O%]!c!}%]#R#S%]#T#o%]",
|
||||
tokenData:
|
||||
":W~RvX^#ipq#iqr$^rs$q}!O%]!P!Q%n!Q![&e!^!_&m!_!`&z!`!a'X!c!}%]#R#S%]#T#U'f#U#V){#V#W%]#W#X*w#X#Y%]#Y#Z,s#Z#`%]#`#a/T#a#b%]#b#c1h#c#d3d#d#h%]#h#i5w#i#k%]#k#l7s#l#o%]#y#z#i$f$g#i#BY#BZ#i$IS$I_#i$Ip$Iq$q$Iq$Ir$q$I|$JO#i$JT$JU#i$KV$KW#i&FU&FV#i~#nYd~X^#ipq#i#y#z#i$f$g#i#BY#BZ#i$IS$I_#i$I|$JO#i$JT$JU#i$KV$KW#i&FU&FV#i~$aP!_!`$d~$iPl~#r#s$l~$qOp~~$tUOr$qrs%Ws$Ip$q$Ip$Iq%W$Iq$Ir%W$Ir~$q~%]OY~P%bSRP}!O%]!c!}%]#R#S%]#T#o%]~%sV[~OY%nZ]%n^!P%n!P!Q&Y!Q#O%n#O#P&_#P~%n~&_O[~~&bPO~%n~&jPX~!Q![&e~&rPf~!_!`&u~&zOj~~'PPk~#r#s'S~'XOo~~'^Pn~!_!`'a~'fOm~R'kWRP}!O%]!c!}%]#R#S%]#T#b%]#b#c(T#c#g%]#g#h)P#h#o%]R(YURP}!O%]!c!}%]#R#S%]#T#W%]#W#X(l#X#o%]R(sSqQRP}!O%]!c!}%]#R#S%]#T#o%]R)UURP}!O%]!c!}%]#R#S%]#T#V%]#V#W)h#W#o%]R)oSuQRP}!O%]!c!}%]#R#S%]#T#o%]R*QURP}!O%]!c!}%]#R#S%]#T#m%]#m#n*d#n#o%]R*kSsQRP}!O%]!c!}%]#R#S%]#T#o%]R*|URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y+`#Y#o%]R+eURP}!O%]!c!}%]#R#S%]#T#g%]#g#h+w#h#o%]R+|URP}!O%]!c!}%]#R#S%]#T#V%]#V#W,`#W#o%]R,gStQRP}!O%]!c!}%]#R#S%]#T#o%]R,xTRP}!O%]!c!}%]#R#S%]#T#U-X#U#o%]R-^URP}!O%]!c!}%]#R#S%]#T#`%]#`#a-p#a#o%]R-uURP}!O%]!c!}%]#R#S%]#T#g%]#g#h.X#h#o%]R.^URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y.p#Y#o%]R.wShQRP}!O%]!c!}%]#R#S%]#T#o%]R/YURP}!O%]!c!}%]#R#S%]#T#]%]#]#^/l#^#o%]R/qURP}!O%]!c!}%]#R#S%]#T#a%]#a#b0T#b#o%]R0YURP}!O%]!c!}%]#R#S%]#T#]%]#]#^0l#^#o%]R0qURP}!O%]!c!}%]#R#S%]#T#h%]#h#i1T#i#o%]R1[SvQRP}!O%]!c!}%]#R#S%]#T#o%]R1mURP}!O%]!c!}%]#R#S%]#T#i%]#i#j2P#j#o%]R2UURP}!O%]!c!}%]#R#S%]#T#`%]#`#a2h#a#o%]R2mURP}!O%]!c!}%]#R#S%]#T#`%]#`#a3P#a#o%]R3WSiQRP}!O%]!c!}%]#R#S%]#T#o%]R3iURP}!O%]!c!}%]#R#S%]#T#f%]#f#g3{#g#o%]R4QURP}!O%]!c!}%]#R#S%]#T#W%]#W#X4d#X#o%]R4iURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y4{#Y#o%]R5QURP}!O%]!c!}%]#R#S%]#T#f%]#f#g5d#g#o%]R5kSrQRP}!O%]!c!}%]#R#S%]#T#o%]R5|URP}!O%]!c!}%]#R#S%]#T#f%]#f#g6`#g#o%]R6eURP}!O%]!c!}%]#R#S%]#T#i%]#i#j6w#j#o%]R6|URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y7`#Y#o%]R7gSgQRP}!O%]!c!}%]#R#S%]#T#o%]R7xURP}!O%]!c!}%]#R#S%]#T#[%]#[#]8[#]#o%]R8aURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y8s#Y#o%]R8xURP}!O%]!c!}%]#R#S%]#T#f%]#f#g9[#g#o%]R9aURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y9s#Y#o%]R9zSeQRP}!O%]!c!}%]#R#S%]#T#o%]",
|
||||
tokenizers: [0, 1],
|
||||
topRules: {"Program":[0,1]},
|
||||
tokenPrec: 0
|
||||
})
|
||||
topRules: { Program: [0, 1] },
|
||||
tokenPrec: 0,
|
||||
});
|
||||
|
||||
@@ -6,6 +6,10 @@ functions:
|
||||
command:
|
||||
name: "Materialized Queries: Update"
|
||||
key: "Alt-q"
|
||||
whiteOutQueriesCommand:
|
||||
path: ./materialized_queries.ts:whiteOutQueriesCommand
|
||||
command:
|
||||
name: "Debug: Whiteout Queries"
|
||||
indexData:
|
||||
path: ./data.ts:indexData
|
||||
events:
|
||||
|
||||
+41
-4
@@ -1,8 +1,45 @@
|
||||
import { addParentPointers, collectNodesMatching, ParseTree, renderToText } from "../../common/tree";
|
||||
|
||||
export const queryRegex =
|
||||
/(<!--\s*#query\s+(.+?)-->)(.+?)(<!--\s*#end\s*-->)/gs;
|
||||
|
||||
export function whiteOutQueries(text: string): string {
|
||||
return text.replaceAll(queryRegex, (match) =>
|
||||
new Array(match.length + 1).join(" ")
|
||||
);
|
||||
export const queryStartRegex = /<!--\s*#query\s+(.+?)-->/s;
|
||||
|
||||
export const queryEndRegex = /<!--\s*#end\s*-->/s;
|
||||
|
||||
// export function whiteOutQueries(text: string): string {
|
||||
// return text.replaceAll(queryRegex, (match) =>
|
||||
// new Array(match.length + 1).join(" ")
|
||||
// );
|
||||
// }
|
||||
|
||||
export function removeQueries(pt: ParseTree) {
|
||||
addParentPointers(pt);
|
||||
collectNodesMatching(pt, (t) => {
|
||||
if (t.type !== "CommentBlock") {
|
||||
return false;
|
||||
}
|
||||
let text = t.children![0].text!;
|
||||
if (!queryStartRegex.exec(text)) {
|
||||
return false;
|
||||
}
|
||||
let parentChildren = t.parent!.children!;
|
||||
let index = parentChildren.indexOf(t);
|
||||
let nodesToReplace: ParseTree[] = [];
|
||||
for (let i = index + 1; i < parentChildren.length; i++) {
|
||||
let n = parentChildren[i];
|
||||
if (n.type === "CommentBlock") {
|
||||
let text = n.children![0].text!;
|
||||
if (queryEndRegex.exec(text)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
nodesToReplace.push(n);
|
||||
}
|
||||
let renderedText = nodesToReplace.map(renderToText).join("");
|
||||
parentChildren.splice(index + 1, nodesToReplace.length, {
|
||||
text: new Array(renderedText.length + 1).join(" "),
|
||||
});
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user