Factored out materialized query providers
This commit is contained in:
Zef Hemel
+22
-3
@@ -2,14 +2,15 @@
|
||||
// data:page@pos
|
||||
|
||||
import { IndexEvent } from "../../webapp/app_event";
|
||||
import { batchSet } from "plugos-silverbullet-syscall";
|
||||
import { batchSet, scanPrefixGlobal } from "plugos-silverbullet-syscall";
|
||||
import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
|
||||
import { collectNodesOfType, findNodeOfType, ParseTree, replaceNodesMatching } from "../../common/tree";
|
||||
import { parse as parseYaml, parseAllDocuments } from "yaml";
|
||||
import YAML, { parse as parseYaml, parseAllDocuments } from "yaml";
|
||||
import { whiteOutQueries } from "./util";
|
||||
import type { QueryProviderEvent } from "./engine";
|
||||
import { applyQuery } from "./engine";
|
||||
|
||||
export async function indexData({ name, text }: IndexEvent) {
|
||||
let e;
|
||||
text = whiteOutQueries(text);
|
||||
// console.log("Now data indexing", name);
|
||||
let mdTree = await parseMarkdown(text);
|
||||
@@ -77,3 +78,21 @@ export function extractMeta(parseTree: ParseTree, remove = false): any {
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
export async function queryProvider({
|
||||
query,
|
||||
}: QueryProviderEvent): Promise<string> {
|
||||
let allData: any[] = [];
|
||||
for (let { key, page, value } of await scanPrefixGlobal("data:")) {
|
||||
let [, pos] = key.split("@");
|
||||
allData.push({
|
||||
...value,
|
||||
page: page,
|
||||
pos: +pos,
|
||||
});
|
||||
}
|
||||
let markdownData = applyQuery(query, allData).map((item) =>
|
||||
YAML.stringify(item)
|
||||
);
|
||||
return `\`\`\`data\n${markdownData.join("---\n")}\`\`\``;
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ test("Test parser", () => {
|
||||
expect(parsedQuery2.filter[0]).toStrictEqual({
|
||||
op: "=~",
|
||||
prop: "name",
|
||||
value: /interview\/.*/,
|
||||
value: "interview\\/.*",
|
||||
});
|
||||
|
||||
let parsedQuery3 = parseQuery(`page where something != null`);
|
||||
|
||||
+11
-5
@@ -4,13 +4,18 @@ import { lezerToParseTree } from "../../common/parse_tree";
|
||||
// @ts-ignore
|
||||
import { parser } from "./parse-query";
|
||||
|
||||
type Filter = {
|
||||
export type QueryProviderEvent = {
|
||||
query: ParsedQuery;
|
||||
pageName: string;
|
||||
};
|
||||
|
||||
export type Filter = {
|
||||
op: string;
|
||||
prop: string;
|
||||
value: any;
|
||||
};
|
||||
|
||||
type ParsedQuery = {
|
||||
export type ParsedQuery = {
|
||||
table: string;
|
||||
orderBy?: string;
|
||||
orderDesc?: boolean;
|
||||
@@ -71,7 +76,7 @@ export function parseQuery(query: string): ParsedQuery {
|
||||
break;
|
||||
case "Regex":
|
||||
val = valNode.children![0].text!;
|
||||
val = new RegExp(val.substring(1, val.length - 1));
|
||||
val = val.substring(1, val.length - 1);
|
||||
break;
|
||||
case "String":
|
||||
val = valNode.children![0].text!;
|
||||
@@ -129,12 +134,13 @@ export function applyQuery<T>(parsedQuery: ParsedQuery, records: T[]): T[] {
|
||||
}
|
||||
break;
|
||||
case "=~":
|
||||
if (!value.exec(recordAny[prop])) {
|
||||
// TODO: Cache regexps somehow
|
||||
if (!new RegExp(value).exec(recordAny[prop])) {
|
||||
continue recordLoop;
|
||||
}
|
||||
break;
|
||||
case "!=~":
|
||||
if (value.exec(recordAny[prop])) {
|
||||
if (new RegExp(value).exec(recordAny[prop])) {
|
||||
continue recordLoop;
|
||||
}
|
||||
break;
|
||||
|
||||
@@ -1,15 +1,11 @@
|
||||
import { flashNotification, getCurrentPage, reloadPage, save } from "plugos-silverbullet-syscall/editor";
|
||||
|
||||
import { listPages, readPage, writePage } from "plugos-silverbullet-syscall/space";
|
||||
import { readPage, writePage } from "plugos-silverbullet-syscall/space";
|
||||
import { invokeFunction } from "plugos-silverbullet-syscall/system";
|
||||
import { scanPrefixGlobal } from "plugos-silverbullet-syscall";
|
||||
import { applyQuery, parseQuery } from "./engine";
|
||||
import { PageMeta } from "../../common/types";
|
||||
import type { Task } from "../tasks/task";
|
||||
import { Item } from "../core/item";
|
||||
import YAML from "yaml";
|
||||
import { parseQuery } from "./engine";
|
||||
import { replaceTemplateVars } from "../core/template";
|
||||
import { queryRegex } from "./util";
|
||||
import { dispatch } from "plugos-syscall/event";
|
||||
|
||||
async function replaceAsync(
|
||||
str: string,
|
||||
@@ -46,79 +42,22 @@ export async function updateMaterializedQueriesOnPage(pageName: string) {
|
||||
text,
|
||||
queryRegex,
|
||||
async (fullMatch, startQuery, query, body, endQuery) => {
|
||||
let parsedQuery = parseQuery(replaceTemplateVars(query));
|
||||
let parsedQuery = parseQuery(replaceTemplateVars(query, pageName));
|
||||
|
||||
console.log("Parsed query", parsedQuery);
|
||||
|
||||
switch (parsedQuery.table) {
|
||||
case "page":
|
||||
let allPages = await listPages();
|
||||
let markdownPages = applyQuery(parsedQuery, allPages).map(
|
||||
(pageMeta: PageMeta) => `* [[${pageMeta.name}]]`
|
||||
);
|
||||
return `${startQuery}\n${markdownPages.join("\n")}\n${endQuery}`;
|
||||
case "task":
|
||||
let allTasks: Task[] = [];
|
||||
for (let { key, page, value } of await scanPrefixGlobal("task:")) {
|
||||
let [, pos] = key.split(":");
|
||||
allTasks.push({
|
||||
...value,
|
||||
page: page,
|
||||
pos: pos,
|
||||
});
|
||||
}
|
||||
let markdownTasks = applyQuery(parsedQuery, allTasks).map(
|
||||
(t) =>
|
||||
`* [${t.done ? "x" : " "}] [[${t.page}@${t.pos}]] ${t.name}` +
|
||||
(t.nested ? "\n " + t.nested : "")
|
||||
);
|
||||
return `${startQuery}\n${markdownTasks.join("\n")}\n${endQuery}`;
|
||||
case "link":
|
||||
let uniqueLinks = new Set<string>();
|
||||
for (let { value: name } of await scanPrefixGlobal(
|
||||
`pl:${pageName}:`
|
||||
)) {
|
||||
uniqueLinks.add(name);
|
||||
}
|
||||
let markdownLinks = applyQuery(
|
||||
parsedQuery,
|
||||
[...uniqueLinks].map((l) => ({ name: l }))
|
||||
).map((pageMeta) => `* [[${pageMeta.name}]]`);
|
||||
return `${startQuery}\n${markdownLinks.join("\n")}\n${endQuery}`;
|
||||
case "item":
|
||||
let allItems: Item[] = [];
|
||||
for (let { key, page, value } of await scanPrefixGlobal("it:")) {
|
||||
let [, pos] = key.split(":");
|
||||
allItems.push({
|
||||
...value,
|
||||
page: page,
|
||||
pos: +pos,
|
||||
});
|
||||
}
|
||||
let markdownItems = applyQuery(parsedQuery, allItems).map(
|
||||
(item) =>
|
||||
`* [[${item.page}@${item.pos}]] ${item.name}` +
|
||||
(item.nested ? "\n " + item.nested : "")
|
||||
);
|
||||
return `${startQuery}\n${markdownItems.join("\n")}\n${endQuery}`;
|
||||
case "data":
|
||||
let allData: Object[] = [];
|
||||
for (let { key, page, value } of await scanPrefixGlobal("data:")) {
|
||||
let [, pos] = key.split("@");
|
||||
allData.push({
|
||||
...value,
|
||||
page: page,
|
||||
pos: +pos,
|
||||
});
|
||||
}
|
||||
let markdownData = applyQuery(parsedQuery, allData).map((item) =>
|
||||
YAML.stringify(item)
|
||||
);
|
||||
return `${startQuery}\n\`\`\`data\n${markdownData.join(
|
||||
"---\n"
|
||||
)}\`\`\`\n${endQuery}`;
|
||||
default:
|
||||
return fullMatch;
|
||||
// Let's dispatch an event and see what happens
|
||||
let results = await dispatch(
|
||||
`query:${parsedQuery.table}`,
|
||||
{ query: parsedQuery, pageName: pageName },
|
||||
5000
|
||||
);
|
||||
if (results.length === 0) {
|
||||
return `${startQuery}\n${endQuery}`;
|
||||
} else if (results.length === 1) {
|
||||
return `${startQuery}\n${results[0]}\n${endQuery}`;
|
||||
} else {
|
||||
console.error("Too many query results", results);
|
||||
return fullMatch;
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
@@ -10,7 +10,7 @@ export const parser = LRParser.deserialize({
|
||||
maxTerm: 38,
|
||||
skippedNodes: [0],
|
||||
repeatNodeCount: 1,
|
||||
tokenData: "4v~RtX^#cpq#cqr$Wrs$k!P!Q%V!Q![%|!^!_&U!_!`&c!`!a&p!c!}&}#T#U'Y#U#V)P#V#W&}#W#X)o#X#Y&}#Y#Z+R#Z#`&}#`#a,s#a#b&}#b#c.h#c#d/z#d#h&}#h#i1o#i#k&}#k#l3R#l#o&}#y#z#c$f$g#c#BY#BZ#c$IS$I_#c$Ip$Iq$k$Iq$Ir$k$I|$JO#c$JT$JU#c$KV$KW#c&FU&FV#c~#hYd~X^#cpq#c#y#z#c$f$g#c#BY#BZ#c$IS$I_#c$I|$JO#c$JT$JU#c$KV$KW#c&FU&FV#c~$ZP!_!`$^~$cPl~#r#s$f~$kOp~~$nUOr$krs%Qs$Ip$k$Ip$Iq%Q$Iq$Ir%Q$Ir~$k~%VOY~~%[V[~OY%VZ]%V^!P%V!P!Q%q!Q#O%V#O#P%v#P~%V~%vO[~~%yPO~%V~&RPX~!Q![%|~&ZPf~!_!`&^~&cOj~~&hPk~#r#s&k~&pOo~~&uPn~!_!`&x~&}Om~P'SQRP!c!}&}#T#o&}R'_URP!c!}&}#T#b&}#b#c'q#c#g&}#g#h(a#h#o&}R'vSRP!c!}&}#T#W&}#W#X(S#X#o&}R(ZQqQRP!c!}&}#T#o&}R(fSRP!c!}&}#T#V&}#V#W(r#W#o&}R(yQuQRP!c!}&}#T#o&}R)USRP!c!}&}#T#m&}#m#n)b#n#o&}R)iQsQRP!c!}&}#T#o&}R)tSRP!c!}&}#T#X&}#X#Y*Q#Y#o&}R*VSRP!c!}&}#T#g&}#g#h*c#h#o&}R*hSRP!c!}&}#T#V&}#V#W*t#W#o&}R*{QtQRP!c!}&}#T#o&}R+WRRP!c!}&}#T#U+a#U#o&}R+fSRP!c!}&}#T#`&}#`#a+r#a#o&}R+wSRP!c!}&}#T#g&}#g#h,T#h#o&}R,YSRP!c!}&}#T#X&}#X#Y,f#Y#o&}R,mQhQRP!c!}&}#T#o&}R,xSRP!c!}&}#T#]&}#]#^-U#^#o&}R-ZSRP!c!}&}#T#a&}#a#b-g#b#o&}R-lSRP!c!}&}#T#]&}#]#^-x#^#o&}R-}SRP!c!}&}#T#h&}#h#i.Z#i#o&}R.bQvQRP!c!}&}#T#o&}R.mSRP!c!}&}#T#i&}#i#j.y#j#o&}R/OSRP!c!}&}#T#`&}#`#a/[#a#o&}R/aSRP!c!}&}#T#`&}#`#a/m#a#o&}R/tQiQRP!c!}&}#T#o&}R0PSRP!c!}&}#T#f&}#f#g0]#g#o&}R0bSRP!c!}&}#T#W&}#W#X0n#X#o&}R0sSRP!c!}&}#T#X&}#X#Y1P#Y#o&}R1USRP!c!}&}#T#f&}#f#g1b#g#o&}R1iQrQRP!c!}&}#T#o&}R1tSRP!c!}&}#T#f&}#f#g2Q#g#o&}R2VSRP!c!}&}#T#i&}#i#j2c#j#o&}R2hSRP!c!}&}#T#X&}#X#Y2t#Y#o&}R2{QgQRP!c!}&}#T#o&}R3WSRP!c!}&}#T#[&}#[#]3d#]#o&}R3iSRP!c!}&}#T#X&}#X#Y3u#Y#o&}R3zSRP!c!}&}#T#f&}#f#g4W#g#o&}R4]SRP!c!}&}#T#X&}#X#Y4i#Y#o&}R4pQeQRP!c!}&}#T#o&}",
|
||||
tokenData: ":W~RvX^#ipq#iqr$^rs$q}!O%]!P!Q%n!Q![&e!^!_&m!_!`&z!`!a'X!c!}%]#R#S%]#T#U'f#U#V){#V#W%]#W#X*w#X#Y%]#Y#Z,s#Z#`%]#`#a/T#a#b%]#b#c1h#c#d3d#d#h%]#h#i5w#i#k%]#k#l7s#l#o%]#y#z#i$f$g#i#BY#BZ#i$IS$I_#i$Ip$Iq$q$Iq$Ir$q$I|$JO#i$JT$JU#i$KV$KW#i&FU&FV#i~#nYd~X^#ipq#i#y#z#i$f$g#i#BY#BZ#i$IS$I_#i$I|$JO#i$JT$JU#i$KV$KW#i&FU&FV#i~$aP!_!`$d~$iPl~#r#s$l~$qOp~~$tUOr$qrs%Ws$Ip$q$Ip$Iq%W$Iq$Ir%W$Ir~$q~%]OY~P%bSRP}!O%]!c!}%]#R#S%]#T#o%]~%sV[~OY%nZ]%n^!P%n!P!Q&Y!Q#O%n#O#P&_#P~%n~&_O[~~&bPO~%n~&jPX~!Q![&e~&rPf~!_!`&u~&zOj~~'PPk~#r#s'S~'XOo~~'^Pn~!_!`'a~'fOm~R'kWRP}!O%]!c!}%]#R#S%]#T#b%]#b#c(T#c#g%]#g#h)P#h#o%]R(YURP}!O%]!c!}%]#R#S%]#T#W%]#W#X(l#X#o%]R(sSqQRP}!O%]!c!}%]#R#S%]#T#o%]R)UURP}!O%]!c!}%]#R#S%]#T#V%]#V#W)h#W#o%]R)oSuQRP}!O%]!c!}%]#R#S%]#T#o%]R*QURP}!O%]!c!}%]#R#S%]#T#m%]#m#n*d#n#o%]R*kSsQRP}!O%]!c!}%]#R#S%]#T#o%]R*|URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y+`#Y#o%]R+eURP}!O%]!c!}%]#R#S%]#T#g%]#g#h+w#h#o%]R+|URP}!O%]!c!}%]#R#S%]#T#V%]#V#W,`#W#o%]R,gStQRP}!O%]!c!}%]#R#S%]#T#o%]R,xTRP}!O%]!c!}%]#R#S%]#T#U-X#U#o%]R-^URP}!O%]!c!}%]#R#S%]#T#`%]#`#a-p#a#o%]R-uURP}!O%]!c!}%]#R#S%]#T#g%]#g#h.X#h#o%]R.^URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y.p#Y#o%]R.wShQRP}!O%]!c!}%]#R#S%]#T#o%]R/YURP}!O%]!c!}%]#R#S%]#T#]%]#]#^/l#^#o%]R/qURP}!O%]!c!}%]#R#S%]#T#a%]#a#b0T#b#o%]R0YURP}!O%]!c!}%]#R#S%]#T#]%]#]#^0l#^#o%]R0qURP}!O%]!c!}%]#R#S%]#T#h%]#h#i1T#i#o%]R1[SvQRP}!O%]!c!}%]#R#S%]#T#o%]R1mURP}!O%]!c!}%]#R#S%]#T#i%]#i#j2P#j#o%]R2UURP}!O%]!c!}%]#R#S%]#T#`%]#`#a2h#a#o%]R2mURP}!O%]!c!}%]#R#S%]#T#`%]#`#a3P#a#o%]R3WSiQRP}!O%]!c!}%]#R#S%]#T#o%]R3iURP}!O%]!c!}%]#R#S%]#T#f%]#f#g3{#g#o%]R4QURP}!O%]!c!}%]#R#S%]#T#W%]#W#X4d#X#o%]R4iURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y4{#Y#o%]R5QURP}!O%]!c!}%]#R#S%]#T#f%]#f#g5d#g#o%]R5kSrQRP}!O%]!c!}%]#R#S%]#T#o%]R5|URP}!O%]!c!}%]#R#S%]#T#f%]#f#g6`#g#o%]R6eURP}!O%]!c!}%]#R#S%]#T#i%]#i#j6w#j#o%]R6|URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y7`#Y#o%]R7gSgQRP}!O%]!c!}%]#R#S%]#T#o%]R7xURP}!O%]!c!}%]#R#S%]#T#[%]#[#]8[#]#o%]R8aURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y8s#Y#o%]R8xURP}!O%]!c!}%]#R#S%]#T#f%]#f#g9[#g#o%]R9aURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y9s#Y#o%]R9zSeQRP}!O%]!c!}%]#R#S%]#T#o%]",
|
||||
tokenizers: [0, 1],
|
||||
topRules: {"Program":[0,1]},
|
||||
tokenPrec: 0
|
||||
|
||||
@@ -43,7 +43,7 @@ Null {
|
||||
|
||||
@tokens {
|
||||
space { std.whitespace+ }
|
||||
Name { std.asciiLetter+ }
|
||||
Name { (std.asciiLetter | "-" | "_")+ }
|
||||
String {
|
||||
("\"" | "“" | "”") ![\"”“]* ("\"" | "“" | "”")
|
||||
}
|
||||
|
||||
@@ -10,3 +10,7 @@ functions:
|
||||
path: ./data.ts:indexData
|
||||
events:
|
||||
- page:index
|
||||
dataQueryProvider:
|
||||
path: ./data.ts:queryProvider
|
||||
events:
|
||||
- query:data
|
||||
|
||||
Reference in New Issue
Block a user