Materialized query parser redo
This commit is contained in:
Zef Hemel
@@ -0,0 +1,70 @@
|
||||
import { expect, test } from "@jest/globals";
|
||||
import { applyQuery, parseQuery } from "./engine";
|
||||
|
||||
test("Test parser", () => {
|
||||
let parsedBasicQuery = parseQuery(`page`);
|
||||
expect(parsedBasicQuery.table).toBe("page");
|
||||
|
||||
let parsedQuery1 = parseQuery(
|
||||
`task where completed = false and dueDate <= "{{today}}" order by dueDate desc limit 5`
|
||||
);
|
||||
expect(parsedQuery1.table).toBe("task");
|
||||
expect(parsedQuery1.orderBy).toBe("dueDate");
|
||||
expect(parsedQuery1.orderDesc).toBe(true);
|
||||
expect(parsedQuery1.limit).toBe(5);
|
||||
expect(parsedQuery1.filter.length).toBe(2);
|
||||
expect(parsedQuery1.filter[0]).toStrictEqual({
|
||||
op: "=",
|
||||
prop: "completed",
|
||||
value: false,
|
||||
});
|
||||
expect(parsedQuery1.filter[1]).toStrictEqual({
|
||||
op: "<=",
|
||||
prop: "dueDate",
|
||||
value: "{{today}}",
|
||||
});
|
||||
|
||||
let parsedQuery2 = parseQuery(`page where name like "interview/%"`);
|
||||
expect(parsedQuery2.table).toBe("page");
|
||||
expect(parsedQuery2.filter.length).toBe(1);
|
||||
expect(parsedQuery2.filter[0]).toStrictEqual({
|
||||
op: "like",
|
||||
prop: "name",
|
||||
value: "interview/%",
|
||||
});
|
||||
});
|
||||
|
||||
test("Test performing the queries", () => {
|
||||
let data: any[] = [
|
||||
{ name: "interview/My Interview", lastModified: 1 },
|
||||
{ name: "interview/My Interview 2", lastModified: 2 },
|
||||
{ name: "Pete", age: 38 },
|
||||
{ name: "Angie", age: 28 },
|
||||
];
|
||||
|
||||
expect(applyQuery(`page where name like "interview/%"`, data)).toStrictEqual([
|
||||
{ name: "interview/My Interview", lastModified: 1 },
|
||||
{ name: "interview/My Interview 2", lastModified: 2 },
|
||||
]);
|
||||
expect(
|
||||
applyQuery(`page where name like "interview/%" order by lastModified`, data)
|
||||
).toStrictEqual([
|
||||
{ name: "interview/My Interview", lastModified: 1 },
|
||||
{ name: "interview/My Interview 2", lastModified: 2 },
|
||||
]);
|
||||
expect(
|
||||
applyQuery(
|
||||
`page where name like "interview/%" order by lastModified desc`,
|
||||
data
|
||||
)
|
||||
).toStrictEqual([
|
||||
{ name: "interview/My Interview 2", lastModified: 2 },
|
||||
{ name: "interview/My Interview", lastModified: 1 },
|
||||
]);
|
||||
expect(applyQuery(`page where age > 30`, data)).toStrictEqual([
|
||||
{ name: "Pete", age: 38 },
|
||||
]);
|
||||
expect(applyQuery(`page where age > 28 and age < 38`, data)).toStrictEqual(
|
||||
[]
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,154 @@
|
||||
import { collectNodesOfType, findNodeOfType, replaceNodesMatching } from "../../common/tree";
|
||||
import { lezerToParseTree } from "../../common/parse_tree";
|
||||
|
||||
// @ts-ignore
|
||||
import { parser } from "./parse-query";
|
||||
|
||||
type Filter = {
|
||||
op: string;
|
||||
prop: string;
|
||||
value: any;
|
||||
};
|
||||
|
||||
type ParsedQuery = {
|
||||
table: string;
|
||||
orderBy?: string;
|
||||
orderDesc?: boolean;
|
||||
limit?: number;
|
||||
filter: Filter[];
|
||||
};
|
||||
|
||||
export function parseQuery(query: string): ParsedQuery {
|
||||
let n = lezerToParseTree(query, parser.parse(query).topNode);
|
||||
// Clean the tree a bit
|
||||
replaceNodesMatching(n, (n) => {
|
||||
if (!n.type) {
|
||||
let trimmed = n.text!.trim();
|
||||
if (!trimmed) {
|
||||
return null;
|
||||
}
|
||||
n.text = trimmed;
|
||||
}
|
||||
});
|
||||
|
||||
let queryNode = n.children![0];
|
||||
let parsedQuery: ParsedQuery = {
|
||||
table: queryNode.children![0].children![0].text!,
|
||||
filter: [],
|
||||
};
|
||||
let orderByNode = findNodeOfType(queryNode, "OrderClause");
|
||||
if (orderByNode) {
|
||||
let nameNode = findNodeOfType(orderByNode, "Name");
|
||||
parsedQuery.orderBy = nameNode!.children![0].text!;
|
||||
let orderNode = findNodeOfType(orderByNode, "Order");
|
||||
parsedQuery.orderDesc = orderNode
|
||||
? orderNode.children![0].text! === "desc"
|
||||
: false;
|
||||
}
|
||||
let limitNode = findNodeOfType(queryNode, "LimitClause");
|
||||
if (limitNode) {
|
||||
let nameNode = findNodeOfType(limitNode, "Number");
|
||||
parsedQuery.limit = +nameNode!.children![0].text!;
|
||||
}
|
||||
let filterNodes = collectNodesOfType(queryNode, "FilterExpr");
|
||||
for (let filterNode of filterNodes) {
|
||||
let val: any = undefined;
|
||||
let valNode = filterNode.children![2].children![0];
|
||||
switch (valNode.type) {
|
||||
case "Number":
|
||||
val = valNode.children![0].text!;
|
||||
break;
|
||||
case "Bool":
|
||||
val = valNode.children![0].text! === "true";
|
||||
break;
|
||||
case "Name":
|
||||
val = valNode.children![0].text!;
|
||||
break;
|
||||
case "String":
|
||||
val = valNode.children![0].text!;
|
||||
val = val.substring(1, val.length - 1);
|
||||
break;
|
||||
}
|
||||
let f: Filter = {
|
||||
prop: filterNode.children![0].children![0].text!,
|
||||
op: filterNode.children![1].text!,
|
||||
value: val,
|
||||
};
|
||||
parsedQuery.filter.push(f);
|
||||
}
|
||||
// console.log(JSON.stringify(queryNode, null, 2));
|
||||
return parsedQuery;
|
||||
}
|
||||
|
||||
export function applyQuery(query: string, records: any[]): any {
|
||||
const parsedQuery = parseQuery(query);
|
||||
|
||||
let resultRecords: any[] = [];
|
||||
if (parsedQuery.filter.length === 0) {
|
||||
resultRecords = records.slice();
|
||||
} else {
|
||||
recordLoop: for (let record of records) {
|
||||
for (let { op, prop, value } of parsedQuery.filter) {
|
||||
switch (op) {
|
||||
case "=":
|
||||
if (!(record[prop] === value)) {
|
||||
continue recordLoop;
|
||||
}
|
||||
break;
|
||||
case "!=":
|
||||
if (!(record[prop] !== value)) {
|
||||
continue recordLoop;
|
||||
}
|
||||
break;
|
||||
case "<":
|
||||
if (!(record[prop] < value)) {
|
||||
continue recordLoop;
|
||||
}
|
||||
break;
|
||||
case "<=":
|
||||
if (!(record[prop] <= value)) {
|
||||
continue recordLoop;
|
||||
}
|
||||
break;
|
||||
case ">":
|
||||
if (!(record[prop] > value)) {
|
||||
continue recordLoop;
|
||||
}
|
||||
break;
|
||||
case ">=":
|
||||
if (!(record[prop] >= value)) {
|
||||
continue recordLoop;
|
||||
}
|
||||
break;
|
||||
case "like":
|
||||
let re = new RegExp(value.replaceAll("%", ".*"));
|
||||
if (!re.exec(record[prop])) {
|
||||
continue recordLoop;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
resultRecords.push(record);
|
||||
}
|
||||
}
|
||||
// Now the sorting
|
||||
if (parsedQuery.orderBy) {
|
||||
resultRecords = resultRecords.sort((a: any, b: any) => {
|
||||
const orderBy = parsedQuery.orderBy!;
|
||||
const orderDesc = parsedQuery.orderDesc!;
|
||||
if (a[orderBy] === b[orderBy]) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (a[orderBy] < b[orderBy]) {
|
||||
return orderDesc ? 1 : -1;
|
||||
} else {
|
||||
return orderDesc ? -1 : 1;
|
||||
}
|
||||
});
|
||||
}
|
||||
if (parsedQuery.limit) {
|
||||
resultRecords = resultRecords.slice(0, parsedQuery.limit);
|
||||
}
|
||||
return resultRecords;
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
import { flashNotification, getCurrentPage, reloadPage, save } from "plugos-silverbullet-syscall/editor";
|
||||
|
||||
import { listPages, readPage, writePage } from "plugos-silverbullet-syscall/space";
|
||||
import { invokeFunction } from "plugos-silverbullet-syscall/system";
|
||||
import { scanPrefixGlobal } from "plugos-silverbullet-syscall";
|
||||
import { niceDate } from "../core/dates";
|
||||
|
||||
export const queryRegex =
|
||||
/(<!--\s*#query\s+(?<table>\w+)\s*(filter\s+["'“”‘’](?<filter>[^"'“”‘’]+)["'“”‘’])?\s*\s*(order by\s+(?<orderBy>\w+)(?<orderDesc>\s+desc)?)?(group by\s+(?<groupBy>\w+))?\s*(limit\s+(?<limit>\d+))?\s*-->)(.+?)(<!--\s*#end\s*-->)/gs;
|
||||
|
||||
export const newQueryRegex =
|
||||
/<!--\s*#query\s+(.+?)(?=\s*-->)-->(.+?)<!--\s*#end\s*-->/gs;
|
||||
|
||||
export function whiteOutQueries(text: string): string {
|
||||
return text.replaceAll(newQueryRegex, (match) =>
|
||||
new Array(match.length + 1).join(" ")
|
||||
);
|
||||
}
|
||||
|
||||
async function replaceAsync(
|
||||
str: string,
|
||||
regex: RegExp,
|
||||
asyncFn: (match: string, ...args: any[]) => Promise<string>
|
||||
) {
|
||||
const promises: Promise<string>[] = [];
|
||||
str.replace(regex, (match: string, ...args: any[]): string => {
|
||||
const promise = asyncFn(match, ...args);
|
||||
promises.push(promise);
|
||||
return "";
|
||||
});
|
||||
const data = await Promise.all(promises);
|
||||
return str.replace(regex, () => data.shift()!);
|
||||
}
|
||||
|
||||
export async function updateMaterializedQueriesCommand() {
|
||||
const currentPage = await getCurrentPage();
|
||||
await save();
|
||||
await invokeFunction(
|
||||
"server",
|
||||
"updateMaterializedQueriesOnPage",
|
||||
currentPage
|
||||
);
|
||||
await reloadPage();
|
||||
await flashNotification("Updated materialized queries");
|
||||
}
|
||||
|
||||
function replaceTemplateVars(s: string): string {
|
||||
return s.replaceAll(/\{\{(\w+)\}\}/g, (match, v) => {
|
||||
switch (v) {
|
||||
case "today":
|
||||
return niceDate(new Date());
|
||||
break;
|
||||
}
|
||||
return match;
|
||||
});
|
||||
}
|
||||
|
||||
// Called from client, running on server
|
||||
export async function updateMaterializedQueriesOnPage(pageName: string) {
|
||||
let { text } = await readPage(pageName);
|
||||
|
||||
text = await replaceAsync(text, queryRegex, async (match, ...args) => {
|
||||
let { table, filter, groupBy, limit, orderBy, orderDesc } =
|
||||
args[args.length - 1];
|
||||
const startQuery = args[0];
|
||||
const endQuery = args[args.length - 4];
|
||||
let results = [];
|
||||
filter = filter && replaceTemplateVars(filter);
|
||||
switch (table) {
|
||||
case "page":
|
||||
let pages = await listPages();
|
||||
if (orderBy) {
|
||||
pages = pages.sort((a: any, b: any) => {
|
||||
if (a[orderBy] === b[orderBy]) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (a[orderBy] < b[orderBy]) {
|
||||
return !!orderDesc ? 1 : -1;
|
||||
} else {
|
||||
return !!orderDesc ? -1 : 1;
|
||||
}
|
||||
});
|
||||
}
|
||||
let matchCount = 0;
|
||||
for (let pageMeta of pages) {
|
||||
if (!filter || (filter && pageMeta.name.includes(filter))) {
|
||||
matchCount++;
|
||||
results.push(`* [[${pageMeta.name}]]`);
|
||||
if (limit && matchCount === +limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return `${startQuery}\n${results.join("\n")}\n${endQuery}`;
|
||||
case "task":
|
||||
for (let {
|
||||
key,
|
||||
page,
|
||||
value: { task, complete, nested },
|
||||
} of await scanPrefixGlobal("task:")) {
|
||||
let [, pos] = key.split(":");
|
||||
if (!filter || (filter && task.includes(filter))) {
|
||||
results.push(
|
||||
`* [${complete ? "x" : " "}] [[${page}@${pos}]] ${task}` +
|
||||
(nested ? "\n " + nested : "")
|
||||
);
|
||||
}
|
||||
}
|
||||
return `${startQuery}\n${results.sort().join("\n")}\n${endQuery}`;
|
||||
case "link":
|
||||
let uniqueLinks = new Set<string>();
|
||||
for (let { key, page, value: name } of await scanPrefixGlobal(
|
||||
`pl:${pageName}:`
|
||||
)) {
|
||||
let [, pos] = key.split(":");
|
||||
if (!filter || (filter && name.includes(filter))) {
|
||||
uniqueLinks.add(name);
|
||||
}
|
||||
}
|
||||
for (const uniqueResult of uniqueLinks) {
|
||||
results.push(`* [[${uniqueResult}]]`);
|
||||
}
|
||||
return `${startQuery}\n${results.sort().join("\n")}\n${endQuery}`;
|
||||
case "item":
|
||||
for (let {
|
||||
key,
|
||||
page,
|
||||
value: { item, nested },
|
||||
} of await scanPrefixGlobal("it:")) {
|
||||
let [, pos] = key.split(":");
|
||||
if (!filter || (filter && item.includes(filter))) {
|
||||
results.push(
|
||||
`* [[${page}@${pos}]] ${item}` + (nested ? "\n " + nested : "")
|
||||
);
|
||||
}
|
||||
}
|
||||
return `${startQuery}\n${results.sort().join("\n")}\n${endQuery}`;
|
||||
default:
|
||||
return match;
|
||||
}
|
||||
});
|
||||
// console.log("New text", text);
|
||||
await writePage(pageName, text);
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
@precedence { logic @left }
|
||||
|
||||
@top Program { Query }
|
||||
|
||||
Query {
|
||||
Name (WhereClause | OrderClause | LimitClause)*
|
||||
}
|
||||
|
||||
WhereClause { "where" LogicalExpr }
|
||||
OrderClause { "order" "by" Name Order? }
|
||||
LimitClause { "limit" Number }
|
||||
|
||||
Order {
|
||||
"desc" | "asc"
|
||||
}
|
||||
|
||||
Value { Number | String | Bool }
|
||||
|
||||
LogicalExpr { AndExpr | FilterExpr }
|
||||
|
||||
AndExpr { FilterExpr !logic "and" FilterExpr }
|
||||
|
||||
FilterExpr {
|
||||
Name "<" Value
|
||||
| Name "<=" Value
|
||||
| Name "=" Value
|
||||
| Name "!=" Value
|
||||
| Name ">=" Value
|
||||
| Name ">" Value
|
||||
| Name "like" Value
|
||||
}
|
||||
|
||||
@skip { space }
|
||||
|
||||
Bool {
|
||||
"true" | "false"
|
||||
}
|
||||
|
||||
@tokens {
|
||||
space { std.whitespace+ }
|
||||
Name { std.asciiLetter+ }
|
||||
String { "\"" ![\"]* "\"" }
|
||||
Number { std.digit+ }
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
functions:
|
||||
updateMaterializedQueriesOnPage:
|
||||
path: ./materialized_queries.ts:updateMaterializedQueriesOnPage
|
||||
updateMaterializedQueriesCommand:
|
||||
path: ./materialized_queries.ts:updateMaterializedQueriesCommand
|
||||
command:
|
||||
name: "Materialized Queries: Update"
|
||||
Reference in New Issue
Block a user