neil/silverbullet
1
0
Fork 0
Code Activity

Massive restructure of plugin API

Browse Source
This commit is contained in:
Zef Hemel
2022-10-14 15:11:33 +02:00
parent 982623fc38
commit 7d28b53b75
70 changed files with 826 additions and 969 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugs/query/complete.ts
+4 -4
View File
@@ -1,11 +1,11 @@
import { listEvents } from "$sb/plugos-syscall/event.ts";
import { matchBefore } from "$sb/silverbullet-syscall/editor.ts";
import { events } from "$sb/plugos-syscall/mod.ts";
import { editor } from "$sb/silverbullet-syscall/mod.ts";
export async function queryComplete() {
const prefix = await matchBefore("#query [\\w\\-_]*");
const prefix = await editor.matchBefore("#query [\\w\\-_]*");
if (prefix) {
const allEvents = await listEvents();
const allEvents = await events.listEvents();
// console.log("All events", allEvents);
return {
plugs/query/data.ts
+23 -27
View File
@@ -1,52 +1,48 @@
// Index key space:
// data:page@pos
import type { IndexTreeEvent } from "../../web/app_event.ts";
import {
batchSet,
queryPrefix,
} from "../../syscall/silverbullet-syscall/index.ts";
import type { IndexTreeEvent, QueryProviderEvent } from "$sb/app_event.ts";
import { index } from "$sb/silverbullet-syscall/mod.ts";
import {
addParentPointers,
collectNodesOfType,
findNodeOfType,
ParseTree,
replaceNodesMatching,
} from "../../common/tree.ts";
import type { QueryProviderEvent } from "./engine.ts";
import { applyQuery } from "./engine.ts";
import { removeQueries } from "./util.ts";
} from "$sb/lib/tree.ts";
import { applyQuery, removeQueries } from "$sb/lib/query.ts";
import * as YAML from "yaml";
export async function indexData({ name, tree }: IndexTreeEvent) {
let dataObjects: { key: string; value: Object }[] = [];
const dataObjects: { key: string; value: any }[] = [];
removeQueries(tree);
collectNodesOfType(tree, "FencedCode").forEach((t) => {
let codeInfoNode = findNodeOfType(t, "CodeInfo");
const codeInfoNode = findNodeOfType(t, "CodeInfo");
if (!codeInfoNode) {
return;
}
if (codeInfoNode.children![0].text !== "data") {
return;
}
let codeTextNode = findNodeOfType(t, "CodeText");
const codeTextNode = findNodeOfType(t, "CodeText");
if (!codeTextNode) {
// Honestly, this shouldn't happen
return;
}
let codeText = codeTextNode.children![0].text!;
const codeText = codeTextNode.children![0].text!;
try {
const docs = codeText.split("---").map((d) => YAML.parse(d));
// We support multiple YAML documents in one block
for (let doc of parseAllDocuments(codeText)) {
if (!doc.contents) {
for (let i = 0; i < docs.length; i++) {
const doc = docs[i];
if (!doc) {
continue;
}
console.log(doc.contents.toJSON());
dataObjects.push({
key: `data:${name}@${t.from! + doc.range[0]}`,
value: doc.contents.toJSON(),
key: `data:${name}@${i}`,
value: doc,
});
}
// console.log("Parsed data", parsedData);
@@ -56,7 +52,7 @@ export async function indexData({ name, tree }: IndexTreeEvent) {
}
});
console.log("Found", dataObjects.length, "data objects");
await batchSet(name, dataObjects);
await index.batchSet(name, dataObjects);
}
export function extractMeta(
@@ -83,23 +79,23 @@ export function extractMeta(
if (t.type !== "FencedCode") {
return;
}
let codeInfoNode = findNodeOfType(t, "CodeInfo");
const codeInfoNode = findNodeOfType(t, "CodeInfo");
if (!codeInfoNode) {
return;
}
if (codeInfoNode.children![0].text !== "meta") {
return;
}
let codeTextNode = findNodeOfType(t, "CodeText");
const codeTextNode = findNodeOfType(t, "CodeText");
if (!codeTextNode) {
// Honestly, this shouldn't happen
return;
}
let codeText = codeTextNode.children![0].text!;
const codeText = codeTextNode.children![0].text!;
data = YAML.parse(codeText);
if (removeKeys.length > 0) {
let newData = { ...data };
for (let key of removeKeys) {
const newData = { ...data };
for (const key of removeKeys) {
delete newData[key];
}
codeTextNode.children![0].text = YAML.stringify(newData).trim();
@@ -117,9 +113,9 @@ export function extractMeta(
export async function queryProvider({
query,
}: QueryProviderEvent): Promise<any[]> {
let allData: any[] = [];
for (let { key, page, value } of await queryPrefix("data:")) {
let [, pos] = key.split("@");
const allData: any[] = [];
for (const { key, page, value } of await index.queryPrefix("data:")) {
const [, pos] = key.split("@");
allData.push({
...value,
page: page,
plugs/query/engine.test.ts
+7 -7
View File
@@ -1,12 +1,12 @@
import { assertEquals } from "../../test_deps.ts";
import { applyQuery } from "./engine.ts";
import { applyQuery } from "$sb/lib/query.ts";
import { parseQuery } from "./parser.ts";
Deno.test("Test parser", () => {
let parsedBasicQuery = parseQuery(`page`);
const parsedBasicQuery = parseQuery(`page`);
assertEquals(parsedBasicQuery.table, "page");
let parsedQuery1 = parseQuery(
const parsedQuery1 = parseQuery(
`task where completed = false and dueDate <= "{{today}}" order by dueDate desc limit 5`,
);
assertEquals(parsedQuery1.table, "task");
@@ -25,7 +25,7 @@ Deno.test("Test parser", () => {
value: "{{today}}",
});
let parsedQuery2 = parseQuery(`page where name =~ /interview\\/.*/"`);
const parsedQuery2 = parseQuery(`page where name =~ /interview\\/.*/"`);
assertEquals(parsedQuery2.table, "page");
assertEquals(parsedQuery2.filter.length, 1);
assertEquals(parsedQuery2.filter[0], {
@@ -34,7 +34,7 @@ Deno.test("Test parser", () => {
value: "interview\\/.*",
});
let parsedQuery3 = parseQuery(`page where something != null`);
const parsedQuery3 = parseQuery(`page where something != null`);
assertEquals(parsedQuery3.table, "page");
assertEquals(parsedQuery3.filter.length, 1);
assertEquals(parsedQuery3.filter[0], {
@@ -77,7 +77,7 @@ Deno.test("Test parser", () => {
});
Deno.test("Test applyQuery", () => {
let data: any[] = [
const data: any[] = [
{ name: "interview/My Interview", lastModified: 1 },
{ name: "interview/My Interview 2", lastModified: 2 },
{ name: "Pete", age: 38 },
@@ -132,7 +132,7 @@ Deno.test("Test applyQuery", () => {
});
Deno.test("Test applyQuery with multi value", () => {
let data: any[] = [
const data: any[] = [
{ name: "Pete", children: ["John", "Angie"] },
{ name: "Angie", children: ["Angie"] },
{ name: "Steve" },
plugs/query/engine.ts
+16 -121
View File
@@ -1,15 +1,10 @@
import { collectNodesOfType, ParseTree } from "../../common/tree.ts";
import { collectNodesOfType, ParseTree } from "$sb/lib/tree.ts";
import Handlebars from "handlebars";
import * as YAML from "yaml";
import { readPage } from "../../syscall/silverbullet-syscall/space.ts";
import { niceDate } from "../core/dates.ts";
import { ParsedQuery } from "./parser.ts";
export type QueryProviderEvent = {
query: ParsedQuery;
pageName: string;
};
import { space } from "$sb/silverbullet-syscall/mod.ts";
import { niceDate } from "$sb/lib/dates.ts";
import { ParsedQuery } from "$sb/lib/query.ts";
export function valueNodeToVal(valNode: ParseTree): any {
switch (valNode.type) {
@@ -21,124 +16,24 @@ export function valueNodeToVal(valNode: ParseTree): any {
return null;
case "Name":
return valNode.children![0].text!;
case "Regex":
let val = valNode.children![0].text!;
case "Regex": {
const val = valNode.children![0].text!;
return val.substring(1, val.length - 1);
case "String":
let stringVal = valNode.children![0].text!;
}
case "String": {
const stringVal = valNode.children![0].text!;
return stringVal.substring(1, stringVal.length - 1);
case "PageRef":
let pageRefVal = valNode.children![0].text!;
}
case "PageRef": {
const pageRefVal = valNode.children![0].text!;
return pageRefVal.substring(2, pageRefVal.length - 2);
case "List":
}
case "List": {
return collectNodesOfType(valNode, "Value").map((t) =>
valueNodeToVal(t.children![0])
);
}
}
export function applyQuery<T>(parsedQuery: ParsedQuery, records: T[]): T[] {
let resultRecords: any[] = [];
if (parsedQuery.filter.length === 0) {
resultRecords = records.slice();
} else {
recordLoop:
for (let record of records) {
const recordAny: any = record;
for (let { op, prop, value } of parsedQuery.filter) {
switch (op) {
case "=":
const recordPropVal = recordAny[prop];
if (Array.isArray(recordPropVal) && !Array.isArray(value)) {
// Record property is an array, and value is a scalar: find the value in the array
if (!recordPropVal.includes(value)) {
continue recordLoop;
}
} else if (Array.isArray(recordPropVal) && Array.isArray(value)) {
// Record property is an array, and value is an array: find the value in the array
if (!recordPropVal.some((v) => value.includes(v))) {
continue recordLoop;
}
} else if (!(recordPropVal == value)) {
// Both are scalars: exact value
continue recordLoop;
}
break;
case "!=":
if (!(recordAny[prop] != value)) {
continue recordLoop;
}
break;
case "<":
if (!(recordAny[prop] < value)) {
continue recordLoop;
}
break;
case "<=":
if (!(recordAny[prop] <= value)) {
continue recordLoop;
}
break;
case ">":
if (!(recordAny[prop] > value)) {
continue recordLoop;
}
break;
case ">=":
if (!(recordAny[prop] >= value)) {
continue recordLoop;
}
break;
case "=~":
// TODO: Cache regexps somehow
if (!new RegExp(value).exec(recordAny[prop])) {
continue recordLoop;
}
break;
case "!=~":
if (new RegExp(value).exec(recordAny[prop])) {
continue recordLoop;
}
break;
case "in":
if (!value.includes(recordAny[prop])) {
continue recordLoop;
}
break;
}
}
resultRecords.push(recordAny);
}
}
// Now the sorting
if (parsedQuery.orderBy) {
resultRecords = resultRecords.sort((a: any, b: any) => {
const orderBy = parsedQuery.orderBy!;
const orderDesc = parsedQuery.orderDesc!;
if (a[orderBy] === b[orderBy]) {
return 0;
}
if (a[orderBy] < b[orderBy]) {
return orderDesc ? 1 : -1;
} else {
return orderDesc ? -1 : 1;
}
});
}
if (parsedQuery.limit) {
resultRecords = resultRecords.slice(0, parsedQuery.limit);
}
if (parsedQuery.select) {
resultRecords = resultRecords.map((rec) => {
let newRec: any = {};
for (let k of parsedQuery.select!) {
newRec[k] = rec[k];
}
return newRec;
});
}
return resultRecords;
}
export async function renderQuery(
@@ -175,9 +70,9 @@ export async function renderQuery(
return YAML.stringify(v).trim();
}
});
let { text: templateText } = await readPage(parsedQuery.render);
let templateText = await space.readPage(parsedQuery.render);
templateText = `{{#each .}}\n${templateText}\n{{/each}}`;
let template = Handlebars.compile(templateText, { noEscape: true });
const template = Handlebars.compile(templateText, { noEscape: true });
return template(data);
}
plugs/query/materialized_queries.ts
+31 -35
View File
@@ -1,26 +1,22 @@
import {
getCurrentPage,
reloadPage,
save,
} from "$sb/silverbullet-syscall/editor.ts";
import { editor } from "$sb/silverbullet-syscall/mod.ts";
import Handlebars from "handlebars";
import { readPage, writePage } from "$sb/silverbullet-syscall/space.ts";
import { markdown, space } from "$sb/silverbullet-syscall/mod.ts";
import { invokeFunction } from "$sb/silverbullet-syscall/system.ts";
import { renderQuery } from "./engine.ts";
import { parseQuery } from "./parser.ts";
import { replaceTemplateVars } from "../core/template.ts";
import { jsonToMDTable, queryRegex } from "./util.ts";
import { dispatch } from "$sb/plugos-syscall/event.ts";
import { replaceAsync } from "../lib/util.ts";
import { parseMarkdown } from "$sb/silverbullet-syscall/markdown.ts";
import { nodeAtPos, renderToText } from "../../common/tree.ts";
import { jsonToMDTable } from "./util.ts";
import { queryRegex } from "$sb/lib/query.ts";
import { events } from "$sb/plugos-syscall/mod.ts";
import { replaceAsync } from "$sb/lib/util.ts";
import { nodeAtPos, renderToText } from "$sb/lib/tree.ts";
import { extractMeta } from "./data.ts";
export async function updateMaterializedQueriesCommand() {
const currentPage = await getCurrentPage();
await save();
const currentPage = await editor.getCurrentPage();
await editor.save();
if (
await invokeFunction(
"server",
@@ -29,7 +25,7 @@ export async function updateMaterializedQueriesCommand() {
)
) {
console.log("Going reload the page");
await reloadPage();
await editor.reloadPage();
}
}
@@ -43,13 +39,13 @@ function updateTemplateInstantiations(
return replaceAsync(
text,
templateInstRegex,
async (fullMatch, startInst, type, template, args, body, endInst) => {
async (fullMatch, startInst, type, template, args, _body, endInst) => {
args = args.trim();
let parsedArgs = {};
if (args) {
try {
parsedArgs = JSON.parse(args);
} catch (e) {
} catch {
console.error("Failed to parse template instantiation args", args);
return fullMatch;
}
@@ -57,21 +53,21 @@ function updateTemplateInstantiations(
let templateText = "";
if (template.startsWith("http://") || template.startsWith("https://")) {
try {
let req = await fetch(template);
const req = await fetch(template);
templateText = await req.text();
} catch (e: any) {
templateText = `ERROR: ${e.message}`;
}
} else {
templateText = (await readPage(template)).text;
templateText = await space.readPage(template);
}
let newBody = templateText;
// if it's a template injection (not a literal "include")
if (type === "use" || type === "use-verbose") {
let tree = await parseMarkdown(templateText);
const tree = await markdown.parseMarkdown(templateText);
extractMeta(tree, ["$disableDirectives"]);
templateText = renderToText(tree);
let templateFn = Handlebars.compile(
const templateFn = Handlebars.compile(
replaceTemplateVars(templateText, pageName),
{ noEscape: true },
);
@@ -87,11 +83,11 @@ function cleanTemplateInstantiations(text: string): Promise<string> {
text,
templateInstRegex,
(
fullMatch,
_fullMatch,
startInst,
type,
template,
args,
_template,
_args,
body,
endInst,
): Promise<string> => {
@@ -99,9 +95,9 @@ function cleanTemplateInstantiations(text: string): Promise<string> {
body = body.replaceAll(
queryRegex,
(
fullMatch: string,
startQuery: string,
query: string,
_fullMatch: string,
_startQuery: string,
_query: string,
body: string,
) => {
return body.trim();
@@ -120,7 +116,7 @@ export async function updateMaterializedQueriesOnPage(
// console.log("Updating queries");
let text = "";
try {
text = (await readPage(pageName)).text;
text = await space.readPage(pageName);
} catch {
console.warn(
"Could not read page",
@@ -130,8 +126,8 @@ export async function updateMaterializedQueriesOnPage(
return false;
}
let newText = await updateTemplateInstantiations(text, pageName);
let tree = await parseMarkdown(newText);
let metaData = extractMeta(tree, ["$disableDirectives"]);
const tree = await markdown.parseMarkdown(newText);
const metaData = extractMeta(tree, ["$disableDirectives"]);
if (metaData.$disableDirectives) {
console.log("Directives disabled, skipping");
return false;
@@ -141,18 +137,18 @@ export async function updateMaterializedQueriesOnPage(
newText = await replaceAsync(
newText,
queryRegex,
async (fullMatch, startQuery, query, body, endQuery, index) => {
let currentNode = nodeAtPos(tree, index + 1);
async (fullMatch, startQuery, query, _body, endQuery, index) => {
const currentNode = nodeAtPos(tree, index + 1);
if (currentNode?.type !== "CommentBlock") {
// If not a comment block, it's likely a code block, ignore
return fullMatch;
}
let parsedQuery = parseQuery(replaceTemplateVars(query, pageName));
const parsedQuery = parseQuery(replaceTemplateVars(query, pageName));
// console.log("Parsed query", parsedQuery);
// Let's dispatch an event and see what happens
let results = await dispatch(
const results = await events.dispatchEvent(
`query:${parsedQuery.table}`,
{ query: parsedQuery, pageName: pageName },
10 * 1000,
@@ -161,7 +157,7 @@ export async function updateMaterializedQueriesOnPage(
return `${startQuery}\n${endQuery}`;
} else if (results.length === 1) {
if (parsedQuery.render) {
let rendered = await renderQuery(parsedQuery, results[0]);
const rendered = await renderQuery(parsedQuery, results[0]);
return `${startQuery}\n${rendered.trim()}\n${endQuery}`;
} else {
return `${startQuery}\n${jsonToMDTable(results[0])}\n${endQuery}`;
@@ -174,7 +170,7 @@ export async function updateMaterializedQueriesOnPage(
);
newText = await cleanTemplateInstantiations(newText);
if (text !== newText) {
await writePage(pageName, newText);
await space.writePage(pageName, newText);
return true;
}
return false;
plugs/query/parser.ts
+17 -36
View File
@@ -2,35 +2,20 @@ import {
collectNodesOfType,
findNodeOfType,
replaceNodesMatching,
} from "../../common/tree.ts";
} from "$sb/lib/tree.ts";
import { lezerToParseTree } from "../../common/parse_tree.ts";
import { valueNodeToVal } from "./engine.ts";
// @ts-ignore auto generated
import { parser } from "./parse-query.js";
export type Filter = {
op: string;
prop: string;
value: any;
};
export type ParsedQuery = {
table: string;
orderBy?: string;
orderDesc?: boolean;
limit?: number;
filter: Filter[];
select?: string[];
render?: string;
};
import { ParsedQuery, QueryFilter } from "$sb/lib/query.ts";
export function parseQuery(query: string): ParsedQuery {
let n = lezerToParseTree(query, parser.parse(query).topNode);
const n = lezerToParseTree(query, parser.parse(query).topNode);
// Clean the tree a bit
replaceNodesMatching(n, (n) => {
if (!n.type) {
let trimmed = n.text!.trim();
const trimmed = n.text!.trim();
if (!trimmed) {
return null;
}
@@ -39,50 +24,47 @@ export function parseQuery(query: string): ParsedQuery {
});
// console.log("Parsed", JSON.stringify(n, null, 2));
let queryNode = n.children![0];
let parsedQuery: ParsedQuery = {
const queryNode = n.children![0];
const parsedQuery: ParsedQuery = {
table: queryNode.children![0].children![0].text!,
filter: [],
};
let orderByNode = findNodeOfType(queryNode, "OrderClause");
const orderByNode = findNodeOfType(queryNode, "OrderClause");
if (orderByNode) {
let nameNode = findNodeOfType(orderByNode, "Name");
const nameNode = findNodeOfType(orderByNode, "Name");
parsedQuery.orderBy = nameNode!.children![0].text!;
let orderNode = findNodeOfType(orderByNode, "Order");
const orderNode = findNodeOfType(orderByNode, "Order");
parsedQuery.orderDesc = orderNode
? orderNode.children![0].text! === "desc"
: false;
}
let limitNode = findNodeOfType(queryNode, "LimitClause");
const limitNode = findNodeOfType(queryNode, "LimitClause");
if (limitNode) {
let nameNode = findNodeOfType(limitNode, "Number");
const nameNode = findNodeOfType(limitNode, "Number");
parsedQuery.limit = valueNodeToVal(nameNode!);
}
let filterNodes = collectNodesOfType(queryNode, "FilterExpr");
for (let filterNode of filterNodes) {
const filterNodes = collectNodesOfType(queryNode, "FilterExpr");
for (const filterNode of filterNodes) {
let val: any = undefined;
let valNode = filterNode.children![2].children![0];
const valNode = filterNode.children![2].children![0];
val = valueNodeToVal(valNode);
let f: Filter = {
const f: QueryFilter = {
prop: filterNode.children![0].children![0].text!,
op: filterNode.children![1].text!,
value: val,
};
parsedQuery.filter.push(f);
}
let selectNode = findNodeOfType(queryNode, "SelectClause");
const selectNode = findNodeOfType(queryNode, "SelectClause");
if (selectNode) {
// console.log("Select node", JSON.stringify(selectNode));
parsedQuery.select = [];
collectNodesOfType(selectNode, "Name").forEach((t) => {
parsedQuery.select!.push(t.children![0].text!);
});
// let nameNode = findNodeOfType(selectNode, "Number");
// parsedQuery.limit = +nameNode!.children![0].text!;
}
let renderNode = findNodeOfType(queryNode, "RenderClause");
const renderNode = findNodeOfType(queryNode, "RenderClause");
if (renderNode) {
let renderNameNode = findNodeOfType(renderNode, "PageRef");
if (!renderNameNode) {
@@ -91,6 +73,5 @@ export function parseQuery(query: string): ParsedQuery {
parsedQuery.render = valueNodeToVal(renderNameNode!);
}
// console.log(JSON.stringify(queryNode, null, 2));
return parsedQuery;
}
plugs/query/util.ts
+3 -51
View File
@@ -1,58 +1,10 @@
import {
addParentPointers,
collectNodesMatching,
ParseTree,
renderToText,
} from "../../common/tree.ts";
export const queryRegex =
/(<!--\s*#query\s+(.+?)-->)(.+?)(<!--\s*\/query\s*-->)/gs;
export const directiveStartRegex = /<!--\s*#([\w\-]+)\s+(.+?)-->/s;
export const directiveEndRegex = /<!--\s*\/([\w\-]+)\s*-->/s;
export function removeQueries(pt: ParseTree) {
addParentPointers(pt);
collectNodesMatching(pt, (t) => {
if (t.type !== "CommentBlock") {
return false;
}
let text = t.children![0].text!;
let match = directiveStartRegex.exec(text);
if (!match) {
return false;
}
let directiveType = match[1];
let parentChildren = t.parent!.children!;
let index = parentChildren.indexOf(t);
let nodesToReplace: ParseTree[] = [];
for (let i = index + 1; i < parentChildren.length; i++) {
let n = parentChildren[i];
if (n.type === "CommentBlock") {
let text = n.children![0].text!;
let match = directiveEndRegex.exec(text);
if (match && match[1] === directiveType) {
break;
}
}
nodesToReplace.push(n);
}
let renderedText = nodesToReplace.map(renderToText).join("");
parentChildren.splice(index + 1, nodesToReplace.length, {
text: new Array(renderedText.length + 1).join(" "),
});
return true;
});
}
const maxWidth = 70;
// Nicely format an array of JSON objects as a Markdown table
export function jsonToMDTable(
jsonArray: any[],
valueTransformer: (k: string, v: any) => string = (k, v) => "" + v,
): string {
let fieldWidths = new Map<string, number>();
const fieldWidths = new Map<string, number>();
for (let entry of jsonArray) {
for (let k of Object.keys(entry)) {
let fieldWidth = fieldWidths.get(k);
@@ -70,8 +22,8 @@ export function jsonToMDTable(
fullWidth += v + 1;
}
let headerList = [...fieldWidths.keys()];
let lines = [];
const headerList = [...fieldWidths.keys()];
const lines = [];
lines.push(
"|" +
headerList