From c268fa9f27888403c49e0592b20ea93786cc2deb Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Tue, 29 Mar 2022 17:02:28 +0200 Subject: [PATCH] Work on materialized queries --- plugos/bin/plugos-server.ts | 12 ++-- plugs/core/core.plug.yaml | 11 ++++ plugs/core/item.ts | 39 +++++++++++++ plugs/core/materialized_queries.ts | 83 ++++++++++++++++++++++++++ plugs/core/navigate.ts | 8 +++ plugs/markdown/yarn.lock | 94 ++++++++++++++++++++++++++++++ plugs/tasks/task.ts | 35 ++++++++--- server/server.ts | 4 +- webapp/styles/editor.scss | 4 ++ 9 files changed, 273 insertions(+), 17 deletions(-) create mode 100644 plugs/core/item.ts create mode 100644 plugs/core/materialized_queries.ts create mode 100644 plugs/markdown/yarn.lock diff --git a/plugos/bin/plugos-server.ts b/plugos/bin/plugos-server.ts index 93ef422..28d1b65 100755 --- a/plugos/bin/plugos-server.ts +++ b/plugos/bin/plugos-server.ts @@ -11,9 +11,9 @@ import {EndpointHook, EndpointHookT} from "../hooks/endpoint"; import {safeRun} from "../util"; import knex from "knex"; import { - ensureTable, - storeReadSyscalls, - storeWriteSyscalls, + ensureTable, + storeReadSyscalls, + storeWriteSyscalls, } from "../syscalls/store.knex_node"; import {fetchSyscalls} from "../syscalls/fetch.node"; import {EventHook, EventHookT} from "../hooks/event"; @@ -21,13 +21,13 @@ import {eventSyscalls} from "../syscalls/event"; let args = yargs(hideBin(process.argv)) .option("port", { - type: "number", - default: 1337, + type: "number", + default: 1337, }) .parse(); if (!args._.length) { - console.error("Usage: plugos-server "); + console.error("Usage: plugos-server "); process.exit(1); } diff --git a/plugs/core/core.plug.yaml b/plugs/core/core.plug.yaml index e38219f..134f251 100644 --- a/plugs/core/core.plug.yaml +++ b/plugs/core/core.plug.yaml @@ -9,6 +9,10 @@ functions: path: "./page.ts:indexLinks" events: - page:index + indexItems: + path: "./item.ts:indexItems" + events: + - page:index deletePage: path: "./page.ts:deletePage" command: @@ -50,3 +54,10 @@ functions: events: - plug:load env: server + updateMaterializedQueriesOnPage: + path: ./materialized_queries.ts:updateMaterializedQueriesOnPage + env: server + updateMaterializedQueriesCommand: + path: ./materialized_queries.ts:updateMaterializedQueriesCommand + command: + name: "Materialized Queries: Update" diff --git a/plugs/core/item.ts b/plugs/core/item.ts new file mode 100644 index 0000000..71a2f03 --- /dev/null +++ b/plugs/core/item.ts @@ -0,0 +1,39 @@ +import { IndexEvent } from "../../webapp/app_event"; +import { whiteOutQueries } from "./materialized_queries"; +import { syscall } from "../lib/syscall"; + +type Item = { + item: string; + children?: string[]; +}; + +const pageRefRe = /\[\[[^\]]+@\d+\]\]/; +const itemFullRe = + /(?[\t ]*)[\-\*]\s*([^\n]+)(\n\k\s+[\-\*][^\n]+)*/g; + +export async function indexItems({ name, text }: IndexEvent) { + let items: { key: string; value: Item }[] = []; + text = whiteOutQueries(text); + for (let match of text.matchAll(itemFullRe)) { + let entire = match[0]; + let item = match[2]; + if (item.match(pageRefRe)) { + continue; + } + let pos = match.index!; + let lines = entire.split("\n"); + + let value: Item = { + item, + }; + if (lines.length > 1) { + value.children = lines.slice(1); + } + items.push({ + key: `it:${pos}`, + value, + }); + } + console.log("Found", items.length, "item(s)"); + await syscall("indexer.batchSet", name, items); +} diff --git a/plugs/core/materialized_queries.ts b/plugs/core/materialized_queries.ts new file mode 100644 index 0000000..a985c7c --- /dev/null +++ b/plugs/core/materialized_queries.ts @@ -0,0 +1,83 @@ +import { syscall } from "../lib/syscall"; + +export const queryRegex = + /()(.+?)()/gs; + +export function whiteOutQueries(text: string): string { + return text.replaceAll(queryRegex, (match) => + new Array(match.length + 1).join(" ") + ); +} + +async function replaceAsync( + str: string, + regex: RegExp, + asyncFn: (match: string, ...args: any[]) => Promise +) { + const promises: Promise[] = []; + str.replace(regex, (match: string, ...args: any[]): string => { + const promise = asyncFn(match, ...args); + promises.push(promise); + return ""; + }); + const data = await Promise.all(promises); + return str.replace(regex, () => data.shift()!); +} + +export async function updateMaterializedQueriesCommand() { + await syscall( + "system.invokeFunctionOnServer", + "updateMaterializedQueriesOnPage", + await syscall("editor.getCurrentPage") + ); + syscall("editor.flashNotification", "Updated materialized queries"); +} + +// Called from client, running on server +export async function updateMaterializedQueriesOnPage(pageName: string) { + let { text } = await syscall("space.readPage", pageName); + text = await replaceAsync(text, queryRegex, async (match, ...args) => { + let { table, filter, groupBy } = args[args.length - 1]; + const startQuery = args[0]; + const endQuery = args[args.length - 4]; + let results = []; + switch (table) { + case "task": + for (let { + key, + page, + value: { task, complete, children }, + } of await syscall("indexer.scanPrefixGlobal", "task:")) { + let [, pos] = key.split(":"); + if (!filter || (filter && task.includes(filter))) { + results.push( + `* [${complete ? "x" : " "}] [[${page}@${pos}]] ${task}` + ); + if (children) { + results.push(children.join("\n")); + } + } + } + return `${startQuery}\n${results.join("\n")}\n${endQuery}`; + case "item": + for (let { + key, + page, + value: { item, children }, + } of await syscall("indexer.scanPrefixGlobal", "it:")) { + let [, pos] = key.split(":"); + if (!filter || (filter && item.includes(filter))) { + results.push(`* [[${page}@${pos}]] ${item}`); + if (children) { + results.push(children.join("\n")); + } + } + } + return `${startQuery}\n${results.join("\n")}\n${endQuery}`; + default: + return match; + } + }); + // console.log("New text", text); + await syscall("space.writePage", pageName, text); +} diff --git a/plugs/core/navigate.ts b/plugs/core/navigate.ts index 35890ec..a4201f7 100644 --- a/plugs/core/navigate.ts +++ b/plugs/core/navigate.ts @@ -1,5 +1,8 @@ import { ClickEvent } from "../../webapp/app_event"; import { syscall } from "../lib/syscall"; +import { updateMaterializedQueriesCommand } from "./materialized_queries"; + +const materializedQueryPrefix = /