From 31254d15e68a44ab2ec8dfa11e431535f46fa9e1 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Wed, 13 Apr 2022 14:46:52 +0200 Subject: [PATCH] Work to reduce bundles size (prebundle modules) --- common/preload_modules.ts | 7 ++ common/spaces/constants.ts | 2 + common/spaces/evented_space_primitives.ts | 23 +++--- common/spaces/space.ts | 7 +- common/types.ts | 7 ++ plugos-silverbullet-syscall/editor.ts | 10 +++ plugos/bin/plugos-bundle.ts | 17 ++++- plugos/environments/node_worker.ts | 12 ++- plugos/environments/sandbox_worker.ts | 15 ++++ plugs/core/core.plug.yaml | 9 ++- plugs/core/dates.ts | 2 +- plugs/core/item.ts | 2 +- plugs/core/template.ts | 51 +++++++++++++ plugs/ghost/ghost.ts | 12 ++- plugs/markdown/markdown.plug.yaml | 2 +- plugs/markdown/markdown.ts | 89 ++--------------------- plugs/markdown/preview.ts | 62 ++++++++++++++++ plugs/markdown/util.ts | 24 ++++++ plugs/query/data.ts | 39 ++++++++-- plugs/query/engine.test.ts | 9 +++ plugs/query/engine.ts | 12 ++- plugs/query/materialized_queries.ts | 25 +------ plugs/query/parse-query.js | 12 +-- plugs/query/parse-query.terms.js | 7 +- plugs/query/query.grammar | 7 +- plugs/query/util.ts | 8 ++ plugs/tasks/task.ts | 7 +- webapp/components/command_palette.tsx | 5 +- webapp/components/filter.tsx | 22 +++--- webapp/components/page_navigator.tsx | 6 +- webapp/editor.tsx | 33 +++++++++ webapp/reducer.ts | 18 +++++ webapp/styles/editor.scss | 10 ++- webapp/syscalls/editor.ts | 10 +++ webapp/syscalls/system.ts | 4 + webapp/types.ts | 23 +++++- 36 files changed, 431 insertions(+), 179 deletions(-) create mode 100644 common/preload_modules.ts create mode 100644 common/spaces/constants.ts create mode 100644 plugs/core/template.ts create mode 100644 plugs/markdown/preview.ts create mode 100644 plugs/markdown/util.ts create mode 100644 plugs/query/util.ts diff --git a/common/preload_modules.ts b/common/preload_modules.ts new file mode 100644 index 0000000..5709d7e --- /dev/null +++ b/common/preload_modules.ts @@ -0,0 +1,7 @@ +// These are the node modules that will be pre-bundled with SB +// as a result they will not be included into plugos bundles and assumed to be loadable +// via require() in the sandbox +// Candidate modules for this are larger modules + +// When adding a module to this list, also manually add it to sandbox_worker.ts +export const preloadModules = ["@lezer/lr", "yaml"]; diff --git a/common/spaces/constants.ts b/common/spaces/constants.ts new file mode 100644 index 0000000..3eb8b4f --- /dev/null +++ b/common/spaces/constants.ts @@ -0,0 +1,2 @@ +export const trashPrefix = "_trash/"; +export const plugPrefix = "_plug/"; diff --git a/common/spaces/evented_space_primitives.ts b/common/spaces/evented_space_primitives.ts index 465cef8..ccf5347 100644 --- a/common/spaces/evented_space_primitives.ts +++ b/common/spaces/evented_space_primitives.ts @@ -2,6 +2,7 @@ import { SpacePrimitives } from "./space_primitives"; import { EventHook } from "../../plugos/hooks/event"; import { PageMeta } from "../types"; import { Plug } from "../../plugos/plug"; +import { trashPrefix } from "./constants"; export class EventedSpacePrimitives implements SpacePrimitives { constructor(private wrapped: SpacePrimitives, private eventHook: EventHook) {} @@ -40,17 +41,19 @@ export class EventedSpacePrimitives implements SpacePrimitives { lastModified ); // This can happen async - this.eventHook - .dispatchEvent("page:saved", pageName) - .then(() => { - return this.eventHook.dispatchEvent("page:index", { - name: pageName, - text: text, + if (!pageName.startsWith(trashPrefix)) { + this.eventHook + .dispatchEvent("page:saved", pageName) + .then(() => { + return this.eventHook.dispatchEvent("page:index", { + name: pageName, + text: text, + }); + }) + .catch((e) => { + console.error("Error dispatching page:saved event", e); }); - }) - .catch((e) => { - console.error("Error dispatching page:saved event", e); - }); + } return newPageMeta; } diff --git a/common/spaces/space.ts b/common/spaces/space.ts index 8da3fe5..2264cf7 100644 --- a/common/spaces/space.ts +++ b/common/spaces/space.ts @@ -4,10 +4,9 @@ import { PageMeta } from "../types"; import { EventEmitter } from "../event"; import { Plug } from "../../plugos/plug"; import { Manifest } from "../manifest"; +import { plugPrefix, trashPrefix } from "./constants"; const pageWatchInterval = 2000; -const trashPrefix = "_trash/"; -const plugPrefix = "_plug/"; export type SpaceEvents = { pageCreated: (meta: PageMeta) => void; @@ -69,7 +68,9 @@ export class Space extends EventEmitter { this.emit("pageCreated", newPageMeta); } else if ( oldPageMeta && - oldPageMeta.lastModified !== newPageMeta.lastModified + oldPageMeta.lastModified !== newPageMeta.lastModified && + (!this.trashEnabled || + (this.trashEnabled && !pageName.startsWith(trashPrefix))) ) { this.emit("pageChanged", newPageMeta); } diff --git a/common/types.ts b/common/types.ts index d782e6f..aaaa1dd 100644 --- a/common/types.ts +++ b/common/types.ts @@ -4,3 +4,10 @@ export type PageMeta = { lastOpened?: number; created?: boolean; }; + +// Used by FilterBox +export type FilterOption = { + name: string; + orderId?: number; + hint?: string; +}; diff --git a/plugos-silverbullet-syscall/editor.ts b/plugos-silverbullet-syscall/editor.ts index 16249f6..e47ef49 100644 --- a/plugos-silverbullet-syscall/editor.ts +++ b/plugos-silverbullet-syscall/editor.ts @@ -1,4 +1,5 @@ import { syscall } from "./syscall"; +import { FilterOption } from "../common/types"; export function getCurrentPage(): Promise { return syscall("editor.getCurrentPage"); @@ -32,6 +33,15 @@ export function flashNotification(message: string): Promise { return syscall("editor.flashNotification", message); } +export function filterBox( + label: string, + options: FilterOption[], + helpText: string = "", + placeHolder: string = "" +): Promise { + return syscall("editor.filterBox", label, options, helpText, placeHolder); +} + export function showRhs(html: string, flex = 1): Promise { return syscall("editor.showRhs", html, flex); } diff --git a/plugos/bin/plugos-bundle.ts b/plugos/bin/plugos-bundle.ts index fe80a7e..19d756b 100755 --- a/plugos/bin/plugos-bundle.ts +++ b/plugos/bin/plugos-bundle.ts @@ -8,8 +8,14 @@ import yargs from "yargs"; import { hideBin } from "yargs/helpers"; import { Manifest } from "../types"; import YAML from "yaml"; +import { preloadModules } from "../../common/preload_modules"; -async function compile(filePath: string, functionName: string, debug: boolean) { +async function compile( + filePath: string, + functionName: string, + debug: boolean, + meta = true +) { let outFile = "_out.tmp"; let inFile = filePath; @@ -23,7 +29,7 @@ async function compile(filePath: string, functionName: string, debug: boolean) { } // TODO: Figure out how to make source maps work correctly with eval() code - let js = await esbuild.build({ + let result = await esbuild.build({ entryPoints: [inFile], bundle: true, format: "iife", @@ -32,8 +38,15 @@ async function compile(filePath: string, functionName: string, debug: boolean) { sourcemap: false, //sourceMap ? "inline" : false, minify: !debug, outfile: outFile, + metafile: true, + external: preloadModules, }); + if (meta) { + let text = await esbuild.analyzeMetafile(result.metafile); + console.log("Bundle info for", functionName, text); + } + let jsCode = (await readFile(outFile)).toString(); await unlink(outFile); if (inFile !== filePath) { diff --git a/plugos/environments/node_worker.ts b/plugos/environments/node_worker.ts index ba5747d..0ce0ac2 100644 --- a/plugos/environments/node_worker.ts +++ b/plugos/environments/node_worker.ts @@ -1,3 +1,5 @@ +import { preloadModules } from "../../common/preload_modules"; + const { parentPort, workerData } = require("worker_threads"); // @ts-ignore let vm2 = `${workerData}/vm2`; @@ -16,11 +18,17 @@ let pendingRequests = new Map< let syscallReqId = 0; -// console.log("Here's crypto", crypto); - let vm = new VM({ sandbox: { console, + require: (moduleName: string): any => { + console.log("Loading", moduleName); + if (preloadModules.includes(moduleName)) { + return require(`${workerData}/${moduleName}`); + } else { + throw Error("Cannot import arbitrary modules"); + } + }, self: { syscall: (name: string, ...args: any[]) => { return new Promise((resolve, reject) => { diff --git a/plugos/environments/sandbox_worker.ts b/plugos/environments/sandbox_worker.ts index 46414d8..068d383 100644 --- a/plugos/environments/sandbox_worker.ts +++ b/plugos/environments/sandbox_worker.ts @@ -20,6 +20,7 @@ function workerPostMessage(msg: ControllerMessage) { declare global { function syscall(name: string, ...args: any[]): Promise; + // function require(moduleName: string): any; } let syscallReqId = 0; @@ -37,6 +38,20 @@ self.syscall = async (name: string, ...args: any[]) => { }); }; +const preloadedModules: { [key: string]: any } = { + "@lezer/lr": require("@lezer/lr"), + yaml: require("yaml"), +}; +// for (const moduleName of preloadModules) { +// preloadedModules[moduleName] = require(moduleName); +// } + +// @ts-ignore +self.require = (moduleName: string): any => { + console.log("Loading", moduleName, preloadedModules[moduleName]); + return preloadedModules[moduleName]; +}; + function wrapScript(code: string) { return `return (${code})["default"]`; } diff --git a/plugs/core/core.plug.yaml b/plugs/core/core.plug.yaml index a93b36d..8b077ec 100644 --- a/plugs/core/core.plug.yaml +++ b/plugs/core/core.plug.yaml @@ -71,10 +71,6 @@ functions: path: "./dates.ts:insertTomorrow" slashCommand: name: tomorrow - indexDates: - path: "./dates.ts:indexDates" - events: - - page:index parseServerCommand: path: ./page.ts:parseServerPageCommand command: @@ -85,3 +81,8 @@ functions: path: ./page.ts:parsePageCommand command: name: "Debug: Parse Document" + + instantiateTemplateCommand: + path: ./template.ts:instantiateTemplateCommand + command: + name: "Template: Instantiate for Page" diff --git a/plugs/core/dates.ts b/plugs/core/dates.ts index 2474c77..0cad52e 100644 --- a/plugs/core/dates.ts +++ b/plugs/core/dates.ts @@ -1,7 +1,7 @@ import { insertAtCursor } from "plugos-silverbullet-syscall/editor"; import { IndexEvent } from "../../webapp/app_event"; import { batchSet } from "plugos-silverbullet-syscall"; -import { whiteOutQueries } from "../query/materialized_queries"; +import { whiteOutQueries } from "../query/util"; const dateMatchRegex = /(\d{4}\-\d{2}\-\d{2})/g; diff --git a/plugs/core/item.ts b/plugs/core/item.ts index 48942d3..20d1ca6 100644 --- a/plugs/core/item.ts +++ b/plugs/core/item.ts @@ -1,9 +1,9 @@ import { IndexEvent } from "../../webapp/app_event"; -import { whiteOutQueries } from "../query/materialized_queries"; import { batchSet } from "plugos-silverbullet-syscall/index"; import { parseMarkdown } from "plugos-silverbullet-syscall/markdown"; import { collectNodesOfType, ParseTree, renderToText } from "../../common/tree"; +import { whiteOutQueries } from "../query/util"; export type Item = { name: string; diff --git a/plugs/core/template.ts b/plugs/core/template.ts new file mode 100644 index 0000000..bbda674 --- /dev/null +++ b/plugs/core/template.ts @@ -0,0 +1,51 @@ +import { listPages, readPage, writePage } from "plugos-silverbullet-syscall/space"; +import { filterBox, navigate, prompt } from "plugos-silverbullet-syscall/editor"; +import { parseMarkdown } from "plugos-silverbullet-syscall/markdown"; +import { extractMeta } from "../query/data"; +import { renderToText } from "../../common/tree"; +import { niceDate } from "./dates"; + +const pageTemplatePrefix = `template/page/`; + +export async function instantiateTemplateCommand() { + let allPages = await listPages(); + let allPageTemplates = allPages.filter((pageMeta) => + pageMeta.name.startsWith(pageTemplatePrefix) + ); + + let selectedTemplate = await filterBox( + "Template", + allPageTemplates, + "Select the template to create a new page from" + ); + + if (!selectedTemplate) { + return; + } + console.log("Selected template", selectedTemplate); + + let { text } = await readPage(selectedTemplate.name); + + let parseTree = await parseMarkdown(text); + let additionalPageMeta = extractMeta(parseTree, true); + console.log("Page meta", additionalPageMeta); + + let pageName = await prompt("Name of new page", additionalPageMeta.name); + if (!pageName) { + return; + } + let pageText = replaceTemplateVars(renderToText(parseTree)); + await writePage(pageName, pageText); + await navigate(pageName); +} + +export function replaceTemplateVars(s: string): string { + return s.replaceAll(/\{\{(\w+)\}\}/g, (match, v) => { + switch (v) { + case "today": + return niceDate(new Date()); + break; + } + return match; + }); +} diff --git a/plugs/ghost/ghost.ts b/plugs/ghost/ghost.ts index 5889186..9d6ee1e 100644 --- a/plugs/ghost/ghost.ts +++ b/plugs/ghost/ghost.ts @@ -1,9 +1,9 @@ import { readPage, writePage } from "plugos-silverbullet-syscall/space"; import { json } from "plugos-syscall/fetch"; -import YAML from "yaml"; +import { parse as parseYaml } from "yaml"; import { invokeFunction } from "plugos-silverbullet-syscall/system"; import { getCurrentPage, getText } from "plugos-silverbullet-syscall/editor"; -import { cleanMarkdown } from "../markdown/markdown"; +import { cleanMarkdown } from "../markdown/util"; type GhostConfig = { url: string; @@ -183,7 +183,13 @@ async function markdownToPost(text: string): Promise> { async function getConfig(): Promise { let configPage = await readPage("ghost-config"); - return YAML.parse(configPage.text) as GhostConfig; + return parseYaml(configPage.text) as GhostConfig; + // return { + // adminKey: "", + // pagePrefix: "", + // postPrefix: "", + // url: "", + // }; } export async function downloadAllPostsCommand() { diff --git a/plugs/markdown/markdown.plug.yaml b/plugs/markdown/markdown.plug.yaml index 4f56324..733fceb 100644 --- a/plugs/markdown/markdown.plug.yaml +++ b/plugs/markdown/markdown.plug.yaml @@ -6,7 +6,7 @@ functions: key: Ctrl-p mac: Cmd-p preview: - path: "./markdown.ts:updateMarkdownPreview" + path: "./preview.ts:updateMarkdownPreview" env: client events: - plug:load diff --git a/plugs/markdown/markdown.ts b/plugs/markdown/markdown.ts index 75f9f4c..e9f2a51 100644 --- a/plugs/markdown/markdown.ts +++ b/plugs/markdown/markdown.ts @@ -1,97 +1,18 @@ -import MarkdownIt from "markdown-it"; -import { getText, hideRhs, showRhs } from "plugos-silverbullet-syscall/editor"; +import { hideRhs } from "plugos-silverbullet-syscall/editor"; +import { invokeFunction } from "plugos-silverbullet-syscall/system"; import * as clientStore from "plugos-silverbullet-syscall/clientStore"; -import { parseMarkdown } from "plugos-silverbullet-syscall/markdown"; -import { renderToText, replaceNodesMatching } from "../../common/tree"; - -const css = ` - -`; - -var taskLists = require("markdown-it-task-lists"); - -const md = new MarkdownIt({ - linkify: true, - html: false, - typographer: true, -}).use(taskLists); export async function togglePreview() { let currentValue = !!(await clientStore.get("enableMarkdownPreview")); await clientStore.set("enableMarkdownPreview", !currentValue); if (!currentValue) { - updateMarkdownPreview(); + await invokeFunction("client", "preview"); + // updateMarkdownPreview(); } else { - hideMarkdownPreview(); + await hideMarkdownPreview(); } } -function encodePageUrl(name: string): string { - return name.replaceAll(" ", "_"); -} - -export async function cleanMarkdown(text: string) { - let mdTree = await parseMarkdown(text); - replaceNodesMatching(mdTree, (n) => { - if (n.type === "WikiLink") { - const page = n.children![1].children![0].text!; - return { - // HACK - text: `[${page}](/${encodePageUrl(page)})`, - }; - } - // Simply get rid of these - if (n.type === "CommentBlock" || n.type === "Comment") { - return null; - } - }); - let html = md.render(renderToText(mdTree)); - return html; -} - -export async function updateMarkdownPreview() { - if (!(await clientStore.get("enableMarkdownPreview"))) { - return; - } - let text = await getText(); - let html = await cleanMarkdown(text); - await showRhs(`${css}${html}`, 2); -} - async function hideMarkdownPreview() { await hideRhs(); } diff --git a/plugs/markdown/preview.ts b/plugs/markdown/preview.ts new file mode 100644 index 0000000..35510f6 --- /dev/null +++ b/plugs/markdown/preview.ts @@ -0,0 +1,62 @@ +import MarkdownIt from "markdown-it"; +import { getText, showRhs } from "plugos-silverbullet-syscall/editor"; +import * as clientStore from "plugos-silverbullet-syscall/clientStore"; +import { cleanMarkdown } from "./util"; + +const css = ` + +`; + +var taskLists = require("markdown-it-task-lists"); + +const md = new MarkdownIt({ + linkify: true, + html: false, + typographer: true, +}).use(taskLists); + +export async function updateMarkdownPreview() { + if (!(await clientStore.get("enableMarkdownPreview"))) { + return; + } + let text = await getText(); + let cleanMd = await cleanMarkdown(text); + await showRhs( + `${css}${md.render(cleanMd)}`, + 2 + ); +} diff --git a/plugs/markdown/util.ts b/plugs/markdown/util.ts new file mode 100644 index 0000000..8142346 --- /dev/null +++ b/plugs/markdown/util.ts @@ -0,0 +1,24 @@ +import { renderToText, replaceNodesMatching } from "../../common/tree"; +import { parseMarkdown } from "plugos-silverbullet-syscall/markdown"; + +export function encodePageUrl(name: string): string { + return name.replaceAll(" ", "_"); +} + +export async function cleanMarkdown(text: string): Promise { + let mdTree = await parseMarkdown(text); + replaceNodesMatching(mdTree, (n) => { + if (n.type === "WikiLink") { + const page = n.children![1].children![0].text!; + return { + // HACK + text: `[${page}](/${encodePageUrl(page)})`, + }; + } + // Simply get rid of these + if (n.type === "CommentBlock" || n.type === "Comment") { + return null; + } + }); + return renderToText(mdTree); +} diff --git a/plugs/query/data.ts b/plugs/query/data.ts index 6aefa63..bf8f1a2 100644 --- a/plugs/query/data.ts +++ b/plugs/query/data.ts @@ -2,17 +2,16 @@ // data:page@pos import { IndexEvent } from "../../webapp/app_event"; -import { whiteOutQueries } from "./materialized_queries"; import { batchSet } from "plugos-silverbullet-syscall"; import { parseMarkdown } from "plugos-silverbullet-syscall/markdown"; -import { collectNodesOfType, findNodeOfType } from "../../common/tree"; -import YAML from "yaml"; +import { collectNodesOfType, findNodeOfType, ParseTree, replaceNodesMatching } from "../../common/tree"; +import { parse as parseYaml, parseAllDocuments } from "yaml"; +import { whiteOutQueries } from "./util"; export async function indexData({ name, text }: IndexEvent) { let e; text = whiteOutQueries(text); - console.log("Now data indexing", name); - console.log("Indexing items", name); + // console.log("Now data indexing", name); let mdTree = await parseMarkdown(text); let dataObjects: { key: string; value: Object }[] = []; @@ -33,7 +32,7 @@ export async function indexData({ name, text }: IndexEvent) { let codeText = codeTextNode.children![0].text!; try { // We support multiple YAML documents in one block - for (let doc of YAML.parseAllDocuments(codeText)) { + for (let doc of parseAllDocuments(codeText)) { if (!doc.contents) { continue; } @@ -49,6 +48,32 @@ export async function indexData({ name, text }: IndexEvent) { return; } }); - console.log("Found", dataObjects, "data objects"); + console.log("Found", dataObjects.length, "data objects"); await batchSet(name, dataObjects); } + +export function extractMeta(parseTree: ParseTree, remove = false): any { + let data = {}; + replaceNodesMatching(parseTree, (t) => { + if (t.type !== "FencedCode") { + return; + } + let codeInfoNode = findNodeOfType(t, "CodeInfo"); + if (!codeInfoNode) { + return; + } + if (codeInfoNode.children![0].text !== "meta") { + return; + } + let codeTextNode = findNodeOfType(t, "CodeText"); + if (!codeTextNode) { + // Honestly, this shouldn't happen + return; + } + let codeText = codeTextNode.children![0].text!; + data = parseYaml(codeText); + return remove ? null : undefined; + }); + + return data; +} diff --git a/plugs/query/engine.test.ts b/plugs/query/engine.test.ts index 918e343..39bd168 100644 --- a/plugs/query/engine.test.ts +++ b/plugs/query/engine.test.ts @@ -32,6 +32,15 @@ test("Test parser", () => { prop: "name", value: /interview\/.*/, }); + + let parsedQuery3 = parseQuery(`page where something != null`); + expect(parsedQuery3.table).toBe("page"); + expect(parsedQuery3.filter.length).toBe(1); + expect(parsedQuery3.filter[0]).toStrictEqual({ + op: "!=", + prop: "something", + value: null, + }); }); test("Test performing the queries", () => { diff --git a/plugs/query/engine.ts b/plugs/query/engine.ts index bb09aaa..6e7943a 100644 --- a/plugs/query/engine.ts +++ b/plugs/query/engine.ts @@ -63,6 +63,9 @@ export function parseQuery(query: string): ParsedQuery { case "Bool": val = valNode.children![0].text! === "true"; break; + case "Null": + val = null; + break; case "Name": val = valNode.children![0].text!; break; @@ -96,12 +99,12 @@ export function applyQuery(parsedQuery: ParsedQuery, records: T[]): T[] { for (let { op, prop, value } of parsedQuery.filter) { switch (op) { case "=": - if (!(recordAny[prop] === value)) { + if (!(recordAny[prop] == value)) { continue recordLoop; } break; case "!=": - if (!(recordAny[prop] !== value)) { + if (!(recordAny[prop] != value)) { continue recordLoop; } break; @@ -130,6 +133,11 @@ export function applyQuery(parsedQuery: ParsedQuery, records: T[]): T[] { continue recordLoop; } break; + case "!=~": + if (value.exec(recordAny[prop])) { + continue recordLoop; + } + break; } } resultRecords.push(recordAny); diff --git a/plugs/query/materialized_queries.ts b/plugs/query/materialized_queries.ts index 2eb70c4..c9e30fc 100644 --- a/plugs/query/materialized_queries.ts +++ b/plugs/query/materialized_queries.ts @@ -3,21 +3,13 @@ import { flashNotification, getCurrentPage, reloadPage, save } from "plugos-silv import { listPages, readPage, writePage } from "plugos-silverbullet-syscall/space"; import { invokeFunction } from "plugos-silverbullet-syscall/system"; import { scanPrefixGlobal } from "plugos-silverbullet-syscall"; -import { niceDate } from "../core/dates"; import { applyQuery, parseQuery } from "./engine"; import { PageMeta } from "../../common/types"; import type { Task } from "../tasks/task"; import { Item } from "../core/item"; import YAML from "yaml"; - -export const queryRegex = - /()(.+?)()/gs; - -export function whiteOutQueries(text: string): string { - return text.replaceAll(queryRegex, (match) => - new Array(match.length + 1).join(" ") - ); -} +import { replaceTemplateVars } from "../core/template"; +import { queryRegex } from "./util"; async function replaceAsync( str: string, @@ -46,17 +38,6 @@ export async function updateMaterializedQueriesCommand() { await flashNotification("Updated materialized queries"); } -function replaceTemplateVars(s: string): string { - return s.replaceAll(/\{\{(\w+)\}\}/g, (match, v) => { - switch (v) { - case "today": - return niceDate(new Date()); - break; - } - return match; - }); -} - // Called from client, running on server export async function updateMaterializedQueriesOnPage(pageName: string) { let { text } = await readPage(pageName); @@ -107,7 +88,7 @@ export async function updateMaterializedQueriesOnPage(pageName: string) { case "item": let allItems: Item[] = []; for (let { key, page, value } of await scanPrefixGlobal("it:")) { - let [, pos] = key.split("@"); + let [, pos] = key.split(":"); allItems.push({ ...value, page: page, diff --git a/plugs/query/parse-query.js b/plugs/query/parse-query.js index 7c3b23e..a8e68a3 100644 --- a/plugs/query/parse-query.js +++ b/plugs/query/parse-query.js @@ -3,14 +3,14 @@ import { LRParser } from "@lezer/lr"; export const parser = LRParser.deserialize({ version: 13, - states: "$UOVQPOOO[QQO'#C^QOQPOOOjQPO'#C`OoQQO'#CiOtQPO'#CkOOQO'#Cl'#ClOyQQO,58xO!XQPO'#CcO!pQQO'#CaOOQO'#Ca'#CaOOQO,58z,58zO#RQPO,59TOOQO,59V,59VOOQO-E6j-E6jO#WQQO,58}OjQPO,58|O#iQQO1G.oOOQO'#Cg'#CgOOQO'#Cd'#CdOOQO1G.i1G.iOOQO1G.h1G.hOOQO'#Cj'#CjOOQO7+$Z7+$Z", - stateData: "#}~OcOS~ORPO~OdROoSOsTOaQX~ORWO~Op[O~OX]O~OdROoSOsTOaQa~Oe_Oh_Oi_Oj_Ok_Ol_Om_O~On`OaTXdTXoTXsTX~ORaO~OXcOYcO[cOfbOgbO~OqfOrfOa]id]io]is]i~O", - goto: "!UaPPbPeilouPPxPe{e!ORQOTUPVRZRRYRQXRRe`Rd_Rc_RgaQVPR^V", - nodeNames: "⚠ Program Query Name WhereClause LogicalExpr AndExpr FilterExpr Value Number String Bool Regex OrderClause Order LimitClause", - maxTerm: 35, + states: "$[OVQPOOO[QQO'#C^QOQPOOOjQPO'#C`OoQQO'#CjOtQPO'#ClOOQO'#Cm'#CmOyQQO,58xO!XQPO'#CcO!sQQO'#CaOOQO'#Ca'#CaOOQO,58z,58zO#UQPO,59UOOQO,59W,59WOOQO-E6k-E6kO#ZQQO,58}OjQPO,58|O#oQQO1G.pOOQO'#Cg'#CgOOQO'#Ci'#CiOOQO'#Cd'#CdOOQO1G.i1G.iOOQO1G.h1G.hOOQO'#Ck'#CkOOQO7+$[7+$[", + stateData: "$T~OdOS~ORPO~OeROrSOvTObQX~ORWO~Os[O~OX]O~OeROrSOvTObQa~Of_Oj_Ok_Ol_Om_On_Oo_Op_O~Oq`ObTXeTXrTXvTX~ORaO~OXdOYdO[dOgbOhbOicO~OtgOugOb^ie^ir^iv^i~O", + goto: "!VbPPcPfjmpvPPyPyf|f!PRQOTUPVRZRRYRQXRRf`Re_Rd_RhaQVPR^V", + nodeNames: "⚠ Program Query Name WhereClause LogicalExpr AndExpr FilterExpr Value Number String Bool Regex Null OrderClause Order LimitClause", + maxTerm: 38, skippedNodes: [0], repeatNodeCount: 1, - tokenData: "3X~RsX^#`pq#`qr$Trs$`!P!Q$z!Q![%q!^!_%y!_!`&W!`!a&e!c!}&r#T#U&}#U#V(t#V#W&r#W#X)d#X#Y&r#Y#Z*v#Z#`&r#`#a,h#a#c&r#c#d.]#d#h&r#h#i0Q#i#k&r#k#l1d#l#o&r#y#z#`$f$g#`#BY#BZ#`$IS$I_#`$Ip$Iq$`$Iq$Ir$`$I|$JO#`$JT$JU#`$KV$KW#`&FU&FV#`~#eYc~X^#`pq#`#y#z#`$f$g#`#BY#BZ#`$IS$I_#`$I|$JO#`$JT$JU#`$KV$KW#`&FU&FV#`~$WP!_!`$Z~$`Oj~~$cUOr$`rs$us$Ip$`$Ip$Iq$u$Iq$Ir$u$Ir~$`~$zOY~~%PV[~OY$zZ]$z^!P$z!P!Q%f!Q#O$z#O#P%k#P~$z~%kO[~~%nPO~$z~%vPX~!Q![%q~&OPe~!_!`&R~&WOh~~&]Pi~#r#s&`~&eOm~~&jPl~!_!`&m~&rOk~P&wQRP!c!}&r#T#o&rR'SURP!c!}&r#T#b&r#b#c'f#c#g&r#g#h(U#h#o&rR'kSRP!c!}&r#T#W&r#W#X'w#X#o&rR(OQnQRP!c!}&r#T#o&rR(ZSRP!c!}&r#T#V&r#V#W(g#W#o&rR(nQrQRP!c!}&r#T#o&rR(ySRP!c!}&r#T#m&r#m#n)V#n#o&rR)^QpQRP!c!}&r#T#o&rR)iSRP!c!}&r#T#X&r#X#Y)u#Y#o&rR)zSRP!c!}&r#T#g&r#g#h*W#h#o&rR*]SRP!c!}&r#T#V&r#V#W*i#W#o&rR*pQqQRP!c!}&r#T#o&rR*{RRP!c!}&r#T#U+U#U#o&rR+ZSRP!c!}&r#T#`&r#`#a+g#a#o&rR+lSRP!c!}&r#T#g&r#g#h+x#h#o&rR+}SRP!c!}&r#T#X&r#X#Y,Z#Y#o&rR,bQgQRP!c!}&r#T#o&rR,mSRP!c!}&r#T#]&r#]#^,y#^#o&rR-OSRP!c!}&r#T#a&r#a#b-[#b#o&rR-aSRP!c!}&r#T#]&r#]#^-m#^#o&rR-rSRP!c!}&r#T#h&r#h#i.O#i#o&rR.VQsQRP!c!}&r#T#o&rR.bSRP!c!}&r#T#f&r#f#g.n#g#o&rR.sSRP!c!}&r#T#W&r#W#X/P#X#o&rR/USRP!c!}&r#T#X&r#X#Y/b#Y#o&rR/gSRP!c!}&r#T#f&r#f#g/s#g#o&rR/zQoQRP!c!}&r#T#o&rR0VSRP!c!}&r#T#f&r#f#g0c#g#o&rR0hSRP!c!}&r#T#i&r#i#j0t#j#o&rR0ySRP!c!}&r#T#X&r#X#Y1V#Y#o&rR1^QfQRP!c!}&r#T#o&rR1iSRP!c!}&r#T#[&r#[#]1u#]#o&rR1zSRP!c!}&r#T#X&r#X#Y2W#Y#o&rR2]SRP!c!}&r#T#f&r#f#g2i#g#o&rR2nSRP!c!}&r#T#X&r#X#Y2z#Y#o&rR3RQdQRP!c!}&r#T#o&r", + tokenData: "4v~RtX^#cpq#cqr$Wrs$k!P!Q%V!Q![%|!^!_&U!_!`&c!`!a&p!c!}&}#T#U'Y#U#V)P#V#W&}#W#X)o#X#Y&}#Y#Z+R#Z#`&}#`#a,s#a#b&}#b#c.h#c#d/z#d#h&}#h#i1o#i#k&}#k#l3R#l#o&}#y#z#c$f$g#c#BY#BZ#c$IS$I_#c$Ip$Iq$k$Iq$Ir$k$I|$JO#c$JT$JU#c$KV$KW#c&FU&FV#c~#hYd~X^#cpq#c#y#z#c$f$g#c#BY#BZ#c$IS$I_#c$I|$JO#c$JT$JU#c$KV$KW#c&FU&FV#c~$ZP!_!`$^~$cPl~#r#s$f~$kOp~~$nUOr$krs%Qs$Ip$k$Ip$Iq%Q$Iq$Ir%Q$Ir~$k~%VOY~~%[V[~OY%VZ]%V^!P%V!P!Q%q!Q#O%V#O#P%v#P~%V~%vO[~~%yPO~%V~&RPX~!Q![%|~&ZPf~!_!`&^~&cOj~~&hPk~#r#s&k~&pOo~~&uPn~!_!`&x~&}Om~P'SQRP!c!}&}#T#o&}R'_URP!c!}&}#T#b&}#b#c'q#c#g&}#g#h(a#h#o&}R'vSRP!c!}&}#T#W&}#W#X(S#X#o&}R(ZQqQRP!c!}&}#T#o&}R(fSRP!c!}&}#T#V&}#V#W(r#W#o&}R(yQuQRP!c!}&}#T#o&}R)USRP!c!}&}#T#m&}#m#n)b#n#o&}R)iQsQRP!c!}&}#T#o&}R)tSRP!c!}&}#T#X&}#X#Y*Q#Y#o&}R*VSRP!c!}&}#T#g&}#g#h*c#h#o&}R*hSRP!c!}&}#T#V&}#V#W*t#W#o&}R*{QtQRP!c!}&}#T#o&}R+WRRP!c!}&}#T#U+a#U#o&}R+fSRP!c!}&}#T#`&}#`#a+r#a#o&}R+wSRP!c!}&}#T#g&}#g#h,T#h#o&}R,YSRP!c!}&}#T#X&}#X#Y,f#Y#o&}R,mQhQRP!c!}&}#T#o&}R,xSRP!c!}&}#T#]&}#]#^-U#^#o&}R-ZSRP!c!}&}#T#a&}#a#b-g#b#o&}R-lSRP!c!}&}#T#]&}#]#^-x#^#o&}R-}SRP!c!}&}#T#h&}#h#i.Z#i#o&}R.bQvQRP!c!}&}#T#o&}R.mSRP!c!}&}#T#i&}#i#j.y#j#o&}R/OSRP!c!}&}#T#`&}#`#a/[#a#o&}R/aSRP!c!}&}#T#`&}#`#a/m#a#o&}R/tQiQRP!c!}&}#T#o&}R0PSRP!c!}&}#T#f&}#f#g0]#g#o&}R0bSRP!c!}&}#T#W&}#W#X0n#X#o&}R0sSRP!c!}&}#T#X&}#X#Y1P#Y#o&}R1USRP!c!}&}#T#f&}#f#g1b#g#o&}R1iQrQRP!c!}&}#T#o&}R1tSRP!c!}&}#T#f&}#f#g2Q#g#o&}R2VSRP!c!}&}#T#i&}#i#j2c#j#o&}R2hSRP!c!}&}#T#X&}#X#Y2t#Y#o&}R2{QgQRP!c!}&}#T#o&}R3WSRP!c!}&}#T#[&}#[#]3d#]#o&}R3iSRP!c!}&}#T#X&}#X#Y3u#Y#o&}R3zSRP!c!}&}#T#f&}#f#g4W#g#o&}R4]SRP!c!}&}#T#X&}#X#Y4i#Y#o&}R4pQeQRP!c!}&}#T#o&}", tokenizers: [0, 1], topRules: {"Program":[0,1]}, tokenPrec: 0 diff --git a/plugs/query/parse-query.terms.js b/plugs/query/parse-query.terms.js index 7c69c48..133a5b6 100644 --- a/plugs/query/parse-query.terms.js +++ b/plugs/query/parse-query.terms.js @@ -12,6 +12,7 @@ export const String = 10, Bool = 11, Regex = 12, - OrderClause = 13, - Order = 14, - LimitClause = 15 + Null = 13, + OrderClause = 14, + Order = 15, + LimitClause = 16 diff --git a/plugs/query/query.grammar b/plugs/query/query.grammar index bc68e76..34b517e 100644 --- a/plugs/query/query.grammar +++ b/plugs/query/query.grammar @@ -14,7 +14,7 @@ Order { "desc" | "asc" } -Value { Number | String | Bool | Regex } +Value { Number | String | Bool | Regex | Null } LogicalExpr { AndExpr | FilterExpr } @@ -28,6 +28,7 @@ FilterExpr { | Name ">=" Value | Name ">" Value | Name "=~" Value +| Name "!=~" Value } @skip { space } @@ -36,6 +37,10 @@ Bool { "true" | "false" } +Null { + "null" +} + @tokens { space { std.whitespace+ } Name { std.asciiLetter+ } diff --git a/plugs/query/util.ts b/plugs/query/util.ts new file mode 100644 index 0000000..2826807 --- /dev/null +++ b/plugs/query/util.ts @@ -0,0 +1,8 @@ +export const queryRegex = + /()(.+?)()/gs; + +export function whiteOutQueries(text: string): string { + return text.replaceAll(queryRegex, (match) => + new Array(match.length + 1).join(" ") + ); +} diff --git a/plugs/tasks/task.ts b/plugs/tasks/task.ts index 922930c..095ff91 100644 --- a/plugs/tasks/task.ts +++ b/plugs/tasks/task.ts @@ -1,7 +1,5 @@ -import type { ClickEvent } from "../../webapp/app_event"; -import { IndexEvent } from "../../webapp/app_event"; +import type { ClickEvent, IndexEvent } from "../../webapp/app_event"; -import { whiteOutQueries } from "../query/materialized_queries"; import { batchSet } from "plugos-silverbullet-syscall/index"; import { readPage, writePage } from "plugos-silverbullet-syscall/space"; import { parseMarkdown } from "plugos-silverbullet-syscall/markdown"; @@ -13,6 +11,7 @@ import { nodeAtPos, renderToText } from "../../common/tree"; +import { whiteOutQueries } from "../query/util"; export type Task = { name: string; @@ -25,7 +24,7 @@ export type Task = { }; export async function indexTasks({ name, text }: IndexEvent) { - console.log("Indexing tasks"); + // console.log("Indexing tasks"); let tasks: { key: string; value: Task }[] = []; text = whiteOutQueries(text); let mdTree = await parseMarkdown(text); diff --git a/webapp/components/command_palette.tsx b/webapp/components/command_palette.tsx index 4bc9973..ef52718 100644 --- a/webapp/components/command_palette.tsx +++ b/webapp/components/command_palette.tsx @@ -1,7 +1,8 @@ import { isMacLike } from "../util"; -import { FilterList, Option } from "./filter"; +import { FilterList } from "./filter"; import { faPersonRunning } from "@fortawesome/free-solid-svg-icons"; import { AppCommand } from "../hooks/command"; +import { FilterOption } from "../../common/types"; export function CommandPalette({ commands, @@ -10,7 +11,7 @@ export function CommandPalette({ commands: Map; onTrigger: (command: AppCommand | undefined) => void; }) { - let options: Option[] = []; + let options: FilterOption[] = []; const isMac = isMacLike(); for (let [name, def] of commands.entries()) { options.push({ diff --git a/webapp/components/filter.tsx b/webapp/components/filter.tsx index 2efaf9b..b66ba34 100644 --- a/webapp/components/filter.tsx +++ b/webapp/components/filter.tsx @@ -1,14 +1,9 @@ import React, { useEffect, useRef, useState } from "react"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { IconDefinition } from "@fortawesome/free-solid-svg-icons"; +import { FilterOption } from "../../common/types"; -export type Option = { - name: string; - orderId?: number; - hint?: string; -}; - -function magicSorter(a: Option, b: Option): number { +function magicSorter(a: FilterOption, b: FilterOption): number { if (a.orderId && b.orderId) { return a.orderId < b.orderId ? -1 : 1; } @@ -19,7 +14,7 @@ function escapeRegExp(str: string): string { return str.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&"); } -function fuzzyFilter(pattern: string, options: Option[]): Option[] { +function fuzzyFilter(pattern: string, options: FilterOption[]): FilterOption[] { let closeMatchRegex = escapeRegExp(pattern); closeMatchRegex = closeMatchRegex.split(/\s+/).join(".*?"); closeMatchRegex = closeMatchRegex.replace(/\\\//g, ".*?\\/.*?"); @@ -51,7 +46,10 @@ function fuzzyFilter(pattern: string, options: Option[]): Option[] { return matches; } -function simpleFilter(pattern: string, options: Option[]): Option[] { +function simpleFilter( + pattern: string, + options: FilterOption[] +): FilterOption[] { const lowerPattern = pattern.toLowerCase(); return options.filter((option) => { return option.name.toLowerCase().includes(lowerPattern); @@ -71,10 +69,10 @@ export function FilterList({ newHint, }: { placeholder: string; - options: Option[]; + options: FilterOption[]; label: string; onKeyPress?: (key: string, currentText: string) => void; - onSelect: (option: Option | undefined) => void; + onSelect: (option: FilterOption | undefined) => void; allowNew?: boolean; completePrefix?: string; helpText: string; @@ -169,7 +167,7 @@ export function FilterList({ onSelect(undefined); break; case " ": - if (completePrefix) { + if (completePrefix && !text) { setText(completePrefix); e.preventDefault(); } diff --git a/webapp/components/page_navigator.tsx b/webapp/components/page_navigator.tsx index 17e75dd..7eca53e 100644 --- a/webapp/components/page_navigator.tsx +++ b/webapp/components/page_navigator.tsx @@ -1,5 +1,5 @@ -import { FilterList, Option } from "./filter"; -import { PageMeta } from "../../common/types"; +import { FilterList } from "./filter"; +import { FilterOption, PageMeta } from "../../common/types"; export function PageNavigator({ allPages, @@ -10,7 +10,7 @@ export function PageNavigator({ onNavigate: (page: string | undefined) => void; currentPage?: string; }) { - let options: Option[] = []; + let options: FilterOption[] = []; for (let pageMeta of allPages) { if (currentPage && currentPage === pageMeta.name) { continue; diff --git a/webapp/editor.tsx b/webapp/editor.tsx index 0592325..9f98fe3 100644 --- a/webapp/editor.tsx +++ b/webapp/editor.tsx @@ -50,6 +50,8 @@ import { markdownSyscalls } from "../common/syscalls/markdown"; import { clientStoreSyscalls } from "./syscalls/clientStore"; import { StatusBar } from "./components/status_bar"; import { loadMarkdownExtensions, MDExt } from "./markdown_ext"; +import { FilterList } from "./components/filter"; +import { FilterOption } from "../common/types"; class PageState { scrollTop: number; @@ -245,6 +247,26 @@ export class Editor implements AppEventDispatcher { }, 2000); } + filterBox( + label: string, + options: FilterOption[], + helpText: string = "", + placeHolder: string = "" + ): Promise { + return new Promise((resolve) => { + this.viewDispatch({ + type: "show-filterbox", + options, + placeHolder, + helpText, + onSelect: (option) => { + this.viewDispatch({ type: "hide-filterbox" }); + resolve(option); + }, + }); + }); + } + async dispatchAppEvent(name: AppEvent, data?: any): Promise { return this.eventHook.dispatchEvent(name, data); } @@ -535,6 +557,17 @@ export class Editor implements AppEventDispatcher { commands={viewState.commands} /> )} + {viewState.showFilterBox && ( + + )} {}, + filterBoxPlaceHolder: "", + filterBoxOptions: [], + filterBoxHelpText: "", + }; } return state; } diff --git a/webapp/styles/editor.scss b/webapp/styles/editor.scss index e24badb..ae9c717 100644 --- a/webapp/styles/editor.scss +++ b/webapp/styles/editor.scss @@ -26,7 +26,7 @@ .line-h1, .line-h2, .line-h3 { - background-color: rgba(0, 15, 52, 0.6); + background-color: rgba(0, 30, 77, 0.5); color: #fff; font-weight: bold; padding: 2px 2px; @@ -195,9 +195,11 @@ } .comment { - color: gray; - background-color: rgba(210, 210, 210, 0.3); + color: #989797; + background-color: rgba(210, 210, 210, 0.2); border-radius: 5px; - padding: 0 2px; + font-style: italic; + font-size: 75%; + line-height: 75%; } } diff --git a/webapp/syscalls/editor.ts b/webapp/syscalls/editor.ts index e654803..bf086a2 100644 --- a/webapp/syscalls/editor.ts +++ b/webapp/syscalls/editor.ts @@ -1,6 +1,7 @@ import { Editor } from "../editor"; import { Transaction } from "@codemirror/state"; import { SysCallMapping } from "../../plugos/system"; +import { FilterOption } from "../../common/types"; type SyntaxNode = { name: string; @@ -51,6 +52,15 @@ export function editorSyscalls(editor: Editor): SysCallMapping { "editor.flashNotification": (ctx, message: string) => { editor.flashNotification(message); }, + "editor.filterBox": ( + ctx, + label: string, + options: FilterOption[], + helpText: string = "", + placeHolder: string = "" + ): Promise => { + return editor.filterBox(label, options, helpText, placeHolder); + }, "editor.showRhs": (ctx, html: string, flex: number) => { editor.viewDispatch({ type: "show-rhs", diff --git a/webapp/syscalls/system.ts b/webapp/syscalls/system.ts index f7e08b6..cddd30e 100644 --- a/webapp/syscalls/system.ts +++ b/webapp/syscalls/system.ts @@ -13,6 +13,10 @@ export function systemSyscalls(space: Space): SysCallMapping { throw Error("No plug associated with context"); } + if (env === "client") { + return ctx.plug.invoke(name, args); + } + return space.invokeFunction(ctx.plug, env, name, args); }, }; diff --git a/webapp/types.ts b/webapp/types.ts index b8ec3fd..ec26231 100644 --- a/webapp/types.ts +++ b/webapp/types.ts @@ -1,5 +1,5 @@ import { AppCommand } from "./hooks/command"; -import { PageMeta } from "../common/types"; +import { FilterOption, PageMeta } from "../common/types"; export const slashCommandRegexp = /\/[\w\-]*/; @@ -21,6 +21,12 @@ export type AppViewState = { allPages: Set; commands: Map; notifications: Notification[]; + + showFilterBox: boolean; + filterBoxPlaceHolder: string; + filterBoxOptions: FilterOption[]; + filterBoxHelpText: string; + filterBoxOnSelect: (option: FilterOption | undefined) => void; }; export const initialViewState: AppViewState = { @@ -34,6 +40,11 @@ export const initialViewState: AppViewState = { allPages: new Set(), commands: new Map(), notifications: [], + showFilterBox: false, + filterBoxHelpText: "", + filterBoxOnSelect: () => {}, + filterBoxOptions: [], + filterBoxPlaceHolder: "", }; export type Action = @@ -51,4 +62,12 @@ export type Action = | { type: "show-rhs"; html: string; flex: number } | { type: "hide-rhs" } | { type: "show-lhs"; html: string; flex: number } - | { type: "hide-lhs" }; + | { type: "hide-lhs" } + | { + type: "show-filterbox"; + options: FilterOption[]; + placeHolder: string; + helpText: string; + onSelect: (option: FilterOption | undefined) => void; + } + | { type: "hide-filterbox" };