neil/silverbullet
1
0
Fork 0
Code

Changed all indexes to use (pre) parsed trees.

Browse Source
This commit is contained in:
Zef Hemel 2022-04-20 10:56:43 +02:00
parent c7176b00fa
commit dbdfc9d631
26 changed files with 264 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
common/spaces/evented_space_primitives.ts
View File

@ -45,9 +45,9 @@ export class EventedSpacePrimitives implements SpacePrimitives {
this.eventHook
.dispatchEvent("page:saved", pageName)
.then(() => {
return this.eventHook.dispatchEvent("page:index", {
return this.eventHook.dispatchEvent("page:index_text", {
name: pageName,
text: text,
text,
});
})
.catch((e) => {

4
common/tree.ts
View File

@ -13,6 +13,10 @@ export function addParentPointers(tree: ParseTree) {
return;
}
for (let child of tree.children) {
if (child.parent) {
// Already added parent pointers before
return;
}
child.parent = tree;
addParentPointers(child);
}

15
plugos-syscall/event.ts
View File

@ -6,13 +6,18 @@ export async function dispatch(
timeout?: number
): Promise<any[]> {
return new Promise((resolve, reject) => {
let timeOut = setTimeout(() => {
console.log("Timeout!");
reject("timeout");
}, timeout);
let timeouter: any = -1;
if (timeout) {
timeouter = setTimeout(() => {
console.log("Timeout!");
reject("timeout");
}, timeout);
}
syscall("event.dispatch", eventName, data)
.then((r) => {
clearTimeout(timeOut);
if (timeouter !== -1) {
clearTimeout(timeouter);
}
resolve(r);
})
.catch(reject);

4
plugs/core/core.plug.yaml
View File

@ -29,6 +29,10 @@ functions:
path: ./page.ts:pageQueryProvider
events:
- query:page
parseIndexTextRepublish:
path: "./page.ts:parseIndexTextRepublish"
events:
- page:index_text
indexLinks:
path: "./page.ts:indexLinks"
events:

23
plugs/core/dates.ts
View File

@ -1,30 +1,9 @@
import { insertAtCursor } from "plugos-silverbullet-syscall/editor";
import { IndexEvent } from "../../webapp/app_event";
import { batchSet } from "plugos-silverbullet-syscall";
import { whiteOutQueries } from "../query/util";
const dateMatchRegex = /(\d{4}\-\d{2}\-\d{2})/g;
// Index key space:
// d:[date]:page@pos
export async function indexDates({ name, text }: IndexEvent) {
let dates: { key: string; value: boolean }[] = [];
text = whiteOutQueries(text);
console.log("Now date indexing", name);
for (let match of text.matchAll(dateMatchRegex)) {
// console.log("Date match", match[0]);
dates.push({
key: `d:${match[0]}:${name}@${match.index}`,
value: true,
});
}
console.log("Found", dates.length, "dates");
await batchSet(name, dates);
}
export function niceDate(d: Date): string {
return new Date().toISOString().split("T")[0];
return d.toISOString().split("T")[0];
}
export async function insertToday() {

12
plugs/core/item.ts
View File

@ -1,9 +1,8 @@
import { IndexEvent } from "../../webapp/app_event";
import { IndexTreeEvent } from "../../webapp/app_event";
import { batchSet, scanPrefixGlobal } from "plugos-silverbullet-syscall/index";
import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
import { collectNodesOfType, ParseTree, renderToText } from "../../common/tree";
import { whiteOutQueries } from "../query/util";
import { removeQueries } from "../query/util";
import { applyQuery, QueryProviderEvent } from "../query/engine";
export type Item = {
@ -14,14 +13,13 @@ export type Item = {
pos?: number;
};
export async function indexItems({ name, text }: IndexEvent) {
export async function indexItems({ name, tree }: IndexTreeEvent) {
let items: { key: string; value: Item }[] = [];
text = whiteOutQueries(text);
removeQueries(tree);
console.log("Indexing items", name);
let mdTree = await parseMarkdown(text);
let coll = collectNodesOfType(mdTree, "ListItem");
let coll = collectNodesOfType(tree, "ListItem");
coll.forEach((n) => {
if (!n.children) {

37
plugs/core/page.ts
View File

@ -1,4 +1,4 @@
import { IndexEvent } from "../../webapp/app_event";
import { IndexEvent, IndexTreeEvent } from "../../webapp/app_event";
import {
batchSet,
clearPageIndex as clearPageIndexSyscall,
@ -28,23 +28,20 @@ import {
import { applyQuery, QueryProviderEvent } from "../query/engine";
import { PageMeta } from "../../common/types";
export async function indexLinks({ name, text }: IndexEvent) {
export async function indexLinks({ name, tree }: IndexTreeEvent) {
let backLinks: { key: string; value: string }[] = [];
// [[Style Links]]
console.log("Now indexing", name);
let mdTree = await parseMarkdown(text);
collectNodesMatching(mdTree, (n) => n.type === "WikiLinkPage").forEach(
(n) => {
let toPage = n.children![0].text!;
if (toPage.includes("@")) {
toPage = toPage.split("@")[0];
}
backLinks.push({
key: `pl:${toPage}:${n.from}`,
value: name,
});
collectNodesMatching(tree, (n) => n.type === "WikiLinkPage").forEach((n) => {
let toPage = n.children![0].text!;
if (toPage.includes("@")) {
toPage = toPage.split("@")[0];
}
);
backLinks.push({
key: `pl:${toPage}:${n.from}`,
value: name,
});
});
console.log("Found", backLinks.length, "wiki link(s)");
await batchSet(name, backLinks);
}
@ -193,10 +190,11 @@ export async function reindexSpace() {
let pages = await listPages();
for (let { name } of pages) {
console.log("Indexing", name);
const pageObj = await readPage(name);
const { text } = await readPage(name);
let parsed = await parseMarkdown(text);
await dispatch("page:index", {
name,
text: pageObj.text,
tree: parsed,
});
}
}
@ -206,6 +204,13 @@ export async function clearPageIndex(page: string) {
await clearPageIndexForPage(page);
}
export async function parseIndexTextRepublish({ name, text }: IndexEvent) {
await dispatch("page:index", {
name,
tree: await parseMarkdown(text),
});
}
export async function parseServerPageCommand() {
console.log(await invokeFunction("server", "parsePage", await getText()));
}

23
plugs/core/template.ts
View File

@ -4,7 +4,6 @@ import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
import { extractMeta } from "../query/data";
import { renderToText } from "../../common/tree";
import { niceDate } from "./dates";
import { dispatch } from "plugos-syscall/event";
import { invokeFunction } from "plugos-silverbullet-syscall/system";
const pageTemplatePrefix = `template/page/`;
@ -60,17 +59,17 @@ export async function replaceTemplateVarsCommand() {
export function replaceTemplateVars(s: string, pageName: string): string {
return s.replaceAll(/\{\{([^\}]+)\}\}/g, (match, v) => {
if (v === "today") {
return niceDate(new Date());
}
if (v.startsWith("placeholder:")) {
// Dispatch event, to be replaced in the file async later
dispatch(v, {
pageName: pageName,
placeholder: v,
}).catch((e) => {
console.error("Failed to dispatch placeholder event", e);
});
switch (v) {
case "today":
return niceDate(new Date());
case "yesterday":
let yesterday = new Date();
yesterday.setDate(yesterday.getDate() - 1);
return niceDate(yesterday);
case "lastWeek":
let lastWeek = new Date();
lastWeek.setDate(lastWeek.getDate() - 7);
return niceDate(lastWeek);
}
return match;
});

18
plugs/markdown/preview.ts
View File

@ -15,6 +15,24 @@ body {
padding-right: 20px;
}
table {
width: 100%;
border-spacing: 0;
}
thead tr {
background-color: #333;
color: #eee;
}
th, td {
padding: 8px;
}
tbody tr:nth-of-type(even) {
background-color: #f3f3f3;
}
a[href] {
text-decoration: none;
}

39
plugs/mattermost/client.ts
View File

@ -1,11 +1,12 @@
import { readPage } from "plugos-silverbullet-syscall/space";
import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
import { extractMeta } from "../query/data";
import { UserProfile } from "@hmhealey/types/lib/users";
import { json } from "plugos-syscall/fetch";
import { Post } from "@hmhealey/types/lib/posts";
import { Channel } from "@hmhealey/types/lib/channels";
import { Team } from "@hmhealey/types/lib/teams";
import type { UserProfile } from "@mattermost/types/lib/users";
import type { Post } from "@mattermost/types/lib/posts";
import type { Channel } from "@mattermost/types/lib/channels";
import type { Team } from "@mattermost/types/lib/teams";
import { niceDate } from "../core/dates";
type MattermostConfig = {
url: string;
@ -19,6 +20,13 @@ async function getConfig(): Promise<MattermostConfig> {
return pageMeta as MattermostConfig;
}
type AugmentedPost = Post & {
// Dates we can use to filter
createdAt: string;
updatedAt: string;
editedAt: string;
};
export class MattermostClient {
userCache = new Map<string, UserProfile>();
channelCache = new Map<string, Channel>();
@ -77,7 +85,10 @@ export class MattermostClient {
return team!;
}
async getFlaggedPosts(userId: string, perPage: number = 10): Promise<Post[]> {
async getFlaggedPosts(
userId: string,
perPage: number = 10
): Promise<AugmentedPost[]> {
let postCollection = await json(
`${this.url}/api/v4/users/${userId}/posts/flagged?per_page=${perPage}`,
{
@ -86,10 +97,24 @@ export class MattermostClient {
},
}
);
let posts: Post[] = [];
let posts: AugmentedPost[] = [];
for (let order of postCollection.order) {
posts.push(postCollection.posts[order]);
let post = postCollection.posts[order];
augmentPost(post);
posts.push(post);
}
return posts;
}
}
function augmentPost(post: AugmentedPost) {
if (post.create_at) {
post.createdAt = niceDate(new Date(post.create_at));
}
if (post.update_at) {
post.updatedAt = niceDate(new Date(post.update_at));
}
if (post.edit_at) {
post.editedAt = niceDate(new Date(post.edit_at));
}
}

9
plugs/mattermost/mattermost.ts
View File

@ -20,17 +20,16 @@ export async function savedPostsQueryProvider({
let savedPostsMd = [];
savedPosts = applyQuery(query, savedPosts);
for (let savedPost of savedPosts) {
// savedPost.
let channel = await client.getChannel(savedPost.channel_id);
let team = await client.getTeam(channel.team_id);
savedPostsMd.push(
`@${
(await client.getUser(savedPost.user_id)).username
} [link](${mattermostDesktopUrlForPost(
`@${(await client.getUser(savedPost.user_id)).username} [${
savedPost.createdAt
}](${mattermostDesktopUrlForPost(
client.url,
team.name,
savedPost.id
)}):\n> ${savedPost.message.replaceAll(/\n/g, "\n> ")}`
)}):\n> ${savedPost.message.substring(0, 1000).replaceAll(/\n/g, "\n> ")}`
);
}
return savedPostsMd.join("\n\n");

40
plugs/package-lock.json generated
View File

@ -8,10 +8,10 @@
"name": "plugs",
"version": "1.0.0",
"dependencies": {
"@hmhealey/types": "^6.6.0-4",
"@jest/globals": "^27.5.1",
"@lezer/generator": "^0.15.4",
"@lezer/lr": "^0.15.8",
"@mattermost/types": "^6.7.0-0",
"@types/yaml": "^1.9.7",
"plugos-silverbullet-syscall": "file:../plugos-silverbullet-syscall",
"plugos-syscall": "file:../plugos-syscall",
@ -120,19 +120,6 @@
"node": ">=4"
}
},
"node_modules/@hmhealey/types": {
"version": "6.6.0-4",
"resolved": "https://registry.npmjs.org/@hmhealey/types/-/types-6.6.0-4.tgz",
"integrity": "sha512-71IxVaXhrUesmLnvQQh4RtUqqhmVL+ejci4qo4R6rTWTdY77BniRtBx269uAz34wzTlAgITysN8x7MBTdt/XBg==",
"peerDependencies": {
"typescript": "^4.3"
},
"peerDependenciesMeta": {
"typescript": {
"optional": true
}
}
},
"node_modules/@jest/environment": {
"version": "27.5.1",
"resolved": "https://registry.npmjs.org/@jest/environment/-/environment-27.5.1.tgz",
@ -216,6 +203,19 @@
"@lezer/common": "^0.15.0"
}
},
"node_modules/@mattermost/types": {
"version": "6.7.0-0",
"resolved": "https://registry.npmjs.org/@mattermost/types/-/types-6.7.0-0.tgz",
"integrity": "sha512-mT8wJwWEp20KPo9D12y7bW7EdUHO7VhUHxr3gH8nPGapWooGcl0Ra0H3u1iCjPpqPWvp7LiodcneU0IysunYKQ==",
"peerDependencies": {
"typescript": "^4.3"
},
"peerDependenciesMeta": {
"typescript": {
"optional": true
}
}
},
"node_modules/@sinonjs/commons": {
"version": "1.8.3",
"resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz",
@ -697,12 +697,6 @@
}
}
},
"@hmhealey/types": {
"version": "6.6.0-4",
"resolved": "https://registry.npmjs.org/@hmhealey/types/-/types-6.6.0-4.tgz",
"integrity": "sha512-71IxVaXhrUesmLnvQQh4RtUqqhmVL+ejci4qo4R6rTWTdY77BniRtBx269uAz34wzTlAgITysN8x7MBTdt/XBg==",
"requires": {}
},
"@jest/environment": {
"version": "27.5.1",
"resolved": "https://registry.npmjs.org/@jest/environment/-/environment-27.5.1.tgz",
@ -771,6 +765,12 @@
"@lezer/common": "^0.15.0"
}
},
"@mattermost/types": {
"version": "6.7.0-0",
"resolved": "https://registry.npmjs.org/@mattermost/types/-/types-6.7.0-0.tgz",
"integrity": "sha512-mT8wJwWEp20KPo9D12y7bW7EdUHO7VhUHxr3gH8nPGapWooGcl0Ra0H3u1iCjPpqPWvp7LiodcneU0IysunYKQ==",
"requires": {}
},
"@sinonjs/commons": {
"version": "1.8.3",
"resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz",

2
plugs/package.json
View File

@ -5,7 +5,7 @@
"generate": "lezer-generator query/query.grammar -o query/parse-query.js"
},
"dependencies": {
"@hmhealey/types": "^6.6.0-4",
"@mattermost/types": "^6.7.0-0",
"@jest/globals": "^27.5.1",
"@lezer/generator": "^0.15.4",
"@lezer/lr": "^0.15.8",

12
plugs/query/data.ts
View File

@ -1,23 +1,17 @@
// Index key space:
// data:page@pos
import { IndexEvent } from "../../webapp/app_event";
import { IndexTreeEvent } from "../../webapp/app_event";
import { batchSet, scanPrefixGlobal } from "plugos-silverbullet-syscall";
import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
import { collectNodesOfType, findNodeOfType, ParseTree, replaceNodesMatching } from "../../common/tree";
import YAML, { parse as parseYaml, parseAllDocuments } from "yaml";
import { whiteOutQueries } from "./util";
import type { QueryProviderEvent } from "./engine";
import { applyQuery } from "./engine";
export async function indexData({ name, text }: IndexEvent) {
text = whiteOutQueries(text);
// console.log("Now data indexing", name);
let mdTree = await parseMarkdown(text);
export async function indexData({ name, tree }: IndexTreeEvent) {
let dataObjects: { key: string; value: Object }[] = [];
collectNodesOfType(mdTree, "FencedCode").forEach((t) => {
collectNodesOfType(tree, "FencedCode").forEach((t) => {
let codeInfoNode = findNodeOfType(t, "CodeInfo");
if (!codeInfoNode) {
return;

30
plugs/query/materialized_queries.ts
View File

@ -1,30 +1,18 @@
import { flashNotification, getCurrentPage, reloadPage, save } from "plugos-silverbullet-syscall/editor";
import { flashNotification, getCurrentPage, getText, reloadPage, save } from "plugos-silverbullet-syscall/editor";
import { readPage, writePage } from "plugos-silverbullet-syscall/space";
import { invokeFunction } from "plugos-silverbullet-syscall/system";
import { parseQuery } from "./engine";
import { replaceTemplateVars } from "../core/template";
import { queryRegex } from "./util";
import { queryRegex, removeQueries } from "./util";
import { dispatch } from "plugos-syscall/event";
async function replaceAsync(
str: string,
regex: RegExp,
asyncFn: (match: string, ...args: any[]) => Promise<string>
) {
const promises: Promise<string>[] = [];
str.replace(regex, (match: string, ...args: any[]): string => {
const promise = asyncFn(match, ...args);
promises.push(promise);
return "";
});
const data = await Promise.all(promises);
return str.replace(regex, () => data.shift()!);
}
import { replaceAsync } from "../lib/util";
import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
export async function updateMaterializedQueriesCommand() {
const currentPage = await getCurrentPage();
await save();
await flashNotification("Updating materialized queries...");
await invokeFunction(
"server",
"updateMaterializedQueriesOnPage",
@ -34,6 +22,12 @@ export async function updateMaterializedQueriesCommand() {
await flashNotification("Updated materialized queries");
}
export async function whiteOutQueriesCommand() {
const text = await getText();
const parsed = await parseMarkdown(text);
console.log(removeQueries(parsed));
}
// Called from client, running on server
export async function updateMaterializedQueriesOnPage(pageName: string) {
let { text } = await readPage(pageName);
@ -49,7 +43,7 @@ export async function updateMaterializedQueriesOnPage(pageName: string) {
let results = await dispatch(
`query:${parsedQuery.table}`,
{ query: parsedQuery, pageName: pageName },
5000
10 * 1000
);
if (results.length === 0) {
return `${startQuery}\n${endQuery}`;

18
plugs/query/parse-query.js
View File

@ -3,15 +3,19 @@ import { LRParser } from "@lezer/lr";
export const parser = LRParser.deserialize({
version: 13,
states: "$[OVQPOOO[QQO'#C^QOQPOOOjQPO'#C`OoQQO'#CjOtQPO'#ClOOQO'#Cm'#CmOyQQO,58xO!XQPO'#CcO!sQQO'#CaOOQO'#Ca'#CaOOQO,58z,58zO#UQPO,59UOOQO,59W,59WOOQO-E6k-E6kO#ZQQO,58}OjQPO,58|O#oQQO1G.pOOQO'#Cg'#CgOOQO'#Ci'#CiOOQO'#Cd'#CdOOQO1G.i1G.iOOQO1G.h1G.hOOQO'#Ck'#CkOOQO7+$[7+$[",
stateData: "$T~OdOS~ORPO~OeROrSOvTObQX~ORWO~Os[O~OX]O~OeROrSOvTObQa~Of_Oj_Ok_Ol_Om_On_Oo_Op_O~Oq`ObTXeTXrTXvTX~ORaO~OXdOYdO[dOgbOhbOicO~OtgOugOb^ie^ir^iv^i~O",
states:
"$[OVQPOOO[QQO'#C^QOQPOOOjQPO'#C`OoQQO'#CjOtQPO'#ClOOQO'#Cm'#CmOyQQO,58xO!XQPO'#CcO!sQQO'#CaOOQO'#Ca'#CaOOQO,58z,58zO#UQPO,59UOOQO,59W,59WOOQO-E6k-E6kO#ZQQO,58}OjQPO,58|O#oQQO1G.pOOQO'#Cg'#CgOOQO'#Ci'#CiOOQO'#Cd'#CdOOQO1G.i1G.iOOQO1G.h1G.hOOQO'#Ck'#CkOOQO7+$[7+$[",
stateData:
"$T~OdOS~ORPO~OeROrSOvTObQX~ORWO~Os[O~OX]O~OeROrSOvTObQa~Of_Oj_Ok_Ol_Om_On_Oo_Op_O~Oq`ObTXeTXrTXvTX~ORaO~OXdOYdO[dOgbOhbOicO~OtgOugOb^ie^ir^iv^i~O",
goto: "!VbPPcPfjmpvPPyPyf|f!PRQOTUPVRZRRYRQXRRf`Re_Rd_RhaQVPR^V",
nodeNames: "⚠ Program Query Name WhereClause LogicalExpr AndExpr FilterExpr Value Number String Bool Regex Null OrderClause Order LimitClause",
nodeNames:
"⚠ Program Query Name WhereClause LogicalExpr AndExpr FilterExpr Value Number String Bool Regex Null OrderClause Order LimitClause",
maxTerm: 38,
skippedNodes: [0],
repeatNodeCount: 1,
tokenData: ":W~RvX^#ipq#iqr$^rs$q}!O%]!P!Q%n!Q![&e!^!_&m!_!`&z!`!a'X!c!}%]#R#S%]#T#U'f#U#V){#V#W%]#W#X*w#X#Y%]#Y#Z,s#Z#`%]#`#a/T#a#b%]#b#c1h#c#d3d#d#h%]#h#i5w#i#k%]#k#l7s#l#o%]#y#z#i$f$g#i#BY#BZ#i$IS$I_#i$Ip$Iq$q$Iq$Ir$q$I|$JO#i$JT$JU#i$KV$KW#i&FU&FV#i~#nYd~X^#ipq#i#y#z#i$f$g#i#BY#BZ#i$IS$I_#i$I|$JO#i$JT$JU#i$KV$KW#i&FU&FV#i~$aP!_!`$d~$iPl~#r#s$l~$qOp~~$tUOr$qrs%Ws$Ip$q$Ip$Iq%W$Iq$Ir%W$Ir~$q~%]OY~P%bSRP}!O%]!c!}%]#R#S%]#T#o%]~%sV[~OY%nZ]%n^!P%n!P!Q&Y!Q#O%n#O#P&_#P~%n~&_O[~~&bPO~%n~&jPX~!Q![&e~&rPf~!_!`&u~&zOj~~'PPk~#r#s'S~'XOo~~'^Pn~!_!`'a~'fOm~R'kWRP}!O%]!c!}%]#R#S%]#T#b%]#b#c(T#c#g%]#g#h)P#h#o%]R(YURP}!O%]!c!}%]#R#S%]#T#W%]#W#X(l#X#o%]R(sSqQRP}!O%]!c!}%]#R#S%]#T#o%]R)UURP}!O%]!c!}%]#R#S%]#T#V%]#V#W)h#W#o%]R)oSuQRP}!O%]!c!}%]#R#S%]#T#o%]R*QURP}!O%]!c!}%]#R#S%]#T#m%]#m#n*d#n#o%]R*kSsQRP}!O%]!c!}%]#R#S%]#T#o%]R*|URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y+`#Y#o%]R+eURP}!O%]!c!}%]#R#S%]#T#g%]#g#h+w#h#o%]R+|URP}!O%]!c!}%]#R#S%]#T#V%]#V#W,`#W#o%]R,gStQRP}!O%]!c!}%]#R#S%]#T#o%]R,xTRP}!O%]!c!}%]#R#S%]#T#U-X#U#o%]R-^URP}!O%]!c!}%]#R#S%]#T#`%]#`#a-p#a#o%]R-uURP}!O%]!c!}%]#R#S%]#T#g%]#g#h.X#h#o%]R.^URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y.p#Y#o%]R.wShQRP}!O%]!c!}%]#R#S%]#T#o%]R/YURP}!O%]!c!}%]#R#S%]#T#]%]#]#^/l#^#o%]R/qURP}!O%]!c!}%]#R#S%]#T#a%]#a#b0T#b#o%]R0YURP}!O%]!c!}%]#R#S%]#T#]%]#]#^0l#^#o%]R0qURP}!O%]!c!}%]#R#S%]#T#h%]#h#i1T#i#o%]R1[SvQRP}!O%]!c!}%]#R#S%]#T#o%]R1mURP}!O%]!c!}%]#R#S%]#T#i%]#i#j2P#j#o%]R2UURP}!O%]!c!}%]#R#S%]#T#`%]#`#a2h#a#o%]R2mURP}!O%]!c!}%]#R#S%]#T#`%]#`#a3P#a#o%]R3WSiQRP}!O%]!c!}%]#R#S%]#T#o%]R3iURP}!O%]!c!}%]#R#S%]#T#f%]#f#g3{#g#o%]R4QURP}!O%]!c!}%]#R#S%]#T#W%]#W#X4d#X#o%]R4iURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y4{#Y#o%]R5QURP}!O%]!c!}%]#R#S%]#T#f%]#f#g5d#g#o%]R5kSrQRP}!O%]!c!}%]#R#S%]#T#o%]R5|URP}!O%]!c!}%]#R#S%]#T#f%]#f#g6`#g#o%]R6eURP}!O%]!c!}%]#R#S%]#T#i%]#i#j6w#j#o%]R6|URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y7`#Y#o%]R7gSgQRP}!O%]!c!}%]#R#S%]#T#o%]R7xURP}!O%]!c!}%]#R#S%]#T#[%]#[#]8[#]#o%]R8aURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y8s#Y#o%]R8xURP}!O%]!c!}%]#R#S%]#T#f%]#f#g9[#g#o%]R9aURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y9s#Y#o%]R9zSeQRP}!O%]!c!}%]#R#S%]#T#o%]",
tokenData:
":W~RvX^#ipq#iqr$^rs$q}!O%]!P!Q%n!Q![&e!^!_&m!_!`&z!`!a'X!c!}%]#R#S%]#T#U'f#U#V){#V#W%]#W#X*w#X#Y%]#Y#Z,s#Z#`%]#`#a/T#a#b%]#b#c1h#c#d3d#d#h%]#h#i5w#i#k%]#k#l7s#l#o%]#y#z#i$f$g#i#BY#BZ#i$IS$I_#i$Ip$Iq$q$Iq$Ir$q$I|$JO#i$JT$JU#i$KV$KW#i&FU&FV#i~#nYd~X^#ipq#i#y#z#i$f$g#i#BY#BZ#i$IS$I_#i$I|$JO#i$JT$JU#i$KV$KW#i&FU&FV#i~$aP!_!`$d~$iPl~#r#s$l~$qOp~~$tUOr$qrs%Ws$Ip$q$Ip$Iq%W$Iq$Ir%W$Ir~$q~%]OY~P%bSRP}!O%]!c!}%]#R#S%]#T#o%]~%sV[~OY%nZ]%n^!P%n!P!Q&Y!Q#O%n#O#P&_#P~%n~&_O[~~&bPO~%n~&jPX~!Q![&e~&rPf~!_!`&u~&zOj~~'PPk~#r#s'S~'XOo~~'^Pn~!_!`'a~'fOm~R'kWRP}!O%]!c!}%]#R#S%]#T#b%]#b#c(T#c#g%]#g#h)P#h#o%]R(YURP}!O%]!c!}%]#R#S%]#T#W%]#W#X(l#X#o%]R(sSqQRP}!O%]!c!}%]#R#S%]#T#o%]R)UURP}!O%]!c!}%]#R#S%]#T#V%]#V#W)h#W#o%]R)oSuQRP}!O%]!c!}%]#R#S%]#T#o%]R*QURP}!O%]!c!}%]#R#S%]#T#m%]#m#n*d#n#o%]R*kSsQRP}!O%]!c!}%]#R#S%]#T#o%]R*|URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y+`#Y#o%]R+eURP}!O%]!c!}%]#R#S%]#T#g%]#g#h+w#h#o%]R+|URP}!O%]!c!}%]#R#S%]#T#V%]#V#W,`#W#o%]R,gStQRP}!O%]!c!}%]#R#S%]#T#o%]R,xTRP}!O%]!c!}%]#R#S%]#T#U-X#U#o%]R-^URP}!O%]!c!}%]#R#S%]#T#`%]#`#a-p#a#o%]R-uURP}!O%]!c!}%]#R#S%]#T#g%]#g#h.X#h#o%]R.^URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y.p#Y#o%]R.wShQRP}!O%]!c!}%]#R#S%]#T#o%]R/YURP}!O%]!c!}%]#R#S%]#T#]%]#]#^/l#^#o%]R/qURP}!O%]!c!}%]#R#S%]#T#a%]#a#b0T#b#o%]R0YURP}!O%]!c!}%]#R#S%]#T#]%]#]#^0l#^#o%]R0qURP}!O%]!c!}%]#R#S%]#T#h%]#h#i1T#i#o%]R1[SvQRP}!O%]!c!}%]#R#S%]#T#o%]R1mURP}!O%]!c!}%]#R#S%]#T#i%]#i#j2P#j#o%]R2UURP}!O%]!c!}%]#R#S%]#T#`%]#`#a2h#a#o%]R2mURP}!O%]!c!}%]#R#S%]#T#`%]#`#a3P#a#o%]R3WSiQRP}!O%]!c!}%]#R#S%]#T#o%]R3iURP}!O%]!c!}%]#R#S%]#T#f%]#f#g3{#g#o%]R4QURP}!O%]!c!}%]#R#S%]#T#W%]#W#X4d#X#o%]R4iURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y4{#Y#o%]R5QURP}!O%]!c!}%]#R#S%]#T#f%]#f#g5d#g#o%]R5kSrQRP}!O%]!c!}%]#R#S%]#T#o%]R5|URP}!O%]!c!}%]#R#S%]#T#f%]#f#g6`#g#o%]R6eURP}!O%]!c!}%]#R#S%]#T#i%]#i#j6w#j#o%]R6|URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y7`#Y#o%]R7gSgQRP}!O%]!c!}%]#R#S%]#T#o%]R7xURP}!O%]!c!}%]#R#S%]#T#[%]#[#]8[#]#o%]R8aURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y8s#Y#o%]R8xURP}!O%]!c!}%]#R#S%]#T#f%]#f#g9[#g#o%]R9aURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y9s#Y#o%]R9zSeQRP}!O%]!c!}%]#R#S%]#T#o%]",
tokenizers: [0, 1],
topRules: {"Program":[0,1]},
tokenPrec: 0
})
topRules: { Program: [0, 1] },
tokenPrec: 0,
});

4
plugs/query/query.plug.yaml
View File

@ -6,6 +6,10 @@ functions:
command:
name: "Materialized Queries: Update"
key: "Alt-q"
whiteOutQueriesCommand:
path: ./materialized_queries.ts:whiteOutQueriesCommand
command:
name: "Debug: Whiteout Queries"
indexData:
path: ./data.ts:indexData
events:

45
plugs/query/util.ts
View File

@ -1,8 +1,45 @@
import { addParentPointers, collectNodesMatching, ParseTree, renderToText } from "../../common/tree";
export const queryRegex =
/(<!--\s*#query\s+(.+?)-->)(.+?)(<!--\s*#end\s*-->)/gs;
export function whiteOutQueries(text: string): string {
return text.replaceAll(queryRegex, (match) =>
new Array(match.length + 1).join(" ")
);
export const queryStartRegex = /<!--\s*#query\s+(.+?)-->/s;
export const queryEndRegex = /<!--\s*#end\s*-->/s;
// export function whiteOutQueries(text: string): string {
// return text.replaceAll(queryRegex, (match) =>
// new Array(match.length + 1).join(" ")
// );
// }
export function removeQueries(pt: ParseTree) {
addParentPointers(pt);
collectNodesMatching(pt, (t) => {
if (t.type !== "CommentBlock") {
return false;
}
let text = t.children![0].text!;
if (!queryStartRegex.exec(text)) {
return false;
}
let parentChildren = t.parent!.children!;
let index = parentChildren.indexOf(t);
let nodesToReplace: ParseTree[] = [];
for (let i = index + 1; i < parentChildren.length; i++) {
let n = parentChildren[i];
if (n.type === "CommentBlock") {
let text = n.children![0].text!;
if (queryEndRegex.exec(text)) {
break;
}
}
nodesToReplace.push(n);
}
let renderedText = nodesToReplace.map(renderToText).join("");
parentChildren.splice(index + 1, nodesToReplace.length, {
text: new Array(renderedText.length + 1).join(" "),
});
return true;
});
}

12
plugs/tasks/task.ts
View File

@ -1,4 +1,4 @@
import type { ClickEvent, IndexEvent } from "../../webapp/app_event";
import type { ClickEvent, IndexTreeEvent } from "../../webapp/app_event";
import { batchSet, scanPrefixGlobal } from "plugos-silverbullet-syscall/index";
import { readPage, writePage } from "plugos-silverbullet-syscall/space";
@ -11,7 +11,7 @@ import {
nodeAtPos,
renderToText
} from "../../common/tree";
import { whiteOutQueries } from "../query/util";
import { removeQueries } from "../query/util";
import { applyQuery, QueryProviderEvent } from "../query/engine";
export type Task = {
@ -24,13 +24,11 @@ export type Task = {
page?: string;
};
export async function indexTasks({ name, text }: IndexEvent) {
export async function indexTasks({ name, tree }: IndexTreeEvent) {
// console.log("Indexing tasks");
let tasks: { key: string; value: Task }[] = [];
text = whiteOutQueries(text);
let mdTree = await parseMarkdown(text);
addParentPointers(mdTree);
collectNodesOfType(mdTree, "Task").forEach((n) => {
removeQueries(tree);
collectNodesOfType(tree, "Task").forEach((n) => {
let task = n.children!.slice(1).map(renderToText).join("").trim();
let complete = n.children![0].children![0].text! !== "[ ]";
let value: Task = {

10
plugs/yarn.lock
View File

@ -23,11 +23,6 @@
"chalk" "^2.0.0"
"js-tokens" "^4.0.0"
"@hmhealey/types@^6.6.0-4":
"integrity" "sha512-71IxVaXhrUesmLnvQQh4RtUqqhmVL+ejci4qo4R6rTWTdY77BniRtBx269uAz34wzTlAgITysN8x7MBTdt/XBg=="
"resolved" "https://registry.npmjs.org/@hmhealey/types/-/types-6.6.0-4.tgz"
"version" "6.6.0-4"
"@jest/environment@^27.5.1":
"integrity" "sha512-/WQjhPJe3/ghaol/4Bq480JKXV/Rfw8nQdN7f41fM8VDHLcxKXou6QyXAh3EFr9/bVG3x74z1NWDkP87EiY8gA=="
"resolved" "https://registry.npmjs.org/@jest/environment/-/environment-27.5.1.tgz"
@ -90,6 +85,11 @@
dependencies:
"@lezer/common" "^0.15.0"
"@mattermost/types@^6.7.0-0":
"integrity" "sha512-mT8wJwWEp20KPo9D12y7bW7EdUHO7VhUHxr3gH8nPGapWooGcl0Ra0H3u1iCjPpqPWvp7LiodcneU0IysunYKQ=="
"resolved" "https://registry.npmjs.org/@mattermost/types/-/types-6.7.0-0.tgz"
"version" "6.7.0-0"
"@sinonjs/commons@^1.7.0":
"integrity" "sha512-xkNcLAn/wZaX14RPlwizcKicDk9G3F8m2nU3L7Ukm5zBgTwiT0wsoFAHx9Jq56fJA1z/7uKGtCRu16sOUCLIHQ=="
"resolved" "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz"

7
webapp/app_event.ts
View File

@ -1,3 +1,5 @@
import type { ParseTree } from "../common/tree";
export type AppEvent = "page:click" | "page:complete";
export type ClickEvent = {
@ -12,3 +14,8 @@ export type IndexEvent = {
name: string;
text: string;
};
export type IndexTreeEvent = {
name: string;
tree: ParseTree;
};

8
webapp/editor.tsx
View File

@ -300,7 +300,6 @@ export class Editor {
closeBrackets(),
autocompletion({
override: [
// this.completerHook.plugCompleter.bind(this.completerHook),
this.completer.bind(this),
this.slashCommandHook.slashCommandCompleter.bind(
this.slashCommandHook
@ -320,6 +319,7 @@ export class Editor {
{ selector: "Comment", class: "line-comment" },
{ selector: "BulletList", class: "line-ul" },
{ selector: "OrderedList", class: "line-ol" },
{ selector: "TableHeader", class: "line-tbl-header" },
]),
keymap.of([
...smartQuoteKeymap,
@ -433,6 +433,9 @@ export class Editor {
editorView.setState(
this.createEditorState(this.currentPage, editorView.state.sliceDoc())
);
if (editorView.contentDOM) {
editorView.contentDOM.spellcheck = true;
}
}
}
@ -503,6 +506,9 @@ export class Editor {
let editorState = this.createEditorState(pageName, doc.text);
let pageState = this.openPages.get(pageName);
editorView.setState(editorState);
if (editorView.contentDOM) {
editorView.contentDOM.spellcheck = true;
}
if (!pageState) {
pageState = new PageState(0, editorState.selection);
this.openPages.set(pageName, pageState!);

13
webapp/markdown/markdown.ts
View File

@ -5,17 +5,10 @@ import {
Language,
languageDataProp,
LanguageDescription,
ParseContext,
ParseContext
} from "@codemirror/language";
import { styleTags, tags as t } from "@codemirror/highlight";
import {
Emoji,
GFM,
MarkdownParser,
parser as baseParser,
Subscript,
Superscript,
} from "@lezer/markdown";
import { Emoji, GFM, MarkdownParser, parser as baseParser, Subscript, Superscript } from "@lezer/markdown";
const data = defineLanguageFacet({ block: { open: "<!--", close: "-->" } });
@ -37,8 +30,6 @@ export const commonmark = baseParser.configure({
"StrongEmphasis/...": t.strong,
"Link/... Image/...": t.link,
"OrderedList/... BulletList/...": t.list,
// "CodeBlock/... FencedCode/...": t.blockComment,
"InlineCode CodeText": t.monospace,
URL: t.url,
"HeaderMark HardBreak QuoteMark ListMark LinkMark EmphasisMark CodeMark":

9
webapp/parser.ts
View File

@ -1,5 +1,5 @@
import { styleTags } from "@codemirror/highlight";
import { BlockContext, LeafBlock, LeafBlockParser, MarkdownConfig, parseCode, TaskList } from "@lezer/markdown";
import { styleTags, tags as t } from "@codemirror/highlight";
import { BlockContext, LeafBlock, LeafBlockParser, MarkdownConfig, parseCode, Table, TaskList } from "@lezer/markdown";
import { commonmark, getCodeParser, mkLang } from "./markdown/markdown";
import * as ct from "./customtags";
import { Language, LanguageDescription, LanguageSupport } from "@codemirror/language";
@ -73,6 +73,7 @@ export default function buildMarkdown(mdExtensions: MDExt[]): Language {
WikiLink,
TaskList,
Comment,
Table,
...mdExtensions.map(mdExtensionSyntaxConfig),
parseCode({
codeParser: getCodeParser([
@ -96,6 +97,10 @@ export default function buildMarkdown(mdExtensions: MDExt[]): Language {
Task: ct.TaskTag,
TaskMarker: ct.TaskMarkerTag,
Comment: ct.CommentTag,
"TableDelimiter SubscriptMark SuperscriptMark StrikethroughMark":
t.processingInstruction,
"TableHeader/...": t.heading,
TableCell: t.content,
}),
...mdExtensions.map((mdExt) =>
styleTags(mdExtensionStyleTags(mdExt))

2
webapp/style.ts
View File

@ -36,6 +36,8 @@ export default function highlightStyles(mdExtension: MDExt[]) {
{ tag: t.variableName, class: "variableName" },
{ tag: t.comment, class: "comment" },
{ tag: t.invalid, class: "invalid" },
{ tag: t.processingInstruction, class: "meta" },
// { tag: t.content, class: "tbl-content" },
{ tag: t.punctuation, class: "punctuation" },
...mdExtension.map((mdExt) => {
return { tag: mdExt.tag, ...mdExt.styles };

31
webapp/styles/editor.scss
View File

@ -58,12 +58,35 @@
}
.line-code {
background-color: #efefef;
margin-left: 30px;
background-color: rgba(72, 72, 72, 0.1);
.code {
background-color: transparent;
}
}
.line-tbl-header {
font-weight: bold;
.meta {
font-weight: normal;
}
}
.struct {
color: darkred;
}
.code {
background-color: rgba(72, 72, 72, 0.1);
}
.line-fenced-code {
background-color: rgba(72, 72, 72, 0.1);
.code {
background-color: transparent;
}
}
.meta {
@ -117,10 +140,6 @@
.code {
background-color: #efefef;
}
// Indentation of follow-up lines
@mixin lineOverflow($baseIndent) {
text-indent: -1 * ($baseIndent + 2ch);