neil/silverbullet
1
0
Fork 0
Code Activity

Complete redo of content indexing and querying (#517)

Browse Source 
Complete redo of data store
Introduces live queries and live templates
This commit is contained in:
Zef Hemel
2023-10-03 14:16:33 +02:00
committed by GitHub
parent 7af98e7c7b
commit 0313565610
200 changed files with 4675 additions and 4363 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugs/builtin_plugs.ts
+1
View File
@@ -7,6 +7,7 @@ export const builtinPlugNames = [
"plug-manager",
"directive",
"emoji",
"query",
"markdown",
"share",
"tasks",
plugs/directive/command.ts
+31 -2
View File
@@ -1,5 +1,8 @@
import { editor, markdown, mq, space, sync } from "$sb/syscalls.ts";
import {
addParentPointers,
findParentMatching,
nodeAtPos,
ParseTree,
removeParentPointers,
renderToText,
@@ -7,9 +10,8 @@ import {
} from "$sb/lib/tree.ts";
import { renderDirectives } from "./directives.ts";
import { extractFrontmatter } from "$sb/lib/frontmatter.ts";
import type { PageMeta } from "../../web/types.ts";
import { isFederationPath } from "$sb/lib/resolve.ts";
import { MQMessage } from "$sb/types.ts";
import { MQMessage, PageMeta } from "$sb/types.ts";
import { sleep } from "$sb/lib/async.ts";
const directiveUpdateQueueName = "directiveUpdateQueue";
@@ -200,3 +202,30 @@ export async function updateDirectives(
}
return text;
}
export async function convertToLiveQuery() {
const text = await editor.getText();
const pos = await editor.getCursor();
const tree = await markdown.parseMarkdown(text);
addParentPointers(tree);
const currentNode = nodeAtPos(tree, pos);
const directive = findParentMatching(
currentNode!,
(node) => node.type === "Directive",
);
if (!directive) {
await editor.flashNotification(
"No directive found at cursor position",
"error",
);
return;
}
const queryText = renderToText(directive!.children![0].children![1]);
await editor.dispatch({
changes: {
from: directive.from,
to: directive.to,
insert: "```query\n" + queryText + "\n```",
},
});
}
plugs/directive/complete.ts
+23 -10
View File
@@ -1,11 +1,11 @@
import { events } from "$sb/syscalls.ts";
import { CompleteEvent } from "$sb/app_event.ts";
import { buildHandebarOptions } from "./util.ts";
import type { PageMeta } from "../../web/types.ts";
import type {
AttributeCompleteEvent,
AttributeCompletion,
} from "../index/attributes.ts";
import { PageMeta } from "$sb/types.ts";
export async function queryComplete(completeEvent: CompleteEvent) {
const querySourceMatch = /#query\s+([\w\-_]*)$/.exec(
@@ -14,18 +14,31 @@ export async function queryComplete(completeEvent: CompleteEvent) {
if (querySourceMatch) {
const allEvents = await events.listEvents();
const completionOptions = allEvents
.filter((eventName) =>
eventName.startsWith("query:") && !eventName.includes("*")
)
.map((source) => ({
label: source.substring("query:".length),
}));
const allObjectTypes: string[] = (await events.dispatchEvent("query_", {}))
.flat();
for (const type of allObjectTypes) {
completionOptions.push({
label: type,
});
}
return {
from: completeEvent.pos - querySourceMatch[1].length,
options: allEvents
.filter((eventName) => eventName.startsWith("query:"))
.map((source) => ({
label: source.substring("query:".length),
})),
options: completionOptions,
};
}
if (completeEvent.parentNodes.includes("DirectiveStart")) {
const querySourceMatch = /#query\s+([\w\-_]+)/.exec(
const querySourceMatch = /#query\s+([\w\-_\/]+)/.exec(
completeEvent.linePrefix,
);
const whereMatch =
@@ -69,9 +82,9 @@ export async function templateVariableComplete(completeEvent: CompleteEvent) {
);
const completions = (await events.dispatchEvent(
`attribute:complete:*`,
`attribute:complete:_`,
{
source: "*",
source: "",
prefix: match[1],
} as AttributeCompleteEvent,
)).flat() as AttributeCompletion[];
@@ -92,7 +105,7 @@ export function attributeCompletionsToCMCompletion(
return completions.map(
(completion) => ({
label: completion.name,
detail: `${completion.type} (${completion.source})`,
detail: `${completion.attributeType} (${completion.source})`,
type: "attribute",
}),
);
plugs/directive/directive.plug.yaml
+6 -44
View File
@@ -20,14 +20,6 @@ functions:
mqSubscriptions:
- queue: directiveUpdateQueue
batchSize: 3
indexData:
path: ./data.ts:indexData
events:
- page:index
dataQueryProvider:
path: ./data.ts:queryProvider
events:
- query:data
queryComplete:
path: ./complete.ts:queryComplete
events:
@@ -37,43 +29,13 @@ functions:
events:
- editor:complete
# Conversion
convertToLiveQuery:
path: command.ts:convertToLiveQuery
command:
name: "Directive: Convert Query to Live Query"
# Templates
insertQuery:
redirect: template.insertTemplateText
slashCommand:
name: query
description: Insert a query
value: |
<!-- #query |^| -->
<!-- /query -->
insertInclude:
redirect: template.insertTemplateText
slashCommand:
name: include
description: Include another page
value: |
<!-- #include [[|^|]] -->
<!-- /include -->
insertUseTemplate:
redirect: template.insertTemplateText
slashCommand:
name: use
description: Use a template
value: |
<!-- #use [[|^|]] {} -->
<!-- /use -->
insertUseVerboseTemplate:
redirect: template.insertTemplateText
slashCommand:
name: use-verbose
description: Use a template (verbose mode)
value: |
<!-- #use-verbose [[|^|]] {} -->
<!-- /use-verbose -->
insertEvalTemplate:
redirect: template.insertTemplateText
slashCommand:
plugs/directive/directives.ts
+31 -14
View File
@@ -1,5 +1,11 @@
import { ParseTree, renderToText } from "$sb/lib/tree.ts";
import { PageMeta } from "../../web/types.ts";
import {
addParentPointers,
findParentMatching,
ParseTree,
renderToText,
} from "$sb/lib/tree.ts";
import { PageMeta } from "$sb/types.ts";
import { editor, markdown } from "$sb/syscalls.ts";
import { evalDirectiveRenderer } from "./eval_directive.ts";
import { queryDirectiveRenderer } from "./query_directive.ts";
@@ -53,8 +59,29 @@ export async function directiveDispatcher(
const directiveStartText = renderToText(directiveStart).trim();
const directiveEndText = renderToText(directiveEnd).trim();
if (directiveStart.children!.length === 1) {
// Everything not #query
const firstPart = directiveStart.children![0].text!;
if (firstPart?.includes("#query")) {
// #query
const newBody = await directiveRenderers["query"](
"query",
pageMeta,
directiveStart.children![1].children![0], // The query ParseTree
);
const result =
`${directiveStartText}\n${newBody.trim()}\n${directiveEndText}`;
return result;
} else if (firstPart?.includes("#eval")) {
console.log("Eval stuff", directiveStart.children![1].children![0]);
const newBody = await directiveRenderers["eval"](
"eval",
pageMeta,
directiveStart.children![1].children![0],
);
const result =
`${directiveStartText}\n${newBody.trim()}\n${directiveEndText}`;
return result;
} else {
// Everything not #query and #eval
const match = directiveStartRegex.exec(directiveStart.children![0].text!);
if (!match) {
throw Error("No match");
@@ -70,16 +97,6 @@ export async function directiveDispatcher(
} catch (e: any) {
return `${directiveStartText}\n**ERROR:** ${e.message}\n${directiveEndText}`;
}
} else {
// #query
const newBody = await directiveRenderers["query"](
"query",
pageMeta,
directiveStart.children![1], // The query ParseTree
);
const result =
`${directiveStartText}\n${newBody.trim()}\n${directiveEndText}`;
return result;
}
}
plugs/directive/eval_directive.ts
+16 -53
View File
@@ -1,23 +1,9 @@
// This is some shocking stuff. My profession would kill me for this.
import { YAML } from "$sb/syscalls.ts";
import { ParseTree } from "$sb/lib/tree.ts";
import { jsonToMDTable, renderTemplate } from "./util.ts";
import type { PageMeta } from "../../web/types.ts";
import { ParseTree, parseTreeToAST } from "$sb/lib/tree.ts";
import { replaceTemplateVars } from "../template/template.ts";
// Enables plugName.functionName(arg1, arg2) syntax in JS expressions
function translateJs(js: string): string {
return js.replaceAll(
/(\w+\.\w+)\s*\(/g,
'await invokeFunction("$1", ',
);
}
// Syntaxes to support:
// - random JS expression
// - random JS expression render [[some/template]]
const expressionRegex = /(.+?)(\s+render\s+\[\[([^\]]+)\]\])?$/;
import { PageMeta } from "$sb/types.ts";
import { expressionToKvQueryExpression } from "$sb/lib/parse-query.ts";
import { evalQueryExpression } from "$sb/lib/query.ts";
import { builtinFunctions } from "$sb/lib/builtin_query_functions.ts";
// This is rather scary and fragile stuff, but it works.
export async function evalDirectiveRenderer(
@@ -25,42 +11,19 @@ export async function evalDirectiveRenderer(
pageMeta: PageMeta,
expression: string | ParseTree,
): Promise<string> {
if (typeof expression !== "string") {
throw new Error("Expected a string");
}
// console.log("Got JS expression", expression);
const match = expressionRegex.exec(expression);
if (!match) {
throw new Error(`Invalid eval directive: ${expression}`);
}
let template = "";
if (match[3]) {
// This is the template reference
expression = match[1];
template = match[3];
}
try {
// Why the weird "eval" call? https://esbuild.github.io/content-types/#direct-eval
const result = await (0, eval)(
`(async () => {
function invokeFunction(name, ...args) {
return syscall("system.invokeFunction", name, ...args);
}
return ${replaceTemplateVars(translateJs(expression), pageMeta)};
})()`,
const result = evalQueryExpression(
expressionToKvQueryExpression(parseTreeToAST(
JSON.parse(
await replaceTemplateVars(JSON.stringify(expression), pageMeta),
),
)),
{},
builtinFunctions,
);
if (template) {
return await renderTemplate(pageMeta, template, result);
}
if (typeof result === "string") {
return result;
} else if (typeof result === "number") {
return "" + result;
} else if (Array.isArray(result)) {
return jsonToMDTable(result);
}
return await YAML.stringify(result);
return Promise.resolve("" + result);
} catch (e: any) {
return `**ERROR:** ${e.message}`;
return Promise.resolve(`**ERROR:** ${e.message}`);
}
}
plugs/directive/handlebar_helpers.ts
-40
View File
@@ -1,40 +0,0 @@
import { niceDate } from "$sb/lib/dates.ts";
export function handlebarHelpers(_pageName: string) {
return {
json: (v: any) => JSON.stringify(v),
niceDate: (ts: any) => niceDate(new Date(ts)),
escapeRegexp: (ts: any) => {
return ts.replace(/[-\/\\^$*+?.()|[\]{}]/g, "\\$&");
},
replaceRegexp: (s: string, regexp: string, replacement: string) => {
return s.replace(new RegExp(regexp, "g"), replacement);
},
prefixLines: (v: string, prefix: string) =>
v.split("\n").map((l) => prefix + l).join("\n"),
substring: (s: string, from: number, to: number, elipsis = "") =>
s.length > to - from ? s.substring(from, to) + elipsis : s,
today: () => niceDate(new Date()),
tomorrow: () => {
const tomorrow = new Date();
tomorrow.setDate(tomorrow.getDate() + 1);
return niceDate(tomorrow);
},
yesterday: () => {
const yesterday = new Date();
yesterday.setDate(yesterday.getDate() - 1);
return niceDate(yesterday);
},
lastWeek: () => {
const lastWeek = new Date();
lastWeek.setDate(lastWeek.getDate() - 7);
return niceDate(lastWeek);
},
nextWeek: () => {
const nextWeek = new Date();
nextWeek.setDate(nextWeek.getDate() + 7);
return niceDate(nextWeek);
},
};
}
plugs/directive/parser.ts
-122
View File
@@ -1,122 +0,0 @@
import {
collectNodesOfType,
findNodeOfType,
ParseTree,
replaceNodesMatching,
} from "$sb/lib/tree.ts";
// @ts-ignore auto generated
import { ParsedQuery, QueryFilter } from "$sb/lib/query.ts";
export function parseQuery(queryTree: ParseTree): ParsedQuery {
// const n = lezerToParseTree(query, parser.parse(query).topNode);
// Clean the tree a bit
replaceNodesMatching(queryTree, (n) => {
if (!n.type) {
const trimmed = n.text!.trim();
if (!trimmed) {
return null;
}
n.text = trimmed;
}
});
// console.log("Parsed", JSON.stringify(n, null, 2));
const queryNode = queryTree.children![0];
const parsedQuery: ParsedQuery = {
table: queryNode.children![0].children![0].text!,
filter: [],
ordering: [],
};
const orderByNodes = collectNodesOfType(queryNode, "OrderClause");
for (const orderByNode of orderByNodes) {
const nameNode = findNodeOfType(orderByNode, "Name");
const orderBy = nameNode!.children![0].text!;
const orderNode = findNodeOfType(orderByNode, "OrderDirection");
const orderDesc = orderNode
? orderNode.children![0].text! === "desc"
: false;
parsedQuery.ordering.push({ orderBy, orderDesc });
}
/**
* @deprecated due to PR #387
* We'll take the first ordering and send that as the deprecated
* fields orderBy and orderDesc. This way it will be backward
* Plugs using the old ParsedQuery.
* Remove this block completely when ParsedQuery no longer have
* those two fields
*/
if (parsedQuery.ordering.length > 0) {
parsedQuery.orderBy = parsedQuery.ordering[0].orderBy;
parsedQuery.orderDesc = parsedQuery.ordering[0].orderDesc;
}
/** @end-deprecation due to PR #387 */
const limitNode = findNodeOfType(queryNode, "LimitClause");
if (limitNode) {
const nameNode = findNodeOfType(limitNode, "Number");
parsedQuery.limit = valueNodeToVal(nameNode!);
}
const filterNodes = collectNodesOfType(queryNode, "FilterExpr");
for (const filterNode of filterNodes) {
let val: any = undefined;
const valNode = filterNode.children![2].children![0];
val = valueNodeToVal(valNode);
const f: QueryFilter = {
prop: filterNode.children![0].children![0].text!,
op: filterNode.children![1].text!,
value: val,
};
parsedQuery.filter.push(f);
}
const selectNode = findNodeOfType(queryNode, "SelectClause");
if (selectNode) {
parsedQuery.select = [];
collectNodesOfType(selectNode, "Name").forEach((t) => {
parsedQuery.select!.push(t.children![0].text!);
});
}
const renderNode = findNodeOfType(queryNode, "RenderClause");
if (renderNode) {
let renderNameNode = findNodeOfType(renderNode, "PageRef");
if (!renderNameNode) {
renderNameNode = findNodeOfType(renderNode, "String");
}
parsedQuery.render = valueNodeToVal(renderNameNode!);
}
return parsedQuery;
}
export function valueNodeToVal(valNode: ParseTree): any {
switch (valNode.type) {
case "Number":
return +valNode.children![0].text!;
case "Bool":
return valNode.children![0].text! === "true";
case "Null":
return null;
case "Name":
return valNode.children![0].text!;
case "Regex": {
const val = valNode.children![0].text!;
return val.substring(1, val.length - 1);
}
case "String": {
const stringVal = valNode.children![0].text!;
return stringVal.substring(1, stringVal.length - 1);
}
case "PageRef": {
const pageRefVal = valNode.children![0].text!;
return pageRefVal.substring(2, pageRefVal.length - 2);
}
case "List": {
return collectNodesOfType(valNode, "Value").map((t) =>
valueNodeToVal(t.children![0])
);
}
}
}
plugs/directive/query.test.ts
-187
View File
@@ -1,187 +0,0 @@
import { assertEquals } from "../../test_deps.ts";
import { applyQuery } from "$sb/lib/query.ts";
import wikiMarkdownLang from "../../common/markdown_parser/parser.ts";
import { parse } from "../../common/markdown_parser/parse_tree.ts";
import { parseQuery as parseQueryQuery } from "./parser.ts";
import { findNodeOfType, renderToText } from "../../plug-api/lib/tree.ts";
function parseQuery(query: string) {
const lang = wikiMarkdownLang([]);
const mdTree = parse(
lang,
`<!-- #query ${query} -->
<!-- /query -->`,
);
const programNode = findNodeOfType(mdTree, "Program")!;
return parseQueryQuery(programNode);
}
Deno.test("Test parser", () => {
const parsedBasicQuery = parseQuery(`page`);
assertEquals(parsedBasicQuery.table, "page");
const parsedQuery1 = parseQuery(
`task where completed = false and dueDate <= "{{today}}" order by dueDate desc limit 5`,
);
assertEquals(parsedQuery1.table, "task");
assertEquals(parsedQuery1.ordering.length, 1);
assertEquals(parsedQuery1.ordering[0].orderBy, "dueDate");
assertEquals(parsedQuery1.ordering[0].orderDesc, true);
assertEquals(parsedQuery1.limit, 5);
assertEquals(parsedQuery1.filter.length, 2);
assertEquals(parsedQuery1.filter[0], {
op: "=",
prop: "completed",
value: false,
});
assertEquals(parsedQuery1.filter[1], {
op: "<=",
prop: "dueDate",
value: "{{today}}",
});
const parsedQuery2 = parseQuery(`page where name =~ /interview\\/.*/"`);
assertEquals(parsedQuery2.table, "page");
assertEquals(parsedQuery2.filter.length, 1);
assertEquals(parsedQuery2.filter[0], {
op: "=~",
prop: "name",
value: "interview\\/.*",
});
const parsedQuery3 = parseQuery(`page where something != null`);
assertEquals(parsedQuery3.table, "page");
assertEquals(parsedQuery3.filter.length, 1);
assertEquals(parsedQuery3.filter[0], {
op: "!=",
prop: "something",
value: null,
});
assertEquals(parseQuery(`page select name`).select, ["name"]);
assertEquals(parseQuery(`page select name, age`).select, [
"name",
"age",
]);
assertEquals(
parseQuery(`gh-events where type in ["PushEvent", "somethingElse"]`),
{
table: "gh-events",
ordering: [],
filter: [
{
op: "in",
prop: "type",
value: ["PushEvent", "somethingElse"],
},
],
},
);
assertEquals(parseQuery(`something render [[template/table]]`), {
table: "something",
ordering: [],
filter: [],
render: "template/table",
});
assertEquals(parseQuery(`something render "template/table"`), {
table: "something",
ordering: [],
filter: [],
render: "template/table",
});
});
Deno.test("Test applyQuery", () => {
const data: any[] = [
{ name: "interview/My Interview", lastModified: 1 },
{ name: "interview/My Interview 2", lastModified: 2 },
{ name: "Pete", age: 38 },
{ name: "Angie", age: 28 },
];
assertEquals(
applyQuery(parseQuery(`page where name =~ /interview\\/.*/`), data),
[
{ name: "interview/My Interview", lastModified: 1 },
{ name: "interview/My Interview 2", lastModified: 2 },
],
);
assertEquals(
applyQuery(
parseQuery(`page where name =~ /interview\\/.*/ order by lastModified`),
data,
),
[
{ name: "interview/My Interview", lastModified: 1 },
{ name: "interview/My Interview 2", lastModified: 2 },
],
);
assertEquals(
applyQuery(
parseQuery(
`page where name =~ /interview\\/.*/ order by lastModified desc`,
),
data,
),
[
{ name: "interview/My Interview 2", lastModified: 2 },
{ name: "interview/My Interview", lastModified: 1 },
],
);
assertEquals(applyQuery(parseQuery(`page where age > 30`), data), [
{ name: "Pete", age: 38 },
]);
assertEquals(
applyQuery(parseQuery(`page where age > 28 and age < 38`), data),
[],
);
assertEquals(
applyQuery(parseQuery(`page where age > 30 select name`), data),
[{ name: "Pete" }],
);
assertEquals(
applyQuery(parseQuery(`page where name in ["Pete"] select name`), data),
[{ name: "Pete" }],
);
});
Deno.test("Test applyQuery with multi value", () => {
const data: any[] = [
{ name: "Pete", children: ["John", "Angie"] },
{ name: "Angie", children: ["Angie"] },
{ name: "Steve" },
];
assertEquals(
applyQuery(parseQuery(`page where children = "Angie"`), data),
[
{ name: "Pete", children: ["John", "Angie"] },
{ name: "Angie", children: ["Angie"] },
],
);
assertEquals(
applyQuery(parseQuery(`page where children = ["Angie", "John"]`), data),
[
{ name: "Pete", children: ["John", "Angie"] },
{ name: "Angie", children: ["Angie"] },
],
);
});
const testQuery = `<!-- #query source where a = 1 and b = "2" and c = "3" -->
<!-- /query -->`;
Deno.test("Query parsing and serialization", () => {
const lang = wikiMarkdownLang([]);
const mdTree = parse(lang, testQuery);
// console.log(JSON.stringify(mdTree, null, 2));
assertEquals(renderToText(mdTree), testQuery);
});
plugs/directive/query_directive.ts
+15 -13
View File
@@ -2,10 +2,10 @@ import { events } from "$sb/syscalls.ts";
import { replaceTemplateVars } from "../template/template.ts";
import { renderTemplate } from "./util.ts";
import { parseQuery } from "./parser.ts";
import { jsonToMDTable } from "./util.ts";
import { ParseTree } from "$sb/lib/tree.ts";
import type { PageMeta } from "../../web/types.ts";
import { ParseTree, parseTreeToAST } from "$sb/lib/tree.ts";
import { astToKvQuery } from "$sb/lib/parse-query.ts";
import { PageMeta, Query } from "$sb/types.ts";
export async function queryDirectiveRenderer(
_directive: string,
@@ -15,11 +15,14 @@ export async function queryDirectiveRenderer(
if (typeof query === "string") {
throw new Error("Argument must be a ParseTree");
}
const parsedQuery = parseQuery(
JSON.parse(replaceTemplateVars(JSON.stringify(query), pageMeta)),
const parsedQuery: Query = astToKvQuery(
parseTreeToAST(
JSON.parse(await replaceTemplateVars(JSON.stringify(query), pageMeta)),
),
);
// console.log("QUERY", parsedQuery);
const eventName = `query:${parsedQuery.table}`;
const eventName = `query:${parsedQuery.querySource}`;
// console.log("Parsed query", parsedQuery);
// Let's dispatch an event and see what happens
@@ -30,24 +33,23 @@ export async function queryDirectiveRenderer(
);
if (results.length === 0) {
// This means there was no handler for the event which means it's unsupported
return `**Error:** Unsupported query source '${parsedQuery.table}'`;
} else if (results.length === 1) {
return `**Error:** Unsupported query source '${parsedQuery.querySource}'`;
} else {
// console.log("Parsed query", parsedQuery);
const allResults = results.flat();
if (parsedQuery.render) {
const rendered = await renderTemplate(
pageMeta,
parsedQuery.render,
results[0],
allResults,
);
return rendered.trim();
} else {
if (results[0].length === 0) {
if (allResults.length === 0) {
return "No results";
} else {
return jsonToMDTable(results[0]);
return jsonToMDTable(allResults);
}
}
} else {
throw new Error(`Too many query results: ${results.length}`);
}
}
plugs/directive/template_directive.ts
+6 -10
View File
@@ -1,15 +1,13 @@
import { queryRegex } from "$sb/lib/query.ts";
import { ParseTree, renderToText } from "$sb/lib/tree.ts";
import { markdown, space } from "$sb/syscalls.ts";
import Handlebars from "handlebars";
import { handlebars, markdown, space } from "$sb/syscalls.ts";
import { replaceTemplateVars } from "../template/template.ts";
import { extractFrontmatter } from "$sb/lib/frontmatter.ts";
import { directiveRegex } from "./directives.ts";
import { updateDirectives } from "./command.ts";
import { buildHandebarOptions } from "./util.ts";
import { PageMeta } from "../../web/types.ts";
import { resolvePath, rewritePageRefs } from "$sb/lib/resolve.ts";
import { PageMeta } from "$sb/types.ts";
const templateRegex = /\[\[([^\]]+)\]\]\s*(.*)\s*/;
@@ -30,7 +28,7 @@ export async function templateDirectiveRenderer(
let parsedArgs = {};
if (args) {
try {
parsedArgs = JSON.parse(replaceTemplateVars(args, pageMeta));
parsedArgs = JSON.parse(await replaceTemplateVars(args, pageMeta));
} catch {
throw new Error(
`Failed to parse template instantiation arg: ${
@@ -65,11 +63,9 @@ export async function templateDirectiveRenderer(
// if it's a template injection (not a literal "include")
if (directive === "use") {
const templateFn = Handlebars.compile(
newBody,
{ noEscape: true },
);
newBody = templateFn(parsedArgs, buildHandebarOptions(pageMeta));
newBody = await handlebars.renderTemplate(newBody, parsedArgs, {
page: pageMeta,
});
// Recursively render directives
const tree = await markdown.parseMarkdown(newBody);
plugs/directive/util.ts
+8 -8
View File
@@ -1,8 +1,6 @@
import Handlebars from "handlebars";
import { space } from "$sb/syscalls.ts";
import type { PageMeta } from "../../web/types.ts";
import { handlebarHelpers } from "./handlebar_helpers.ts";
import { handlebars, space } from "$sb/syscalls.ts";
import { handlebarHelpers } from "../../common/syscalls/handlebar_helpers.ts";
import { PageMeta } from "$sb/types.ts";
const maxWidth = 70;
@@ -10,6 +8,9 @@ export function defaultJsonTransformer(_k: string, v: any) {
if (v === undefined) {
return "";
}
if (typeof v === "string") {
return v.replaceAll("\n", " ").replaceAll("|", "\\|");
}
return "" + v;
}
@@ -86,13 +87,12 @@ export async function renderTemplate(
): Promise<string> {
let templateText = await space.readPage(renderTemplate);
templateText = `{{#each .}}\n${templateText}\n{{/each}}`;
const template = Handlebars.compile(templateText, { noEscape: true });
return template(data, buildHandebarOptions(pageMeta));
return handlebars.renderTemplate(templateText, data, { page: pageMeta });
}
export function buildHandebarOptions(pageMeta: PageMeta) {
return {
helpers: handlebarHelpers(pageMeta.name),
helpers: handlebarHelpers(),
data: { page: pageMeta },
};
}
plugs/editor/complete.ts
+1 -1
View File
@@ -1,6 +1,6 @@
import { CompleteEvent } from "$sb/app_event.ts";
import { space } from "$sb/syscalls.ts";
import { PageMeta } from "../../web/types.ts";
import { PageMeta } from "$sb/types.ts";
import { cacheFileListing } from "../federation/federation.ts";
// Completion
plugs/editor/link.ts
+1 -1
View File
@@ -57,7 +57,7 @@ export async function titleUnfurl(url: string): Promise<string> {
const response = await fetch(url);
if (response.status < 200 || response.status >= 300) {
console.error("Unfurl failed", await response.text());
throw new Error(`Failed to fetch: ${await response.statusText}`);
throw new Error(`Failed to fetch: ${response.statusText}`);
}
const body = await response.text();
const match = titleRegex.exec(body);
plugs/editor/page.ts
-4
View File
@@ -1,9 +1,5 @@
import type { CompleteEvent } from "$sb/app_event.ts";
import { editor, space } from "$sb/syscalls.ts";
import { cacheFileListing } from "../federation/federation.ts";
import type { PageMeta } from "../../web/types.ts";
export async function deletePage() {
const pageName = await editor.getCurrentPage();
if (
plugs/editor/vim.ts
+4 -4
View File
@@ -1,11 +1,11 @@
import { readCodeBlockPage } from "$sb/lib/yaml_page.ts";
import { editor, store } from "$sb/syscalls.ts";
import { clientStore, editor } from "$sb/syscalls.ts";
export async function toggleVimMode() {
let vimMode = await store.get("vimMode");
let vimMode = await clientStore.get("vimMode");
vimMode = !vimMode;
await editor.setUiOption("vimMode", vimMode);
await store.set("vimMode", vimMode);
await clientStore.set("vimMode", vimMode);
}
export async function loadVimRc() {
@@ -28,7 +28,7 @@ export async function loadVimRc() {
}
}
}
} catch (e: any) {
} catch {
// No VIMRC page found
}
}
plugs/federation/federation.ts
+5 -5
View File
@@ -1,7 +1,7 @@
import "$sb/lib/fetch.ts";
import { federatedPathToUrl } from "$sb/lib/resolve.ts";
import { readFederationConfigs } from "./config.ts";
import { store } from "$sb/syscalls.ts";
import { datastore } from "$sb/syscalls.ts";
import type { FileMeta } from "$sb/types.ts";
async function responseToFileMeta(
@@ -29,7 +29,7 @@ async function responseToFileMeta(
};
}
const fileListingPrefixCacheKey = `federationListCache:`;
const fileListingPrefixCacheKey = `federationListCache`;
const listingCacheTimeout = 1000 * 30;
const listingFetchTimeout = 2000;
@@ -56,8 +56,8 @@ export async function listFiles(): Promise<FileMeta[]> {
}
export async function cacheFileListing(uri: string): Promise<FileMeta[]> {
const cachedListing = await store.get(
`${fileListingPrefixCacheKey}${uri}`,
const cachedListing = await datastore.get(
[fileListingPrefixCacheKey, uri],
) as FileListingCacheEntry;
if (
cachedListing &&
@@ -99,7 +99,7 @@ export async function cacheFileListing(uri: string): Promise<FileMeta[]> {
perm: "ro",
name: `${rootUri}/${meta.name}`,
}));
await store.set(`${fileListingPrefixCacheKey}${uri}`, {
await datastore.set([fileListingPrefixCacheKey, uri], {
items,
lastUpdated: Date.now(),
} as FileListingCacheEntry);
plugs/index/anchor.ts
+18 -11
View File
@@ -1,24 +1,31 @@
import { collectNodesOfType } from "$sb/lib/tree.ts";
import { index } from "$sb/syscalls.ts";
import type { CompleteEvent, IndexTreeEvent } from "$sb/app_event.ts";
import { removeQueries } from "$sb/lib/query.ts";
import { ObjectValue } from "$sb/types.ts";
import { indexObjects, queryObjects } from "./api.ts";
// Key space
// a:pageName:anchorName => pos
type AnchorObject = ObjectValue<{
name: string;
page: string;
pos: number;
}>;
export async function indexAnchors({ name: pageName, tree }: IndexTreeEvent) {
removeQueries(tree);
const anchors: { key: string; value: string }[] = [];
const anchors: ObjectValue<AnchorObject>[] = [];
collectNodesOfType(tree, "NamedAnchor").forEach((n) => {
const aName = n.children![0].text!.substring(1);
anchors.push({
key: `a:${pageName}:${aName}`,
value: "" + n.from,
ref: `${pageName}@${aName}`,
tags: ["anchor"],
name: aName,
page: pageName,
pos: n.from!,
});
});
// console.log("Found", anchors.length, "anchors(s)");
await index.batchSet(pageName, anchors);
await indexObjects(pageName, anchors);
}
export async function anchorComplete(completeEvent: CompleteEvent) {
@@ -31,13 +38,13 @@ export async function anchorComplete(completeEvent: CompleteEvent) {
if (!pageRef) {
pageRef = completeEvent.pageName;
}
const allAnchors = await index.queryPrefix(
`a:${pageRef}:${anchorRef}`,
);
const allAnchors = await queryObjects<AnchorObject>("anchor", {
filter: ["=", ["attr", "page"], ["string", pageRef]],
});
return {
from: completeEvent.pos - anchorRef.length,
options: allAnchors.map((a) => ({
label: a.key.split(":")[2],
label: a.name,
type: "anchor",
})),
};
plugs/index/api.ts
+156
View File
@@ -0,0 +1,156 @@
import { datastore } from "$sb/syscalls.ts";
import { KV, KvKey, ObjectQuery, ObjectValue } from "$sb/types.ts";
import { QueryProviderEvent } from "$sb/app_event.ts";
import { builtins } from "./builtins.ts";
import { AttributeObject, determineType } from "./attributes.ts";
const indexKey = "idx";
const pageKey = "ridx";
/*
* Key namespace:
* [indexKey, type, ...key, page] -> value
* [pageKey, page, ...key] -> true // for fast page clearing
* ["type", type] -> true // for fast type listing
*/
export function batchSet(page: string, kvs: KV[]): Promise<void> {
const finalBatch: KV[] = [];
for (const { key, value } of kvs) {
finalBatch.push({
key: [indexKey, ...key, page],
value,
}, {
key: [pageKey, page, ...key],
value: true,
});
}
return datastore.batchSet(finalBatch);
}
/**
* Clears all keys for a given page
* @param page
*/
export async function clearPageIndex(page: string): Promise<void> {
const allKeys: KvKey[] = [];
for (
const { key } of await datastore.query({
prefix: [pageKey, page],
})
) {
allKeys.push(key);
allKeys.push([indexKey, ...key.slice(2), page]);
}
await datastore.batchDel(allKeys);
}
/**
* Clears the entire datastore for this indexKey plug
*/
export async function clearIndex(): Promise<void> {
const allKeys: KvKey[] = [];
for (
const { key } of await datastore.query({ prefix: [] })
) {
allKeys.push(key);
}
await datastore.batchDel(allKeys);
console.log("Deleted", allKeys.length, "keys from the index");
}
// ENTITIES API
/**
* Indexes entities in the data store
*/
export async function indexObjects<T>(
page: string,
objects: ObjectValue<T>[],
): Promise<void> {
const kvs: KV<T>[] = [];
const allAttributes = new Map<string, string>(); // tag:name -> attributeType
for (const obj of objects) {
for (const tag of obj.tags) {
kvs.push({
key: [tag, cleanKey(obj.ref, page)],
value: obj,
});
// Index attributes
if (!builtins[tag]) {
// But only for non-builtin tags
for (
const [attrName, attrValue] of Object.entries(
obj as Record<string, any>,
)
) {
if (attrName.startsWith("$")) {
continue;
}
allAttributes.set(`${tag}:${attrName}`, determineType(attrValue));
}
}
}
}
if (allAttributes.size > 0) {
await indexObjects<AttributeObject>(
page,
[...allAttributes].map(([key, value]) => {
const [tag, name] = key.split(":");
return {
ref: key,
tags: ["attribute"],
tag,
name,
attributeType: value,
page,
};
}),
);
}
return batchSet(page, kvs);
}
function cleanKey(ref: string, page: string) {
if (ref.startsWith(`${page}@`)) {
return ref.substring(page.length + 1);
} else {
return ref;
}
}
export async function queryObjects<T>(
tag: string,
query: ObjectQuery,
): Promise<ObjectValue<T>[]> {
return (await datastore.query({
...query,
prefix: [indexKey, tag],
})).map(({ value }) => value);
}
export async function getObjectByRef<T>(
page: string,
tag: string,
ref: string,
): Promise<ObjectValue<T> | undefined> {
console.log("Fetching!!!!!", [indexKey, tag, cleanKey(ref, page), page]);
return (await datastore.get([indexKey, tag, cleanKey(ref, page), page]));
}
export async function objectSourceProvider({
query,
}: QueryProviderEvent): Promise<any[]> {
const tag = query.querySource!;
const results = await datastore.query({
...query,
prefix: [indexKey, tag],
});
return results.map((r) => r.value);
}
export async function discoverSources() {
return (await datastore.query({ prefix: [indexKey, "tag"] })).map((
{ key },
) => key[2]);
}
plugs/index/asset/linked_mentions.js
+17
View File
@@ -0,0 +1,17 @@
function processClick(e) {
const dataEl = e.target.closest("[data-ref]");
syscall(
"system.invokeFunction",
"index.navigateToMention",
dataEl.getAttribute("data-ref"),
).catch(console.error);
}
document.getElementById("link-ul").addEventListener("click", processClick);
document.getElementById("hide-button").addEventListener("click", function () {
console.log("HERE")
syscall(
"system.invokeFunction",
"index.toggleMentions",
).catch(console.error);
});
plugs/index/asset/style.css
+25
View File
@@ -0,0 +1,25 @@
body {
font-family: var(--ui-font);
background-color: var(--root-background-color);
color: var(--root-color);
overflow: scroll;
}
.sb-line-h2 {
border-top-right-radius: 5px;
border-top-left-radius: 5px;
margin: 0;
padding: 10px !important;
background-color: rgba(233, 233, 233, 0.5);
}
#hide-button {
position: absolute;
right: 15px;
top: 15px;
}
li code {
font-size: 80%;
color: #a5a4a4;
}
plugs/index/attributes.ts
+25 -98
View File
@@ -1,12 +1,15 @@
import { index } from "$sb/silverbullet-syscall/mod.ts";
import type { CompleteEvent } from "$sb/app_event.ts";
import { events } from "$sb/syscalls.ts";
import { queryObjects } from "./api.ts";
import { ObjectValue, QueryExpression } from "$sb/types.ts";
import { builtinPseudoPage } from "./builtins.ts";
export type AttributeContext = "page" | "item" | "task";
type AttributeEntry = {
type: string;
};
export type AttributeObject = ObjectValue<{
name: string;
attributeType: string;
tag: string;
page: string;
}>;
export type AttributeCompleteEvent = {
source: string;
@@ -16,41 +19,11 @@ export type AttributeCompleteEvent = {
export type AttributeCompletion = {
name: string;
source: string;
type: string;
attributeType: string;
builtin?: boolean;
};
const builtinAttributes: Record<string, Record<string, string>> = {
page: {
name: "string",
lastModified: "number",
perm: "rw|ro",
contentType: "string",
size: "number",
tags: "array",
},
task: {
name: "string",
done: "boolean",
page: "string",
state: "string",
deadline: "string",
pos: "number",
tags: "array",
},
item: {
name: "string",
page: "string",
pos: "number",
tags: "array",
},
tag: {
name: "string",
freq: "number",
},
};
function determineType(v: any): string {
export function determineType(v: any): string {
const t = typeof v;
if (t === "object") {
if (Array.isArray(v)) {
@@ -60,69 +33,23 @@ function determineType(v: any): string {
return t;
}
const attributeKeyPrefix = "attr:";
export async function indexAttributes(
pageName: string,
attributes: Record<string, any>,
context: AttributeContext,
) {
await index.batchSet(
pageName,
Object.entries(attributes).map(([k, v]) => {
return {
key: `${attributeKeyPrefix}${context}:${k}`,
value: {
type: determineType(v),
} as AttributeEntry,
};
}),
);
}
export async function customAttributeCompleter(
export async function objectAttributeCompleter(
attributeCompleteEvent: AttributeCompleteEvent,
): Promise<AttributeCompletion[]> {
const sourcePrefix = attributeCompleteEvent.source === "*"
? ""
: `${attributeCompleteEvent.source}:`;
const allAttributes = await index.queryPrefix(
`${attributeKeyPrefix}${sourcePrefix}`,
);
return allAttributes.map((attr) => {
const [_prefix, context, name] = attr.key.split(":");
return {
name,
source: context,
type: attr.value.type,
};
const attributeFilter: QueryExpression | undefined =
attributeCompleteEvent.source === ""
? undefined
: ["=", ["attr", "tag"], ["string", attributeCompleteEvent.source]];
const allAttributes = await queryObjects<AttributeObject>("attribute", {
filter: attributeFilter,
});
}
export function builtinAttributeCompleter(
attributeCompleteEvent: AttributeCompleteEvent,
): AttributeCompletion[] {
let allAttributes = builtinAttributes[attributeCompleteEvent.source];
if (attributeCompleteEvent.source === "*") {
allAttributes = {};
for (const [source, attributes] of Object.entries(builtinAttributes)) {
for (const [name, type] of Object.entries(attributes)) {
allAttributes[name] = `${type}|${source}`;
}
}
}
if (!allAttributes) {
return [];
}
return Object.entries(allAttributes).map(([name, type]) => {
return allAttributes.map((value) => {
return {
name,
source: attributeCompleteEvent.source === "*"
? type.split("|")[1]
: attributeCompleteEvent.source,
type: attributeCompleteEvent.source === "*" ? type.split("|")[0] : type,
builtin: true,
};
name: value.name,
source: value.tag,
attributeType: value.attributeType,
builtin: value.page === builtinPseudoPage,
} as AttributeCompletion;
});
}
@@ -184,7 +111,7 @@ export function attributeCompletionsToCMCompletion(
(completion) => ({
label: completion.name,
apply: `${completion.name}: `,
detail: `${completion.type} (${completion.source})`,
detail: `${completion.attributeType} (${completion.source})`,
type: "attribute",
}),
);
plugs/index/builtins.ts
+79
View File
@@ -0,0 +1,79 @@
import { ObjectValue } from "$sb/types.ts";
import { indexObjects } from "./api.ts";
import { AttributeObject } from "./attributes.ts";
import { TagObject } from "./tags.ts";
export const builtinPseudoPage = ":builtin:";
export const builtins: Record<string, Record<string, string>> = {
page: {
name: "string",
lastModified: "date",
perm: "rw|ro",
contentType: "string",
size: "number",
tags: "array",
},
task: {
name: "string",
done: "boolean",
page: "string",
state: "string",
deadline: "string",
pos: "number",
tags: "array",
},
tag: {
name: "string",
page: "string",
context: "string",
},
attribute: {
name: "string",
attributeType: "string",
type: "string",
page: "string",
},
anchor: {
name: "string",
page: "string",
pos: "number",
},
link: {
name: "string",
page: "string",
pos: "number",
alias: "string",
inDirective: "boolean",
asTemplate: "boolean",
},
};
export async function loadBuiltinsIntoIndex() {
console.log("Loading builtins attributes into index");
const allTags: ObjectValue<TagObject>[] = [];
for (const [tag, attributes] of Object.entries(builtins)) {
allTags.push({
ref: tag,
tags: ["tag"],
name: tag,
page: builtinPseudoPage,
parent: "builtin",
});
await indexObjects<AttributeObject>(
builtinPseudoPage,
Object.entries(attributes).map(([name, attributeType]) => {
return {
ref: `${tag}:${name}`,
tags: ["attribute"],
tag,
name,
attributeType,
builtinPseudoPage,
page: builtinPseudoPage,
};
}),
);
}
await indexObjects(builtinPseudoPage, allTags);
}
plugs/index/command.ts
+51
View File
@@ -0,0 +1,51 @@
import { editor, events, markdown, mq, space, system } from "$sb/syscalls.ts";
import { sleep } from "$sb/lib/async.ts";
import { IndexEvent } from "$sb/app_event.ts";
import { MQMessage } from "$sb/types.ts";
export async function reindexCommand() {
await editor.flashNotification("Performing full page reindex...");
await system.invokeFunction("reindexSpace");
await editor.flashNotification("Done with page index!");
}
export async function reindexSpace() {
console.log("Clearing page index...");
// Executed this way to not have to embed the search plug code here
await system.invokeFunction("search.clearIndex");
await system.invokeFunction("index.clearIndex");
const pages = await space.listPages();
// Queue all page names to be indexed
await mq.batchSend("indexQueue", pages.map((page) => page.name));
// Now let's wait for the processing to finish
let queueStats = await mq.getQueueStats("indexQueue");
while (queueStats.queued > 0 || queueStats.processing > 0) {
sleep(1000);
queueStats = await mq.getQueueStats("indexQueue");
}
// And notify the user
console.log("Indexing completed!");
}
export async function processIndexQueue(messages: MQMessage[]) {
for (const message of messages) {
const name: string = message.body;
console.log(`Indexing page ${name}`);
const text = await space.readPage(name);
const parsed = await markdown.parseMarkdown(text);
await events.dispatchEvent("page:index", {
name,
tree: parsed,
});
}
}
export async function parseIndexTextRepublish({ name, text }: IndexEvent) {
// console.log("Reindexing", name);
await events.dispatchEvent("page:index", {
name,
tree: await markdown.parseMarkdown(text),
});
}
plugs/directive/data.ts → plugs/index/data.ts
+33 -26
View File
@@ -1,13 +1,20 @@
// Index key space:
// data:page@pos
import type { IndexTreeEvent, QueryProviderEvent } from "$sb/app_event.ts";
import { index, YAML } from "$sb/syscalls.ts";
import type { IndexTreeEvent } from "$sb/app_event.ts";
import { YAML } from "$sb/syscalls.ts";
import { collectNodesOfType, findNodeOfType } from "$sb/lib/tree.ts";
import { applyQuery, removeQueries } from "$sb/lib/query.ts";
import { removeQueries } from "$sb/lib/query.ts";
import { ObjectValue } from "$sb/types.ts";
import { indexObjects } from "./api.ts";
import { TagObject } from "./tags.ts";
type DataObject = ObjectValue<
{
pos: number;
page: string;
} & Record<string, any>
>;
export async function indexData({ name, tree }: IndexTreeEvent) {
const dataObjects: { key: string; value: any }[] = [];
const dataObjects: ObjectValue<DataObject>[] = [];
removeQueries(tree);
@@ -17,7 +24,8 @@ export async function indexData({ name, tree }: IndexTreeEvent) {
if (!codeInfoNode) {
return;
}
if (codeInfoNode.children![0].text !== "data") {
const fenceType = codeInfoNode.children![0].text!;
if (fenceType !== "data" && !fenceType.startsWith("#")) {
return;
}
const codeTextNode = findNodeOfType(t, "CodeText");
@@ -26,6 +34,7 @@ export async function indexData({ name, tree }: IndexTreeEvent) {
return;
}
const codeText = codeTextNode.children![0].text!;
const dataType = fenceType === "data" ? "data" : fenceType.substring(1);
try {
const docs = codeText.split("---");
// We support multiple YAML documents in one block
@@ -34,12 +43,25 @@ export async function indexData({ name, tree }: IndexTreeEvent) {
if (!doc) {
continue;
}
const pos = t.from! + i;
dataObjects.push({
key: `data:${name}@${t.from! + i}`,
value: doc,
ref: `${name}@${pos}`,
tags: [dataType],
...doc,
pos,
page: name,
});
}
// console.log("Parsed data", parsedData);
await indexObjects<TagObject>(name, [
{
ref: dataType,
tags: ["tag"],
name: dataType,
page: name,
parent: "data",
},
]);
} catch (e) {
console.error("Could not parse data", codeText, "error:", e);
return;
@@ -47,20 +69,5 @@ export async function indexData({ name, tree }: IndexTreeEvent) {
}),
);
// console.log("Found", dataObjects.length, "data objects");
await index.batchSet(name, dataObjects);
}
export async function queryProvider({
query,
}: QueryProviderEvent): Promise<any[]> {
const allData: any[] = [];
for (const { key, page, value } of await index.queryPrefix("data:")) {
const [, pos] = key.split("@");
allData.push({
...value,
page: page,
pos: +pos,
});
}
return applyQuery(query, allData);
await indexObjects(name, dataObjects);
}
plugs/index/index.plug.yaml
+76 -36
View File
@@ -10,34 +10,67 @@ syntax:
- "$"
regex: "\\$[a-zA-Z\\.\\-\\/]+[\\w\\.\\-\\/]*"
className: sb-named-anchor
assets:
- asset/*
functions:
clearPageIndex:
path: "./page.ts:clearPageIndex"
loadBuiltinsIntoIndex:
path: builtins.ts:loadBuiltinsIntoIndex
env: server
events:
- system:ready
# Public API
batchSet:
path: api.ts:batchSet
env: server
indexObjects:
path: api.ts:indexObjects
env: server
queryObjects:
path: api.ts:queryObjects
env: server
getObjectByRef:
path: api.ts:getObjectByRef
env: server
objectSourceProvider:
path: api.ts:objectSourceProvider
events:
- query:*
discoverSources:
path: api.ts:discoverSources
events:
- query_
clearIndex:
path: api.ts:clearIndex
env: server
clearDSIndex:
path: api.ts:clearPageIndex
env: server
events:
- page:saved
- page:deleted
pageQueryProvider:
path: ./page.ts:pageQueryProvider
events:
- query:page
parseIndexTextRepublish:
path: "./page.ts:parseIndexTextRepublish"
path: "./command.ts:parseIndexTextRepublish"
env: server
events:
- page:index_text
reindexSpaceCommand:
path: "./page.ts:reindexCommand"
path: "./command.ts:reindexCommand"
command:
name: "Space: Reindex"
processIndexQueue:
path: ./page.ts:processIndexQueue
path: ./command.ts:processIndexQueue
mqSubscriptions:
- queue: indexQueue
batchSize: 10
autoAck: true
reindexSpace:
path: "./page.ts:reindexSpace"
path: "./command.ts:reindexSpace"
env: server
# Attachments
attachmentQueryProvider:
@@ -45,46 +78,37 @@ functions:
events:
- query:attachment
indexPage:
path: page.ts:indexPage
events:
- page:index
# Backlinks
indexLinks:
path: "./page_links.ts:indexLinks"
events:
- page:index
linkQueryProvider:
path: ./page_links.ts:linkQueryProvider
events:
- query:link
attributeComplete:
path: "./attributes.ts:attributeComplete"
events:
- editor:complete
customAttributeCompleter:
path: ./attributes.ts:customAttributeCompleter
objectAttributeCompleter:
path: ./attributes.ts:objectAttributeCompleter
events:
- attribute:complete:page
- attribute:complete:task
- attribute:complete:item
- attribute:complete:*
builtinAttributeCompleter:
path: ./attributes.ts:builtinAttributeCompleter
events:
- attribute:complete:page
- attribute:complete:task
- attribute:complete:item
- attribute:complete:*
# builtinAttributeCompleter:
# path: ./attributes.ts:builtinAttributeCompleter
# events:
# - attribute:complete:*
# Item indexing
indexItem:
path: "./item.ts:indexItems"
events:
- page:index
itemQueryProvider:
path: "./item.ts:queryProvider"
events:
- query:item
# Anchors
indexAnchors:
@@ -96,19 +120,21 @@ functions:
events:
- editor:complete
# Data
indexData:
path: data.ts:indexData
events:
- page:index
# Hashtags
indexTags:
path: "./tags.ts:indexTags"
path: tags.ts:indexTags
events:
- page:index
tagComplete:
path: "./tags.ts:tagComplete"
path: tags.ts:tagComplete
events:
- editor:complete
tagProvider:
path: "./tags.ts:tagProvider"
events:
- query:tag
renamePageCommand:
path: "./refactor.ts:renamePageCommand"
@@ -128,4 +154,18 @@ functions:
command:
name: "Page: Extract"
# Mentions panel (postscript)
toggleMentions:
path: "./mentions_ps.ts:toggleMentions"
command:
name: "Mentions: Toggle"
key: ctrl-alt-m
updateMentions:
path: "./mentions_ps.ts:updateMentions"
env: client
events:
- plug:load
- editor:pageLoaded
navigateToMention:
path: "./mentions_ps.ts:navigate"
plugs/index/item.ts
+28 -50
View File
@@ -1,31 +1,28 @@
import type { IndexTreeEvent, QueryProviderEvent } from "$sb/app_event.ts";
import type { IndexTreeEvent } from "$sb/app_event.ts";
import { index } from "$sb/syscalls.ts";
import { collectNodesOfType, ParseTree, renderToText } from "$sb/lib/tree.ts";
import { applyQuery, removeQueries } from "$sb/lib/query.ts";
import { removeQueries } from "$sb/lib/query.ts";
import { extractAttributes } from "$sb/lib/attribute.ts";
import { rewritePageRefs } from "$sb/lib/resolve.ts";
import { indexAttributes } from "./attributes.ts";
import { ObjectValue } from "$sb/types.ts";
import { indexObjects } from "./api.ts";
export type Item = {
name: string;
nested?: string;
tags?: string[];
// Not stored in DB
page?: string;
pos?: number;
} & Record<string, any>;
export type ItemObject = ObjectValue<
{
name: string;
page: string;
pos: number;
} & Record<string, any>
>;
export async function indexItems({ name, tree }: IndexTreeEvent) {
const items: { key: string; value: Item }[] = [];
const items: ObjectValue<ItemObject>[] = [];
removeQueries(tree);
// console.log("Indexing items", name);
const coll = collectNodesOfType(tree, "ListItem");
const allAttributes: Record<string, any> = {};
for (const n of coll) {
if (!n.children) {
continue;
@@ -35,16 +32,24 @@ export async function indexItems({ name, tree }: IndexTreeEvent) {
continue;
}
const item: Item = {
const item: ItemObject = {
ref: `${name}@${n.from}`,
tags: [],
name: "", // to be replaced
page: name,
pos: n.from!,
};
const textNodes: ParseTree[] = [];
let nested: string | undefined;
collectNodesOfType(n, "Hashtag").forEach((h) => {
// Push tag to the list, removing the initial #
item.tags.push(h.children![0].text!.substring(1));
});
for (const child of n.children!.slice(1)) {
rewritePageRefs(child, name);
if (child.type === "OrderedList" || child.type === "BulletList") {
nested = renderToText(child);
break;
}
// Extract attributes and remove from tree
@@ -52,44 +57,17 @@ export async function indexItems({ name, tree }: IndexTreeEvent) {
for (const [key, value] of Object.entries(extractedAttributes)) {
item[key] = value;
allAttributes[key] = value;
}
textNodes.push(child);
}
item.name = textNodes.map(renderToText).join("").trim();
if (nested) {
item.nested = nested;
}
collectNodesOfType(n, "Hashtag").forEach((h) => {
if (!item.tags) {
item.tags = [];
}
// Push tag to the list, removinn the initial #
item.tags.push(h.children![0].text!.substring(1));
});
items.push({
key: `it:${n.from}`,
value: item,
});
if (item.tags.length > 0) {
// Only index items with tags
items.push(item);
}
}
// console.log("Found", items, "item(s)");
await index.batchSet(name, items);
await indexAttributes(name, allAttributes, "item");
}
export async function queryProvider({
query,
}: QueryProviderEvent): Promise<any[]> {
const allItems: Item[] = [];
for (const { key, page, value } of await index.queryPrefix("it:")) {
const [, pos] = key.split(":");
allItems.push({
...value,
page: page,
pos: +pos,
});
}
return applyQuery(query, allItems);
await indexObjects(name, items);
}
plugs/index/mentions_ps.ts
+79
View File
@@ -0,0 +1,79 @@
import { asset } from "$sb/plugos-syscall/mod.ts";
import { clientStore, editor } from "$sb/silverbullet-syscall/mod.ts";
import { queryObjects } from "./api.ts";
import { LinkObject } from "./page_links.ts";
const hideMentionsKey = "hideMentions";
export async function toggleMentions() {
let hideMentions = await clientStore.get(hideMentionsKey);
hideMentions = !hideMentions;
await clientStore.set(hideMentionsKey, hideMentions);
if (!hideMentions) {
const name = await editor.getCurrentPage();
await renderMentions(name);
} else {
await editor.hidePanel("ps");
}
}
// Triggered when switching pages or upon first load
export async function updateMentions() {
if (await clientStore.get(hideMentionsKey)) {
return;
}
const name = await editor.getCurrentPage();
await renderMentions(name);
}
// use internal navigation via syscall to prevent reloading the full page.
export async function navigate(ref: string) {
const [page, pos] = ref.split("@");
await editor.navigate(page, +pos);
}
function escapeHtml(unsafe: string) {
return unsafe.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(
/>/g,
"&gt;",
);
}
async function renderMentions(page: string) {
const linksResult = await queryObjects<LinkObject>("link", {
// Query all links that point to this page, excluding those that are inside directives and self pointers.
filter: ["and", ["!=", ["attr", "page"], ["string", page]], ["and", ["=", [
"attr",
"toPage",
], ["string", page]], ["=", ["attr", "inDirective"], ["boolean", false]]]],
});
if (linksResult.length === 0) {
// Don't show the panel if there are no links here.
await editor.hidePanel("ps");
} else {
const css = await asset.readAsset("asset/style.css");
const js = await asset.readAsset("asset/linked_mentions.js");
await editor.showPanel(
"ps",
1,
` <style>${css}</style>
<link rel="stylesheet" href="/.client/main.css" />
<div id="sb-main"><div id="sb-editor"><div class="cm-editor">
<button id="hide-button">Hide</button>
<div class="cm-line sb-line-h2">Linked Mentions</div>
<ul id="link-ul">
${
linksResult.map((link) =>
`<li data-ref="${link.ref}"><span class="sb-wiki-link-page">${link.ref}</span>: <code>...${
escapeHtml(link.snippet)
}...</code></li>`
).join("")
}
</ul>
</div></div></div>
`,
js,
);
}
}
plugs/index/page.ts
+28 -66
View File
@@ -1,75 +1,37 @@
import type { IndexEvent, QueryProviderEvent } from "$sb/app_event.ts";
import {
editor,
events,
index,
markdown,
mq,
space,
system,
} from "$sb/syscalls.ts";
import type { IndexTreeEvent } from "$sb/app_event.ts";
import { space } from "$sb/syscalls.ts";
import { applyQuery } from "$sb/lib/query.ts";
import type { MQMessage } from "$sb/types.ts";
import { sleep } from "$sb/lib/async.ts";
import type { ObjectValue, PageMeta } from "$sb/types.ts";
import { extractFrontmatter } from "$sb/lib/frontmatter.ts";
import { extractAttributes } from "$sb/lib/attribute.ts";
import { indexObjects } from "./api.ts";
// Key space:
// meta: => metaJson
type PageObject = ObjectValue<
// The base is PageMeta, but we override lastModified to be a string
Omit<PageMeta, "lastModified"> & {
lastModified: string; // indexing it as a string
} & Record<string, any>
>;
export async function pageQueryProvider({
query,
}: QueryProviderEvent): Promise<any[]> {
return applyQuery(query, await space.listPages());
}
export async function indexPage({ name, tree }: IndexTreeEvent) {
const pageMeta = await space.getPageMeta(name);
let pageObj: PageObject = {
ref: name,
tags: [], // will be overridden in a bit
...pageMeta,
lastModified: new Date(pageMeta.lastModified).toISOString(),
};
export async function reindexCommand() {
await editor.flashNotification("Performing full page reindex...");
await reindexSpace();
await editor.flashNotification("Done with page index!");
}
const frontmatter: Record<string, any> = await extractFrontmatter(tree);
const toplevelAttributes = await extractAttributes(tree, false);
export async function reindexSpace() {
console.log("Clearing page index...");
await index.clearPageIndex();
// Executed this way to not have to embed the search plug code here
await system.invokeFunction("search.clearIndex");
const pages = await space.listPages();
// Push them all into the page object
pageObj = { ...pageObj, ...frontmatter, ...toplevelAttributes };
// Queue all page names to be indexed
await mq.batchSend("indexQueue", pages.map((page) => page.name));
pageObj.tags = ["page", ...pageObj.tags || []];
// Now let's wait for the processing to finish
let queueStats = await mq.getQueueStats("indexQueue");
while (queueStats.queued > 0 || queueStats.processing > 0) {
sleep(1000);
queueStats = await mq.getQueueStats("indexQueue");
}
// And notify the user
console.log("Indexing completed!");
}
// console.log("Page object", pageObj);
export async function processIndexQueue(messages: MQMessage[]) {
for (const message of messages) {
const name: string = message.body;
console.log(`Indexing page ${name}`);
const text = await space.readPage(name);
const parsed = await markdown.parseMarkdown(text);
await events.dispatchEvent("page:index", {
name,
tree: parsed,
});
}
}
export async function clearPageIndex(page: string) {
// console.log("Clearing page index for page", page);
await index.clearPageIndexForPage(page);
}
export async function parseIndexTextRepublish({ name, text }: IndexEvent) {
// console.log("Reindexing", name);
await events.dispatchEvent("page:index", {
name,
tree: await markdown.parseMarkdown(text),
});
// console.log("Extracted page meta data", pageMeta);
await indexObjects<PageObject>(name, [pageObj]);
}
plugs/index/page_links.ts
+83 -93
View File
@@ -1,48 +1,53 @@
import { index } from "$sb/syscalls.ts";
import { findNodeOfType, traverseTree } from "$sb/lib/tree.ts";
import { extractFrontmatter } from "$sb/lib/frontmatter.ts";
import { extractAttributes } from "$sb/lib/attribute.ts";
import { IndexTreeEvent, QueryProviderEvent } from "$sb/app_event.ts";
import { applyQuery } from "$sb/lib/query.ts";
import { findNodeOfType, renderToText, traverseTree } from "$sb/lib/tree.ts";
import { IndexTreeEvent } from "$sb/app_event.ts";
import { resolvePath } from "$sb/lib/resolve.ts";
import { indexAttributes } from "./attributes.ts";
import { indexObjects, queryObjects } from "./api.ts";
import { ObjectValue } from "$sb/types.ts";
// Key space:
// l:toPage:pos => {name: pageName, inDirective: true, asTemplate: true}
export const backlinkPrefix = `l:`;
export type BacklinkEntry = {
name: string;
export type LinkObject = {
ref: string;
tags: string[];
// The page the link points to
toPage: string;
// The page the link occurs in
page: string;
pos: number;
snippet: string;
alias?: string;
inDirective?: boolean;
asTemplate?: boolean;
inDirective: boolean;
asTemplate: boolean;
};
export function extractSnippet(text: string, pos: number): string {
let prefix = "";
for (let i = pos - 1; i > 0; i--) {
if (text[i] === "\n") {
break;
}
prefix = text[i] + prefix;
if (prefix.length > 25) {
break;
}
}
let suffix = "";
for (let i = pos; i < text.length; i++) {
if (text[i] === "\n") {
break;
}
suffix += text[i];
if (suffix.length > 25) {
break;
}
}
return prefix + suffix;
}
export async function indexLinks({ name, tree }: IndexTreeEvent) {
const backLinks: { key: string; value: BacklinkEntry }[] = [];
const links: ObjectValue<LinkObject>[] = [];
// [[Style Links]]
// console.log("Now indexing links for", name);
const pageMeta = await extractFrontmatter(tree);
const toplevelAttributes = await extractAttributes(tree, false);
if (
Object.keys(pageMeta).length > 0 ||
Object.keys(toplevelAttributes).length > 0
) {
for (const [k, v] of Object.entries(toplevelAttributes)) {
pageMeta[k] = v;
}
// Don't index meta data starting with $
for (const key in pageMeta) {
if (key.startsWith("$")) {
delete pageMeta[key];
}
}
// console.log("Extracted page meta data", pageMeta);
await index.set(name, "meta:", pageMeta);
}
await indexAttributes(name, pageMeta, "page");
const pageText = renderToText(tree);
let directiveDepth = 0;
traverseTree(tree, (n): boolean => {
@@ -54,9 +59,16 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) {
name,
pageRef.children![0].text!.slice(2, -2),
);
backLinks.push({
key: `${backlinkPrefix}${pageRefName}:${pageRef.from! + 2}`,
value: { name, asTemplate: true },
const pos = pageRef.from! + 2;
links.push({
ref: `${name}@${pos}`,
tags: ["link"],
toPage: pageRefName,
pos: pos,
snippet: extractSnippet(pageText, pos),
page: name,
asTemplate: true,
inDirective: false,
});
}
const directiveText = n.children![0].text;
@@ -65,18 +77,23 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) {
const match = /\[\[(.+)\]\]/.exec(directiveText);
if (match) {
const pageRefName = resolvePath(name, match[1]);
backLinks.push({
key: `${backlinkPrefix}${pageRefName}:${
n.from! + match.index! + 2
}`,
value: { name, asTemplate: true },
const pos = n.from! + match.index! + 2;
links.push({
ref: `${name}@${pos}`,
tags: ["link"],
toPage: pageRefName,
page: name,
snippet: extractSnippet(pageText, pos),
pos: pos,
asTemplate: true,
inDirective: false,
});
}
}
return true;
}
if (n.type === "DirectiveStop") {
if (n.type === "DirectiveEnd") {
directiveDepth--;
return true;
}
@@ -85,66 +102,39 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) {
const wikiLinkPage = findNodeOfType(n, "WikiLinkPage")!;
const wikiLinkAlias = findNodeOfType(n, "WikiLinkAlias");
let toPage = resolvePath(name, wikiLinkPage.children![0].text!);
const pos = wikiLinkPage.from!;
if (toPage.includes("@")) {
toPage = toPage.split("@")[0];
}
const blEntry: BacklinkEntry = { name };
const link: LinkObject = {
ref: `${name}@${pos}`,
tags: ["link"],
toPage: toPage,
snippet: extractSnippet(pageText, pos),
pos,
page: name,
inDirective: false,
asTemplate: false,
};
if (directiveDepth > 0) {
blEntry.inDirective = true;
link.inDirective = true;
}
if (wikiLinkAlias) {
blEntry.alias = wikiLinkAlias.children![0].text!;
link.alias = wikiLinkAlias.children![0].text!;
}
backLinks.push({
key: `${backlinkPrefix}${toPage}:${wikiLinkPage.from}`,
value: blEntry,
});
links.push(link);
return true;
}
return false;
});
// console.log("Found", backLinks, "page link(s)");
await index.batchSet(name, backLinks);
await indexObjects(name, links);
}
export async function linkQueryProvider({
query,
pageName,
}: QueryProviderEvent): Promise<any[]> {
const links: any[] = [];
for (
const { value: blEntry, key } of await index.queryPrefix(
`${backlinkPrefix}${pageName}:`,
)
) {
const [, , pos] = key.split(":"); // Key: l:page:pos
if (!blEntry.inDirective) {
blEntry.inDirective = false;
}
if (!blEntry.asTemplate) {
blEntry.asTemplate = false;
}
links.push({ ...blEntry, pos });
}
return applyQuery(query, links);
}
type BackLinkPage = {
page: string;
pos: number;
};
export async function getBackLinks(pageName: string): Promise<BackLinkPage[]> {
const allBackLinks = await index.queryPrefix(
`${backlinkPrefix}${pageName}:`,
);
const pagesToUpdate: BackLinkPage[] = [];
for (const { key, value: { name } } of allBackLinks) {
const keyParts = key.split(":");
pagesToUpdate.push({
page: name,
pos: +keyParts[keyParts.length - 1],
});
}
return pagesToUpdate;
export async function getBackLinks(
pageName: string,
): Promise<LinkObject[]> {
return (await queryObjects<LinkObject>("link", {
filter: ["=", ["attr", "toPage"], ["string", pageName]],
}));
}
plugs/index/plug_api.ts
+24
View File
@@ -0,0 +1,24 @@
import { ObjectQuery, ObjectValue } from "$sb/types.ts";
import { invokeFunction } from "$sb/silverbullet-syscall/system.ts";
export function indexObjects<T>(
page: string,
objects: ObjectValue<T>[],
): Promise<void> {
return invokeFunction("index.indexObjects", page, objects);
}
export function queryObjects<T>(
tag: string,
query: ObjectQuery,
): Promise<ObjectValue<T>[]> {
return invokeFunction("index.queryObjects", tag, query);
}
export function getObjectByRef<T>(
page: string,
tag: string,
ref: string,
): Promise<ObjectValue<T>[]> {
return invokeFunction("index.getObjectByRef", page, tag, ref);
}
plugs/index/snippet.test.ts
+17
View File
@@ -0,0 +1,17 @@
import { assertEquals } from "../../test_deps.ts";
import { extractSnippet } from "./page_links.ts";
Deno.test("Snippet extraction", () => {
const sample1 = `This is a test
and a [[new]] line that runs super duper duper duper duper duper long
[[SETTINGS]]
super`;
assertEquals(
extractSnippet(sample1, sample1.indexOf("[[new]]")),
"and a [[new]] line that runs sup",
);
assertEquals(
extractSnippet(sample1, sample1.indexOf("[[SETTINGS]]")),
"[[SETTINGS]]",
);
});
plugs/index/tags.ts
+54 -43
View File
@@ -1,67 +1,78 @@
import { collectNodesOfType } from "$sb/lib/tree.ts";
import { index } from "$sb/syscalls.ts";
import type {
CompleteEvent,
IndexTreeEvent,
QueryProviderEvent,
} from "$sb/app_event.ts";
import { applyQuery, removeQueries } from "$sb/lib/query.ts";
import type { CompleteEvent, IndexTreeEvent } from "$sb/app_event.ts";
import { removeQueries } from "$sb/lib/query.ts";
import { extractFrontmatter } from "$sb/lib/frontmatter.ts";
import { indexObjects, queryObjects } from "./api.ts";
import {
addParentPointers,
collectNodesOfType,
findParentMatching,
} from "$sb/lib/tree.ts";
// Key space
// tag:TAG => true (for completion)
export type TagObject = {
ref: string;
tags: string[];
name: string;
page: string;
parent: string;
};
export async function indexTags({ name, tree }: IndexTreeEvent) {
removeQueries(tree);
const allTags = new Set<string>();
const { tags } = await extractFrontmatter(tree);
if (Array.isArray(tags)) {
tags.forEach((t) => allTags.add(t));
const tags = new Set<string>(); // name:parent
addParentPointers(tree);
const pageTags: string[] = (await extractFrontmatter(tree)).tags || [];
for (const pageTag of pageTags) {
tags.add(`${pageTag}:page`);
}
collectNodesOfType(tree, "Hashtag").forEach((n) => {
allTags.add(n.children![0].text!.substring(1));
collectNodesOfType(tree, "Hashtag").forEach((h) => {
const tagName = h.children![0].text!.substring(1);
// Check if this occurs in the context of a task
if (findParentMatching(h, (n) => n.type === "Task")) {
tags.add(`${tagName}:task`);
} else if (findParentMatching(h, (n) => n.type === "ListItem")) {
// Or an item
tags.add(`${tagName}:item`);
}
});
await index.batchSet(
// console.log("Indexing these tags", tags);
await indexObjects<TagObject>(
name,
[...allTags].map((t) => ({ key: `tag:${t}`, value: t })),
[...tags].map((tag) => {
const [tagName, parent] = tag.split(":");
return {
ref: tagName,
tags: ["tag"],
name: tagName,
page: name,
parent,
};
}),
);
}
const taskPrefixRegex = /^\s*[\-\*]\s+\[([^\]]+)\]/;
const itemPrefixRegex = /^\s*[\-\*]\s+/;
export async function tagComplete(completeEvent: CompleteEvent) {
const match = /#[^#\s]+$/.exec(completeEvent.linePrefix);
if (!match) {
return null;
}
const tagPrefix = match[0].substring(1);
const allTags = await index.queryPrefix(`tag:${tagPrefix}`);
let parent = "page";
if (taskPrefixRegex.test(completeEvent.linePrefix)) {
parent = "task";
} else if (itemPrefixRegex.test(completeEvent.linePrefix)) {
parent = "item";
}
const allTags = await queryObjects<TagObject>("tag", {
filter: ["=", ["attr", "parent"], ["string", parent]],
});
return {
from: completeEvent.pos - tagPrefix.length,
options: allTags.map((tag) => ({
label: tag.value,
label: tag.name,
type: "tag",
})),
};
}
type Tag = {
name: string;
freq: number;
};
export async function tagProvider({ query }: QueryProviderEvent) {
const allTags = new Map<string, number>();
for (const { value } of await index.queryPrefix("tag:")) {
let currentFreq = allTags.get(value);
if (!currentFreq) {
currentFreq = 0;
}
allTags.set(value, currentFreq + 1);
}
return applyQuery(
query,
[...allTags.entries()].map(([name, freq]) => ({
name,
freq,
})),
);
}
plugs/markdown/markdown_render.test.ts
+2 -2
View File
@@ -22,7 +22,7 @@ Deno.test("Markdown render", async () => {
new URL("test/example.md", import.meta.url).pathname,
);
const tree = parse(lang, testFile);
await renderMarkdownToHtml(tree, {
renderMarkdownToHtml(tree, {
failOnUnknown: true,
});
// console.log("HTML", html);
@@ -34,7 +34,7 @@ Deno.test("Smart hard break test", async () => {
*world!*`;
const lang = buildMarkdown([]);
const tree = parse(lang, example);
const html = await renderMarkdownToHtml(tree, {
const html = renderMarkdownToHtml(tree, {
failOnUnknown: true,
smartHardBreak: true,
});
plugs/markdown/markdown_render.ts
+23 -4
View File
@@ -1,4 +1,5 @@
import {
collectNodesOfType,
findNodeOfType,
ParseTree,
renderToText,
@@ -115,9 +116,13 @@ function render(
case "FencedCode":
case "CodeBlock": {
// Clear out top-level indent blocks
const lang = findNodeOfType(t, "CodeInfo");
t.children = t.children!.filter((c) => c.type);
return {
name: "pre",
attrs: {
"data-lang": lang ? lang.children![0].text : undefined,
},
body: cleanTags(mapRender(t.children!)),
};
}
@@ -235,6 +240,7 @@ function render(
name: "a",
attrs: {
href: `/${ref.replace("@", "#")}`,
"data-ref": ref,
},
body: linkText,
};
@@ -255,12 +261,25 @@ function render(
body: t.children![0].text!,
};
case "Task":
case "Task": {
let externalTaskRef = "";
collectNodesOfType(t, "WikiLinkPage").forEach((wikilink) => {
const ref = wikilink.children![0].text!;
if (!externalTaskRef && ref.includes("@")) {
externalTaskRef = ref;
}
});
return {
name: "span",
attrs: externalTaskRef
? {
"data-external-task-ref": externalTaskRef,
}
: {},
body: cleanTags(mapRender(t.children!)),
};
}
case "TaskState": {
// child[0] = marker, child[1] = state, child[2] = marker
const stateText = t.children![1].text!;
@@ -269,8 +288,8 @@ function render(
name: "input",
attrs: {
type: "checkbox",
checked: t.children![0].text !== "[ ]" ? "checked" : undefined,
"data-onclick": JSON.stringify(["task", t.to]),
checked: stateText !== " " ? "checked" : undefined,
"data-state": stateText,
},
body: "",
};
plugs/markdown/widget.ts
+1 -1
View File
@@ -12,7 +12,7 @@ export async function markdownWidget(
});
return Promise.resolve({
html: html,
script: `updateHeight();
script: `
document.addEventListener("click", () => {
api({type: "blur"});
});`,
plugs/query/assets/common.js
+102
View File
@@ -0,0 +1,102 @@
async function init() {
// Make edit button send the "blur" API call so that the MD code is visible
document.getElementById("edit-button").addEventListener("click", () => {
api({ type: "blur" });
});
document.getElementById("reload-button").addEventListener("click", () => {
api({ type: "reload" });
});
document.querySelectorAll("a[data-ref]").forEach((el) => {
el.addEventListener("click", (e) => {
e.preventDefault();
syscall("editor.navigate", el.dataset.ref);
});
});
// Find all fenced code blocks and replace them with iframes (if a code widget is defined for them)
const allWidgets = document.querySelectorAll("pre[data-lang]");
for (const widget of allWidgets) {
const lang = widget.getAttribute("data-lang");
const body = widget.innerText;
try {
const result = await syscall("widget.render", lang, body);
const iframe = document.createElement("iframe");
iframe.srcdoc = panelHtml; // set as a global
iframe.onload = () => {
iframe.contentWindow.postMessage({
type: "html",
theme: document.getElementsByTagName("html")[0].getAttribute(
"data-theme",
),
...result,
}, "*");
};
widget.parentNode.replaceChild(iframe, widget);
globalThis.addEventListener("message", (e) => {
if (e.source !== iframe.contentWindow) {
return;
}
const messageData = e.data;
switch (messageData.type) {
case "setHeight":
iframe.style.height = messageData.height + "px";
// Propagate height setting to parent
updateHeight();
break;
case "syscall": {
// Intercept syscall messages and send them to the parent
const { id, name, args } = messageData;
syscall(name, ...args).then((result) => {
iframe.contentWindow.postMessage(
{ id, type: "syscall-response", result },
"*",
);
}).catch((error) => {
iframe.contentWindow.postMessage({
id,
type: "syscall-response",
error,
}, "*");
});
break;
}
default:
// Bubble up any other messages to parent iframe
window.parent.postMessage(messageData, "*");
}
});
} catch (e) {
if (e.message.includes("not found")) {
// Not a code widget, ignore
} else {
console.error("Error rendering widget", e);
}
}
}
// Find all task toggles and propagate their state
document.querySelectorAll("span[data-external-task-ref]").forEach((el) => {
const taskRef = el.dataset.externalTaskRef;
el.querySelector("input[type=checkbox]").addEventListener("change", (e) => {
const oldState = e.target.dataset.state;
const newState = oldState === " " ? "x" : " ";
// Update state in DOM as well for future toggles
e.target.dataset.state = newState;
console.log("Toggling task", taskRef);
syscall(
"system.invokeFunction",
"tasks.updateTaskState",
taskRef,
oldState,
newState,
).catch(
console.error,
);
});
});
}
init().catch(console.error);
plugs/query/assets/style.css
+45
View File
@@ -0,0 +1,45 @@
body {
font-family: var(--editor-font);
background-color: var(--root-background-color);
color: var(--root-color);
overflow: scroll;
}
ul li p {
margin: 0;
}
body:hover #button-bar,
body:active #button-bar {
display: block;
}
#button-bar {
position: absolute;
right: 12px;
top: 3px;
display: none;
background: rgb(255 255 255 / 0.9);
padding: 0 3px;
}
#button-bar button {
border: none;
background: none;
cursor: pointer;
color: var(--root-color);
}
#edit-button {
margin-left: -10px;
}
li code {
font-size: 80%;
color: #a5a4a4;
}
iframe {
border: none;
width: 100%;
}
plugs/query/complete.ts
+80
View File
@@ -0,0 +1,80 @@
import { CompleteEvent } from "$sb/app_event.ts";
import { events } from "$sb/syscalls.ts";
import {
AttributeCompleteEvent,
AttributeCompletion,
} from "../index/attributes.ts";
export async function queryComplete(completeEvent: CompleteEvent) {
const fencedParent = completeEvent.parentNodes.find((node) =>
node === "FencedCode:query"
);
if (!fencedParent) {
return null;
}
let querySourceMatch = /^\s*([\w\-_]*)$/.exec(
completeEvent.linePrefix,
);
if (querySourceMatch) {
const allEvents = await events.listEvents();
const completionOptions = allEvents
.filter((eventName) =>
eventName.startsWith("query:") && !eventName.includes("*")
)
.map((source) => ({
label: source.substring("query:".length),
}));
const allObjectTypes: string[] = (await events.dispatchEvent("query_", {}))
.flat();
for (const type of allObjectTypes) {
completionOptions.push({
label: type,
});
}
return {
from: completeEvent.pos - querySourceMatch[1].length,
options: completionOptions,
};
}
querySourceMatch = /^\s*([\w\-_]*)/.exec(
completeEvent.linePrefix,
);
const whereMatch =
/(where|order\s+by|and|or|select(\s+[\w\s,]+)?)\s+([\w\-_]*)$/
.exec(
completeEvent.linePrefix,
);
if (querySourceMatch && whereMatch) {
const type = querySourceMatch[1];
const attributePrefix = whereMatch[3];
const completions = (await events.dispatchEvent(
`attribute:complete:${type}`,
{
source: type,
prefix: attributePrefix,
} as AttributeCompleteEvent,
)).flat() as AttributeCompletion[];
return {
from: completeEvent.pos - attributePrefix.length,
options: attributeCompletionsToCMCompletion(completions),
};
}
return null;
}
function attributeCompletionsToCMCompletion(
completions: AttributeCompletion[],
) {
return completions.map(
(completion) => ({
label: completion.name,
detail: `${completion.attributeType} (${completion.source})`,
type: "attribute",
}),
);
}
plugs/query/query.plug.yaml
+45
View File
@@ -0,0 +1,45 @@
name: query
assets:
- "assets/*"
functions:
queryWidget:
path: query.ts:widget
codeWidget: query
templateWidget:
path: template.ts:widget
codeWidget: template
queryComplete:
path: complete.ts:queryComplete
events:
- editor:complete
# Slash commands
insertQuery:
redirect: template.insertTemplateText
slashCommand:
name: query
description: Insert a query
value: |
```query
|^|
```
insertInclude:
redirect: template.insertTemplateText
slashCommand:
name: include
description: Include another page
value: |
<!-- #include [[|^|]] -->
<!-- /include -->
insertUseTemplate:
redirect: template.insertTemplateText
slashCommand:
name: template
description: Use a template
value: |
```template
page: "[[|^|]]"
```
plugs/query/query.ts
+79
View File
@@ -0,0 +1,79 @@
import type { WidgetContent } from "$sb/app_event.ts";
import { editor, events, language, markdown, space } from "$sb/syscalls.ts";
import { parseTreeToAST } from "$sb/lib/tree.ts";
import { astToKvQuery } from "$sb/lib/parse-query.ts";
import { jsonToMDTable, renderTemplate } from "../directive/util.ts";
import { renderMarkdownToHtml } from "../markdown/markdown_render.ts";
import { replaceTemplateVars } from "../template/template.ts";
import { prepareJS, wrapHTML } from "./util.ts";
export async function widget(bodyText: string): Promise<WidgetContent> {
const pageMeta = await space.getPageMeta(await editor.getCurrentPage());
try {
const queryAST = parseTreeToAST(
await language.parseLanguage("query", bodyText),
);
const parsedQuery = astToKvQuery(
JSON.parse(
await replaceTemplateVars(JSON.stringify(queryAST[1]), pageMeta),
),
);
// console.log("actual query", parsedQuery);
const eventName = `query:${parsedQuery.querySource}`;
let resultMarkdown = "";
// console.log("Parsed query", parsedQuery);
// Let's dispatch an event and see what happens
const results = await events.dispatchEvent(
eventName,
{ query: parsedQuery, pageName: pageMeta.name },
30 * 1000,
);
if (results.length === 0) {
// This means there was no handler for the event which means it's unsupported
return {
html:
`**Error:** Unsupported query source '${parsedQuery.querySource}'`,
};
} else {
const allResults = results.flat();
if (allResults.length === 0) {
resultMarkdown = "No results";
} else {
if (parsedQuery.render) {
// Configured a custom rendering template, let's use it!
const rendered = await renderTemplate(
pageMeta,
parsedQuery.render,
allResults,
);
resultMarkdown = rendered.trim();
} else {
// TODO: At this point it's a bit pointless to first render a markdown table, and then convert that to HTML
// We should just render the HTML table directly
resultMarkdown = jsonToMDTable(allResults);
}
}
}
// Parse markdown to a ParseTree
const mdTree = await markdown.parseMarkdown(resultMarkdown);
// And then render it to HTML
const html = renderMarkdownToHtml(mdTree, { smartHardBreak: true });
return {
html: await wrapHTML(`
${parsedQuery.render ? "" : `<div class="sb-table-widget">`}
${html}
${parsedQuery.render ? "" : `</div>`}
`),
script: await prepareJS(),
};
} catch (e: any) {
return {
html: await wrapHTML(`<b>Error:</b> ${e.message}`),
};
}
}
plugs/query/template.ts
+58
View File
@@ -0,0 +1,58 @@
import { WidgetContent } from "$sb/app_event.ts";
import { editor, handlebars, markdown, space, YAML } from "$sb/syscalls.ts";
import { renderMarkdownToHtml } from "../markdown/markdown_render.ts";
import { prepareJS, wrapHTML } from "./util.ts";
type TemplateConfig = {
// Pull the template from a page
page?: string;
// Or use a string directly
template?: string;
// Optional argument to pass
value?: any;
// If true, don't render the template, just use it as-is
raw?: boolean;
};
export async function widget(bodyText: string): Promise<WidgetContent> {
const pageMeta = await space.getPageMeta(await editor.getCurrentPage());
try {
const config: TemplateConfig = await YAML.parse(bodyText);
let templateText = config.template || "";
if (config.page) {
let page = config.page;
if (!page) {
throw new Error("Missing `page`");
}
if (page.startsWith("[[")) {
page = page.slice(2, -2);
}
templateText = await space.readPage(page);
}
const rendered = config.raw
? templateText
: await handlebars.renderTemplate(
templateText,
config.value,
{
page: pageMeta,
},
);
const parsedMarkdown = await markdown.parseMarkdown(rendered);
const html = renderMarkdownToHtml(parsedMarkdown, {
smartHardBreak: true,
});
return {
html: await wrapHTML(html),
script: await prepareJS(),
};
} catch (e: any) {
return {
html: `<b>Error:</b> ${e.message}`,
};
}
}
plugs/query/util.ts
+31
View File
@@ -0,0 +1,31 @@
import { asset } from "$sb/syscalls.ts";
import { panelHtml } from "../../web/components/panel_html.ts";
export async function prepareJS() {
const iframeJS = await asset.readAsset("assets/common.js");
return `
const panelHtml = \`${panelHtml}\`;
${iframeJS}
`;
}
export async function wrapHTML(html: string): Promise<string> {
const css = await asset.readAsset("assets/style.css");
return `
<!-- Load SB's own CSS here too -->
<link rel="stylesheet" href="/.client/main.css" />
<!-- In addition to some custom CSS -->
<style>${css}</style>
<!-- Wrap the whole thing in something SB-like to get access to styles -->
<div id="sb-main"><div id="sb-editor"><div class="cm-editor">
<!-- And add an edit button -->
<div id="button-bar">
<button id="reload-button" title="Reload"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="23 4 23 10 17 10"></polyline><polyline points="1 20 1 14 7 14"></polyline><path d="M3.51 9a9 9 0 0 1 14.85-3.36L23 10M1 14l4.64 4.36A9 9 0 0 0 20.49 15"></path></svg></button>
<button id="edit-button" title="Edit"><svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-edit"><path d="M11 4H4a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h14a2 2 0 0 0 2-2v-7"></path><path d="M18.5 2.5a2.121 2.121 0 0 1 3 3L12 15l-4 1 1-4 9.5-9.5z"></path></svg></button>
</div>
${html}
</div></div></div>
`;
}
plugs/search/engine.test.ts
+16 -18
View File
@@ -1,44 +1,42 @@
import { KV, KvKey } from "$sb/types.ts";
import { assertEquals } from "../../test_deps.ts";
import { BatchKVStore, SimpleSearchEngine } from "./engine.ts";
class InMemoryBatchKVStore implements BatchKVStore {
private store = new Map<string, any>();
get(keys: string[]): Promise<(any | undefined)[]> {
const results: (any | undefined)[] = keys.map((key) => this.store.get(key));
return Promise.resolve(results);
}
queryPrefix(prefix: string): Promise<[string, any][]> {
const results: [string, any][] = [];
query({ prefix }: { prefix: KvKey }): Promise<KV[]> {
const results: KV[] = [];
entries:
for (const [key, value] of this.store.entries()) {
if (key.startsWith(prefix)) {
results.push([key, value]);
const parsedKey: string[] = JSON.parse(key);
for (let i = 0; i < prefix.length; i++) {
if (prefix[i] !== parsedKey[i]) {
continue entries;
}
}
results.push({ key: parsedKey, value });
}
return Promise.resolve(results);
}
set(entries: Map<string, any>): Promise<void> {
for (const [key, value] of entries) {
this.store.set(key, value);
batchSet(kvs: KV[]): Promise<void> {
for (const { key, value } of kvs) {
this.store.set(JSON.stringify(key), value);
}
return Promise.resolve();
}
delete(keys: string[]): Promise<void> {
batchDel(keys: KvKey[]): Promise<void> {
for (const key of keys) {
this.store.delete(key);
this.store.delete(JSON.stringify(key));
}
return Promise.resolve();
}
}
Deno.test("Test full text search", async () => {
const engine = new SimpleSearchEngine(
new InMemoryBatchKVStore(),
new InMemoryBatchKVStore(),
);
const engine = new SimpleSearchEngine(new InMemoryBatchKVStore());
await engine.indexDocument({ id: "1", text: "The quick brown fox" });
await engine.indexDocument({ id: "2", text: "jumps over the lazy dogs" });
plugs/search/engine.ts
+27 -21
View File
@@ -1,4 +1,5 @@
import { stemmer } from "https://esm.sh/porter-stemmer@0.9.1";
import { KV, KvKey } from "$sb/types.ts";
export type Document = {
id: string;
@@ -6,10 +7,9 @@ export type Document = {
};
export interface BatchKVStore {
get(keys: string[]): Promise<(any | undefined)[]>;
set(entries: Map<string, any>): Promise<void>;
delete(keys: string[]): Promise<void>;
queryPrefix(prefix: string): Promise<[string, any][]>;
batchSet(kvs: KV[]): Promise<void>;
batchDel(keys: KvKey[]): Promise<void>;
query(options: { prefix: KvKey }): Promise<KV[]>;
}
type ResultObject = {
@@ -22,7 +22,7 @@ export class SimpleSearchEngine {
constructor(
public index: BatchKVStore,
public reverseIndex: BatchKVStore,
// public reverseIndex: BatchKVStore,
) {
}
@@ -68,8 +68,16 @@ export class SimpleSearchEngine {
// console.log("updateIndexMap", updateIndexMap);
await this.index.set(updateIndexMap);
await this.reverseIndex.set(updateReverseIndexMap);
await this.index.batchSet(
[...updateIndexMap.entries()].map((
[key, value],
) => ({ key: ["fts", ...key.split("!")], value: value })),
);
await this.index.batchSet(
[...updateReverseIndexMap.entries()].map((
[key, value],
) => ({ key: ["fts_rev", ...key.split("!")], value: value })),
);
}
// Search for a phrase and return document ids sorted by match count
@@ -82,9 +90,9 @@ export class SimpleSearchEngine {
const matchCounts: Map<string, number> = new Map(); // pageName -> count
for (const stemmedWord of stemmedWords) {
const entries = await this.index.queryPrefix(`${stemmedWord}!`);
for (const [key, value] of entries) {
const id = key.split("!").slice(1).join("!");
const entries = await this.index.query({ prefix: ["fts", stemmedWord] });
for (const { key, value } of entries) {
const id = key[2];
if (matchCounts.has(id)) {
matchCounts.set(id, matchCounts.get(id)! + value);
} else {
@@ -102,17 +110,15 @@ export class SimpleSearchEngine {
// Delete a document from the index
public async deleteDocument(documentId: string): Promise<void> {
const words: [string, boolean][] = await this.reverseIndex.queryPrefix(
`${documentId}!`,
);
const keysToDelete: string[] = [];
const revKeysToDelete: string[] = [];
for (const [wordKey] of words) {
const word = wordKey.split("!").slice(1).join("!");
keysToDelete.push(`${word}!${documentId}`);
revKeysToDelete.push(wordKey);
const words = await this.index.query({
prefix: ["fts_rev", documentId],
});
const keysToDelete: KvKey[] = [];
for (const { key } of words) {
const word = key[2];
keysToDelete.push(["fts", word, documentId]);
keysToDelete.push(key);
}
await this.index.delete(keysToDelete);
await this.reverseIndex.delete(revKeysToDelete);
await this.index.batchDel(keysToDelete);
}
}
plugs/search/search.ts
+21 -51
View File
@@ -1,41 +1,18 @@
import { IndexTreeEvent, QueryProviderEvent } from "$sb/app_event.ts";
import { renderToText } from "$sb/lib/tree.ts";
import { applyQuery } from "$sb/lib/query.ts";
import { editor, index, store } from "$sb/syscalls.ts";
import { BatchKVStore, SimpleSearchEngine } from "./engine.ts";
import { FileMeta } from "$sb/types.ts";
import {
applyQuery,
evalQueryExpression,
liftAttributeFilter,
} from "$sb/lib/query.ts";
import { datastore, editor } from "$sb/syscalls.ts";
import { SimpleSearchEngine } from "./engine.ts";
import { FileMeta, KvKey } from "$sb/types.ts";
import { PromiseQueue } from "$sb/lib/async.ts";
const searchPrefix = "🔍 ";
class StoreKVStore implements BatchKVStore {
constructor(private prefix: string) {
}
async queryPrefix(prefix: string): Promise<[string, any][]> {
const results = await store.queryPrefix(this.prefix + prefix);
return results.map((
{ key, value },
) => [key.substring(this.prefix.length), value]);
}
get(keys: string[]): Promise<(string[] | undefined)[]> {
return store.batchGet(keys.map((key) => this.prefix + key));
}
set(entries: Map<string, string[]>): Promise<void> {
return store.batchSet(
Array.from(entries.entries()).map((
[key, value],
) => ({ key: this.prefix + key, value })),
);
}
delete(keys: string[]): Promise<void> {
return store.batchDel(keys.map((key) => this.prefix + key));
}
}
const ftsKvStore = new StoreKVStore("fts:");
const ftsRevKvStore = new StoreKVStore("fts_rev:");
const engine = new SimpleSearchEngine(ftsKvStore, ftsRevKvStore);
const engine = new SimpleSearchEngine(datastore);
// Search indexing is prone to concurrency issues, so we queue all write operations
const promiseQueue = new PromiseQueue();
@@ -50,8 +27,14 @@ export function indexPage({ name, tree }: IndexTreeEvent) {
}
export async function clearIndex() {
await store.deletePrefix("fts:");
await store.deletePrefix("fts_rev:");
const keysToDelete: KvKey[] = [];
for (const { key } of await datastore.query({ prefix: ["fts"] })) {
keysToDelete.push(key);
}
for (const { key } of await datastore.query({ prefix: ["fts_rev"] })) {
keysToDelete.push(key);
}
await datastore.batchDel(keysToDelete);
}
export function pageUnindex(pageName: string) {
@@ -63,11 +46,13 @@ export function pageUnindex(pageName: string) {
export async function queryProvider({
query,
}: QueryProviderEvent): Promise<any[]> {
const phraseFilter = query.filter.find((f) => f.prop === "phrase");
const phraseFilter = liftAttributeFilter(query.filter, "phrase");
if (!phraseFilter) {
throw Error("No 'phrase' filter specified, this is mandatory");
}
let results: any[] = await engine.search(phraseFilter.value);
const phrase = evalQueryExpression(phraseFilter, {});
// console.log("Phrase", phrase);
let results: any[] = await engine.search(phrase);
// Patch the object to a format that users expect (translate id to name)
for (const r of results) {
@@ -75,21 +60,6 @@ export async function queryProvider({
delete r.id;
}
const allPageMap: Map<string, any> = new Map(
results.map((r: any) => [r.name, r]),
);
for (const { page, value } of await index.queryPrefix("meta:")) {
const p = allPageMap.get(page);
if (p) {
for (const [k, v] of Object.entries(value)) {
p[k] = v;
}
}
}
// Remove the "phrase" filter
query.filter.splice(query.filter.indexOf(phraseFilter), 1);
results = applyQuery(query, results);
return results;
}
plugs/tasks/complete.ts
+4 -3
View File
@@ -1,5 +1,6 @@
import { CompleteEvent } from "$sb/app_event.ts";
import { index } from "$sb/syscalls.ts";
import { queryObjects } from "../index/plug_api.ts";
import { TaskStateObject } from "./task.ts";
export async function completeTaskState(completeEvent: CompleteEvent) {
const taskMatch = /([\-\*]\s+\[)([^\[\]]+)$/.exec(
@@ -8,8 +9,8 @@ export async function completeTaskState(completeEvent: CompleteEvent) {
if (!taskMatch) {
return null;
}
const allStates = await index.queryPrefix("taskState:");
const states = [...new Set(allStates.map((s) => s.key.split(":")[1]))];
const allStates = await queryObjects<TaskStateObject>("taskstate", {});
const states = [...new Set(allStates.map((s) => s.state))];
return {
from: completeEvent.pos - taskMatch[2].length,
plugs/tasks/task.ts
+113 -116
View File
@@ -1,10 +1,6 @@
import type {
ClickEvent,
IndexTreeEvent,
QueryProviderEvent,
} from "$sb/app_event.ts";
import type { ClickEvent, IndexTreeEvent } from "$sb/app_event.ts";
import { editor, index, markdown, space, sync } from "$sb/syscalls.ts";
import { editor, markdown, space, sync } from "$sb/syscalls.ts";
import {
addParentPointers,
@@ -17,24 +13,32 @@ import {
replaceNodesMatching,
traverseTreeAsync,
} from "$sb/lib/tree.ts";
import { applyQuery, removeQueries } from "$sb/lib/query.ts";
import { removeQueries } from "$sb/lib/query.ts";
import { niceDate } from "$sb/lib/dates.ts";
import { extractAttributes } from "$sb/lib/attribute.ts";
import { rewritePageRefs } from "$sb/lib/resolve.ts";
import { indexAttributes } from "../index/attributes.ts";
import { ObjectValue } from "$sb/types.ts";
import { indexObjects, queryObjects } from "../index/plug_api.ts";
export type Task = {
export type TaskObject = {
ref: string;
tags: string[];
page: string;
pos: number;
name: string;
done: boolean;
state: string;
deadline?: string;
tags?: string[];
nested?: string;
// Not saved in DB, just added when pulled out (from key)
pos?: number;
page?: string;
} & Record<string, any>;
export type TaskStateObject = {
ref: string;
tags: string[];
state: string;
count: number;
page: string;
};
function getDeadline(deadlineNode: ParseTree): string {
return deadlineNode.children![0].text!.replace(/📅\s*/, "");
}
@@ -43,27 +47,32 @@ const completeStates = ["x", "X"];
const incompleteStates = [" "];
export async function indexTasks({ name, tree }: IndexTreeEvent) {
const tasks: { key: string; value: Task }[] = [];
const taskStates = new Map<string, number>();
const tasks: ObjectValue<TaskObject>[] = [];
const taskStates = new Map<string, { count: number; firstPos: number }>();
removeQueries(tree);
addParentPointers(tree);
const allAttributes: Record<string, any> = {};
// const allAttributes: AttributeObject[] = [];
// const allTags = new Set<string>();
await traverseTreeAsync(tree, async (n) => {
if (n.type !== "Task") {
return false;
}
const state = n.children![0].children![1].text!;
if (!incompleteStates.includes(state) && !completeStates.includes(state)) {
if (!taskStates.has(state)) {
taskStates.set(state, 1);
} else {
taskStates.set(state, taskStates.get(state)! + 1);
let currentState = taskStates.get(state);
if (!currentState) {
currentState = { count: 0, firstPos: n.from! };
}
currentState.count++;
}
const complete = completeStates.includes(state);
const task: Task = {
const task: TaskObject = {
ref: `${name}@${n.from}`,
tags: [],
name: "",
done: complete,
page: name,
pos: n.from!,
state,
};
@@ -76,65 +85,60 @@ export async function indexTasks({ name, tree }: IndexTreeEvent) {
return null;
}
if (tree.type === "Hashtag") {
if (!task.tags) {
task.tags = [];
}
// Push the tag to the list, removing the initial #
task.tags.push(tree.children![0].text!.substring(1));
// Remove this node from the tree
// return null;
const tagName = tree.children![0].text!.substring(1);
task.tags.push(tagName);
}
});
task.tags = ["task", ...task.tags];
// Extract attributes and remove from tree
const extractedAttributes = await extractAttributes(n, true);
for (const [key, value] of Object.entries(extractedAttributes)) {
task[key] = value;
allAttributes[key] = value;
}
task.name = n.children!.slice(1).map(renderToText).join("").trim();
const taskIndex = n.parent!.children!.indexOf(n);
const nestedItems = n.parent!.children!.slice(taskIndex + 1);
if (nestedItems.length > 0) {
task.nested = nestedItems.map(renderToText).join("").trim();
}
tasks.push({
key: `task:${n.from}`,
value: task,
});
tasks.push(task);
return true;
});
// console.log("Found", tasks, "task(s)");
await index.batchSet(name, tasks);
await indexAttributes(name, allAttributes, "task");
await index.batchSet(
name,
Array.from(taskStates.entries()).map(([state, count]) => ({
key: `taskState:${state}`,
value: count,
})),
);
// Index task states
if (taskStates.size > 0) {
await indexObjects<TaskStateObject>(
name,
Array.from(taskStates.entries()).map(([state, { firstPos, count }]) => ({
ref: `${name}@${firstPos}`,
tags: ["taskstate"],
state,
count,
page: name,
})),
);
}
// Index tasks themselves
if (tasks.length > 0) {
await indexObjects(name, tasks);
}
}
export function taskToggle(event: ClickEvent) {
if (event.altKey) {
return;
}
return taskCycleAtPos(event.page, event.pos);
return taskCycleAtPos(event.pos);
}
export async function previewTaskToggle(eventString: string) {
export function previewTaskToggle(eventString: string) {
const [eventName, pos] = JSON.parse(eventString);
if (eventName === "task") {
return taskCycleAtPos(await editor.getCurrentPage(), +pos);
return taskCycleAtPos(+pos);
}
}
async function cycleTaskState(
pageName: string,
node: ParseTree,
) {
const stateText = node.children![1].text!;
@@ -145,8 +149,8 @@ async function cycleTaskState(
changeTo = "x";
} else {
// Not a checkbox, but a custom state
const allStates = await index.queryPrefix("taskState:");
const states = [...new Set(allStates.map((s) => s.key.split(":")[1]))];
const allStates = await queryObjects<TaskStateObject>("taskstate", {});
const states = [...new Set(allStates.map((s) => s.state))];
states.sort();
// Select a next state
const currentStateIndex = states.indexOf(stateText);
@@ -174,64 +178,73 @@ async function cycleTaskState(
for (const wikiLink of parentWikiLinks) {
const ref = wikiLink.children![0].text!;
if (ref.includes("@")) {
const [page, posS] = ref.split("@");
const pos = +posS;
if (page === pageName) {
// In current page, just update the task marker with dispatch
const editorText = await editor.getText();
// Check if the task state marker is still there
const targetText = editorText.substring(
pos + 1,
pos + 1 + stateText.length,
);
if (targetText !== stateText) {
console.error(
"Reference not a task marker, out of date?",
targetText,
);
return;
}
await editor.dispatch({
changes: {
from: pos + 1,
to: pos + 1 + stateText.length,
insert: changeTo,
},
});
} else {
let text = await space.readPage(page);
const referenceMdTree = await markdown.parseMarkdown(text);
// Adding +1 to immediately hit the task state node
const taskStateNode = nodeAtPos(referenceMdTree, pos + 1);
if (!taskStateNode || taskStateNode.type !== "TaskState") {
console.error(
"Reference not a task marker, out of date?",
taskStateNode,
);
return;
}
taskStateNode.children![1].text = changeTo;
text = renderToText(referenceMdTree);
await space.writePage(page, text);
sync.scheduleFileSync(`${page}.md`);
}
await updateTaskState(ref, stateText, changeTo);
}
}
}
export async function taskCycleAtPos(pageName: string, pos: number) {
export async function updateTaskState(
ref: string,
oldState: string,
newState: string,
) {
const currentPage = await editor.getCurrentPage();
const [page, posS] = ref.split("@");
const pos = +posS;
if (page === currentPage) {
// In current page, just update the task marker with dispatch
const editorText = await editor.getText();
// Check if the task state marker is still there
const targetText = editorText.substring(
pos + 1,
pos + 1 + oldState.length,
);
if (targetText !== oldState) {
console.error(
"Reference not a task marker, out of date?",
targetText,
);
return;
}
await editor.dispatch({
changes: {
from: pos + 1,
to: pos + 1 + oldState.length,
insert: newState,
},
});
} else {
let text = await space.readPage(page);
const referenceMdTree = await markdown.parseMarkdown(text);
// Adding +1 to immediately hit the task state node
const taskStateNode = nodeAtPos(referenceMdTree, pos + 1);
if (!taskStateNode || taskStateNode.type !== "TaskState") {
console.error(
"Reference not a task marker, out of date?",
taskStateNode,
);
return;
}
taskStateNode.children![1].text = newState;
text = renderToText(referenceMdTree);
await space.writePage(page, text);
sync.scheduleFileSync(`${page}.md`);
}
}
export async function taskCycleAtPos(pos: number) {
const text = await editor.getText();
const mdTree = await markdown.parseMarkdown(text);
addParentPointers(mdTree);
let node = nodeAtPos(mdTree, pos);
if (node) {
if (node.type === "TaskMarker") {
if (node.type === "TaskMark") {
node = node.parent!;
}
if (node.type === "TaskState") {
await cycleTaskState(pageName, node);
await cycleTaskState(node);
}
}
}
@@ -265,7 +278,7 @@ export async function taskCycleCommand() {
}
const taskState = findNodeOfType(taskNode!, "TaskState");
if (taskState) {
await cycleTaskState(await editor.getCurrentPage(), taskState);
await cycleTaskState(taskState);
}
}
@@ -318,19 +331,3 @@ export async function postponeCommand() {
},
});
}
export async function queryProvider({
query,
}: QueryProviderEvent): Promise<Task[]> {
const allTasks: Task[] = [];
for (const { key, page, value } of await index.queryPrefix("task:")) {
const pos = key.split(":")[1];
allTasks.push({
...value,
page: page,
pos: +pos,
});
}
return applyQuery(query, allTasks);
}
plugs/tasks/tasks.plug.yaml
+9 -4
View File
@@ -19,6 +19,11 @@ syntax:
styles:
backgroundColor: "rgba(22,22,22,0.07)"
functions:
# API
updateTaskState:
path: task.ts:updateTaskState
turnIntoTask:
redirect: template.applyLineReplace
slashCommand:
@@ -35,10 +40,10 @@ functions:
path: "./task.ts:taskToggle"
events:
- page:click
itemQueryProvider:
path: ./task.ts:queryProvider
events:
- query:task
# itemQueryProvider:
# path: ./task.ts:queryProvider
# events:
# - query:task
taskToggleCommand:
path: ./task.ts:taskCycleCommand
command:
plugs/template/template.ts
+19 -18
View File
@@ -1,13 +1,10 @@
import { editor, markdown, space } from "$sb/syscalls.ts";
import { editor, handlebars, markdown, space } from "$sb/syscalls.ts";
import { extractFrontmatter } from "$sb/lib/frontmatter.ts";
import { renderToText } from "$sb/lib/tree.ts";
import { niceDate } from "$sb/lib/dates.ts";
import { readSettings } from "$sb/lib/settings_page.ts";
import { PageMeta } from "../../web/types.ts";
import { buildHandebarOptions } from "../directive/util.ts";
import Handlebars from "handlebars";
import { cleanPageRef } from "$sb/lib/resolve.ts";
import { PageMeta } from "$sb/types.ts";
export async function instantiateTemplateCommand() {
const allPages = await space.listPages();
@@ -48,7 +45,7 @@ export async function instantiateTemplateCommand() {
};
if (additionalPageMeta.$name) {
additionalPageMeta.$name = replaceTemplateVars(
additionalPageMeta.$name = await replaceTemplateVars(
additionalPageMeta.$name,
tempPageMeta,
);
@@ -79,7 +76,10 @@ export async function instantiateTemplateCommand() {
// The preferred scenario, let's keep going
}
const pageText = replaceTemplateVars(renderToText(parseTree), tempPageMeta);
const pageText = await replaceTemplateVars(
renderToText(parseTree),
tempPageMeta,
);
await space.writePage(pageName, pageText);
await editor.navigate(pageName);
}
@@ -110,10 +110,10 @@ export async function insertSnippet() {
}
const text = await space.readPage(`${snippetPrefix}${selectedSnippet.name}`);
let templateText = replaceTemplateVars(text, pageMeta);
let templateText = await replaceTemplateVars(text, pageMeta);
const carretPos = templateText.indexOf("|^|");
templateText = templateText.replace("|^|", "");
templateText = replaceTemplateVars(templateText, pageMeta);
templateText = await replaceTemplateVars(templateText, pageMeta);
await editor.insertAtCursor(templateText);
if (carretPos !== -1) {
await editor.moveCursor(cursorPos + carretPos);
@@ -148,20 +148,21 @@ export async function applyPageTemplateCommand() {
const text = await space.readPage(
`${pageTemplatePrefix}${selectedPage.name}`,
);
let templateText = replaceTemplateVars(text, pageMeta);
let templateText = await replaceTemplateVars(text, pageMeta);
const carretPos = templateText.indexOf("|^|");
templateText = templateText.replace("|^|", "");
templateText = replaceTemplateVars(templateText, pageMeta);
templateText = await replaceTemplateVars(templateText, pageMeta);
await editor.insertAtCursor(templateText);
if (carretPos !== -1) {
await editor.moveCursor(cursorPos + carretPos);
}
}
// TODO: This should probably be replaced with handlebards somehow?
export function replaceTemplateVars(s: string, pageMeta: PageMeta): string {
const template = Handlebars.compile(s, { noEscape: true });
return template({}, buildHandebarOptions(pageMeta));
export function replaceTemplateVars(
s: string,
pageMeta: PageMeta,
): Promise<string> {
return handlebars.renderTemplate(s, {}, { page: pageMeta });
}
export async function quickNoteCommand() {
@@ -204,7 +205,7 @@ export async function dailyNoteCommand() {
await space.writePage(
pageName,
replaceTemplateVars(dailyNoteTemplateText, {
await replaceTemplateVars(dailyNoteTemplateText, {
name: pageName,
lastModified: 0,
perm: "rw",
@@ -248,7 +249,7 @@ export async function weeklyNoteCommand() {
// Doesn't exist, let's create
await space.writePage(
pageName,
replaceTemplateVars(weeklyNoteTemplateText, {
await replaceTemplateVars(weeklyNoteTemplateText, {
name: pageName,
lastModified: 0,
perm: "rw",
@@ -278,7 +279,7 @@ export async function insertTemplateText(cmdDef: any) {
let templateText: string = cmdDef.value;
const carretPos = templateText.indexOf("|^|");
templateText = templateText.replace("|^|", "");
templateText = replaceTemplateVars(templateText, pageMeta);
templateText = await replaceTemplateVars(templateText, pageMeta!);
await editor.insertAtCursor(templateText);
if (carretPos !== -1) {
await editor.moveCursor(cursorPos + carretPos);