neil/silverbullet
1
0
Fork 0
Code Activity

Fixes #529 by removing directives (#613)

Browse Source 
* Fixes #529 by removing directives
* Load builtin tags on space reindex
This commit is contained in:
Zef Hemel
2024-01-02 14:47:02 +01:00
committed by GitHub
parent 5f4e584e46
commit 8a2e081672
46 changed files with 159 additions and 1445 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugs/builtin_plugs.ts
-1
View File
@@ -5,7 +5,6 @@ export const builtinPlugNames = [
"sync",
"template",
"plug-manager",
"directive",
"emoji",
"query",
"markdown",
plugs/directive/command.ts
-317
View File
@@ -1,317 +0,0 @@
import { editor, markdown, mq, space, sync } from "$sb/syscalls.ts";
import {
addParentPointers,
collectNodesOfType,
findParentMatching,
nodeAtPos,
ParseTree,
removeParentPointers,
renderToText,
traverseTree,
} from "$sb/lib/tree.ts";
import { renderDirectives } from "./directives.ts";
import { extractFrontmatter } from "$sb/lib/frontmatter.ts";
import { isFederationPath } from "$sb/lib/resolve.ts";
import { MQMessage, PageMeta } from "$sb/types.ts";
import { sleep } from "$sb/lib/async.ts";
const directiveUpdateQueueName = "directiveUpdateQueue";
export async function updateDirectivesOnPageCommand() {
// If `arg` is a string, it's triggered automatically via an event, not explicitly via a command
const currentPage = await editor.getCurrentPage();
let pageMeta: PageMeta | undefined;
try {
pageMeta = await space.getPageMeta(currentPage);
} catch {
console.info("Page not found, not updating directives");
return;
}
const text = await editor.getText();
const tree = await markdown.parseMarkdown(text);
const metaData = await extractFrontmatter(tree, {
removeKeys: ["$disableDirectives"],
});
if (isFederationPath(currentPage)) {
console.info("Current page is a federation page, not updating directives.");
return;
}
if (metaData.$disableDirectives) {
console.info("Directives disabled in page meta, not updating them.");
return;
}
if (!(await sync.hasInitialSyncCompleted())) {
console.info(
"Initial sync hasn't completed yet, not updating directives.",
);
return;
}
await editor.save();
const replacements = await findReplacements(tree, text, pageMeta);
// Iterate again and replace the bodies. Iterating again (not using previous positions)
// because text may have changed in the mean time (directive processing may take some time)
// Hypothetically in the mean time directives in text may have been changed/swapped, in which
// case this will break. This would be a rare edge case, however.
for (const replacement of replacements) {
// Fetch the text every time, because dispatch() will have been made changes
const text = await editor.getText();
// Determine the current position
const index = text.indexOf(replacement.fullMatch);
// This may happen if the query itself, or the user is editing inside the directive block (WHY!?)
if (index === -1) {
console.warn(
"Text I got",
text,
);
console.warn(
"Could not find directive in text, skipping",
replacement.fullMatch,
);
continue;
}
const from = index, to = index + replacement.fullMatch.length;
const newText = await replacement.textPromise;
if (text.substring(from, to) === newText) {
// No change, skip
continue;
}
await editor.dispatch({
changes: {
from,
to,
insert: newText,
},
});
}
}
export async function updateDirectivesInSpaceCommand() {
await editor.flashNotification(
"Updating directives in entire space, this can take a while...",
);
await updateDirectivesInSpace();
// And notify the user
await editor.flashNotification("Updating of all directives completed!");
}
export async function processUpdateQueue(messages: MQMessage[]) {
for (const message of messages) {
const pageName: string = message.body;
console.log("Updating directives in page", pageName);
await updateDirectivesForPage(pageName);
await mq.ack(directiveUpdateQueueName, message.id);
}
}
async function findReplacements(
tree: ParseTree,
text: string,
pageMeta: PageMeta,
) {
// Collect all directives and their body replacements
const replacements: { fullMatch: string; textPromise: Promise<string> }[] =
[];
// Convenience array to wait for all promises to resolve
const allPromises: Promise<string>[] = [];
removeParentPointers(tree);
traverseTree(tree, (tree) => {
if (tree.type !== "Directive") {
return false;
}
const fullMatch = text.substring(tree.from!, tree.to!);
try {
const promise = renderDirectives(pageMeta, tree);
replacements.push({
textPromise: promise,
fullMatch,
});
allPromises.push(promise);
} catch (e: any) {
replacements.push({
fullMatch,
textPromise: Promise.resolve(
`${renderToText(tree.children![0])}\n**ERROR:** ${e.message}\n${
renderToText(tree.children![tree.children!.length - 1])
}`,
),
});
}
return true;
});
// Wait for all to have processed
await Promise.all(allPromises);
return replacements;
}
export async function updateDirectivesInSpace() {
const pages = await space.listPages();
await mq.batchSend(directiveUpdateQueueName, pages.map((page) => page.name));
// Now let's wait for the processing to finish
let queueStats = await mq.getQueueStats(directiveUpdateQueueName);
while (queueStats.queued > 0 || queueStats.processing > 0) {
sleep(1000);
queueStats = await mq.getQueueStats(directiveUpdateQueueName);
}
console.log("Done updating directives in space!");
}
async function updateDirectivesForPage(
pageName: string,
) {
const pageMeta = await space.getPageMeta(pageName);
const currentText = await space.readPage(pageName);
const tree = await markdown.parseMarkdown(currentText);
const metaData = await extractFrontmatter(tree, {
removeKeys: ["$disableDirectives"],
});
if (isFederationPath(pageName)) {
console.info("Current page is a federation page, not updating directives.");
return;
}
if (metaData.$disableDirectives) {
console.info("Directives disabled in page meta, not updating them.");
return;
}
const newText = await updateDirectives(pageMeta, tree, currentText);
if (newText !== currentText) {
console.info("Content of page changed, saving", pageName);
await space.writePage(pageName, newText);
}
}
export async function updateDirectives(
pageMeta: PageMeta,
tree: ParseTree,
text: string,
) {
const replacements = await findReplacements(tree, text, pageMeta);
// Iterate again and replace the bodies.
for (const replacement of replacements) {
text = text.replace(
replacement.fullMatch,
await replacement.textPromise,
);
}
return text;
}
export async function convertToLive() {
const text = await editor.getText();
const pos = await editor.getCursor();
const tree = await markdown.parseMarkdown(text);
addParentPointers(tree);
const currentNode = nodeAtPos(tree, pos);
const directive = findParentMatching(
currentNode!,
(node) => node.type === "Directive",
);
if (!directive) {
await editor.flashNotification(
"No directive found at cursor position",
"error",
);
return;
}
console.log("Got this directive", directive);
const startNode = directive.children![0];
const startNodeText = renderToText(startNode);
if (startNodeText.includes("#query")) {
const queryText = renderToText(startNode.children![1]);
await editor.dispatch({
changes: {
from: directive.from,
to: directive.to,
insert: "```query\n" + queryText + "\n```",
},
});
} else if (
startNodeText.includes("#use") || startNodeText.includes("#include")
) {
const pageRefMatch = /\[\[([^\]]+)\]\]\s*([^\-]+)?/.exec(startNodeText);
if (!pageRefMatch) {
await editor.flashNotification(
"No page reference found in directive",
"error",
);
return;
}
const val = pageRefMatch[2];
await editor.dispatch({
changes: {
from: directive.from,
to: directive.to,
insert: '```template\npage: "[[' + pageRefMatch[1] + ']]"\n' +
(val ? `val: ${val}\n` : "") + "```",
},
});
}
}
export async function convertSpaceToLive() {
if (
!await editor.confirm(
"This will convert all directives in the space to live queries. Are you sure?",
)
) {
return;
}
const pages = await space.listPages();
for (const page of pages) {
console.log("Now converting", page);
const text = await space.readPage(page.name);
const newText = await convertDirectivesOnPage(text);
if (text !== newText) {
console.log("Changes were made, writing", page.name);
await space.writePage(page.name, newText);
}
}
await editor.flashNotification("All done!");
}
export async function convertDirectivesOnPage(text: string) {
const tree = await markdown.parseMarkdown(text);
collectNodesOfType(tree, "Directive").forEach((directive) => {
const directiveText = renderToText(directive);
console.log("Got this directive", directiveText);
const startNode = directive.children![0];
const startNodeText = renderToText(startNode);
if (startNodeText.includes("#query")) {
const queryText = renderToText(startNode.children![1]);
text = text.replace(directiveText, "```query\n" + queryText + "\n```");
} else if (
startNodeText.includes("#use") || startNodeText.includes("#include")
) {
const pageRefMatch = /\[\[([^\]]+)\]\]\s*([^\-]+)?/.exec(startNodeText);
if (!pageRefMatch) {
return;
}
const val = pageRefMatch[2];
text = text.replace(
directiveText,
'```template\npage: "[[' + pageRefMatch[1] + ']]"\n' +
(val ? `val: ${val}\n` : "") + "```",
);
}
});
// console.log("Converted page", text);
return text;
}
plugs/directive/complete.ts
-77
View File
@@ -1,77 +0,0 @@
import { events } from "$sb/syscalls.ts";
import { CompleteEvent } from "$sb/app_event.ts";
import { buildHandebarOptions } from "./util.ts";
import type {
AttributeCompleteEvent,
AttributeCompletion,
} from "../index/attributes.ts";
import { PageMeta } from "$sb/types.ts";
export async function queryComplete(completeEvent: CompleteEvent) {
const querySourceMatch = /#query\s+([\w\-_]*)$/.exec(
completeEvent.linePrefix,
);
if (querySourceMatch) {
const allEvents = await events.listEvents();
const completionOptions = allEvents
.filter((eventName) =>
eventName.startsWith("query:") && !eventName.includes("*")
)
.map((source) => ({
label: source.substring("query:".length),
}));
const allObjectTypes: string[] = (await events.dispatchEvent("query_", {}))
.flat();
for (const type of allObjectTypes) {
completionOptions.push({
label: type,
});
}
return {
from: completeEvent.pos - querySourceMatch[1].length,
options: completionOptions,
};
}
if (completeEvent.parentNodes.includes("DirectiveStart")) {
const querySourceMatch = /#query\s+([\w\-_\/]+)/.exec(
completeEvent.linePrefix,
);
const whereMatch =
/(where|order\s+by|and|select(\s+[\w\s,]+)?)\s+([\w\-_]*)$/.exec(
completeEvent.linePrefix,
);
if (querySourceMatch && whereMatch) {
const type = querySourceMatch[1];
const attributePrefix = whereMatch[3];
const completions = (await events.dispatchEvent(
`attribute:complete:${type}`,
{
source: type,
prefix: attributePrefix,
} as AttributeCompleteEvent,
)).flat() as AttributeCompletion[];
return {
from: completeEvent.pos - attributePrefix.length,
options: attributeCompletionsToCMCompletion(completions),
};
}
}
return null;
}
export function attributeCompletionsToCMCompletion(
completions: AttributeCompletion[],
) {
return completions.map(
(completion) => ({
label: completion.name,
detail: `${completion.attributeType} (${completion.source})`,
type: "attribute",
}),
);
}
plugs/directive/directive.plug.yaml
-35
View File
@@ -1,35 +0,0 @@
name: directive
requiredPermissions:
- fetch
functions:
updateDirectivesOnPageCommand:
path: ./command.ts:updateDirectivesOnPageCommand
command:
name: "Directives: Update"
events:
- editor:pageLoaded
updateDirectivesInSpace:
path: ./command.ts:updateDirectivesInSpace
updateDirectivesInSpaceCommand:
path: ./command.ts:updateDirectivesInSpaceCommand
command:
name: "Directives: Update Entire Space"
processUpdateQueue:
path: ./command.ts:processUpdateQueue
mqSubscriptions:
- queue: directiveUpdateQueue
batchSize: 3
queryComplete:
path: ./complete.ts:queryComplete
events:
- editor:complete
# Conversion
convertToLiveQuery:
path: command.ts:convertToLive
command:
name: "Directive: Convert to Live Query/Template"
convertSpaceToLive:
path: command.ts:convertSpaceToLive
command:
name: "Directive: Convert Entire Space to Live/Templates"
plugs/directive/directives.ts
-122
View File
@@ -1,122 +0,0 @@
import {
addParentPointers,
findParentMatching,
ParseTree,
renderToText,
} from "$sb/lib/tree.ts";
import { PageMeta } from "$sb/types.ts";
import { editor, markdown } from "$sb/syscalls.ts";
import { evalDirectiveRenderer } from "./eval_directive.ts";
import { queryDirectiveRenderer } from "./query_directive.ts";
import {
cleanTemplateInstantiations,
templateDirectiveRenderer,
} from "./template_directive.ts";
/** An error that occurs while a directive is being rendered.
* Mostly annotates the underlying error with page metadata.
*/
export class RenderDirectiveError extends Error {
pageMeta: PageMeta;
directive: string;
cause: Error;
constructor(pageMeta: PageMeta, directive: string, cause: Error) {
super(`In directive "${directive}" from "${pageMeta.name}": ${cause}`, {
cause: cause,
});
this.pageMeta = pageMeta;
this.directive = directive;
this.cause = cause;
}
}
export const directiveStartRegex =
/<!--\s*#(use|use-verbose|include|eval|query)\s+(.*?)-->/i;
export const directiveRegex =
/(<!--\s*#(use|use-verbose|include|eval|query)\s+(.*?)-->)(.+?)(<!--\s*\/\2\s*-->)/gs;
/**
* Looks for directives in the text dispatches them based on name
*/
export async function directiveDispatcher(
pageMeta: PageMeta,
directiveTree: ParseTree,
directiveRenderers: Record<
string,
(
directive: string,
pageMeta: PageMeta,
arg: string | ParseTree,
) => Promise<string>
>,
): Promise<string> {
const directiveStart = directiveTree.children![0]; // <!-- #directive -->
const directiveEnd = directiveTree.children![2]; // <!-- /directive -->
const directiveStartText = renderToText(directiveStart).trim();
const directiveEndText = renderToText(directiveEnd).trim();
const firstPart = directiveStart.children![0].text!;
if (firstPart?.includes("#query")) {
// #query
const newBody = await directiveRenderers["query"](
"query",
pageMeta,
directiveStart.children![1].children![0], // The query ParseTree
);
const result =
`${directiveStartText}\n${newBody.trim()}\n${directiveEndText}`;
return result;
} else if (firstPart?.includes("#eval")) {
console.log("Eval stuff", directiveStart.children![1].children![0]);
const newBody = await directiveRenderers["eval"](
"eval",
pageMeta,
directiveStart.children![1].children![0],
);
const result =
`${directiveStartText}\n${newBody.trim()}\n${directiveEndText}`;
return result;
} else {
// Everything not #query and #eval
const match = directiveStartRegex.exec(directiveStart.children![0].text!);
if (!match) {
throw Error("No match");
}
let [_fullMatch, type, arg] = match;
try {
arg = arg.trim();
const newBody = await directiveRenderers[type](type, pageMeta, arg);
const result =
`${directiveStartText}\n${newBody.trim()}\n${directiveEndText}`;
return result;
} catch (e: any) {
return `${directiveStartText}\n**ERROR:** ${e.message}\n${directiveEndText}`;
}
}
}
export async function renderDirectives(
pageMeta: PageMeta,
directiveTree: ParseTree,
): Promise<string> {
try {
const replacementText = await directiveDispatcher(pageMeta, directiveTree, {
use: templateDirectiveRenderer,
include: templateDirectiveRenderer,
query: queryDirectiveRenderer,
eval: evalDirectiveRenderer,
});
return cleanTemplateInstantiations(replacementText);
} catch (e) {
throw new RenderDirectiveError(
pageMeta,
renderToText(directiveTree.children![0].children![1]).trim(),
e,
);
}
}
plugs/directive/eval_directive.ts
-29
View File
@@ -1,29 +0,0 @@
import { ParseTree, parseTreeToAST } from "$sb/lib/tree.ts";
import { replaceTemplateVars } from "../template/template.ts";
import { PageMeta } from "$sb/types.ts";
import { expressionToKvQueryExpression } from "$sb/lib/parse-query.ts";
import { evalQueryExpression } from "$sb/lib/query.ts";
import { builtinFunctions } from "$sb/lib/builtin_query_functions.ts";
// This is rather scary and fragile stuff, but it works.
export async function evalDirectiveRenderer(
_directive: string,
pageMeta: PageMeta,
expression: string | ParseTree,
): Promise<string> {
try {
const result = evalQueryExpression(
expressionToKvQueryExpression(parseTreeToAST(
JSON.parse(
await replaceTemplateVars(JSON.stringify(expression), pageMeta),
),
)),
{},
builtinFunctions,
);
return Promise.resolve("" + result);
} catch (e: any) {
return Promise.resolve(`**ERROR:** ${e.message}`);
}
}
plugs/directive/query_directive.ts
-56
View File
@@ -1,56 +0,0 @@
import { events } from "$sb/syscalls.ts";
import { replaceTemplateVars } from "../template/template.ts";
import { renderQueryTemplate } from "./util.ts";
import { jsonToMDTable } from "./util.ts";
import { ParseTree, parseTreeToAST } from "$sb/lib/tree.ts";
import { astToKvQuery } from "$sb/lib/parse-query.ts";
import { PageMeta, Query } from "$sb/types.ts";
export async function queryDirectiveRenderer(
_directive: string,
pageMeta: PageMeta,
query: string | ParseTree,
): Promise<string> {
if (typeof query === "string") {
throw new Error("Argument must be a ParseTree");
}
const parsedQuery: Query = astToKvQuery(
parseTreeToAST(
JSON.parse(await replaceTemplateVars(JSON.stringify(query), pageMeta)),
),
);
// console.log("QUERY", parsedQuery);
const eventName = `query:${parsedQuery.querySource}`;
// console.log("Parsed query", parsedQuery);
// Let's dispatch an event and see what happens
const results = await events.dispatchEvent(
eventName,
{ query: parsedQuery, pageName: pageMeta.name },
30 * 1000,
);
if (results.length === 0) {
// This means there was no handler for the event which means it's unsupported
return `**Error:** Unsupported query source '${parsedQuery.querySource}'`;
} else {
// console.log("Parsed query", parsedQuery);
const allResults = results.flat();
if (parsedQuery.render) {
const rendered = await renderQueryTemplate(
pageMeta,
parsedQuery.render,
allResults,
parsedQuery.renderAll!,
);
return rendered.trim();
} else {
if (allResults.length === 0) {
return "No results";
} else {
return jsonToMDTable(allResults);
}
}
}
}
plugs/directive/template_directive.ts
-100
View File
@@ -1,100 +0,0 @@
import { queryRegex } from "$sb/lib/query.ts";
import { ParseTree, renderToText } from "$sb/lib/tree.ts";
import { handlebars, markdown, space } from "$sb/syscalls.ts";
import { replaceTemplateVars } from "../template/template.ts";
import { extractFrontmatter } from "$sb/lib/frontmatter.ts";
import { directiveRegex } from "./directives.ts";
import { updateDirectives } from "./command.ts";
import { resolvePath, rewritePageRefs } from "$sb/lib/resolve.ts";
import { PageMeta } from "$sb/types.ts";
import { renderTemplate } from "../template/plug_api.ts";
const templateRegex = /\[\[([^\]]+)\]\]\s*(.*)\s*/;
export async function templateDirectiveRenderer(
directive: string,
pageMeta: PageMeta,
arg: string | ParseTree,
): Promise<string> {
if (typeof arg !== "string") {
throw new Error("Template directives must be a string");
}
const match = arg.match(templateRegex);
if (!match) {
throw new Error(`Invalid template directive: ${arg}`);
}
let templatePath = match[1];
const args = match[2];
let parsedArgs = {};
if (args) {
try {
parsedArgs = JSON.parse(await replaceTemplateVars(args, pageMeta));
} catch {
throw new Error(
`Failed to parse template instantiation arg: ${
replaceTemplateVars(args, pageMeta)
}`,
);
}
}
let templateText = "";
if (
templatePath.startsWith("http://") || templatePath.startsWith("https://")
) {
try {
const req = await fetch(templatePath);
templateText = await req.text();
} catch (e: any) {
templateText = `ERROR: ${e.message}`;
}
} else {
templatePath = resolvePath(pageMeta.name, templatePath);
templateText = await space.readPage(templatePath);
}
const tree = await markdown.parseMarkdown(templateText);
await extractFrontmatter(tree, { removeFrontmatterSection: true }); // Remove entire frontmatter section, if any
// Resolve paths in the template
rewritePageRefs(tree, templatePath);
let newBody = renderToText(tree);
// console.log("Rewritten template:", newBody);
// if it's a template injection (not a literal "include")
if (directive === "use") {
newBody = (await renderTemplate(newBody, pageMeta, parsedArgs)).text;
// Recursively render directives
const tree = await markdown.parseMarkdown(newBody);
newBody = await updateDirectives(pageMeta, tree, newBody);
}
return newBody.trim();
}
export function cleanTemplateInstantiations(text: string) {
return text.replaceAll(directiveRegex, (
_fullMatch,
startInst,
type,
_args,
body,
endInst,
): string => {
if (type === "use") {
body = body.replaceAll(
queryRegex,
(
_fullMatch: string,
_startQuery: string,
_query: string,
body: string,
) => {
return body.trim();
},
);
}
return `${startInst}${body}${endInst}`;
});
}
plugs/directive/util.ts
-77
View File
@@ -1,77 +0,0 @@
import { handlebars, space } from "$sb/syscalls.ts";
import { handlebarHelpers } from "../../common/syscalls/handlebar_helpers.ts";
import { PageMeta } from "$sb/types.ts";
import { cleanTemplate } from "../template/plug_api.ts";
export function defaultJsonTransformer(_k: string, v: any) {
if (v === undefined) {
return "";
}
if (typeof v === "string") {
return v.replaceAll("\n", " ").replaceAll("|", "\\|");
}
return "" + v;
}
// Nicely format an array of JSON objects as a Markdown table
export function jsonToMDTable(
jsonArray: any[],
valueTransformer: (k: string, v: any) => string = defaultJsonTransformer,
): string {
const headers = new Set<string>();
for (const entry of jsonArray) {
for (const k of Object.keys(entry)) {
headers.add(k);
}
}
const headerList = [...headers];
const lines = [];
lines.push(
"|" +
headerList
.map(
(headerName) => headerName,
)
.join("|") +
"|",
);
lines.push(
"|" +
headerList
.map(() => "--")
.join("|") +
"|",
);
for (const val of jsonArray) {
const el = [];
for (const prop of headerList) {
const s = valueTransformer(prop, val[prop]);
el.push(s);
}
lines.push("|" + el.join("|") + "|");
}
return lines.join("\n");
}
export async function renderQueryTemplate(
pageMeta: PageMeta,
templatePage: string,
data: any[],
renderAll: boolean,
): Promise<string> {
let templateText = await space.readPage(templatePage);
templateText = await cleanTemplate(templateText);
if (!renderAll) {
templateText = `{{#each .}}\n${templateText}\n{{/each}}`;
}
return handlebars.renderTemplate(templateText, data, { page: pageMeta });
}
export function buildHandebarOptions(pageMeta: PageMeta) {
return {
helpers: handlebarHelpers(),
data: { page: pageMeta },
};
}
plugs/editor/broken_links.ts
-5
View File
@@ -43,11 +43,6 @@ export async function brokenLinksCommand() {
}
}
if (tree.type === "DirectiveBody") {
// Don't look inside directive bodies
return true;
}
return false;
});
}
plugs/index/anchor.ts
-2
View File
@@ -1,6 +1,5 @@
import { collectNodesOfType } from "$sb/lib/tree.ts";
import type { CompleteEvent, IndexTreeEvent } from "$sb/app_event.ts";
import { removeQueries } from "$sb/lib/query.ts";
import { ObjectValue, QueryExpression } from "$sb/types.ts";
import { indexObjects, queryObjects } from "./api.ts";
@@ -11,7 +10,6 @@ type AnchorObject = ObjectValue<{
}>;
export async function indexAnchors({ name: pageName, tree }: IndexTreeEvent) {
removeQueries(tree);
const anchors: ObjectValue<AnchorObject>[] = [];
collectNodesOfType(tree, "NamedAnchor").forEach((n) => {
plugs/index/builtins.ts
-1
View File
@@ -62,7 +62,6 @@ export const builtins: Record<string, Record<string, string>> = {
page: "!string",
pos: "!number",
alias: "!string",
inDirective: "!boolean",
asTemplate: "!boolean",
},
paragraph: {
plugs/index/command.ts
+4
View File
@@ -14,6 +14,10 @@ export async function reindexSpace() {
console.log("Clearing page index...");
// Executed this way to not have to embed the search plug code here
await system.invokeFunction("index.clearIndex");
// Load builtins
await system.invokeFunction("index.loadBuiltinsIntoIndex");
const pages = await space.listPages();
// Queue all page names to be indexed
plugs/index/data.ts
-3
View File
@@ -1,7 +1,6 @@
import type { IndexTreeEvent } from "$sb/app_event.ts";
import { YAML } from "$sb/syscalls.ts";
import { collectNodesOfType, findNodeOfType } from "$sb/lib/tree.ts";
import { removeQueries } from "$sb/lib/query.ts";
import { ObjectValue } from "$sb/types.ts";
import { indexObjects } from "./api.ts";
import { TagObject } from "./tags.ts";
@@ -16,8 +15,6 @@ type DataObject = ObjectValue<
export async function indexData({ name, tree }: IndexTreeEvent) {
const dataObjects: ObjectValue<DataObject>[] = [];
removeQueries(tree);
await Promise.all(
collectNodesOfType(tree, "FencedCode").map(async (t) => {
const codeInfoNode = findNodeOfType(t, "CodeInfo");
plugs/index/item.ts
-2
View File
@@ -1,7 +1,6 @@
import type { IndexTreeEvent } from "$sb/app_event.ts";
import { collectNodesOfType, ParseTree, renderToText } from "$sb/lib/tree.ts";
import { removeQueries } from "$sb/lib/query.ts";
import { extractAttributes } from "$sb/lib/attribute.ts";
import { rewritePageRefs } from "$sb/lib/resolve.ts";
import { ObjectValue } from "$sb/types.ts";
@@ -17,7 +16,6 @@ export type ItemObject = ObjectValue<
export async function indexItems({ name, tree }: IndexTreeEvent) {
const items: ObjectValue<ItemObject>[] = [];
removeQueries(tree);
// console.log("Indexing items", name);
plugs/index/linked_mentions.ts
+3 -3
View File
@@ -19,11 +19,11 @@ export async function renderMentions(): Promise<CodeWidgetContent | null> {
const page = await editor.getCurrentPage();
const linksResult = await queryObjects<LinkObject>("link", {
// Query all links that point to this page, excluding those that are inside directives and self pointers.
filter: ["and", ["!=", ["attr", "page"], ["string", page]], ["and", ["=", [
// Query all links that point to this page
filter: ["and", ["!=", ["attr", "page"], ["string", page]], ["=", [
"attr",
"toPage",
], ["string", page]], ["=", ["attr", "inDirective"], ["boolean", false]]]],
], ["string", page]]],
});
if (linksResult.length === 0) {
// Don't show the panel if there are no links here.
plugs/index/page_links.ts
-54
View File
@@ -16,7 +16,6 @@ export type LinkObject = {
pos: number;
snippet: string;
alias?: string;
inDirective: boolean;
asTemplate: boolean;
};
@@ -51,55 +50,7 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) {
const pageText = renderToText(tree);
let directiveDepth = 0;
traverseTree(tree, (n): boolean => {
if (n.type === "DirectiveStart") {
directiveDepth++;
const pageRef = findNodeOfType(n, "PageRef")!;
if (pageRef) {
const pageRefName = resolvePath(
name,
pageRef.children![0].text!.slice(2, -2),
);
const pos = pageRef.from! + 2;
links.push({
ref: `${name}@${pos}`,
tags: ["link"],
toPage: pageRefName,
pos: pos,
snippet: extractSnippet(pageText, pos),
page: name,
asTemplate: true,
inDirective: false,
});
}
const directiveText = n.children![0].text;
// #use or #import
if (directiveText) {
const match = /\[\[(.+)\]\]/.exec(directiveText);
if (match) {
const pageRefName = resolvePath(name, match[1]);
const pos = n.from! + match.index! + 2;
links.push({
ref: `${name}@${pos}`,
tags: ["link"],
toPage: pageRefName,
page: name,
snippet: extractSnippet(pageText, pos),
pos: pos,
asTemplate: true,
inDirective: false,
});
}
}
return true;
}
if (n.type === "DirectiveEnd") {
directiveDepth--;
return true;
}
if (n.type === "WikiLink") {
const wikiLinkPage = findNodeOfType(n, "WikiLinkPage")!;
const wikiLinkAlias = findNodeOfType(n, "WikiLinkAlias");
@@ -113,12 +64,8 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) {
snippet: extractSnippet(pageText, pos),
pos,
page: name,
inDirective: false,
asTemplate: false,
};
if (directiveDepth > 0) {
link.inDirective = true;
}
if (wikiLinkAlias) {
link.alias = wikiLinkAlias.children![0].text!;
}
@@ -151,7 +98,6 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) {
snippet: extractSnippet(pageText, pos),
pos: pos,
asTemplate: true,
inDirective: false,
});
}
}
plugs/index/tags.ts
-2
View File
@@ -1,5 +1,4 @@
import type { CompleteEvent, IndexTreeEvent } from "$sb/app_event.ts";
import { removeQueries } from "$sb/lib/query.ts";
import { extractFrontmatter } from "$sb/lib/frontmatter.ts";
import { indexObjects, queryObjects } from "./api.ts";
import {
@@ -16,7 +15,6 @@ export type TagObject = ObjectValue<{
}>;
export async function indexTags({ name, tree }: IndexTreeEvent) {
removeQueries(tree);
const tags = new Set<string>(); // name:parent
addParentPointers(tree);
const pageTags: string[] = (await extractFrontmatter(tree)).tags;
plugs/markdown/markdown_render.ts
-4
View File
@@ -391,10 +391,6 @@ function render(
body: cleanTags(mapRender(newChildren)),
};
}
case "Directive": {
const body = findNodeOfType(t, "DirectiveBody")!;
return posPreservingRender(body.children![0], options);
}
case "Attribute":
if (options.preserveAttributes) {
return {
plugs/query/query.plug.yaml
-9
View File
@@ -45,15 +45,6 @@ functions:
```query
|^|
```
insertInclude:
redirect: template.insertTemplateText
slashCommand:
name: include
description: Include another page
value: |
<!-- #include [[|^|]] -->
<!-- /include -->
insertUseTemplate:
redirect: template.insertTemplateText
slashCommand:
plugs/query/query.ts
+2 -2
View File
@@ -1,4 +1,4 @@
import type { LintEvent, WidgetContent } from "$sb/app_event.ts";
import type { LintEvent } from "$sb/app_event.ts";
import { events, language, space } from "$sb/syscalls.ts";
import {
findNodeOfType,
@@ -6,10 +6,10 @@ import {
traverseTreeAsync,
} from "$sb/lib/tree.ts";
import { astToKvQuery } from "$sb/lib/parse-query.ts";
import { jsonToMDTable, renderQueryTemplate } from "../directive/util.ts";
import { loadPageObject, replaceTemplateVars } from "../template/template.ts";
import { cleanPageRef, resolvePath } from "$sb/lib/resolve.ts";
import { CodeWidgetContent, LintDiagnostic } from "$sb/types.ts";
import { jsonToMDTable, renderQueryTemplate } from "../template/util.ts";
export async function widget(
bodyText: string,
plugs/tasks/task.ts
-2
View File
@@ -13,7 +13,6 @@ import {
replaceNodesMatching,
traverseTreeAsync,
} from "$sb/lib/tree.ts";
import { removeQueries } from "$sb/lib/query.ts";
import { niceDate } from "$sb/lib/dates.ts";
import { extractAttributes } from "$sb/lib/attribute.ts";
import { rewritePageRefs } from "$sb/lib/resolve.ts";
@@ -47,7 +46,6 @@ const incompleteStates = [" "];
export async function indexTasks({ name, tree }: IndexTreeEvent) {
const tasks: ObjectValue<TaskObject>[] = [];
const taskStates = new Map<string, { count: number; firstPos: number }>();
removeQueries(tree);
addParentPointers(tree);
// const allAttributes: AttributeObject[] = [];
// const allTags = new Set<string>();
plugs/template/complete.ts
+1 -1
View File
@@ -1,7 +1,6 @@
import { CompleteEvent, SlashCompletion } from "$sb/app_event.ts";
import { PageMeta } from "$sb/types.ts";
import { editor, events, markdown, space } from "$sb/syscalls.ts";
import { buildHandebarOptions } from "../directive/util.ts";
import type {
AttributeCompleteEvent,
AttributeCompletion,
@@ -11,6 +10,7 @@ import { TemplateObject } from "./types.ts";
import { loadPageObject } from "./template.ts";
import { renderTemplate } from "./api.ts";
import { prepareFrontmatterDispatch } from "$sb/lib/frontmatter.ts";
import { buildHandebarOptions } from "./util.ts";
export async function templateVariableComplete(completeEvent: CompleteEvent) {
const match = /\{\{([\w@]*)$/.exec(completeEvent.linePrefix);
plugs/template/util.ts
+77
View File
@@ -1,4 +1,8 @@
import { determineTags } from "$sb/lib/cheap_yaml.ts";
import { handlebarHelpers } from "../../common/syscalls/handlebar_helpers.ts";
import { PageMeta } from "$sb/types.ts";
import { handlebars, space } from "$sb/syscalls.ts";
import { cleanTemplate } from "./plug_api.ts";
const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/;
@@ -24,3 +28,76 @@ export function isTemplate(pageText: string): boolean {
}
return false;
}
export function buildHandebarOptions(pageMeta: PageMeta) {
return {
helpers: handlebarHelpers(),
data: { page: pageMeta },
};
}
export function defaultJsonTransformer(_k: string, v: any) {
if (v === undefined) {
return "";
}
if (typeof v === "string") {
return v.replaceAll("\n", " ").replaceAll("|", "\\|");
}
return "" + v;
}
// Nicely format an array of JSON objects as a Markdown table
export function jsonToMDTable(
jsonArray: any[],
valueTransformer: (k: string, v: any) => string = defaultJsonTransformer,
): string {
const headers = new Set<string>();
for (const entry of jsonArray) {
for (const k of Object.keys(entry)) {
headers.add(k);
}
}
const headerList = [...headers];
const lines = [];
lines.push(
"|" +
headerList
.map(
(headerName) => headerName,
)
.join("|") +
"|",
);
lines.push(
"|" +
headerList
.map(() => "--")
.join("|") +
"|",
);
for (const val of jsonArray) {
const el = [];
for (const prop of headerList) {
const s = valueTransformer(prop, val[prop]);
el.push(s);
}
lines.push("|" + el.join("|") + "|");
}
return lines.join("\n");
}
export async function renderQueryTemplate(
pageMeta: PageMeta,
templatePage: string,
data: any[],
renderAll: boolean,
): Promise<string> {
let templateText = await space.readPage(templatePage);
templateText = await cleanTemplate(templateText);
if (!renderAll) {
templateText = `{{#each .}}\n${templateText}\n{{/each}}`;
}
return handlebars.renderTemplate(templateText, data, { page: pageMeta });
}