From 848211120c2d8fc0b9c81c0c1e5c8bff3b3bcb8b Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Thu, 11 Jan 2024 13:20:50 +0100 Subject: [PATCH] Tags redo (#624) Introduction of `tag` and `itags` --- common/space_index.ts | 30 +++++++++++++ plug-api/lib/cheap_yaml.ts | 3 +- plug-api/lib/frontmatter.ts | 72 ++++++++++++++++++++----------- plug-api/lib/tags.ts | 14 ++++++ plug-api/types.ts | 4 +- plugs/editor/complete.ts | 2 +- plugs/index/anchor.ts | 2 +- plugs/index/api.ts | 16 ++++--- plugs/index/attributes.ts | 6 +-- plugs/index/builtins.ts | 22 ++++++---- plugs/index/data.ts | 13 ++++-- plugs/index/index.plug.yaml | 12 +++--- plugs/index/item.ts | 15 ++++--- plugs/index/lint.ts | 2 +- plugs/index/page.ts | 20 +++------ plugs/index/page_links.ts | 21 +++++---- plugs/index/paragraph.ts | 47 +++++++++++++------- plugs/index/tags.ts | 4 +- plugs/index/toc.ts | 15 ++++--- plugs/tasks/task.ts | 17 +++++--- plugs/tasks/tasks.plug.yaml | 6 +-- plugs/template/template.ts | 6 +-- plugs/template/util.ts | 2 +- server/server_system.ts | 18 +++----- web/client.ts | 26 +++++++++-- web/components/page_navigator.tsx | 6 +-- web/space.ts | 2 +- web/styles/editor.scss | 2 - website/CHANGELOG.md | 5 +++ website/Objects.md | 43 ++++++++++-------- website/Table of Contents.md | 3 +- 31 files changed, 286 insertions(+), 170 deletions(-) create mode 100644 common/space_index.ts create mode 100644 plug-api/lib/tags.ts diff --git a/common/space_index.ts b/common/space_index.ts new file mode 100644 index 0000000..233d9d2 --- /dev/null +++ b/common/space_index.ts @@ -0,0 +1,30 @@ +import { DataStore } from "../plugos/lib/datastore.ts"; +import { System } from "../plugos/system.ts"; + +const indexVersionKey = ["$indexVersion"]; + +// Bump this one every time a full reinxex is needed +const desiredIndexVersion = 2; + +let indexOngoing = false; + +export async function ensureSpaceIndex(ds: DataStore, system: System) { + const currentIndexVersion = await ds.get(indexVersionKey); + + console.info("Current space index version", currentIndexVersion); + + if (currentIndexVersion !== desiredIndexVersion && !indexOngoing) { + console.info("Performing a full space reindex, this could take a while..."); + indexOngoing = true; + await system.loadedPlugs.get("index")!.invoke("reindexSpace", []); + console.info("Full space index complete."); + await markFullSpaceIndexComplete(ds); + indexOngoing = false; + } else { + console.info("Space index is up to date"); + } +} + +export async function markFullSpaceIndexComplete(ds: DataStore) { + await ds.set(indexVersionKey, desiredIndexVersion); +} diff --git a/plug-api/lib/cheap_yaml.ts b/plug-api/lib/cheap_yaml.ts index 547ee3e..3350072 100644 --- a/plug-api/lib/cheap_yaml.ts +++ b/plug-api/lib/cheap_yaml.ts @@ -1,5 +1,6 @@ const yamlKvRegex = /^\s*(\w+):\s*["']?([^'"]*)["']?$/; const yamlListItemRegex = /^\s*-\s+["']?([^'"]+)["']?$/; +const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/; /** * Cheap YAML parser to determine tags (ugly, regex based but fast) @@ -35,8 +36,6 @@ export function determineTags(yamlText: string): string[] { return tags; } -const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/; - /** * Quick and dirty way to check if a page is a template or not * @param pageText diff --git a/plug-api/lib/frontmatter.ts b/plug-api/lib/frontmatter.ts index ecb1dee..722464b 100644 --- a/plug-api/lib/frontmatter.ts +++ b/plug-api/lib/frontmatter.ts @@ -2,14 +2,13 @@ import { YAML } from "$sb/plugos-syscall/mod.ts"; import { addParentPointers, - collectNodesOfType, ParseTree, renderToText, replaceNodesMatchingAsync, traverseTreeAsync, } from "$sb/lib/tree.ts"; -export type FrontMatter = { tags: string[] } & Record; +export type FrontMatter = { tags?: string[] } & Record; export type FrontmatterExtractOptions = { removeKeys?: string[]; @@ -17,8 +16,11 @@ export type FrontmatterExtractOptions = { removeFrontmatterSection?: boolean; }; -// Extracts front matter from a markdown document -// optionally removes certain keys from the front matter +/** + * Extracts front matter from a markdown document, as well as extracting tags that are to apply to the page + * optionally removes certain keys from the front matter + * Side effect: will add parent pointers + */ export async function extractFrontmatter( tree: ParseTree, options: FrontmatterExtractOptions = {}, @@ -26,29 +28,44 @@ export async function extractFrontmatter( let data: FrontMatter = { tags: [], }; + const tags: string[] = []; addParentPointers(tree); - let paragraphCounter = 0; await replaceNodesMatchingAsync(tree, async (t) => { - // Find tags in the first paragraph to attach to the page - if (t.type === "Paragraph") { - paragraphCounter++; - // Only attach hashtags in the first paragraph to the page - if (paragraphCounter !== 1) { - return; + // Find tags in paragraphs directly nested under the document where the only content is tags + if (t.type === "Paragraph" && t.parent?.type === "Document") { + let onlyTags = true; + const collectedTags = new Set(); + for (const child of t.children!) { + if (child.text) { + if (child.text.startsWith("\n") && child.text !== "\n") { + // Multi line paragraph, cut it off here + break; + } + if (child.text.trim()) { + // Text node with actual text (not just whitespace): not a page tag line! + onlyTags = false; + break; + } + } else if (child.type === "Hashtag") { + const tagname = child.children![0].text!.substring(1); + collectedTags.add(tagname); + + if ( + options.removeTags === true || options.removeTags?.includes(tagname) + ) { + // Ugly hack to remove the hashtag + child.children![0].text = ""; + } + } else if (child.type) { + // Found something else than tags, so... nope! + onlyTags = false; + break; + } + } + if (onlyTags) { + tags.push(...collectedTags); } - collectNodesOfType(t, "Hashtag").forEach((h) => { - const tagname = h.children![0].text!.substring(1); - if (!data.tags.includes(tagname)) { - data.tags.push(tagname); - } - if ( - options.removeTags === true || options.removeTags?.includes(tagname) - ) { - // Ugly hack to remove the hashtag - h.children![0].text = ""; - } - }); } // Find FrontMatter and parse it if (t.type === "FrontMatter") { @@ -65,11 +82,9 @@ export async function extractFrontmatter( // Normalize tags to an array // support "tag1, tag2" as well as "tag1 tag2" as well as "#tag1 #tag2" notations if (typeof data.tags === "string") { - data.tags = (data.tags as string).split(/,\s*|\s+/); + tags.push(...(data.tags as string).split(/,\s*|\s+/)); } - // Strip # from tags - data.tags = data.tags.map((t) => t.replace(/^#/, "")); if (options.removeKeys && options.removeKeys.length > 0) { let removedOne = false; @@ -97,6 +112,11 @@ export async function extractFrontmatter( return undefined; }); + // Strip # from tags + data.tags = [...new Set([...tags.map((t) => t.replace(/^#/, ""))])]; + + // console.log("Extracted tags", data.tags); + return data; } diff --git a/plug-api/lib/tags.ts b/plug-api/lib/tags.ts new file mode 100644 index 0000000..f0358b8 --- /dev/null +++ b/plug-api/lib/tags.ts @@ -0,0 +1,14 @@ +import { FrontMatter } from "$sb/lib/frontmatter.ts"; +import { ObjectValue } from "$sb/types.ts"; + +export function updateITags(obj: ObjectValue, frontmatter: FrontMatter) { + const itags = [obj.tag, ...frontmatter.tags || []]; + if (obj.tags) { + for (const tag of obj.tags) { + if (!itags.includes(tag)) { + itags.push(tag); + } + } + } + obj.itags = itags; +} diff --git a/plug-api/types.ts b/plug-api/types.ts index 2c0b93d..0786cde 100644 --- a/plug-api/types.ts +++ b/plug-api/types.ts @@ -118,7 +118,9 @@ export type FunctionMap = Record any>; */ export type ObjectValue = { ref: string; - tags: string[]; + tag: string; // main tag + tags?: string[]; + itags?: string[]; // implicit or inherited tags (inherited from the page for instance) } & T; export type ObjectQuery = Omit; diff --git a/plugs/editor/complete.ts b/plugs/editor/complete.ts index 04f9c70..41f8c94 100644 --- a/plugs/editor/complete.ts +++ b/plugs/editor/complete.ts @@ -83,7 +83,7 @@ function fileMetaToPageMeta(fileMeta: FileMeta): PageMeta { return { ...fileMeta, ref: fileMeta.name, - tags: ["page"], + tag: "page", name, created: new Date(fileMeta.created).toISOString(), lastModified: new Date(fileMeta.lastModified).toISOString(), diff --git a/plugs/index/anchor.ts b/plugs/index/anchor.ts index 78c6785..c28ae07 100644 --- a/plugs/index/anchor.ts +++ b/plugs/index/anchor.ts @@ -16,7 +16,7 @@ export async function indexAnchors({ name: pageName, tree }: IndexTreeEvent) { const aName = n.children![0].text!.substring(1); anchors.push({ ref: `${pageName}$${aName}`, - tags: ["anchor"], + tag: "anchor", name: aName, page: pageName, pos: n.from!, diff --git a/plugs/index/api.ts b/plugs/index/api.ts index 6aee939..59dc80a 100644 --- a/plugs/index/api.ts +++ b/plugs/index/api.ts @@ -71,7 +71,13 @@ export async function indexObjects( const kvs: KV[] = []; const allAttributes = new Map(); // tag:name -> attributeType for (const obj of objects) { - for (const tag of obj.tags) { + if (!obj.tag) { + console.error("Object has no tag", obj, "this shouldn't happen"); + continue; + } + // Index as all the tag + any additional tags specified + const allTags = [obj.tag, ...obj.tags || []]; + for (const tag of allTags) { // The object itself kvs.push({ key: [tag, cleanKey(obj.ref, page)], @@ -91,7 +97,7 @@ export async function indexObjects( } // Check for all tags attached to this object if they're builtins // If so: if `attrName` is defined in the builtin, use the attributeType from there (mostly to preserve readOnly aspects) - for (const otherTag of obj.tags) { + for (const otherTag of allTags) { const builtinAttributes = builtins[otherTag]; if (builtinAttributes && builtinAttributes[attrName]) { allAttributes.set( @@ -124,14 +130,14 @@ export async function indexObjects( await indexObjects( page, [...allAttributes].map(([key, value]) => { - const [tag, name] = key.split(":"); + const [tagName, name] = key.split(":"); const attributeType = value.startsWith("!") ? value.substring(1) : value; return { ref: key, - tags: ["attribute"], - tag, + tag: "attribute", + tagName, name, attributeType, readOnly: value.startsWith("!"), diff --git a/plugs/index/attributes.ts b/plugs/index/attributes.ts index aa85c10..bb59f93 100644 --- a/plugs/index/attributes.ts +++ b/plugs/index/attributes.ts @@ -7,7 +7,7 @@ import { determineTags } from "../../plug-api/lib/cheap_yaml.ts"; export type AttributeObject = ObjectValue<{ name: string; attributeType: string; - tag: string; + tagName: string; page: string; readOnly: boolean; }>; @@ -49,7 +49,7 @@ export async function objectAttributeCompleter( const attributeFilter: QueryExpression | undefined = attributeCompleteEvent.source === "" ? prefixFilter - : ["and", prefixFilter, ["=", ["attr", "tag"], [ + : ["and", prefixFilter, ["=", ["attr", "tagName"], [ "string", attributeCompleteEvent.source, ]]]; @@ -63,7 +63,7 @@ export async function objectAttributeCompleter( return allAttributes.map((value) => { return { name: value.name, - source: value.tag, + source: value.tagName, attributeType: value.attributeType, readOnly: value.readOnly, } as AttributeCompletion; diff --git a/plugs/index/builtins.ts b/plugs/index/builtins.ts index d6ee569..bda714d 100644 --- a/plugs/index/builtins.ts +++ b/plugs/index/builtins.ts @@ -29,6 +29,12 @@ export const builtins: Record> = { pos: "!number", tags: "string[]", }, + item: { + ref: "!string", + name: "!string", + page: "!string", + tags: "string[]", + }, taskstate: { ref: "!string", tags: "!string[]", @@ -46,7 +52,7 @@ export const builtins: Record> = { ref: "!string", name: "!string", attributeType: "!string", - type: "!string", + tagName: "!string", page: "!string", readOnly: "!boolean", }, @@ -84,11 +90,11 @@ export const builtins: Record> = { export async function loadBuiltinsIntoIndex() { console.log("Loading builtins attributes into index"); const allTags: ObjectValue[] = []; - for (const [tag, attributes] of Object.entries(builtins)) { + for (const [tagName, attributes] of Object.entries(builtins)) { allTags.push({ - ref: tag, - tags: ["tag"], - name: tag, + ref: tagName, + tag: "tag", + name: tagName, page: builtinPseudoPage, parent: "builtin", }); @@ -96,9 +102,9 @@ export async function loadBuiltinsIntoIndex() { builtinPseudoPage, Object.entries(attributes).map(([name, attributeType]) => { return { - ref: `${tag}:${name}`, - tags: ["attribute"], - tag, + ref: `${tagName}:${name}`, + tag: "attribute", + tagName, name, attributeType: attributeType.startsWith("!") ? attributeType.substring(1) diff --git a/plugs/index/data.ts b/plugs/index/data.ts index ab9e647..56d397e 100644 --- a/plugs/index/data.ts +++ b/plugs/index/data.ts @@ -4,6 +4,8 @@ import { collectNodesOfType, findNodeOfType } from "$sb/lib/tree.ts"; import { ObjectValue } from "$sb/types.ts"; import { indexObjects } from "./api.ts"; import { TagObject } from "./tags.ts"; +import { extractFrontmatter } from "$sb/lib/frontmatter.ts"; +import { updateITags } from "$sb/lib/tags.ts"; type DataObject = ObjectValue< { @@ -14,6 +16,7 @@ type DataObject = ObjectValue< export async function indexData({ name, tree }: IndexTreeEvent) { const dataObjects: ObjectValue[] = []; + const frontmatter = await extractFrontmatter(tree); await Promise.all( collectNodesOfType(tree, "FencedCode").map(async (t) => { @@ -41,19 +44,21 @@ export async function indexData({ name, tree }: IndexTreeEvent) { continue; } const pos = t.from! + i; - dataObjects.push({ + const dataObj = { ref: `${name}@${pos}`, - tags: [dataType], + tag: dataType, ...doc, pos, page: name, - }); + }; + updateITags(dataObj, frontmatter); + dataObjects.push(dataObj); } // console.log("Parsed data", parsedData); await indexObjects(name, [ { ref: dataType, - tags: ["tag"], + tag: "tag", name: dataType, page: name, parent: "data", diff --git a/plugs/index/index.plug.yaml b/plugs/index/index.plug.yaml index 2c05f16..d1d8c50 100644 --- a/plugs/index/index.plug.yaml +++ b/plugs/index/index.plug.yaml @@ -87,13 +87,13 @@ functions: indexParagraphs: path: "./paragraph.ts:indexParagraphs" events: - - page:index + - page:index # Backlinks indexLinks: path: "./page_links.ts:indexLinks" events: - - page:index + - page:index attributeComplete: path: "./attributes.ts:attributeComplete" @@ -109,13 +109,13 @@ functions: indexItem: path: "./item.ts:indexItems" events: - - page:index + - page:index # Anchors indexAnchors: path: "./anchor.ts:indexAnchors" events: - - page:index + - page:index anchorComplete: path: "./anchor.ts:anchorComplete" events: @@ -125,13 +125,13 @@ functions: indexData: path: data.ts:indexData events: - - page:index + - page:index # Hashtags indexTags: path: tags.ts:indexTags events: - - page:index + - page:index tagComplete: path: tags.ts:tagComplete events: diff --git a/plugs/index/item.ts b/plugs/index/item.ts index cb418c8..8abc05f 100644 --- a/plugs/index/item.ts +++ b/plugs/index/item.ts @@ -5,6 +5,8 @@ import { extractAttributes } from "$sb/lib/attribute.ts"; import { rewritePageRefs } from "$sb/lib/resolve.ts"; import { ObjectValue } from "$sb/types.ts"; import { indexObjects } from "./api.ts"; +import { updateITags } from "$sb/lib/tags.ts"; +import { extractFrontmatter } from "$sb/lib/frontmatter.ts"; export type ItemObject = ObjectValue< { @@ -17,7 +19,7 @@ export type ItemObject = ObjectValue< export async function indexItems({ name, tree }: IndexTreeEvent) { const items: ObjectValue[] = []; - // console.log("Indexing items", name); + const frontmatter = await extractFrontmatter(tree); const coll = collectNodesOfType(tree, "ListItem"); @@ -30,11 +32,10 @@ export async function indexItems({ name, tree }: IndexTreeEvent) { continue; } - const tags = new Set(["item"]); - + const tags = new Set(); const item: ItemObject = { ref: `${name}@${n.from}`, - tags: [], + tag: "item", name: "", // to be replaced page: name, pos: n.from!, @@ -62,7 +63,11 @@ export async function indexItems({ name, tree }: IndexTreeEvent) { } item.name = textNodes.map(renderToText).join("").trim(); - item.tags = [...tags.values()]; + if (tags.size > 0) { + item.tags = [...tags]; + } + + updateITags(item, frontmatter); items.push(item); } diff --git a/plugs/index/lint.ts b/plugs/index/lint.ts index 99ff3f0..82179f9 100644 --- a/plugs/index/lint.ts +++ b/plugs/index/lint.ts @@ -16,7 +16,7 @@ export async function lintYAML({ tree }: LintEvent): Promise { const tags = ["page", ...frontmatter.tags || []]; // Query all readOnly attributes for pages with this tag set const readOnlyAttributes = await queryObjects("attribute", { - filter: ["and", ["=", ["attr", "tag"], [ + filter: ["and", ["=", ["attr", "tagName"], [ "array", tags.map((tag): QueryExpression => ["string", tag]), ]], [ diff --git a/plugs/index/page.ts b/plugs/index/page.ts index bee82ac..9175731 100644 --- a/plugs/index/page.ts +++ b/plugs/index/page.ts @@ -10,6 +10,7 @@ import { renderToText, traverseTreeAsync, } from "$sb/lib/tree.ts"; +import { updateITags } from "$sb/lib/tags.ts"; export async function indexPage({ name, tree }: IndexTreeEvent) { if (name.startsWith("_")) { @@ -24,7 +25,7 @@ export async function indexPage({ name, tree }: IndexTreeEvent) { // Note the order here, making sure that the actual page meta data overrules // any attempt to manually set built-in attributes like 'name' or 'lastModified' // pageMeta appears at the beginning and the end due to the ordering behavior of ojects in JS (making builtin attributes appear first) - const combinedPageMeta = { + const combinedPageMeta: PageMeta = { ...pageMeta, ...frontmatter, ...toplevelAttributes, @@ -33,18 +34,16 @@ export async function indexPage({ name, tree }: IndexTreeEvent) { combinedPageMeta.tags = [ ...new Set([ - "page", ...frontmatter.tags || [], ...toplevelAttributes.tags || [], ]), ]; - // if (pageMeta.tags.includes("template")) { - // // If this is a template, we don't want to index it as a page or anything else, just a template - // pageMeta.tags = ["template"]; - // } + combinedPageMeta.tag = "page"; - // console.log("Page object", pageObj); + updateITags(combinedPageMeta, frontmatter); + + // console.log("Page object", combinedPageMeta); await indexObjects(name, [combinedPageMeta]); } @@ -109,13 +108,6 @@ async function lintYaml( const errorMatch = errorRegex.exec(e.message); if (errorMatch) { console.log("YAML error", e.message); - // const line = parseInt(errorMatch[1], 10) - 1; - // const yamlLines = yamlText.split("\n"); - // let pos = posOffset; - // for (let i = 0; i < line; i++) { - // pos += yamlLines[i].length + 1; - // } - // const endPos = pos + yamlLines[line].length; return { from, diff --git a/plugs/index/page_links.ts b/plugs/index/page_links.ts index 0767ad5..96212c4 100644 --- a/plugs/index/page_links.ts +++ b/plugs/index/page_links.ts @@ -3,12 +3,12 @@ import { IndexTreeEvent } from "$sb/app_event.ts"; import { resolvePath } from "$sb/lib/resolve.ts"; import { indexObjects, queryObjects } from "./api.ts"; import { ObjectValue } from "$sb/types.ts"; +import { extractFrontmatter } from "$sb/lib/frontmatter.ts"; +import { updateITags } from "$sb/lib/tags.ts"; const pageRefRegex = /\[\[([^\]]+)\]\]/g; -export type LinkObject = { - ref: string; - tags: string[]; +export type LinkObject = ObjectValue<{ // The page the link points to toPage: string; // The page the link occurs in @@ -17,7 +17,7 @@ export type LinkObject = { snippet: string; alias?: string; asTemplate: boolean; -}; +}>; export function extractSnippet(text: string, pos: number): string { let prefix = ""; @@ -47,7 +47,7 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) { const links: ObjectValue[] = []; // [[Style Links]] // console.log("Now indexing links for", name); - + const frontmatter = await extractFrontmatter(tree); const pageText = renderToText(tree); traverseTree(tree, (n): boolean => { @@ -59,7 +59,7 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) { toPage = toPage.split(/[@$]/)[0]; const link: LinkObject = { ref: `${name}@${pos}`, - tags: ["link"], + tag: "link", toPage: toPage, snippet: extractSnippet(pageText, pos), pos, @@ -69,6 +69,7 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) { if (wikiLinkAlias) { link.alias = wikiLinkAlias.children![0].text!; } + updateITags(link, frontmatter); links.push(link); return true; } @@ -90,15 +91,17 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) { for (const match of matches) { const pageRefName = resolvePath(name, match[1]); const pos = codeText.from! + match.index! + 2; - links.push({ + const link = { ref: `${name}@${pos}`, - tags: ["link"], + tag: "link", toPage: pageRefName, page: name, snippet: extractSnippet(pageText, pos), pos: pos, asTemplate: true, - }); + }; + updateITags(link, frontmatter); + links.push(link); } } } diff --git a/plugs/index/paragraph.ts b/plugs/index/paragraph.ts index 0116e67..35fa4ad 100644 --- a/plugs/index/paragraph.ts +++ b/plugs/index/paragraph.ts @@ -9,6 +9,9 @@ import { } from "$sb/lib/tree.ts"; import { extractAttributes } from "$sb/lib/attribute.ts"; import { ObjectValue } from "$sb/types.ts"; +import a from "https://esm.sh/v135/node_process.js"; +import { updateITags } from "$sb/lib/tags.ts"; +import { extractFrontmatter } from "$sb/lib/frontmatter.ts"; /** ParagraphObject An index object for the top level text nodes */ export type ParagraphObject = ObjectValue< @@ -21,41 +24,53 @@ export type ParagraphObject = ObjectValue< export async function indexParagraphs({ name: page, tree }: IndexTreeEvent) { const objects: ParagraphObject[] = []; - addParentPointers(tree); - let paragraphCounter = 0; + + const frontmatter = await extractFrontmatter(tree); await traverseTreeAsync(tree, async (p) => { if (p.type !== "Paragraph") { return false; } - paragraphCounter++; if (findParentMatching(p, (n) => n.type === "ListItem")) { // Not looking at paragraphs nested in a list return false; } - // So we're looking at indexable a paragraph now - const tags = new Set(["paragraph"]); - if (paragraphCounter > 1) { - // Only attach hashtags to later paragraphs than the first + const attrs = await extractAttributes(p, true); + const tags = new Set(); + const text = renderToText(p); - // tag the paragraph with any hashtags inside it - collectNodesOfType(p, "Hashtag").forEach((tagNode) => { - tags.add(tagNode.children![0].text!.substring(1)); - }); + // So we're looking at indexable a paragraph now + collectNodesOfType(p, "Hashtag").forEach((tagNode) => { + tags.add(tagNode.children![0].text!.substring(1)); + // Hacky way to remove the hashtag + tagNode.children = []; + }); + + const textWithoutTags = renderToText(p); + + if (!textWithoutTags.trim()) { + // Empty paragraph, just tags and attributes maybe + return true; } - const attrs = await extractAttributes(p, false); const pos = p.from!; - objects.push({ + const paragraph: ParagraphObject = { ref: `${page}@${pos}`, - text: renderToText(p), - tags: [...tags.values()], + text, + tag: "paragraph", page, pos, ...attrs, - }); + }; + if (tags.size > 0) { + paragraph.tags = [...tags]; + paragraph.itags = [...tags]; + } + + updateITags(paragraph, frontmatter); + objects.push(paragraph); // stop on every element except document, including paragraphs return true; diff --git a/plugs/index/tags.ts b/plugs/index/tags.ts index 311523e..37e3fa3 100644 --- a/plugs/index/tags.ts +++ b/plugs/index/tags.ts @@ -17,7 +17,7 @@ export type TagObject = ObjectValue<{ export async function indexTags({ name, tree }: IndexTreeEvent) { const tags = new Set(); // name:parent addParentPointers(tree); - const pageTags: string[] = (await extractFrontmatter(tree)).tags; + const pageTags: string[] = (await extractFrontmatter(tree)).tags || []; for (const pageTag of pageTags) { tags.add(`${pageTag}:page`); } @@ -41,7 +41,7 @@ export async function indexTags({ name, tree }: IndexTreeEvent) { const [tagName, parent] = tag.split(":"); return { ref: tag, - tags: ["tag"], + tag: "tag", name: tagName, page: name, parent, diff --git a/plugs/index/toc.ts b/plugs/index/toc.ts index 2007719..8dc7d8c 100644 --- a/plugs/index/toc.ts +++ b/plugs/index/toc.ts @@ -2,8 +2,6 @@ import { editor, markdown, YAML } from "$sb/syscalls.ts"; import { CodeWidgetContent } from "$sb/types.ts"; import { renderToText, traverseTree } from "$sb/lib/tree.ts"; -const defaultHeaderThreshold = 0; - type Header = { name: string; pos: number; @@ -11,7 +9,10 @@ type Header = { }; type TocConfig = { + // Only show the TOC if there are at least this many headers minHeaders?: number; + // Don't show the TOC if there are more than this many headers + maxHeaders?: number; header?: boolean; }; @@ -40,14 +41,14 @@ export async function widget( return false; }); - let headerThreshold = defaultHeaderThreshold; - if (config.minHeaders) { - headerThreshold = config.minHeaders; - } - if (headers.length < headerThreshold) { + if (config.minHeaders && headers.length < config.minHeaders) { // Not enough headers, not showing TOC return null; } + if (config.maxHeaders && headers.length > config.maxHeaders) { + // Too many headers, not showing TOC + return null; + } let headerText = "# Table of Contents\n"; if (config.header === false) { headerText = ""; diff --git a/plugs/tasks/task.ts b/plugs/tasks/task.ts index 8d4dffe..836d602 100644 --- a/plugs/tasks/task.ts +++ b/plugs/tasks/task.ts @@ -18,6 +18,8 @@ import { extractAttributes } from "$sb/lib/attribute.ts"; import { rewritePageRefs } from "$sb/lib/resolve.ts"; import { ObjectValue } from "$sb/types.ts"; import { indexObjects, queryObjects } from "../index/plug_api.ts"; +import { updateITags } from "$sb/lib/tags.ts"; +import { extractFrontmatter } from "$sb/lib/frontmatter.ts"; export type TaskObject = ObjectValue< { @@ -46,9 +48,8 @@ const incompleteStates = [" "]; export async function indexTasks({ name, tree }: IndexTreeEvent) { const tasks: ObjectValue[] = []; const taskStates = new Map(); - addParentPointers(tree); - // const allAttributes: AttributeObject[] = []; - // const allTags = new Set(); + const frontmatter = await extractFrontmatter(tree); + await traverseTreeAsync(tree, async (n) => { if (n.type !== "Task") { return false; @@ -65,7 +66,7 @@ export async function indexTasks({ name, tree }: IndexTreeEvent) { const complete = completeStates.includes(state); const task: TaskObject = { ref: `${name}@${n.from}`, - tags: [], + tag: "task", name: "", done: complete, page: name, @@ -84,10 +85,12 @@ export async function indexTasks({ name, tree }: IndexTreeEvent) { if (tree.type === "Hashtag") { // Push the tag to the list, removing the initial # const tagName = tree.children![0].text!.substring(1); + if (!task.tags) { + task.tags = []; + } task.tags.push(tagName); } }); - task.tags = ["task", ...task.tags]; // Extract attributes and remove from tree const extractedAttributes = await extractAttributes(n, true); @@ -97,6 +100,8 @@ export async function indexTasks({ name, tree }: IndexTreeEvent) { task.name = n.children!.slice(1).map(renderToText).join("").trim(); + updateITags(task, frontmatter); + tasks.push(task); return true; }); @@ -107,7 +112,7 @@ export async function indexTasks({ name, tree }: IndexTreeEvent) { name, Array.from(taskStates.entries()).map(([state, { firstPos, count }]) => ({ ref: `${name}@${firstPos}`, - tags: ["taskstate"], + tag: "taskstate", state, count, page: name, diff --git a/plugs/tasks/tasks.plug.yaml b/plugs/tasks/tasks.plug.yaml index 99a5ea2..cd857fc 100644 --- a/plugs/tasks/tasks.plug.yaml +++ b/plugs/tasks/tasks.plug.yaml @@ -35,15 +35,11 @@ functions: indexTasks: path: "./task.ts:indexTasks" events: - - page:index + - page:index taskToggle: path: "./task.ts:taskToggle" events: - page:click - # itemQueryProvider: - # path: ./task.ts:queryProvider - # events: - # - query:task taskToggleCommand: path: ./task.ts:taskCycleCommand command: diff --git a/plugs/template/template.ts b/plugs/template/template.ts index 7b25e48..c6d3b0b 100644 --- a/plugs/template/template.ts +++ b/plugs/template/template.ts @@ -37,7 +37,7 @@ export async function newPageCommand( const templateText = await space.readPage(templateName!); const tempPageMeta: PageMeta = { - tags: ["page"], + tag: "page", ref: "", name: "", created: "", @@ -169,7 +169,7 @@ export async function dailyNoteCommand() { await space.writePage( pageName, await replaceTemplateVars(dailyNoteTemplateText, { - tags: ["page"], + tag: "page", ref: pageName, name: pageName, created: "", @@ -218,7 +218,7 @@ export async function weeklyNoteCommand() { await replaceTemplateVars(weeklyNoteTemplateText, { name: pageName, ref: pageName, - tags: ["page"], + tag: "page", created: "", lastModified: "", perm: "rw", diff --git a/plugs/template/util.ts b/plugs/template/util.ts index 1855d79..ee287c4 100644 --- a/plugs/template/util.ts +++ b/plugs/template/util.ts @@ -19,7 +19,7 @@ export function defaultJsonTransformer(v: any): string { } if (Array.isArray(v)) { return v.map(defaultJsonTransformer).join(", "); - } else if (typeof v === "object") { + } else if (v && typeof v === "object") { return Object.entries(v).map(([k, v]: [string, any]) => `${k}: ${defaultJsonTransformer(v)}` ).join(", "); diff --git a/server/server_system.ts b/server/server_system.ts index 9b0fcb9..7905a36 100644 --- a/server/server_system.ts +++ b/server/server_system.ts @@ -33,6 +33,7 @@ import { CodeWidgetHook } from "../web/hooks/code_widget.ts"; import { KVPrimitivesManifestCache } from "../plugos/manifest_cache.ts"; import { KvPrimitives } from "../plugos/lib/kv_primitives.ts"; import { ShellBackend } from "./shell_backend.ts"; +import { ensureSpaceIndex } from "../common/space_index.ts"; const fileListInterval = 30 * 1000; // 30s @@ -161,19 +162,10 @@ export class ServerSystem { })().catch(console.error); }); - // Check if this space was ever indexed before - if (!await this.ds.get(["$initialIndexDone"])) { - console.log("Indexing space for the first time (in the background)"); - const indexPromise = this.system.loadedPlugs.get("index")!.invoke( - "reindexSpace", - [], - ).then(() => { - console.log("Initial index completed!"); - this.ds.set(["$initialIndexDone"], true); - }).catch(console.error); - if (awaitIndex) { - await indexPromise; - } + // Ensure a valid index + const indexPromise = ensureSpaceIndex(this.ds, this.system); + if (awaitIndex) { + await indexPromise; } await eventHook.dispatchEvent("system:ready"); diff --git a/web/client.ts b/web/client.ts index 2ca5550..d7d37e3 100644 --- a/web/client.ts +++ b/web/client.ts @@ -47,6 +47,10 @@ import { EncryptedSpacePrimitives, } from "../common/spaces/encrypted_space_primitives.ts"; import { LimitedMap } from "../common/limited_map.ts"; +import { + ensureSpaceIndex, + markFullSpaceIndexComplete, +} from "../common/space_index.ts"; const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/; const autoSaveInterval = 1000; @@ -216,7 +220,7 @@ export class Client { await this.loadPlugs(); this.initNavigator(); - this.initSync(); + await this.initSync(); this.loadCustomStyles().catch(console.error); @@ -235,9 +239,12 @@ export class Client { this.updatePageListCache().catch(console.error); } - private initSync() { + private async initSync() { this.syncService.start(); + // We're still booting, if a initial sync has already been completed we know this is the initial sync + const initialSync = !await this.syncService.hasInitialSyncCompleted(); + this.eventHook.addLocalListener("sync:success", async (operations) => { // console.log("Operations", operations); if (operations > 0) { @@ -247,13 +254,24 @@ export class Client { if (operations !== undefined) { // "sync:success" is called with a number of operations only from syncSpace(), not from syncing individual pages this.fullSyncCompleted = true; + + console.log("Full sync completed"); + + // A full sync just completed + if (!initialSync) { + // If this was NOT the initial sync let's check if we need to perform a space reindex + ensureSpaceIndex(this.stateDataStore, this.system.system).catch( + console.error, + ); + } else { + // This was the initial sync, let's mark a full index as completed + await markFullSpaceIndexComplete(this.stateDataStore); + } } - // if (this.system.plugsUpdated) { if (operations) { // Likely initial sync so let's show visually that we're synced now this.showProgress(100); } - // } this.ui.viewDispatch({ type: "sync-change", syncSuccess: true }); }); diff --git a/web/components/page_navigator.tsx b/web/components/page_navigator.tsx index b8f7d01..f71069b 100644 --- a/web/components/page_navigator.tsx +++ b/web/components/page_navigator.tsx @@ -53,11 +53,9 @@ export function PageNavigator({ if (aliases.length > 0) { description = "(a.k.a. " + aliases.join(", ") + ") "; } - if (pageMeta.tags.length > 1) { - // Every page has the "page" tag, so it only gets interesting beyond that - const interestingTags = pageMeta.tags.filter((tag) => tag !== "page"); + if (pageMeta.tags) { description = (description || "") + - interestingTags.map((tag) => `#${tag}`).join(" "); + pageMeta.tags.map((tag) => `#${tag}`).join(" "); } options.push({ ...pageMeta, diff --git a/web/space.ts b/web/space.ts index 604959c..9275768 100644 --- a/web/space.ts +++ b/web/space.ts @@ -187,7 +187,7 @@ export function fileMetaToPageMeta(fileMeta: FileMeta): PageMeta { return { ...fileMeta, ref: name, - tags: ["page"], + tag: "page", name, created: new Date(fileMeta.created).toISOString(), lastModified: new Date(fileMeta.lastModified).toISOString(), diff --git a/web/styles/editor.scss b/web/styles/editor.scss index f3c4f48..eadf0b2 100644 --- a/web/styles/editor.scss +++ b/web/styles/editor.scss @@ -414,8 +414,6 @@ .sb-markdown-top-widget h1, .sb-markdown-bottom-widget h1 { - border-top-right-radius: 5px; - border-top-left-radius: 5px; margin: 0 0 5px 0; padding: 10px !important; background-color: var(--editor-widget-background-color); diff --git a/website/CHANGELOG.md b/website/CHANGELOG.md index aa69cab..5be8e25 100644 --- a/website/CHANGELOG.md +++ b/website/CHANGELOG.md @@ -6,6 +6,11 @@ release. _Not yet released, this will likely become 0.6.0._ * **Directives have now been removed** from the code base. Please use [[Live Queries]] and [[Live Templates]] instead. If you hadn’t migrated yet and want to auto migrate, downgrade your SilverBullet version to 0.5.11 (e.g. using the `zefhemel/silverbullet:0.5.11` docker image) and run the {[Directive: Convert Entire Space to Live/Templates]} command with that version. +* (Hopefully subtle) **breaking change** in how tags work (see [[Objects]]): + * Every object now has a `tag` attribute, signifying the “main” tag for that object (e.g. `page`, `item`) + * The `tags` attribute will now _only_ contain explicitly assigned tags (so not the built-in tag, which moved to `tag`) + * The new `itags` attribute (available in many objects) includes both the `tag`, `tags` as well as any tags inherited from the page the object appears in. + * Page tags now no longer need to appear at the top of the page, but can appear anywhere as long as they are the only thing appearing in a paragraph with no additional text, see [[Objects$page]]. * New [[Markdown/Code Widgets|Code Widget]]: `toc` to manually include a [[Table of Contents]] * New template type: [[Live Template Widgets]] allowing you to automatically add templates to the top or bottom of your pages (based on some criteria). Using this feature it possible to implement [[Table of Contents]] and [[Linked Mentions]] without having “hard coded” into SilverBullet itself. * **“Breaking” change:** Two features are now no longer hardcoded into SilverBullet, but can be activated quite easily using [[Live Template Widgets]] (see their respective documentation pages on instructions on how to do this): diff --git a/website/Objects.md b/website/Objects.md index 54e780b..8e023e2 100644 --- a/website/Objects.md +++ b/website/Objects.md @@ -1,33 +1,31 @@ -#core - SilverBullet automatically builds and maintains an index of _objects_ extracted from all markdown pages in your space. It subsequently allows you to [[Live Queries|query]] this database in (potentially) useful ways. -Some examples of things you can query for: -* Give me a list of all books that I have marked as _want to read_ -* Give me a list of all tasks not yet completed that have today as a due date -* Give me a list of items tagged with `#quote` -* Give me a list of not-completed tasks that reference the current page - -By design, the truth remains in the markdown: all data indexed as objects will have a representation in markdown text as well. The index can be flushed at any time and be rebuilt from its source markdown files kept in your space. +By design, the truth remains in the markdown: all data indexed as objects will have a representation in markdown text as well. This index can be flushed at any time and be rebuilt from its source markdown files kept in your space (and you can do so on demand if you like using the {[Space: Reindex]} command). # Object representation -Every object has a set of [[Attributes]]. +Every object has a set of [[Attributes]], some predefined, but you can add any additional custom attributes that you like. -At the very least: -* `ref`: a unique _identifier_ (unique to the page, at least), often represented as a pointer to the place (page, position) in your space where the object is defined. For instance, a _page_ object will use the page name as its `ref` attribute, and a `task` will use `page@pos` (where `pos` is the location the task appears in `page`). -* `tags`: an array of type(s) of an object, see [[$tags]]. +The following attributes are predefined, and you can expect all objects to have them: +* `ref`: a globally unique _identifier_, often represented as a pointer to the place (page, position) in your space where the object is defined. For instance, a _page_ object will use the page name as its `ref` attribute, and a `task` will use `page@pos` (where `pos` is the location the task appears in `page`). +* `tag`: the main type, or “tag” of the page, usually a built-in type of the object (see below). -In addition, any number of additional tag-specific and custom [[Attributes]] can be defined (see below). +In addition, many objects will also contain: +* `tags`: an optional set of additional, explicitly assigned tags. +* `itags`: a set of _implicit_ or _inherited_ tags: including the object’s `tag`, `tags` as well as any tags _assigned to its containing page_. This is useful to answer queries like, “give me all tasks on pages where that page is tagged with `person`“, which would be expressed as `task where itags = "person"` (although technically that would also match any tags that have the `#person` explicitly assigned). + +Beside these, any number of additional tag-specific and custom [[Attributes]] can be defined (see below). # Tags $tags -Every object has one or more tags, defining the _types_ of an object. Some tags are built-in (as described below), but you can easily define new tags by simply using the #hashtag notation in strategic locations (more on these locations later). +Every object has a main `tag`, which signifies the type of object being described. In addition, any number of additional tags can be assigned as well via the `tags` attribute. You can use either the main `tag` or any of the `tags` as query sources in [[Live Queries]] — examples below. Here are the currently built-in tags: ## page $page -Every page in your space is available via the `page` tag. You can attach _additional tags_ to a page, by either specifying them in the `tags` attribute [[Frontmatter]], or by putting additional [[Tags]] in the _first paragraph of your page_, as is done with the #core tag at the beginning of this page. +Every page in your space is available via the `page` tag. You can attach _additional_ tags to a page, by either specifying them in the `tags` attribute [[Frontmatter]], or by putting additional [[Tags]] in a stand alone paragraph with no other (textual) content in them, e.g.: + +#example-tag #another-tag In addition to `ref` and `tags`, the `page` tag defines a bunch of additional attributes as can be seen in this example query: @@ -35,6 +33,12 @@ In addition to `ref` and `tags`, the `page` tag defines a bunch of additional at page where name = "{{@page.name}}" ``` +Note that you can also query this page using the `example-tag` directly: + +```query +example-tag +``` + ## task $task Every task in your space is tagged with the `task` tag by default. You tag it with additional tags by using [[Tags]] in the task name, e.g. @@ -48,6 +52,7 @@ The following query shows all attributes available for tasks: ```query upnext ``` + Although you may want to render it using a template such as [[template/task]] instead: ```query @@ -71,13 +76,13 @@ $template Indexes all pages tagged with `#template`. See [[Templates]] for more information on templates. ```query -template select name +template select name limit 5 ``` ## item $item -List items (both bullet point and numbered items) are indexed by default with the `item` tag, and additional tags can be added using [[Tags]]. +List items (both bullet point and numbered items) are indexed with the `item` tag, and additional tags can be added using [[Tags]]. Here is an example of a #quote item using a custom [[Attributes|attribute]]: @@ -86,7 +91,7 @@ Here is an example of a #quote item using a custom [[Attributes|attribute]]: And then queried via the #quote tag: ```query -quote where tags = "item" select name, by +quote where page = "{{@page.name}}" and tag = "item" select name, by ``` ## paragraph diff --git a/website/Table of Contents.md b/website/Table of Contents.md index f5196ec..3a4c039 100644 --- a/website/Table of Contents.md +++ b/website/Table of Contents.md @@ -21,7 +21,8 @@ federation: In the body of the `toc` code widget you can configure a few options: * `header`: by default a “Table of Contents” header is added to the ToC, set this to `false` to disable rendering this header -* `minHeaders`: only renders a ToC if the number of headers in the current page exceeds this number, otherwise render an empty widget +* `minHeaders`: only renders a ToC if the number of headers in the current page exceeds this number, otherwise renders an empty widget +* `maxHeaders`: only renders a ToC if the number of headers in the current page is below this number, otherwise renders an empty widget Example: ```toc