neil/silverbullet
1
0
Fork 0
Code Activity

SilverBullet pivot to become an offline-first PWA (#403)

Browse Source
This commit is contained in:
Zef Hemel
2023-05-23 20:53:53 +02:00
committed by GitHub
parent b256269897
commit 5f484bed57
389 changed files with 4484 additions and 291129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugs/builtin_plugs.ts
+11
View File
@@ -0,0 +1,11 @@
// TODO: Figure out how to keep this up-to-date automatically
export const builtinPlugNames = [
"collab",
"core",
"directive",
"emoji",
"markdown",
"share",
"tasks",
"search",
];
plugs/collab/collab.plug.yaml
-5
View File
@@ -1,6 +1,4 @@
name: collab
imports:
- https://get.silverbullet.md/global.plug.json
functions:
detectCollabPage:
path: "./collab.ts:detectPage"
@@ -23,19 +21,16 @@ functions:
# Space extension
readPageCollab:
path: ./collab.ts:readFileCollab
env: client
pageNamespace:
pattern: "collab:.+"
operation: readFile
writePageCollab:
path: ./collab.ts:writeFileCollab
env: client
pageNamespace:
pattern: "collab:.+"
operation: writeFile
getPageMetaCollab:
path: ./collab.ts:getFileMetaCollab
env: client
pageNamespace:
pattern: "collab:.+"
operation: getFileMeta
plugs/collab/collab.ts
+11 -21
View File
@@ -9,26 +9,17 @@ import {
extractFrontmatter,
prepareFrontmatterDispatch,
} from "$sb/lib/frontmatter.ts";
import * as YAML from "yaml";
import {
clientStore,
collab,
editor,
markdown,
} from "$sb/silverbullet-syscall/mod.ts";
import { store, YAML } from "$sb/plugos-syscall/mod.ts";
import { collab, editor, markdown } from "$sb/silverbullet-syscall/mod.ts";
import { nanoid } from "https://esm.sh/nanoid@4.0.0";
import {
FileData,
FileEncoding,
} from "../../common/spaces/space_primitives.ts";
import { FileMeta } from "../../common/types.ts";
import { base64EncodedDataUrl } from "../../plugos/asset_bundle/base64.ts";
const defaultServer = "wss://collab.silverbullet.md";
async function ensureUsername(): Promise<string> {
let username = await clientStore.get("collabUsername");
let username = await store.get("collabUsername");
if (!username) {
username = await editor.prompt(
"Please enter a publicly visible user name (or cancel for 'anonymous'):",
@@ -36,7 +27,7 @@ async function ensureUsername(): Promise<string> {
if (!username) {
return "anonymous";
} else {
await clientStore.set("collabUsername", username);
await store.set("collabUsername", username);
}
}
return username;
@@ -67,7 +58,7 @@ export async function shareCommand() {
await editor.save();
const text = await editor.getText();
const tree = await markdown.parseMarkdown(text);
let { $share } = extractFrontmatter(tree);
let { $share } = await extractFrontmatter(tree);
if (!$share) {
$share = [];
}
@@ -76,7 +67,7 @@ export async function shareCommand() {
}
removeParentPointers(tree);
const dispatchData = prepareFrontmatterDispatch(tree, {
const dispatchData = await prepareFrontmatterDispatch(tree, {
$share: [...$share, `collab:${serverUrl}/${roomId}`],
});
@@ -95,7 +86,7 @@ export async function detectPage() {
if (frontMatter) {
const yamlText = renderToText(frontMatter.children![1].children![0]);
try {
let { $share } = YAML.parse(yamlText) as any;
let { $share } = await YAML.parse(yamlText) as any;
if (!$share) {
return;
}
@@ -125,19 +116,18 @@ export function shareNoop() {
return true;
}
export async function readFileCollab(
export function readFileCollab(
name: string,
encoding: FileEncoding,
): Promise<{ data: FileData; meta: FileMeta }> {
): { data: string; meta: FileMeta } {
if (!name.endsWith(".md")) {
throw new Error("File not found");
throw new Error("Not found");
}
const collabUri = name.substring(0, name.length - ".md".length);
const text = `---\n$share: ${collabUri}\n---\n`;
return {
// encoding === "arraybuffer" is not an option, so either it's "utf8" or "dataurl"
data: encoding === "utf8" ? text : base64EncodedDataUrl(
data: base64EncodedDataUrl(
"text/markdown",
new TextEncoder().encode(text),
),
plugs/core/cloud.ts
+9 -7
View File
@@ -1,7 +1,3 @@
import type {
FileData,
FileEncoding,
} from "../../common/spaces/space_primitives.ts";
import { renderToText, replaceNodesMatching } from "$sb/lib/tree.ts";
import type { FileMeta } from "../../common/types.ts";
import { parseMarkdown } from "$sb/silverbullet-syscall/markdown.ts";
@@ -11,8 +7,7 @@ export const cloudPrefix = "💭 ";
export async function readFileCloud(
name: string,
encoding: FileEncoding,
): Promise<{ data: FileData; meta: FileMeta } | undefined> {
): Promise<{ data: string; meta: FileMeta } | undefined> {
const originalUrl = name.substring(
cloudPrefix.length,
name.length - ".md".length,
@@ -43,7 +38,7 @@ export async function readFileCloud(
`${cloudPrefix}${originalUrl.split("/")[0]}/`,
);
return {
data: encoding === "utf8" ? text : base64EncodedDataUrl(
data: base64EncodedDataUrl(
"text/markdown",
new TextEncoder().encode(text),
),
@@ -57,6 +52,13 @@ export async function readFileCloud(
};
}
export function writeFileCloud(
name: string,
): Promise<FileMeta> {
console.log("Writing cloud file", name);
return getFileMetaCloud(name);
}
async function translateLinksWithPrefix(
text: string,
prefix: string,
plugs/core/core.plug.yaml
+8 -63
View File
@@ -1,6 +1,6 @@
name: core
imports:
- https://get.silverbullet.md/global.plug.json
requiredPermissions:
- fetch
syntax:
Hashtag:
firstCharacters:
@@ -26,10 +26,8 @@ functions:
path: "./editor.ts:toggleDarkMode"
command:
name: "Editor: Toggle Dark Mode"
clearPageIndex:
path: "./page.ts:clearPageIndex"
env: server
events:
- page:saved
- page:deleted
@@ -45,9 +43,6 @@ functions:
path: "./page.ts:reindexCommand"
command:
name: "Space: Reindex"
reindexSpace:
path: "./page.ts:reindexSpace"
env: server
deletePage:
path: "./page.ts:deletePage"
command:
@@ -138,40 +133,7 @@ functions:
path: "./anchor.ts:anchorComplete"
events:
- editor:complete
# Full text search
searchIndex:
path: ./search.ts:pageIndex
events:
- page:index
searchUnindex:
path: "./search.ts:pageUnindex"
env: server
events:
- page:deleted
searchQueryProvider:
path: ./search.ts:queryProvider
events:
- query:full-text
searchCommand:
path: ./search.ts:searchCommand
command:
name: "Search Space"
key: Ctrl-Shift-f
mac: Cmd-Shift-f
readPageSearch:
path: ./search.ts:readFileSearch
env: server
pageNamespace:
pattern: "🔍 .+"
operation: readFile
getPageMetaSearch:
path: ./search.ts:getFileMetaSearch
env: server
pageNamespace:
pattern: "🔍 .+"
operation: getFileMeta
# Template commands
insertTemplateText:
path: "./template.ts:insertTemplateText"
@@ -342,9 +304,6 @@ functions:
name: "Plugs: Update"
key: "Ctrl-Shift-p"
mac: "Cmd-Shift-p"
updatePlugs:
path: ./plugmanager.ts:updatePlugs
env: server
getPlugHTTPS:
path: "./plugmanager.ts:getPlugHTTPS"
events:
@@ -367,20 +326,6 @@ functions:
path: ./debug.ts:parsePageCommand
command:
name: "Debug: Parse Document"
showLogsCommand:
path: ./debug.ts:showLogsCommand
command:
name: "Show Logs"
key: "Ctrl-Alt-l"
mac: "Cmd-Alt-l"
hideBhsCommand:
path: ./debug.ts:hideBhsCommand
command:
name: "UI: Hide BHS"
key: "Ctrl-Alt-b"
mac: "Cmd-Alt-b"
events:
- log:hide
# Link unfurl infrastructure
unfurlLink:
@@ -391,9 +336,6 @@ functions:
mac: "Cmd-Shift-u"
contexts:
- NakedURL
unfurlExec:
env: server
path: ./link.ts:unfurlExec
# Title-based link unfurl
titleUnfurlOptions:
@@ -418,13 +360,16 @@ functions:
# Cloud pages
readPageCloud:
path: ./cloud.ts:readFileCloud
env: server
pageNamespace:
pattern: "💭 .+"
operation: readFile
writePageCloud:
path: ./cloud.ts:writeFileCloud
pageNamespace:
pattern: "💭 .+"
operation: writeFile
getPageMetaCloud:
path: ./cloud.ts:getFileMetaCloud
env: server
pageNamespace:
pattern: "💭 .+"
operation: getFileMeta
plugs/core/debug.ts
-80
View File
@@ -10,83 +10,3 @@ export async function parsePageCommand() {
),
);
}
export async function showLogsCommand() {
await editor.showPanel(
"bhs",
1,
`
<style>
#close {
width: 100%;
}
#client-log-header {
position: absolute;
left: 0;
top: 35px;
}
#server-log-header {
position: absolute;
right: 0;
top: 35px;
width: 50%;
}
#client-log {
position: absolute;
left: 0;
top: 60px;
bottom: 0;
width: 50%;
overflow: scroll;
}
#server-log {
position: absolute;
right: 0;
top: 60px;
bottom: 0;
width: 50%;
overflow: scroll;
}
</style>
<button onclick="self.close()" id="close">Close</button>
<div id="client-log-header">Client logs (max 100)</div>
<div id="client-log">Loading...</div>
<div id="server-log-header">Server logs (max 100)</div>
<div id="server-log">Loading...</div>`,
`
const clientDiv = document.getElementById("client-log");
clientDiv.scrollTop = clientDiv.scrollHeight;
const serverDiv = document.getElementById("server-log");
serverDiv.scrollTop = serverDiv.scrollHeight;
self.close = () => {
syscall("event.dispatch", "log:hide");
};
syscall("system.getEnv").then((env) => {
const clientServerMode = !!env;
if (!clientServerMode) {
// Running in hybrid mode (mobile), so let's ignore server logs (they're the same as client logs)
serverDiv.style.display = "none";
clientDiv.style.width = "100%";
document.getElementById("server-log-header").style.display = "none";
}
setInterval(() => {
Promise.resolve().then(async () => {
if(clientServerMode) {
const serverLogs = await syscall("sandbox.getServerLogs");
serverDiv.innerHTML = "<pre>" + serverLogs.map((le) => "[" + le.level + "] " + le.message).join("\\n") + "</pre>";
}
const clientLogs = await syscall("sandbox.getLogs");
clientDiv.innerHTML = "<pre>" + clientLogs.map((le) => "[" + le.level + "] " + le.message).join("\\n") + "</pre>";
}).catch(console.error);
}, 1000);
});
`,
);
}
export async function hideBhsCommand() {
await editor.hidePanel("bhs");
}
plugs/core/editor.ts
+6 -5
View File
@@ -1,18 +1,19 @@
import { clientStore, editor } from "$sb/silverbullet-syscall/mod.ts";
import { editor } from "$sb/silverbullet-syscall/mod.ts";
import { store } from "$sb/plugos-syscall/mod.ts";
// Run on "editor:init"
export async function setEditorMode() {
if (await clientStore.get("vimMode")) {
if (await store.get("vimMode")) {
await editor.setUiOption("vimMode", true);
}
if (await clientStore.get("darkMode")) {
if (await store.get("darkMode")) {
await editor.setUiOption("darkMode", true);
}
}
export async function toggleDarkMode() {
let darkMode = await clientStore.get("darkMode");
let darkMode = await store.get("darkMode");
darkMode = !darkMode;
await editor.setUiOption("darkMode", darkMode);
await clientStore.set("darkMode", darkMode);
await store.set("darkMode", darkMode);
}
plugs/core/embed.ts
+4 -4
View File
@@ -1,4 +1,4 @@
import * as YAML from "yaml";
import { YAML } from "$sb/plugos-syscall/mod.ts";
import type { WidgetContent } from "$sb/app_event.ts";
type EmbedConfig = {
@@ -20,11 +20,11 @@ function extractYoutubeVideoId(url: string) {
return null;
}
export function embedWidget(
export async function embedWidget(
bodyText: string,
): WidgetContent {
): Promise<WidgetContent> {
try {
const data: EmbedConfig = YAML.parse(bodyText) as any;
const data: EmbedConfig = await YAML.parse(bodyText) as any;
let url = data.url;
const youtubeVideoId = extractYoutubeVideoId(url);
if (youtubeVideoId) {
plugs/core/link.ts
+7 -16
View File
@@ -1,5 +1,5 @@
import { nodeAtPos } from "$sb/lib/tree.ts";
import { editor, markdown, system } from "$sb/silverbullet-syscall/mod.ts";
import { editor, markdown } from "$sb/silverbullet-syscall/mod.ts";
import { events } from "$sb/plugos-syscall/mod.ts";
type UnfurlOption = {
@@ -26,16 +26,17 @@ export async function unfurlCommand() {
return;
}
try {
const replacement = await system.invokeFunction(
"server",
"unfurlExec",
selectedUnfurl.id,
const replacement = await events.dispatchEvent(
`unfurl:${selectedUnfurl.id}`,
url,
);
if (replacement.length === 0) {
throw new Error("Unfurl failed");
}
await editor.replaceRange(
nakedUrlNode?.from!,
nakedUrlNode?.to!,
replacement,
replacement[0],
);
} catch (e: any) {
await editor.flashNotification(e.message, "error");
@@ -51,16 +52,6 @@ export function titleUnfurlOptions(): UnfurlOption[] {
];
}
// Run on the server because plugs will likely rely on fetch for this
export async function unfurlExec(id: string, url: string): Promise<string> {
const replacement = await events.dispatchEvent(`unfurl:${id}`, url);
if (replacement.length === 0) {
throw new Error("Unfurl failed");
} else {
return replacement[0];
}
}
const titleRegex = /<title[^>]*>\s*([^<]+)\s*<\/title\s*>/i;
export async function titleUnfurl(url: string): Promise<string> {
plugs/core/navigate.ts
+5 -9
View File
@@ -1,10 +1,5 @@
import type { ClickEvent } from "$sb/app_event.ts";
import {
editor,
markdown,
space,
system,
} from "$sb/silverbullet-syscall/mod.ts";
import { editor, markdown, system } from "$sb/silverbullet-syscall/mod.ts";
import {
addParentPointers,
findNodeOfType,
@@ -82,9 +77,10 @@ async function actionClickOrActionEnter(
}
if (url.indexOf("://") === -1 && !url.startsWith("mailto:")) {
url = decodeURIComponent(url);
// attachment URL, let's fetch as a data url
const dataUrl = await space.readAttachment(url);
return editor.downloadFile(url, dataUrl);
// // attachment URL, let's fetch as a data url
// const dataUrl = await space.readAttachment(url);
// return editor.downloadFile(url, dataUrl);
return editor.openUrl(`/.fs/${url}`);
} else {
await editor.openUrl(url);
}
plugs/core/page.ts
+10 -6
View File
@@ -23,6 +23,7 @@ import {
} from "$sb/lib/tree.ts";
import { applyQuery } from "$sb/lib/query.ts";
import { extractFrontmatter } from "$sb/lib/frontmatter.ts";
import { invokeFunction } from "../../plug-api/silverbullet-syscall/system.ts";
// Key space:
// pl:toPage:pos => pageName
@@ -31,8 +32,8 @@ import { extractFrontmatter } from "$sb/lib/frontmatter.ts";
export async function indexLinks({ name, tree }: IndexTreeEvent) {
const backLinks: { key: string; value: string }[] = [];
// [[Style Links]]
// console.log("Now indexing", name);
const pageMeta = extractFrontmatter(tree);
// console.log("Now indexing links for", name);
const pageMeta = await extractFrontmatter(tree);
if (Object.keys(pageMeta).length > 0) {
// console.log("Extracted page meta data", pageMeta);
// Don't index meta data starting with $
@@ -44,6 +45,8 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) {
await index.set(name, "meta:", pageMeta);
}
// throw new Error("Boom");
collectNodesMatching(tree, (n) => n.type === "WikiLinkPage").forEach((n) => {
let toPage = n.children![0].text!;
if (toPage.includes("@")) {
@@ -118,7 +121,7 @@ export async function renamePage(cmdDef: any) {
`Page ${newName} already exists, cannot rename to existing page.`,
);
} catch (e: any) {
if (e.message.includes("not found")) {
if (e.message === "Not found") {
// Expected not found error, so we can continue
} else {
await editor.flashNotification(e.message, "error");
@@ -224,7 +227,7 @@ async function getBackLinks(pageName: string): Promise<BackLink[]> {
export async function reindexCommand() {
await editor.flashNotification("Reindexing...");
await system.invokeFunction("server", "reindexSpace");
await reindexSpace();
await editor.flashNotification("Reindexing done");
}
@@ -245,10 +248,11 @@ export async function pageComplete(completeEvent: CompleteEvent) {
};
}
// Server functions
export async function reindexSpace() {
console.log("Clearing page index...");
await index.clearPageIndex();
// Executed this way to not have to embed the search plug code here
await invokeFunction("client", "search.clearIndex");
console.log("Listing all pages");
const pages = await space.listPages();
let counter = 0;
@@ -272,7 +276,7 @@ export async function clearPageIndex(page: string) {
}
export async function parseIndexTextRepublish({ name, text }: IndexEvent) {
console.log("Reindexing", name);
// console.log("Reindexing", name);
await events.dispatchEvent("page:index", {
name,
tree: await markdown.parseMarkdown(text),
plugs/core/plugmanager.ts
+71 -62
View File
@@ -2,6 +2,7 @@ import { events } from "$sb/plugos-syscall/mod.ts";
import type { Manifest } from "../../common/manifest.ts";
import { editor, space, system } from "$sb/silverbullet-syscall/mod.ts";
import { readYamlPage } from "$sb/lib/yaml_page.ts";
import { builtinPlugNames } from "../builtin_plugs.ts";
const plugsPrelude =
"This file lists all plugs that SilverBullet will load. Run the {[Plugs: Update]} command to update and reload this list of plugs.\n\n";
@@ -10,9 +11,69 @@ export async function updatePlugsCommand() {
await editor.save();
await editor.flashNotification("Updating plugs...");
try {
await system.invokeFunction("server", "updatePlugs");
let plugList: string[] = [];
try {
const plugListRead: any[] = await readYamlPage("PLUGS");
plugList = plugListRead.filter((plug) => typeof plug === "string");
if (plugList.length !== plugListRead.length) {
throw new Error(
`Some of the plugs were not in a yaml list format, they were ignored`,
);
}
} catch (e: any) {
if (e.message.includes("Could not read file")) {
console.warn("No PLUGS page found, not loading anything");
return;
}
throw new Error(`Error processing PLUGS: ${e.message}`);
}
console.log("Plug YAML", plugList);
const allCustomPlugNames: string[] = [];
for (const plugUri of plugList) {
const [protocol, ...rest] = plugUri.split(":");
const plugNameMatch = /\/([^\/]+)\.plug\.js$/.exec(plugUri);
if (!plugNameMatch) {
console.error(
"Could not extract plug name from ",
plugUri,
"ignoring...",
);
continue;
}
const plugName = plugNameMatch[1];
const manifests = await events.dispatchEvent(
`get-plug:${protocol}`,
rest.join(":"),
);
if (manifests.length === 0) {
console.error("Could not resolve plug", plugUri);
}
// console.log("Got manifests", plugUri, protocol, manifests);
const workerCode = manifests[0] as string;
allCustomPlugNames.push(plugName);
// console.log("Writing", `_plug/${plugName}.plug.js`, workerCode);
await space.writeAttachment(
`_plug/${plugName}.plug.js`,
"utf8",
workerCode,
);
}
const allPlugNames = [...builtinPlugNames, ...allCustomPlugNames];
// And delete extra ones
for (const existingPlug of await space.listPlugs()) {
const plugName = existingPlug.substring(
"_plug/".length,
existingPlug.length - ".plug.js".length,
);
if (!allPlugNames.includes(plugName)) {
await space.deleteAttachment(existingPlug);
}
}
await editor.flashNotification("And... done!");
system.reloadPlugs();
} catch (e: any) {
editor.flashNotification("Error updating plugs: " + e.message, "error");
}
@@ -45,74 +106,22 @@ export async function addPlugCommand() {
"\n```",
);
await editor.navigate("PLUGS");
await system.invokeFunction("server", "updatePlugs");
await updatePlugsCommand();
await editor.flashNotification("Plug added!");
system.reloadPlugs();
}
export async function updatePlugs() {
let plugList: string[] = [];
try {
const plugListRead: any[] = await readYamlPage("PLUGS");
plugList = plugListRead.filter((plug) => typeof plug === "string");
if (plugList.length !== plugListRead.length) {
throw new Error(
`Some of the plugs were not in a yaml list format, they were ignored`,
);
}
} catch (e: any) {
if (e.message.includes("Could not read file")) {
console.warn("No PLUGS page found, not loading anything");
return;
}
throw new Error(`Error processing PLUGS: ${e.message}`);
}
console.log("Plug YAML", plugList);
const allPlugNames: string[] = [];
for (const plugUri of plugList) {
const [protocol, ...rest] = plugUri.split(":");
const manifests = await events.dispatchEvent(
`get-plug:${protocol}`,
rest.join(":"),
);
if (manifests.length === 0) {
console.error("Could not resolve plug", plugUri);
}
// console.log("Got manifests", plugUri, protocol, manifests);
const manifest = manifests[0];
allPlugNames.push(manifest.name);
// console.log("Writing", `_plug/${manifest.name}`);
await space.writeAttachment(
`_plug/${manifest.name}.plug.json`,
"utf8",
JSON.stringify(manifest),
);
}
// And delete extra ones
for (const existingPlug of await space.listPlugs()) {
const plugName = existingPlug.substring(
"_plug/".length,
existingPlug.length - ".plug.json".length,
);
if (!allPlugNames.includes(plugName)) {
await space.deleteAttachment(existingPlug);
}
}
system.reloadPlugs();
}
export async function getPlugHTTPS(url: string): Promise<Manifest> {
export async function getPlugHTTPS(url: string): Promise<string> {
const fullUrl = `https:${url}`;
console.log("Now fetching plug manifest from", fullUrl);
console.log("Now fetching plug code from", fullUrl);
const req = await fetch(fullUrl);
if (req.status !== 200) {
throw new Error(`Could not fetch plug manifest from ${fullUrl}`);
throw new Error(`Could not fetch plug code from ${fullUrl}`);
}
return req.json();
return req.text();
}
export function getPlugGithub(identifier: string): Promise<Manifest> {
export function getPlugGithub(identifier: string): Promise<string> {
const [owner, repo, path] = identifier.split("/");
let [repoClean, branch] = repo.split("@");
if (!branch) {
@@ -125,7 +134,7 @@ export function getPlugGithub(identifier: string): Promise<Manifest> {
export async function getPlugGithubRelease(
identifier: string,
): Promise<Manifest> {
): Promise<string> {
let [owner, repo, version] = identifier.split("/");
if (!version || version === "latest") {
console.log("fetching the latest version");
@@ -141,6 +150,6 @@ export async function getPlugGithubRelease(
version = result.name;
}
const finalUrl =
`//github.com/${owner}/${repo}/releases/download/${version}/${repo}.plug.json`;
`//github.com/${owner}/${repo}/releases/download/${version}/${repo}.plug.js`;
return getPlugHTTPS(finalUrl);
}
plugs/core/refactor.ts
+1 -1
View File
@@ -16,7 +16,7 @@ export async function extractToPage() {
`Page ${newName} already exists, cannot rename to existing page.`,
);
} catch (e: any) {
if (e.message.includes("not found")) {
if (e.message === "Not found") {
// Expected not found error, so we can continue
} else {
await editor.flashNotification(e.message, "error");
plugs/core/tags.ts
+1 -1
View File
@@ -14,7 +14,7 @@ import { extractFrontmatter } from "../../plug-api/lib/frontmatter.ts";
export async function indexTags({ name, tree }: IndexTreeEvent) {
removeQueries(tree);
const allTags = new Set<string>();
const { tags } = extractFrontmatter(tree);
const { tags } = await extractFrontmatter(tree);
if (Array.isArray(tags)) {
tags.forEach((t) => allTags.add(t));
}
plugs/core/template.ts
+14 -17
View File
@@ -31,7 +31,7 @@ export async function instantiateTemplateCommand() {
);
const parseTree = await markdown.parseMarkdown(text);
const additionalPageMeta = extractFrontmatter(parseTree, [
const additionalPageMeta = await extractFrontmatter(parseTree, [
"$name",
"$disableDirectives",
]);
@@ -157,28 +157,25 @@ export async function dailyNoteCommand() {
dailyNoteTemplate: "template/page/Daily Note",
dailyNotePrefix: "📅 ",
});
let dailyNoteTemplateText = "";
try {
dailyNoteTemplateText = await space.readPage(dailyNoteTemplate);
} catch {
console.warn(`No daily note template found at ${dailyNoteTemplate}`);
}
const date = niceDate(new Date());
const pageName = `${dailyNotePrefix}${date}`;
if (dailyNoteTemplateText) {
try {
await space.getPageMeta(pageName);
} catch {
// Doesn't exist, let's create
let dailyNoteTemplateText = "";
try {
await space.getPageMeta(pageName);
dailyNoteTemplateText = await space.readPage(dailyNoteTemplate);
} catch {
// Doesn't exist, let's create
await space.writePage(
pageName,
replaceTemplateVars(dailyNoteTemplateText, pageName),
);
console.warn(`No daily note template found at ${dailyNoteTemplate}`);
}
await editor.navigate(pageName);
} else {
await editor.navigate(pageName);
await space.writePage(
pageName,
replaceTemplateVars(dailyNoteTemplateText, pageName),
);
}
await editor.navigate(pageName);
}
function getWeekStartDate(monday = false) {
plugs/core/vim.ts
+4 -3
View File
@@ -1,11 +1,12 @@
import { readCodeBlockPage } from "../../plug-api/lib/yaml_page.ts";
import { clientStore, editor } from "$sb/silverbullet-syscall/mod.ts";
import { editor } from "$sb/silverbullet-syscall/mod.ts";
import { store } from "$sb/plugos-syscall/mod.ts";
export async function toggleVimMode() {
let vimMode = await clientStore.get("vimMode");
let vimMode = await store.get("vimMode");
vimMode = !vimMode;
await editor.setUiOption("vimMode", vimMode);
await clientStore.set("vimMode", vimMode);
await store.set("vimMode", vimMode);
}
export async function loadVimRc() {
plugs/directive/command.ts
+9 -19
View File
@@ -1,6 +1,5 @@
import { editor, markdown, system } from "$sb/silverbullet-syscall/mod.ts";
import { editor, markdown, sync } from "$sb/silverbullet-syscall/mod.ts";
import {
ParseTree,
removeParentPointers,
renderToText,
traverseTree,
@@ -14,12 +13,17 @@ export async function updateDirectivesOnPageCommand(arg: any) {
const pageName = await editor.getCurrentPage();
const text = await editor.getText();
const tree = await markdown.parseMarkdown(text);
const metaData = extractFrontmatter(tree, ["$disableDirectives"]);
const metaData = await extractFrontmatter(tree, ["$disableDirectives"]);
if (metaData.$disableDirectives) {
// Not updating, directives disabled
return;
}
// if (!(await sync.hasInitialSyncCompleted())) {
// console.info("Initial sync hasn't completed yet, not updating directives.");
// return;
// }
// If this page is shared ($share) via collab: disable directives as well
// due to security concerns
if (metaData.$share) {
@@ -51,12 +55,7 @@ export async function updateDirectivesOnPageCommand(arg: any) {
}
const fullMatch = text.substring(tree.from!, tree.to!);
try {
const promise = system.invokeFunction(
"server",
"serverRenderDirective",
pageName,
tree,
);
const promise = renderDirectives(pageName, tree);
replacements.push({
textPromise: promise,
fullMatch,
@@ -116,17 +115,8 @@ export async function updateDirectivesOnPageCommand(arg: any) {
}
}
// Called from client, running on server
// The text passed here is going to be a single directive block (not a full page)
export function serverRenderDirective(
pageName: string,
tree: ParseTree,
): Promise<string> {
return renderDirectives(pageName, tree);
}
// Pure server driven implementation of directive updating
export async function serverUpdateDirectives(
export async function updateDirectives(
pageName: string,
text: string,
) {
plugs/directive/complete.ts
+1 -1
View File
@@ -1,5 +1,5 @@
import { events } from "$sb/plugos-syscall/mod.ts";
import { CompleteEvent } from "../../plug-api/app_event.ts";
import { CompleteEvent } from "$sb/app_event.ts";
export async function queryComplete(completeEvent: CompleteEvent) {
const match = /#query ([\w\-_]+)*$/.exec(completeEvent.linePrefix);
plugs/directive/data.ts
+35 -33
View File
@@ -5,46 +5,48 @@ import type { IndexTreeEvent, QueryProviderEvent } from "$sb/app_event.ts";
import { index } from "$sb/silverbullet-syscall/mod.ts";
import { collectNodesOfType, findNodeOfType } from "$sb/lib/tree.ts";
import { applyQuery, removeQueries } from "$sb/lib/query.ts";
import * as YAML from "yaml";
import { YAML } from "$sb/plugos-syscall/mod.ts";
export async function indexData({ name, tree }: IndexTreeEvent) {
const dataObjects: { key: string; value: any }[] = [];
removeQueries(tree);
collectNodesOfType(tree, "FencedCode").forEach((t) => {
const codeInfoNode = findNodeOfType(t, "CodeInfo");
if (!codeInfoNode) {
return;
}
if (codeInfoNode.children![0].text !== "data") {
return;
}
const codeTextNode = findNodeOfType(t, "CodeText");
if (!codeTextNode) {
// Honestly, this shouldn't happen
return;
}
const codeText = codeTextNode.children![0].text!;
try {
const docs = codeText.split("---").map((d) => YAML.parse(d));
// We support multiple YAML documents in one block
for (let i = 0; i < docs.length; i++) {
const doc = docs[i];
if (!doc) {
continue;
}
dataObjects.push({
key: `data:${name}@${t.from! + i}`,
value: doc,
});
await Promise.all(
collectNodesOfType(tree, "FencedCode").map(async (t) => {
const codeInfoNode = findNodeOfType(t, "CodeInfo");
if (!codeInfoNode) {
return;
}
// console.log("Parsed data", parsedData);
} catch (e) {
console.error("Could not parse data", codeText, "error:", e);
return;
}
});
if (codeInfoNode.children![0].text !== "data") {
return;
}
const codeTextNode = findNodeOfType(t, "CodeText");
if (!codeTextNode) {
// Honestly, this shouldn't happen
return;
}
const codeText = codeTextNode.children![0].text!;
try {
const docs = codeText.split("---");
// We support multiple YAML documents in one block
for (let i = 0; i < docs.length; i++) {
const doc = await YAML.parse(docs[i]);
if (!doc) {
continue;
}
dataObjects.push({
key: `data:${name}@${t.from! + i}`,
value: doc,
});
}
// console.log("Parsed data", parsedData);
} catch (e) {
console.error("Could not parse data", codeText, "error:", e);
return;
}
}),
);
// console.log("Found", dataObjects.length, "data objects");
await index.batchSet(name, dataObjects);
}
plugs/directive/directive.plug.yaml
+2 -4
View File
@@ -1,9 +1,7 @@
name: directive
imports:
- https://get.silverbullet.md/global.plug.json
requiredPermissions:
- fetch
functions:
serverRenderDirective:
path: ./command.ts:serverRenderDirective
updateDirectivesOnPageCommand:
path: ./command.ts:updateDirectivesOnPageCommand
command:
plugs/directive/directives.ts
+9
View File
@@ -1,4 +1,5 @@
import { ParseTree, renderToText } from "$sb/lib/tree.ts";
import { sync } from "../../plug-api/silverbullet-syscall/mod.ts";
import { evalDirectiveRenderer } from "./eval_directive.ts";
import { queryDirectiveRenderer } from "./query_directive.ts";
@@ -52,6 +53,14 @@ export async function directiveDispatcher(
}
} else {
// #query
if (!(await sync.hasInitialSyncCompleted())) {
console.info(
"Initial sync hasn't completed yet, not updating query directives.",
);
// Render the query directive as-is
return renderToText(directiveTree);
}
const newBody = await directiveRenderers["query"](
"query",
pageName,
plugs/directive/eval_directive.ts
+2 -2
View File
@@ -1,6 +1,6 @@
// This is some shocking stuff. My profession would kill me for this.
import * as YAML from "yaml";
import { YAML } from "$sb/plugos-syscall/mod.ts";
import { ParseTree } from "$sb/lib/tree.ts";
import { jsonToMDTable, renderTemplate } from "./util.ts";
@@ -57,7 +57,7 @@ export async function evalDirectiveRenderer(
} else if (Array.isArray(result)) {
return jsonToMDTable(result);
}
return YAML.stringify(result);
return await YAML.stringify(result);
} catch (e: any) {
return `**ERROR:** ${e.message}`;
}
plugs/directive/query_directive.ts
+6 -2
View File
@@ -4,7 +4,7 @@ import { replaceTemplateVars } from "../core/template.ts";
import { renderTemplate } from "./util.ts";
import { parseQuery } from "./parser.ts";
import { jsonToMDTable } from "./util.ts";
import { ParseTree } from "../../plug-api/lib/tree.ts";
import { ParseTree } from "$sb/lib/tree.ts";
export async function queryDirectiveRenderer(
_directive: string,
@@ -31,8 +31,12 @@ export async function queryDirectiveRenderer(
// This means there was no handler for the event which means it's unsupported
return `**Error:** Unsupported query source '${parsedQuery.table}'`;
} else if (results.length === 1) {
// console.log("Parsed query", parsedQuery);
if (parsedQuery.render) {
const rendered = await renderTemplate(parsedQuery.render, results[0]);
const rendered = await renderTemplate(
parsedQuery.render,
results[0],
);
return rendered.trim();
} else {
if (results[0].length === 0) {
plugs/directive/template_directive.ts
+8 -3
View File
@@ -7,7 +7,8 @@ import Handlebars from "handlebars";
import { replaceTemplateVars } from "../core/template.ts";
import { extractFrontmatter } from "$sb/lib/frontmatter.ts";
import { directiveRegex } from "./directives.ts";
import { serverUpdateDirectives } from "./command.ts";
import { updateDirectives } from "./command.ts";
import { registerHandlebarsHelpers } from "./util.ts";
const templateRegex = /\[\[([^\]]+)\]\]\s*(.*)\s*/;
@@ -48,16 +49,20 @@ export async function templateDirectiveRenderer(
// if it's a template injection (not a literal "include")
if (directive === "use") {
const tree = await markdown.parseMarkdown(templateText);
extractFrontmatter(tree, ["$disableDirectives"]);
await extractFrontmatter(tree, ["$disableDirectives"]);
templateText = renderToText(tree);
registerHandlebarsHelpers();
const templateFn = Handlebars.compile(
replaceTemplateVars(templateText, pageName),
{ noEscape: true },
);
if (typeof parsedArgs !== "string") {
(parsedArgs as any).page = pageName;
}
newBody = templateFn(parsedArgs);
// Recursively render directives
newBody = await serverUpdateDirectives(pageName, newBody);
newBody = await updateDirectives(pageName, newBody);
}
return newBody.trim();
}
plugs/directive/util.ts
+27 -21
View File
@@ -1,5 +1,4 @@
import Handlebars from "handlebars";
import * as YAML from "yaml";
import { space } from "$sb/silverbullet-syscall/mod.ts";
import { niceDate } from "$sb/lib/dates.ts";
@@ -83,8 +82,35 @@ export async function renderTemplate(
renderTemplate: string,
data: any[],
): Promise<string> {
registerHandlebarsHelpers();
// Handlebars.registerHelper("yaml", (v: any, prefix: string) => {
// if (typeof prefix === "string") {
// let yaml = (await YAML.stringify(v))
// .split("\n")
// .join("\n" + prefix)
// .trim();
// if (Array.isArray(v)) {
// return "\n" + prefix + yaml;
// } else {
// return yaml;
// }
// } else {
// return YAML.stringify(v).trim();
// }
// });
let templateText = await space.readPage(renderTemplate);
templateText = `{{#each .}}\n${templateText}\n{{/each}}`;
const template = Handlebars.compile(templateText, { noEscape: true });
return template(data);
}
export function registerHandlebarsHelpers() {
Handlebars.registerHelper("json", (v: any) => JSON.stringify(v));
Handlebars.registerHelper("niceDate", (ts: any) => niceDate(new Date(ts)));
Handlebars.registerHelper("escapeRegexp", (ts: any) => {
return ts.replace(/[-\/\\^$*+?.()|[\]{}]/g, "\\$&");
});
Handlebars.registerHelper("prefixLines", (v: string, prefix: string) =>
v
.split("\n")
@@ -96,24 +122,4 @@ export async function renderTemplate(
(s: string, from: number, to: number, elipsis = "") =>
s.length > to - from ? s.substring(from, to) + elipsis : s,
);
Handlebars.registerHelper("yaml", (v: any, prefix: string) => {
if (typeof prefix === "string") {
let yaml = YAML.stringify(v)
.split("\n")
.join("\n" + prefix)
.trim();
if (Array.isArray(v)) {
return "\n" + prefix + yaml;
} else {
return yaml;
}
} else {
return YAML.stringify(v).trim();
}
});
let templateText = await space.readPage(renderTemplate);
templateText = `{{#each .}}\n${templateText}\n{{/each}}`;
const template = Handlebars.compile(templateText, { noEscape: true });
return template(data);
}
plugs/emoji/emoji.plug.yaml
-2
View File
@@ -1,6 +1,4 @@
name: emoji
imports:
- https://get.silverbullet.md/global.plug.json
functions:
emojiCompleter:
path: "./emoji.ts:emojiCompleter"
plugs/global.plug.yaml
-4
View File
@@ -1,4 +0,0 @@
name: global
dependencies:
"https://esm.sh/handlebars": "https://esm.sh/handlebars@4.7.7"
"https://deno.land/std@0.184.0/yaml/mod.ts": "https://deno.land/std@0.184.0/yaml/mod.ts"
plugs/markdown/markdown.plug.yaml
+1 -10
View File
@@ -1,6 +1,4 @@
name: markdown
imports:
- https://get.silverbullet.md/global.plug.json
assets:
- "assets/*"
requiredPermissions:
@@ -12,9 +10,9 @@ functions:
name: "Markdown Preview: Toggle"
key: Ctrl-p
mac: Cmd-p
preview:
path: "./preview.ts:updateMarkdownPreview"
env: client
events:
- plug:load
- editor:updated
@@ -22,16 +20,9 @@ functions:
- editor:pageReloaded
previewClickHandler:
path: "./preview.ts:previewClickHandler"
env: client
events:
- preview:click
# $share: file:* publisher for markdown files
sharePublisher:
path: ./share.ts:sharePublisher
events:
- share:file
markdownWidget:
path: ./widget.ts:markdownWidget
codeWidget: markdown
plugs/markdown/markdown.ts
+6 -4
View File
@@ -1,11 +1,13 @@
import { clientStore, editor, system } from "$sb/silverbullet-syscall/mod.ts";
import { editor } from "$sb/silverbullet-syscall/mod.ts";
import { readSettings } from "$sb/lib/settings_page.ts";
import { updateMarkdownPreview } from "./preview.ts";
import { store } from "$sb/plugos-syscall/mod.ts";
export async function togglePreview() {
const currentValue = !!(await clientStore.get("enableMarkdownPreview"));
await clientStore.set("enableMarkdownPreview", !currentValue);
const currentValue = !!(await store.get("enableMarkdownPreview"));
await store.set("enableMarkdownPreview", !currentValue);
if (!currentValue) {
await system.invokeFunction("client", "preview");
await updateMarkdownPreview();
} else {
await hideMarkdownPreview();
}
plugs/markdown/markdown_render.test.ts
+11 -12
View File
@@ -2,32 +2,31 @@ import buildMarkdown from "../../common/markdown_parser/parser.ts";
import { parse } from "../../common/markdown_parser/parse_tree.ts";
import { System } from "../../plugos/system.ts";
import corePlug from "../../dist_bundle/_plug/core.plug.json" assert {
type: "json",
};
import tasksPlug from "../../dist_bundle/_plug/tasks.plug.json" assert {
type: "json",
};
import { createSandbox } from "../../plugos/environments/deno_sandbox.ts";
import { loadMarkdownExtensions } from "../../common/markdown_parser/markdown_ext.ts";
import { renderMarkdownToHtml } from "./markdown_render.ts";
import { assertEquals } from "../../test_deps.ts";
import { urlToPathname } from "../../plugos/util.ts";
Deno.test("Markdown render", async () => {
const system = new System<any>("server");
await system.load(corePlug, createSandbox);
await system.load(tasksPlug, createSandbox);
await system.load(
new URL("../../dist_plug_bundle/_plug/core.plug.js", import.meta.url),
createSandbox,
);
await system.load(
new URL("../../dist_plug_bundle/_plug/tasks.plug.js", import.meta.url),
createSandbox,
);
const lang = buildMarkdown(loadMarkdownExtensions(system));
const testFile = Deno.readTextFileSync(
urlToPathname(new URL("test/example.md", import.meta.url)),
new URL("test/example.md", import.meta.url).pathname,
);
const tree = parse(lang, testFile);
renderMarkdownToHtml(tree, {
await renderMarkdownToHtml(tree, {
failOnUnknown: true,
renderFrontMatter: true,
});
// console.log("HTML", html);
await system.unloadAll();
});
Deno.test("Smart hard break test", async () => {
plugs/markdown/markdown_render.ts
+2 -30
View File
@@ -4,14 +4,12 @@ import {
renderToText,
traverseTree,
} from "$sb/lib/tree.ts";
import * as YAML from "yaml";
import { Fragment, renderHtml, Tag } from "./html_render.ts";
type MarkdownRenderOptions = {
failOnUnknown?: true;
smartHardBreak?: true;
annotationPositions?: true;
renderFrontMatter?: true;
attachmentUrlPrefix?: string;
// When defined, use to inline images as data: urls
inlineAttachments?: (url: string) => Promise<string>;
@@ -78,33 +76,7 @@ function render(
body: cleanTags(mapRender(t.children!)),
};
case "FrontMatter":
if (options.renderFrontMatter) {
const yamlCode = renderToText(t.children![1]);
const parsedYaml = YAML.parse(yamlCode) as Record<string, any>;
const rows: Tag[] = [];
for (const [k, v] of Object.entries(parsedYaml)) {
rows.push({
name: "tr",
body: [
{ name: "td", attrs: { class: "key" }, body: k },
{
name: "td",
attrs: { class: "value" },
body: YAML.stringify(v),
},
],
});
}
return {
name: "table",
attrs: {
class: "front-matter",
},
body: rows,
};
} else {
return null;
}
return null;
case "CommentBlock":
// Remove, for now
return null;
@@ -261,7 +233,7 @@ function render(
return {
name: "a",
attrs: {
href: `/${ref.replaceAll(" ", "_").replace("@", "#")}`,
href: `/${ref.replace("@", "#")}`,
},
body: linkText,
};
plugs/markdown/preview.ts
+4 -10
View File
@@ -1,15 +1,10 @@
import {
clientStore,
editor,
space,
system,
} from "$sb/silverbullet-syscall/mod.ts";
import { asset } from "$sb/plugos-syscall/mod.ts";
import { parseMarkdown } from "../../plug-api/silverbullet-syscall/markdown.ts";
import { editor, space, system } from "$sb/silverbullet-syscall/mod.ts";
import { asset, store } from "$sb/plugos-syscall/mod.ts";
import { parseMarkdown } from "$sb/silverbullet-syscall/markdown.ts";
import { renderMarkdownToHtml } from "./markdown_render.ts";
export async function updateMarkdownPreview() {
if (!(await clientStore.get("enableMarkdownPreview"))) {
if (!(await store.get("enableMarkdownPreview"))) {
return;
}
const text = await editor.getText();
@@ -20,7 +15,6 @@ export async function updateMarkdownPreview() {
const html = await renderMarkdownToHtml(mdTree, {
smartHardBreak: true,
annotationPositions: true,
renderFrontMatter: true,
inlineAttachments: async (url): Promise<string> => {
if (!url.includes("://")) {
try {
plugs/markdown/share.ts
-24
View File
@@ -1,24 +0,0 @@
import { markdown, space } from "$sb/silverbullet-syscall/mod.ts";
import { LocalFileSystem } from "$sb/plugos-syscall/mod.ts";
import { asset } from "$sb/plugos-syscall/mod.ts";
import { renderMarkdownToHtml } from "./markdown_render.ts";
import { PublishEvent } from "$sb/app_event.ts";
export async function sharePublisher(event: PublishEvent) {
const path = event.uri.split(":")[1];
const pageName = event.name;
const text = await space.readPage(pageName);
const tree = await markdown.parseMarkdown(text);
const rootFS = new LocalFileSystem("");
const css = await asset.readAsset("assets/styles.css");
const markdownHtml = renderMarkdownToHtml(tree, {
smartHardBreak: true,
});
const html =
`<html><head><style>${css}</style></head><body><div id="root">${markdownHtml}</div></body></html>`;
await rootFS.writeFile(path, html, "utf8");
return true;
}
plugs/markdown/util.ts
+1 -1
View File
@@ -6,7 +6,7 @@ import {
import { markdown } from "$sb/silverbullet-syscall/mod.ts";
export function encodePageUrl(name: string): string {
return name.replaceAll(" ", "_");
return name;
}
export async function cleanMarkdown(
plugs/plugmd/plugmd.plug.yaml
-26
View File
@@ -1,26 +0,0 @@
name: plugmd
imports:
- https://get.silverbullet.md/global.plug.json
functions:
check:
path: "./plugmd.ts:checkCommand"
command:
name: "Plug: Check"
mac: "Cmd-Alt-c"
key: "Ctrl-Alt-c"
compile:
path: "./plugmd.ts:compileCommand"
command:
name: "Plug: Compile"
mac: "Cmd-Shift-c"
key: "Ctrl-Shift-c"
compileJS:
path: "./plugmd.ts:compileJS"
env: server
compileModule:
path: "./plugmd.ts:compileModule"
env: server
getPlugPlugMd:
path: "./plugmd.ts:getPlugPlugMd"
events:
- get-plug:plugmd
plugs/plugmd/plugmd.ts
-125
View File
@@ -1,125 +0,0 @@
import { collectNodesOfType, findNodeOfType } from "$sb/lib/tree.ts";
import {
editor,
markdown,
space,
system,
} from "$sb/silverbullet-syscall/mod.ts";
import { syscall } from "$sb/plugos-syscall/mod.ts";
import * as YAML from "yaml";
import type { Manifest } from "../../common/manifest.ts";
export async function compileCommand() {
const text = await editor.getText();
try {
const manifest = await compileDefinition(text);
await space.writePage(
`_plug/${manifest.name}`,
JSON.stringify(manifest, null, 2),
);
console.log("Wrote this plug", manifest);
await editor.hidePanel("bhs");
system.reloadPlugs();
} catch (e: any) {
await editor.showPanel("bhs", 1, e.message);
// console.error("Got this error from compiler", e.message);
}
}
export async function checkCommand() {
const text = await editor.getText();
try {
await compileDefinition(text);
await editor.hidePanel("bhs");
system.reloadPlugs();
} catch (e: any) {
await editor.showPanel("bhs", 1, e.message);
// console.error("Got this error from compiler", e.message);
}
}
async function compileDefinition(text: string): Promise<Manifest> {
const tree = await markdown.parseMarkdown(text);
const codeNodes = collectNodesOfType(tree, "FencedCode");
let manifest: Manifest | undefined;
let code: string | undefined;
let language = "js";
for (const codeNode of codeNodes) {
const codeInfo = findNodeOfType(codeNode, "CodeInfo")!.children![0].text!;
const codeText = findNodeOfType(codeNode, "CodeText")!.children![0].text!;
if (codeInfo === "yaml") {
manifest = YAML.parse(codeText) as Manifest;
continue;
}
if (codeInfo === "typescript" || codeInfo === "ts") {
language = "ts";
}
code = codeText;
}
if (!manifest) {
throw new Error("No meta found");
}
if (!code) {
throw new Error("No code found");
}
manifest.dependencies = manifest.dependencies || {};
for (const [dep, depSpec] of Object.entries(manifest.dependencies)) {
const compiled = await system.invokeFunction(
"server",
"compileModule",
depSpec,
);
manifest.dependencies![dep] = compiled;
}
manifest.functions = manifest.functions || {};
for (const [name, func] of Object.entries(manifest.functions)) {
const compiled = await system.invokeFunction(
"server",
"compileJS",
`file.${language}`,
code,
name,
Object.keys(manifest.dependencies),
);
func.code = compiled;
}
console.log("Doing the whole manifest thing");
return manifest;
}
export function compileJS(
filename: string,
code: string,
functionName: string,
excludeModules: string[],
): Promise<string> {
// console.log("Compiling JS", filename, excludeModules);
return syscall(
"esbuild.compile",
filename,
code,
functionName,
excludeModules,
);
}
export function compileModule(moduleName: string): Promise<string> {
return syscall("esbuild.compileModule", moduleName);
}
export async function getPlugPlugMd(pageName: string): Promise<Manifest> {
const text = await space.readPage(pageName);
console.log("Compiling", pageName);
return compileDefinition(text);
}
plugs/search/engine.test.ts
+64
View File
@@ -0,0 +1,64 @@
import { assertEquals } from "../../test_deps.ts";
import { BatchKVStore, SimpleSearchEngine } from "./engine.ts";
class InMemoryBatchKVStore<K, V> implements BatchKVStore<K, V> {
private store = new Map<K, V>();
get(keys: K[]): Promise<(V | undefined)[]> {
const results: (V | undefined)[] = keys.map((key) => this.store.get(key));
return Promise.resolve(results);
}
set(entries: Map<K, V>): Promise<void> {
for (const [key, value] of entries) {
this.store.set(key, value);
}
return Promise.resolve();
}
delete(keys: K[]): Promise<void> {
for (const key of keys) {
this.store.delete(key);
}
return Promise.resolve();
}
}
Deno.test("Test full text search", async () => {
const engine = new SimpleSearchEngine(
new InMemoryBatchKVStore(),
new InMemoryBatchKVStore(),
);
await engine.indexDocument({ id: "1", text: "The quick brown fox" });
await engine.indexDocument({ id: "2", text: "jumps over the lazy dogs" });
await engine.indexDocument({
id: "3",
text: "Hello world, jumping jump jumps",
});
await engine.indexDocument({ id: "4", text: "TypeScript is awesome" });
await engine.indexDocument({ id: "5", text: "The brown dogs jumps zęf" });
console.log(engine.index);
const results = await engine.search("Brown fox");
console.log(results);
assertEquals(results.length, 2);
assertEquals(results[0].id, "1");
assertEquals(results[0].score, 2);
assertEquals(results[1].id, "5");
assertEquals(results[1].score, 1);
const results2 = await engine.search("jump");
console.log(results2);
assertEquals(results2.length, 3);
await engine.deleteDocument("3");
const results3 = await engine.search("jump");
console.log(results3);
assertEquals(results3.length, 2);
const results4 = await engine.search("zęf");
console.log(results4);
assertEquals(results4.length, 1);
});
plugs/search/engine.ts
+140
View File
@@ -0,0 +1,140 @@
import { c } from "https://esm.sh/@codemirror/legacy-modes@6.3.1/mode/clike?external=@codemirror/language";
import { stemmer } from "https://esm.sh/porter-stemmer@0.9.1";
export type Document = {
id: string;
text: string;
};
export interface BatchKVStore<K, V> {
get(keys: K[]): Promise<(V | undefined)[]>;
set(entries: Map<K, V>): Promise<void>;
delete(keys: K[]): Promise<void>;
}
type ResultObject = {
score: number;
id: string;
};
export class SimpleSearchEngine {
private stopWords = ["and", "or", "the", "a", "an"];
constructor(
public index: BatchKVStore<string, string[]>,
public reverseIndex: BatchKVStore<string, string[]>,
) {
}
// Tokenize text into words
private tokenize(text: string): string[] {
return text.toLowerCase().split(/[^\p{L}]+/u);
}
// Remove stop words from array of words
private removeStopWords(words: string[]): string[] {
return words.filter((word) =>
word.length > 2 &&
!this.stopWords.includes(word) && /^\p{L}+$/u.test(word)
);
}
// Basic stemming function
private stem(word: string): string {
return stemmer(word);
}
// Index an array of documents
public async indexDocument(document: Document): Promise<void> {
const updateIndexMap = new Map<string, string[]>();
const updateReverseIndexMap = new Map<string, string[]>();
const words = this.tokenize(document.text);
const filteredWords = this.removeStopWords(words);
const stemmedWords = filteredWords.map((word) => this.stem(word));
// Get the current IDs for these stemmed words
const uniqueStemmedWords = [...new Set(stemmedWords)];
const currentIdsArray = await this.index.get(uniqueStemmedWords);
stemmedWords.forEach((stemmedWord, i) => {
const currentIds =
currentIdsArray[uniqueStemmedWords.indexOf(stemmedWord)] || [];
currentIds.push(document.id);
updateIndexMap.set(stemmedWord, currentIds);
if (!updateReverseIndexMap.has(document.id)) {
updateReverseIndexMap.set(document.id, []);
}
if (!updateReverseIndexMap.get(document.id)!.includes(stemmedWord)) {
updateReverseIndexMap.get(document.id)!.push(stemmedWord);
}
});
// console.log("updateIndexMap", updateIndexMap);
await this.index.set(updateIndexMap);
await this.reverseIndex.set(updateReverseIndexMap);
}
// Search for a phrase and return document ids sorted by match count
public async search(phrase: string): Promise<ResultObject[]> {
const words = this.tokenize(phrase);
const filteredWords = this.removeStopWords(words);
const stemmedWords = filteredWords.map((word) => this.stem(word));
const wordIdsArray = await this.index.get(stemmedWords);
const matchCounts: Map<string, number> = new Map();
wordIdsArray.forEach((wordIds) => {
if (wordIds) {
wordIds.forEach((id) => {
if (matchCounts.has(id)) {
matchCounts.set(id, matchCounts.get(id)! + 1);
} else {
matchCounts.set(id, 1);
}
});
}
});
const results = Array.from(matchCounts.entries()).map(
([id, score]) => ({ id, score }),
);
return results.sort((a, b) => b.score - a.score);
}
// Delete a document from the index
public async deleteDocument(documentId: string): Promise<void> {
const words = await this.reverseIndex.get([documentId]);
if (words && words[0]) {
const currentIdsArray = await this.index.get(words[0]);
const deleteKeys: string[] = [];
const updateMap = new Map<string, string[]>();
words[0].forEach((word, i) => {
const currentIds = currentIdsArray[i];
if (currentIds) {
const updatedIds = currentIds.filter((id) => id !== documentId);
if (updatedIds.length > 0) {
updateMap.set(word, updatedIds);
} else {
deleteKeys.push(word);
}
}
});
if (deleteKeys.length > 0) {
await this.index.delete(deleteKeys);
}
if (updateMap.size > 0) {
await this.index.set(updateMap);
}
await this.reverseIndex.delete([documentId]);
}
}
}
plugs/search/search.plug.yaml
+42
View File
@@ -0,0 +1,42 @@
name: search
functions:
indexPage:
path: search.ts:indexPage
events:
- page:index
clearIndex:
path: search.ts:clearIndex
searchUnindex:
path: "./search.ts:pageUnindex"
events:
- page:deleted
searchQueryProvider:
path: ./search.ts:queryProvider
events:
- query:full-text
searchCommand:
path: ./search.ts:searchCommand
command:
name: "Search Space"
key: Ctrl-Shift-f
mac: Cmd-Shift-f
readPageSearch:
path: ./search.ts:readFileSearch
pageNamespace:
pattern: "🔍 .+"
operation: readFile
writePageSearch:
path: ./search.ts:writeFileSearch
pageNamespace:
pattern: "🔍 .+"
operation: readFile
getPageMetaSearch:
path: ./search.ts:getFileMetaSearch
pageNamespace:
pattern: "🔍 .+"
operation: getFileMeta
plugs/core/search.ts → plugs/search/search.ts
+60 -33
View File
@@ -1,25 +1,51 @@
import { fulltext } from "$sb/plugos-syscall/mod.ts";
import { renderToText } from "$sb/lib/tree.ts";
import type { FileMeta } from "../../common/types.ts";
import { editor, index } from "$sb/silverbullet-syscall/mod.ts";
import { IndexTreeEvent, QueryProviderEvent } from "$sb/app_event.ts";
import { applyQuery, removeQueries } from "$sb/lib/query.ts";
import {
FileData,
FileEncoding,
} from "../../common/spaces/space_primitives.ts";
import { renderToText } from "$sb/lib/tree.ts";
import { store } from "$sb/plugos-syscall/mod.ts";
import { applyQuery } from "$sb/lib/query.ts";
import { editor, index } from "$sb/silverbullet-syscall/mod.ts";
import { base64EncodedDataUrl } from "../../plugos/asset_bundle/base64.ts";
import { BatchKVStore, SimpleSearchEngine } from "./engine.ts";
import { FileMeta } from "../../common/types.ts";
const searchPrefix = "🔍 ";
export async function pageIndex(data: IndexTreeEvent) {
removeQueries(data.tree);
const cleanText = renderToText(data.tree);
await fulltext.fullTextIndex(data.name, cleanText);
class StoreKVStore implements BatchKVStore<string, string[]> {
constructor(private prefix: string) {
}
get(keys: string[]): Promise<(string[] | undefined)[]> {
return store.batchGet(keys.map((key) => this.prefix + key));
}
set(entries: Map<string, string[]>): Promise<void> {
return store.batchSet(
Array.from(entries.entries()).map((
[key, value],
) => ({ key: this.prefix + key, value })),
);
}
delete(keys: string[]): Promise<void> {
return store.batchDel(keys.map((key) => this.prefix + key));
}
}
const engine = new SimpleSearchEngine(
new StoreKVStore("fts:"),
new StoreKVStore("fts_rev:"),
);
export async function indexPage({ name, tree }: IndexTreeEvent) {
const text = renderToText(tree);
// console.log("Now FTS indexing", name);
await engine.deleteDocument(name);
await engine.indexDocument({ id: name, text });
}
export async function clearIndex() {
await store.deletePrefix("fts:");
await store.deletePrefix("fts_rev:");
}
export async function pageUnindex(pageName: string) {
await fulltext.fullTextDelete(pageName);
await engine.deleteDocument(pageName);
}
export async function queryProvider({
@@ -29,10 +55,13 @@ export async function queryProvider({
if (!phraseFilter) {
throw Error("No 'phrase' filter specified, this is mandatory");
}
let results = await fulltext.fullTextSearch(phraseFilter.value, {
highlightEllipsis: "...",
limit: 100,
});
let results: any[] = await engine.search(phraseFilter.value);
// Patch the object to a format that users expect (translate id to name)
for (const r of results) {
r.name = r.id;
delete r.id;
}
const allPageMap: Map<string, any> = new Map(
results.map((r: any) => [r.name, r]),
@@ -62,31 +91,22 @@ export async function searchCommand() {
export async function readFileSearch(
name: string,
encoding: FileEncoding,
): Promise<{ data: FileData; meta: FileMeta }> {
): Promise<{ data: string; meta: FileMeta }> {
const phrase = name.substring(
searchPrefix.length,
name.length - ".md".length,
);
const results = await fulltext.fullTextSearch(phrase, {
highlightEllipsis: "...",
highlightPostfix: "==",
highlightPrefix: "==",
summaryMaxLength: 30,
limit: 100,
});
const results = await engine.search(phrase);
const text = `# Search results for "${phrase}"\n${
results
.map((r: any) =>
`[[${r.name}]]:\n> ${r.snippet.split("\n").join("\n> ")}`
)
.join("\n\n")
.map((r) => `* [[${r.id}]] (score ${r.score})`)
.join("\n")
}
`;
`;
return {
// encoding === "arraybuffer" is not an option, so either it's "utf8" or "dataurl"
data: encoding === "utf8" ? text : base64EncodedDataUrl(
data: base64EncodedDataUrl(
"text/markdown",
new TextEncoder().encode(text),
),
@@ -100,6 +120,13 @@ export async function readFileSearch(
};
}
export function writeFileSearch(
name: string,
): FileMeta {
// Never actually writing this
return getFileMetaSearch(name);
}
export function getFileMetaSearch(name: string): FileMeta {
return {
name,
plugs/share/publish.ts
+4 -5
View File
@@ -1,5 +1,5 @@
import { events } from "$sb/plugos-syscall/mod.ts";
import { editor, markdown, system } from "$sb/silverbullet-syscall/mod.ts";
import { editor, markdown } from "$sb/silverbullet-syscall/mod.ts";
import { extractFrontmatter } from "$sb/lib/frontmatter.ts";
import { PublishEvent } from "$sb/app_event.ts";
@@ -8,7 +8,7 @@ export async function publishCommand() {
const text = await editor.getText();
const pageName = await editor.getCurrentPage();
const tree = await markdown.parseMarkdown(text);
const { $share } = extractFrontmatter(tree);
const { $share } = await extractFrontmatter(tree);
if (!$share) {
await editor.flashNotification("Saved.");
return;
@@ -23,15 +23,14 @@ export async function publishCommand() {
await editor.flashNotification("Sharing...");
// Delegate actual publishing to the server
try {
await system.invokeFunction("server", "publish", pageName, $share);
await publish(pageName, $share);
await editor.flashNotification("Done!");
} catch (e: any) {
await editor.flashNotification(e.message, "error");
}
}
// Runs on server side
export async function publish(pageName: string, uris: string[]) {
async function publish(pageName: string, uris: string[]) {
for (const uri of uris) {
const publisher = uri.split(":")[0];
const results = await events.dispatchEvent(
plugs/share/share.plug.yaml
+1 -4
View File
@@ -5,7 +5,4 @@ functions:
command:
name: "Share: Publish"
key: "Ctrl-s"
mac: "Cmd-s"
publish:
path: publish.ts:publish
env: server
mac: "Cmd-s"
plugs/sync/sync.plug.yaml
-44
View File
@@ -1,44 +0,0 @@
name: sync
functions:
configureCommand:
path: sync.ts:configureCommand
command:
name: "Sync: Configure"
disableCommand:
path: sync.ts:disableCommand
command:
name: "Sync: Disable"
syncCommand:
path: sync.ts:syncCommand
command:
name: "Sync: Sync"
key: "Shift-Alt-s"
wipeAndSyncCommand:
path: sync.ts:localWipeAndSyncCommand
command:
name: "Sync: Wipe Local Space and Sync"
syncOpenedPage:
path: sync.ts:syncOpenedPage
events:
- editor:pageLoaded
check:
env: server
path: sync.ts:check
performSync:
env: server
path: sync.ts:performSync
# Sync every minute
cron: "* * * * *"
# Automatically sync the current page upon change
syncPage:
path: sync.ts:syncPage
env: server
events:
- page:saved
plugs/sync/sync.ts
-251
View File
@@ -1,251 +0,0 @@
import { store } from "$sb/plugos-syscall/mod.ts";
import { editor, space, sync, system } from "$sb/silverbullet-syscall/mod.ts";
import type { SyncEndpoint } from "$sb/silverbullet-syscall/sync.ts";
import { readSetting } from "$sb/lib/settings_page.ts";
export async function configureCommand() {
const url = await editor.prompt(
"Enter the URL of the remote space to sync with",
"https://",
);
if (!url) {
return;
}
const user = await editor.prompt("Username (if any):");
let password = undefined;
if (user) {
password = await editor.prompt("Password:");
}
const syncConfig: SyncEndpoint = {
url,
user,
password,
};
try {
await system.invokeFunction("server", "check", syncConfig);
} catch (e: any) {
await editor.flashNotification(
`Sync configuration failed: ${e.message}`,
"error",
);
return;
}
await store.batchSet([
{ key: "sync.config", value: syncConfig },
// Empty initial snapshot
{ key: "sync.snapshot", value: {} },
]);
await editor.flashNotification("Sync configuration saved.");
return syncConfig;
}
export async function syncCommand() {
let config: SyncEndpoint | undefined = await store.get("sync.config");
if (!config) {
config = await configureCommand();
if (!config) {
return;
}
}
await editor.flashNotification("Starting sync...");
try {
await system.invokeFunction("server", "check", config);
const operations = await system.invokeFunction("server", "performSync");
await editor.flashNotification(
`Sync complete. Performed ${operations} operations.`,
);
} catch (e: any) {
await editor.flashNotification(
`Sync failed: ${e.message}`,
"error",
);
}
}
export async function disableCommand() {
if (
!(await editor.confirm(
"Are you sure you want to disable sync?",
))
) {
return;
}
// remove all sync related keys from the store
await store.deletePrefix("sync.");
await editor.flashNotification("Sync disabled.");
}
export async function localWipeAndSyncCommand() {
let config: SyncEndpoint | undefined = await store.get("sync.config");
if (!config) {
config = await configureCommand();
if (!config) {
return;
}
}
if (
!(await editor.confirm(
"Are you sure you want to wipe your local space and sync with the remote?",
))
) {
return;
}
if (
!(await editor.confirm(
"To be clear: this means all local content will be deleted with no way to recover it. Are you sure?",
))
) {
return;
}
console.log("Wiping local pages");
await editor.flashNotification("Now wiping all pages");
for (const page of await space.listPages()) {
console.log("Deleting page", page.name);
await space.deletePage(page.name);
}
console.log("Wiping local attachments");
await editor.flashNotification("Now wiping all attachments");
for (const attachment of await space.listAttachments()) {
console.log("Deleting attachment", attachment.name);
await space.deleteAttachment(attachment.name);
}
console.log("Wiping local sync state");
await store.set("sync.snapshot", {});
// Starting actual sync
await syncCommand();
// And finally loading all plugs
await system.invokeFunction("client", "core.updatePlugsCommand");
}
export async function syncOpenedPage() {
// Is sync on?
if (!(await store.has("sync.config"))) {
// Nope -> exit
return;
}
await system.invokeFunction(
"server",
"syncPage",
await editor.getCurrentPage(),
);
}
// Run on server
export function check(config: SyncEndpoint) {
return sync.check(config);
}
// If a sync takes longer than this, we'll consider it timed out
const syncTimeout = 1000 * 60 * 10; // 10 minutes
// Run on server
export async function performSync() {
const config: SyncEndpoint = await store.get("sync.config");
if (!config) {
// Sync not configured
return;
}
await augmentSettings(config);
// Check if sync not already in progress
const ongoingSync: number | undefined = await store.get("sync.startTime");
if (ongoingSync) {
if (Date.now() - ongoingSync > syncTimeout) {
console.log("Sync timed out, continuing");
} else {
console.log("Sync already in progress");
return;
}
}
// Keep track of sync start time
await store.set("sync.startTime", Date.now());
try {
// Perform actual sync
const snapshot = await store.get("sync.snapshot");
const { snapshot: newSnapshot, operations, error } = await sync.syncAll(
config,
snapshot,
);
// Store snapshot
await store.set("sync.snapshot", newSnapshot);
// Clear sync start time
await store.del("sync.startTime");
if (error) {
console.error("Sync error", error);
throw new Error(error);
}
return operations;
} catch (e: any) {
// Clear sync start time
await store.del("sync.startTime");
console.error("Sync error", e);
}
}
async function augmentSettings(endpoint: SyncEndpoint) {
const syncSettings = await readSetting("sync", {});
if (syncSettings.excludePrefixes) {
endpoint.excludePrefixes = syncSettings.excludePrefixes;
}
}
export async function syncPage(page: string) {
const config: SyncEndpoint = await store.get("sync.config");
if (!config) {
// Sync not configured
return;
}
await augmentSettings(config);
// Check if sync not already in progress
const ongoingSync: number | undefined = await store.get("sync.startTime");
if (ongoingSync) {
if (Date.now() - ongoingSync > syncTimeout) {
console.log("Sync timed out, continuing");
} else {
console.log("Sync already in progress");
return;
}
}
// Keep track of sync start time
await store.set("sync.startTime", Date.now());
const snapshot = await store.get("sync.snapshot");
console.log("Syncing page", page);
try {
const { snapshot: newSnapshot, error } = await sync.syncFile(
config,
snapshot,
`${page}.md`,
);
// Store snapshot
await store.set("sync.snapshot", newSnapshot);
// Clear sync start time
await store.del("sync.startTime");
if (error) {
console.error("Sync error", error);
throw new Error(error);
}
} catch (e: any) {
// Clear sync start time
await store.del("sync.startTime");
console.error("Sync error", e);
}
}
plugs/tasks/tasks.plug.yaml
-3
View File
@@ -1,6 +1,4 @@
name: tasks
imports:
- https://get.silverbullet.md/global.plug.json
syntax:
DeadlineDate:
firstCharacters:
@@ -56,7 +54,6 @@ functions:
contexts:
- DeadlineDate
previewTaskToggle:
env: client
path: ./task.ts:previewTaskToggle
events:
- preview:click