neil/silverbullet
1
0
Fork 0
Code Activity

SilverBullet pivot to become an offline-first PWA (#403)

Browse Source
This commit is contained in:
Zef Hemel
2023-05-23 20:53:53 +02:00
committed by GitHub
parent b256269897
commit 5f484bed57
389 changed files with 4484 additions and 291129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plug-api/lib/frontmatter.ts
+15 -15
View File
@@ -1,24 +1,24 @@
import * as YAML from "yaml";
import { YAML } from "$sb/plugos-syscall/mod.ts";
import {
addParentPointers,
findNodeOfType,
ParseTree,
renderToText,
replaceNodesMatching,
traverseTree,
replaceNodesMatchingAsync,
traverseTreeAsync,
} from "$sb/lib/tree.ts";
// Extracts front matter (or legacy "meta" code blocks) from a markdown document
// optionally removes certain keys from the front matter
export function extractFrontmatter(
export async function extractFrontmatter(
tree: ParseTree,
removeKeys: string[] = [],
): any {
): Promise<any> {
let data: any = {};
addParentPointers(tree);
replaceNodesMatching(tree, (t) => {
await replaceNodesMatchingAsync(tree, async (t) => {
// Find top-level hash tags
if (t.type === "Hashtag") {
// Check if if nested directly into a Paragraph
@@ -38,7 +38,7 @@ export function extractFrontmatter(
const yamlNode = t.children![1].children![0];
const yamlText = renderToText(yamlNode);
try {
const parsedData: any = YAML.parse(yamlText);
const parsedData: any = await YAML.parse(yamlText);
const newData = { ...parsedData };
data = { ...data, ...parsedData };
if (removeKeys.length > 0) {
@@ -51,7 +51,7 @@ export function extractFrontmatter(
}
}
if (removedOne) {
yamlNode.text = YAML.stringify(newData);
yamlNode.text = await YAML.stringify(newData);
}
}
// If nothing is left, let's just delete this whole block
@@ -92,7 +92,7 @@ export function extractFrontmatter(
}
}
if (removedOne) {
codeTextNode.children![0].text = YAML.stringify(newData).trim();
codeTextNode.children![0].text = (await YAML.stringify(newData)).trim();
}
}
// If nothing is left, let's just delete this whole block
@@ -112,26 +112,26 @@ export function extractFrontmatter(
}
// Updates the front matter of a markdown document and returns the text as a rendered string
export function prepareFrontmatterDispatch(
export async function prepareFrontmatterDispatch(
tree: ParseTree,
data: Record<string, any>,
): any {
): Promise<any> {
let dispatchData: any = null;
traverseTree(tree, (t) => {
await traverseTreeAsync(tree, async (t) => {
// Find FrontMatter and parse it
if (t.type === "FrontMatter") {
const bodyNode = t.children![1].children![0];
const yamlText = renderToText(bodyNode);
try {
const parsedYaml = YAML.parse(yamlText) as any;
const parsedYaml = await YAML.parse(yamlText) as any;
const newData = { ...parsedYaml, ...data };
// Patch inline
dispatchData = {
changes: {
from: bodyNode.from,
to: bodyNode.to,
insert: YAML.stringify(newData, { noArrayIndent: true }),
insert: await YAML.stringify(newData),
},
};
} catch (e: any) {
@@ -147,7 +147,7 @@ export function prepareFrontmatterDispatch(
changes: {
from: 0,
to: 0,
insert: "---\n" + YAML.stringify(data, { noArrayIndent: true }) +
insert: "---\n" + await YAML.stringify(data) +
"---\n",
},
};
plug-api/lib/secrets_page.ts
+2 -2
View File
@@ -17,7 +17,7 @@ export async function readSecrets(keys: string[]): Promise<any[]> {
}
return collectedSecrets;
} catch (e: any) {
if (e.message === "Page not found") {
if (e.message === "Not found") {
throw new Error(`No such secret: ${keys[0]}`);
}
throw e;
@@ -34,7 +34,7 @@ export async function readSecret(key: string): Promise<any> {
}
return val;
} catch (e: any) {
if (e.message === "Page not found") {
if (e.message === "Not found") {
throw new Error(`No such secret: ${key}`);
}
throw e;
plug-api/lib/settings_page.ts
+6 -7
View File
@@ -1,6 +1,6 @@
import { readYamlPage } from "./yaml_page.ts";
import { notifyUser } from "./util.ts";
import * as YAML from "yaml";
import { YAML } from "$sb/plugos-syscall/mod.ts";
import { space } from "$sb/silverbullet-syscall/mod.ts";
@@ -30,7 +30,7 @@ export async function readSettings<T extends object>(settings: T): Promise<T> {
}
return collectedSettings as T;
} catch (e: any) {
if (e.message === "Page not found") {
if (e.message === "Not found") {
// No settings yet, return default values for all
return settings;
}
@@ -47,7 +47,7 @@ export async function readSetting(
const val = allSettings[key];
return val === undefined ? defaultValue : val;
} catch (e: any) {
if (e.message === "Page not found") {
if (e.message === "Not found") {
// No settings yet, return default values for all
return defaultValue;
}
@@ -71,10 +71,9 @@ export async function writeSettings<T extends object>(settings: T) {
// const doc = new YAML.Document();
// doc.contents = writeSettings;
const contents =
`This page contains settings for configuring SilverBullet and its Plugs.\nAny changes outside of the yaml block will be overwritten.\n\`\`\`yaml\n${
YAML.stringify(
`This page contains settings for configuring SilverBullet and its Plugs.\nAny changes outside of the yaml block will be overwritten.\n\`\`\`yaml\n${await YAML
.stringify(
writeSettings,
)
}\n\`\`\``; // might need \r\n for windows?
)}\n\`\`\``; // might need \r\n for windows?
await space.writePage(SETTINGS_PAGE, contents);
}
plug-api/lib/tree.test.ts
+2 -2
View File
@@ -73,7 +73,7 @@ Deno.test("Test parsing", () => {
};
}
});
console.log(JSON.stringify(mdTree, null, 2));
// console.log(JSON.stringify(mdTree, null, 2));
let mdTree3 = parse(lang, mdTest3);
console.log(JSON.stringify(mdTree3, null, 2));
// console.log(JSON.stringify(mdTree3, null, 2));
});
plug-api/lib/tree.ts
+51
View File
@@ -69,6 +69,25 @@ export function collectNodesMatching(
return results;
}
export async function collectNodesMatchingAsync(
tree: ParseTree,
matchFn: (tree: ParseTree) => Promise<boolean>,
): Promise<ParseTree[]> {
if (await matchFn(tree)) {
return [tree];
}
let results: ParseTree[] = [];
if (tree.children) {
for (const child of tree.children) {
results = [
...results,
...await collectNodesMatchingAsync(child, matchFn),
];
}
}
return results;
}
// return value: returning undefined = not matched, continue, null = delete, new node = replace
export function replaceNodesMatching(
tree: ParseTree,
@@ -93,6 +112,29 @@ export function replaceNodesMatching(
}
}
export async function replaceNodesMatchingAsync(
tree: ParseTree,
substituteFn: (tree: ParseTree) => Promise<ParseTree | null | undefined>,
) {
if (tree.children) {
const children = tree.children.slice();
for (const child of children) {
const subst = await substituteFn(child);
if (subst !== undefined) {
const pos = tree.children.indexOf(child);
if (subst) {
tree.children.splice(pos, 1, subst);
} else {
// null = delete
tree.children.splice(pos, 1);
}
} else {
replaceNodesMatchingAsync(child, substituteFn);
}
}
}
}
export function findNodeMatching(
tree: ParseTree,
matchFn: (tree: ParseTree) => boolean,
@@ -116,6 +158,15 @@ export function traverseTree(
collectNodesMatching(tree, matchFn);
}
export async function traverseTreeAsync(
tree: ParseTree,
// Return value = should stop traversal?
matchFn: (tree: ParseTree) => Promise<boolean>,
): Promise<void> {
// Do a collect, but ignore the result
await collectNodesMatchingAsync(tree, matchFn);
}
// Finds non-text node at position
export function nodeAtPos(tree: ParseTree, pos: number): ParseTree | null {
if (pos < tree.from! || pos >= tree.to!) {
plug-api/lib/yaml_page.ts
+2 -4
View File
@@ -1,6 +1,6 @@
import { findNodeOfType, traverseTree } from "$sb/lib/tree.ts";
import { markdown, space } from "$sb/silverbullet-syscall/mod.ts";
import * as YAML from "yaml";
import { YAML } from "$sb/plugos-syscall/mod.ts";
export async function readCodeBlockPage(
pageName: string,
@@ -58,8 +58,6 @@ export async function writeYamlPage(
data: any,
prelude = "",
): Promise<void> {
const text = YAML.stringify(data, {
noCompatMode: true,
});
const text = await YAML.stringify(data);
await space.writePage(pageName, prelude + "```yaml\n" + text + "\n```");
}
plug-api/plugos-syscall/fetch.ts
+6 -16
View File
@@ -1,23 +1,13 @@
import type {
ProxyFetchRequest,
ProxyFetchResponse,
} from "../../common/proxy_fetch.ts";
import { base64Decode } from "../../plugos/asset_bundle/base64.ts";
export type SandboxFetchRequest = {
method?: string;
headers?: Record<string, string>;
body?: string;
};
export type SandboxFetchResponse = {
ok: boolean;
status: number;
headers: Record<string, string>;
// We base64 encode the body because the body can be binary data that we have to push through the worker boundary
base64Body: string;
};
export function sandboxFetch(
url: string,
options?: SandboxFetchRequest,
): Promise<SandboxFetchResponse> {
options?: ProxyFetchRequest,
): Promise<ProxyFetchResponse> {
// @ts-ignore: monkey patching fetch
return syscall("sandboxFetch.fetch", url, options);
}
plug-api/plugos-syscall/fs.ts
-48
View File
@@ -1,48 +0,0 @@
import { syscall } from "./syscall.ts";
import type { FileMeta, ProxyFileSystem } from "./types.ts";
export class LocalFileSystem implements ProxyFileSystem {
constructor(readonly root: string) {
}
readFile(
path: string,
encoding: "utf8" | "dataurl" = "utf8",
): Promise<string> {
return syscall("fs.readFile", `${this.root}/${path}`, encoding);
}
async getFileMeta(path: string): Promise<FileMeta> {
return this.removeRootDir(
await syscall("fs.getFileMeta", `${this.root}/${path}`),
);
}
writeFile(
path: string,
text: string,
encoding: "utf8" | "dataurl" = "utf8",
): Promise<FileMeta> {
return syscall("fs.writeFile", `${this.root}/${path}`, text, encoding);
}
deleteFile(path: string): Promise<void> {
return syscall("fs.deleteFile", `${this.root}/${path}`);
}
async listFiles(
dirName: string,
recursive = false,
): Promise<FileMeta[]> {
return (await syscall(
"fs.listFiles",
`${this.root}/${dirName}`,
recursive,
)).map(this.removeRootDir.bind(this));
}
private removeRootDir(fileMeta: FileMeta): FileMeta {
fileMeta.name = fileMeta.name.substring(this.root.length + 1);
return fileMeta;
}
}
plug-api/plugos-syscall/fulltext.ts
-24
View File
@@ -1,24 +0,0 @@
import { syscall } from "./syscall.ts";
export function fullTextIndex(key: string, value: string) {
return syscall("fulltext.index", key, value);
}
export function fullTextDelete(key: string) {
return syscall("fulltext.delete", key);
}
export type FullTextSearchOptions = {
limit?: number;
highlightPrefix?: string;
highlightPostfix?: string;
highlightEllipsis?: string;
summaryMaxLength?: number;
};
export function fullTextSearch(
phrase: string,
options: FullTextSearchOptions = {},
) {
return syscall("fulltext.search", phrase, options);
}
plug-api/plugos-syscall/mod.ts
+1 -4
View File
@@ -1,9 +1,6 @@
export * as asset from "./asset.ts";
export * as events from "./event.ts";
// export * as fs from "./fs.ts";
export { LocalFileSystem } from "./fs.ts";
export * as sandbox from "./sandbox.ts";
export * as fulltext from "./fulltext.ts";
export * as shell from "./shell.ts";
export * as store from "./store.ts";
export * as YAML from "./yaml.ts";
export * from "./syscall.ts";
plug-api/plugos-syscall/sandbox.ts
-5
View File
@@ -1,5 +0,0 @@
import type { LogEntry } from "../../plugos/sandbox.ts";
export function getLogs(): Promise<LogEntry[]> {
return syscall("sandbox.getLogs");
}
plug-api/plugos-syscall/shell.ts
+1 -1
View File
@@ -3,6 +3,6 @@ import { syscall } from "./syscall.ts";
export function run(
cmd: string,
args: string[],
): Promise<{ stdout: string; stderr: string }> {
): Promise<{ stdout: string; stderr: string; code: number }> {
return syscall("shell.run", cmd, args);
}
plug-api/plugos-syscall/store.ts
+4
View File
@@ -31,6 +31,10 @@ export function get(key: string): Promise<any> {
return syscall("store.get", key);
}
export function batchGet(keys: string[]): Promise<(any | undefined)[]> {
return syscall("store.batchGet", keys);
}
export function has(key: string): Promise<boolean> {
return syscall("store.has", key);
}
plug-api/plugos-syscall/types.ts
-25
View File
@@ -1,25 +0,0 @@
export type FileMeta = {
name: string;
lastModified: number;
};
export interface ProxyFileSystem {
readFile(
path: string,
encoding: "utf8" | "dataurl",
): Promise<string>;
getFileMeta(path: string): Promise<FileMeta>;
writeFile(
path: string,
text: string,
encoding: "utf8" | "dataurl",
): Promise<FileMeta>;
deleteFile(path: string): Promise<void>;
listFiles(
path: string,
): Promise<FileMeta[]>;
}
plug-api/plugos-syscall/yaml.ts
+13
View File
@@ -0,0 +1,13 @@
import { syscall } from "./syscall.ts";
export function parse(
text: string,
): Promise<any> {
return syscall("yaml.parse", text);
}
export function stringify(
obj: any,
): Promise<string> {
return syscall("yaml.stringify", obj);
}
plug-api/silverbullet-syscall/editor.ts
+1 -1
View File
@@ -1,5 +1,5 @@
import type { FilterOption } from "../../web/types.ts";
import { syscall } from "./syscall.ts";
import { FilterOption } from "../../common/types.ts";
export function getCurrentPage(): Promise<string> {
return syscall("editor.getCurrentPage");
plug-api/silverbullet-syscall/mod.ts
+3 -3
View File
@@ -1,9 +1,9 @@
export * as clientStore from "./clientStore.ts";
export * as editor from "./editor.ts";
export * as index from "./index.ts";
export * as markdown from "./markdown.ts";
export * as sandbox from "./sandbox.ts";
export { default as space } from "./space.ts";
export * as space from "./space.ts";
export * as system from "./system.ts";
export * as collab from "./collab.ts";
// Legacy redirect, use "store" in $sb/plugos-syscall/mod.ts instead
export * as clientStore from "./store.ts";
export * as sync from "./sync.ts";
plug-api/silverbullet-syscall/sandbox.ts
-5
View File
@@ -1,5 +0,0 @@
import type { LogEntry } from "../../plugos/sandbox.ts";
export function getServerLogs(): Promise<LogEntry[]> {
return syscall("sandbox.getServerLogs");
}
plug-api/silverbullet-syscall/space.ts
+66 -94
View File
@@ -1,98 +1,70 @@
import { syscall } from "./syscall.ts";
import { AttachmentMeta, PageMeta } from "../../common/types.ts";
import { FileMeta, ProxyFileSystem } from "../plugos-syscall/types.ts";
import type { AttachmentMeta, PageMeta } from "../../web/types.ts";
export class SpaceFileSystem implements ProxyFileSystem {
// More space-specific methods
listPages(unfiltered = false): Promise<PageMeta[]> {
return syscall("space.listPages", unfiltered);
}
getPageMeta(name: string): Promise<PageMeta> {
return syscall("space.getPageMeta", name);
}
readPage(
name: string,
): Promise<string> {
return syscall("space.readPage", name);
}
writePage(name: string, text: string): Promise<PageMeta> {
return syscall("space.writePage", name, text);
}
deletePage(name: string): Promise<void> {
return syscall("space.deletePage", name);
}
listPlugs(): Promise<string[]> {
return syscall("space.listPlugs");
}
listAttachments(): Promise<PageMeta[]> {
return syscall("space.listAttachments");
}
getAttachmentMeta(name: string): Promise<AttachmentMeta> {
return syscall("space.getAttachmentMeta", name);
}
/**
* Read an attachment from the space
* @param name path of the attachment to read
* @returns the attachment data encoded as a data URL
*/
readAttachment(
name: string,
): Promise<string> {
return syscall("space.readAttachment", name);
}
/**
* Writes an attachment to the space
* @param name path of the attachment to write
* @param encoding encoding of the data ("utf8" or "dataurl)
* @param data data itself
* @returns
*/
writeAttachment(
name: string,
encoding: "utf8" | "dataurl",
data: string,
): Promise<AttachmentMeta> {
return syscall("space.writeAttachment", name, encoding, data);
}
/**
* Deletes an attachment from the space
* @param name path of the attachment to delete
*/
deleteAttachment(name: string): Promise<void> {
return syscall("space.deleteAttachment", name);
}
// Filesystem implementation
readFile(path: string, encoding: "dataurl" | "utf8"): Promise<string> {
return syscall("space.readFile", path, encoding);
}
getFileMeta(path: string): Promise<FileMeta> {
return syscall("space.getFileMeta", path);
}
writeFile(
path: string,
text: string,
encoding: "dataurl" | "utf8",
): Promise<FileMeta> {
return syscall("space.writeFile", path, text, encoding);
}
deleteFile(path: string): Promise<void> {
return syscall("space.deleteFile", path);
}
listFiles(path: string): Promise<FileMeta[]> {
return syscall("space.listFiles", path);
}
export function listPages(unfiltered = false): Promise<PageMeta[]> {
return syscall("space.listPages", unfiltered);
}
export default new SpaceFileSystem();
export function getPageMeta(name: string): Promise<PageMeta> {
return syscall("space.getPageMeta", name);
}
export function readPage(
name: string,
): Promise<string> {
return syscall("space.readPage", name);
}
export function writePage(name: string, text: string): Promise<PageMeta> {
return syscall("space.writePage", name, text);
}
export function deletePage(name: string): Promise<void> {
return syscall("space.deletePage", name);
}
export function listPlugs(): Promise<string[]> {
return syscall("space.listPlugs");
}
export function listAttachments(): Promise<PageMeta[]> {
return syscall("space.listAttachments");
}
export function getAttachmentMeta(name: string): Promise<AttachmentMeta> {
return syscall("space.getAttachmentMeta", name);
}
/**
* Read an attachment from the space
* @param name path of the attachment to read
* @returns the attachment data encoded as a data URL
*/
export function readAttachment(
name: string,
): Promise<string> {
return syscall("space.readAttachment", name);
}
/**
* Writes an attachment to the space
* @param name path of the attachment to write
* @param encoding encoding of the data ("utf8" or "dataurl)
* @param data data itself
* @returns
*/
export function writeAttachment(
name: string,
encoding: "utf8" | "dataurl",
data: string,
): Promise<AttachmentMeta> {
return syscall("space.writeAttachment", name, encoding, data);
}
/**
* Deletes an attachment from the space
* @param name path of the attachment to delete
*/
export function deleteAttachment(name: string): Promise<void> {
return syscall("space.deleteAttachment", name);
}
plug-api/silverbullet-syscall/clientStore.ts → plug-api/silverbullet-syscall/store.ts
+3 -3
View File
@@ -1,13 +1,13 @@
import { syscall } from "./syscall.ts";
export function set(key: string, value: any): Promise<void> {
return syscall("clientStore.set", key, value);
return syscall("store.set", key, value);
}
export function get(key: string): Promise<any> {
return syscall("clientStore.get", key);
return syscall("store.get", key);
}
export function del(key: string): Promise<void> {
return syscall("clientStore.delete", key);
return syscall("store.delete", key);
}
plug-api/silverbullet-syscall/sync.ts
+5 -41
View File
@@ -1,45 +1,9 @@
import type { SyncStatusItem } from "../../common/spaces/sync.ts";
import { syscall } from "./syscall.ts";
import { syscall } from "$sb/silverbullet-syscall/syscall.ts";
export type SyncEndpoint = {
url: string;
user?: string;
password?: string;
excludePrefixes?: string[];
};
// Perform a sync with the server, based on the given status (to be persisted)
// returns a new sync status to persist
export function syncAll(
endpoint: SyncEndpoint,
snapshot: Record<string, SyncStatusItem>,
): Promise<
{
snapshot: Record<string, SyncStatusItem>;
operations: number;
error?: string;
}
> {
return syscall("sync.syncAll", endpoint, snapshot);
export function isSyncing(): Promise<boolean> {
return syscall("sync.isSyncing");
}
// Perform a sync with the server, based on the given status (to be persisted)
// returns a new sync status to persist
export function syncFile(
endpoint: SyncEndpoint,
snapshot: Record<string, SyncStatusItem>,
name: string,
): Promise<
{
snapshot: Record<string, SyncStatusItem>;
operations: number;
error?: string;
}
> {
return syscall("sync.syncFile", endpoint, snapshot, name);
}
// Checks the sync endpoint for connectivity and authentication, throws and Error on failure
export function check(endpoint: SyncEndpoint): Promise<void> {
return syscall("sync.check", endpoint);
export function hasInitialSyncCompleted(): Promise<boolean> {
return syscall("sync.hasInitialSyncCompleted");
}