neil/silverbullet
1
0
Fork 0
Code Activity

SilverBullet pivot to become an offline-first PWA (#403)

Browse Source
This commit is contained in:
Zef Hemel
2023-05-23 20:53:53 +02:00
committed by GitHub
parent b256269897
commit 5f484bed57
389 changed files with 4484 additions and 291129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/crypto.ts
+12
View File
@@ -0,0 +1,12 @@
export function simpleHash(s: string): number {
let hash = 0,
i,
chr;
if (s.length === 0) return hash;
for (i = 0; i < s.length; i++) {
chr = s.charCodeAt(i);
hash = ((hash << 5) - hash) + chr;
hash |= 0; // Convert to 32bit integer
}
return hash;
}
common/deps.ts
+2
View File
@@ -122,3 +122,5 @@ export {
javascriptLanguage,
typescriptLanguage,
} from "https://esm.sh/@codemirror/lang-javascript@6.1.4?external=@codemirror/language,@codemirror/autocomplete,@codemirror/view,@codemirror/state,@codemirror/lint,@lezer/common,@lezer/lr,@lezer/javascript,@codemirror/commands";
export { mime } from "https://deno.land/x/mimetypes@v1.0.0/mod.ts";
common/manifest.ts
-2
View File
@@ -1,5 +1,4 @@
import * as plugos from "../plugos/types.ts";
import { EndpointHookT } from "../plugos/hooks/endpoint.ts";
import { CronHookT } from "../plugos/hooks/cron.ts";
import { EventHookT } from "../plugos/hooks/event.ts";
import { CommandHookT } from "../web/hooks/command.ts";
@@ -10,7 +9,6 @@ import { CodeWidgetT } from "../web/hooks/code_widget.ts";
export type SilverBulletHooks =
& CommandHookT
& SlashCommandHookT
& EndpointHookT
& CronHookT
& EventHookT
& CodeWidgetT
common/proxy_fetch.ts
+37
View File
@@ -0,0 +1,37 @@
import { base64Encode } from "../plugos/asset_bundle/base64.ts";
export type ProxyFetchRequest = {
method?: string;
headers?: Record<string, string>;
body?: string;
};
export type ProxyFetchResponse = {
ok: boolean;
status: number;
headers: Record<string, string>;
// We base64 encode the body because the body can be binary data that we have to push through the worker boundary
base64Body: string;
};
export async function performLocalFetch(
url: string,
req: ProxyFetchRequest,
): Promise<ProxyFetchResponse> {
const result = await fetch(
url,
req && {
method: req.method,
headers: req.headers,
body: req.body,
},
);
return {
ok: result.ok,
status: result.status,
headers: Object.fromEntries(result.headers.entries()),
base64Body: base64Encode(
new Uint8Array(await (await result.blob()).arrayBuffer()),
),
};
}
common/spaces/asset_bundle_space_primitives.ts
+21 -27
View File
@@ -1,7 +1,7 @@
import { Plug } from "../../plugos/plug.ts";
import { FileMeta } from "../types.ts";
import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives.ts";
import { SpacePrimitives } from "./space_primitives.ts";
import { AssetBundle } from "../../plugos/asset_bundle/bundle.ts";
import { mime } from "../deps.ts";
const bootTime = Date.now();
export class AssetBundlePlugSpacePrimitives implements SpacePrimitives {
@@ -13,10 +13,10 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives {
async fetchFileList(): Promise<FileMeta[]> {
const files = await this.wrapped.fetchFileList();
return this.assetBundle.listFiles().filter((p) => p.startsWith("_plug/"))
return this.assetBundle.listFiles()
.map((p) => ({
name: p,
contentType: "application/json",
contentType: mime.getType(p) || "application/octet-stream",
lastModified: bootTime,
perm: "ro",
size: -1,
@@ -25,22 +25,21 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives {
readFile(
name: string,
encoding: FileEncoding,
): Promise<{ data: FileData; meta: FileMeta }> {
): Promise<{ data: Uint8Array; meta: FileMeta }> {
if (this.assetBundle.has(name)) {
const data = this.assetBundle.readFileSync(name);
// console.log("Requested encoding", encoding);
return Promise.resolve({
data: encoding === "utf8" ? new TextDecoder().decode(data) : data,
data,
meta: {
lastModified: bootTime,
size: data.byteLength,
perm: "ro",
contentType: "application/json",
contentType: this.assetBundle.getMimeType(name),
} as FileMeta,
});
}
return this.wrapped.readFile(name, encoding);
return this.wrapped.readFile(name);
}
getFileMeta(name: string): Promise<FileMeta> {
@@ -50,7 +49,7 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives {
lastModified: bootTime,
size: data.byteLength,
perm: "ro",
contentType: "application/json",
contentType: this.assetBundle.getMimeType(name),
} as FileMeta);
}
return this.wrapped.getFileMeta(name);
@@ -58,11 +57,20 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives {
writeFile(
name: string,
encoding: FileEncoding,
data: FileData,
data: Uint8Array,
selfUpdate?: boolean,
lastModified?: number,
): Promise<FileMeta> {
return this.wrapped.writeFile(name, encoding, data, selfUpdate);
if (this.assetBundle.has(name)) {
console.warn("Attempted to write to read-only asset file", name);
return this.getFileMeta(name);
}
return this.wrapped.writeFile(
name,
data,
selfUpdate,
lastModified,
);
}
deleteFile(name: string): Promise<void> {
@@ -72,18 +80,4 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives {
}
return this.wrapped.deleteFile(name);
}
// deno-lint-ignore no-explicit-any
proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
return this.wrapped.proxySyscall(plug, name, args);
}
invokeFunction(
plug: Plug<any>,
env: string,
name: string,
args: any[],
): Promise<any> {
return this.wrapped.invokeFunction(plug, env, name, args);
}
}
common/spaces/disk_space_primitives.ts
+33 -61
View File
@@ -2,14 +2,9 @@
import { path } from "../deps.ts";
import { readAll } from "../deps.ts";
import { FileMeta } from "../types.ts";
import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives.ts";
import { Plug } from "../../plugos/plug.ts";
import { SpacePrimitives } from "./space_primitives.ts";
import { mime } from "https://deno.land/x/mimetypes@v1.0.0/mod.ts";
import {
base64DecodeDataUrl,
base64EncodedDataUrl,
} from "../../plugos/asset_bundle/base64.ts";
import { walk } from "../../plugos/deps.ts";
import { walk } from "https://deno.land/std@0.165.0/fs/walk.ts";
function lookupContentType(path: string): string {
return mime.getType(path) || "application/octet-stream";
@@ -21,10 +16,14 @@ function normalizeForwardSlashPath(path: string) {
const excludedFiles = ["data.db", "data.db-journal", "sync.json"];
export type DiskSpaceOptions = {
maxFileSizeMB?: number;
};
export class DiskSpacePrimitives implements SpacePrimitives {
rootPath: string;
constructor(rootPath: string) {
constructor(rootPath: string, private options: DiskSpaceOptions = {}) {
this.rootPath = Deno.realPathSync(rootPath);
}
@@ -46,36 +45,16 @@ export class DiskSpacePrimitives implements SpacePrimitives {
async readFile(
name: string,
encoding: FileEncoding,
): Promise<{ data: FileData; meta: FileMeta }> {
): Promise<{ data: Uint8Array; meta: FileMeta }> {
const localPath = this.filenameToPath(name);
try {
const s = await Deno.stat(localPath);
let data: FileData | null = null;
const contentType = lookupContentType(name);
switch (encoding) {
case "utf8":
data = await Deno.readTextFile(localPath);
break;
case "dataurl":
{
const f = await Deno.open(localPath, { read: true });
const buf = await readAll(f);
Deno.close(f.rid);
data = base64EncodedDataUrl(contentType, buf);
}
break;
case "arraybuffer":
{
const f = await Deno.open(localPath, { read: true });
const buf = await readAll(f);
Deno.close(f.rid);
const f = await Deno.open(localPath, { read: true });
const data = await readAll(f);
Deno.close(f.rid);
data = buf.buffer;
}
break;
}
return {
data,
meta: {
@@ -94,29 +73,29 @@ export class DiskSpacePrimitives implements SpacePrimitives {
async writeFile(
name: string,
encoding: FileEncoding,
data: FileData,
data: Uint8Array,
_selfUpdate?: boolean,
lastModified?: number,
): Promise<FileMeta> {
const localPath = this.filenameToPath(name);
try {
// Ensure parent folder exists
await Deno.mkdir(path.dirname(localPath), { recursive: true });
const file = await Deno.open(localPath, {
write: true,
create: true,
truncate: true,
});
// Actually write the file
switch (encoding) {
case "utf8":
await Deno.writeTextFile(`${localPath}`, data as string);
break;
case "dataurl":
await Deno.writeFile(
localPath,
base64DecodeDataUrl(data as string),
);
break;
case "arraybuffer":
await Deno.writeFile(localPath, new Uint8Array(data as ArrayBuffer));
break;
await Deno.write(file.rid, data);
if (lastModified) {
console.log("Seting mtime to", new Date(lastModified));
await Deno.futime(file.rid, new Date(), new Date(lastModified));
}
file.close();
// Fetch new metadata
const s = await Deno.stat(localPath);
@@ -171,6 +150,13 @@ export class DiskSpacePrimitives implements SpacePrimitives {
const fullPath = file.path;
try {
const s = await Deno.stat(fullPath);
// Don't list file exceeding the maximum file size
if (
this.options.maxFileSizeMB &&
s.size / (1024 * 1024) > this.options.maxFileSizeMB
) {
continue;
}
const name = fullPath.substring(this.rootPath.length + 1);
if (excludedFiles.includes(name)) {
continue;
@@ -193,20 +179,6 @@ export class DiskSpacePrimitives implements SpacePrimitives {
return allFiles;
}
// Plugs
invokeFunction(
plug: Plug<any>,
_env: string,
name: string,
args: any[],
): Promise<any> {
return plug.invoke(name, args);
}
proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
return plug.syscall(name, args);
}
}
function escapeRegExp(string: string) {
common/spaces/evented_space_primitives.ts
+11 -35
View File
@@ -1,8 +1,7 @@
import { EventHook } from "../../plugos/hooks/event.ts";
import { Plug } from "../../plugos/plug.ts";
import { FileMeta } from "../types.ts";
import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives.ts";
import type { SpacePrimitives } from "./space_primitives.ts";
export class EventedSpacePrimitives implements SpacePrimitives {
constructor(private wrapped: SpacePrimitives, private eventHook: EventHook) {}
@@ -11,56 +10,30 @@ export class EventedSpacePrimitives implements SpacePrimitives {
return this.wrapped.fetchFileList();
}
proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
return this.wrapped.proxySyscall(plug, name, args);
}
invokeFunction(
plug: Plug<any>,
env: string,
name: string,
args: any[],
): Promise<any> {
return this.wrapped.invokeFunction(plug, env, name, args);
}
readFile(
name: string,
encoding: FileEncoding,
): Promise<{ data: FileData; meta: FileMeta }> {
return this.wrapped.readFile(name, encoding);
): Promise<{ data: Uint8Array; meta: FileMeta }> {
return this.wrapped.readFile(name);
}
async writeFile(
name: string,
encoding: FileEncoding,
data: FileData,
data: Uint8Array,
selfUpdate?: boolean,
lastModified?: number,
): Promise<FileMeta> {
const newMeta = await this.wrapped.writeFile(
name,
encoding,
data,
selfUpdate,
lastModified,
);
// This can happen async
if (name.endsWith(".md")) {
const pageName = name.substring(0, name.length - 3);
let text = "";
switch (encoding) {
case "utf8":
text = data as string;
break;
case "arraybuffer":
{
const decoder = new TextDecoder("utf-8");
text = decoder.decode(data as ArrayBuffer);
}
break;
case "dataurl":
throw Error("Data urls not supported in this context");
}
const decoder = new TextDecoder("utf-8");
text = decoder.decode(data);
this.eventHook
.dispatchEvent("page:saved", pageName)
@@ -74,6 +47,9 @@ export class EventedSpacePrimitives implements SpacePrimitives {
console.error("Error dispatching page:saved event", e);
});
}
if (name.endsWith(".plug.js")) {
await this.eventHook.dispatchEvent("plug:changed", name);
}
return newMeta;
}
common/spaces/fallback_space_primitives.ts
+45
View File
@@ -0,0 +1,45 @@
import { FileMeta } from "../types.ts";
import type { SpacePrimitives } from "./space_primitives.ts";
/**
* FallbackSpacePrimitives is a SpacePrimitives implementation that will try to fall back to another SpacePrimitives implementation for two
* operations:
* - readFile
* - getFileMeta
* The use case is primarily sync: when sync hasn't completed yet, we can fall back to HttpSpacePrimitives to fetch the file from the server.
*/
export class FallbackSpacePrimitives implements SpacePrimitives {
constructor(
private primary: SpacePrimitives,
private fallback: SpacePrimitives,
) {
}
fetchFileList(): Promise<FileMeta[]> {
return this.primary.fetchFileList();
}
async readFile(name: string): Promise<{ data: Uint8Array; meta: FileMeta }> {
try {
return await this.primary.readFile(name);
} catch {
return this.fallback.readFile(name);
}
}
async getFileMeta(name: string): Promise<FileMeta> {
try {
return await this.primary.getFileMeta(name);
} catch {
return this.fallback.getFileMeta(name);
}
}
writeFile(
name: string,
data: Uint8Array,
selfUpdate?: boolean | undefined,
lastModified?: number | undefined,
): Promise<FileMeta> {
return this.primary.writeFile(name, data, selfUpdate, lastModified);
}
deleteFile(name: string): Promise<void> {
return this.primary.deleteFile(name);
}
}
common/spaces/file_meta_space_primitives.ts
+11 -22
View File
@@ -1,6 +1,5 @@
import { Plug } from "../../plugos/plug.ts";
import { FileMeta } from "../types.ts";
import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives.ts";
import { SpacePrimitives } from "./space_primitives.ts";
import type { SysCallMapping } from "../../plugos/system.ts";
// Enriches the file list listing with custom metadata from the page index
@@ -40,9 +39,8 @@ export class FileMetaSpacePrimitives implements SpacePrimitives {
readFile(
name: string,
encoding: FileEncoding,
): Promise<{ data: FileData; meta: FileMeta }> {
return this.wrapped.readFile(name, encoding);
): Promise<{ data: Uint8Array; meta: FileMeta }> {
return this.wrapped.readFile(name);
}
getFileMeta(name: string): Promise<FileMeta> {
@@ -51,28 +49,19 @@ export class FileMetaSpacePrimitives implements SpacePrimitives {
writeFile(
name: string,
encoding: FileEncoding,
data: FileData,
data: Uint8Array,
selfUpdate?: boolean,
lastModified?: number,
): Promise<FileMeta> {
return this.wrapped.writeFile(name, encoding, data, selfUpdate);
return this.wrapped.writeFile(
name,
data,
selfUpdate,
lastModified,
);
}
deleteFile(name: string): Promise<void> {
return this.wrapped.deleteFile(name);
}
// deno-lint-ignore no-explicit-any
proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
return this.wrapped.proxySyscall(plug, name, args);
}
invokeFunction(
plug: Plug<any>,
env: string,
name: string,
args: any[],
): Promise<any> {
return this.wrapped.invokeFunction(plug, env, name, args);
}
}
common/spaces/http_space_primitives.ts
+46 -143
View File
@@ -1,132 +1,93 @@
import { FileMeta } from "../types.ts";
import { Plug } from "../../plugos/plug.ts";
import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives.ts";
import {
base64DecodeDataUrl,
base64Encode,
base64EncodedDataUrl,
} from "../../plugos/asset_bundle/base64.ts";
import { mime } from "../../plugos/deps.ts";
import { SpacePrimitives } from "./space_primitives.ts";
import { flushCachesAndUnregisterServiceWorker } from "../sw_util.ts";
export class HttpSpacePrimitives implements SpacePrimitives {
private fsUrl: string;
private plugUrl: string;
constructor(
url: string,
readonly user?: string,
readonly password?: string,
readonly base64Put?: boolean,
readonly url: string,
readonly expectedSpacePath?: string,
readonly syncMode = false,
) {
this.fsUrl = url + "/fs";
this.plugUrl = url + "/plug";
}
private async authenticatedFetch(
public async authenticatedFetch(
url: string,
options: Record<string, any>,
options: RequestInit,
): Promise<Response> {
if (this.user && this.password) {
// Explicitly set an auth cookie
if (!options.headers) {
options.headers = {};
}
options.headers["cookie"] = `auth=${
btoa(`${this.user}:${this.password}`)
}`;
if (!options.headers) {
options.headers = {};
}
if (this.syncMode) {
options.headers = { ...options.headers, ...{ "X-Sync-Mode": "true" } };
}
const result = await fetch(url, options);
if (result.status === 401 || result.redirected) {
// Invalid credentials, reloading the browser should trigger authentication
if (typeof location !== "undefined") {
location.reload();
}
throw Error("Unauthorized");
const result = await fetch(url, { ...options });
if (
result.status === 401
) {
// Invalid credentials, reloading the browser should trigger authentication
console.log("Going to redirect after", url);
location.href = "/.auth?refer=" + location.pathname;
throw new Error("Invalid credentials");
}
return result;
}
async fetchFileList(): Promise<FileMeta[]> {
const req = await this.authenticatedFetch(this.fsUrl, {
const resp = await this.authenticatedFetch(this.url, {
method: "GET",
});
return req.json();
if (
resp.status === 200 &&
this.expectedSpacePath &&
resp.headers.get("X-Space-Path") !== this.expectedSpacePath
) {
await flushCachesAndUnregisterServiceWorker();
alert("Space folder path different on server, reloading the page");
location.reload();
}
return resp.json();
}
async readFile(
name: string,
encoding: FileEncoding,
): Promise<{ data: FileData; meta: FileMeta }> {
): Promise<{ data: Uint8Array; meta: FileMeta }> {
const res = await this.authenticatedFetch(
`${this.fsUrl}/${encodeURI(name)}`,
`${this.url}/${encodeURI(name)}`,
{
method: "GET",
},
);
if (res.status === 404) {
throw new Error(`Page not found`);
}
let data: FileData | null = null;
switch (encoding) {
case "arraybuffer":
{
data = await res.arrayBuffer();
}
break;
case "dataurl":
{
data = base64EncodedDataUrl(
mime.getType(name) || "application/octet-stream",
new Uint8Array(await res.arrayBuffer()),
);
}
break;
case "utf8":
data = await res.text();
break;
throw new Error(`Not found`);
}
return {
data: data,
data: new Uint8Array(await res.arrayBuffer()),
meta: this.responseToMeta(name, res),
};
}
async writeFile(
name: string,
encoding: FileEncoding,
data: FileData,
data: Uint8Array,
_selfUpdate?: boolean,
lastModified?: number,
): Promise<FileMeta> {
let body: any = null;
switch (encoding) {
case "arraybuffer":
// actually we want an Uint8Array
body = data instanceof ArrayBuffer ? new Uint8Array(data) : data;
break;
case "utf8":
body = data;
break;
case "dataurl":
data = base64DecodeDataUrl(data as string);
break;
}
const headers: Record<string, string> = {
"Content-Type": "application/octet-stream",
};
if (this.base64Put) {
headers["X-Content-Base64"] = "true";
headers["Content-Type"] = "text/plain";
body = base64Encode(body);
if (lastModified) {
headers["X-Last-Modified"] = "" + lastModified;
}
const res = await this.authenticatedFetch(
`${this.fsUrl}/${encodeURI(name)}`,
`${this.url}/${encodeURI(name)}`,
{
method: "PUT",
headers,
body,
body: data,
},
);
const newMeta = this.responseToMeta(name, res);
@@ -135,7 +96,7 @@ export class HttpSpacePrimitives implements SpacePrimitives {
async deleteFile(name: string): Promise<void> {
const req = await this.authenticatedFetch(
`${this.fsUrl}/${encodeURI(name)}`,
`${this.url}/${encodeURI(name)}`,
{
method: "DELETE",
},
@@ -147,13 +108,13 @@ export class HttpSpacePrimitives implements SpacePrimitives {
async getFileMeta(name: string): Promise<FileMeta> {
const res = await this.authenticatedFetch(
`${this.fsUrl}/${encodeURI(name)}`,
`${this.url}/${encodeURI(name)}`,
{
method: "OPTIONS",
},
);
if (res.status === 404) {
throw new Error(`File not found`);
throw new Error(`Not found`);
}
return this.responseToMeta(name, res);
}
@@ -167,62 +128,4 @@ export class HttpSpacePrimitives implements SpacePrimitives {
perm: (res.headers.get("X-Permission") as "rw" | "ro") || "rw",
};
}
// Plugs
async proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
const req = await this.authenticatedFetch(
`${this.plugUrl}/${plug.name}/syscall/${name}`,
{
method: "POST",
headers: {
"Content-type": "application/json",
},
body: JSON.stringify(args),
},
);
if (req.status !== 200) {
const error = await req.text();
throw Error(error);
}
if (req.headers.get("Content-length") === "0") {
return;
}
return await req.json();
}
async invokeFunction(
plug: Plug<any>,
env: string,
name: string,
args: any[],
): Promise<any> {
// Invoke locally
if (!env || env === "client") {
return plug.invoke(name, args);
}
// Or dispatch to server
const req = await this.authenticatedFetch(
`${this.plugUrl}/${plug.name}/function/${name}`,
{
method: "POST",
headers: {
"Content-type": "application/json",
},
body: JSON.stringify(args),
},
);
if (req.status !== 200) {
const error = await req.text();
throw Error(error);
}
if (req.headers.get("Content-length") === "0") {
return;
}
if (req.headers.get("Content-type")?.includes("application/json")) {
return await req.json();
} else {
return await req.text();
}
}
}
common/spaces/indexeddb_space_primitives.test.ts
+35
View File
@@ -0,0 +1,35 @@
import { indexedDB } from "https://deno.land/x/indexeddb@v1.1.0/ponyfill_memory.ts";
import { IndexedDBSpacePrimitives } from "./indexeddb_space_primitives.ts";
import { assertEquals } from "../../test_deps.ts";
Deno.test("IndexedDBSpacePrimitives", async () => {
const space = new IndexedDBSpacePrimitives("test", indexedDB);
const files = await space.fetchFileList();
assertEquals(files, []);
// Write text file
const fileMeta = await space.writeFile(
"test.txt",
stringToBytes("Hello World"),
);
assertEquals(
(await space.readFile("test.txt")).data,
stringToBytes("Hello World"),
);
const fbContent = (await space.readFile("test.txt"))
.data;
assertEquals(new TextDecoder().decode(fbContent), "Hello World");
assertEquals(await space.fetchFileList(), [fileMeta]);
const buf = new Uint8Array([1, 2, 3, 4, 5]);
// Write binary file
await space.writeFile("test.bin", buf);
const fMeta = await space.getFileMeta("test.bin");
assertEquals(fMeta.size, 5);
assertEquals((await space.fetchFileList()).length, 2);
await space.deleteFile("test.bin");
assertEquals(await space.fetchFileList(), [fileMeta]);
});
function stringToBytes(str: string): Uint8Array {
return new TextEncoder().encode(str);
}
common/spaces/indexeddb_space_primitives.ts
+87
View File
@@ -0,0 +1,87 @@
import type { FileMeta } from "../types.ts";
import type { SpacePrimitives } from "./space_primitives.ts";
import Dexie, { Table } from "dexie";
import { mime } from "../deps.ts";
export type FileContent = {
name: string;
data: Uint8Array;
};
export class IndexedDBSpacePrimitives implements SpacePrimitives {
private db: Dexie;
filesMetaTable: Table<FileMeta, string>;
filesContentTable: Table<FileContent, string>;
constructor(
dbName: string,
indexedDB?: any,
) {
this.db = new Dexie(dbName, {
indexedDB,
});
this.db.version(1).stores({
fileMeta: "name",
fileContent: "name",
});
this.filesMetaTable = this.db.table("fileMeta");
this.filesContentTable = this.db.table<FileContent, string>("fileContent");
}
fetchFileList(): Promise<FileMeta[]> {
return this.filesMetaTable.toArray();
}
async readFile(
name: string,
): Promise<{ data: Uint8Array; meta: FileMeta }> {
const fileMeta = await this.filesMetaTable.get(name);
if (!fileMeta) {
throw new Error("Not found");
}
const fileContent = await this.filesContentTable.get(name);
if (!fileContent) {
throw new Error("Not found");
}
return {
data: fileContent.data,
meta: fileMeta,
};
}
async writeFile(
name: string,
data: Uint8Array,
_selfUpdate?: boolean,
lastModified?: number,
): Promise<FileMeta> {
const fileMeta: FileMeta = {
name,
lastModified: lastModified || Date.now(),
contentType: mime.getType(name) || "application/octet-stream",
size: data.byteLength,
perm: "rw",
};
await this.filesContentTable.put({ name, data });
await this.filesMetaTable.put(fileMeta);
return fileMeta;
}
async deleteFile(name: string): Promise<void> {
const fileMeta = await this.filesMetaTable.get(name);
if (!fileMeta) {
throw new Error("Not found");
}
await this.filesMetaTable.delete(name);
await this.filesContentTable.delete(name);
}
async getFileMeta(name: string): Promise<FileMeta> {
const fileMeta = await this.filesMetaTable.get(name);
if (!fileMeta) {
throw new Error("Not found");
}
return fileMeta;
}
}
common/spaces/plug_space_primitives.ts
+43 -42
View File
@@ -1,15 +1,14 @@
import { Plug } from "../../plugos/plug.ts";
import {
FileData,
FileEncoding,
SpacePrimitives,
} from "../../common/spaces/space_primitives.ts";
import { SpacePrimitives } from "../../common/spaces/space_primitives.ts";
import { FileMeta } from "../../common/types.ts";
import {
NamespaceOperation,
PageNamespaceHook,
} from "../hooks/page_namespace.ts";
import { base64DecodeDataUrl } from "../../plugos/asset_bundle/base64.ts";
import {
base64DecodeDataUrl,
base64EncodedDataUrl,
} from "../../plugos/asset_bundle/base64.ts";
import { mime } from "../deps.ts";
export class PlugSpacePrimitives implements SpacePrimitives {
constructor(
@@ -18,19 +17,34 @@ export class PlugSpacePrimitives implements SpacePrimitives {
private env?: string,
) {}
// Used e.g. by the sync engine to see if it should sync a certain path (likely not the case when we have a plug space override)
public isLikelyHandled(path: string): boolean {
for (
const { pattern, env } of this.hook.spaceFunctions
) {
if (
path.match(pattern) &&
(!this.env || (env && env === this.env))
) {
return true;
}
}
return false;
}
performOperation(
type: NamespaceOperation,
pageName: string,
path: string,
...args: any[]
): Promise<any> | false {
for (
const { operation, pattern, plug, name, env } of this.hook.spaceFunctions
) {
if (
operation === type && pageName.match(pattern) &&
operation === type && path.match(pattern) &&
(!this.env || (env && env === this.env))
) {
return plug.invoke(name, [pageName, ...args]);
return plug.invoke(name, [path, ...args]);
}
}
return false;
@@ -58,26 +72,19 @@ export class PlugSpacePrimitives implements SpacePrimitives {
async readFile(
name: string,
encoding: FileEncoding,
): Promise<{ data: FileData; meta: FileMeta }> {
const wantArrayBuffer = encoding === "arraybuffer";
const result: { data: FileData; meta: FileMeta } | false = await this
): Promise<{ data: Uint8Array; meta: FileMeta }> {
const result: { data: string; meta: FileMeta } | false = await this
.performOperation(
"readFile",
name,
wantArrayBuffer ? "dataurl" : encoding,
);
if (result) {
if (wantArrayBuffer) {
return {
data: base64DecodeDataUrl(result.data as string),
meta: result.meta,
};
} else {
return result;
}
return {
data: base64DecodeDataUrl(result.data),
meta: result.meta,
};
}
return this.wrapped.readFile(name, encoding);
return this.wrapped.readFile(name);
}
getFileMeta(name: string): Promise<FileMeta> {
@@ -90,22 +97,29 @@ export class PlugSpacePrimitives implements SpacePrimitives {
writeFile(
name: string,
encoding: FileEncoding,
data: FileData,
data: Uint8Array,
selfUpdate?: boolean,
lastModified?: number,
): Promise<FileMeta> {
const result = this.performOperation(
"writeFile",
name,
encoding,
data,
base64EncodedDataUrl(
mime.getType(name) || "application/octet-stream",
data,
),
selfUpdate,
);
if (result) {
return result;
}
return this.wrapped.writeFile(name, encoding, data, selfUpdate);
return this.wrapped.writeFile(
name,
data,
selfUpdate,
lastModified,
);
}
deleteFile(name: string): Promise<void> {
@@ -115,17 +129,4 @@ export class PlugSpacePrimitives implements SpacePrimitives {
}
return this.wrapped.deleteFile(name);
}
proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
return this.wrapped.proxySyscall(plug, name, args);
}
invokeFunction(
plug: Plug<any>,
env: string,
name: string,
args: any[],
): Promise<any> {
return this.wrapped.invokeFunction(plug, env, name, args);
}
}
common/spaces/space.ts
-280
View File
@@ -1,280 +0,0 @@
import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives.ts";
import { AttachmentMeta, PageMeta } from "../types.ts";
import { EventEmitter } from "../../plugos/event.ts";
import { Plug } from "../../plugos/plug.ts";
import { plugPrefix } from "./constants.ts";
import { safeRun } from "../util.ts";
import {
FileMeta,
ProxyFileSystem,
} from "../../plug-api/plugos-syscall/types.ts";
const pageWatchInterval = 2000;
export type SpaceEvents = {
pageCreated: (meta: PageMeta) => void;
pageChanged: (meta: PageMeta) => void;
pageDeleted: (name: string) => void;
pageListUpdated: (pages: Set<PageMeta>) => void;
};
export class Space extends EventEmitter<SpaceEvents>
implements ProxyFileSystem {
pageMetaCache = new Map<string, PageMeta>();
watchedPages = new Set<string>();
private initialPageListLoad = true;
private saving = false;
watchInterval?: number;
constructor(readonly spacePrimitives: SpacePrimitives) {
super();
}
// Filesystem interface implementation
async readFile(path: string, encoding: "dataurl" | "utf8"): Promise<string> {
return (await this.spacePrimitives.readFile(path, encoding)).data as string;
}
getFileMeta(path: string): Promise<FileMeta> {
return this.spacePrimitives.getFileMeta(path);
}
writeFile(
path: string,
text: string,
encoding: "dataurl" | "utf8",
): Promise<FileMeta> {
return this.spacePrimitives.writeFile(path, encoding, text);
}
deleteFile(path: string): Promise<void> {
return this.spacePrimitives.deleteFile(path);
}
async listFiles(path: string): Promise<FileMeta[]> {
return (await this.spacePrimitives.fetchFileList()).filter((f) =>
f.name.startsWith(path)
);
}
// The more domain-specific methods
public async updatePageList() {
const newPageList = await this.fetchPageList();
const deletedPages = new Set<string>(this.pageMetaCache.keys());
newPageList.forEach((meta) => {
const pageName = meta.name;
const oldPageMeta = this.pageMetaCache.get(pageName);
const newPageMeta: PageMeta = { ...meta };
if (
!oldPageMeta &&
(pageName.startsWith(plugPrefix) || !this.initialPageListLoad)
) {
this.emit("pageCreated", newPageMeta);
} else if (
oldPageMeta &&
oldPageMeta.lastModified !== newPageMeta.lastModified
) {
this.emit("pageChanged", newPageMeta);
}
// Page found, not deleted
deletedPages.delete(pageName);
// Update in cache
this.pageMetaCache.set(pageName, newPageMeta);
});
for (const deletedPage of deletedPages) {
this.pageMetaCache.delete(deletedPage);
this.emit("pageDeleted", deletedPage);
}
this.emit("pageListUpdated", this.listPages());
this.initialPageListLoad = false;
}
watch() {
if (this.watchInterval) {
clearInterval(this.watchInterval);
}
this.watchInterval = setInterval(() => {
safeRun(async () => {
if (this.saving) {
return;
}
for (const pageName of this.watchedPages) {
const oldMeta = this.pageMetaCache.get(pageName);
if (!oldMeta) {
// No longer in cache, meaning probably deleted let's unwatch
this.watchedPages.delete(pageName);
continue;
}
// This seems weird, but simply fetching it will compare to local cache and trigger an event if necessary
await this.getPageMeta(pageName);
}
});
}, pageWatchInterval);
this.updatePageList().catch(console.error);
}
unwatch() {
if (this.watchInterval) {
clearInterval(this.watchInterval);
}
}
async deletePage(name: string): Promise<void> {
await this.getPageMeta(name); // Check if page exists, if not throws Error
await this.spacePrimitives.deleteFile(`${name}.md`);
this.pageMetaCache.delete(name);
this.emit("pageDeleted", name);
this.emit("pageListUpdated", new Set([...this.pageMetaCache.values()]));
}
async getPageMeta(name: string): Promise<PageMeta> {
const oldMeta = this.pageMetaCache.get(name);
const newMeta = fileMetaToPageMeta(
await this.spacePrimitives.getFileMeta(`${name}.md`),
);
if (oldMeta) {
if (oldMeta.lastModified !== newMeta.lastModified) {
// Changed on disk, trigger event
this.emit("pageChanged", newMeta);
}
}
return this.metaCacher(name, newMeta);
}
invokeFunction(
plug: Plug<any>,
env: string,
name: string,
args: any[],
): Promise<any> {
return this.spacePrimitives.invokeFunction(plug, env, name, args);
}
listPages(): PageMeta[] {
return [...new Set(this.pageMetaCache.values())];
}
async listPlugs(): Promise<string[]> {
const files = await this.spacePrimitives.fetchFileList();
return files
.filter((fileMeta) => fileMeta.name.endsWith(".plug.json"))
.map((fileMeta) => fileMeta.name);
}
proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
return this.spacePrimitives.proxySyscall(plug, name, args);
}
async readPage(name: string): Promise<{ text: string; meta: PageMeta }> {
const pageData = await this.spacePrimitives.readFile(
`${name}.md`,
"utf8",
);
const previousMeta = this.pageMetaCache.get(name);
const newMeta = fileMetaToPageMeta(pageData.meta);
if (previousMeta) {
if (previousMeta.lastModified !== newMeta.lastModified) {
// Page changed since last cached metadata, trigger event
this.emit("pageChanged", newMeta);
}
}
const meta = this.metaCacher(name, newMeta);
return {
text: pageData.data as string,
meta: meta,
};
}
watchPage(pageName: string) {
this.watchedPages.add(pageName);
}
unwatchPage(pageName: string) {
this.watchedPages.delete(pageName);
}
async writePage(
name: string,
text: string,
selfUpdate?: boolean,
): Promise<PageMeta> {
try {
this.saving = true;
const pageMeta = fileMetaToPageMeta(
await this.spacePrimitives.writeFile(
`${name}.md`,
"utf8",
text,
selfUpdate,
),
);
if (!selfUpdate) {
this.emit("pageChanged", pageMeta);
}
return this.metaCacher(name, pageMeta);
} finally {
this.saving = false;
}
}
async fetchPageList(): Promise<PageMeta[]> {
return (await this.spacePrimitives.fetchFileList())
.filter((fileMeta) => fileMeta.name.endsWith(".md"))
.map(fileMetaToPageMeta);
}
async fetchAttachmentList(): Promise<AttachmentMeta[]> {
return (await this.spacePrimitives.fetchFileList()).filter(
(fileMeta) =>
!fileMeta.name.endsWith(".md") &&
!fileMeta.name.endsWith(".plug.json") &&
fileMeta.name !== "data.db",
);
}
/**
* Reads an attachment
* @param name path of the attachment
* @param encoding how the return value is expected to be encoded
* @returns
*/
readAttachment(
name: string,
encoding: FileEncoding,
): Promise<{ data: FileData; meta: AttachmentMeta }> {
return this.spacePrimitives.readFile(name, encoding);
}
getAttachmentMeta(name: string): Promise<AttachmentMeta> {
return this.spacePrimitives.getFileMeta(name);
}
writeAttachment(
name: string,
encoding: FileEncoding,
data: FileData,
selfUpdate?: boolean | undefined,
): Promise<AttachmentMeta> {
return this.spacePrimitives.writeFile(name, encoding, data, selfUpdate);
}
deleteAttachment(name: string): Promise<void> {
return this.spacePrimitives.deleteFile(name);
}
private metaCacher(name: string, meta: PageMeta): PageMeta {
if (meta.lastModified !== 0) {
// Don't cache metadata for pages with a 0 lastModified timestamp (usualy dynamically generated pages)
this.pageMetaCache.set(name, meta);
}
return meta;
}
}
function fileMetaToPageMeta(fileMeta: FileMeta): PageMeta {
return {
...fileMeta,
name: fileMeta.name.substring(0, fileMeta.name.length - 3),
} as PageMeta;
}
common/spaces/space_primitives.ts
+7 -17
View File
@@ -1,31 +1,21 @@
import { Plug } from "../../plugos/plug.ts";
import { FileMeta } from "../types.ts";
import type { FileMeta } from "../types.ts";
// export type FileEncoding = "utf8" | "arraybuffer" | "dataurl";
// export type FileData = ArrayBuffer | string;
export type FileEncoding = "utf8" | "arraybuffer" | "dataurl";
export type FileData = ArrayBuffer | string;
export interface SpacePrimitives {
// Returns a list of file meta data as well as the timestamp of this snapshot
fetchFileList(): Promise<FileMeta[]>;
readFile(
name: string,
encoding: FileEncoding,
): Promise<{ data: FileData; meta: FileMeta }>;
): Promise<{ data: Uint8Array; meta: FileMeta }>;
getFileMeta(name: string): Promise<FileMeta>;
writeFile(
name: string,
encoding: FileEncoding,
data: FileData,
data: Uint8Array,
// Used to decide whether or not to emit change events
selfUpdate?: boolean,
lastModified?: number,
): Promise<FileMeta>;
deleteFile(name: string): Promise<void>;
// Plugs
proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any>;
invokeFunction(
plug: Plug<any>,
env: string,
name: string,
args: any[],
): Promise<any>;
}
common/spaces/sync.test.ts
+54 -33
View File
@@ -9,23 +9,28 @@ Deno.test("Test store", async () => {
console.log("Secondary", secondaryPath);
const primary = new DiskSpacePrimitives(primaryPath);
const secondary = new DiskSpacePrimitives(secondaryPath);
const statusMap = new Map<string, SyncStatusItem>();
const sync = new SpaceSync(primary, secondary, statusMap, {});
const snapshot = new Map<string, SyncStatusItem>();
const sync = new SpaceSync(primary, secondary, {
conflictResolver: SpaceSync.primaryConflictResolver,
});
// Write one page to primary
await primary.writeFile("index", "utf8", "Hello");
await primary.writeFile("index", stringToBytes("Hello"));
assertEquals((await secondary.fetchFileList()).length, 0);
console.log("Initial sync ops", await doSync());
assertEquals((await secondary.fetchFileList()).length, 1);
assertEquals((await secondary.readFile("index", "utf8")).data, "Hello");
assertEquals(
(await secondary.readFile("index")).data,
stringToBytes("Hello"),
);
// Should be a no-op
assertEquals(await doSync(), 0);
// Now let's make a change on the secondary
await secondary.writeFile("index", "utf8", "Hello!!");
await secondary.writeFile("test", "utf8", "Test page");
await secondary.writeFile("index", stringToBytes("Hello!!"));
await secondary.writeFile("test", stringToBytes("Test page"));
// And sync it
await doSync();
@@ -33,13 +38,16 @@ Deno.test("Test store", async () => {
assertEquals((await primary.fetchFileList()).length, 2);
assertEquals((await secondary.fetchFileList()).length, 2);
assertEquals((await primary.readFile("index", "utf8")).data, "Hello!!");
assertEquals(
(await primary.readFile("index")).data,
stringToBytes("Hello!!"),
);
// Let's make some random edits on both ends
await primary.writeFile("index", "utf8", "1");
await primary.writeFile("index2", "utf8", "2");
await secondary.writeFile("index3", "utf8", "3");
await secondary.writeFile("index4", "utf8", "4");
await primary.writeFile("index", stringToBytes("1"));
await primary.writeFile("index2", stringToBytes("2"));
await secondary.writeFile("index3", stringToBytes("3"));
await secondary.writeFile("index4", stringToBytes("4"));
await doSync();
assertEquals((await primary.fetchFileList()).length, 5);
@@ -72,16 +80,19 @@ Deno.test("Test store", async () => {
// No-op
assertEquals(await doSync(), 0);
await secondary.writeFile("index", "utf8", "I'm back");
await secondary.writeFile("index", stringToBytes("I'm back"));
await doSync();
assertEquals((await primary.readFile("index", "utf8")).data, "I'm back");
assertEquals(
(await primary.readFile("index")).data,
stringToBytes("I'm back"),
);
// Cause a conflict
console.log("Introducing a conflict now");
await primary.writeFile("index", "utf8", "Hello 1");
await secondary.writeFile("index", "utf8", "Hello 2");
await primary.writeFile("index", stringToBytes("Hello 1"));
await secondary.writeFile("index", stringToBytes("Hello 2"));
await doSync();
@@ -89,27 +100,33 @@ Deno.test("Test store", async () => {
await doSync();
// Verify that primary won
assertEquals((await primary.readFile("index", "utf8")).data, "Hello 1");
assertEquals((await secondary.readFile("index", "utf8")).data, "Hello 1");
assertEquals(
(await primary.readFile("index")).data,
stringToBytes("Hello 1"),
);
assertEquals(
(await secondary.readFile("index")).data,
stringToBytes("Hello 1"),
);
// test + index + index.conflicting copy
assertEquals((await primary.fetchFileList()).length, 3);
assertEquals((await secondary.fetchFileList()).length, 3);
// Introducing a fake conflict (same content, so not really conflicting)
await primary.writeFile("index", "utf8", "Hello 1");
await secondary.writeFile("index", "utf8", "Hello 1");
await primary.writeFile("index", stringToBytes("Hello 1"));
await secondary.writeFile("index", stringToBytes("Hello 1"));
// And two more files with different bodies, but only within a query directive — shouldn't conflict
await primary.writeFile(
"index.md",
"utf8",
"Hello\n<!-- #query page -->\nHello 1\n<!-- /query -->",
stringToBytes(
"Hello\n<!-- #query page -->\nHello 1\n<!-- /query -->",
),
);
await secondary.writeFile(
"index.md",
"utf8",
"Hello\n<!-- #query page -->\nHello 2\n<!-- /query -->",
stringToBytes("Hello\n<!-- #query page -->\nHello 2\n<!-- /query -->"),
);
await doSync();
@@ -128,15 +145,17 @@ Deno.test("Test store", async () => {
const sync2 = new SpaceSync(
secondary,
ternary,
new Map<string, SyncStatusItem>(),
{},
{
conflictResolver: SpaceSync.primaryConflictResolver,
},
);
const snapshot2 = new Map<string, SyncStatusItem>();
console.log(
"N ops",
await sync2.syncFiles(SpaceSync.primaryConflictResolver),
await sync2.syncFiles(snapshot2),
);
await sleep(2);
assertEquals(await sync2.syncFiles(SpaceSync.primaryConflictResolver), 0);
assertEquals(await sync2.syncFiles(snapshot2), 0);
// I had to look up what follows ternary (https://english.stackexchange.com/questions/25116/what-follows-next-in-the-sequence-unary-binary-ternary)
const quaternaryPath = await Deno.makeTempDir();
@@ -144,12 +163,12 @@ Deno.test("Test store", async () => {
const sync3 = new SpaceSync(
secondary,
quaternary,
new Map<string, SyncStatusItem>(),
{
excludePrefixes: ["index"],
isSyncCandidate: (path) => !path.startsWith("index"),
conflictResolver: SpaceSync.primaryConflictResolver,
},
);
const selectingOps = await sync3.syncFiles(SpaceSync.primaryConflictResolver);
const selectingOps = await sync3.syncFiles(new Map());
assertEquals(selectingOps, 1);
@@ -160,9 +179,7 @@ Deno.test("Test store", async () => {
async function doSync() {
await sleep();
const r = await sync.syncFiles(
SpaceSync.primaryConflictResolver,
);
const r = await sync.syncFiles(snapshot);
await sleep();
return r;
}
@@ -193,3 +210,7 @@ Hello
`,
);
});
function stringToBytes(s: string): Uint8Array {
return new TextEncoder().encode(s);
}
common/spaces/sync.ts
+131 -126
View File
@@ -10,52 +10,41 @@ type SyncHash = number;
// and the second item the lastModified value of the secondary space
export type SyncStatusItem = [SyncHash, SyncHash];
export interface Logger {
log(level: string, ...messageBits: any[]): void;
}
class ConsoleLogger implements Logger {
log(_level: string, ...messageBits: any[]) {
console.log(...messageBits);
}
}
export type SyncStatus = {
filesProcessed: number;
totalFiles: number;
snapshot: Map<string, SyncStatusItem>;
};
export type SyncOptions = {
logger?: Logger;
excludePrefixes?: string[];
conflictResolver: (
name: string,
snapshot: Map<string, SyncStatusItem>,
primarySpace: SpacePrimitives,
secondarySpace: SpacePrimitives,
) => Promise<number>;
isSyncCandidate?: (path: string) => boolean;
// Used to track progress, may want to pass more specific info later
onSyncProgress?: (syncStatus: SyncStatus) => void;
};
// Implementation of this algorithm https://unterwaditzer.net/2016/sync-algorithm.html
export class SpaceSync {
logger: ConsoleLogger;
excludePrefixes: string[];
constructor(
private primary: SpacePrimitives,
private secondary: SpacePrimitives,
readonly snapshot: Map<string, SyncStatusItem>,
readonly options: SyncOptions,
) {
this.logger = options.logger || new ConsoleLogger();
this.excludePrefixes = options.excludePrefixes || [];
}
async syncFiles(
conflictResolver: (
name: string,
snapshot: Map<string, SyncStatusItem>,
primarySpace: SpacePrimitives,
secondarySpace: SpacePrimitives,
logger: Logger,
) => Promise<number>,
): Promise<number> {
async syncFiles(snapshot: Map<string, SyncStatusItem>): Promise<number> {
let operations = 0;
this.logger.log("info", "Fetching snapshot from primary");
console.log("[sync]", "Fetching snapshot from primary");
const primaryAllPages = this.syncCandidates(
await this.primary.fetchFileList(),
);
this.logger.log("info", "Fetching snapshot from secondary");
console.log("[sync]", "Fetching snapshot from secondary");
try {
const secondaryAllPages = this.syncCandidates(
await this.secondary.fetchFileList(),
@@ -69,177 +58,188 @@ export class SpaceSync {
);
const allFilesToProcess = new Set([
...this.snapshot.keys(),
...snapshot.keys(),
...primaryFileMap.keys(),
...secondaryFileMap.keys(),
]);
this.logger.log("info", "Iterating over all files");
for (const name of allFilesToProcess) {
const sortedFilenames = [...allFilesToProcess];
sortedFilenames.sort((a) => {
// Just make sure that _plug/ files appear first
// This is important for the initial sync: plugs are loaded the moment they are pulled into the space,
// which would activate e.g. any indexing logic for the remaining space content
return a.startsWith("_plug/") ? -1 : 1;
});
// console.log("[sync]", "Iterating over all files");
let filesProcessed = 0;
for (const name of sortedFilenames) {
try {
operations += await this.syncFile(
snapshot,
name,
primaryFileMap.get(name),
secondaryFileMap.get(name),
conflictResolver,
);
filesProcessed++;
// Only report something significant
if (operations > 1 && this.options.onSyncProgress) {
this.options.onSyncProgress({
filesProcessed,
totalFiles: sortedFilenames.length,
snapshot,
});
}
} catch (e: any) {
this.logger.log("error", "Error syncing file", name, e.message);
console.log("error", "Error syncing file", name, e.message);
}
}
} catch (e: any) {
this.logger.log("error", "General sync error:", e.message);
console.log("error", "General sync error:", e.message);
throw e;
}
this.logger.log("info", "Sync complete, operations performed", operations);
console.log("[sync]", "Sync complete, operations performed", operations);
return operations;
}
async syncFile(
snapshot: Map<string, SyncStatusItem>,
name: string,
primaryHash: SyncHash | undefined,
secondaryHash: SyncHash | undefined,
conflictResolver: (
name: string,
snapshot: Map<string, SyncStatusItem>,
primarySpace: SpacePrimitives,
secondarySpace: SpacePrimitives,
logger: Logger,
) => Promise<number>,
): Promise<number> {
if (this.options.isSyncCandidate && !this.options.isSyncCandidate(name)) {
return 0;
}
// console.log("Syncing", name, primaryHash, secondaryHash);
let operations = 0;
// Check if not matching one of the excluded prefixes
for (const prefix of this.excludePrefixes) {
if (name.startsWith(prefix)) {
return operations;
}
}
if (
primaryHash !== undefined && secondaryHash === undefined &&
!this.snapshot.has(name)
!snapshot.has(name)
) {
// New file, created on primary, copy from primary to secondary
this.logger.log(
"info",
console.log(
"[sync]",
"New file created on primary, copying to secondary",
name,
);
const { data } = await this.primary.readFile(name, "arraybuffer");
const { data, meta } = await this.primary.readFile(name);
const writtenMeta = await this.secondary.writeFile(
name,
"arraybuffer",
data,
false,
meta.lastModified,
);
this.snapshot.set(name, [
snapshot.set(name, [
primaryHash,
writtenMeta.lastModified,
]);
operations++;
} else if (
secondaryHash !== undefined && primaryHash === undefined &&
!this.snapshot.has(name)
!snapshot.has(name)
) {
// New file, created on secondary, copy from secondary to primary
this.logger.log(
"info",
console.log(
"[sync]",
"New file created on secondary, copying from secondary to primary",
name,
);
const { data } = await this.secondary.readFile(name, "arraybuffer");
const { data, meta } = await this.secondary.readFile(name);
const writtenMeta = await this.primary.writeFile(
name,
"arraybuffer",
data,
false,
meta.lastModified,
);
this.snapshot.set(name, [
snapshot.set(name, [
writtenMeta.lastModified,
secondaryHash,
]);
operations++;
} else if (
primaryHash !== undefined && this.snapshot.has(name) &&
primaryHash !== undefined && snapshot.has(name) &&
secondaryHash === undefined
) {
// File deleted on B
this.logger.log(
"info",
console.log(
"[sync]",
"File deleted on secondary, deleting from primary",
name,
);
await this.primary.deleteFile(name);
this.snapshot.delete(name);
snapshot.delete(name);
operations++;
} else if (
secondaryHash !== undefined && this.snapshot.has(name) &&
secondaryHash !== undefined && snapshot.has(name) &&
primaryHash === undefined
) {
// File deleted on A
this.logger.log(
"info",
console.log(
"[sync]",
"File deleted on primary, deleting from secondary",
name,
);
await this.secondary.deleteFile(name);
this.snapshot.delete(name);
snapshot.delete(name);
operations++;
} else if (
this.snapshot.has(name) && primaryHash === undefined &&
snapshot.has(name) && primaryHash === undefined &&
secondaryHash === undefined
) {
// File deleted on both sides, :shrug:
this.logger.log(
"info",
console.log(
"[sync]",
"File deleted on both ends, deleting from status",
name,
);
this.snapshot.delete(name);
snapshot.delete(name);
operations++;
} else if (
primaryHash !== undefined && secondaryHash !== undefined &&
this.snapshot.get(name) &&
primaryHash !== this.snapshot.get(name)![0] &&
secondaryHash === this.snapshot.get(name)![1]
snapshot.get(name) &&
primaryHash !== snapshot.get(name)![0] &&
secondaryHash === snapshot.get(name)![1]
) {
// File has changed on primary, but not secondary: copy from primary to secondary
this.logger.log(
"info",
console.log(
"[sync]",
"File changed on primary, copying to secondary",
name,
);
const { data } = await this.primary.readFile(name, "arraybuffer");
const { data, meta } = await this.primary.readFile(name);
const writtenMeta = await this.secondary.writeFile(
name,
"arraybuffer",
data,
false,
meta.lastModified,
);
this.snapshot.set(name, [
snapshot.set(name, [
primaryHash,
writtenMeta.lastModified,
]);
operations++;
} else if (
primaryHash !== undefined && secondaryHash !== undefined &&
this.snapshot.get(name) &&
secondaryHash !== this.snapshot.get(name)![1] &&
primaryHash === this.snapshot.get(name)![0]
snapshot.get(name) &&
secondaryHash !== snapshot.get(name)![1] &&
primaryHash === snapshot.get(name)![0]
) {
// File has changed on secondary, but not primary: copy from secondary to primary
this.logger.log(
"info",
console.log(
"[sync]",
"File has changed on secondary, but not primary: copy from secondary to primary",
name,
);
const { data } = await this.secondary.readFile(name, "arraybuffer");
const { data, meta } = await this.secondary.readFile(name);
const writtenMeta = await this.primary.writeFile(
name,
"arraybuffer",
data,
false,
meta.lastModified,
);
this.snapshot.set(name, [
snapshot.set(name, [
writtenMeta.lastModified,
secondaryHash,
]);
@@ -247,26 +247,25 @@ export class SpaceSync {
} else if (
( // File changed on both ends, but we don't have any info in the snapshot (resync scenario?): have to run through conflict handling
primaryHash !== undefined && secondaryHash !== undefined &&
!this.snapshot.has(name)
!snapshot.has(name)
) ||
( // File changed on both ends, CONFLICT!
primaryHash && secondaryHash &&
this.snapshot.get(name) &&
secondaryHash !== this.snapshot.get(name)![1] &&
primaryHash !== this.snapshot.get(name)![0]
snapshot.get(name) &&
secondaryHash !== snapshot.get(name)![1] &&
primaryHash !== snapshot.get(name)![0]
)
) {
this.logger.log(
"info",
console.log(
"[sync]",
"File changed on both ends, potential conflict",
name,
);
operations += await conflictResolver(
operations += await this.options.conflictResolver!(
name,
this.snapshot,
snapshot,
this.primary,
this.secondary,
this.logger,
);
} else {
// Nothing needs to happen
@@ -280,27 +279,29 @@ export class SpaceSync {
snapshot: Map<string, SyncStatusItem>,
primary: SpacePrimitives,
secondary: SpacePrimitives,
logger: Logger,
): Promise<number> {
logger.log("info", "Starting conflict resolution for", name);
console.log("[sync]", "Starting conflict resolution for", name);
const filePieces = name.split(".");
const fileNameBase = filePieces.slice(0, -1).join(".");
const fileNameExt = filePieces[filePieces.length - 1];
const pageData1 = await primary.readFile(name, "arraybuffer");
const pageData2 = await secondary.readFile(name, "arraybuffer");
const pageData1 = await primary.readFile(name);
const pageData2 = await secondary.readFile(name);
if (name.endsWith(".md")) {
logger.log("info", "File is markdown, using smart conflict resolution");
console.log(
"[sync]",
"File is markdown, using smart conflict resolution",
);
// Let's use a smartert check for markdown files, ignoring directive bodies
const pageText1 = removeDirectiveBody(
new TextDecoder().decode(pageData1.data as Uint8Array),
new TextDecoder().decode(pageData1.data),
);
const pageText2 = removeDirectiveBody(
new TextDecoder().decode(pageData2.data as Uint8Array),
new TextDecoder().decode(pageData2.data),
);
if (pageText1 === pageText2) {
logger.log(
"info",
console.log(
"[sync]",
"Files are the same (eliminating the directive bodies), no conflict",
);
snapshot.set(name, [
@@ -311,8 +312,8 @@ export class SpaceSync {
}
} else {
let byteWiseMatch = true;
const arrayBuffer1 = new Uint8Array(pageData1.data as ArrayBuffer);
const arrayBuffer2 = new Uint8Array(pageData2.data as ArrayBuffer);
const arrayBuffer1 = pageData1.data;
const arrayBuffer2 = pageData2.data;
if (arrayBuffer1.byteLength !== arrayBuffer2.byteLength) {
byteWiseMatch = false;
}
@@ -326,7 +327,8 @@ export class SpaceSync {
}
// Byte wise they're still the same, so no confict
if (byteWiseMatch) {
logger.log("info", "Files are the same, no conflict");
console.log("[sync]", "Files are the same, no conflict");
snapshot.set(name, [
pageData1.meta.lastModified,
pageData2.meta.lastModified,
@@ -335,11 +337,12 @@ export class SpaceSync {
}
}
}
let operations = 0;
const revisionFileName = filePieces.length === 1
? `${name}.conflicted.${pageData2.meta.lastModified}`
: `${fileNameBase}.conflicted.${pageData2.meta.lastModified}.${fileNameExt}`;
logger.log(
"info",
console.log(
"[sync]",
"Going to create conflicting copy",
revisionFileName,
);
@@ -347,14 +350,22 @@ export class SpaceSync {
// Copy secondary to conflict copy
const localConflictMeta = await primary.writeFile(
revisionFileName,
"arraybuffer",
pageData2.data,
);
operations++;
const remoteConflictMeta = await secondary.writeFile(
revisionFileName,
"arraybuffer",
pageData2.data,
);
operations++;
// Write replacement on top
const writeMeta = await secondary.writeFile(
name,
pageData1.data,
true,
);
operations++;
// Updating snapshot
snapshot.set(revisionFileName, [
@@ -362,22 +373,16 @@ export class SpaceSync {
remoteConflictMeta.lastModified,
]);
// Write replacement on top
const writeMeta = await secondary.writeFile(
name,
"arraybuffer",
pageData1.data,
true,
);
snapshot.set(name, [pageData1.meta.lastModified, writeMeta.lastModified]);
return 1;
return operations;
}
syncCandidates(files: FileMeta[]): FileMeta[] {
return files.filter((f) =>
!f.name.startsWith("_plug/") && f.lastModified > 0
);
if (this.options.isSyncCandidate) {
return files.filter((meta) => this.options.isSyncCandidate!(meta.name));
} else {
return files;
}
}
}
common/sw_util.ts
+27
View File
@@ -0,0 +1,27 @@
export function flushCachesAndUnregisterServiceWorker() {
return new Promise<void>((resolve) => {
if (!navigator.serviceWorker) {
console.log("No service worker active");
return resolve();
}
navigator.serviceWorker.addEventListener("message", (event) => {
if (event.data.type === "cacheFlushed") {
console.log("Cache flushed");
// Then unregister all service workers
navigator.serviceWorker.getRegistrations().then((registrations) => {
for (const registration of registrations) {
registration.unregister();
console.log("Service worker unregistered");
resolve();
}
});
}
});
// First flush active cache
navigator.serviceWorker.ready.then((registration) => {
registration.active!.postMessage({ type: "flushCache" });
});
});
}
common/syscalls/markdown.ts
-12
View File
@@ -1,12 +0,0 @@
import { SysCallMapping } from "../../plugos/system.ts";
import { parse } from "../markdown_parser/parse_tree.ts";
import { Language } from "../deps.ts";
import type { ParseTree } from "$sb/lib/tree.ts";
export function markdownSyscalls(lang: Language): SysCallMapping {
return {
"markdown.parseMarkdown": (_ctx, text: string): ParseTree => {
return parse(lang, text);
},
};
}
common/syscalls/space.ts
-69
View File
@@ -1,69 +0,0 @@
import { AttachmentMeta, PageMeta } from "../../common/types.ts";
import { SysCallMapping } from "../../plugos/system.ts";
import { Space } from "../../common/spaces/space.ts";
import {
FileData,
FileEncoding,
} from "../../common/spaces/space_primitives.ts";
import { FileMeta as PlugFileMeta } from "../../plug-api/plugos-syscall/types.ts";
export default (space: Space): SysCallMapping => {
return {
"space.listPages": (): PageMeta[] => {
return space.listPages();
},
"space.readPage": async (
_ctx,
name: string,
): Promise<string> => {
return (await space.readPage(name)).text;
},
"space.getPageMeta": (_ctx, name: string): Promise<PageMeta> => {
return space.getPageMeta(name);
},
"space.writePage": (
_ctx,
name: string,
text: string,
): Promise<PageMeta> => {
return space.writePage(name, text);
},
"space.deletePage": (_ctx, name: string) => {
return space.deletePage(name);
},
"space.listPlugs": (): Promise<string[]> => {
return space.listPlugs();
},
"space.listAttachments": async (): Promise<AttachmentMeta[]> => {
return await space.fetchAttachmentList();
},
"space.readAttachment": async (
_ctx,
name: string,
): Promise<FileData> => {
return (await space.readAttachment(name, "dataurl")).data;
},
"space.getAttachmentMeta": async (
_ctx,
name: string,
): Promise<AttachmentMeta> => {
return await space.getAttachmentMeta(name);
},
"space.writeAttachment": async (
_ctx,
name: string,
encoding: FileEncoding,
data: string,
): Promise<AttachmentMeta> => {
return await space.writeAttachment(name, encoding, data);
},
"space.deleteAttachment": async (_ctx, name: string) => {
await space.deleteAttachment(name);
},
"space.listFiles": (_ctx, path: string): Promise<PlugFileMeta[]> => {
return space.listFiles(path);
},
};
};
common/syscalls/sync.ts
-132
View File
@@ -1,132 +0,0 @@
import { SysCallMapping, System } from "../../plugos/system.ts";
import type { SyncEndpoint } from "../../plug-api/silverbullet-syscall/sync.ts";
import { SpaceSync, SyncStatusItem } from "../spaces/sync.ts";
import { HttpSpacePrimitives } from "../spaces/http_space_primitives.ts";
import { SpacePrimitives } from "../spaces/space_primitives.ts";
export function syncSyscalls(
localSpace: SpacePrimitives,
system: System<any>,
): SysCallMapping {
return {
"sync.syncAll": async (
_ctx,
endpoint: SyncEndpoint,
snapshot: Record<string, SyncStatusItem>,
): Promise<
{
snapshot: Record<string, SyncStatusItem>;
operations: number;
// The reason to not just throw an Error is so that the partially updated snapshot can still be saved
error?: string;
}
> => {
const { spaceSync } = setupSync(endpoint, snapshot);
try {
const operations = await spaceSync.syncFiles(
SpaceSync.primaryConflictResolver,
);
return {
// And convert back to JSON
snapshot: Object.fromEntries(spaceSync.snapshot),
operations,
};
} catch (e: any) {
return {
snapshot: Object.fromEntries(spaceSync.snapshot),
operations: -1,
error: e.message,
};
}
},
"sync.syncFile": async (
_ctx,
endpoint: SyncEndpoint,
snapshot: Record<string, SyncStatusItem>,
name: string,
): Promise<
{
snapshot: Record<string, SyncStatusItem>;
operations: number;
// The reason to not just throw an Error is so that the partially updated snapshot can still be saved
error?: string;
}
> => {
const { spaceSync, remoteSpace } = setupSync(endpoint, snapshot);
try {
const localHash = (await localSpace.getFileMeta(name)).lastModified;
let remoteHash: number | undefined = undefined;
try {
remoteHash = (await remoteSpace.getFileMeta(name)).lastModified;
} catch (e: any) {
if (e.message.includes("File not found")) {
// File doesn't exist remotely, that's ok
} else {
throw e;
}
}
const operations = await spaceSync.syncFile(
name,
localHash,
remoteHash,
SpaceSync.primaryConflictResolver,
);
return {
// And convert back to JSON
snapshot: Object.fromEntries(spaceSync.snapshot),
operations,
};
} catch (e: any) {
return {
snapshot: Object.fromEntries(spaceSync.snapshot),
operations: -1,
error: e.message,
};
}
},
"sync.check": async (_ctx, endpoint: SyncEndpoint): Promise<void> => {
const syncSpace = new HttpSpacePrimitives(
endpoint.url,
endpoint.user,
endpoint.password,
);
// Let's just fetch the file list and see if it works
try {
await syncSpace.fetchFileList();
} catch (e: any) {
console.error("Sync check failure", e.message);
throw e;
}
},
};
function setupSync(
endpoint: SyncEndpoint,
snapshot: Record<string, SyncStatusItem>,
) {
const remoteSpace = new HttpSpacePrimitives(
endpoint.url,
endpoint.user,
endpoint.password,
// Base64 PUTs to support mobile
true,
);
// Convert from JSON to a Map
const syncStatusMap = new Map<string, SyncStatusItem>(
Object.entries(snapshot),
);
const spaceSync = new SpaceSync(
localSpace,
remoteSpace,
syncStatusMap,
{
excludePrefixes: endpoint.excludePrefixes,
// Log to the "sync" plug sandbox
logger: system.loadedPlugs.get("sync")!.sandbox!,
},
);
return { spaceSync, remoteSpace };
}
}
common/types.ts
+1 -23
View File
@@ -1,4 +1,4 @@
export const maximumAttachmentSize = 100 * 1024 * 1024; // 100 MB
export const maximumAttachmentSize = 20 * 1024 * 1024; // 10 MB
export type FileMeta = {
name: string;
@@ -7,25 +7,3 @@ export type FileMeta = {
size: number;
perm: "ro" | "rw";
} & Record<string, any>;
export type PageMeta = {
name: string;
lastModified: number;
lastOpened?: number;
perm: "ro" | "rw";
} & Record<string, any>;
export type AttachmentMeta = {
name: string;
contentType: string;
lastModified: number;
size: number;
perm: "ro" | "rw";
};
// Used by FilterBox
export type FilterOption = {
name: string;
orderId?: number;
hint?: string;
} & Record<string, any>;
common/util.ts
+18 -30
View File
@@ -1,6 +1,6 @@
import { SETTINGS_TEMPLATE } from "./settings_template.ts";
import { YAML } from "./deps.ts";
import { Space } from "./spaces/space.ts";
import { SpacePrimitives } from "./spaces/space_primitives.ts";
export function safeRun(fn: () => Promise<void>) {
fn().catch((e) => {
@@ -33,43 +33,31 @@ export function parseYamlSettings(settingsMarkdown: string): {
}
}
export async function ensureAndLoadSettings(
space: Space,
dontCreate: boolean,
export async function ensureSettingsAndIndex(
space: SpacePrimitives,
): Promise<any> {
if (dontCreate) {
return {
indexPage: "index",
};
}
try {
await space.getPageMeta("SETTINGS");
await space.getFileMeta("SETTINGS.md");
} catch {
await space.writePage(
"SETTINGS",
SETTINGS_TEMPLATE,
await space.writeFile(
"SETTINGS.md",
new TextEncoder().encode(SETTINGS_TEMPLATE),
true,
);
}
const { text: settingsText } = await space.readPage("SETTINGS");
const settings = parseYamlSettings(settingsText);
if (!settings.indexPage) {
settings.indexPage = "index";
}
try {
await space.getPageMeta(settings.indexPage);
} catch {
await space.writePage(
settings.indexPage,
`Hello! And welcome to your brand new SilverBullet space!
// Ok, then let's also write the index page
try {
await space.getFileMeta("index.md");
} catch {
await space.writeFile(
"index.md",
new TextEncoder().encode(
`Hello! And welcome to your brand new SilverBullet space!
<!-- #use [[💭 silverbullet.md/Getting Started]] -->
Loading some onboarding content for you (but doing so does require a working internet connection)...
<!-- /use -->`,
);
),
);
}
}
return settings;
}