From a56e14bff1dbe36f24f12b1d6ad147ef69e38cb6 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Fri, 13 Jan 2023 15:41:29 +0100 Subject: [PATCH] Sync engine (#298) Fixes #261 --- .github/workflows/desktop.yml | 3 +- build_mobile.ts | 2 +- build.ts => build_web.ts | 0 cmd/server.ts | 2 + {server => common}/hooks/page_namespace.ts | 0 common/manifest.ts | 2 +- .../spaces/asset_bundle_space_primitives.ts | 8 +- common/spaces/disk_space_primitives.ts | 12 +- common/spaces/evented_space_primitives.ts | 4 +- common/spaces/file_meta_space_primitives.ts | 6 +- common/spaces/http_space_primitives.ts | 87 ++- .../spaces}/plug_space_primitives.ts | 9 +- common/spaces/space.ts | 72 ++- common/spaces/space_primitives.ts | 5 +- common/spaces/sync.test.ts | 143 +++++ common/spaces/sync.ts | 271 +++++++++ {server => common}/syscalls/space.ts | 6 + common/syscalls/sync.ts | 65 +++ common/util.ts | 11 +- deno.jsonc | 6 +- desktop/src/menu.ts | 13 +- mobile/boot.ts | 9 +- mobile/spaces/capacitor_space_primitives.ts | 13 +- plug-api/plugos-syscall/fs.ts | 76 +-- plug-api/plugos-syscall/mod.ts | 3 +- plug-api/plugos-syscall/types.ts | 25 + plug-api/silverbullet-syscall/mod.ts | 3 +- plug-api/silverbullet-syscall/space.ts | 158 +++--- plug-api/silverbullet-syscall/sync.ts | 28 + plugos/sqlite/deno-sqlite/src/db.test.ts | 462 ---------------- plugos/sqlite/deno-sqlite/src/error.test.ts | 55 -- plugos/sqlite/deno-sqlite/src/query.test.ts | 521 ------------------ plugos/sqlite/deno-sqlite/src/readme.test.ts | 80 --- plugos/sqlite/deno-sqlite/src/wasm.test.ts | 40 -- plugs/collab/collab.ts | 4 +- plugs/core/cloud.ts | 2 +- plugs/core/plugmanager.ts | 4 +- plugs/core/search.ts | 4 +- plugs/markdown/markdown.plug.yaml | 6 +- plugs/markdown/share.ts | 6 +- plugs/sync/sync.plug.yaml | 19 + plugs/sync/sync.ts | 88 +++ server/http_server.ts | 23 +- server/space_system.ts | 10 +- silverbullet.ts | 3 + web/boot.ts | 6 +- web/editor.tsx | 4 +- web/syscalls/space.ts | 77 +-- web/types.ts | 1 + website/.gitignore | 1 - website/CHANGELOG.md | 1 + 51 files changed, 1037 insertions(+), 1422 deletions(-) rename build.ts => build_web.ts (100%) rename {server => common}/hooks/page_namespace.ts (100%) rename {server/hooks => common/spaces}/plug_space_primitives.ts (94%) create mode 100644 common/spaces/sync.test.ts create mode 100644 common/spaces/sync.ts rename {server => common}/syscalls/space.ts (89%) create mode 100644 common/syscalls/sync.ts create mode 100644 plug-api/plugos-syscall/types.ts create mode 100644 plug-api/silverbullet-syscall/sync.ts delete mode 100644 plugos/sqlite/deno-sqlite/src/db.test.ts delete mode 100644 plugos/sqlite/deno-sqlite/src/error.test.ts delete mode 100644 plugos/sqlite/deno-sqlite/src/query.test.ts delete mode 100644 plugos/sqlite/deno-sqlite/src/readme.test.ts delete mode 100644 plugos/sqlite/deno-sqlite/src/wasm.test.ts create mode 100644 plugs/sync/sync.plug.yaml create mode 100644 plugs/sync/sync.ts diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 576e6e1..c104677 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -3,7 +3,7 @@ name: Build & Release on: push: tags: - - '*' + - "*" jobs: build: name: Build (${{ matrix.os }} - ${{ matrix.arch }}) @@ -72,6 +72,7 @@ jobs: files: | desktop/out/**/*.deb desktop/out/**/*Setup.exe + desktop/out/**/RELEASES desktop/out/**/*.rpm desktop/out/**/*.zip dist/silverbullet.js diff --git a/build_mobile.ts b/build_mobile.ts index f8b2dfd..8a33339 100644 --- a/build_mobile.ts +++ b/build_mobile.ts @@ -1,4 +1,4 @@ -import { bundle, esbuild } from "./build.ts"; +import { bundle, esbuild } from "./build_web.ts"; import * as flags from "https://deno.land/std@0.165.0/flags/mod.ts"; import { copy } from "https://deno.land/std@0.165.0/fs/copy.ts"; diff --git a/build.ts b/build_web.ts similarity index 100% rename from build.ts rename to build_web.ts diff --git a/cmd/server.ts b/cmd/server.ts index 0d84f18..4b273ec 100644 --- a/cmd/server.ts +++ b/cmd/server.ts @@ -7,6 +7,7 @@ export function serveCommand(options: any, folder: string) { const pagesPath = path.resolve(Deno.cwd(), folder); const hostname = options.hostname || "127.0.0.1"; const port = options.port || 3000; + const bareMode = options.bare; console.log( "Going to start Silver Bullet binding to", @@ -27,6 +28,7 @@ export function serveCommand(options: any, folder: string) { dbPath: path.join(pagesPath, options.db), assetBundle: new AssetBundle(assetBundle as AssetJson), user: options.user, + bareMode, }); httpServer.start().catch((e) => { console.error("HTTP Server error", e); diff --git a/server/hooks/page_namespace.ts b/common/hooks/page_namespace.ts similarity index 100% rename from server/hooks/page_namespace.ts rename to common/hooks/page_namespace.ts diff --git a/common/manifest.ts b/common/manifest.ts index f0ce2ad..4609d79 100644 --- a/common/manifest.ts +++ b/common/manifest.ts @@ -4,7 +4,7 @@ import { CronHookT } from "../plugos/hooks/cron.deno.ts"; import { EventHookT } from "../plugos/hooks/event.ts"; import { CommandHookT } from "../web/hooks/command.ts"; import { SlashCommandHookT } from "../web/hooks/slash_command.ts"; -import { PageNamespaceHookT } from "../server/hooks/page_namespace.ts"; +import { PageNamespaceHookT } from "./hooks/page_namespace.ts"; import { CodeWidgetT } from "../web/hooks/code_widget.ts"; export type SilverBulletHooks = diff --git a/common/spaces/asset_bundle_space_primitives.ts b/common/spaces/asset_bundle_space_primitives.ts index 3d80792..e0cdcc4 100644 --- a/common/spaces/asset_bundle_space_primitives.ts +++ b/common/spaces/asset_bundle_space_primitives.ts @@ -12,7 +12,7 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives { } async fetchFileList(): Promise { - const l = await this.wrapped.fetchFileList(); + const files = await this.wrapped.fetchFileList(); return this.assetBundle.listFiles().filter((p) => p.startsWith("_plug/")) .map((p) => ({ name: p, @@ -20,7 +20,7 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives { lastModified: bootTime, perm: "ro", size: -1, - } as FileMeta)).concat(l); + } as FileMeta)).concat(files); } readFile( @@ -31,7 +31,7 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives { const data = this.assetBundle.readFileSync(name); // console.log("Requested encoding", encoding); return Promise.resolve({ - data: encoding === "string" ? new TextDecoder().decode(data) : data, + data: encoding === "utf8" ? new TextDecoder().decode(data) : data, meta: { lastModified: bootTime, size: data.byteLength, @@ -60,7 +60,7 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives { name: string, encoding: FileEncoding, data: FileData, - selfUpdate?: boolean | undefined, + selfUpdate?: boolean, ): Promise { return this.wrapped.writeFile(name, encoding, data, selfUpdate); } diff --git a/common/spaces/disk_space_primitives.ts b/common/spaces/disk_space_primitives.ts index 44200cf..5c87402 100644 --- a/common/spaces/disk_space_primitives.ts +++ b/common/spaces/disk_space_primitives.ts @@ -15,6 +15,8 @@ function lookupContentType(path: string): string { return mime.getType(path) || "application/octet-stream"; } +const excludedFiles = ["data.db", "data.db-journal", "sync.json"]; + export class DiskSpacePrimitives implements SpacePrimitives { rootPath: string; @@ -48,7 +50,7 @@ export class DiskSpacePrimitives implements SpacePrimitives { let data: FileData | null = null; const contentType = lookupContentType(name); switch (encoding) { - case "string": + case "utf8": data = await Deno.readTextFile(localPath); break; case "dataurl": @@ -98,7 +100,7 @@ export class DiskSpacePrimitives implements SpacePrimitives { // Actually write the file switch (encoding) { - case "string": + case "utf8": await Deno.writeTextFile(`${localPath}`, data as string); break; case "dataurl": @@ -165,8 +167,12 @@ export class DiskSpacePrimitives implements SpacePrimitives { const fullPath = file.path; try { const s = await Deno.stat(fullPath); + const name = fullPath.substring(this.rootPath.length + 1); + if (excludedFiles.includes(name)) { + continue; + } allFiles.push({ - name: fullPath.substring(this.rootPath.length + 1), + name: name, lastModified: s.mtime!.getTime(), contentType: mime.getType(fullPath) || "application/octet-stream", size: s.size, diff --git a/common/spaces/evented_space_primitives.ts b/common/spaces/evented_space_primitives.ts index 42d3ca0..3fbed70 100644 --- a/common/spaces/evented_space_primitives.ts +++ b/common/spaces/evented_space_primitives.ts @@ -35,7 +35,7 @@ export class EventedSpacePrimitives implements SpacePrimitives { name: string, encoding: FileEncoding, data: FileData, - selfUpdate: boolean, + selfUpdate?: boolean, ): Promise { const newMeta = await this.wrapped.writeFile( name, @@ -48,7 +48,7 @@ export class EventedSpacePrimitives implements SpacePrimitives { const pageName = name.substring(0, name.length - 3); let text = ""; switch (encoding) { - case "string": + case "utf8": text = data as string; break; case "arraybuffer": diff --git a/common/spaces/file_meta_space_primitives.ts b/common/spaces/file_meta_space_primitives.ts index 9cc010b..c52ba63 100644 --- a/common/spaces/file_meta_space_primitives.ts +++ b/common/spaces/file_meta_space_primitives.ts @@ -12,10 +12,10 @@ export class FileMetaSpacePrimitives implements SpacePrimitives { } async fetchFileList(): Promise { - const list = await this.wrapped.fetchFileList(); + const files = await this.wrapped.fetchFileList(); // Enrich the file list with custom meta data (for pages) const allFilesMap: Map = new Map( - list.map((fm) => [fm.name, fm]), + files.map((fm) => [fm.name, fm]), ); for ( const { page, value } of await this.indexSyscalls["index.queryPrefix"]( @@ -53,7 +53,7 @@ export class FileMetaSpacePrimitives implements SpacePrimitives { name: string, encoding: FileEncoding, data: FileData, - selfUpdate?: boolean | undefined, + selfUpdate?: boolean, ): Promise { return this.wrapped.writeFile(name, encoding, data, selfUpdate); } diff --git a/common/spaces/http_space_primitives.ts b/common/spaces/http_space_primitives.ts index 31585e9..6ea118a 100644 --- a/common/spaces/http_space_primitives.ts +++ b/common/spaces/http_space_primitives.ts @@ -3,33 +3,51 @@ import { Plug } from "../../plugos/plug.ts"; import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives.ts"; import { base64DecodeDataUrl, + base64Encode, base64EncodedDataUrl, } from "../../plugos/asset_bundle/base64.ts"; import { mime } from "../../plugos/deps.ts"; export class HttpSpacePrimitives implements SpacePrimitives { - fsUrl: string; + private fsUrl: string; private plugUrl: string; - constructor(url: string) { + constructor( + url: string, + readonly user?: string, + readonly password?: string, + readonly base64Put?: boolean, + ) { this.fsUrl = url + "/fs"; this.plugUrl = url + "/plug"; } private async authenticatedFetch( url: string, - options: any, + options: Record, ): Promise { + if (this.user && this.password) { + // Explicitly set an auth cookie + if (!options.headers) { + options.headers = {}; + } + options.headers["cookie"] = `auth=${ + btoa(`${this.user}:${this.password}`) + }`; + } const result = await fetch(url, options); - if (result.status === 401) { + if (result.status === 401 || result.redirected) { // Invalid credentials, reloading the browser should trigger authentication - location.reload(); + if (typeof location !== "undefined") { + location.reload(); + } + throw Error("Unauthorized"); } return result; } - public async fetchFileList(): Promise { + async fetchFileList(): Promise { const req = await this.authenticatedFetch(this.fsUrl, { method: "GET", }); @@ -41,9 +59,12 @@ export class HttpSpacePrimitives implements SpacePrimitives { name: string, encoding: FileEncoding, ): Promise<{ data: FileData; meta: FileMeta }> { - const res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, { - method: "GET", - }); + const res = await this.authenticatedFetch( + `${this.fsUrl}/${encodeURI(name)}`, + { + method: "GET", + }, + ); if (res.status === 404) { throw new Error(`Page not found`); } @@ -52,7 +73,6 @@ export class HttpSpacePrimitives implements SpacePrimitives { case "arraybuffer": { data = await res.arrayBuffer(); - // data = await abBlob.arrayBuffer(); } break; case "dataurl": @@ -63,7 +83,7 @@ export class HttpSpacePrimitives implements SpacePrimitives { ); } break; - case "string": + case "utf8": data = await res.text(); break; } @@ -82,37 +102,56 @@ export class HttpSpacePrimitives implements SpacePrimitives { switch (encoding) { case "arraybuffer": - case "string": + // actually we want an Uint8Array + body = data instanceof ArrayBuffer ? new Uint8Array(data) : data; + break; + case "utf8": body = data; break; case "dataurl": data = base64DecodeDataUrl(data as string); break; } - const res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, { - method: "PUT", - headers: { - "Content-type": "application/octet-stream", + const headers: Record = { + "Content-Type": "application/octet-stream", + }; + if (this.base64Put) { + headers["X-Content-Base64"] = "true"; + headers["Content-Type"] = "text/plain"; + body = base64Encode(body); + } + + const res = await this.authenticatedFetch( + `${this.fsUrl}/${encodeURI(name)}`, + { + method: "PUT", + headers, + body, }, - body, - }); + ); const newMeta = this.responseToMeta(name, res); return newMeta; } async deleteFile(name: string): Promise { - const req = await this.authenticatedFetch(`${this.fsUrl}/${name}`, { - method: "DELETE", - }); + const req = await this.authenticatedFetch( + `${this.fsUrl}/${encodeURI(name)}`, + { + method: "DELETE", + }, + ); if (req.status !== 200) { throw Error(`Failed to delete file: ${req.statusText}`); } } async getFileMeta(name: string): Promise { - const res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, { - method: "OPTIONS", - }); + const res = await this.authenticatedFetch( + `${this.fsUrl}/${encodeURI(name)}`, + { + method: "OPTIONS", + }, + ); if (res.status === 404) { throw new Error(`File not found`); } diff --git a/server/hooks/plug_space_primitives.ts b/common/spaces/plug_space_primitives.ts similarity index 94% rename from server/hooks/plug_space_primitives.ts rename to common/spaces/plug_space_primitives.ts index 74166f0..892cbee 100644 --- a/server/hooks/plug_space_primitives.ts +++ b/common/spaces/plug_space_primitives.ts @@ -5,7 +5,10 @@ import { SpacePrimitives, } from "../../common/spaces/space_primitives.ts"; import { FileMeta } from "../../common/types.ts"; -import { NamespaceOperation, PageNamespaceHook } from "./page_namespace.ts"; +import { + NamespaceOperation, + PageNamespaceHook, +} from "../hooks/page_namespace.ts"; import { base64DecodeDataUrl } from "../../plugos/asset_bundle/base64.ts"; export class PlugSpacePrimitives implements SpacePrimitives { @@ -46,8 +49,8 @@ export class PlugSpacePrimitives implements SpacePrimitives { } } } - const result = await this.wrapped.fetchFileList(); - for (const pm of result) { + const files = await this.wrapped.fetchFileList(); + for (const pm of files) { allFiles.push(pm); } return allFiles; diff --git a/common/spaces/space.ts b/common/spaces/space.ts index 7bdbeb1..b3610be 100644 --- a/common/spaces/space.ts +++ b/common/spaces/space.ts @@ -1,9 +1,13 @@ import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives.ts"; -import { AttachmentMeta, FileMeta, PageMeta } from "../types.ts"; +import { AttachmentMeta, PageMeta } from "../types.ts"; import { EventEmitter } from "../../plugos/event.ts"; import { Plug } from "../../plugos/plug.ts"; import { plugPrefix } from "./constants.ts"; import { safeRun } from "../util.ts"; +import { + FileMeta, + ProxyFileSystem, +} from "../../plug-api/plugos-syscall/types.ts"; const pageWatchInterval = 2000; @@ -14,16 +18,42 @@ export type SpaceEvents = { pageListUpdated: (pages: Set) => void; }; -export class Space extends EventEmitter { +export class Space extends EventEmitter + implements ProxyFileSystem { pageMetaCache = new Map(); watchedPages = new Set(); private initialPageListLoad = true; private saving = false; - constructor(private space: SpacePrimitives) { + constructor(readonly spacePrimitives: SpacePrimitives) { super(); } + // Filesystem interface implementation + async readFile(path: string, encoding: "dataurl" | "utf8"): Promise { + return (await this.spacePrimitives.readFile(path, encoding)).data as string; + } + getFileMeta(path: string): Promise { + return this.spacePrimitives.getFileMeta(path); + } + writeFile( + path: string, + text: string, + encoding: "dataurl" | "utf8", + ): Promise { + return this.spacePrimitives.writeFile(path, encoding, text); + } + deleteFile(path: string): Promise { + return this.spacePrimitives.deleteFile(path); + } + async listFiles(path: string): Promise { + return (await this.spacePrimitives.fetchFileList()).filter((f) => + f.name.startsWith(path) + ); + } + + // The more domain-specific methods + public async updatePageList() { const newPageList = await this.fetchPageList(); const deletedPages = new Set(this.pageMetaCache.keys()); @@ -81,7 +111,7 @@ export class Space extends EventEmitter { async deletePage(name: string): Promise { await this.getPageMeta(name); // Check if page exists, if not throws Error - await this.space.deleteFile(`${name}.md`); + await this.spacePrimitives.deleteFile(`${name}.md`); this.pageMetaCache.delete(name); this.emit("pageDeleted", name); @@ -91,7 +121,7 @@ export class Space extends EventEmitter { async getPageMeta(name: string): Promise { const oldMeta = this.pageMetaCache.get(name); const newMeta = fileMetaToPageMeta( - await this.space.getFileMeta(`${name}.md`), + await this.spacePrimitives.getFileMeta(`${name}.md`), ); if (oldMeta) { if (oldMeta.lastModified !== newMeta.lastModified) { @@ -108,7 +138,7 @@ export class Space extends EventEmitter { name: string, args: any[], ): Promise { - return this.space.invokeFunction(plug, env, name, args); + return this.spacePrimitives.invokeFunction(plug, env, name, args); } listPages(): Set { @@ -116,18 +146,21 @@ export class Space extends EventEmitter { } async listPlugs(): Promise { - const allFiles = await this.space.fetchFileList(); - return allFiles + const files = await this.spacePrimitives.fetchFileList(); + return files .filter((fileMeta) => fileMeta.name.endsWith(".plug.json")) .map((fileMeta) => fileMeta.name); } proxySyscall(plug: Plug, name: string, args: any[]): Promise { - return this.space.proxySyscall(plug, name, args); + return this.spacePrimitives.proxySyscall(plug, name, args); } async readPage(name: string): Promise<{ text: string; meta: PageMeta }> { - const pageData = await this.space.readFile(`${name}.md`, "string"); + const pageData = await this.spacePrimitives.readFile( + `${name}.md`, + "utf8", + ); const previousMeta = this.pageMetaCache.get(name); const newMeta = fileMetaToPageMeta(pageData.meta); if (previousMeta) { @@ -159,7 +192,12 @@ export class Space extends EventEmitter { try { this.saving = true; const pageMeta = fileMetaToPageMeta( - await this.space.writeFile(`${name}.md`, "string", text, selfUpdate), + await this.spacePrimitives.writeFile( + `${name}.md`, + "utf8", + text, + selfUpdate, + ), ); if (!selfUpdate) { this.emit("pageChanged", pageMeta); @@ -171,13 +209,13 @@ export class Space extends EventEmitter { } async fetchPageList(): Promise { - return (await this.space.fetchFileList()) + return (await this.spacePrimitives.fetchFileList()) .filter((fileMeta) => fileMeta.name.endsWith(".md")) .map(fileMetaToPageMeta); } async fetchAttachmentList(): Promise { - return (await this.space.fetchFileList()).filter( + return (await this.spacePrimitives.fetchFileList()).filter( (fileMeta) => !fileMeta.name.endsWith(".md") && !fileMeta.name.endsWith(".plug.json") && @@ -195,11 +233,11 @@ export class Space extends EventEmitter { name: string, encoding: FileEncoding, ): Promise<{ data: FileData; meta: AttachmentMeta }> { - return this.space.readFile(name, encoding); + return this.spacePrimitives.readFile(name, encoding); } getAttachmentMeta(name: string): Promise { - return this.space.getFileMeta(name); + return this.spacePrimitives.getFileMeta(name); } writeAttachment( @@ -208,11 +246,11 @@ export class Space extends EventEmitter { data: FileData, selfUpdate?: boolean | undefined, ): Promise { - return this.space.writeFile(name, encoding, data, selfUpdate); + return this.spacePrimitives.writeFile(name, encoding, data, selfUpdate); } deleteAttachment(name: string): Promise { - return this.space.deleteFile(name); + return this.spacePrimitives.deleteFile(name); } private metaCacher(name: string, meta: PageMeta): PageMeta { diff --git a/common/spaces/space_primitives.ts b/common/spaces/space_primitives.ts index d4e346e..2af5359 100644 --- a/common/spaces/space_primitives.ts +++ b/common/spaces/space_primitives.ts @@ -1,10 +1,10 @@ import { Plug } from "../../plugos/plug.ts"; import { FileMeta } from "../types.ts"; -export type FileEncoding = "string" | "arraybuffer" | "dataurl"; +export type FileEncoding = "utf8" | "arraybuffer" | "dataurl"; export type FileData = ArrayBuffer | string; export interface SpacePrimitives { - // Pages + // Returns a list of file meta data as well as the timestamp of this snapshot fetchFileList(): Promise; readFile( name: string, @@ -15,6 +15,7 @@ export interface SpacePrimitives { name: string, encoding: FileEncoding, data: FileData, + // Used to decide whether or not to emit change events selfUpdate?: boolean, ): Promise; deleteFile(name: string): Promise; diff --git a/common/spaces/sync.test.ts b/common/spaces/sync.test.ts new file mode 100644 index 0000000..b7bd102 --- /dev/null +++ b/common/spaces/sync.test.ts @@ -0,0 +1,143 @@ +import { SpaceSync, SyncStatusItem } from "./sync.ts"; +import { DiskSpacePrimitives } from "./disk_space_primitives.ts"; +import { assertEquals } from "../../test_deps.ts"; + +Deno.test("Test store", async () => { + const primaryPath = await Deno.makeTempDir(); + const secondaryPath = await Deno.makeTempDir(); + console.log("Primary", primaryPath); + console.log("Secondary", secondaryPath); + const primary = new DiskSpacePrimitives(primaryPath); + const secondary = new DiskSpacePrimitives(secondaryPath); + const statusMap = new Map(); + const sync = new SpaceSync(primary, secondary, statusMap); + + // Write one page to primary + await primary.writeFile("index", "utf8", "Hello"); + assertEquals((await secondary.fetchFileList()).length, 0); + console.log("Initial sync ops", await doSync()); + + assertEquals((await secondary.fetchFileList()).length, 1); + assertEquals((await secondary.readFile("index", "utf8")).data, "Hello"); + + // Should be a no-op + assertEquals(await doSync(), 0); + + // Now let's make a change on the secondary + await secondary.writeFile("index", "utf8", "Hello!!"); + await secondary.writeFile("test", "utf8", "Test page"); + + // And sync it + await doSync(); + + assertEquals((await primary.fetchFileList()).length, 2); + assertEquals((await secondary.fetchFileList()).length, 2); + + assertEquals((await primary.readFile("index", "utf8")).data, "Hello!!"); + + // Let's make some random edits on both ends + await primary.writeFile("index", "utf8", "1"); + await primary.writeFile("index2", "utf8", "2"); + await secondary.writeFile("index3", "utf8", "3"); + await secondary.writeFile("index4", "utf8", "4"); + await doSync(); + + assertEquals((await primary.fetchFileList()).length, 5); + assertEquals((await secondary.fetchFileList()).length, 5); + + assertEquals(await doSync(), 0); + + console.log("Deleting pages"); + // Delete some pages + await primary.deleteFile("index"); + await primary.deleteFile("index3"); + + await doSync(); + + assertEquals((await primary.fetchFileList()).length, 3); + assertEquals((await secondary.fetchFileList()).length, 3); + + // No-op + assertEquals(await doSync(), 0); + + await secondary.deleteFile("index4"); + await primary.deleteFile("index2"); + + await doSync(); + + // Just "test" left + assertEquals((await primary.fetchFileList()).length, 1); + assertEquals((await secondary.fetchFileList()).length, 1); + + // No-op + assertEquals(await doSync(), 0); + + await secondary.writeFile("index", "utf8", "I'm back"); + + await doSync(); + + assertEquals((await primary.readFile("index", "utf8")).data, "I'm back"); + + // Cause a conflict + console.log("Introducing a conflict now"); + await primary.writeFile("index", "utf8", "Hello 1"); + await secondary.writeFile("index", "utf8", "Hello 2"); + + await doSync(); + + // Sync conflicting copy back + await doSync(); + + // Verify that primary won + assertEquals((await primary.readFile("index", "utf8")).data, "Hello 1"); + assertEquals((await secondary.readFile("index", "utf8")).data, "Hello 1"); + + // test + index + index.conflicting copy + assertEquals((await primary.fetchFileList()).length, 3); + assertEquals((await secondary.fetchFileList()).length, 3); + + // Introducing a fake conflict (same content, so not really conflicting) + await primary.writeFile("index", "utf8", "Hello 1"); + await secondary.writeFile("index", "utf8", "Hello 1"); + + await doSync(); + await doSync(); + + // test + index + previous index.conflicting copy but nothing more + assertEquals((await primary.fetchFileList()).length, 3); + + console.log("Bringing a third device in the mix"); + + const ternaryPath = await Deno.makeTempDir(); + + console.log("Ternary", ternaryPath); + + const ternary = new DiskSpacePrimitives(ternaryPath); + const sync2 = new SpaceSync( + secondary, + ternary, + new Map(), + ); + console.log("N ops", await sync2.syncFiles()); + await sleep(2); + assertEquals(await sync2.syncFiles(), 0); + + await Deno.remove(primaryPath, { recursive: true }); + await Deno.remove(secondaryPath, { recursive: true }); + await Deno.remove(ternaryPath, { recursive: true }); + + async function doSync() { + await sleep(); + const r = await sync.syncFiles( + SpaceSync.primaryConflictResolver, + ); + await sleep(); + return r; + } +}); + +function sleep(ms = 10): Promise { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} diff --git a/common/spaces/sync.ts b/common/spaces/sync.ts new file mode 100644 index 0000000..c5df889 --- /dev/null +++ b/common/spaces/sync.ts @@ -0,0 +1,271 @@ +import type { FileMeta } from "../types.ts"; +import { SpacePrimitives } from "./space_primitives.ts"; + +type SyncHash = number; + +// Tuple where the first value represents a lastModified timestamp for the primary space +// and the second item the lastModified value of the secondary space +export type SyncStatusItem = [SyncHash, SyncHash]; + +// Implementation of this algorithm https://unterwaditzer.net/2016/sync-algorithm.html +export class SpaceSync { + constructor( + private primary: SpacePrimitives, + private secondary: SpacePrimitives, + readonly snapshot: Map, + ) {} + + async syncFiles( + conflictResolver?: ( + name: string, + snapshot: Map, + primarySpace: SpacePrimitives, + secondarySpace: SpacePrimitives, + ) => Promise, + ): Promise { + let operations = 0; + console.log("Fetching snapshot from primary"); + const primaryAllPages = this.syncCandidates( + await this.primary.fetchFileList(), + ); + + console.log("Fetching snapshot from secondary"); + try { + const secondaryAllPages = this.syncCandidates( + await this.secondary.fetchFileList(), + ); + + const primaryFileMap = new Map( + primaryAllPages.map((m) => [m.name, m.lastModified]), + ); + const secondaryFileMap = new Map( + secondaryAllPages.map((m) => [m.name, m.lastModified]), + ); + + const allFilesToProcess = new Set([ + ...this.snapshot.keys(), + ...primaryFileMap.keys(), + ...secondaryFileMap.keys(), + ]); + + console.log("Iterating over all files"); + for (const name of allFilesToProcess) { + if ( + primaryFileMap.has(name) && !secondaryFileMap.has(name) && + !this.snapshot.has(name) + ) { + // New file, created on primary, copy from primary to secondary + console.log( + "New file created on primary, copying to secondary", + name, + ); + const { data } = await this.primary.readFile(name, "arraybuffer"); + const writtenMeta = await this.secondary.writeFile( + name, + "arraybuffer", + data, + ); + this.snapshot.set(name, [ + primaryFileMap.get(name)!, + writtenMeta.lastModified, + ]); + operations++; + } else if ( + secondaryFileMap.has(name) && !primaryFileMap.has(name) && + !this.snapshot.has(name) + ) { + // New file, created on secondary, copy from secondary to primary + console.log( + "New file created on secondary, copying from secondary to primary", + name, + ); + const { data } = await this.secondary.readFile(name, "arraybuffer"); + const writtenMeta = await this.primary.writeFile( + name, + "arraybuffer", + data, + ); + this.snapshot.set(name, [ + writtenMeta.lastModified, + secondaryFileMap.get(name)!, + ]); + operations++; + } else if ( + primaryFileMap.has(name) && this.snapshot.has(name) && + !secondaryFileMap.has(name) + ) { + // File deleted on B + console.log("File deleted on secondary, deleting from primary", name); + await this.primary.deleteFile(name); + this.snapshot.delete(name); + operations++; + } else if ( + secondaryFileMap.has(name) && this.snapshot.has(name) && + !primaryFileMap.has(name) + ) { + // File deleted on A + console.log("File deleted on primary, deleting from secondary", name); + await this.secondary.deleteFile(name); + this.snapshot.delete(name); + operations++; + } else if ( + this.snapshot.has(name) && !primaryFileMap.has(name) && + !secondaryFileMap.has(name) + ) { + // File deleted on both sides, :shrug: + console.log("File deleted on both ends, deleting from status", name); + this.snapshot.delete(name); + operations++; + } else if ( + primaryFileMap.has(name) && secondaryFileMap.has(name) && + this.snapshot.get(name) && + primaryFileMap.get(name) !== this.snapshot.get(name)![0] && + secondaryFileMap.get(name) === this.snapshot.get(name)![1] + ) { + // File has changed on primary, but not secondary: copy from primary to secondary + console.log("File changed on primary, copying to secondary", name); + const { data } = await this.primary.readFile(name, "arraybuffer"); + const writtenMeta = await this.secondary.writeFile( + name, + "arraybuffer", + data, + ); + this.snapshot.set(name, [ + primaryFileMap.get(name)!, + writtenMeta.lastModified, + ]); + operations++; + } else if ( + primaryFileMap.has(name) && secondaryFileMap.has(name) && + this.snapshot.get(name) && + secondaryFileMap.get(name) !== this.snapshot.get(name)![1] && + primaryFileMap.get(name) === this.snapshot.get(name)![0] + ) { + // File has changed on secondary, but not primary: copy from secondary to primary + const { data } = await this.secondary.readFile(name, "arraybuffer"); + const writtenMeta = await this.primary.writeFile( + name, + "arraybuffer", + data, + ); + this.snapshot.set(name, [ + writtenMeta.lastModified, + secondaryFileMap.get(name)!, + ]); + operations++; + } else if ( + ( // File changed on both ends, but we don't have any info in the snapshot (resync scenario?): have to run through conflict handling + primaryFileMap.has(name) && secondaryFileMap.has(name) && + !this.snapshot.has(name) + ) || + ( // File changed on both ends, CONFLICT! + primaryFileMap.has(name) && secondaryFileMap.has(name) && + this.snapshot.get(name) && + secondaryFileMap.get(name) !== this.snapshot.get(name)![1] && + primaryFileMap.get(name) !== this.snapshot.get(name)![0] + ) + ) { + console.log("File changed on both ends, conflict!", name); + if (conflictResolver) { + await conflictResolver( + name, + this.snapshot, + this.primary, + this.secondary, + ); + } else { + throw Error( + `Sync conflict for ${name} with no conflict resolver specified`, + ); + } + operations++; + } else { + // Nothing needs to happen + } + } + } catch (e: any) { + console.error("Boom", e.message); + throw e; + } + + return operations; + } + + // Strategy: Primary wins + public static async primaryConflictResolver( + name: string, + snapshot: Map, + primary: SpacePrimitives, + secondary: SpacePrimitives, + ): Promise { + console.log("Hit a conflict for", name); + const filePieces = name.split("."); + const fileNameBase = filePieces.slice(0, -1).join("."); + const fileNameExt = filePieces[filePieces.length - 1]; + const pageData1 = await primary.readFile(name, "arraybuffer"); + const pageData2 = await secondary.readFile(name, "arraybuffer"); + + let byteWiseMatch = true; + const arrayBuffer1 = new Uint8Array(pageData1.data as ArrayBuffer); + const arrayBuffer2 = new Uint8Array(pageData2.data as ArrayBuffer); + if (arrayBuffer1.byteLength !== arrayBuffer2.byteLength) { + byteWiseMatch = false; + } + if (byteWiseMatch) { + // Byte-wise comparison + for (let i = 0; i < arrayBuffer1.byteLength; i++) { + if (arrayBuffer1[i] !== arrayBuffer2[i]) { + byteWiseMatch = false; + break; + } + } + // Byte wise they're still the same, so no confict + if (byteWiseMatch) { + snapshot.set(name, [ + pageData1.meta.lastModified, + pageData2.meta.lastModified, + ]); + return; + } + } + const revisionFileName = filePieces.length === 1 + ? `${name}.conflicted.${pageData2.meta.lastModified}` + : `${fileNameBase}.conflicted.${pageData2.meta.lastModified}.${fileNameExt}`; + console.log( + "Going to create conflicting copy", + revisionFileName, + ); + + // Copy secondary to conflict copy + const localConflictMeta = await primary.writeFile( + revisionFileName, + "arraybuffer", + pageData2.data, + ); + const remoteConflictMeta = await secondary.writeFile( + revisionFileName, + "arraybuffer", + pageData2.data, + ); + + // Updating snapshot + snapshot.set(revisionFileName, [ + localConflictMeta.lastModified, + remoteConflictMeta.lastModified, + ]); + + // Write replacement on top + const writeMeta = await secondary.writeFile( + name, + "arraybuffer", + pageData1.data, + true, + ); + + snapshot.set(name, [pageData1.meta.lastModified, writeMeta.lastModified]); + } + + syncCandidates(files: FileMeta[]): FileMeta[] { + return files.filter((f) => !f.name.startsWith("_plug/")); + } +} diff --git a/server/syscalls/space.ts b/common/syscalls/space.ts similarity index 89% rename from server/syscalls/space.ts rename to common/syscalls/space.ts index e4af791..e634180 100644 --- a/server/syscalls/space.ts +++ b/common/syscalls/space.ts @@ -6,6 +6,8 @@ import { FileEncoding, } from "../../common/spaces/space_primitives.ts"; +import { FileMeta as PlugFileMeta } from "../../plug-api/plugos-syscall/types.ts"; + export default (space: Space): SysCallMapping => { return { "space.listPages": (): PageMeta[] => { @@ -59,5 +61,9 @@ export default (space: Space): SysCallMapping => { "space.deleteAttachment": async (_ctx, name: string) => { await space.deleteAttachment(name); }, + + "space.listFiles": (_ctx, path: string): Promise => { + return space.listFiles(path); + }, }; }; diff --git a/common/syscalls/sync.ts b/common/syscalls/sync.ts new file mode 100644 index 0000000..89ce9c1 --- /dev/null +++ b/common/syscalls/sync.ts @@ -0,0 +1,65 @@ +import { SysCallMapping } from "../../plugos/system.ts"; +import type { SyncEndpoint } from "../../plug-api/silverbullet-syscall/sync.ts"; +import { SpaceSync, SyncStatusItem } from "../spaces/sync.ts"; +import { HttpSpacePrimitives } from "../spaces/http_space_primitives.ts"; +import { SpacePrimitives } from "../spaces/space_primitives.ts"; + +export function syncSyscalls(localSpace: SpacePrimitives): SysCallMapping { + return { + "sync.sync": async ( + _ctx, + endpoint: SyncEndpoint, + snapshot: Record, + ): Promise< + { + snapshot: Record; + operations: number; + // The reason to not just throw an Error is so that the partially updated snapshot can still be saved + error?: string; + } + > => { + const syncSpace = new HttpSpacePrimitives( + endpoint.url, + endpoint.user, + endpoint.password, + // Base64 PUTs to support mobile + true, + ); + // Convert from JSON to a Map + const syncStatusMap = new Map( + Object.entries(snapshot), + ); + const spaceSync = new SpaceSync( + localSpace, + syncSpace, + syncStatusMap, + ); + + try { + const operations = await spaceSync.syncFiles( + SpaceSync.primaryConflictResolver, + ); + return { + // And convert back to JSON + snapshot: Object.fromEntries(spaceSync.snapshot), + operations, + }; + } catch (e: any) { + return { + snapshot: Object.fromEntries(spaceSync.snapshot), + operations: -1, + error: e.message, + }; + } + }, + "sync.check": async (_ctx, endpoint: SyncEndpoint): Promise => { + const syncSpace = new HttpSpacePrimitives( + endpoint.url, + endpoint.user, + endpoint.password, + ); + // Let's just fetch the file list to see if it works + await syncSpace.fetchFileList(); + }, + }; +} diff --git a/common/util.ts b/common/util.ts index 4c85a4d..5b7a8eb 100644 --- a/common/util.ts +++ b/common/util.ts @@ -1,6 +1,7 @@ import { SETTINGS_TEMPLATE } from "./settings_template.ts"; import { YAML } from "./deps.ts"; import { Space } from "./spaces/space.ts"; +import { BuiltinSettings } from "../web/types.ts"; export function safeRun(fn: () => Promise) { fn().catch((e) => { @@ -45,7 +46,15 @@ export function parseYamlSettings(settingsMarkdown: string): { } } -export async function ensureAndLoadSettings(space: Space) { +export async function ensureAndLoadSettings( + space: Space, + dontCreate: boolean, +): Promise { + if (dontCreate) { + return { + indexPage: "index", + }; + } try { await space.getPageMeta("SETTINGS"); } catch { diff --git a/deno.jsonc b/deno.jsonc index 1d81124..c55c583 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -3,9 +3,9 @@ "clean": "rm -rf dist dist_bundle", "install": "deno install -f -A --unstable silverbullet.ts", "test": "deno test -A --unstable", - "build": "deno run -A --unstable --check build_plugs.ts && deno run -A --unstable --check build.ts", + "build": "deno run -A --unstable --check build_plugs.ts && deno run -A --unstable --check build_web.ts", "plugs": "deno run -A --unstable --check build_plugs.ts", - "watch-web": "deno run -A --unstable --check build.ts --watch", + "watch-web": "deno run -A --unstable --check build_web.ts --watch", "watch-mobile": "deno run -A --unstable --check build_mobile.ts --watch", "watch-server": "deno run -A --unstable --check --watch silverbullet.ts", // The only reason to run a shell script is that deno task doesn't support globs yet (e.g. *.plug.yaml) @@ -22,7 +22,7 @@ "desktop:build": "deno task build && deno task bundle && cd desktop && npm run make", // Mobile "mobile:deps": "cd mobile && npm install && npx cap sync", - "mobile:build": "deno run -A --unstable --check build_mobile.ts && cd mobile && npx cap copy && npx cap open ios" + "mobile:build": "deno task clean && deno task plugs && deno run -A --unstable --check build_mobile.ts && cd mobile && npx cap copy && npx cap open ios" }, "compilerOptions": { diff --git a/desktop/src/menu.ts b/desktop/src/menu.ts index 7baf66b..f462cdc 100644 --- a/desktop/src/menu.ts +++ b/desktop/src/menu.ts @@ -155,7 +155,18 @@ const template: MenuItemConstructorOptions[] = [ if (process.platform === "darwin") { const name = app.getName(); - template.unshift({ label: name, submenu: [] }); + template.unshift({ + label: name, + submenu: [ + { role: "services" }, + { type: "separator" }, + { role: "hide" }, + { role: "hideOthers" }, + { role: "unhide" }, + { type: "separator" }, + { role: "quit" }, + ], + }); } export const menu = Menu.buildFromTemplate(template); diff --git a/mobile/boot.ts b/mobile/boot.ts index da0afa8..2c4242e 100644 --- a/mobile/boot.ts +++ b/mobile/boot.ts @@ -1,8 +1,8 @@ import { Editor } from "../web/editor.tsx"; import { ensureAndLoadSettings, safeRun } from "../common/util.ts"; import { Space } from "../common/spaces/space.ts"; -import { PlugSpacePrimitives } from "../server/hooks/plug_space_primitives.ts"; -import { PageNamespaceHook } from "../server/hooks/page_namespace.ts"; +import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts"; +import { PageNamespaceHook } from "../common/hooks/page_namespace.ts"; import { SilverBulletHooks } from "../common/manifest.ts"; import { System } from "../plugos/system.ts"; import { BuiltinSettings } from "../web/types.ts"; @@ -75,7 +75,10 @@ safeRun(async () => { const serverSpace = new Space(spacePrimitives); serverSpace.watch(); - const settings = await ensureAndLoadSettings(serverSpace) as BuiltinSettings; + const settings = await ensureAndLoadSettings( + serverSpace, + false, + ) as BuiltinSettings; // Register some mobile-specific syscall implementations system.registerSyscalls( diff --git a/mobile/spaces/capacitor_space_primitives.ts b/mobile/spaces/capacitor_space_primitives.ts index da20517..672c1c0 100644 --- a/mobile/spaces/capacitor_space_primitives.ts +++ b/mobile/spaces/capacitor_space_primitives.ts @@ -13,7 +13,10 @@ import { Directory, Encoding, Filesystem } from "../deps.ts"; import { mime } from "../../plugos/deps.ts"; export class CapacitorSpacePrimitives implements SpacePrimitives { - constructor(readonly source: Directory, readonly root: string) { + constructor( + readonly source: Directory, + readonly root: string, + ) { } async fetchFileList(): Promise { @@ -28,8 +31,9 @@ export class CapacitorSpacePrimitives implements SpacePrimitives { }); for (const file of files.files) { if (file.type === "file") { + const name = `${dir}/${file.name}`.substring(1); allFiles.push({ - name: `${dir}/${file.name}`.substring(1), + name: name, lastModified: file.mtime, perm: "rw", contentType: mime.getType(file.name) || "application/octet-stream", @@ -41,7 +45,6 @@ export class CapacitorSpacePrimitives implements SpacePrimitives { } } await readAllFiles(""); - console.log("allFiles", allFiles); return allFiles; } async readFile( @@ -51,7 +54,7 @@ export class CapacitorSpacePrimitives implements SpacePrimitives { let data: FileData | undefined; try { switch (encoding) { - case "string": + case "utf8": data = (await Filesystem.readFile({ path: this.root + name, directory: this.source, @@ -109,7 +112,7 @@ export class CapacitorSpacePrimitives implements SpacePrimitives { data: FileData, ): Promise { switch (encoding) { - case "string": + case "utf8": await Filesystem.writeFile({ path: this.root + name, directory: this.source, diff --git a/plug-api/plugos-syscall/fs.ts b/plug-api/plugos-syscall/fs.ts index 6602e63..4f849e3 100644 --- a/plug-api/plugos-syscall/fs.ts +++ b/plug-api/plugos-syscall/fs.ts @@ -1,36 +1,48 @@ import { syscall } from "./syscall.ts"; +import type { FileMeta, ProxyFileSystem } from "./types.ts"; -export type FileMeta = { - name: string; - lastModified: number; -}; +export class LocalFileSystem implements ProxyFileSystem { + constructor(readonly root: string) { + } -export function readFile( - path: string, - encoding: "utf8" | "dataurl" = "utf8", -): Promise { - return syscall("fs.readFile", path, encoding); -} - -export function getFileMeta(path: string): Promise { - return syscall("fs.getFileMeta", path); -} - -export function writeFile( - path: string, - text: string, - encoding: "utf8" | "dataurl" = "utf8", -): Promise { - return syscall("fs.writeFile", path, text, encoding); -} - -export function deleteFile(path: string): Promise { - return syscall("fs.deleteFile", path); -} - -export function listFiles( - dirName: string, - recursive = false, -): Promise { - return syscall("fs.listFiles", dirName, recursive); + readFile( + path: string, + encoding: "utf8" | "dataurl" = "utf8", + ): Promise { + return syscall("fs.readFile", `${this.root}/${path}`, encoding); + } + + async getFileMeta(path: string): Promise { + return this.removeRootDir( + await syscall("fs.getFileMeta", `${this.root}/${path}`), + ); + } + + writeFile( + path: string, + text: string, + encoding: "utf8" | "dataurl" = "utf8", + ): Promise { + return syscall("fs.writeFile", `${this.root}/${path}`, text, encoding); + } + + deleteFile(path: string): Promise { + return syscall("fs.deleteFile", `${this.root}/${path}`); + } + + async listFiles( + dirName: string, + recursive = false, + ): Promise { + return (await syscall( + "fs.listFiles", + `${this.root}/${dirName}`, + recursive, + )).map(this.removeRootDir.bind(this)); + } + + private removeRootDir(fileMeta: FileMeta): FileMeta { + fileMeta.name = fileMeta.name.substring(this.root.length + 1); + return fileMeta; + } } diff --git a/plug-api/plugos-syscall/mod.ts b/plug-api/plugos-syscall/mod.ts index cf1fa24..b87abe9 100644 --- a/plug-api/plugos-syscall/mod.ts +++ b/plug-api/plugos-syscall/mod.ts @@ -1,6 +1,7 @@ export * as asset from "./asset.ts"; export * as events from "./event.ts"; -export * as fs from "./fs.ts"; +// export * as fs from "./fs.ts"; +export { LocalFileSystem } from "./fs.ts"; export * as sandbox from "./sandbox.ts"; export * as fulltext from "./fulltext.ts"; export * as shell from "./shell.ts"; diff --git a/plug-api/plugos-syscall/types.ts b/plug-api/plugos-syscall/types.ts new file mode 100644 index 0000000..e39c251 --- /dev/null +++ b/plug-api/plugos-syscall/types.ts @@ -0,0 +1,25 @@ +export type FileMeta = { + name: string; + lastModified: number; +}; + +export interface ProxyFileSystem { + readFile( + path: string, + encoding: "utf8" | "dataurl", + ): Promise; + + getFileMeta(path: string): Promise; + + writeFile( + path: string, + text: string, + encoding: "utf8" | "dataurl", + ): Promise; + + deleteFile(path: string): Promise; + + listFiles( + path: string, + ): Promise; +} diff --git a/plug-api/silverbullet-syscall/mod.ts b/plug-api/silverbullet-syscall/mod.ts index 6dca8a4..b4b9033 100644 --- a/plug-api/silverbullet-syscall/mod.ts +++ b/plug-api/silverbullet-syscall/mod.ts @@ -3,6 +3,7 @@ export * as editor from "./editor.ts"; export * as index from "./index.ts"; export * as markdown from "./markdown.ts"; export * as sandbox from "./sandbox.ts"; -export * as space from "./space.ts"; +export { default as space } from "./space.ts"; export * as system from "./system.ts"; export * as collab from "./collab.ts"; +export * as sync from "./sync.ts"; diff --git a/plug-api/silverbullet-syscall/space.ts b/plug-api/silverbullet-syscall/space.ts index 6d42f46..3a8fcf8 100644 --- a/plug-api/silverbullet-syscall/space.ts +++ b/plug-api/silverbullet-syscall/space.ts @@ -1,70 +1,98 @@ import { syscall } from "./syscall.ts"; import { AttachmentMeta, PageMeta } from "../../common/types.ts"; +import { FileMeta, ProxyFileSystem } from "../plugos-syscall/types.ts"; -export function listPages(unfiltered = false): Promise { - return syscall("space.listPages", unfiltered); +export class SpaceFileSystem implements ProxyFileSystem { + // More space-specific methods + + listPages(unfiltered = false): Promise { + return syscall("space.listPages", unfiltered); + } + + getPageMeta(name: string): Promise { + return syscall("space.getPageMeta", name); + } + + readPage( + name: string, + ): Promise { + return syscall("space.readPage", name); + } + + writePage(name: string, text: string): Promise { + return syscall("space.writePage", name, text); + } + + deletePage(name: string): Promise { + return syscall("space.deletePage", name); + } + + listPlugs(): Promise { + return syscall("space.listPlugs"); + } + + listAttachments(): Promise { + return syscall("space.listAttachments"); + } + + getAttachmentMeta(name: string): Promise { + return syscall("space.getAttachmentMeta", name); + } + + /** + * Read an attachment from the space + * @param name path of the attachment to read + * @returns the attachment data encoded as a data URL + */ + readAttachment( + name: string, + ): Promise { + return syscall("space.readAttachment", name); + } + + /** + * Writes an attachment to the space + * @param name path of the attachment to write + * @param encoding encoding of the data ("utf8" or "dataurl) + * @param data data itself + * @returns + */ + writeAttachment( + name: string, + encoding: "utf8" | "dataurl", + data: string, + ): Promise { + return syscall("space.writeAttachment", name, encoding, data); + } + + /** + * Deletes an attachment from the space + * @param name path of the attachment to delete + */ + deleteAttachment(name: string): Promise { + return syscall("space.deleteAttachment", name); + } + + // Filesystem implementation + readFile(path: string, encoding: "dataurl" | "utf8"): Promise { + return syscall("space.readFile", path, encoding); + } + getFileMeta(path: string): Promise { + return syscall("space.getFileMeta", path); + } + writeFile( + path: string, + text: string, + encoding: "dataurl" | "utf8", + ): Promise { + return syscall("space.writeFile", path, text, encoding); + } + deleteFile(path: string): Promise { + return syscall("space.deleteFile", path); + } + listFiles(path: string): Promise { + return syscall("space.listFiles", path); + } } -export function getPageMeta(name: string): Promise { - return syscall("space.getPageMeta", name); -} - -export function readPage( - name: string, -): Promise { - return syscall("space.readPage", name); -} - -export function writePage(name: string, text: string): Promise { - return syscall("space.writePage", name, text); -} - -export function deletePage(name: string): Promise { - return syscall("space.deletePage", name); -} - -export function listPlugs(): Promise { - return syscall("space.listPlugs"); -} - -export function listAttachments(): Promise { - return syscall("space.listAttachments"); -} - -export function getAttachmentMeta(name: string): Promise { - return syscall("space.getAttachmentMeta", name); -} - -/** - * Read an attachment from the space - * @param name path of the attachment to read - * @returns the attachment data encoded as a data URL - */ -export function readAttachment( - name: string, -): Promise { - return syscall("space.readAttachment", name); -} - -/** - * Writes an attachment to the space - * @param name path of the attachment to write - * @param encoding encoding of the data ("string" or "dataurl) - * @param data data itself - * @returns - */ -export function writeAttachment( - name: string, - encoding: "string" | "dataurl", - data: string, -): Promise { - return syscall("space.writeAttachment", name, encoding, data); -} - -/** - * Deletes an attachment from the space - * @param name path of the attachment to delete - */ -export function deleteAttachment(name: string): Promise { - return syscall("space.deleteAttachment", name); -} +export default new SpaceFileSystem(); diff --git a/plug-api/silverbullet-syscall/sync.ts b/plug-api/silverbullet-syscall/sync.ts new file mode 100644 index 0000000..5fe00c7 --- /dev/null +++ b/plug-api/silverbullet-syscall/sync.ts @@ -0,0 +1,28 @@ +import type { SyncStatusItem } from "../../common/spaces/sync.ts"; +import { syscall } from "./syscall.ts"; + +export type SyncEndpoint = { + url: string; + user?: string; + password?: string; +}; + +// Perform a sync with the server, based on the given status (to be persisted) +// returns a new sync status to persist +export function sync( + endpoint: SyncEndpoint, + snapshot: Record, +): Promise< + { + snapshot: Record; + operations: number; + error?: string; + } +> { + return syscall("sync.sync", endpoint, snapshot); +} + +// Checks the sync endpoint for connectivity and authentication, throws and Error on failure +export function check(endpoint: SyncEndpoint): Promise { + return syscall("sync.check", endpoint); +} diff --git a/plugos/sqlite/deno-sqlite/src/db.test.ts b/plugos/sqlite/deno-sqlite/src/db.test.ts deleted file mode 100644 index 25677aa..0000000 --- a/plugos/sqlite/deno-sqlite/src/db.test.ts +++ /dev/null @@ -1,462 +0,0 @@ -import { - assertAlmostEquals, - assertEquals, - assertThrows, -} from "https://deno.land/std@0.154.0/testing/asserts.ts"; - -import { DB } from "../mod.ts"; - -const TEST_DB = "test.db"; -const LARGE_TEST_DB = "build/2GB_test.db"; - -async function dbPermissions(path: string): Promise { - const query = async (name: "read" | "write") => - (await Deno.permissions.query({ name, path })).state === - "granted"; - return await query("read") && await query("write"); -} - -const TEST_DB_PERMISSIONS = await dbPermissions(TEST_DB); -const LARGE_TEST_DB_PERMISSIONS = await dbPermissions(LARGE_TEST_DB); - -async function deleteDatabase(file: string) { - try { - await Deno.remove(file); - } catch { /* no op */ } - try { - await Deno.remove(`${file}-journal`); - } catch { /* no op */ } -} - -Deno.test("execute multiple statements", function () { - const db = new DB(); - - db.execute(` - CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT); - - INSERT INTO test (id) VALUES (1); - INSERT INTO test (id) VALUES (2); - INSERT INTO test (id) VALUES (3); - `); - assertEquals(db.query("SELECT id FROM test"), [[1], [2], [3]]); - - // table `test` already exists ... - assertThrows(function () { - db.execute(` - CREATE TABLE test2 (id INTEGER); - CREATE TABLE test (id INTEGER); - `); - }); - - // ... but table `test2` was created before the error - assertEquals(db.query("SELECT id FROM test2"), []); - - // syntax error after first valid statement - assertThrows(() => db.execute("SELECT id FROM test; NOT SQL ANYMORE")); -}); - -Deno.test("foreign key constraints enabled", function () { - const db = new DB(); - db.execute(` - CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT); - CREATE TABLE orders (id INTEGER PRIMARY KEY AUTOINCREMENT, user INTEGER, FOREIGN KEY(user) REFERENCES users(id)); - `); - - db.query("INSERT INTO users (id) VALUES (1)"); - const [{ id }] = db.queryEntries<{ id: number }>("SELECT id FROM users"); - - // user must exist - assertThrows(() => - db.query("INSERT INTO orders (user) VALUES (?)", [id + 1]) - ); - db.query("INSERT INTO orders (user) VALUES (?)", [id]); - - // can't delete if that violates the constraint ... - assertThrows(() => { - db.query("DELETE FROM users WHERE id = ?", [id]); - }); - - // ... after deleting the order, deleting is OK - db.query("DELETE FROM orders WHERE user = ?", [id]); - db.query("DELETE FROM users WHERE id = ?", [id]); -}); - -Deno.test("json functions exist", function () { - const db = new DB(); - - // The JSON1 functions should exist and we should be able to call them without unexpected errors - db.query(`SELECT json('{"this is": ["json"]}')`); - - // We should expect an error if we pass invalid JSON where valid JSON is expected - assertThrows(() => { - db.query(`SELECT json('this is not json')`); - }); - - // We should be able to use bound values as arguments to the JSON1 functions, - // and they should produce the expected results for these simple expressions. - const [[objectType]] = db.query(`SELECT json_type('{}')`); - assertEquals(objectType, "object"); - - const [[integerType]] = db.query(`SELECT json_type(?)`, ["2"]); - assertEquals(integerType, "integer"); - - const [[realType]] = db.query(`SELECT json_type(?)`, ["2.5"]); - assertEquals(realType, "real"); - - const [[stringType]] = db.query(`SELECT json_type(?)`, [`"hello"`]); - assertEquals(stringType, "text"); - - const [[integerTypeAtPath]] = db.query( - `SELECT json_type(?, ?)`, - [`["hello", 2, {"world": 4}]`, `$[2].world`], - ); - assertEquals(integerTypeAtPath, "integer"); -}); - -Deno.test("date time is correct", function () { - const db = new DB(); - // the date/ time is passed from JS and should be current (note that it is GMT) - const [[now]] = [...db.query("SELECT STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')")]; - const jsTime = new Date().getTime(); - const dbTime = new Date(`${now}Z`).getTime(); - // to account for runtime latency, a small difference is ok - const tolerance = 10; - assertAlmostEquals(jsTime, dbTime, tolerance); - db.close(); -}); - -Deno.test("SQL localtime reflects system locale", function () { - const db = new DB(); - const [[timeDb]] = db.query("SELECT datetime('now', 'localtime')"); - const now = new Date(); - - const jsMonth = `${now.getMonth() + 1}`.padStart(2, "0"); - const jsDate = `${now.getDate()}`.padStart(2, "0"); - const jsHour = `${now.getHours()}`.padStart(2, "0"); - const jsMinute = `${now.getMinutes()}`.padStart(2, "0"); - const jsSecond = `${now.getSeconds()}`.padStart(2, "0"); - const timeJs = - `${now.getFullYear()}-${jsMonth}-${jsDate} ${jsHour}:${jsMinute}:${jsSecond}`; - - assertEquals(timeDb, timeJs); -}); - -Deno.test("database has correct changes and totalChanges", function () { - const db = new DB(); - - db.execute( - "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)", - ); - - for (const name of ["a", "b", "c"]) { - db.query("INSERT INTO test (name) VALUES (?)", [name]); - assertEquals(1, db.changes); - } - - assertEquals(3, db.totalChanges); - - db.query("UPDATE test SET name = ?", ["new name"]); - assertEquals(3, db.changes); - assertEquals(6, db.totalChanges); -}); - -Deno.test("last inserted id", function () { - const db = new DB(); - - // By default, lastInsertRowId must be 0 - assertEquals(db.lastInsertRowId, 0); - - // Create table and insert value - db.query("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT)"); - - const insertRowIds = []; - - // Insert data to table and collect their ids - for (let i = 0; i < 10; i++) { - db.query("INSERT INTO users (name) VALUES ('John Doe')"); - insertRowIds.push(db.lastInsertRowId); - } - - // Now, the last inserted row id must be 10 - assertEquals(db.lastInsertRowId, 10); - - // All collected row ids must be the same as in the database - assertEquals( - insertRowIds, - [...db.query("SELECT id FROM users")].map(([i]) => i), - ); - - db.close(); - - // When the database is closed, the value - // will be reset to 0 again - assertEquals(db.lastInsertRowId, 0); -}); - -Deno.test("close database", function () { - const db = new DB(); - db.close(); - assertThrows(() => db.query("CREATE TABLE test (name TEXT PRIMARY KEY)")); - db.close(); // check close is idempotent and won't throw -}); - -Deno.test("open queries block close", function () { - const db = new DB(); - db.query("CREATE TABLE test (name TEXT PRIMARY KEY)"); - - const query = db.prepareQuery("SELECT name FROM test"); - assertThrows(() => db.close()); - query.finalize(); - - db.close(); -}); - -Deno.test("open queries cleaned up by forced close", function () { - const db = new DB(); - db.query("CREATE TABLE test (name TEXT PRIMARY KEY)"); - db.query("INSERT INTO test (name) VALUES (?)", ["Deno"]); - - db.prepareQuery("SELECT name FROM test WHERE name like '%test%'"); - - assertThrows(() => db.close()); - db.close(true); -}); - -Deno.test("invalid bind does not leak statements", function () { - const db = new DB(); - db.query("CREATE TABLE test (id INTEGER)"); - - for (let n = 0; n < 100; n++) { - assertThrows(() => { - // deno-lint-ignore no-explicit-any - const badBinding: any = [{}]; - db.query("INSERT INTO test (id) VALUES (?)", badBinding); - }); - assertThrows(() => { - const badBinding = { missingKey: null }; - db.query("INSERT INTO test (id) VALUES (?)", badBinding); - }); - } - - db.query("INSERT INTO test (id) VALUES (1)"); - - db.close(); -}); - -Deno.test("transactions can be nested", function () { - const db = new DB(); - db.query("CREATE TABLE test (id INTEGER PRIMARY KEY)"); - - db.transaction(() => { - db.query("INSERT INTO test (id) VALUES (1)"); - try { - db.transaction(() => { - db.query("INSERT INTO test (id) VALUES (2)"); - throw new Error("boom!"); - }); - } catch (_) { /* ignore */ } - }); - - assertEquals([{ id: 1 }], db.queryEntries("SELECT * FROM test")); -}); - -Deno.test("transactions commit when closure exists", function () { - const db = new DB(); - db.query("CREATE TABLE test (id INTEGER PRIMARY KEY)"); - - db.transaction(() => { - db.query("INSERT INTO test (id) VALUES (1)"); - }); - assertThrows(() => db.query("ROLLBACK")); - - assertEquals([{ id: 1 }], db.queryEntries("SELECT * FROM test")); -}); - -Deno.test("transaction rolls back on throw", function () { - const db = new DB(); - db.query("CREATE TABLE test (id INTEGER PRIMARY KEY)"); - - assertThrows(() => { - db.transaction(() => { - db.query("INSERT INTO test (id) VALUES (1)"); - throw new Error("boom!"); - }); - }); - - assertEquals([], db.query("SELECT * FROM test")); -}); - -Deno.test( - "persist database to file", - { - ignore: !TEST_DB_PERMISSIONS, - permissions: { read: true, write: true }, - sanitizeResources: true, - }, - async function () { - const data = [ - "Hello World!", - "Hello Deno!", - "JavaScript <3", - "This costs 0€ / $0 / £0", - "Wéll, hällö thėrè¿", - ]; - - // ensure the test database file does not exist - await deleteDatabase(TEST_DB); - - const db = new DB(TEST_DB); - db.execute( - "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, val TEXT)", - ); - for (const val of data) { - db.query("INSERT INTO test (val) VALUES (?)", [val]); - } - - // open the same database with a separate connection - const readOnlyDb = await new DB(TEST_DB, { mode: "read" }); - for ( - const [id, val] of readOnlyDb.query<[number, string]>( - "SELECT * FROM test", - ) - ) { - assertEquals(data[id - 1], val); - } - - await Deno.remove(TEST_DB); - db.close(); - readOnlyDb.close(); - }, -); - -Deno.test( - "temporary file database read / write", - { - ignore: !TEST_DB_PERMISSIONS, - permissions: { read: true, write: true }, - sanitizeResources: true, - }, - function () { - const data = [ - "Hello World!", - "Hello Deno!", - "JavaScript <3", - "This costs 0€ / $0 / £0", - "Wéll, hällö thėrè¿", - ]; - - const tempDb = new DB(""); - tempDb.execute( - "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, val TEXT)", - ); - for (const val of data) { - tempDb.query("INSERT INTO test (val) VALUES (?)", [val]); - } - - for ( - const [id, val] of tempDb.query<[number, string]>("SELECT * FROM test") - ) { - assertEquals(data[id - 1], val); - } - - tempDb.close(); - }, -); - -Deno.test( - "database open options", - { - ignore: !TEST_DB_PERMISSIONS, - permissions: { read: true, write: true }, - sanitizeResources: true, - }, - async function () { - await deleteDatabase(TEST_DB); - - // when no file exists, these should error - assertThrows(() => new DB(TEST_DB, { mode: "write" })); - assertThrows(() => new DB(TEST_DB, { mode: "read" })); - - // create the database - const dbCreate = new DB(TEST_DB, { mode: "create" }); - dbCreate.execute( - "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL)", - ); - dbCreate.close(); - - // the default mode is create - await deleteDatabase(TEST_DB); - const dbCreateDefault = new DB(TEST_DB, { mode: "create" }); - dbCreateDefault.execute( - "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL)", - ); - dbCreateDefault.close(); - - // in write mode, we can run INSERT queries ... - const dbWrite = new DB(TEST_DB, { mode: "write" }); - dbWrite.query("INSERT INTO test (name) VALUES (?)", ["open-options-test"]); - dbWrite.close(); - - // ... which we can read in read-only mode ... - const dbRead = new DB(TEST_DB, { mode: "read" }); - const rows = [...dbRead.query("SELECT id, name FROM test")]; - assertEquals(rows, [[1, "open-options-test"]]); - - // ... but we can't write with a read-only connection - assertThrows(() => - dbRead.query("INTERT INTO test (name) VALUES (?)", ["this-fails"]) - ); - dbRead.close(); - }, -); - -Deno.test( - "create / write mode require write permissions", - { - ignore: !TEST_DB_PERMISSIONS, - permissions: { read: true, write: false }, - sanitizeResources: true, - }, - function () { - // opening with these modes requires write permissions ... - assertThrows(() => new DB(TEST_DB, { mode: "create" })); - assertThrows(() => new DB(TEST_DB, { mode: "write" })); - - // ... and the default mode is create - assertThrows(() => new DB(TEST_DB)); - - // however, opening in read-only mode should work (the file was created - // in the previous test) - (new DB(TEST_DB, { mode: "read" })).close(); - - // with memory flag set, the database will be in memory and - // not require any permissions - (new DB(TEST_DB, { mode: "create", memory: true })).close(); - - // the mode can also be specified via a URI flag - (new DB(`file:${TEST_DB}?mode=memory`, { uri: true })).close(); - }, -); - -Deno.test( - "database larger than 2GB read / write", - { - ignore: !LARGE_TEST_DB_PERMISSIONS, - permissions: { read: true, write: true }, - sanitizeResources: true, - }, - function () { - // generated with `cd build && make testdb` - const db = new DB(LARGE_TEST_DB, { mode: "write" }); - - db.query("INSERT INTO test (value) VALUES (?)", ["This is a test..."]); - - const rows = [ - ...db.query("SELECT value FROM test ORDER BY id DESC LIMIT 10"), - ]; - assertEquals(rows.length, 10); - assertEquals(rows[0][0], "This is a test..."); - - db.close(); - }, -); diff --git a/plugos/sqlite/deno-sqlite/src/error.test.ts b/plugos/sqlite/deno-sqlite/src/error.test.ts deleted file mode 100644 index ce21c38..0000000 --- a/plugos/sqlite/deno-sqlite/src/error.test.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { - assertEquals, - assertInstanceOf, - assertThrows, -} from "https://deno.land/std@0.154.0/testing/asserts.ts"; - -import { DB, SqliteError, Status } from "../mod.ts"; - -Deno.test("invalid SQL", function () { - const db = new DB(); - const queries = [ - "INSERT INTO does_not_exist (balance) VALUES (5)", - "this is not sql", - ";;;", - ]; - for (const query of queries) assertThrows(() => db.query(query)); - - db.close(); -}); - -Deno.test("constraint error code is correct", function () { - const db = new DB(); - db.query("CREATE TABLE test (name TEXT PRIMARY KEY)"); - db.query("INSERT INTO test (name) VALUES (?)", ["A"]); - - assertThrows( - () => db.query("INSERT INTO test (name) VALUES (?)", ["A"]), - (e: Error) => { - assertInstanceOf(e, SqliteError); - assertEquals(e.code, Status.SqliteConstraint, "Got wrong error code"); - assertEquals( - Status[e.codeName], - Status.SqliteConstraint, - "Got wrong error code name", - ); - }, - ); -}); - -Deno.test("syntax error code is correct", function () { - const db = new DB(); - - assertThrows( - () => db.query("CREATE TABLEX test (name TEXT PRIMARY KEY)"), - (e: Error) => { - assertInstanceOf(e, SqliteError); - assertEquals(e.code, Status.SqliteError, "Got wrong error code"); - assertEquals( - Status[e.codeName], - Status.SqliteError, - "Got wrong error code name", - ); - }, - ); -}); diff --git a/plugos/sqlite/deno-sqlite/src/query.test.ts b/plugos/sqlite/deno-sqlite/src/query.test.ts deleted file mode 100644 index 0bad3aa..0000000 --- a/plugos/sqlite/deno-sqlite/src/query.test.ts +++ /dev/null @@ -1,521 +0,0 @@ -import { - assertEquals, - assertThrows, -} from "https://deno.land/std@0.154.0/testing/asserts.ts"; - -import { DB, QueryParameter } from "../mod.ts"; - -function roundTripValues(values: T[]): unknown[] { - const db = new DB(); - db.execute( - "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, datum ANY)", - ); - - for (const value of values) { - db.query("INSERT INTO test (datum) VALUES (?)", [value]); - } - - return db - .queryEntries<{ datum: unknown }>("SELECT datum FROM test") - .map(({ datum }) => datum); -} - -Deno.test("bind string values", function () { - const values = ["Hello World!", "I love Deno.", "Täst strüng..."]; - assertEquals(values, roundTripValues(values)); -}); - -Deno.test("bind integer values", function () { - const values = [42, 1, 2, 3, 4, 3453246, 4536787093, 45536787093]; - assertEquals(values, roundTripValues(values)); -}); - -Deno.test("bind float values", function () { - const values = [42.1, 1.235, 2.999, 1 / 3, 4.2345, 345.3246, 4536787.953e-8]; - assertEquals(values, roundTripValues(values)); -}); - -Deno.test("bind boolean values", function () { - assertEquals([1, 0], roundTripValues([true, false])); -}); - -Deno.test("bind date values", function () { - const values = [new Date(), new Date("2018-11-20"), new Date(123456789)]; - assertEquals( - values.map((date) => date.toISOString()), - roundTripValues(values), - ); -}); - -Deno.test("bind blob values", function () { - const values = [ - new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0]), - new Uint8Array([3, 57, 45]), - ]; - assertEquals(values, roundTripValues(values)); -}); - -Deno.test("blobs are copies", function () { - const db = new DB(); - - db.query( - "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, val BLOB)", - ); - const data = new Uint8Array([1, 2, 3, 4, 5]); - db.query("INSERT INTO test (val) VALUES (?)", [data]); - - const [[a]] = db.query<[Uint8Array]>("SELECT val FROM test"); - const [[b]] = db.query<[Uint8Array]>("SELECT val FROM test"); - - assertEquals(data, a); - assertEquals(data, b); - assertEquals(a, b); - - a[0] = 100; - assertEquals(a[0], 100); - assertEquals(b[0], 1); - assertEquals(data[0], 1); - - data[0] = 5; - const [[c]] = db.query<[Uint8Array]>("SELECT val FROM test"); - assertEquals(c[0], 1); -}); - -Deno.test("bind bigint values", function () { - assertEquals( - [9007199254741991n, 100], - roundTripValues([9007199254741991n, 100n]), - ); -}); - -Deno.test("bind null / undefined", function () { - assertEquals([null, null], roundTripValues([null, undefined])); -}); - -Deno.test("bind mixed values", function () { - const values = [42, "Hello World!", 0.33333, null]; - assertEquals(values, roundTripValues(values)); -}); - -Deno.test("omitting a value binds NULL", function () { - const db = new DB(); - db.execute("CREATE TABLE test (id INTEGER PRIMARY KEY, datum ANY)"); - - const insert = db.prepareQuery( - "INSERT INTO test (datum) VALUES (?) RETURNING datum", - ); - - assertEquals([null], insert.first()); - assertEquals([null], insert.first([])); - assertEquals([null], insert.first({})); - - // previously bound values are cleared - insert.execute(["this is not null"]); - assertEquals([null], insert.first()); -}); - -Deno.test("prepared query clears bindings before reused", function () { - const db = new DB(); - db.query("CREATE TABLE test (id INTEGER PRIMARY KEY, value INTEGER)"); - - const query = db.prepareQuery("INSERT INTO test (value) VALUES (?)"); - query.execute([1]); - query.execute(); - - assertEquals([[1], [null]], db.query("SELECT value FROM test")); - - query.finalize(); - db.close(); -}); - -Deno.test("bind very large floating point numbers", function () { - const db = new DB(); - - db.query("CREATE TABLE numbers (id INTEGER PRIMARY KEY, number REAL)"); - - db.query("INSERT INTO numbers (number) VALUES (?)", [+Infinity]); - db.query("INSERT INTO numbers (number) VALUES (?)", [-Infinity]); - db.query("INSERT INTO numbers (number) VALUES (?)", [+20e20]); - db.query("INSERT INTO numbers (number) VALUES (?)", [-20e20]); - - const [ - [positiveInfinity], - [negativeInfinity], - [positiveTwentyTwenty], - [negativeTwentyTwenty], - ] = db.query("SELECT number FROM numbers"); - - assertEquals(negativeInfinity, -Infinity); - assertEquals(positiveInfinity, +Infinity); - assertEquals(positiveTwentyTwenty, +20e20); - assertEquals(negativeTwentyTwenty, -20e20); -}); - -Deno.test("big very large integers", function () { - const db = new DB(); - db.query( - "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, val INTEGER)", - ); - - const goodValues = [ - 0n, - 42n, - -42n, - 9223372036854775807n, - -9223372036854775808n, - ]; - const overflowValues = [ - 9223372036854775807n + 1n, - -9223372036854775808n - 1n, - 2352359223372036854775807n, - -32453249223372036854775807n, - ]; - - const query = db.prepareQuery("INSERT INTO test (val) VALUES (?)"); - for (const val of goodValues) { - query.execute([val]); - } - - const dbValues = db.query<[number | bigint]>( - "SELECT val FROM test ORDER BY id", - ).map(( - [id], - ) => BigInt(id)); - assertEquals(goodValues, dbValues); - - for (const bigVal of overflowValues) { - assertThrows(() => { - query.execute([bigVal]); - }); - } - - query.finalize(); - db.close(); -}); - -Deno.test("bind named parameters", function () { - const db = new DB(); - - db.query( - "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, val TEXT)", - ); - - // :name - db.query("INSERT INTO test (val) VALUES (:val)", { val: "value" }); - db.query( - "INSERT INTO test (val) VALUES (:otherVal)", - { otherVal: "value other" }, - ); - db.query( - "INSERT INTO test (val) VALUES (:explicitColon)", - { ":explicitColon": "value explicit" }, - ); - - // @name - db.query( - "INSERT INTO test (val) VALUES (@someName)", - { "@someName": "@value" }, - ); - - // $name - db.query( - "INSERT INTO test (val) VALUES ($var::Name)", - { "$var::Name": "$value" }, - ); - - // explicit positional syntax - db.query("INSERT INTO test (id, val) VALUES (?2, ?1)", ["this-is-it", 1000]); - - // names must exist - assertThrows(() => { - db.query( - "INSERT INTO test (val) VALUES (:val)", - { Val: "miss-spelled name" }, - ); - }); - - // make sure the data came through correctly - const vals = [...db.query("SELECT val FROM test ORDER BY id ASC")] - .map(([datum]) => datum); - assertEquals( - vals, - [ - "value", - "value other", - "value explicit", - "@value", - "$value", - "this-is-it", - ], - ); -}); - -Deno.test("iterate from prepared query", function () { - const db = new DB(); - db.execute("CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT)"); - db.execute("INSERT INTO test (id) VALUES (1), (2), (3)"); - - const res = []; - const query = db.prepareQuery<[number]>("SELECT id FROM test"); - for (const [id] of query.iter()) { - res.push(id); - } - assertEquals(res, [1, 2, 3]); - - query.finalize(); - db.close(); -}); - -Deno.test("query all from prepared query", function () { - const db = new DB(); - db.query("CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT)"); - const query = db.prepareQuery("SELECT id FROM test"); - - assertEquals(query.all(), []); - db.query("INSERT INTO test (id) VALUES (1), (2), (3)"); - assertEquals(query.all(), [[1], [2], [3]]); - - query.finalize(); - db.close(); -}); - -Deno.test("query first from prepared query", function () { - const db = new DB(); - db.query("CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT)"); - db.query("INSERT INTO test (id) VALUES (1), (2), (3)"); - - const querySingle = db.prepareQuery("SELECT id FROM test WHERE id = ?"); - assertEquals(querySingle.first([42]), undefined); - assertEquals(querySingle.first([2]), [2]); - - const queryAll = db.prepareQuery("SELECT id FROM test ORDER BY id ASC"); - assertEquals(queryAll.first(), [1]); - - querySingle.finalize(); - queryAll.finalize(); - db.close(); -}); - -Deno.test("query one from prepared query", function () { - const db = new DB(); - db.query("CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT)"); - db.query("INSERT INTO test (id) VALUES (1), (2), (3)"); - - const queryOne = db.prepareQuery<[number]>( - "SELECT id FROM test WHERE id = ?", - ); - assertThrows(() => queryOne.one([42])); - assertEquals(queryOne.one([2]), [2]); - - const queryAll = db.prepareQuery("SELECT id FROM test"); - assertThrows(() => queryAll.one()); - - queryOne.finalize(); - queryAll.finalize(); - db.close(); -}); - -Deno.test("execute from prepared query", function () { - const db = new DB(); - db.query("CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT)"); - - const insert = db.prepareQuery("INSERT INTO test (id) VALUES (:id)"); - for (const id of [1, 2, 3]) { - insert.execute({ id }); - } - insert.finalize(); - assertEquals(db.query("SELECT id FROM test"), [[1], [2], [3]]); - - db.close(); -}); - -Deno.test("empty query returns empty array", function () { - const db = new DB(); - db.query("CREATE TABLE test (id INTEGER PRIMARY KEY)"); - assertEquals([], db.query("SELECT * FROM test")); - db.close(); -}); - -Deno.test("query entries returns correct object shapes", function () { - const db = new DB(); - db.query( - "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, height REAL)", - ); - - const rowsOrig = [ - { id: 1, name: "Peter Parker", height: 1.5 }, - { id: 2, name: "Clark Kent", height: 1.9 }, - { id: 3, name: "Robert Paar", height: 2.1 }, - ]; - - const insertQuery = db.prepareQuery( - "INSERT INTO test (id, name, height) VALUES (:id, :name, :height)", - ); - for (const row of rowsOrig) { - insertQuery.execute(row); - } - insertQuery.finalize(); - - const query = db.prepareQuery("SELECT * FROM test"); - assertEquals(rowsOrig, [...query.iterEntries()]); - assertEquals(rowsOrig, query.allEntries()); - assertEquals(rowsOrig[0], query.firstEntry()); - assertEquals(rowsOrig, db.queryEntries("SELECT * FROM test")); - - query.finalize(); - db.close(); -}); - -Deno.test("prepared query can be reused", function () { - const db = new DB(); - db.query("CREATE TABLE test (id INTEGER PRIMARY KEY)"); - - const query = db.prepareQuery("INSERT INTO test (id) VALUES (?)"); - query.execute([1]); - query.execute([2]); - query.execute([3]); - - assertEquals([[1], [2], [3]], db.query("SELECT id FROM test")); - - query.finalize(); - db.close(); -}); - -Deno.test("get columns from select query", function () { - const db = new DB(); - - db.query( - "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)", - ); - - const query = db.prepareQuery("SELECT id, name from test"); - - assertEquals(query.columns(), [ - { name: "id", originName: "id", tableName: "test" }, - { name: "name", originName: "name", tableName: "test" }, - ]); -}); - -Deno.test("get columns from returning query", function () { - const db = new DB(); - - db.query( - "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)", - ); - const query = db.prepareQuery( - "INSERT INTO test (name) VALUES (?) RETURNING *", - ); - - assertEquals(query.columns(), [ - { name: "id", originName: "id", tableName: "test" }, - { name: "name", originName: "name", tableName: "test" }, - ]); - - assertEquals(query.all(["name"]), [[1, "name"]]); -}); - -Deno.test("get columns with renamed column", function () { - const db = new DB(); - - db.query( - "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)", - ); - db.query("INSERT INTO test (name) VALUES (?)", ["name"]); - - const query = db.prepareQuery( - "SELECT id AS test_id, name AS test_name from test", - ); - const columns = query.columns(); - - assertEquals(columns, [ - { name: "test_id", originName: "id", tableName: "test" }, - { name: "test_name", originName: "name", tableName: "test" }, - ]); -}); - -Deno.test("columns can be obtained from empty prepared query", function () { - const db = new DB(); - db.query( - "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEST, age INTEGER)", - ); - db.query("INSERT INTO test (name, age) VALUES (?, ?)", ["Peter Parker", 21]); - - const query = db.prepareQuery("SELECT * FROM test"); - const columnsFromPreparedQuery = query.columns(); - query.finalize(); - - const queryEmpty = db.prepareQuery("SELECT * FROM test WHERE 1 = 0"); - const columnsFromPreparedQueryWithEmptyQuery = queryEmpty.columns(); - assertEquals(queryEmpty.all(), []); - query.finalize(); - - assertEquals( - [{ name: "id", originName: "id", tableName: "test" }, { - name: "name", - originName: "name", - tableName: "test", - }, { name: "age", originName: "age", tableName: "test" }], - columnsFromPreparedQuery, - ); - assertEquals( - columnsFromPreparedQueryWithEmptyQuery, - columnsFromPreparedQuery, - ); -}); - -Deno.test("invalid number of bound parameters throws", function () { - const db = new DB(); - db.execute("CREATE TABLE test (id INTEGER PRIMARY KEY)"); - - // too many - assertThrows(() => { - db.query("SELECT * FROM test", [null]); - }); - assertThrows(() => { - db.query("SELECT * FROM test LIMIT ?", [5, "extra"]); - }); - - // too few - assertThrows(() => db.query("SELECT * FROM test LIMIT ?", [])); - assertThrows(() => { - db.query( - "SELECT * FROM test WHERE id >= ? AND id <= ? LIMIT ?", - [42], - ); - }); -}); - -Deno.test("using finalized prepared query throws", function () { - const db = new DB(); - db.query("CREATE TABLE test (name TEXT)"); - const query = db.prepareQuery("INSERT INTO test (name) VALUES (?)"); - query.finalize(); - - assertThrows(() => query.execute(["test"])); - db.close(); -}); - -Deno.test("invalid binding throws", function () { - const db = new DB(); - db.query("CREATE TABLE test (id INTEGER)"); - assertThrows(() => { - // deno-lint-ignore no-explicit-any - const badBinding: any = [{}]; - db.query("SELECT * FORM test WHERE id = ?", badBinding); - }); - db.close(); -}); - -Deno.test("get columns from finalized query throws", function () { - const db = new DB(); - - db.query("CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT)"); - - const query = db.prepareQuery("SELECT id from test"); - query.finalize(); - - // after iteration is done - assertThrows(() => { - query.columns(); - }); -}); diff --git a/plugos/sqlite/deno-sqlite/src/readme.test.ts b/plugos/sqlite/deno-sqlite/src/readme.test.ts deleted file mode 100644 index 973e7c9..0000000 --- a/plugos/sqlite/deno-sqlite/src/readme.test.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { - assertEquals, - assertMatch, -} from "https://deno.land/std@0.154.0/testing/asserts.ts"; - -import { DB } from "../mod.ts"; - -Deno.test("README example", function () { - const db = new DB(/* in memory */); - db.execute(` - CREATE TABLE IF NOT EXISTS people ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT - ) - `); - - const name = - ["Peter Parker", "Clark Kent", "Bruce Wane"][Math.floor(Math.random() * 3)]; - - // Run a simple query - db.query("INSERT INTO people (name) VALUES (?)", [name]); - - // Print out data in table - for (const [_name] of db.query("SELECT name FROM people")) continue; // no console.log ;) - - db.close(); -}); - -Deno.test("old README example", function () { - const db = new DB(); - const first = ["Bruce", "Clark", "Peter"]; - const last = ["Wane", "Kent", "Parker"]; - db.query( - "CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT, email TEXT, subscribed INTEGER)", - ); - - for (let i = 0; i < 100; i++) { - const name = `${first[Math.floor(Math.random() * first.length)]} ${ - last[ - Math.floor( - Math.random() * last.length, - ) - ] - }`; - const email = `${name.replace(" ", "-")}@deno.land`; - const subscribed = Math.random() > 0.5 ? true : false; - db.query("INSERT INTO users (name, email, subscribed) VALUES (?, ?, ?)", [ - name, - email, - subscribed, - ]); - } - - for ( - const [ - name, - email, - ] of db.query<[string, string]>( - "SELECT name, email FROM users WHERE subscribed = ? LIMIT 100", - [true], - ) - ) { - assertMatch(name, /(Bruce|Clark|Peter) (Wane|Kent|Parker)/); - assertEquals(email, `${name.replace(" ", "-")}@deno.land`); - } - - const res = db.query("SELECT email FROM users WHERE name LIKE ?", [ - "Robert Parr", - ]); - assertEquals(res, []); - - const subscribers = db.query( - "SELECT name, email FROM users WHERE subscribed = ?", - [true], - ); - for (const [_name, _email] of subscribers) { - if (Math.random() > 0.5) continue; - break; - } -}); diff --git a/plugos/sqlite/deno-sqlite/src/wasm.test.ts b/plugos/sqlite/deno-sqlite/src/wasm.test.ts deleted file mode 100644 index 1c0ff2c..0000000 --- a/plugos/sqlite/deno-sqlite/src/wasm.test.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { - assertEquals, - assertThrows, -} from "https://deno.land/std@0.154.0/testing/asserts.ts"; - -import { Wasm } from "../build/sqlite.js"; -import * as wasm from "./wasm.ts"; - -function mock( - malloc: () => number = () => 1, - free: (pts: number) => void = () => {}, -): Wasm { - const memory = new Uint8Array(2048); - return { - malloc, - free, - str_len: (ptr: number) => { - let len = 0; - for (let idx = ptr; memory.at(idx) != 0; idx++) len++; - return len; - }, - memory, - } as unknown as Wasm; -} - -Deno.test("round trip string", function () { - const mockWasm = mock(); - const testCases = ["Hello world!", "Söme, fünky lëttêrß", "你好👋"]; - for (const input of testCases) { - const output = wasm.setStr(mockWasm, input, (ptr) => { - return wasm.getStr(mockWasm, ptr); - }); - assertEquals(input, output); - } -}); - -Deno.test("throws on allocation error", function () { - const mockWasm = mock(() => 0); - assertThrows(() => wasm.setStr(mockWasm, "Hello world!", (_) => null)); -}); diff --git a/plugs/collab/collab.ts b/plugs/collab/collab.ts index 6be051d..b6054b2 100644 --- a/plugs/collab/collab.ts +++ b/plugs/collab/collab.ts @@ -136,8 +136,8 @@ export async function readFileCollab( const text = `---\n$share: ${collabUri}\n---\n`; return { - // encoding === "arraybuffer" is not an option, so either it's "string" or "dataurl" - data: encoding === "string" ? text : base64EncodedDataUrl( + // encoding === "arraybuffer" is not an option, so either it's "utf8" or "dataurl" + data: encoding === "utf8" ? text : base64EncodedDataUrl( "text/markdown", new TextEncoder().encode(text), ), diff --git a/plugs/core/cloud.ts b/plugs/core/cloud.ts index 773b252..a2b407a 100644 --- a/plugs/core/cloud.ts +++ b/plugs/core/cloud.ts @@ -45,7 +45,7 @@ export async function readFileCloud( `${pagePrefix}${originalUrl.split("/")[0]}/`, ); return { - data: encoding === "string" ? text : base64EncodedDataUrl( + data: encoding === "utf8" ? text : base64EncodedDataUrl( "text/markdown", new TextEncoder().encode(text), ), diff --git a/plugs/core/plugmanager.ts b/plugs/core/plugmanager.ts index fa6b94f..a565af7 100644 --- a/plugs/core/plugmanager.ts +++ b/plugs/core/plugmanager.ts @@ -1,9 +1,7 @@ import { events } from "$sb/plugos-syscall/mod.ts"; import type { Manifest } from "../../common/manifest.ts"; import { editor, space, system } from "$sb/silverbullet-syscall/mod.ts"; - import { readYamlPage } from "$sb/lib/yaml_page.ts"; -import { writePage } from "$sb/silverbullet-syscall/space.ts"; const plugsPrelude = "This file lists all plugs that SilverBullet will load. Run the {[Plugs: Update]} command to update and reload this list of plugs.\n\n"; @@ -82,7 +80,7 @@ export async function updatePlugs() { // console.log("Writing", `_plug/${manifest.name}`); await space.writeAttachment( `_plug/${manifest.name}.plug.json`, - "string", + "utf8", JSON.stringify(manifest), ); } diff --git a/plugs/core/search.ts b/plugs/core/search.ts index 6c11d4a..49eb34f 100644 --- a/plugs/core/search.ts +++ b/plugs/core/search.ts @@ -85,8 +85,8 @@ export async function readFileSearch( `; return { - // encoding === "arraybuffer" is not an option, so either it's "string" or "dataurl" - data: encoding === "string" ? text : base64EncodedDataUrl( + // encoding === "arraybuffer" is not an option, so either it's "utf8" or "dataurl" + data: encoding === "utf8" ? text : base64EncodedDataUrl( "text/markdown", new TextEncoder().encode(text), ), diff --git a/plugs/markdown/markdown.plug.yaml b/plugs/markdown/markdown.plug.yaml index 2bd5245..d383a0b 100644 --- a/plugs/markdown/markdown.plug.yaml +++ b/plugs/markdown/markdown.plug.yaml @@ -24,14 +24,14 @@ functions: path: "./preview.ts:previewClickHandler" env: client events: - - preview:click + - preview:click # $share: file:* publisher for markdown files sharePublisher: path: ./share.ts:sharePublisher events: - - share:file + - share:file markdownWidget: path: ./widget.ts:markdownWidget - codeWidget: markdown \ No newline at end of file + codeWidget: markdown diff --git a/plugs/markdown/share.ts b/plugs/markdown/share.ts index 823e7a3..5e903b2 100644 --- a/plugs/markdown/share.ts +++ b/plugs/markdown/share.ts @@ -1,5 +1,5 @@ import { markdown, space } from "$sb/silverbullet-syscall/mod.ts"; -import { fs } from "$sb/plugos-syscall/mod.ts"; +import { LocalFileSystem } from "$sb/plugos-syscall/mod.ts"; import { asset } from "$sb/plugos-syscall/mod.ts"; import { renderMarkdownToHtml } from "./markdown_render.ts"; import { PublishEvent } from "$sb/app_event.ts"; @@ -10,12 +10,14 @@ export async function sharePublisher(event: PublishEvent) { const text = await space.readPage(pageName); const tree = await markdown.parseMarkdown(text); + const rootFS = new LocalFileSystem(""); + const css = await asset.readAsset("assets/styles.css"); const markdownHtml = renderMarkdownToHtml(tree, { smartHardBreak: true, }); const html = `
${markdownHtml}
`; - await fs.writeFile(path, html, "utf8"); + await rootFS.writeFile(path, html, "utf8"); return true; } diff --git a/plugs/sync/sync.plug.yaml b/plugs/sync/sync.plug.yaml new file mode 100644 index 0000000..ab4ae36 --- /dev/null +++ b/plugs/sync/sync.plug.yaml @@ -0,0 +1,19 @@ +name: sync +functions: + configureCommand: + path: sync.ts:configureCommand + command: + name: "Sync: Configure" + + syncCommand: + path: sync.ts:syncCommand + command: + name: "Sync: Sync" + + check: + env: server + path: sync.ts:check + + performSync: + env: server + path: sync.ts:performSync diff --git a/plugs/sync/sync.ts b/plugs/sync/sync.ts new file mode 100644 index 0000000..47cce72 --- /dev/null +++ b/plugs/sync/sync.ts @@ -0,0 +1,88 @@ +import { store } from "$sb/plugos-syscall/mod.ts"; +import { editor, sync, system } from "$sb/silverbullet-syscall/mod.ts"; +import type { SyncEndpoint } from "$sb/silverbullet-syscall/sync.ts"; + +export async function configureCommand() { + const url = await editor.prompt( + "Enter the URL of the remote space to sync with", + "https://", + ); + if (!url) { + return; + } + + const user = await editor.prompt("Username (if any):"); + let password = undefined; + if (user) { + password = await editor.prompt("Password:"); + } + + const syncConfig: SyncEndpoint = { + url, + user, + password, + }; + + try { + await system.invokeFunction("server", "check", syncConfig); + } catch (e: any) { + await editor.flashNotification( + `Sync configuration failed: ${e.message}`, + "error", + ); + return; + } + + await store.batchSet([ + { key: "sync.config", value: syncConfig }, + // Empty initial snapshot + { key: "sync.snapshot", value: {} }, + ]); + + await editor.flashNotification("Sync configuration saved."); + + return syncConfig; +} + +export async function syncCommand() { + let config: SyncEndpoint | undefined = await store.get("sync.config"); + if (!config) { + config = await configureCommand(); + if (!config) { + return; + } + } + await editor.flashNotification("Starting sync..."); + try { + const operations = await system.invokeFunction("server", "performSync"); + await editor.flashNotification( + `Sync complete. Performed ${operations} operations.`, + ); + } catch (e: any) { + await editor.flashNotification( + `Sync failed: ${e.message}`, + "error", + ); + } +} + +// Run on server +export function check(config: SyncEndpoint) { + return sync.check(config); +} + +// Run on server +export async function performSync() { + const config: SyncEndpoint = await store.get("sync.config"); + const snapshot = await store.get("sync.snapshot"); + const { snapshot: newSnapshot, operations, error } = await sync.sync( + config, + snapshot, + ); + await store.set("sync.snapshot", newSnapshot); + if (error) { + console.error("Sync error", error); + throw new Error(error); + } + return operations; +} diff --git a/server/http_server.ts b/server/http_server.ts index 32a2610..6090d9f 100644 --- a/server/http_server.ts +++ b/server/http_server.ts @@ -5,6 +5,7 @@ import { EndpointHook } from "../plugos/hooks/endpoint.ts"; import { AssetBundle } from "../plugos/asset_bundle/bundle.ts"; import { SpaceSystem } from "./space_system.ts"; import { ensureAndLoadSettings } from "../common/util.ts"; +import { base64Decode } from "../plugos/asset_bundle/base64.ts"; export type ServerOptions = { hostname: string; @@ -14,6 +15,7 @@ export type ServerOptions = { assetBundle: AssetBundle; user?: string; pass?: string; + bareMode?: boolean; }; const staticLastModified = new Date().toUTCString(); @@ -26,6 +28,7 @@ export class HttpServer { user?: string; settings: { [key: string]: any } = {}; abortController?: AbortController; + bareMode: boolean; constructor(options: ServerOptions) { this.hostname = options.hostname; @@ -37,6 +40,7 @@ export class HttpServer { options.pagesPath, options.dbPath, ); + this.bareMode = options.bareMode || false; // Second, for loading plug JSON files with absolute or relative (from CWD) paths this.systemBoot.eventHook.addLocalListener( @@ -66,7 +70,7 @@ export class HttpServer { async start() { await this.systemBoot.start(); await this.systemBoot.ensureSpaceIndex(); - await ensureAndLoadSettings(this.systemBoot.space); + await ensureAndLoadSettings(this.systemBoot.space, this.bareMode); this.addPasswordAuth(this.app); @@ -207,7 +211,8 @@ export class HttpServer { // File list fsRouter.get("/", async ({ response }) => { response.headers.set("Content-type", "application/json"); - response.body = JSON.stringify(await spacePrimitives.fetchFileList()); + const files = await spacePrimitives.fetchFileList(); + response.body = JSON.stringify(files); }); fsRouter @@ -248,12 +253,21 @@ export class HttpServer { const name = params[0]; console.log("Saving file", name); + let body: Uint8Array; + if ( + request.headers.get("X-Content-Base64") + ) { + const content = await request.body({ type: "text" }).value; + body = base64Decode(content); + } else { + body = await request.body({ type: "bytes" }).value; + } + try { const meta = await spacePrimitives.writeFile( name, "arraybuffer", - await request.body().value, - false, + body, ); response.status = 200; response.headers.set("Content-Type", meta.contentType); @@ -299,7 +313,6 @@ export class HttpServer { private buildPlugRouter(): Router { const plugRouter = new Router(); - // this.addPasswordAuth(plugRouter); const system = this.systemBoot.system; plugRouter.post( diff --git a/server/space_system.ts b/server/space_system.ts index 6f641a5..d880ba3 100644 --- a/server/space_system.ts +++ b/server/space_system.ts @@ -23,13 +23,13 @@ import { storeSyscalls, } from "../plugos/syscalls/store.sqlite.ts"; import { System } from "../plugos/system.ts"; -import { PageNamespaceHook } from "./hooks/page_namespace.ts"; -import { PlugSpacePrimitives } from "./hooks/plug_space_primitives.ts"; +import { PageNamespaceHook } from "../common/hooks/page_namespace.ts"; +import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts"; import { ensureTable as ensureIndexTable, pageIndexSyscalls, } from "./syscalls/index.ts"; -import spaceSyscalls from "./syscalls/space.ts"; +import spaceSyscalls from "../common/syscalls/space.ts"; import { systemSyscalls } from "./syscalls/system.ts"; import { AssetBundlePlugSpacePrimitives } from "../common/spaces/asset_bundle_space_primitives.ts"; import assetSyscalls from "../plugos/syscalls/asset.ts"; @@ -37,6 +37,7 @@ import { AssetBundle } from "../plugos/asset_bundle/bundle.ts"; import { AsyncSQLite } from "../plugos/sqlite/async_sqlite.ts"; import { FileMetaSpacePrimitives } from "../common/spaces/file_meta_space_primitives.ts"; import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts"; +import { syncSyscalls } from "../common/syscalls/sync.ts"; export const indexRequiredKey = "$spaceIndexed"; // A composition of a PlugOS system attached to a Space for server-side use @@ -111,6 +112,7 @@ export class SpaceSystem { storeSyscalls(this.db, "store"), fullTextSearchSyscalls(this.db, "fts"), spaceSyscalls(this.space), + syncSyscalls(this.spacePrimitives), eventSyscalls(this.eventHook), markdownSyscalls(buildMarkdown([])), esbuildSyscalls([globalModules]), @@ -145,7 +147,7 @@ export class SpaceSystem { console.log("Going to load", allPlugs.length, "plugs..."); await Promise.all(allPlugs.map(async (plugName) => { - const { data } = await this.space.readAttachment(plugName, "string"); + const { data } = await this.space.readAttachment(plugName, "utf8"); await this.system.load(JSON.parse(data as string), createSandbox); })); diff --git a/silverbullet.ts b/silverbullet.ts index e15bd79..519ea7d 100755 --- a/silverbullet.ts +++ b/silverbullet.ts @@ -21,6 +21,9 @@ await new Command() .arguments("") .option("--hostname ", "Hostname or address to listen on") .option("-p, --port ", "Port to listen on") + .option("--bare [type:boolean]", "Don't auto generate pages", { + default: false, + }) .option("--db ", "Filename for the database", { default: "data.db", }) diff --git a/web/boot.ts b/web/boot.ts index ea1ba5a..cc2ea05 100644 --- a/web/boot.ts +++ b/web/boot.ts @@ -2,8 +2,8 @@ import { Editor } from "./editor.tsx"; import { parseYamlSettings, safeRun } from "../common/util.ts"; import { Space } from "../common/spaces/space.ts"; import { HttpSpacePrimitives } from "../common/spaces/http_space_primitives.ts"; -import { PlugSpacePrimitives } from "../server/hooks/plug_space_primitives.ts"; -import { PageNamespaceHook } from "../server/hooks/page_namespace.ts"; +import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts"; +import { PageNamespaceHook } from "../common/hooks/page_namespace.ts"; import { SilverBulletHooks } from "../common/manifest.ts"; import { System } from "../plugos/system.ts"; import { BuiltinSettings } from "./types.ts"; @@ -19,7 +19,7 @@ safeRun(async () => { let settingsPageText = ""; try { settingsPageText = ( - await httpPrimitives.readFile("SETTINGS.md", "string") + await httpPrimitives.readFile("SETTINGS.md", "utf8") ).data as string; } catch (e: any) { console.error("No settings page found", e.message); diff --git a/web/editor.tsx b/web/editor.tsx index 522beda..569231a 100644 --- a/web/editor.tsx +++ b/web/editor.tsx @@ -97,6 +97,7 @@ import type { } from "../plug-api/app_event.ts"; import { CodeWidgetHook } from "./hooks/code_widget.ts"; import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts"; +import { syncSyscalls } from "../common/syscalls/sync.ts"; const frontMatterRegex = /^---\n(.*?)---\n/ms; @@ -195,6 +196,7 @@ export class Editor { markdownSyscalls(buildMarkdown(this.mdExtensions)), sandboxSyscalls(this.system), assetSyscalls(this.system), + syncSyscalls(this.space.spacePrimitives), collabSyscalls(this), ); @@ -659,7 +661,7 @@ export class Editor { await this.system.unloadAll(); console.log("(Re)loading plugs"); await Promise.all((await this.space.listPlugs()).map(async (plugName) => { - const { data } = await this.space.readAttachment(plugName, "string"); + const { data } = await this.space.readAttachment(plugName, "utf8"); await this.system.load(JSON.parse(data as string), createSandbox); })); this.rebuildEditorState(); diff --git a/web/syscalls/space.ts b/web/syscalls/space.ts index 747c07b..d496667 100644 --- a/web/syscalls/space.ts +++ b/web/syscalls/space.ts @@ -1,70 +1,19 @@ import { Editor } from "../editor.tsx"; import { SysCallMapping } from "../../plugos/system.ts"; -import { AttachmentMeta, PageMeta } from "../../common/types.ts"; -import { - FileData, - FileEncoding, -} from "../../common/spaces/space_primitives.ts"; + +import commonSpaceSyscalls from "../../common/syscalls/space.ts"; export function spaceSyscalls(editor: Editor): SysCallMapping { - return { - "space.listPages": (): PageMeta[] => { - return [...editor.space.listPages()]; - }, - "space.readPage": async ( - _ctx, - name: string, - ): Promise => { - return (await editor.space.readPage(name)).text; - }, - "space.getPageMeta": async (_ctx, name: string): Promise => { - return await editor.space.getPageMeta(name); - }, - "space.writePage": async ( - _ctx, - name: string, - text: string, - ): Promise => { - return await editor.space.writePage(name, text); - }, - "space.deletePage": async (_ctx, name: string) => { - // If we're deleting the current page, navigate to the index page - if (editor.currentPage === name) { - await editor.navigate(""); - } - // Remove page from open pages in editor - editor.openPages.delete(name); - console.log("Deleting page"); - await editor.space.deletePage(name); - }, - "space.listPlugs": (): Promise => { - return editor.space.listPlugs(); - }, - "space.listAttachments": (): Promise => { - return editor.space.fetchAttachmentList(); - }, - "space.readAttachment": async ( - _ctx, - name: string, - ): Promise => { - return (await editor.space.readAttachment(name, "dataurl")).data; - }, - "space.getAttachmentMeta": async ( - _ctx, - name: string, - ): Promise => { - return await editor.space.getAttachmentMeta(name); - }, - "space.writeAttachment": async ( - _ctx, - name: string, - encoding: FileEncoding, - data: FileData, - ): Promise => { - return await editor.space.writeAttachment(name, encoding, data); - }, - "space.deleteAttachment": async (_ctx, name: string) => { - await editor.space.deleteAttachment(name); - }, + const syscalls = commonSpaceSyscalls(editor.space); + syscalls["space.deletePage"] = async (_ctx, name: string) => { + // If we're deleting the current page, navigate to the index page + if (editor.currentPage === name) { + await editor.navigate(""); + } + // Remove page from open pages in editor + editor.openPages.delete(name); + console.log("Deleting page"); + await editor.space.deletePage(name); }; + return syscalls; } diff --git a/web/types.ts b/web/types.ts index 7ef9970..aae4035 100644 --- a/web/types.ts +++ b/web/types.ts @@ -14,6 +14,7 @@ export type PanelMode = number; export type BuiltinSettings = { indexPage: string; + syncUrl?: string; }; export type PanelConfig = { diff --git a/website/.gitignore b/website/.gitignore index c553972..0f2c5e4 100644 --- a/website/.gitignore +++ b/website/.gitignore @@ -1,3 +1,2 @@ data.db _plug -_trash \ No newline at end of file diff --git a/website/CHANGELOG.md b/website/CHANGELOG.md index cdb385f..3021627 100644 --- a/website/CHANGELOG.md +++ b/website/CHANGELOG.md @@ -14,6 +14,7 @@ release. select * from my_table; ``` * Merged code for experimental mobile app (iOS only for now) +* Experimental sync engine, to be documented once it matures ---