neil/silverbullet
1
0
Fork 0
Code

Sync engine (#298)

Browse Source 
Fixes #261
This commit is contained in:
Zef Hemel 2023-01-13 15:41:29 +01:00 committed by GitHub
parent de6f531e91
commit a56e14bff1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
51 changed files with 1037 additions and 1422 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.github/workflows/desktop.yml vendored
View File

@ -3,7 +3,7 @@ name: Build & Release
on:
push:
tags:
- '*'
- "*"
jobs:
build:
name: Build (${{ matrix.os }} - ${{ matrix.arch }})
@ -72,6 +72,7 @@ jobs:
files: |
desktop/out/**/*.deb
desktop/out/**/*Setup.exe
desktop/out/**/RELEASES
desktop/out/**/*.rpm
desktop/out/**/*.zip
dist/silverbullet.js

2
build_mobile.ts
View File

@ -1,4 +1,4 @@
import { bundle, esbuild } from "./build.ts";
import { bundle, esbuild } from "./build_web.ts";
import * as flags from "https://deno.land/std@0.165.0/flags/mod.ts";
import { copy } from "https://deno.land/std@0.165.0/fs/copy.ts";

0
build.ts → build_web.ts
View File

2
cmd/server.ts
View File

@ -7,6 +7,7 @@ export function serveCommand(options: any, folder: string) {
const pagesPath = path.resolve(Deno.cwd(), folder);
const hostname = options.hostname || "127.0.0.1";
const port = options.port || 3000;
const bareMode = options.bare;
console.log(
"Going to start Silver Bullet binding to",
@ -27,6 +28,7 @@ export function serveCommand(options: any, folder: string) {
dbPath: path.join(pagesPath, options.db),
assetBundle: new AssetBundle(assetBundle as AssetJson),
user: options.user,
bareMode,
});
httpServer.start().catch((e) => {
console.error("HTTP Server error", e);

0
server/hooks/page_namespace.ts → common/hooks/page_namespace.ts
View File

2
common/manifest.ts
View File

@ -4,7 +4,7 @@ import { CronHookT } from "../plugos/hooks/cron.deno.ts";
import { EventHookT } from "../plugos/hooks/event.ts";
import { CommandHookT } from "../web/hooks/command.ts";
import { SlashCommandHookT } from "../web/hooks/slash_command.ts";
import { PageNamespaceHookT } from "../server/hooks/page_namespace.ts";
import { PageNamespaceHookT } from "./hooks/page_namespace.ts";
import { CodeWidgetT } from "../web/hooks/code_widget.ts";
export type SilverBulletHooks =

8
common/spaces/asset_bundle_space_primitives.ts
View File

@ -12,7 +12,7 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives {
}
async fetchFileList(): Promise<FileMeta[]> {
const l = await this.wrapped.fetchFileList();
const files = await this.wrapped.fetchFileList();
return this.assetBundle.listFiles().filter((p) => p.startsWith("_plug/"))
.map((p) => ({
name: p,
@ -20,7 +20,7 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives {
lastModified: bootTime,
perm: "ro",
size: -1,
} as FileMeta)).concat(l);
} as FileMeta)).concat(files);
}
readFile(
@ -31,7 +31,7 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives {
const data = this.assetBundle.readFileSync(name);
// console.log("Requested encoding", encoding);
return Promise.resolve({
data: encoding === "string" ? new TextDecoder().decode(data) : data,
data: encoding === "utf8" ? new TextDecoder().decode(data) : data,
meta: {
lastModified: bootTime,
size: data.byteLength,
@ -60,7 +60,7 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives {
name: string,
encoding: FileEncoding,
data: FileData,
selfUpdate?: boolean | undefined,
selfUpdate?: boolean,
): Promise<FileMeta> {
return this.wrapped.writeFile(name, encoding, data, selfUpdate);
}

12
common/spaces/disk_space_primitives.ts
View File

@ -15,6 +15,8 @@ function lookupContentType(path: string): string {
return mime.getType(path) || "application/octet-stream";
}
const excludedFiles = ["data.db", "data.db-journal", "sync.json"];
export class DiskSpacePrimitives implements SpacePrimitives {
rootPath: string;
@ -48,7 +50,7 @@ export class DiskSpacePrimitives implements SpacePrimitives {
let data: FileData | null = null;
const contentType = lookupContentType(name);
switch (encoding) {
case "string":
case "utf8":
data = await Deno.readTextFile(localPath);
break;
case "dataurl":
@ -98,7 +100,7 @@ export class DiskSpacePrimitives implements SpacePrimitives {
// Actually write the file
switch (encoding) {
case "string":
case "utf8":
await Deno.writeTextFile(`${localPath}`, data as string);
break;
case "dataurl":
@ -165,8 +167,12 @@ export class DiskSpacePrimitives implements SpacePrimitives {
const fullPath = file.path;
try {
const s = await Deno.stat(fullPath);
const name = fullPath.substring(this.rootPath.length + 1);
if (excludedFiles.includes(name)) {
continue;
}
allFiles.push({
name: fullPath.substring(this.rootPath.length + 1),
name: name,
lastModified: s.mtime!.getTime(),
contentType: mime.getType(fullPath) || "application/octet-stream",
size: s.size,

4
common/spaces/evented_space_primitives.ts
View File

@ -35,7 +35,7 @@ export class EventedSpacePrimitives implements SpacePrimitives {
name: string,
encoding: FileEncoding,
data: FileData,
selfUpdate: boolean,
selfUpdate?: boolean,
): Promise<FileMeta> {
const newMeta = await this.wrapped.writeFile(
name,
@ -48,7 +48,7 @@ export class EventedSpacePrimitives implements SpacePrimitives {
const pageName = name.substring(0, name.length - 3);
let text = "";
switch (encoding) {
case "string":
case "utf8":
text = data as string;
break;
case "arraybuffer":

6
common/spaces/file_meta_space_primitives.ts
View File

@ -12,10 +12,10 @@ export class FileMetaSpacePrimitives implements SpacePrimitives {
}
async fetchFileList(): Promise<FileMeta[]> {
const list = await this.wrapped.fetchFileList();
const files = await this.wrapped.fetchFileList();
// Enrich the file list with custom meta data (for pages)
const allFilesMap: Map<string, any> = new Map(
list.map((fm) => [fm.name, fm]),
files.map((fm) => [fm.name, fm]),
);
for (
const { page, value } of await this.indexSyscalls["index.queryPrefix"](
@ -53,7 +53,7 @@ export class FileMetaSpacePrimitives implements SpacePrimitives {
name: string,
encoding: FileEncoding,
data: FileData,
selfUpdate?: boolean | undefined,
selfUpdate?: boolean,
): Promise<FileMeta> {
return this.wrapped.writeFile(name, encoding, data, selfUpdate);
}

77
common/spaces/http_space_primitives.ts
View File

@ -3,33 +3,51 @@ import { Plug } from "../../plugos/plug.ts";
import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives.ts";
import {
base64DecodeDataUrl,
base64Encode,
base64EncodedDataUrl,
} from "../../plugos/asset_bundle/base64.ts";
import { mime } from "../../plugos/deps.ts";
export class HttpSpacePrimitives implements SpacePrimitives {
fsUrl: string;
private fsUrl: string;
private plugUrl: string;
constructor(url: string) {
constructor(
url: string,
readonly user?: string,
readonly password?: string,
readonly base64Put?: boolean,
) {
this.fsUrl = url + "/fs";
this.plugUrl = url + "/plug";
}
private async authenticatedFetch(
url: string,
options: any,
options: Record<string, any>,
): Promise<Response> {
if (this.user && this.password) {
// Explicitly set an auth cookie
if (!options.headers) {
options.headers = {};
}
options.headers["cookie"] = `auth=${
btoa(`${this.user}:${this.password}`)
}`;
}
const result = await fetch(url, options);
if (result.status === 401) {
if (result.status === 401 || result.redirected) {
// Invalid credentials, reloading the browser should trigger authentication
if (typeof location !== "undefined") {
location.reload();
}
throw Error("Unauthorized");
}
return result;
}
public async fetchFileList(): Promise<FileMeta[]> {
async fetchFileList(): Promise<FileMeta[]> {
const req = await this.authenticatedFetch(this.fsUrl, {
method: "GET",
});
@ -41,9 +59,12 @@ export class HttpSpacePrimitives implements SpacePrimitives {
name: string,
encoding: FileEncoding,
): Promise<{ data: FileData; meta: FileMeta }> {
const res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
const res = await this.authenticatedFetch(
`${this.fsUrl}/${encodeURI(name)}`,
{
method: "GET",
});
},
);
if (res.status === 404) {
throw new Error(`Page not found`);
}
@ -52,7 +73,6 @@ export class HttpSpacePrimitives implements SpacePrimitives {
case "arraybuffer":
{
data = await res.arrayBuffer();
// data = await abBlob.arrayBuffer();
}
break;
case "dataurl":
@ -63,7 +83,7 @@ export class HttpSpacePrimitives implements SpacePrimitives {
);
}
break;
case "string":
case "utf8":
data = await res.text();
break;
}
@ -82,37 +102,56 @@ export class HttpSpacePrimitives implements SpacePrimitives {
switch (encoding) {
case "arraybuffer":
case "string":
// actually we want an Uint8Array
body = data instanceof ArrayBuffer ? new Uint8Array(data) : data;
break;
case "utf8":
body = data;
break;
case "dataurl":
data = base64DecodeDataUrl(data as string);
break;
}
const res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
const headers: Record<string, string> = {
"Content-Type": "application/octet-stream",
};
if (this.base64Put) {
headers["X-Content-Base64"] = "true";
headers["Content-Type"] = "text/plain";
body = base64Encode(body);
}
const res = await this.authenticatedFetch(
`${this.fsUrl}/${encodeURI(name)}`,
{
method: "PUT",
headers: {
"Content-type": "application/octet-stream",
},
headers,
body,
});
},
);
const newMeta = this.responseToMeta(name, res);
return newMeta;
}
async deleteFile(name: string): Promise<void> {
const req = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
const req = await this.authenticatedFetch(
`${this.fsUrl}/${encodeURI(name)}`,
{
method: "DELETE",
});
},
);
if (req.status !== 200) {
throw Error(`Failed to delete file: ${req.statusText}`);
}
}
async getFileMeta(name: string): Promise<FileMeta> {
const res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
const res = await this.authenticatedFetch(
`${this.fsUrl}/${encodeURI(name)}`,
{
method: "OPTIONS",
});
},
);
if (res.status === 404) {
throw new Error(`File not found`);
}

9
server/hooks/plug_space_primitives.ts → common/spaces/plug_space_primitives.ts
View File

@ -5,7 +5,10 @@ import {
SpacePrimitives,
} from "../../common/spaces/space_primitives.ts";
import { FileMeta } from "../../common/types.ts";
import { NamespaceOperation, PageNamespaceHook } from "./page_namespace.ts";
import {
NamespaceOperation,
PageNamespaceHook,
} from "../hooks/page_namespace.ts";
import { base64DecodeDataUrl } from "../../plugos/asset_bundle/base64.ts";
export class PlugSpacePrimitives implements SpacePrimitives {
@ -46,8 +49,8 @@ export class PlugSpacePrimitives implements SpacePrimitives {
}
}
}
const result = await this.wrapped.fetchFileList();
for (const pm of result) {
const files = await this.wrapped.fetchFileList();
for (const pm of files) {
allFiles.push(pm);
}
return allFiles;

72
common/spaces/space.ts
View File

@ -1,9 +1,13 @@
import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives.ts";
import { AttachmentMeta, FileMeta, PageMeta } from "../types.ts";
import { AttachmentMeta, PageMeta } from "../types.ts";
import { EventEmitter } from "../../plugos/event.ts";
import { Plug } from "../../plugos/plug.ts";
import { plugPrefix } from "./constants.ts";
import { safeRun } from "../util.ts";
import {
FileMeta,
ProxyFileSystem,
} from "../../plug-api/plugos-syscall/types.ts";
const pageWatchInterval = 2000;
@ -14,16 +18,42 @@ export type SpaceEvents = {
pageListUpdated: (pages: Set<PageMeta>) => void;
};
export class Space extends EventEmitter<SpaceEvents> {
export class Space extends EventEmitter<SpaceEvents>
implements ProxyFileSystem {
pageMetaCache = new Map<string, PageMeta>();
watchedPages = new Set<string>();
private initialPageListLoad = true;
private saving = false;
constructor(private space: SpacePrimitives) {
constructor(readonly spacePrimitives: SpacePrimitives) {
super();
}
// Filesystem interface implementation
async readFile(path: string, encoding: "dataurl" | "utf8"): Promise<string> {
return (await this.spacePrimitives.readFile(path, encoding)).data as string;
}
getFileMeta(path: string): Promise<FileMeta> {
return this.spacePrimitives.getFileMeta(path);
}
writeFile(
path: string,
text: string,
encoding: "dataurl" | "utf8",
): Promise<FileMeta> {
return this.spacePrimitives.writeFile(path, encoding, text);
}
deleteFile(path: string): Promise<void> {
return this.spacePrimitives.deleteFile(path);
}
async listFiles(path: string): Promise<FileMeta[]> {
return (await this.spacePrimitives.fetchFileList()).filter((f) =>
f.name.startsWith(path)
);
}
// The more domain-specific methods
public async updatePageList() {
const newPageList = await this.fetchPageList();
const deletedPages = new Set<string>(this.pageMetaCache.keys());
@ -81,7 +111,7 @@ export class Space extends EventEmitter<SpaceEvents> {
async deletePage(name: string): Promise<void> {
await this.getPageMeta(name); // Check if page exists, if not throws Error
await this.space.deleteFile(`${name}.md`);
await this.spacePrimitives.deleteFile(`${name}.md`);
this.pageMetaCache.delete(name);
this.emit("pageDeleted", name);
@ -91,7 +121,7 @@ export class Space extends EventEmitter<SpaceEvents> {
async getPageMeta(name: string): Promise<PageMeta> {
const oldMeta = this.pageMetaCache.get(name);
const newMeta = fileMetaToPageMeta(
await this.space.getFileMeta(`${name}.md`),
await this.spacePrimitives.getFileMeta(`${name}.md`),
);
if (oldMeta) {
if (oldMeta.lastModified !== newMeta.lastModified) {
@ -108,7 +138,7 @@ export class Space extends EventEmitter<SpaceEvents> {
name: string,
args: any[],
): Promise<any> {
return this.space.invokeFunction(plug, env, name, args);
return this.spacePrimitives.invokeFunction(plug, env, name, args);
}
listPages(): Set<PageMeta> {
@ -116,18 +146,21 @@ export class Space extends EventEmitter<SpaceEvents> {
}
async listPlugs(): Promise<string[]> {
const allFiles = await this.space.fetchFileList();
return allFiles
const files = await this.spacePrimitives.fetchFileList();
return files
.filter((fileMeta) => fileMeta.name.endsWith(".plug.json"))
.map((fileMeta) => fileMeta.name);
}
proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
return this.space.proxySyscall(plug, name, args);
return this.spacePrimitives.proxySyscall(plug, name, args);
}
async readPage(name: string): Promise<{ text: string; meta: PageMeta }> {
const pageData = await this.space.readFile(`${name}.md`, "string");
const pageData = await this.spacePrimitives.readFile(
`${name}.md`,
"utf8",
);
const previousMeta = this.pageMetaCache.get(name);
const newMeta = fileMetaToPageMeta(pageData.meta);
if (previousMeta) {
@ -159,7 +192,12 @@ export class Space extends EventEmitter<SpaceEvents> {
try {
this.saving = true;
const pageMeta = fileMetaToPageMeta(
await this.space.writeFile(`${name}.md`, "string", text, selfUpdate),
await this.spacePrimitives.writeFile(
`${name}.md`,
"utf8",
text,
selfUpdate,
),
);
if (!selfUpdate) {
this.emit("pageChanged", pageMeta);
@ -171,13 +209,13 @@ export class Space extends EventEmitter<SpaceEvents> {
}
async fetchPageList(): Promise<PageMeta[]> {
return (await this.space.fetchFileList())
return (await this.spacePrimitives.fetchFileList())
.filter((fileMeta) => fileMeta.name.endsWith(".md"))
.map(fileMetaToPageMeta);
}
async fetchAttachmentList(): Promise<AttachmentMeta[]> {
return (await this.space.fetchFileList()).filter(
return (await this.spacePrimitives.fetchFileList()).filter(
(fileMeta) =>
!fileMeta.name.endsWith(".md") &&
!fileMeta.name.endsWith(".plug.json") &&
@ -195,11 +233,11 @@ export class Space extends EventEmitter<SpaceEvents> {
name: string,
encoding: FileEncoding,
): Promise<{ data: FileData; meta: AttachmentMeta }> {
return this.space.readFile(name, encoding);
return this.spacePrimitives.readFile(name, encoding);
}
getAttachmentMeta(name: string): Promise<AttachmentMeta> {
return this.space.getFileMeta(name);
return this.spacePrimitives.getFileMeta(name);
}
writeAttachment(
@ -208,11 +246,11 @@ export class Space extends EventEmitter<SpaceEvents> {
data: FileData,
selfUpdate?: boolean | undefined,
): Promise<AttachmentMeta> {
return this.space.writeFile(name, encoding, data, selfUpdate);
return this.spacePrimitives.writeFile(name, encoding, data, selfUpdate);
}
deleteAttachment(name: string): Promise<void> {
return this.space.deleteFile(name);
return this.spacePrimitives.deleteFile(name);
}
private metaCacher(name: string, meta: PageMeta): PageMeta {

5
common/spaces/space_primitives.ts
View File

@ -1,10 +1,10 @@
import { Plug } from "../../plugos/plug.ts";
import { FileMeta } from "../types.ts";
export type FileEncoding = "string" | "arraybuffer" | "dataurl";
export type FileEncoding = "utf8" | "arraybuffer" | "dataurl";
export type FileData = ArrayBuffer | string;
export interface SpacePrimitives {
// Pages
// Returns a list of file meta data as well as the timestamp of this snapshot
fetchFileList(): Promise<FileMeta[]>;
readFile(
name: string,
@ -15,6 +15,7 @@ export interface SpacePrimitives {
name: string,
encoding: FileEncoding,
data: FileData,
// Used to decide whether or not to emit change events
selfUpdate?: boolean,
): Promise<FileMeta>;
deleteFile(name: string): Promise<void>;

143
common/spaces/sync.test.ts Normal file
View File

@ -0,0 +1,143 @@
import { SpaceSync, SyncStatusItem } from "./sync.ts";
import { DiskSpacePrimitives } from "./disk_space_primitives.ts";
import { assertEquals } from "../../test_deps.ts";
Deno.test("Test store", async () => {
const primaryPath = await Deno.makeTempDir();
const secondaryPath = await Deno.makeTempDir();
console.log("Primary", primaryPath);
console.log("Secondary", secondaryPath);
const primary = new DiskSpacePrimitives(primaryPath);
const secondary = new DiskSpacePrimitives(secondaryPath);
const statusMap = new Map<string, SyncStatusItem>();
const sync = new SpaceSync(primary, secondary, statusMap);
// Write one page to primary
await primary.writeFile("index", "utf8", "Hello");
assertEquals((await secondary.fetchFileList()).length, 0);
console.log("Initial sync ops", await doSync());
assertEquals((await secondary.fetchFileList()).length, 1);
assertEquals((await secondary.readFile("index", "utf8")).data, "Hello");
// Should be a no-op
assertEquals(await doSync(), 0);
// Now let's make a change on the secondary
await secondary.writeFile("index", "utf8", "Hello!!");
await secondary.writeFile("test", "utf8", "Test page");
// And sync it
await doSync();
assertEquals((await primary.fetchFileList()).length, 2);
assertEquals((await secondary.fetchFileList()).length, 2);
assertEquals((await primary.readFile("index", "utf8")).data, "Hello!!");
// Let's make some random edits on both ends
await primary.writeFile("index", "utf8", "1");
await primary.writeFile("index2", "utf8", "2");
await secondary.writeFile("index3", "utf8", "3");
await secondary.writeFile("index4", "utf8", "4");
await doSync();
assertEquals((await primary.fetchFileList()).length, 5);
assertEquals((await secondary.fetchFileList()).length, 5);
assertEquals(await doSync(), 0);
console.log("Deleting pages");
// Delete some pages
await primary.deleteFile("index");
await primary.deleteFile("index3");
await doSync();
assertEquals((await primary.fetchFileList()).length, 3);
assertEquals((await secondary.fetchFileList()).length, 3);
// No-op
assertEquals(await doSync(), 0);
await secondary.deleteFile("index4");
await primary.deleteFile("index2");
await doSync();
// Just "test" left
assertEquals((await primary.fetchFileList()).length, 1);
assertEquals((await secondary.fetchFileList()).length, 1);
// No-op
assertEquals(await doSync(), 0);
await secondary.writeFile("index", "utf8", "I'm back");
await doSync();
assertEquals((await primary.readFile("index", "utf8")).data, "I'm back");
// Cause a conflict
console.log("Introducing a conflict now");
await primary.writeFile("index", "utf8", "Hello 1");
await secondary.writeFile("index", "utf8", "Hello 2");
await doSync();
// Sync conflicting copy back
await doSync();
// Verify that primary won
assertEquals((await primary.readFile("index", "utf8")).data, "Hello 1");
assertEquals((await secondary.readFile("index", "utf8")).data, "Hello 1");
// test + index + index.conflicting copy
assertEquals((await primary.fetchFileList()).length, 3);
assertEquals((await secondary.fetchFileList()).length, 3);
// Introducing a fake conflict (same content, so not really conflicting)
await primary.writeFile("index", "utf8", "Hello 1");
await secondary.writeFile("index", "utf8", "Hello 1");
await doSync();
await doSync();
// test + index + previous index.conflicting copy but nothing more
assertEquals((await primary.fetchFileList()).length, 3);
console.log("Bringing a third device in the mix");
const ternaryPath = await Deno.makeTempDir();
console.log("Ternary", ternaryPath);
const ternary = new DiskSpacePrimitives(ternaryPath);
const sync2 = new SpaceSync(
secondary,
ternary,
new Map<string, SyncStatusItem>(),
);
console.log("N ops", await sync2.syncFiles());
await sleep(2);
assertEquals(await sync2.syncFiles(), 0);
await Deno.remove(primaryPath, { recursive: true });
await Deno.remove(secondaryPath, { recursive: true });
await Deno.remove(ternaryPath, { recursive: true });
async function doSync() {
await sleep();
const r = await sync.syncFiles(
SpaceSync.primaryConflictResolver,
);
await sleep();
return r;
}
});
function sleep(ms = 10): Promise<void> {
return new Promise((resolve) => {
setTimeout(resolve, ms);
});
}

271
common/spaces/sync.ts Normal file
View File

@ -0,0 +1,271 @@
import type { FileMeta } from "../types.ts";
import { SpacePrimitives } from "./space_primitives.ts";
type SyncHash = number;
// Tuple where the first value represents a lastModified timestamp for the primary space
// and the second item the lastModified value of the secondary space
export type SyncStatusItem = [SyncHash, SyncHash];
// Implementation of this algorithm https://unterwaditzer.net/2016/sync-algorithm.html
export class SpaceSync {
constructor(
private primary: SpacePrimitives,
private secondary: SpacePrimitives,
readonly snapshot: Map<string, SyncStatusItem>,
) {}
async syncFiles(
conflictResolver?: (
name: string,
snapshot: Map<string, SyncStatusItem>,
primarySpace: SpacePrimitives,
secondarySpace: SpacePrimitives,
) => Promise<void>,
): Promise<number> {
let operations = 0;
console.log("Fetching snapshot from primary");
const primaryAllPages = this.syncCandidates(
await this.primary.fetchFileList(),
);
console.log("Fetching snapshot from secondary");
try {
const secondaryAllPages = this.syncCandidates(
await this.secondary.fetchFileList(),
);
const primaryFileMap = new Map<string, SyncHash>(
primaryAllPages.map((m) => [m.name, m.lastModified]),
);
const secondaryFileMap = new Map<string, SyncHash>(
secondaryAllPages.map((m) => [m.name, m.lastModified]),
);
const allFilesToProcess = new Set([
...this.snapshot.keys(),
...primaryFileMap.keys(),
...secondaryFileMap.keys(),
]);
console.log("Iterating over all files");
for (const name of allFilesToProcess) {
if (
primaryFileMap.has(name) && !secondaryFileMap.has(name) &&
!this.snapshot.has(name)
) {
// New file, created on primary, copy from primary to secondary
console.log(
"New file created on primary, copying to secondary",
name,
);
const { data } = await this.primary.readFile(name, "arraybuffer");
const writtenMeta = await this.secondary.writeFile(
name,
"arraybuffer",
data,
);
this.snapshot.set(name, [
primaryFileMap.get(name)!,
writtenMeta.lastModified,
]);
operations++;
} else if (
secondaryFileMap.has(name) && !primaryFileMap.has(name) &&
!this.snapshot.has(name)
) {
// New file, created on secondary, copy from secondary to primary
console.log(
"New file created on secondary, copying from secondary to primary",
name,
);
const { data } = await this.secondary.readFile(name, "arraybuffer");
const writtenMeta = await this.primary.writeFile(
name,
"arraybuffer",
data,
);
this.snapshot.set(name, [
writtenMeta.lastModified,
secondaryFileMap.get(name)!,
]);
operations++;
} else if (
primaryFileMap.has(name) && this.snapshot.has(name) &&
!secondaryFileMap.has(name)
) {
// File deleted on B
console.log("File deleted on secondary, deleting from primary", name);
await this.primary.deleteFile(name);
this.snapshot.delete(name);
operations++;
} else if (
secondaryFileMap.has(name) && this.snapshot.has(name) &&
!primaryFileMap.has(name)
) {
// File deleted on A
console.log("File deleted on primary, deleting from secondary", name);
await this.secondary.deleteFile(name);
this.snapshot.delete(name);
operations++;
} else if (
this.snapshot.has(name) && !primaryFileMap.has(name) &&
!secondaryFileMap.has(name)
) {
// File deleted on both sides, :shrug:
console.log("File deleted on both ends, deleting from status", name);
this.snapshot.delete(name);
operations++;
} else if (
primaryFileMap.has(name) && secondaryFileMap.has(name) &&
this.snapshot.get(name) &&
primaryFileMap.get(name) !== this.snapshot.get(name)![0] &&
secondaryFileMap.get(name) === this.snapshot.get(name)![1]
) {
// File has changed on primary, but not secondary: copy from primary to secondary
console.log("File changed on primary, copying to secondary", name);
const { data } = await this.primary.readFile(name, "arraybuffer");
const writtenMeta = await this.secondary.writeFile(
name,
"arraybuffer",
data,
);
this.snapshot.set(name, [
primaryFileMap.get(name)!,
writtenMeta.lastModified,
]);
operations++;
} else if (
primaryFileMap.has(name) && secondaryFileMap.has(name) &&
this.snapshot.get(name) &&
secondaryFileMap.get(name) !== this.snapshot.get(name)![1] &&
primaryFileMap.get(name) === this.snapshot.get(name)![0]
) {
// File has changed on secondary, but not primary: copy from secondary to primary
const { data } = await this.secondary.readFile(name, "arraybuffer");
const writtenMeta = await this.primary.writeFile(
name,
"arraybuffer",
data,
);
this.snapshot.set(name, [
writtenMeta.lastModified,
secondaryFileMap.get(name)!,
]);
operations++;
} else if (
( // File changed on both ends, but we don't have any info in the snapshot (resync scenario?): have to run through conflict handling
primaryFileMap.has(name) && secondaryFileMap.has(name) &&
!this.snapshot.has(name)
) ||
( // File changed on both ends, CONFLICT!
primaryFileMap.has(name) && secondaryFileMap.has(name) &&
this.snapshot.get(name) &&
secondaryFileMap.get(name) !== this.snapshot.get(name)![1] &&
primaryFileMap.get(name) !== this.snapshot.get(name)![0]
)
) {
console.log("File changed on both ends, conflict!", name);
if (conflictResolver) {
await conflictResolver(
name,
this.snapshot,
this.primary,
this.secondary,
);
} else {
throw Error(
`Sync conflict for ${name} with no conflict resolver specified`,
);
}
operations++;
} else {
// Nothing needs to happen
}
}
} catch (e: any) {
console.error("Boom", e.message);
throw e;
}
return operations;
}
// Strategy: Primary wins
public static async primaryConflictResolver(
name: string,
snapshot: Map<string, SyncStatusItem>,
primary: SpacePrimitives,
secondary: SpacePrimitives,
): Promise<void> {
console.log("Hit a conflict for", name);
const filePieces = name.split(".");
const fileNameBase = filePieces.slice(0, -1).join(".");
const fileNameExt = filePieces[filePieces.length - 1];
const pageData1 = await primary.readFile(name, "arraybuffer");
const pageData2 = await secondary.readFile(name, "arraybuffer");
let byteWiseMatch = true;
const arrayBuffer1 = new Uint8Array(pageData1.data as ArrayBuffer);
const arrayBuffer2 = new Uint8Array(pageData2.data as ArrayBuffer);
if (arrayBuffer1.byteLength !== arrayBuffer2.byteLength) {
byteWiseMatch = false;
}
if (byteWiseMatch) {
// Byte-wise comparison
for (let i = 0; i < arrayBuffer1.byteLength; i++) {
if (arrayBuffer1[i] !== arrayBuffer2[i]) {
byteWiseMatch = false;
break;
}
}
// Byte wise they're still the same, so no confict
if (byteWiseMatch) {
snapshot.set(name, [
pageData1.meta.lastModified,
pageData2.meta.lastModified,
]);
return;
}
}
const revisionFileName = filePieces.length === 1
? `${name}.conflicted.${pageData2.meta.lastModified}`
: `${fileNameBase}.conflicted.${pageData2.meta.lastModified}.${fileNameExt}`;
console.log(
"Going to create conflicting copy",
revisionFileName,
);
// Copy secondary to conflict copy
const localConflictMeta = await primary.writeFile(
revisionFileName,
"arraybuffer",
pageData2.data,
);
const remoteConflictMeta = await secondary.writeFile(
revisionFileName,
"arraybuffer",
pageData2.data,
);
// Updating snapshot
snapshot.set(revisionFileName, [
localConflictMeta.lastModified,
remoteConflictMeta.lastModified,
]);
// Write replacement on top
const writeMeta = await secondary.writeFile(
name,
"arraybuffer",
pageData1.data,
true,
);
snapshot.set(name, [pageData1.meta.lastModified, writeMeta.lastModified]);
}
syncCandidates(files: FileMeta[]): FileMeta[] {
return files.filter((f) => !f.name.startsWith("_plug/"));
}
}

6
server/syscalls/space.ts → common/syscalls/space.ts
View File

@ -6,6 +6,8 @@ import {
FileEncoding,
} from "../../common/spaces/space_primitives.ts";
import { FileMeta as PlugFileMeta } from "../../plug-api/plugos-syscall/types.ts";
export default (space: Space): SysCallMapping => {
return {
"space.listPages": (): PageMeta[] => {
@ -59,5 +61,9 @@ export default (space: Space): SysCallMapping => {
"space.deleteAttachment": async (_ctx, name: string) => {
await space.deleteAttachment(name);
},
"space.listFiles": (_ctx, path: string): Promise<PlugFileMeta[]> => {
return space.listFiles(path);
},
};
};

65
common/syscalls/sync.ts Normal file
View File

@ -0,0 +1,65 @@
import { SysCallMapping } from "../../plugos/system.ts";
import type { SyncEndpoint } from "../../plug-api/silverbullet-syscall/sync.ts";
import { SpaceSync, SyncStatusItem } from "../spaces/sync.ts";
import { HttpSpacePrimitives } from "../spaces/http_space_primitives.ts";
import { SpacePrimitives } from "../spaces/space_primitives.ts";
export function syncSyscalls(localSpace: SpacePrimitives): SysCallMapping {
return {
"sync.sync": async (
_ctx,
endpoint: SyncEndpoint,
snapshot: Record<string, SyncStatusItem>,
): Promise<
{
snapshot: Record<string, SyncStatusItem>;
operations: number;
// The reason to not just throw an Error is so that the partially updated snapshot can still be saved
error?: string;
}
> => {
const syncSpace = new HttpSpacePrimitives(
endpoint.url,
endpoint.user,
endpoint.password,
// Base64 PUTs to support mobile
true,
);
// Convert from JSON to a Map
const syncStatusMap = new Map<string, SyncStatusItem>(
Object.entries(snapshot),
);
const spaceSync = new SpaceSync(
localSpace,
syncSpace,
syncStatusMap,
);
try {
const operations = await spaceSync.syncFiles(
SpaceSync.primaryConflictResolver,
);
return {
// And convert back to JSON
snapshot: Object.fromEntries(spaceSync.snapshot),
operations,
};
} catch (e: any) {
return {
snapshot: Object.fromEntries(spaceSync.snapshot),
operations: -1,
error: e.message,
};
}
},
"sync.check": async (_ctx, endpoint: SyncEndpoint): Promise<void> => {
const syncSpace = new HttpSpacePrimitives(
endpoint.url,
endpoint.user,
endpoint.password,
);
// Let's just fetch the file list to see if it works
await syncSpace.fetchFileList();
},
};
}

11
common/util.ts
View File

@ -1,6 +1,7 @@
import { SETTINGS_TEMPLATE } from "./settings_template.ts";
import { YAML } from "./deps.ts";
import { Space } from "./spaces/space.ts";
import { BuiltinSettings } from "../web/types.ts";
export function safeRun(fn: () => Promise<void>) {
fn().catch((e) => {
@ -45,7 +46,15 @@ export function parseYamlSettings(settingsMarkdown: string): {
}
}
export async function ensureAndLoadSettings(space: Space) {
export async function ensureAndLoadSettings(
space: Space,
dontCreate: boolean,
): Promise<any> {
if (dontCreate) {
return {
indexPage: "index",
};
}
try {
await space.getPageMeta("SETTINGS");
} catch {

6
deno.jsonc
View File

@ -3,9 +3,9 @@
"clean": "rm -rf dist dist_bundle",
"install": "deno install -f -A --unstable silverbullet.ts",
"test": "deno test -A --unstable",
"build": "deno run -A --unstable --check build_plugs.ts && deno run -A --unstable --check build.ts",
"build": "deno run -A --unstable --check build_plugs.ts && deno run -A --unstable --check build_web.ts",
"plugs": "deno run -A --unstable --check build_plugs.ts",
"watch-web": "deno run -A --unstable --check build.ts --watch",
"watch-web": "deno run -A --unstable --check build_web.ts --watch",
"watch-mobile": "deno run -A --unstable --check build_mobile.ts --watch",
"watch-server": "deno run -A --unstable --check --watch silverbullet.ts",
// The only reason to run a shell script is that deno task doesn't support globs yet (e.g. *.plug.yaml)
@ -22,7 +22,7 @@
"desktop:build": "deno task build && deno task bundle && cd desktop && npm run make",
// Mobile
"mobile:deps": "cd mobile && npm install && npx cap sync",
"mobile:build": "deno run -A --unstable --check build_mobile.ts && cd mobile && npx cap copy && npx cap open ios"
"mobile:build": "deno task clean && deno task plugs && deno run -A --unstable --check build_mobile.ts && cd mobile && npx cap copy && npx cap open ios"
},
"compilerOptions": {

13
desktop/src/menu.ts
View File

@ -155,7 +155,18 @@ const template: MenuItemConstructorOptions[] = [
if (process.platform === "darwin") {
const name = app.getName();
template.unshift({ label: name, submenu: [] });
template.unshift({
label: name,
submenu: [
{ role: "services" },
{ type: "separator" },
{ role: "hide" },
{ role: "hideOthers" },
{ role: "unhide" },
{ type: "separator" },
{ role: "quit" },
],
});
}
export const menu = Menu.buildFromTemplate(template);

9
mobile/boot.ts
View File

@ -1,8 +1,8 @@
import { Editor } from "../web/editor.tsx";
import { ensureAndLoadSettings, safeRun } from "../common/util.ts";
import { Space } from "../common/spaces/space.ts";
import { PlugSpacePrimitives } from "../server/hooks/plug_space_primitives.ts";
import { PageNamespaceHook } from "../server/hooks/page_namespace.ts";
import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts";
import { PageNamespaceHook } from "../common/hooks/page_namespace.ts";
import { SilverBulletHooks } from "../common/manifest.ts";
import { System } from "../plugos/system.ts";
import { BuiltinSettings } from "../web/types.ts";
@ -75,7 +75,10 @@ safeRun(async () => {
const serverSpace = new Space(spacePrimitives);
serverSpace.watch();
const settings = await ensureAndLoadSettings(serverSpace) as BuiltinSettings;
const settings = await ensureAndLoadSettings(
serverSpace,
false,
) as BuiltinSettings;
// Register some mobile-specific syscall implementations
system.registerSyscalls(

13
mobile/spaces/capacitor_space_primitives.ts
View File

@ -13,7 +13,10 @@ import { Directory, Encoding, Filesystem } from "../deps.ts";
import { mime } from "../../plugos/deps.ts";
export class CapacitorSpacePrimitives implements SpacePrimitives {
constructor(readonly source: Directory, readonly root: string) {
constructor(
readonly source: Directory,
readonly root: string,
) {
}
async fetchFileList(): Promise<FileMeta[]> {
@ -28,8 +31,9 @@ export class CapacitorSpacePrimitives implements SpacePrimitives {
});
for (const file of files.files) {
if (file.type === "file") {
const name = `${dir}/${file.name}`.substring(1);
allFiles.push({
name: `${dir}/${file.name}`.substring(1),
name: name,
lastModified: file.mtime,
perm: "rw",
contentType: mime.getType(file.name) || "application/octet-stream",
@ -41,7 +45,6 @@ export class CapacitorSpacePrimitives implements SpacePrimitives {
}
}
await readAllFiles("");
console.log("allFiles", allFiles);
return allFiles;
}
async readFile(
@ -51,7 +54,7 @@ export class CapacitorSpacePrimitives implements SpacePrimitives {
let data: FileData | undefined;
try {
switch (encoding) {
case "string":
case "utf8":
data = (await Filesystem.readFile({
path: this.root + name,
directory: this.source,
@ -109,7 +112,7 @@ export class CapacitorSpacePrimitives implements SpacePrimitives {
data: FileData,
): Promise<FileMeta> {
switch (encoding) {
case "string":
case "utf8":
await Filesystem.writeFile({
path: this.root + name,
directory: this.source,

40
plug-api/plugos-syscall/fs.ts
View File

@ -1,36 +1,48 @@
import { syscall } from "./syscall.ts";
import type { FileMeta, ProxyFileSystem } from "./types.ts";
export type FileMeta = {
name: string;
lastModified: number;
};
export class LocalFileSystem implements ProxyFileSystem {
constructor(readonly root: string) {
}
export function readFile(
readFile(
path: string,
encoding: "utf8" | "dataurl" = "utf8",
): Promise<string> {
return syscall("fs.readFile", path, encoding);
return syscall("fs.readFile", `${this.root}/${path}`, encoding);
}
export function getFileMeta(path: string): Promise<FileMeta> {
return syscall("fs.getFileMeta", path);
async getFileMeta(path: string): Promise<FileMeta> {
return this.removeRootDir(
await syscall("fs.getFileMeta", `${this.root}/${path}`),
);
}
export function writeFile(
writeFile(
path: string,
text: string,
encoding: "utf8" | "dataurl" = "utf8",
): Promise<FileMeta> {
return syscall("fs.writeFile", path, text, encoding);
return syscall("fs.writeFile", `${this.root}/${path}`, text, encoding);
}
export function deleteFile(path: string): Promise<void> {
return syscall("fs.deleteFile", path);
deleteFile(path: string): Promise<void> {
return syscall("fs.deleteFile", `${this.root}/${path}`);
}
export function listFiles(
async listFiles(
dirName: string,
recursive = false,
): Promise<FileMeta[]> {
return syscall("fs.listFiles", dirName, recursive);
return (await syscall(
"fs.listFiles",
`${this.root}/${dirName}`,
recursive,
)).map(this.removeRootDir.bind(this));
}
private removeRootDir(fileMeta: FileMeta): FileMeta {
fileMeta.name = fileMeta.name.substring(this.root.length + 1);
return fileMeta;
}
}

3
plug-api/plugos-syscall/mod.ts
View File

@ -1,6 +1,7 @@
export * as asset from "./asset.ts";
export * as events from "./event.ts";
export * as fs from "./fs.ts";
// export * as fs from "./fs.ts";
export { LocalFileSystem } from "./fs.ts";
export * as sandbox from "./sandbox.ts";
export * as fulltext from "./fulltext.ts";
export * as shell from "./shell.ts";

25
plug-api/plugos-syscall/types.ts Normal file
View File

@ -0,0 +1,25 @@
export type FileMeta = {
name: string;
lastModified: number;
};
export interface ProxyFileSystem {
readFile(
path: string,
encoding: "utf8" | "dataurl",
): Promise<string>;
getFileMeta(path: string): Promise<FileMeta>;
writeFile(
path: string,
text: string,
encoding: "utf8" | "dataurl",
): Promise<FileMeta>;
deleteFile(path: string): Promise<void>;
listFiles(
path: string,
): Promise<FileMeta[]>;
}

3
plug-api/silverbullet-syscall/mod.ts
View File

@ -3,6 +3,7 @@ export * as editor from "./editor.ts";
export * as index from "./index.ts";
export * as markdown from "./markdown.ts";
export * as sandbox from "./sandbox.ts";
export * as space from "./space.ts";
export { default as space } from "./space.ts";
export * as system from "./system.ts";
export * as collab from "./collab.ts";
export * as sync from "./sync.ts";

54
plug-api/silverbullet-syscall/space.ts
View File

@ -1,37 +1,41 @@
import { syscall } from "./syscall.ts";
import { AttachmentMeta, PageMeta } from "../../common/types.ts";
import { FileMeta, ProxyFileSystem } from "../plugos-syscall/types.ts";
export function listPages(unfiltered = false): Promise<PageMeta[]> {
export class SpaceFileSystem implements ProxyFileSystem {
// More space-specific methods
listPages(unfiltered = false): Promise<PageMeta[]> {
return syscall("space.listPages", unfiltered);
}
export function getPageMeta(name: string): Promise<PageMeta> {
getPageMeta(name: string): Promise<PageMeta> {
return syscall("space.getPageMeta", name);
}
export function readPage(
readPage(
name: string,
): Promise<string> {
return syscall("space.readPage", name);
}
export function writePage(name: string, text: string): Promise<PageMeta> {
writePage(name: string, text: string): Promise<PageMeta> {
return syscall("space.writePage", name, text);
}
export function deletePage(name: string): Promise<void> {
deletePage(name: string): Promise<void> {
return syscall("space.deletePage", name);
}
export function listPlugs(): Promise<string[]> {
listPlugs(): Promise<string[]> {
return syscall("space.listPlugs");
}
export function listAttachments(): Promise<PageMeta[]> {
listAttachments(): Promise<PageMeta[]> {
return syscall("space.listAttachments");
}
export function getAttachmentMeta(name: string): Promise<AttachmentMeta> {
getAttachmentMeta(name: string): Promise<AttachmentMeta> {
return syscall("space.getAttachmentMeta", name);
}
@ -40,7 +44,7 @@ export function getAttachmentMeta(name: string): Promise<AttachmentMeta> {
* @param name path of the attachment to read
* @returns the attachment data encoded as a data URL
*/
export function readAttachment(
readAttachment(
name: string,
): Promise<string> {
return syscall("space.readAttachment", name);
@ -49,13 +53,13 @@ export function readAttachment(
/**
* Writes an attachment to the space
* @param name path of the attachment to write
* @param encoding encoding of the data ("string" or "dataurl)
* @param encoding encoding of the data ("utf8" or "dataurl)
* @param data data itself
* @returns
*/
export function writeAttachment(
writeAttachment(
name: string,
encoding: "string" | "dataurl",
encoding: "utf8" | "dataurl",
data: string,
): Promise<AttachmentMeta> {
return syscall("space.writeAttachment", name, encoding, data);
@ -65,6 +69,30 @@ export function writeAttachment(
* Deletes an attachment from the space
* @param name path of the attachment to delete
*/
export function deleteAttachment(name: string): Promise<void> {
deleteAttachment(name: string): Promise<void> {
return syscall("space.deleteAttachment", name);
}
// Filesystem implementation
readFile(path: string, encoding: "dataurl" | "utf8"): Promise<string> {
return syscall("space.readFile", path, encoding);
}
getFileMeta(path: string): Promise<FileMeta> {
return syscall("space.getFileMeta", path);
}
writeFile(
path: string,
text: string,
encoding: "dataurl" | "utf8",
): Promise<FileMeta> {
return syscall("space.writeFile", path, text, encoding);
}
deleteFile(path: string): Promise<void> {
return syscall("space.deleteFile", path);
}
listFiles(path: string): Promise<FileMeta[]> {
return syscall("space.listFiles", path);
}
}
export default new SpaceFileSystem();

28
plug-api/silverbullet-syscall/sync.ts Normal file
View File

@ -0,0 +1,28 @@
import type { SyncStatusItem } from "../../common/spaces/sync.ts";
import { syscall } from "./syscall.ts";
export type SyncEndpoint = {
url: string;
user?: string;
password?: string;
};
// Perform a sync with the server, based on the given status (to be persisted)
// returns a new sync status to persist
export function sync(
endpoint: SyncEndpoint,
snapshot: Record<string, SyncStatusItem>,
): Promise<
{
snapshot: Record<string, SyncStatusItem>;
operations: number;
error?: string;
}
> {
return syscall("sync.sync", endpoint, snapshot);
}
// Checks the sync endpoint for connectivity and authentication, throws and Error on failure
export function check(endpoint: SyncEndpoint): Promise<void> {
return syscall("sync.check", endpoint);
}

462
plugos/sqlite/deno-sqlite/src/db.test.ts
View File

@ -1,462 +0,0 @@
import {
assertAlmostEquals,
assertEquals,
assertThrows,
} from "https://deno.land/std@0.154.0/testing/asserts.ts";
import { DB } from "../mod.ts";
const TEST_DB = "test.db";
const LARGE_TEST_DB = "build/2GB_test.db";
async function dbPermissions(path: string): Promise<boolean> {
const query = async (name: "read" | "write") =>
(await Deno.permissions.query({ name, path })).state ===
"granted";
return await query("read") && await query("write");
}
const TEST_DB_PERMISSIONS = await dbPermissions(TEST_DB);
const LARGE_TEST_DB_PERMISSIONS = await dbPermissions(LARGE_TEST_DB);
async function deleteDatabase(file: string) {
try {
await Deno.remove(file);
} catch { /* no op */ }
try {
await Deno.remove(`${file}-journal`);
} catch { /* no op */ }
}
Deno.test("execute multiple statements", function () {
const db = new DB();
db.execute(`
CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT);
INSERT INTO test (id) VALUES (1);
INSERT INTO test (id) VALUES (2);
INSERT INTO test (id) VALUES (3);
`);
assertEquals(db.query("SELECT id FROM test"), [[1], [2], [3]]);
// table `test` already exists ...
assertThrows(function () {
db.execute(`
CREATE TABLE test2 (id INTEGER);
CREATE TABLE test (id INTEGER);
`);
});
// ... but table `test2` was created before the error
assertEquals(db.query("SELECT id FROM test2"), []);
// syntax error after first valid statement
assertThrows(() => db.execute("SELECT id FROM test; NOT SQL ANYMORE"));
});
Deno.test("foreign key constraints enabled", function () {
const db = new DB();
db.execute(`
CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT);
CREATE TABLE orders (id INTEGER PRIMARY KEY AUTOINCREMENT, user INTEGER, FOREIGN KEY(user) REFERENCES users(id));
`);
db.query("INSERT INTO users (id) VALUES (1)");
const [{ id }] = db.queryEntries<{ id: number }>("SELECT id FROM users");
// user must exist
assertThrows(() =>
db.query("INSERT INTO orders (user) VALUES (?)", [id + 1])
);
db.query("INSERT INTO orders (user) VALUES (?)", [id]);
// can't delete if that violates the constraint ...
assertThrows(() => {
db.query("DELETE FROM users WHERE id = ?", [id]);
});
// ... after deleting the order, deleting is OK
db.query("DELETE FROM orders WHERE user = ?", [id]);
db.query("DELETE FROM users WHERE id = ?", [id]);
});
Deno.test("json functions exist", function () {
const db = new DB();
// The JSON1 functions should exist and we should be able to call them without unexpected errors
db.query(`SELECT json('{"this is": ["json"]}')`);
// We should expect an error if we pass invalid JSON where valid JSON is expected
assertThrows(() => {
db.query(`SELECT json('this is not json')`);
});
// We should be able to use bound values as arguments to the JSON1 functions,
// and they should produce the expected results for these simple expressions.
const [[objectType]] = db.query(`SELECT json_type('{}')`);
assertEquals(objectType, "object");
const [[integerType]] = db.query(`SELECT json_type(?)`, ["2"]);
assertEquals(integerType, "integer");
const [[realType]] = db.query(`SELECT json_type(?)`, ["2.5"]);
assertEquals(realType, "real");
const [[stringType]] = db.query(`SELECT json_type(?)`, [`"hello"`]);
assertEquals(stringType, "text");
const [[integerTypeAtPath]] = db.query(
`SELECT json_type(?, ?)`,
[`["hello", 2, {"world": 4}]`, `$[2].world`],
);
assertEquals(integerTypeAtPath, "integer");
});
Deno.test("date time is correct", function () {
const db = new DB();
// the date/ time is passed from JS and should be current (note that it is GMT)
const [[now]] = [...db.query("SELECT STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')")];
const jsTime = new Date().getTime();
const dbTime = new Date(`${now}Z`).getTime();
// to account for runtime latency, a small difference is ok
const tolerance = 10;
assertAlmostEquals(jsTime, dbTime, tolerance);
db.close();
});
Deno.test("SQL localtime reflects system locale", function () {
const db = new DB();
const [[timeDb]] = db.query("SELECT datetime('now', 'localtime')");
const now = new Date();
const jsMonth = `${now.getMonth() + 1}`.padStart(2, "0");
const jsDate = `${now.getDate()}`.padStart(2, "0");
const jsHour = `${now.getHours()}`.padStart(2, "0");
const jsMinute = `${now.getMinutes()}`.padStart(2, "0");
const jsSecond = `${now.getSeconds()}`.padStart(2, "0");
const timeJs =
`${now.getFullYear()}-${jsMonth}-${jsDate} ${jsHour}:${jsMinute}:${jsSecond}`;
assertEquals(timeDb, timeJs);
});
Deno.test("database has correct changes and totalChanges", function () {
const db = new DB();
db.execute(
"CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)",
);
for (const name of ["a", "b", "c"]) {
db.query("INSERT INTO test (name) VALUES (?)", [name]);
assertEquals(1, db.changes);
}
assertEquals(3, db.totalChanges);
db.query("UPDATE test SET name = ?", ["new name"]);
assertEquals(3, db.changes);
assertEquals(6, db.totalChanges);
});
Deno.test("last inserted id", function () {
const db = new DB();
// By default, lastInsertRowId must be 0
assertEquals(db.lastInsertRowId, 0);
// Create table and insert value
db.query("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT)");
const insertRowIds = [];
// Insert data to table and collect their ids
for (let i = 0; i < 10; i++) {
db.query("INSERT INTO users (name) VALUES ('John Doe')");
insertRowIds.push(db.lastInsertRowId);
}
// Now, the last inserted row id must be 10
assertEquals(db.lastInsertRowId, 10);
// All collected row ids must be the same as in the database
assertEquals(
insertRowIds,
[...db.query("SELECT id FROM users")].map(([i]) => i),
);
db.close();
// When the database is closed, the value
// will be reset to 0 again
assertEquals(db.lastInsertRowId, 0);
});
Deno.test("close database", function () {
const db = new DB();
db.close();
assertThrows(() => db.query("CREATE TABLE test (name TEXT PRIMARY KEY)"));
db.close(); // check close is idempotent and won't throw
});
Deno.test("open queries block close", function () {
const db = new DB();
db.query("CREATE TABLE test (name TEXT PRIMARY KEY)");
const query = db.prepareQuery("SELECT name FROM test");
assertThrows(() => db.close());
query.finalize();
db.close();
});
Deno.test("open queries cleaned up by forced close", function () {
const db = new DB();
db.query("CREATE TABLE test (name TEXT PRIMARY KEY)");
db.query("INSERT INTO test (name) VALUES (?)", ["Deno"]);
db.prepareQuery("SELECT name FROM test WHERE name like '%test%'");
assertThrows(() => db.close());
db.close(true);
});
Deno.test("invalid bind does not leak statements", function () {
const db = new DB();
db.query("CREATE TABLE test (id INTEGER)");
for (let n = 0; n < 100; n++) {
assertThrows(() => {
// deno-lint-ignore no-explicit-any
const badBinding: any = [{}];
db.query("INSERT INTO test (id) VALUES (?)", badBinding);
});
assertThrows(() => {
const badBinding = { missingKey: null };
db.query("INSERT INTO test (id) VALUES (?)", badBinding);
});
}
db.query("INSERT INTO test (id) VALUES (1)");
db.close();
});
Deno.test("transactions can be nested", function () {
const db = new DB();
db.query("CREATE TABLE test (id INTEGER PRIMARY KEY)");
db.transaction(() => {
db.query("INSERT INTO test (id) VALUES (1)");
try {
db.transaction(() => {
db.query("INSERT INTO test (id) VALUES (2)");
throw new Error("boom!");
});
} catch (_) { /* ignore */ }
});
assertEquals([{ id: 1 }], db.queryEntries("SELECT * FROM test"));
});
Deno.test("transactions commit when closure exists", function () {
const db = new DB();
db.query("CREATE TABLE test (id INTEGER PRIMARY KEY)");
db.transaction(() => {
db.query("INSERT INTO test (id) VALUES (1)");
});
assertThrows(() => db.query("ROLLBACK"));
assertEquals([{ id: 1 }], db.queryEntries("SELECT * FROM test"));
});
Deno.test("transaction rolls back on throw", function () {
const db = new DB();
db.query("CREATE TABLE test (id INTEGER PRIMARY KEY)");
assertThrows(() => {
db.transaction(() => {
db.query("INSERT INTO test (id) VALUES (1)");
throw new Error("boom!");
});
});
assertEquals([], db.query("SELECT * FROM test"));
});
Deno.test(
"persist database to file",
{
ignore: !TEST_DB_PERMISSIONS,
permissions: { read: true, write: true },
sanitizeResources: true,
},
async function () {
const data = [
"Hello World!",
"Hello Deno!",
"JavaScript <3",
"This costs 0€ / $0 / £0",
"Wéll, hällö thėrè¿",
];
// ensure the test database file does not exist
await deleteDatabase(TEST_DB);
const db = new DB(TEST_DB);
db.execute(
"CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, val TEXT)",
);
for (const val of data) {
db.query("INSERT INTO test (val) VALUES (?)", [val]);
}
// open the same database with a separate connection
const readOnlyDb = await new DB(TEST_DB, { mode: "read" });
for (
const [id, val] of readOnlyDb.query<[number, string]>(
"SELECT * FROM test",
)
) {
assertEquals(data[id - 1], val);
}
await Deno.remove(TEST_DB);
db.close();
readOnlyDb.close();
},
);
Deno.test(
"temporary file database read / write",
{
ignore: !TEST_DB_PERMISSIONS,
permissions: { read: true, write: true },
sanitizeResources: true,
},
function () {
const data = [
"Hello World!",
"Hello Deno!",
"JavaScript <3",
"This costs 0€ / $0 / £0",
"Wéll, hällö thėrè¿",
];
const tempDb = new DB("");
tempDb.execute(
"CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, val TEXT)",
);
for (const val of data) {
tempDb.query("INSERT INTO test (val) VALUES (?)", [val]);
}
for (
const [id, val] of tempDb.query<[number, string]>("SELECT * FROM test")
) {
assertEquals(data[id - 1], val);
}
tempDb.close();
},
);
Deno.test(
"database open options",
{
ignore: !TEST_DB_PERMISSIONS,
permissions: { read: true, write: true },
sanitizeResources: true,
},
async function () {
await deleteDatabase(TEST_DB);
// when no file exists, these should error
assertThrows(() => new DB(TEST_DB, { mode: "write" }));
assertThrows(() => new DB(TEST_DB, { mode: "read" }));
// create the database
const dbCreate = new DB(TEST_DB, { mode: "create" });
dbCreate.execute(
"CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL)",
);
dbCreate.close();
// the default mode is create
await deleteDatabase(TEST_DB);
const dbCreateDefault = new DB(TEST_DB, { mode: "create" });
dbCreateDefault.execute(
"CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL)",
);
dbCreateDefault.close();
// in write mode, we can run INSERT queries ...
const dbWrite = new DB(TEST_DB, { mode: "write" });
dbWrite.query("INSERT INTO test (name) VALUES (?)", ["open-options-test"]);
dbWrite.close();
// ... which we can read in read-only mode ...
const dbRead = new DB(TEST_DB, { mode: "read" });
const rows = [...dbRead.query("SELECT id, name FROM test")];
assertEquals(rows, [[1, "open-options-test"]]);
// ... but we can't write with a read-only connection
assertThrows(() =>
dbRead.query("INTERT INTO test (name) VALUES (?)", ["this-fails"])
);
dbRead.close();
},
);
Deno.test(
"create / write mode require write permissions",
{
ignore: !TEST_DB_PERMISSIONS,
permissions: { read: true, write: false },
sanitizeResources: true,
},
function () {
// opening with these modes requires write permissions ...
assertThrows(() => new DB(TEST_DB, { mode: "create" }));
assertThrows(() => new DB(TEST_DB, { mode: "write" }));
// ... and the default mode is create
assertThrows(() => new DB(TEST_DB));
// however, opening in read-only mode should work (the file was created
// in the previous test)
(new DB(TEST_DB, { mode: "read" })).close();
// with memory flag set, the database will be in memory and
// not require any permissions
(new DB(TEST_DB, { mode: "create", memory: true })).close();
// the mode can also be specified via a URI flag
(new DB(`file:${TEST_DB}?mode=memory`, { uri: true })).close();
},
);
Deno.test(
"database larger than 2GB read / write",
{
ignore: !LARGE_TEST_DB_PERMISSIONS,
permissions: { read: true, write: true },
sanitizeResources: true,
},
function () {
// generated with `cd build && make testdb`
const db = new DB(LARGE_TEST_DB, { mode: "write" });
db.query("INSERT INTO test (value) VALUES (?)", ["This is a test..."]);
const rows = [
...db.query("SELECT value FROM test ORDER BY id DESC LIMIT 10"),
];
assertEquals(rows.length, 10);
assertEquals(rows[0][0], "This is a test...");
db.close();
},
);

55
plugos/sqlite/deno-sqlite/src/error.test.ts
View File

@ -1,55 +0,0 @@
import {
assertEquals,
assertInstanceOf,
assertThrows,
} from "https://deno.land/std@0.154.0/testing/asserts.ts";
import { DB, SqliteError, Status } from "../mod.ts";
Deno.test("invalid SQL", function () {
const db = new DB();
const queries = [
"INSERT INTO does_not_exist (balance) VALUES (5)",
"this is not sql",
";;;",
];
for (const query of queries) assertThrows(() => db.query(query));
db.close();
});
Deno.test("constraint error code is correct", function () {
const db = new DB();
db.query("CREATE TABLE test (name TEXT PRIMARY KEY)");
db.query("INSERT INTO test (name) VALUES (?)", ["A"]);
assertThrows(
() => db.query("INSERT INTO test (name) VALUES (?)", ["A"]),
(e: Error) => {
assertInstanceOf(e, SqliteError);
assertEquals(e.code, Status.SqliteConstraint, "Got wrong error code");
assertEquals(
Status[e.codeName],
Status.SqliteConstraint,
"Got wrong error code name",
);
},
);
});
Deno.test("syntax error code is correct", function () {
const db = new DB();
assertThrows(
() => db.query("CREATE TABLEX test (name TEXT PRIMARY KEY)"),
(e: Error) => {
assertInstanceOf(e, SqliteError);
assertEquals(e.code, Status.SqliteError, "Got wrong error code");
assertEquals(
Status[e.codeName],
Status.SqliteError,
"Got wrong error code name",
);
},
);
});

521
plugos/sqlite/deno-sqlite/src/query.test.ts
View File

@ -1,521 +0,0 @@
import {
assertEquals,
assertThrows,
} from "https://deno.land/std@0.154.0/testing/asserts.ts";
import { DB, QueryParameter } from "../mod.ts";
function roundTripValues<T extends QueryParameter>(values: T[]): unknown[] {
const db = new DB();
db.execute(
"CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, datum ANY)",
);
for (const value of values) {
db.query("INSERT INTO test (datum) VALUES (?)", [value]);
}
return db
.queryEntries<{ datum: unknown }>("SELECT datum FROM test")
.map(({ datum }) => datum);
}
Deno.test("bind string values", function () {
const values = ["Hello World!", "I love Deno.", "Täst strüng..."];
assertEquals(values, roundTripValues(values));
});
Deno.test("bind integer values", function () {
const values = [42, 1, 2, 3, 4, 3453246, 4536787093, 45536787093];
assertEquals(values, roundTripValues(values));
});
Deno.test("bind float values", function () {
const values = [42.1, 1.235, 2.999, 1 / 3, 4.2345, 345.3246, 4536787.953e-8];
assertEquals(values, roundTripValues(values));
});
Deno.test("bind boolean values", function () {
assertEquals([1, 0], roundTripValues([true, false]));
});
Deno.test("bind date values", function () {
const values = [new Date(), new Date("2018-11-20"), new Date(123456789)];
assertEquals(
values.map((date) => date.toISOString()),
roundTripValues(values),
);
});
Deno.test("bind blob values", function () {
const values = [
new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0]),
new Uint8Array([3, 57, 45]),
];
assertEquals(values, roundTripValues(values));
});
Deno.test("blobs are copies", function () {
const db = new DB();
db.query(
"CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, val BLOB)",
);
const data = new Uint8Array([1, 2, 3, 4, 5]);
db.query("INSERT INTO test (val) VALUES (?)", [data]);
const [[a]] = db.query<[Uint8Array]>("SELECT val FROM test");
const [[b]] = db.query<[Uint8Array]>("SELECT val FROM test");
assertEquals(data, a);
assertEquals(data, b);
assertEquals(a, b);
a[0] = 100;
assertEquals(a[0], 100);
assertEquals(b[0], 1);
assertEquals(data[0], 1);
data[0] = 5;
const [[c]] = db.query<[Uint8Array]>("SELECT val FROM test");
assertEquals(c[0], 1);
});
Deno.test("bind bigint values", function () {
assertEquals(
[9007199254741991n, 100],
roundTripValues([9007199254741991n, 100n]),
);
});
Deno.test("bind null / undefined", function () {
assertEquals([null, null], roundTripValues([null, undefined]));
});
Deno.test("bind mixed values", function () {
const values = [42, "Hello World!", 0.33333, null];
assertEquals(values, roundTripValues(values));
});
Deno.test("omitting a value binds NULL", function () {
const db = new DB();
db.execute("CREATE TABLE test (id INTEGER PRIMARY KEY, datum ANY)");
const insert = db.prepareQuery(
"INSERT INTO test (datum) VALUES (?) RETURNING datum",
);
assertEquals([null], insert.first());
assertEquals([null], insert.first([]));
assertEquals([null], insert.first({}));
// previously bound values are cleared
insert.execute(["this is not null"]);
assertEquals([null], insert.first());
});
Deno.test("prepared query clears bindings before reused", function () {
const db = new DB();
db.query("CREATE TABLE test (id INTEGER PRIMARY KEY, value INTEGER)");
const query = db.prepareQuery("INSERT INTO test (value) VALUES (?)");
query.execute([1]);
query.execute();
assertEquals([[1], [null]], db.query("SELECT value FROM test"));
query.finalize();
db.close();
});
Deno.test("bind very large floating point numbers", function () {
const db = new DB();
db.query("CREATE TABLE numbers (id INTEGER PRIMARY KEY, number REAL)");
db.query("INSERT INTO numbers (number) VALUES (?)", [+Infinity]);
db.query("INSERT INTO numbers (number) VALUES (?)", [-Infinity]);
db.query("INSERT INTO numbers (number) VALUES (?)", [+20e20]);
db.query("INSERT INTO numbers (number) VALUES (?)", [-20e20]);
const [
[positiveInfinity],
[negativeInfinity],
[positiveTwentyTwenty],
[negativeTwentyTwenty],
] = db.query("SELECT number FROM numbers");
assertEquals(negativeInfinity, -Infinity);
assertEquals(positiveInfinity, +Infinity);
assertEquals(positiveTwentyTwenty, +20e20);
assertEquals(negativeTwentyTwenty, -20e20);
});
Deno.test("big very large integers", function () {
const db = new DB();
db.query(
"CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, val INTEGER)",
);
const goodValues = [
0n,
42n,
-42n,
9223372036854775807n,
-9223372036854775808n,
];
const overflowValues = [
9223372036854775807n + 1n,
-9223372036854775808n - 1n,
2352359223372036854775807n,
-32453249223372036854775807n,
];
const query = db.prepareQuery("INSERT INTO test (val) VALUES (?)");
for (const val of goodValues) {
query.execute([val]);
}
const dbValues = db.query<[number | bigint]>(
"SELECT val FROM test ORDER BY id",
).map((
[id],
) => BigInt(id));
assertEquals(goodValues, dbValues);
for (const bigVal of overflowValues) {
assertThrows(() => {
query.execute([bigVal]);
});
}
query.finalize();
db.close();
});
Deno.test("bind named parameters", function () {
const db = new DB();
db.query(
"CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, val TEXT)",
);
// :name
db.query("INSERT INTO test (val) VALUES (:val)", { val: "value" });
db.query(
"INSERT INTO test (val) VALUES (:otherVal)",
{ otherVal: "value other" },
);
db.query(
"INSERT INTO test (val) VALUES (:explicitColon)",
{ ":explicitColon": "value explicit" },
);
// @name
db.query(
"INSERT INTO test (val) VALUES (@someName)",
{ "@someName": "@value" },
);
// $name
db.query(
"INSERT INTO test (val) VALUES ($var::Name)",
{ "$var::Name": "$value" },
);
// explicit positional syntax
db.query("INSERT INTO test (id, val) VALUES (?2, ?1)", ["this-is-it", 1000]);
// names must exist
assertThrows(() => {
db.query(
"INSERT INTO test (val) VALUES (:val)",
{ Val: "miss-spelled name" },
);
});
// make sure the data came through correctly
const vals = [...db.query("SELECT val FROM test ORDER BY id ASC")]
.map(([datum]) => datum);
assertEquals(
vals,
[
"value",
"value other",
"value explicit",
"@value",
"$value",
"this-is-it",
],
);
});
Deno.test("iterate from prepared query", function () {
const db = new DB();
db.execute("CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT)");
db.execute("INSERT INTO test (id) VALUES (1), (2), (3)");
const res = [];
const query = db.prepareQuery<[number]>("SELECT id FROM test");
for (const [id] of query.iter()) {
res.push(id);
}
assertEquals(res, [1, 2, 3]);
query.finalize();
db.close();
});
Deno.test("query all from prepared query", function () {
const db = new DB();
db.query("CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT)");
const query = db.prepareQuery("SELECT id FROM test");
assertEquals(query.all(), []);
db.query("INSERT INTO test (id) VALUES (1), (2), (3)");
assertEquals(query.all(), [[1], [2], [3]]);
query.finalize();
db.close();
});
Deno.test("query first from prepared query", function () {
const db = new DB();
db.query("CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT)");
db.query("INSERT INTO test (id) VALUES (1), (2), (3)");
const querySingle = db.prepareQuery("SELECT id FROM test WHERE id = ?");
assertEquals(querySingle.first([42]), undefined);
assertEquals(querySingle.first([2]), [2]);
const queryAll = db.prepareQuery("SELECT id FROM test ORDER BY id ASC");
assertEquals(queryAll.first(), [1]);
querySingle.finalize();
queryAll.finalize();
db.close();
});
Deno.test("query one from prepared query", function () {
const db = new DB();
db.query("CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT)");
db.query("INSERT INTO test (id) VALUES (1), (2), (3)");
const queryOne = db.prepareQuery<[number]>(
"SELECT id FROM test WHERE id = ?",
);
assertThrows(() => queryOne.one([42]));
assertEquals(queryOne.one([2]), [2]);
const queryAll = db.prepareQuery("SELECT id FROM test");
assertThrows(() => queryAll.one());
queryOne.finalize();
queryAll.finalize();
db.close();
});
Deno.test("execute from prepared query", function () {
const db = new DB();
db.query("CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT)");
const insert = db.prepareQuery("INSERT INTO test (id) VALUES (:id)");
for (const id of [1, 2, 3]) {
insert.execute({ id });
}
insert.finalize();
assertEquals(db.query("SELECT id FROM test"), [[1], [2], [3]]);
db.close();
});
Deno.test("empty query returns empty array", function () {
const db = new DB();
db.query("CREATE TABLE test (id INTEGER PRIMARY KEY)");
assertEquals([], db.query("SELECT * FROM test"));
db.close();
});
Deno.test("query entries returns correct object shapes", function () {
const db = new DB();
db.query(
"CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, height REAL)",
);
const rowsOrig = [
{ id: 1, name: "Peter Parker", height: 1.5 },
{ id: 2, name: "Clark Kent", height: 1.9 },
{ id: 3, name: "Robert Paar", height: 2.1 },
];
const insertQuery = db.prepareQuery(
"INSERT INTO test (id, name, height) VALUES (:id, :name, :height)",
);
for (const row of rowsOrig) {
insertQuery.execute(row);
}
insertQuery.finalize();
const query = db.prepareQuery("SELECT * FROM test");
assertEquals(rowsOrig, [...query.iterEntries()]);
assertEquals(rowsOrig, query.allEntries());
assertEquals(rowsOrig[0], query.firstEntry());
assertEquals(rowsOrig, db.queryEntries("SELECT * FROM test"));
query.finalize();
db.close();
});
Deno.test("prepared query can be reused", function () {
const db = new DB();
db.query("CREATE TABLE test (id INTEGER PRIMARY KEY)");
const query = db.prepareQuery("INSERT INTO test (id) VALUES (?)");
query.execute([1]);
query.execute([2]);
query.execute([3]);
assertEquals([[1], [2], [3]], db.query("SELECT id FROM test"));
query.finalize();
db.close();
});
Deno.test("get columns from select query", function () {
const db = new DB();
db.query(
"CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)",
);
const query = db.prepareQuery("SELECT id, name from test");
assertEquals(query.columns(), [
{ name: "id", originName: "id", tableName: "test" },
{ name: "name", originName: "name", tableName: "test" },
]);
});
Deno.test("get columns from returning query", function () {
const db = new DB();
db.query(
"CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)",
);
const query = db.prepareQuery(
"INSERT INTO test (name) VALUES (?) RETURNING *",
);
assertEquals(query.columns(), [
{ name: "id", originName: "id", tableName: "test" },
{ name: "name", originName: "name", tableName: "test" },
]);
assertEquals(query.all(["name"]), [[1, "name"]]);
});
Deno.test("get columns with renamed column", function () {
const db = new DB();
db.query(
"CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)",
);
db.query("INSERT INTO test (name) VALUES (?)", ["name"]);
const query = db.prepareQuery(
"SELECT id AS test_id, name AS test_name from test",
);
const columns = query.columns();
assertEquals(columns, [
{ name: "test_id", originName: "id", tableName: "test" },
{ name: "test_name", originName: "name", tableName: "test" },
]);
});
Deno.test("columns can be obtained from empty prepared query", function () {
const db = new DB();
db.query(
"CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEST, age INTEGER)",
);
db.query("INSERT INTO test (name, age) VALUES (?, ?)", ["Peter Parker", 21]);
const query = db.prepareQuery("SELECT * FROM test");
const columnsFromPreparedQuery = query.columns();
query.finalize();
const queryEmpty = db.prepareQuery("SELECT * FROM test WHERE 1 = 0");
const columnsFromPreparedQueryWithEmptyQuery = queryEmpty.columns();
assertEquals(queryEmpty.all(), []);
query.finalize();
assertEquals(
[{ name: "id", originName: "id", tableName: "test" }, {
name: "name",
originName: "name",
tableName: "test",
}, { name: "age", originName: "age", tableName: "test" }],
columnsFromPreparedQuery,
);
assertEquals(
columnsFromPreparedQueryWithEmptyQuery,
columnsFromPreparedQuery,
);
});
Deno.test("invalid number of bound parameters throws", function () {
const db = new DB();
db.execute("CREATE TABLE test (id INTEGER PRIMARY KEY)");
// too many
assertThrows(() => {
db.query("SELECT * FROM test", [null]);
});
assertThrows(() => {
db.query("SELECT * FROM test LIMIT ?", [5, "extra"]);
});
// too few
assertThrows(() => db.query("SELECT * FROM test LIMIT ?", []));
assertThrows(() => {
db.query(
"SELECT * FROM test WHERE id >= ? AND id <= ? LIMIT ?",
[42],
);
});
});
Deno.test("using finalized prepared query throws", function () {
const db = new DB();
db.query("CREATE TABLE test (name TEXT)");
const query = db.prepareQuery("INSERT INTO test (name) VALUES (?)");
query.finalize();
assertThrows(() => query.execute(["test"]));
db.close();
});
Deno.test("invalid binding throws", function () {
const db = new DB();
db.query("CREATE TABLE test (id INTEGER)");
assertThrows(() => {
// deno-lint-ignore no-explicit-any
const badBinding: any = [{}];
db.query("SELECT * FORM test WHERE id = ?", badBinding);
});
db.close();
});
Deno.test("get columns from finalized query throws", function () {
const db = new DB();
db.query("CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT)");
const query = db.prepareQuery("SELECT id from test");
query.finalize();
// after iteration is done
assertThrows(() => {
query.columns();
});
});

80
plugos/sqlite/deno-sqlite/src/readme.test.ts
View File

@ -1,80 +0,0 @@
import {
assertEquals,
assertMatch,
} from "https://deno.land/std@0.154.0/testing/asserts.ts";
import { DB } from "../mod.ts";
Deno.test("README example", function () {
const db = new DB(/* in memory */);
db.execute(`
CREATE TABLE IF NOT EXISTS people (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT
)
`);
const name =
["Peter Parker", "Clark Kent", "Bruce Wane"][Math.floor(Math.random() * 3)];
// Run a simple query
db.query("INSERT INTO people (name) VALUES (?)", [name]);
// Print out data in table
for (const [_name] of db.query("SELECT name FROM people")) continue; // no console.log ;)
db.close();
});
Deno.test("old README example", function () {
const db = new DB();
const first = ["Bruce", "Clark", "Peter"];
const last = ["Wane", "Kent", "Parker"];
db.query(
"CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT, email TEXT, subscribed INTEGER)",
);
for (let i = 0; i < 100; i++) {
const name = `${first[Math.floor(Math.random() * first.length)]} ${
last[
Math.floor(
Math.random() * last.length,
)
]
}`;
const email = `${name.replace(" ", "-")}@deno.land`;
const subscribed = Math.random() > 0.5 ? true : false;
db.query("INSERT INTO users (name, email, subscribed) VALUES (?, ?, ?)", [
name,
email,
subscribed,
]);
}
for (
const [
name,
email,
] of db.query<[string, string]>(
"SELECT name, email FROM users WHERE subscribed = ? LIMIT 100",
[true],
)
) {
assertMatch(name, /(Bruce|Clark|Peter) (Wane|Kent|Parker)/);
assertEquals(email, `${name.replace(" ", "-")}@deno.land`);
}
const res = db.query("SELECT email FROM users WHERE name LIKE ?", [
"Robert Parr",
]);
assertEquals(res, []);
const subscribers = db.query(
"SELECT name, email FROM users WHERE subscribed = ?",
[true],
);
for (const [_name, _email] of subscribers) {
if (Math.random() > 0.5) continue;
break;
}
});

40
plugos/sqlite/deno-sqlite/src/wasm.test.ts
View File

@ -1,40 +0,0 @@
import {
assertEquals,
assertThrows,
} from "https://deno.land/std@0.154.0/testing/asserts.ts";
import { Wasm } from "../build/sqlite.js";
import * as wasm from "./wasm.ts";
function mock(
malloc: () => number = () => 1,
free: (pts: number) => void = () => {},
): Wasm {
const memory = new Uint8Array(2048);
return {
malloc,
free,
str_len: (ptr: number) => {
let len = 0;
for (let idx = ptr; memory.at(idx) != 0; idx++) len++;
return len;
},
memory,
} as unknown as Wasm;
}
Deno.test("round trip string", function () {
const mockWasm = mock();
const testCases = ["Hello world!", "Söme, fünky lëttêrß", "你好👋"];
for (const input of testCases) {
const output = wasm.setStr(mockWasm, input, (ptr) => {
return wasm.getStr(mockWasm, ptr);
});
assertEquals(input, output);
}
});
Deno.test("throws on allocation error", function () {
const mockWasm = mock(() => 0);
assertThrows(() => wasm.setStr(mockWasm, "Hello world!", (_) => null));
});

4
plugs/collab/collab.ts
View File

@ -136,8 +136,8 @@ export async function readFileCollab(
const text = `---\n$share: ${collabUri}\n---\n`;
return {
// encoding === "arraybuffer" is not an option, so either it's "string" or "dataurl"
data: encoding === "string" ? text : base64EncodedDataUrl(
// encoding === "arraybuffer" is not an option, so either it's "utf8" or "dataurl"
data: encoding === "utf8" ? text : base64EncodedDataUrl(
"text/markdown",
new TextEncoder().encode(text),
),

2
plugs/core/cloud.ts
View File

@ -45,7 +45,7 @@ export async function readFileCloud(
`${pagePrefix}${originalUrl.split("/")[0]}/`,
);
return {
data: encoding === "string" ? text : base64EncodedDataUrl(
data: encoding === "utf8" ? text : base64EncodedDataUrl(
"text/markdown",
new TextEncoder().encode(text),
),

4
plugs/core/plugmanager.ts
View File

@ -1,9 +1,7 @@
import { events } from "$sb/plugos-syscall/mod.ts";
import type { Manifest } from "../../common/manifest.ts";
import { editor, space, system } from "$sb/silverbullet-syscall/mod.ts";
import { readYamlPage } from "$sb/lib/yaml_page.ts";
import { writePage } from "$sb/silverbullet-syscall/space.ts";
const plugsPrelude =
"This file lists all plugs that SilverBullet will load. Run the {[Plugs: Update]} command to update and reload this list of plugs.\n\n";
@ -82,7 +80,7 @@ export async function updatePlugs() {
// console.log("Writing", `_plug/${manifest.name}`);
await space.writeAttachment(
`_plug/${manifest.name}.plug.json`,
"string",
"utf8",
JSON.stringify(manifest),
);
}

4
plugs/core/search.ts
View File

@ -85,8 +85,8 @@ export async function readFileSearch(
`;
return {
// encoding === "arraybuffer" is not an option, so either it's "string" or "dataurl"
data: encoding === "string" ? text : base64EncodedDataUrl(
// encoding === "arraybuffer" is not an option, so either it's "utf8" or "dataurl"
data: encoding === "utf8" ? text : base64EncodedDataUrl(
"text/markdown",
new TextEncoder().encode(text),
),

6
plugs/markdown/share.ts
View File

@ -1,5 +1,5 @@
import { markdown, space } from "$sb/silverbullet-syscall/mod.ts";
import { fs } from "$sb/plugos-syscall/mod.ts";
import { LocalFileSystem } from "$sb/plugos-syscall/mod.ts";
import { asset } from "$sb/plugos-syscall/mod.ts";
import { renderMarkdownToHtml } from "./markdown_render.ts";
import { PublishEvent } from "$sb/app_event.ts";
@ -10,12 +10,14 @@ export async function sharePublisher(event: PublishEvent) {
const text = await space.readPage(pageName);
const tree = await markdown.parseMarkdown(text);
const rootFS = new LocalFileSystem("");
const css = await asset.readAsset("assets/styles.css");
const markdownHtml = renderMarkdownToHtml(tree, {
smartHardBreak: true,
});
const html =
`<html><head><style>${css}</style></head><body><div id="root">${markdownHtml}</div></body></html>`;
await fs.writeFile(path, html, "utf8");
await rootFS.writeFile(path, html, "utf8");
return true;
}

19
plugs/sync/sync.plug.yaml Normal file
View File

@ -0,0 +1,19 @@
name: sync
functions:
configureCommand:
path: sync.ts:configureCommand
command:
name: "Sync: Configure"
syncCommand:
path: sync.ts:syncCommand
command:
name: "Sync: Sync"
check:
env: server
path: sync.ts:check
performSync:
env: server
path: sync.ts:performSync

88
plugs/sync/sync.ts Normal file
View File

@ -0,0 +1,88 @@
import { store } from "$sb/plugos-syscall/mod.ts";
import { editor, sync, system } from "$sb/silverbullet-syscall/mod.ts";
import type { SyncEndpoint } from "$sb/silverbullet-syscall/sync.ts";
export async function configureCommand() {
const url = await editor.prompt(
"Enter the URL of the remote space to sync with",
"https://",
);
if (!url) {
return;
}
const user = await editor.prompt("Username (if any):");
let password = undefined;
if (user) {
password = await editor.prompt("Password:");
}
const syncConfig: SyncEndpoint = {
url,
user,
password,
};
try {
await system.invokeFunction("server", "check", syncConfig);
} catch (e: any) {
await editor.flashNotification(
`Sync configuration failed: ${e.message}`,
"error",
);
return;
}
await store.batchSet([
{ key: "sync.config", value: syncConfig },
// Empty initial snapshot
{ key: "sync.snapshot", value: {} },
]);
await editor.flashNotification("Sync configuration saved.");
return syncConfig;
}
export async function syncCommand() {
let config: SyncEndpoint | undefined = await store.get("sync.config");
if (!config) {
config = await configureCommand();
if (!config) {
return;
}
}
await editor.flashNotification("Starting sync...");
try {
const operations = await system.invokeFunction("server", "performSync");
await editor.flashNotification(
`Sync complete. Performed ${operations} operations.`,
);
} catch (e: any) {
await editor.flashNotification(
`Sync failed: ${e.message}`,
"error",
);
}
}
// Run on server
export function check(config: SyncEndpoint) {
return sync.check(config);
}
// Run on server
export async function performSync() {
const config: SyncEndpoint = await store.get("sync.config");
const snapshot = await store.get("sync.snapshot");
const { snapshot: newSnapshot, operations, error } = await sync.sync(
config,
snapshot,
);
await store.set("sync.snapshot", newSnapshot);
if (error) {
console.error("Sync error", error);
throw new Error(error);
}
return operations;
}

23
server/http_server.ts
View File

@ -5,6 +5,7 @@ import { EndpointHook } from "../plugos/hooks/endpoint.ts";
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
import { SpaceSystem } from "./space_system.ts";
import { ensureAndLoadSettings } from "../common/util.ts";
import { base64Decode } from "../plugos/asset_bundle/base64.ts";
export type ServerOptions = {
hostname: string;
@ -14,6 +15,7 @@ export type ServerOptions = {
assetBundle: AssetBundle;
user?: string;
pass?: string;
bareMode?: boolean;
};
const staticLastModified = new Date().toUTCString();
@ -26,6 +28,7 @@ export class HttpServer {
user?: string;
settings: { [key: string]: any } = {};
abortController?: AbortController;
bareMode: boolean;
constructor(options: ServerOptions) {
this.hostname = options.hostname;
@ -37,6 +40,7 @@ export class HttpServer {
options.pagesPath,
options.dbPath,
);
this.bareMode = options.bareMode || false;
// Second, for loading plug JSON files with absolute or relative (from CWD) paths
this.systemBoot.eventHook.addLocalListener(
@ -66,7 +70,7 @@ export class HttpServer {
async start() {
await this.systemBoot.start();
await this.systemBoot.ensureSpaceIndex();
await ensureAndLoadSettings(this.systemBoot.space);
await ensureAndLoadSettings(this.systemBoot.space, this.bareMode);
this.addPasswordAuth(this.app);
@ -207,7 +211,8 @@ export class HttpServer {
// File list
fsRouter.get("/", async ({ response }) => {
response.headers.set("Content-type", "application/json");
response.body = JSON.stringify(await spacePrimitives.fetchFileList());
const files = await spacePrimitives.fetchFileList();
response.body = JSON.stringify(files);
});
fsRouter
@ -248,12 +253,21 @@ export class HttpServer {
const name = params[0];
console.log("Saving file", name);
let body: Uint8Array;
if (
request.headers.get("X-Content-Base64")
) {
const content = await request.body({ type: "text" }).value;
body = base64Decode(content);
} else {
body = await request.body({ type: "bytes" }).value;
}
try {
const meta = await spacePrimitives.writeFile(
name,
"arraybuffer",
await request.body().value,
false,
body,
);
response.status = 200;
response.headers.set("Content-Type", meta.contentType);
@ -299,7 +313,6 @@ export class HttpServer {
private buildPlugRouter(): Router {
const plugRouter = new Router();
// this.addPasswordAuth(plugRouter);
const system = this.systemBoot.system;
plugRouter.post(

10
server/space_system.ts
View File

@ -23,13 +23,13 @@ import {
storeSyscalls,
} from "../plugos/syscalls/store.sqlite.ts";
import { System } from "../plugos/system.ts";
import { PageNamespaceHook } from "./hooks/page_namespace.ts";
import { PlugSpacePrimitives } from "./hooks/plug_space_primitives.ts";
import { PageNamespaceHook } from "../common/hooks/page_namespace.ts";
import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts";
import {
ensureTable as ensureIndexTable,
pageIndexSyscalls,
} from "./syscalls/index.ts";
import spaceSyscalls from "./syscalls/space.ts";
import spaceSyscalls from "../common/syscalls/space.ts";
import { systemSyscalls } from "./syscalls/system.ts";
import { AssetBundlePlugSpacePrimitives } from "../common/spaces/asset_bundle_space_primitives.ts";
import assetSyscalls from "../plugos/syscalls/asset.ts";
@ -37,6 +37,7 @@ import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
import { AsyncSQLite } from "../plugos/sqlite/async_sqlite.ts";
import { FileMetaSpacePrimitives } from "../common/spaces/file_meta_space_primitives.ts";
import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts";
import { syncSyscalls } from "../common/syscalls/sync.ts";
export const indexRequiredKey = "$spaceIndexed";
// A composition of a PlugOS system attached to a Space for server-side use
@ -111,6 +112,7 @@ export class SpaceSystem {
storeSyscalls(this.db, "store"),
fullTextSearchSyscalls(this.db, "fts"),
spaceSyscalls(this.space),
syncSyscalls(this.spacePrimitives),
eventSyscalls(this.eventHook),
markdownSyscalls(buildMarkdown([])),
esbuildSyscalls([globalModules]),
@ -145,7 +147,7 @@ export class SpaceSystem {
console.log("Going to load", allPlugs.length, "plugs...");
await Promise.all(allPlugs.map(async (plugName) => {
const { data } = await this.space.readAttachment(plugName, "string");
const { data } = await this.space.readAttachment(plugName, "utf8");
await this.system.load(JSON.parse(data as string), createSandbox);
}));

3
silverbullet.ts
View File

@ -21,6 +21,9 @@ await new Command()
.arguments("<folder:string>")
.option("--hostname <hostname:string>", "Hostname or address to listen on")
.option("-p, --port <port:number>", "Port to listen on")
.option("--bare [type:boolean]", "Don't auto generate pages", {
default: false,
})
.option("--db <dbfile:string>", "Filename for the database", {
default: "data.db",
})

6
web/boot.ts
View File

@ -2,8 +2,8 @@ import { Editor } from "./editor.tsx";
import { parseYamlSettings, safeRun } from "../common/util.ts";
import { Space } from "../common/spaces/space.ts";
import { HttpSpacePrimitives } from "../common/spaces/http_space_primitives.ts";
import { PlugSpacePrimitives } from "../server/hooks/plug_space_primitives.ts";
import { PageNamespaceHook } from "../server/hooks/page_namespace.ts";
import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts";
import { PageNamespaceHook } from "../common/hooks/page_namespace.ts";
import { SilverBulletHooks } from "../common/manifest.ts";
import { System } from "../plugos/system.ts";
import { BuiltinSettings } from "./types.ts";
@ -19,7 +19,7 @@ safeRun(async () => {
let settingsPageText = "";
try {
settingsPageText = (
await httpPrimitives.readFile("SETTINGS.md", "string")
await httpPrimitives.readFile("SETTINGS.md", "utf8")
).data as string;
} catch (e: any) {
console.error("No settings page found", e.message);

4
web/editor.tsx
View File

@ -97,6 +97,7 @@ import type {
} from "../plug-api/app_event.ts";
import { CodeWidgetHook } from "./hooks/code_widget.ts";
import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts";
import { syncSyscalls } from "../common/syscalls/sync.ts";
const frontMatterRegex = /^---\n(.*?)---\n/ms;
@ -195,6 +196,7 @@ export class Editor {
markdownSyscalls(buildMarkdown(this.mdExtensions)),
sandboxSyscalls(this.system),
assetSyscalls(this.system),
syncSyscalls(this.space.spacePrimitives),
collabSyscalls(this),
);
@ -659,7 +661,7 @@ export class Editor {
await this.system.unloadAll();
console.log("(Re)loading plugs");
await Promise.all((await this.space.listPlugs()).map(async (plugName) => {
const { data } = await this.space.readAttachment(plugName, "string");
const { data } = await this.space.readAttachment(plugName, "utf8");
await this.system.load(JSON.parse(data as string), createSandbox);
}));
this.rebuildEditorState();

61
web/syscalls/space.ts
View File

@ -1,33 +1,11 @@
import { Editor } from "../editor.tsx";
import { SysCallMapping } from "../../plugos/system.ts";
import { AttachmentMeta, PageMeta } from "../../common/types.ts";
import {
FileData,
FileEncoding,
} from "../../common/spaces/space_primitives.ts";
import commonSpaceSyscalls from "../../common/syscalls/space.ts";
export function spaceSyscalls(editor: Editor): SysCallMapping {
return {
"space.listPages": (): PageMeta[] => {
return [...editor.space.listPages()];
},
"space.readPage": async (
_ctx,
name: string,
): Promise<string> => {
return (await editor.space.readPage(name)).text;
},
"space.getPageMeta": async (_ctx, name: string): Promise<PageMeta> => {
return await editor.space.getPageMeta(name);
},
"space.writePage": async (
_ctx,
name: string,
text: string,
): Promise<PageMeta> => {
return await editor.space.writePage(name, text);
},
"space.deletePage": async (_ctx, name: string) => {
const syscalls = commonSpaceSyscalls(editor.space);
syscalls["space.deletePage"] = async (_ctx, name: string) => {
// If we're deleting the current page, navigate to the index page
if (editor.currentPage === name) {
await editor.navigate("");
@ -36,35 +14,6 @@ export function spaceSyscalls(editor: Editor): SysCallMapping {
editor.openPages.delete(name);
console.log("Deleting page");
await editor.space.deletePage(name);
},
"space.listPlugs": (): Promise<string[]> => {
return editor.space.listPlugs();
},
"space.listAttachments": (): Promise<AttachmentMeta[]> => {
return editor.space.fetchAttachmentList();
},
"space.readAttachment": async (
_ctx,
name: string,
): Promise<FileData> => {
return (await editor.space.readAttachment(name, "dataurl")).data;
},
"space.getAttachmentMeta": async (
_ctx,
name: string,
): Promise<AttachmentMeta> => {
return await editor.space.getAttachmentMeta(name);
},
"space.writeAttachment": async (
_ctx,
name: string,
encoding: FileEncoding,
data: FileData,
): Promise<AttachmentMeta> => {
return await editor.space.writeAttachment(name, encoding, data);
},
"space.deleteAttachment": async (_ctx, name: string) => {
await editor.space.deleteAttachment(name);
},
};
return syscalls;
}

1
web/types.ts
View File

@ -14,6 +14,7 @@ export type PanelMode = number;
export type BuiltinSettings = {
indexPage: string;
syncUrl?: string;
};
export type PanelConfig = {

1
website/.gitignore vendored
View File

@ -1,3 +1,2 @@
data.db
_plug
_trash

1
website/CHANGELOG.md
View File

@ -14,6 +14,7 @@ release.
select * from my_table;
```
* Merged code for experimental mobile app (iOS only for now)
* Experimental sync engine, to be documented once it matures
---