Step 1 in major backend refactor
This commit is contained in:
Zef Hemel
@@ -1,2 +1 @@
|
||||
export const trashPrefix = "_trash/";
|
||||
export const plugPrefix = "_plug/";
|
||||
|
||||
@@ -1,30 +1,20 @@
|
||||
import {
|
||||
mkdir,
|
||||
readdir,
|
||||
readFile,
|
||||
stat,
|
||||
unlink,
|
||||
utimes,
|
||||
writeFile,
|
||||
} from "fs/promises";
|
||||
import { mkdir, readdir, readFile, stat, unlink, writeFile } from "fs/promises";
|
||||
import * as path from "path";
|
||||
import { AttachmentMeta, PageMeta } from "../types";
|
||||
import {
|
||||
AttachmentData,
|
||||
AttachmentEncoding,
|
||||
SpacePrimitives,
|
||||
} from "./space_primitives";
|
||||
import { FileMeta } from "../types";
|
||||
import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives";
|
||||
import { Plug } from "@plugos/plugos/plug";
|
||||
import { realpathSync } from "fs";
|
||||
import mime from "mime-types";
|
||||
|
||||
function lookupContentType(path: string): string {
|
||||
return mime.lookup(path) || "application/octet-stream";
|
||||
}
|
||||
|
||||
export class DiskSpacePrimitives implements SpacePrimitives {
|
||||
rootPath: string;
|
||||
plugPrefix: string;
|
||||
|
||||
constructor(rootPath: string, plugPrefix: string = "_plug/") {
|
||||
constructor(rootPath: string) {
|
||||
this.rootPath = realpathSync(rootPath);
|
||||
this.plugPrefix = plugPrefix;
|
||||
}
|
||||
|
||||
safePath(p: string): string {
|
||||
@@ -35,111 +25,136 @@ export class DiskSpacePrimitives implements SpacePrimitives {
|
||||
return realPath;
|
||||
}
|
||||
|
||||
pageNameToPath(pageName: string) {
|
||||
if (pageName.startsWith(this.plugPrefix)) {
|
||||
return this.safePath(path.join(this.rootPath, pageName + ".plug.json"));
|
||||
}
|
||||
return this.safePath(path.join(this.rootPath, pageName + ".md"));
|
||||
filenameToPath(pageName: string) {
|
||||
return this.safePath(path.join(this.rootPath, pageName));
|
||||
}
|
||||
|
||||
pathToPageName(fullPath: string): string {
|
||||
let extLength = fullPath.endsWith(".plug.json")
|
||||
? ".plug.json".length
|
||||
: ".md".length;
|
||||
return fullPath.substring(
|
||||
this.rootPath.length + 1,
|
||||
fullPath.length - extLength
|
||||
);
|
||||
pathToFilename(fullPath: string): string {
|
||||
return fullPath.substring(this.rootPath.length + 1);
|
||||
}
|
||||
|
||||
// Pages
|
||||
async readPage(pageName: string): Promise<{ text: string; meta: PageMeta }> {
|
||||
const localPath = this.pageNameToPath(pageName);
|
||||
async readFile(
|
||||
name: string,
|
||||
encoding: FileEncoding
|
||||
): Promise<{ data: FileData; meta: FileMeta }> {
|
||||
const localPath = this.filenameToPath(name);
|
||||
try {
|
||||
const s = await stat(localPath);
|
||||
let data: FileData | null = null;
|
||||
let contentType = lookupContentType(name);
|
||||
switch (encoding) {
|
||||
case "string":
|
||||
data = await readFile(localPath, "utf8");
|
||||
break;
|
||||
case "dataurl":
|
||||
let fileBuffer = await readFile(localPath, {
|
||||
encoding: "base64",
|
||||
});
|
||||
data = `data:${contentType};base64,${fileBuffer}`;
|
||||
break;
|
||||
case "arraybuffer":
|
||||
let arrayBuffer = await readFile(localPath);
|
||||
data = arrayBuffer.buffer;
|
||||
break;
|
||||
}
|
||||
return {
|
||||
text: await readFile(localPath, "utf8"),
|
||||
data,
|
||||
meta: {
|
||||
name: pageName,
|
||||
name: name,
|
||||
lastModified: s.mtime.getTime(),
|
||||
perm: "rw",
|
||||
size: s.size,
|
||||
contentType: contentType,
|
||||
},
|
||||
};
|
||||
} catch (e) {
|
||||
// console.error("Error while reading page", pageName, e);
|
||||
throw Error(`Could not read page ${pageName}`);
|
||||
console.error("Error while reading file", name, e);
|
||||
throw Error(`Could not read file ${name}`);
|
||||
}
|
||||
}
|
||||
|
||||
async writePage(
|
||||
pageName: string,
|
||||
text: string,
|
||||
selfUpdate: boolean,
|
||||
lastModified?: number
|
||||
): Promise<PageMeta> {
|
||||
let localPath = this.pageNameToPath(pageName);
|
||||
async writeFile(
|
||||
name: string,
|
||||
encoding: FileEncoding,
|
||||
data: FileData,
|
||||
selfUpdate?: boolean
|
||||
): Promise<FileMeta> {
|
||||
let localPath = this.filenameToPath(name);
|
||||
try {
|
||||
// Ensure parent folder exists
|
||||
await mkdir(path.dirname(localPath), { recursive: true });
|
||||
|
||||
// Actually write the file
|
||||
await writeFile(localPath, text);
|
||||
|
||||
if (lastModified) {
|
||||
let d = new Date(lastModified);
|
||||
console.log("Going to set the modified time", d);
|
||||
await utimes(localPath, d, d);
|
||||
switch (encoding) {
|
||||
case "string":
|
||||
await writeFile(localPath, data as string, "utf8");
|
||||
break;
|
||||
case "dataurl":
|
||||
await writeFile(localPath, (data as string).split(",")[1], {
|
||||
encoding: "base64",
|
||||
});
|
||||
break;
|
||||
case "arraybuffer":
|
||||
await writeFile(localPath, Buffer.from(data as ArrayBuffer));
|
||||
break;
|
||||
}
|
||||
|
||||
// Fetch new metadata
|
||||
const s = await stat(localPath);
|
||||
return {
|
||||
name: pageName,
|
||||
name: name,
|
||||
size: s.size,
|
||||
contentType: lookupContentType(name),
|
||||
lastModified: s.mtime.getTime(),
|
||||
perm: "rw",
|
||||
};
|
||||
} catch (e) {
|
||||
console.error("Error while writing page", pageName, e);
|
||||
throw Error(`Could not write ${pageName}`);
|
||||
console.error("Error while writing file", name, e);
|
||||
throw Error(`Could not write ${name}`);
|
||||
}
|
||||
}
|
||||
|
||||
async getPageMeta(pageName: string): Promise<PageMeta> {
|
||||
let localPath = this.pageNameToPath(pageName);
|
||||
async getFileMeta(name: string): Promise<FileMeta> {
|
||||
let localPath = this.filenameToPath(name);
|
||||
try {
|
||||
const s = await stat(localPath);
|
||||
return {
|
||||
name: pageName,
|
||||
name: name,
|
||||
size: s.size,
|
||||
contentType: lookupContentType(name),
|
||||
lastModified: s.mtime.getTime(),
|
||||
perm: "rw",
|
||||
};
|
||||
} catch (e) {
|
||||
// console.error("Error while getting page meta", pageName, e);
|
||||
throw Error(`Could not get meta for ${pageName}`);
|
||||
throw Error(`Could not get meta for ${name}`);
|
||||
}
|
||||
}
|
||||
|
||||
async deletePage(pageName: string): Promise<void> {
|
||||
let localPath = this.pageNameToPath(pageName);
|
||||
async deleteFile(name: string): Promise<void> {
|
||||
let localPath = this.filenameToPath(name);
|
||||
await unlink(localPath);
|
||||
}
|
||||
|
||||
async fetchPageList(): Promise<{
|
||||
pages: Set<PageMeta>;
|
||||
nowTimestamp: number;
|
||||
}> {
|
||||
let pages = new Set<PageMeta>();
|
||||
async fetchFileList(): Promise<FileMeta[]> {
|
||||
let fileList: FileMeta[] = [];
|
||||
|
||||
const walkPath = async (dir: string) => {
|
||||
let files = await readdir(dir);
|
||||
for (let file of files) {
|
||||
if (file.startsWith(".")) {
|
||||
continue;
|
||||
}
|
||||
const fullPath = path.join(dir, file);
|
||||
let s = await stat(fullPath);
|
||||
if (s.isDirectory()) {
|
||||
await walkPath(fullPath);
|
||||
} else {
|
||||
if (file.endsWith(".md") || file.endsWith(".json")) {
|
||||
pages.add({
|
||||
name: this.pathToPageName(fullPath),
|
||||
if (!file.startsWith(".")) {
|
||||
fileList.push({
|
||||
name: this.pathToFilename(fullPath),
|
||||
size: s.size,
|
||||
contentType: lookupContentType(fullPath),
|
||||
lastModified: s.mtime.getTime(),
|
||||
perm: "rw",
|
||||
});
|
||||
@@ -148,150 +163,7 @@ export class DiskSpacePrimitives implements SpacePrimitives {
|
||||
}
|
||||
};
|
||||
await walkPath(this.rootPath);
|
||||
return {
|
||||
pages: pages,
|
||||
nowTimestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
// Attachments
|
||||
attachmentNameToPath(name: string) {
|
||||
return this.safePath(path.join(this.rootPath, name));
|
||||
}
|
||||
|
||||
pathToAttachmentName(fullPath: string): string {
|
||||
return fullPath.substring(this.rootPath.length + 1);
|
||||
}
|
||||
|
||||
async fetchAttachmentList(): Promise<{
|
||||
attachments: Set<AttachmentMeta>;
|
||||
nowTimestamp: number;
|
||||
}> {
|
||||
let attachments = new Set<AttachmentMeta>();
|
||||
|
||||
const walkPath = async (dir: string) => {
|
||||
let files = await readdir(dir);
|
||||
for (let file of files) {
|
||||
const fullPath = path.join(dir, file);
|
||||
let s = await stat(fullPath);
|
||||
if (s.isDirectory()) {
|
||||
if (!file.startsWith(".")) {
|
||||
await walkPath(fullPath);
|
||||
}
|
||||
} else {
|
||||
if (
|
||||
!file.startsWith(".") &&
|
||||
!file.endsWith(".md") &&
|
||||
!file.endsWith(".json")
|
||||
) {
|
||||
attachments.add({
|
||||
name: this.pathToAttachmentName(fullPath),
|
||||
lastModified: s.mtime.getTime(),
|
||||
size: s.size,
|
||||
contentType: mime.lookup(file) || "application/octet-stream",
|
||||
perm: "rw",
|
||||
} as AttachmentMeta);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
await walkPath(this.rootPath);
|
||||
return {
|
||||
attachments,
|
||||
nowTimestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
async readAttachment(
|
||||
name: string,
|
||||
encoding: AttachmentEncoding
|
||||
): Promise<{ data: AttachmentData; meta: AttachmentMeta }> {
|
||||
const localPath = this.attachmentNameToPath(name);
|
||||
let fileBuffer = await readFile(localPath, {
|
||||
encoding: encoding === "dataurl" ? "base64" : null,
|
||||
});
|
||||
|
||||
try {
|
||||
const s = await stat(localPath);
|
||||
let contentType = mime.lookup(name) || "application/octet-stream";
|
||||
return {
|
||||
data:
|
||||
encoding === "dataurl"
|
||||
? `data:${contentType};base64,${fileBuffer}`
|
||||
: (fileBuffer as Buffer).buffer,
|
||||
meta: {
|
||||
name: name,
|
||||
lastModified: s.mtime.getTime(),
|
||||
size: s.size,
|
||||
contentType: contentType,
|
||||
perm: "rw",
|
||||
},
|
||||
};
|
||||
} catch (e) {
|
||||
// console.error("Error while reading attachment", name, e);
|
||||
throw Error(`Could not read attachment ${name}`);
|
||||
}
|
||||
}
|
||||
|
||||
async getAttachmentMeta(name: string): Promise<AttachmentMeta> {
|
||||
const localPath = this.attachmentNameToPath(name);
|
||||
try {
|
||||
const s = await stat(localPath);
|
||||
return {
|
||||
name: name,
|
||||
lastModified: s.mtime.getTime(),
|
||||
size: s.size,
|
||||
contentType: mime.lookup(name) || "application/octet-stream",
|
||||
perm: "rw",
|
||||
};
|
||||
} catch (e) {
|
||||
// console.error("Error while getting attachment meta", name, e);
|
||||
throw Error(`Could not get meta for ${name}`);
|
||||
}
|
||||
}
|
||||
|
||||
async writeAttachment(
|
||||
name: string,
|
||||
data: AttachmentData,
|
||||
selfUpdate?: boolean,
|
||||
lastModified?: number
|
||||
): Promise<AttachmentMeta> {
|
||||
let localPath = this.attachmentNameToPath(name);
|
||||
try {
|
||||
// Ensure parent folder exists
|
||||
await mkdir(path.dirname(localPath), { recursive: true });
|
||||
|
||||
// Actually write the file
|
||||
if (typeof data === "string") {
|
||||
await writeFile(localPath, data.split(",")[1], { encoding: "base64" });
|
||||
} else {
|
||||
await writeFile(localPath, Buffer.from(data));
|
||||
}
|
||||
|
||||
if (lastModified) {
|
||||
let d = new Date(lastModified);
|
||||
console.log("Going to set the modified time", d);
|
||||
await utimes(localPath, d, d);
|
||||
}
|
||||
|
||||
// Fetch new metadata
|
||||
const s = await stat(localPath);
|
||||
return {
|
||||
name: name,
|
||||
lastModified: s.mtime.getTime(),
|
||||
size: s.size,
|
||||
contentType: mime.lookup(name) || "application/octet-stream",
|
||||
perm: "rw",
|
||||
};
|
||||
} catch (e) {
|
||||
console.error("Error while writing attachment", name, e);
|
||||
throw Error(`Could not write ${name}`);
|
||||
}
|
||||
}
|
||||
|
||||
async deleteAttachment(name: string): Promise<void> {
|
||||
let localPath = this.attachmentNameToPath(name);
|
||||
await unlink(localPath);
|
||||
return fileList;
|
||||
}
|
||||
|
||||
// Plugs
|
||||
|
||||
@@ -1,19 +1,14 @@
|
||||
import { EventHook } from "@plugos/plugos/hooks/event";
|
||||
import { Plug } from "@plugos/plugos/plug";
|
||||
|
||||
import { AttachmentMeta, PageMeta } from "../types";
|
||||
import { plugPrefix, trashPrefix } from "./constants";
|
||||
import {
|
||||
AttachmentData,
|
||||
AttachmentEncoding,
|
||||
SpacePrimitives,
|
||||
} from "./space_primitives";
|
||||
import { FileMeta } from "../types";
|
||||
import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives";
|
||||
|
||||
export class EventedSpacePrimitives implements SpacePrimitives {
|
||||
constructor(private wrapped: SpacePrimitives, private eventHook: EventHook) {}
|
||||
|
||||
fetchPageList(): Promise<{ pages: Set<PageMeta>; nowTimestamp: number }> {
|
||||
return this.wrapped.fetchPageList();
|
||||
fetchFileList(): Promise<FileMeta[]> {
|
||||
return this.wrapped.fetchFileList();
|
||||
}
|
||||
|
||||
proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
|
||||
@@ -29,26 +24,43 @@ export class EventedSpacePrimitives implements SpacePrimitives {
|
||||
return this.wrapped.invokeFunction(plug, env, name, args);
|
||||
}
|
||||
|
||||
readPage(pageName: string): Promise<{ text: string; meta: PageMeta }> {
|
||||
return this.wrapped.readPage(pageName);
|
||||
readFile(
|
||||
name: string,
|
||||
encoding: FileEncoding
|
||||
): Promise<{ data: FileData; meta: FileMeta }> {
|
||||
return this.wrapped.readFile(name, encoding);
|
||||
}
|
||||
|
||||
async writePage(
|
||||
pageName: string,
|
||||
text: string,
|
||||
selfUpdate: boolean,
|
||||
lastModified?: number
|
||||
): Promise<PageMeta> {
|
||||
const newPageMeta = await this.wrapped.writePage(
|
||||
pageName,
|
||||
text,
|
||||
selfUpdate,
|
||||
lastModified
|
||||
async writeFile(
|
||||
name: string,
|
||||
encoding: FileEncoding,
|
||||
data: FileData,
|
||||
selfUpdate: boolean
|
||||
): Promise<FileMeta> {
|
||||
const newMeta = await this.wrapped.writeFile(
|
||||
name,
|
||||
encoding,
|
||||
data,
|
||||
selfUpdate
|
||||
);
|
||||
// This can happen async
|
||||
if (!pageName.startsWith(trashPrefix) && !pageName.startsWith(plugPrefix)) {
|
||||
if (name.endsWith(".md")) {
|
||||
const pageName = name.substring(0, name.length - 3);
|
||||
let text = "";
|
||||
switch (encoding) {
|
||||
case "string":
|
||||
text = data as string;
|
||||
break;
|
||||
case "arraybuffer":
|
||||
const decoder = new TextDecoder("utf-8");
|
||||
text = decoder.decode(data as ArrayBuffer);
|
||||
break;
|
||||
case "dataurl":
|
||||
throw Error("Data urls not supported in this context");
|
||||
}
|
||||
|
||||
this.eventHook
|
||||
.dispatchEvent("page:saved", pageName)
|
||||
.dispatchEvent("page:saved")
|
||||
.then(() => {
|
||||
return this.eventHook.dispatchEvent("page:index_text", {
|
||||
name: pageName,
|
||||
@@ -59,54 +71,18 @@ export class EventedSpacePrimitives implements SpacePrimitives {
|
||||
console.error("Error dispatching page:saved event", e);
|
||||
});
|
||||
}
|
||||
return newPageMeta;
|
||||
return newMeta;
|
||||
}
|
||||
|
||||
getPageMeta(pageName: string): Promise<PageMeta> {
|
||||
return this.wrapped.getPageMeta(pageName);
|
||||
getFileMeta(name: string): Promise<FileMeta> {
|
||||
return this.wrapped.getFileMeta(name);
|
||||
}
|
||||
|
||||
async deletePage(pageName: string): Promise<void> {
|
||||
await this.eventHook.dispatchEvent("page:deleted", pageName);
|
||||
return this.wrapped.deletePage(pageName);
|
||||
}
|
||||
|
||||
fetchAttachmentList(): Promise<{
|
||||
attachments: Set<AttachmentMeta>;
|
||||
nowTimestamp: number;
|
||||
}> {
|
||||
return this.wrapped.fetchAttachmentList();
|
||||
}
|
||||
|
||||
readAttachment(
|
||||
name: string,
|
||||
encoding: AttachmentEncoding
|
||||
): Promise<{ data: AttachmentData; meta: AttachmentMeta }> {
|
||||
return this.wrapped.readAttachment(name, encoding);
|
||||
}
|
||||
|
||||
getAttachmentMeta(name: string): Promise<AttachmentMeta> {
|
||||
return this.wrapped.getAttachmentMeta(name);
|
||||
}
|
||||
|
||||
async writeAttachment(
|
||||
name: string,
|
||||
blob: ArrayBuffer,
|
||||
selfUpdate?: boolean | undefined,
|
||||
lastModified?: number | undefined
|
||||
): Promise<AttachmentMeta> {
|
||||
let meta = await this.wrapped.writeAttachment(
|
||||
name,
|
||||
blob,
|
||||
selfUpdate,
|
||||
lastModified
|
||||
);
|
||||
await this.eventHook.dispatchEvent("attachment:saved", name);
|
||||
return meta;
|
||||
}
|
||||
|
||||
async deleteAttachment(name: string): Promise<void> {
|
||||
await this.eventHook.dispatchEvent("attachment:deleted", name);
|
||||
return this.wrapped.deleteAttachment(name);
|
||||
async deleteFile(name: string): Promise<void> {
|
||||
if (name.endsWith(".md")) {
|
||||
const pageName = name.substring(0, name.length - 3);
|
||||
await this.eventHook.dispatchEvent("page:deleted", pageName);
|
||||
}
|
||||
return this.wrapped.deleteFile(name);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,20 +1,14 @@
|
||||
import { AttachmentMeta, PageMeta } from "../types";
|
||||
import { AttachmentMeta, FileMeta, PageMeta } from "../types";
|
||||
import { Plug } from "@plugos/plugos/plug";
|
||||
import {
|
||||
AttachmentData,
|
||||
AttachmentEncoding,
|
||||
SpacePrimitives,
|
||||
} from "./space_primitives";
|
||||
import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives";
|
||||
|
||||
export class HttpSpacePrimitives implements SpacePrimitives {
|
||||
fsUrl: string;
|
||||
fsaUrl: string;
|
||||
private plugUrl: string;
|
||||
token?: string;
|
||||
|
||||
constructor(url: string, token?: string) {
|
||||
this.fsUrl = url + "/page";
|
||||
this.fsaUrl = url + "/attachment";
|
||||
this.fsUrl = url + "/fs";
|
||||
this.plugUrl = url + "/plug";
|
||||
this.token = token;
|
||||
}
|
||||
@@ -34,72 +28,105 @@ export class HttpSpacePrimitives implements SpacePrimitives {
|
||||
return result;
|
||||
}
|
||||
|
||||
public async fetchPageList(): Promise<{
|
||||
pages: Set<PageMeta>;
|
||||
nowTimestamp: number;
|
||||
}> {
|
||||
public async fetchFileList(): Promise<FileMeta[]> {
|
||||
let req = await this.authenticatedFetch(this.fsUrl, {
|
||||
method: "GET",
|
||||
});
|
||||
|
||||
let result = new Set<PageMeta>();
|
||||
((await req.json()) as any[]).forEach((meta: any) => {
|
||||
const pageName = meta.name;
|
||||
result.add({
|
||||
name: pageName,
|
||||
lastModified: meta.lastModified,
|
||||
perm: "rw",
|
||||
});
|
||||
});
|
||||
let result: FileMeta[] = await req.json();
|
||||
|
||||
return {
|
||||
pages: result,
|
||||
nowTimestamp: +req.headers.get("Now-Timestamp")!,
|
||||
};
|
||||
return result;
|
||||
}
|
||||
|
||||
async readPage(name: string): Promise<{ text: string; meta: PageMeta }> {
|
||||
async readFile(
|
||||
name: string,
|
||||
encoding: FileEncoding
|
||||
): Promise<{ data: FileData; meta: FileMeta }> {
|
||||
let res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
|
||||
method: "GET",
|
||||
});
|
||||
if (res.headers.get("X-Status") === "404") {
|
||||
if (res.status === 404) {
|
||||
throw new Error(`Page not found`);
|
||||
}
|
||||
let data: FileData | null = null;
|
||||
switch (encoding) {
|
||||
case "arraybuffer":
|
||||
let abBlob = await res.blob();
|
||||
data = await abBlob.arrayBuffer();
|
||||
break;
|
||||
case "dataurl":
|
||||
let dUBlob = await res.blob();
|
||||
data = arrayBufferToDataUrl(await dUBlob.arrayBuffer());
|
||||
break;
|
||||
case "string":
|
||||
data = await res.text();
|
||||
break;
|
||||
}
|
||||
return {
|
||||
text: await res.text(),
|
||||
meta: this.responseToPageMeta(name, res),
|
||||
data: data,
|
||||
meta: this.responseToMeta(name, res),
|
||||
};
|
||||
}
|
||||
|
||||
async writePage(
|
||||
async writeFile(
|
||||
name: string,
|
||||
text: string,
|
||||
selfUpdate?: boolean,
|
||||
lastModified?: number
|
||||
): Promise<PageMeta> {
|
||||
// TODO: lastModified ignored for now
|
||||
encoding: FileEncoding,
|
||||
data: FileData,
|
||||
selfUpdate?: boolean
|
||||
): Promise<FileMeta> {
|
||||
let body: any = null;
|
||||
|
||||
switch (encoding) {
|
||||
case "arraybuffer":
|
||||
case "string":
|
||||
body = data;
|
||||
break;
|
||||
case "dataurl":
|
||||
data = dataUrlToArrayBuffer(data as string);
|
||||
break;
|
||||
}
|
||||
let res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
|
||||
method: "PUT",
|
||||
body: text,
|
||||
headers: lastModified
|
||||
? {
|
||||
"Last-Modified": "" + lastModified,
|
||||
}
|
||||
: undefined,
|
||||
headers: {
|
||||
"Content-type": "application/octet-stream",
|
||||
},
|
||||
body,
|
||||
});
|
||||
const newMeta = this.responseToPageMeta(name, res);
|
||||
const newMeta = this.responseToMeta(name, res);
|
||||
return newMeta;
|
||||
}
|
||||
|
||||
async deletePage(name: string): Promise<void> {
|
||||
async deleteFile(name: string): Promise<void> {
|
||||
let req = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
|
||||
method: "DELETE",
|
||||
});
|
||||
if (req.status !== 200) {
|
||||
throw Error(`Failed to delete page: ${req.statusText}`);
|
||||
throw Error(`Failed to delete file: ${req.statusText}`);
|
||||
}
|
||||
}
|
||||
|
||||
async getFileMeta(name: string): Promise<FileMeta> {
|
||||
let res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
|
||||
method: "OPTIONS",
|
||||
});
|
||||
if (res.status === 404) {
|
||||
throw new Error(`File not found`);
|
||||
}
|
||||
return this.responseToMeta(name, res);
|
||||
}
|
||||
|
||||
private responseToMeta(name: string, res: Response): FileMeta {
|
||||
return {
|
||||
name,
|
||||
size: +res.headers.get("Content-length")!,
|
||||
contentType: res.headers.get("Content-type")!,
|
||||
lastModified: +(res.headers.get("Last-Modified") || "0"),
|
||||
perm: (res.headers.get("X-Permission") as "rw" | "ro") || "rw",
|
||||
};
|
||||
}
|
||||
|
||||
// Plugs
|
||||
|
||||
async proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
|
||||
let req = await this.authenticatedFetch(
|
||||
`${this.plugUrl}/${plug.name}/syscall/${name}`,
|
||||
@@ -121,95 +148,6 @@ export class HttpSpacePrimitives implements SpacePrimitives {
|
||||
return await req.json();
|
||||
}
|
||||
|
||||
// Attachments
|
||||
public async fetchAttachmentList(): Promise<{
|
||||
attachments: Set<AttachmentMeta>;
|
||||
nowTimestamp: number;
|
||||
}> {
|
||||
let req = await this.authenticatedFetch(this.fsaUrl, {
|
||||
method: "GET",
|
||||
});
|
||||
|
||||
let result = new Set<AttachmentMeta>();
|
||||
((await req.json()) as any[]).forEach((meta: any) => {
|
||||
const pageName = meta.name;
|
||||
result.add({
|
||||
name: pageName,
|
||||
size: meta.size,
|
||||
lastModified: meta.lastModified,
|
||||
contentType: meta.contentType,
|
||||
perm: "rw",
|
||||
});
|
||||
});
|
||||
|
||||
return {
|
||||
attachments: result,
|
||||
nowTimestamp: +req.headers.get("Now-Timestamp")!,
|
||||
};
|
||||
}
|
||||
|
||||
async readAttachment(
|
||||
name: string,
|
||||
encoding: AttachmentEncoding
|
||||
): Promise<{ data: AttachmentData; meta: AttachmentMeta }> {
|
||||
let res = await this.authenticatedFetch(`${this.fsaUrl}/${name}`, {
|
||||
method: "GET",
|
||||
});
|
||||
if (res.headers.get("X-Status") === "404") {
|
||||
throw new Error(`Page not found`);
|
||||
}
|
||||
let blob = await res.blob();
|
||||
return {
|
||||
data:
|
||||
encoding === "arraybuffer"
|
||||
? await blob.arrayBuffer()
|
||||
: arrayBufferToDataUrl(await blob.arrayBuffer()),
|
||||
meta: this.responseToAttachmentMeta(name, res),
|
||||
};
|
||||
}
|
||||
|
||||
async writeAttachment(
|
||||
name: string,
|
||||
data: AttachmentData,
|
||||
selfUpdate?: boolean,
|
||||
lastModified?: number
|
||||
): Promise<AttachmentMeta> {
|
||||
if (typeof data === "string") {
|
||||
data = dataUrlToArrayBuffer(data);
|
||||
}
|
||||
let res = await this.authenticatedFetch(`${this.fsaUrl}/${name}`, {
|
||||
method: "PUT",
|
||||
body: data,
|
||||
headers: {
|
||||
"Last-Modified": lastModified ? "" + lastModified : undefined,
|
||||
"Content-type": "application/octet-stream",
|
||||
},
|
||||
});
|
||||
const newMeta = this.responseToAttachmentMeta(name, res);
|
||||
return newMeta;
|
||||
}
|
||||
|
||||
async getAttachmentMeta(name: string): Promise<AttachmentMeta> {
|
||||
let res = await this.authenticatedFetch(`${this.fsaUrl}/${name}`, {
|
||||
method: "OPTIONS",
|
||||
});
|
||||
if (res.headers.get("X-Status") === "404") {
|
||||
throw new Error(`Page not found`);
|
||||
}
|
||||
return this.responseToAttachmentMeta(name, res);
|
||||
}
|
||||
|
||||
async deleteAttachment(name: string): Promise<void> {
|
||||
let req = await this.authenticatedFetch(`${this.fsaUrl}/${name}`, {
|
||||
method: "DELETE",
|
||||
});
|
||||
if (req.status !== 200) {
|
||||
throw Error(`Failed to delete attachment: ${req.statusText}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Plugs
|
||||
|
||||
async invokeFunction(
|
||||
plug: Plug<any>,
|
||||
env: string,
|
||||
@@ -244,38 +182,6 @@ export class HttpSpacePrimitives implements SpacePrimitives {
|
||||
return await req.text();
|
||||
}
|
||||
}
|
||||
|
||||
async getPageMeta(name: string): Promise<PageMeta> {
|
||||
let res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
|
||||
method: "OPTIONS",
|
||||
});
|
||||
if (res.headers.get("X-Status") === "404") {
|
||||
throw new Error(`Page not found`);
|
||||
}
|
||||
return this.responseToPageMeta(name, res);
|
||||
}
|
||||
|
||||
private responseToPageMeta(name: string, res: Response): PageMeta {
|
||||
return {
|
||||
name,
|
||||
lastModified: +(res.headers.get("Last-Modified") || "0"),
|
||||
perm: (res.headers.get("X-Permission") as "rw" | "ro") || "rw",
|
||||
};
|
||||
}
|
||||
|
||||
private responseToAttachmentMeta(
|
||||
name: string,
|
||||
res: Response
|
||||
): AttachmentMeta {
|
||||
return {
|
||||
name,
|
||||
lastModified: +(res.headers.get("Last-Modified") || "0"),
|
||||
size: +(res.headers.get("Content-Length") || "0"),
|
||||
contentType:
|
||||
res.headers.get("Content-Type") || "application/octet-stream",
|
||||
perm: (res.headers.get("X-Permission") as "rw" | "ro") || "rw",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function dataUrlToArrayBuffer(dataUrl: string): ArrayBuffer {
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
import {
|
||||
AttachmentData,
|
||||
AttachmentEncoding,
|
||||
SpacePrimitives,
|
||||
} from "./space_primitives";
|
||||
import { AttachmentMeta, PageMeta } from "../types";
|
||||
import Dexie, { Table } from "dexie";
|
||||
import { Plug } from "@plugos/plugos/plug";
|
||||
|
||||
type Page = {
|
||||
name: string;
|
||||
text: string;
|
||||
meta: PageMeta;
|
||||
};
|
||||
|
||||
export class IndexedDBSpacePrimitives implements SpacePrimitives {
|
||||
private pageTable: Table<Page, string>;
|
||||
|
||||
constructor(dbName: string, readonly timeSkew: number = 0) {
|
||||
const db = new Dexie(dbName);
|
||||
db.version(1).stores({
|
||||
page: "name",
|
||||
});
|
||||
this.pageTable = db.table("page");
|
||||
}
|
||||
fetchAttachmentList(): Promise<{
|
||||
attachments: Set<AttachmentMeta>;
|
||||
nowTimestamp: number;
|
||||
}> {
|
||||
throw new Error("Method not implemented.");
|
||||
}
|
||||
readAttachment(
|
||||
name: string,
|
||||
encoding: AttachmentEncoding
|
||||
): Promise<{ data: AttachmentData; meta: AttachmentMeta }> {
|
||||
throw new Error("Method not implemented.");
|
||||
}
|
||||
getAttachmentMeta(name: string): Promise<AttachmentMeta> {
|
||||
throw new Error("Method not implemented.");
|
||||
}
|
||||
writeAttachment(
|
||||
name: string,
|
||||
blob: ArrayBuffer,
|
||||
selfUpdate?: boolean | undefined,
|
||||
lastModified?: number | undefined
|
||||
): Promise<AttachmentMeta> {
|
||||
throw new Error("Method not implemented.");
|
||||
}
|
||||
deleteAttachment(name: string): Promise<void> {
|
||||
throw new Error("Method not implemented.");
|
||||
}
|
||||
|
||||
async deletePage(name: string): Promise<void> {
|
||||
return this.pageTable.delete(name);
|
||||
}
|
||||
|
||||
async getPageMeta(name: string): Promise<PageMeta> {
|
||||
let entry = await this.pageTable.get(name);
|
||||
if (entry) {
|
||||
return entry.meta;
|
||||
} else {
|
||||
throw Error(`Page not found`);
|
||||
}
|
||||
}
|
||||
|
||||
invokeFunction(
|
||||
plug: Plug<any>,
|
||||
env: string,
|
||||
name: string,
|
||||
args: any[]
|
||||
): Promise<any> {
|
||||
return plug.invoke(name, args);
|
||||
}
|
||||
|
||||
async fetchPageList(): Promise<{
|
||||
pages: Set<PageMeta>;
|
||||
nowTimestamp: number;
|
||||
}> {
|
||||
let allPages = await this.pageTable.toArray();
|
||||
return {
|
||||
pages: new Set(allPages.map((p) => p.meta)),
|
||||
nowTimestamp: Date.now() + this.timeSkew,
|
||||
};
|
||||
}
|
||||
|
||||
proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
|
||||
return plug.syscall(name, args);
|
||||
}
|
||||
|
||||
async readPage(name: string): Promise<{ text: string; meta: PageMeta }> {
|
||||
let page = await this.pageTable.get(name);
|
||||
if (page) {
|
||||
return page;
|
||||
} else {
|
||||
throw new Error("Page not found");
|
||||
}
|
||||
}
|
||||
|
||||
async writePage(
|
||||
name: string,
|
||||
text: string,
|
||||
selfUpdate?: boolean,
|
||||
lastModified?: number
|
||||
): Promise<PageMeta> {
|
||||
const meta: PageMeta = {
|
||||
name,
|
||||
lastModified: lastModified ? lastModified : Date.now() + this.timeSkew,
|
||||
perm: "rw",
|
||||
};
|
||||
await this.pageTable.put({
|
||||
name,
|
||||
text,
|
||||
meta,
|
||||
});
|
||||
return meta;
|
||||
}
|
||||
}
|
||||
+72
-102
@@ -1,13 +1,8 @@
|
||||
import {
|
||||
AttachmentData,
|
||||
AttachmentEncoding,
|
||||
SpacePrimitives,
|
||||
} from "./space_primitives";
|
||||
import { AttachmentMeta, PageMeta } from "../types";
|
||||
import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives";
|
||||
import { AttachmentMeta, FileMeta, PageMeta } from "../types";
|
||||
import { EventEmitter } from "@plugos/plugos/event";
|
||||
import { Plug } from "@plugos/plugos/plug";
|
||||
import { Manifest } from "../manifest";
|
||||
import { plugPrefix, trashPrefix } from "./constants";
|
||||
import { plugPrefix } from "./constants";
|
||||
import { safeRun } from "../util";
|
||||
|
||||
const pageWatchInterval = 2000;
|
||||
@@ -19,23 +14,21 @@ export type SpaceEvents = {
|
||||
pageListUpdated: (pages: Set<PageMeta>) => void;
|
||||
};
|
||||
|
||||
export class Space
|
||||
extends EventEmitter<SpaceEvents>
|
||||
implements SpacePrimitives
|
||||
{
|
||||
export class Space extends EventEmitter<SpaceEvents> {
|
||||
pageMetaCache = new Map<string, PageMeta>();
|
||||
watchedPages = new Set<string>();
|
||||
private initialPageListLoad = true;
|
||||
private saving = false;
|
||||
|
||||
constructor(private space: SpacePrimitives, private trashEnabled = true) {
|
||||
constructor(private space: SpacePrimitives) {
|
||||
super();
|
||||
}
|
||||
|
||||
public async updatePageList() {
|
||||
let newPageList = await this.space.fetchPageList();
|
||||
let newPageList = await this.fetchPageList();
|
||||
// console.log("Updating page list", newPageList);
|
||||
let deletedPages = new Set<string>(this.pageMetaCache.keys());
|
||||
newPageList.pages.forEach((meta) => {
|
||||
newPageList.forEach((meta) => {
|
||||
const pageName = meta.name;
|
||||
const oldPageMeta = this.pageMetaCache.get(pageName);
|
||||
const newPageMeta: PageMeta = {
|
||||
@@ -50,9 +43,7 @@ export class Space
|
||||
this.emit("pageCreated", newPageMeta);
|
||||
} else if (
|
||||
oldPageMeta &&
|
||||
oldPageMeta.lastModified !== newPageMeta.lastModified &&
|
||||
(!this.trashEnabled ||
|
||||
(this.trashEnabled && !pageName.startsWith(trashPrefix)))
|
||||
oldPageMeta.lastModified !== newPageMeta.lastModified
|
||||
) {
|
||||
this.emit("pageChanged", newPageMeta);
|
||||
}
|
||||
@@ -95,17 +86,7 @@ export class Space
|
||||
|
||||
async deletePage(name: string, deleteDate?: number): Promise<void> {
|
||||
await this.getPageMeta(name); // Check if page exists, if not throws Error
|
||||
if (this.trashEnabled) {
|
||||
let pageData = await this.readPage(name);
|
||||
// Move to trash
|
||||
await this.writePage(
|
||||
`${trashPrefix}${name}`,
|
||||
pageData.text,
|
||||
true,
|
||||
deleteDate
|
||||
);
|
||||
}
|
||||
await this.space.deletePage(name);
|
||||
await this.space.deleteFile(`${name}.md`);
|
||||
|
||||
this.pageMetaCache.delete(name);
|
||||
this.emit("pageDeleted", name);
|
||||
@@ -114,7 +95,9 @@ export class Space
|
||||
|
||||
async getPageMeta(name: string): Promise<PageMeta> {
|
||||
let oldMeta = this.pageMetaCache.get(name);
|
||||
let newMeta = await this.space.getPageMeta(name);
|
||||
let newMeta = fileMetaToPageMeta(
|
||||
await this.space.getFileMeta(`${name}.md`)
|
||||
);
|
||||
if (oldMeta) {
|
||||
if (oldMeta.lastModified !== newMeta.lastModified) {
|
||||
// Changed on disk, trigger event
|
||||
@@ -133,41 +116,15 @@ export class Space
|
||||
return this.space.invokeFunction(plug, env, name, args);
|
||||
}
|
||||
|
||||
listPages(unfiltered = false): Set<PageMeta> {
|
||||
if (unfiltered) {
|
||||
return new Set(this.pageMetaCache.values());
|
||||
} else {
|
||||
return new Set(
|
||||
[...this.pageMetaCache.values()].filter(
|
||||
(pageMeta) =>
|
||||
!pageMeta.name.startsWith(trashPrefix) &&
|
||||
!pageMeta.name.startsWith(plugPrefix)
|
||||
)
|
||||
);
|
||||
}
|
||||
listPages(): Set<PageMeta> {
|
||||
return new Set(this.pageMetaCache.values());
|
||||
}
|
||||
|
||||
listTrash(): Set<PageMeta> {
|
||||
return new Set(
|
||||
[...this.pageMetaCache.values()]
|
||||
.filter(
|
||||
(pageMeta) =>
|
||||
pageMeta.name.startsWith(trashPrefix) &&
|
||||
!pageMeta.name.startsWith(plugPrefix)
|
||||
)
|
||||
.map((pageMeta) => ({
|
||||
...pageMeta,
|
||||
name: pageMeta.name.substring(trashPrefix.length),
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
listPlugs(): Set<PageMeta> {
|
||||
return new Set(
|
||||
[...this.pageMetaCache.values()].filter((pageMeta) =>
|
||||
pageMeta.name.startsWith(plugPrefix)
|
||||
)
|
||||
);
|
||||
async listPlugs(): Promise<string[]> {
|
||||
let allFiles = await this.space.fetchFileList();
|
||||
return allFiles
|
||||
.filter((fileMeta) => fileMeta.name.endsWith(".plug.json"))
|
||||
.map((fileMeta) => fileMeta.name);
|
||||
}
|
||||
|
||||
proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
|
||||
@@ -175,16 +132,20 @@ export class Space
|
||||
}
|
||||
|
||||
async readPage(name: string): Promise<{ text: string; meta: PageMeta }> {
|
||||
let pageData = await this.space.readPage(name);
|
||||
let pageData = await this.space.readFile(`${name}.md`, "string");
|
||||
let previousMeta = this.pageMetaCache.get(name);
|
||||
let newMeta = fileMetaToPageMeta(pageData.meta);
|
||||
if (previousMeta) {
|
||||
if (previousMeta.lastModified !== pageData.meta.lastModified) {
|
||||
if (previousMeta.lastModified !== newMeta.lastModified) {
|
||||
// Page changed since last cached metadata, trigger event
|
||||
this.emit("pageChanged", pageData.meta);
|
||||
this.emit("pageChanged", newMeta);
|
||||
}
|
||||
}
|
||||
this.pageMetaCache.set(name, pageData.meta);
|
||||
return pageData;
|
||||
let meta = this.metaCacher(name, newMeta);
|
||||
return {
|
||||
text: pageData.data as string,
|
||||
meta: meta,
|
||||
};
|
||||
}
|
||||
|
||||
watchPage(pageName: string) {
|
||||
@@ -198,16 +159,12 @@ export class Space
|
||||
async writePage(
|
||||
name: string,
|
||||
text: string,
|
||||
selfUpdate?: boolean,
|
||||
lastModified?: number
|
||||
selfUpdate?: boolean
|
||||
): Promise<PageMeta> {
|
||||
try {
|
||||
this.saving = true;
|
||||
let pageMeta = await this.space.writePage(
|
||||
name,
|
||||
text,
|
||||
selfUpdate,
|
||||
lastModified
|
||||
let pageMeta = fileMetaToPageMeta(
|
||||
await this.space.writeFile(`${name}.md`, "string", text, selfUpdate)
|
||||
);
|
||||
if (!selfUpdate) {
|
||||
this.emit("pageChanged", pageMeta);
|
||||
@@ -218,39 +175,52 @@ export class Space
|
||||
}
|
||||
}
|
||||
|
||||
fetchPageList(): Promise<{ pages: Set<PageMeta>; nowTimestamp: number }> {
|
||||
return this.space.fetchPageList();
|
||||
async fetchPageList(): Promise<PageMeta[]> {
|
||||
return (await this.space.fetchFileList())
|
||||
.filter((fileMeta) => fileMeta.name.endsWith(".md"))
|
||||
.map(fileMetaToPageMeta);
|
||||
}
|
||||
|
||||
fetchAttachmentList(): Promise<{
|
||||
attachments: Set<AttachmentMeta>;
|
||||
nowTimestamp: number;
|
||||
}> {
|
||||
return this.space.fetchAttachmentList();
|
||||
async fetchAttachmentList(): Promise<AttachmentMeta[]> {
|
||||
return (await this.space.fetchFileList()).filter(
|
||||
(fileMeta) =>
|
||||
!fileMeta.name.endsWith(".md") && !fileMeta.name.endsWith(".plug.json")
|
||||
);
|
||||
}
|
||||
|
||||
readAttachment(
|
||||
name: string,
|
||||
encoding: AttachmentEncoding
|
||||
): Promise<{ data: AttachmentData; meta: AttachmentMeta }> {
|
||||
return this.space.readAttachment(name, encoding);
|
||||
}
|
||||
getAttachmentMeta(name: string): Promise<AttachmentMeta> {
|
||||
return this.space.getAttachmentMeta(name);
|
||||
}
|
||||
writeAttachment(
|
||||
name: string,
|
||||
data: AttachmentData,
|
||||
selfUpdate?: boolean | undefined,
|
||||
lastModified?: number | undefined
|
||||
): Promise<AttachmentMeta> {
|
||||
return this.space.writeAttachment(name, data, selfUpdate, lastModified);
|
||||
}
|
||||
deleteAttachment(name: string): Promise<void> {
|
||||
return this.space.deleteAttachment(name);
|
||||
encoding: FileEncoding
|
||||
): Promise<{ data: FileData; meta: AttachmentMeta }> {
|
||||
return this.space.readFile(name, encoding);
|
||||
}
|
||||
|
||||
private metaCacher(name: string, pageMeta: PageMeta): PageMeta {
|
||||
this.pageMetaCache.set(name, pageMeta);
|
||||
return pageMeta;
|
||||
getAttachmentMeta(name: string): Promise<AttachmentMeta> {
|
||||
return this.space.getFileMeta(name);
|
||||
}
|
||||
|
||||
writeAttachment(
|
||||
name: string,
|
||||
encoding: FileEncoding,
|
||||
data: FileData,
|
||||
selfUpdate?: boolean | undefined
|
||||
): Promise<AttachmentMeta> {
|
||||
return this.space.writeFile(name, encoding, data, selfUpdate);
|
||||
}
|
||||
|
||||
deleteAttachment(name: string): Promise<void> {
|
||||
return this.space.deleteFile(name);
|
||||
}
|
||||
|
||||
private metaCacher(name: string, meta: PageMeta): PageMeta {
|
||||
this.pageMetaCache.set(name, meta);
|
||||
return meta;
|
||||
}
|
||||
}
|
||||
|
||||
function fileMetaToPageMeta(fileMeta: FileMeta): PageMeta {
|
||||
return {
|
||||
...fileMeta,
|
||||
name: fileMeta.name.substring(0, fileMeta.name.length - 3),
|
||||
} as PageMeta;
|
||||
}
|
||||
|
||||
@@ -1,38 +1,23 @@
|
||||
import { Plug } from "@plugos/plugos/plug";
|
||||
import { AttachmentMeta, PageMeta } from "../types";
|
||||
import { FileMeta } from "../types";
|
||||
|
||||
export type AttachmentEncoding = "arraybuffer" | "dataurl";
|
||||
export type AttachmentData = ArrayBuffer | string;
|
||||
export type FileEncoding = "string" | "arraybuffer" | "dataurl";
|
||||
export type FileData = ArrayBuffer | string;
|
||||
export interface SpacePrimitives {
|
||||
// Pages
|
||||
fetchPageList(): Promise<{ pages: Set<PageMeta>; nowTimestamp: number }>;
|
||||
readPage(name: string): Promise<{ text: string; meta: PageMeta }>;
|
||||
getPageMeta(name: string): Promise<PageMeta>;
|
||||
writePage(
|
||||
fetchFileList(): Promise<FileMeta[]>;
|
||||
readFile(
|
||||
name: string,
|
||||
text: string,
|
||||
selfUpdate?: boolean,
|
||||
lastModified?: number
|
||||
): Promise<PageMeta>;
|
||||
deletePage(name: string): Promise<void>;
|
||||
|
||||
// Attachments
|
||||
fetchAttachmentList(): Promise<{
|
||||
attachments: Set<AttachmentMeta>;
|
||||
nowTimestamp: number;
|
||||
}>;
|
||||
readAttachment(
|
||||
encoding: FileEncoding
|
||||
): Promise<{ data: FileData; meta: FileMeta }>;
|
||||
getFileMeta(name: string): Promise<FileMeta>;
|
||||
writeFile(
|
||||
name: string,
|
||||
encoding: AttachmentEncoding
|
||||
): Promise<{ data: AttachmentData; meta: AttachmentMeta }>;
|
||||
getAttachmentMeta(name: string): Promise<AttachmentMeta>;
|
||||
writeAttachment(
|
||||
name: string,
|
||||
data: AttachmentData,
|
||||
selfUpdate?: boolean,
|
||||
lastModified?: number
|
||||
): Promise<AttachmentMeta>;
|
||||
deleteAttachment(name: string): Promise<void>;
|
||||
encoding: FileEncoding,
|
||||
data: FileData,
|
||||
selfUpdate?: boolean
|
||||
): Promise<FileMeta>;
|
||||
deleteFile(name: string): Promise<void>;
|
||||
|
||||
// Plugs
|
||||
proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any>;
|
||||
|
||||
@@ -1,123 +0,0 @@
|
||||
import { expect, test } from "@jest/globals";
|
||||
import { IndexedDBSpacePrimitives } from "./indexeddb_space_primitives";
|
||||
import { SpaceSync } from "./sync";
|
||||
import { PageMeta } from "../types";
|
||||
import { Space } from "./space";
|
||||
|
||||
// For testing in node.js
|
||||
require("fake-indexeddb/auto");
|
||||
|
||||
test("Test store", async () => {
|
||||
let primary = new Space(new IndexedDBSpacePrimitives("primary"), true);
|
||||
let secondary = new Space(
|
||||
new IndexedDBSpacePrimitives("secondary", -5000),
|
||||
true
|
||||
);
|
||||
let sync = new SpaceSync(primary, secondary, 0, 0, "_trash/");
|
||||
|
||||
async function conflictResolver(pageMeta1: PageMeta, pageMeta2: PageMeta) {}
|
||||
|
||||
// Write one page to primary
|
||||
await primary.writePage("index", "Hello");
|
||||
expect((await secondary.listPages()).size).toBe(0);
|
||||
await syncPages(conflictResolver);
|
||||
expect((await secondary.listPages()).size).toBe(1);
|
||||
expect((await secondary.readPage("index")).text).toBe("Hello");
|
||||
|
||||
// Should be a no-op
|
||||
expect(await syncPages()).toBe(0);
|
||||
|
||||
// Now let's make a change on the secondary
|
||||
await secondary.writePage("index", "Hello!!");
|
||||
await secondary.writePage("test", "Test page");
|
||||
|
||||
// And sync it
|
||||
await syncPages();
|
||||
|
||||
expect(primary.listPages().size).toBe(2);
|
||||
expect(secondary.listPages().size).toBe(2);
|
||||
|
||||
expect((await primary.readPage("index")).text).toBe("Hello!!");
|
||||
|
||||
// Let's make some random edits on both ends
|
||||
await primary.writePage("index", "1");
|
||||
await primary.writePage("index2", "2");
|
||||
await secondary.writePage("index3", "3");
|
||||
await secondary.writePage("index4", "4");
|
||||
await syncPages();
|
||||
|
||||
expect((await primary.listPages()).size).toBe(5);
|
||||
expect((await secondary.listPages()).size).toBe(5);
|
||||
|
||||
expect(await syncPages()).toBe(0);
|
||||
|
||||
console.log("Deleting pages");
|
||||
// Delete some pages
|
||||
await primary.deletePage("index");
|
||||
await primary.deletePage("index3");
|
||||
|
||||
console.log("Pages", await primary.listPages());
|
||||
console.log("Trash", await primary.listTrash());
|
||||
|
||||
await syncPages();
|
||||
|
||||
expect((await primary.listPages()).size).toBe(3);
|
||||
expect((await secondary.listPages()).size).toBe(3);
|
||||
|
||||
// No-op
|
||||
expect(await syncPages()).toBe(0);
|
||||
|
||||
await secondary.deletePage("index4");
|
||||
await primary.deletePage("index2");
|
||||
|
||||
await syncPages();
|
||||
|
||||
// Just "test" left
|
||||
expect((await primary.listPages()).size).toBe(1);
|
||||
expect((await secondary.listPages()).size).toBe(1);
|
||||
|
||||
// No-op
|
||||
expect(await syncPages()).toBe(0);
|
||||
|
||||
await secondary.writePage("index", "I'm back");
|
||||
|
||||
await syncPages();
|
||||
|
||||
expect((await primary.readPage("index")).text).toBe("I'm back");
|
||||
|
||||
// Cause a conflict
|
||||
await primary.writePage("index", "Hello 1");
|
||||
await secondary.writePage("index", "Hello 2");
|
||||
|
||||
await syncPages(SpaceSync.primaryConflictResolver(primary, secondary));
|
||||
|
||||
// Sync conflicting copy back
|
||||
await syncPages();
|
||||
|
||||
// Verify that primary won
|
||||
expect((await primary.readPage("index")).text).toBe("Hello 1");
|
||||
expect((await secondary.readPage("index")).text).toBe("Hello 1");
|
||||
|
||||
// test + index + index.conflicting copy
|
||||
expect((await primary.listPages()).size).toBe(3);
|
||||
expect((await secondary.listPages()).size).toBe(3);
|
||||
|
||||
async function syncPages(
|
||||
conflictResolver?: (
|
||||
pageMeta1: PageMeta,
|
||||
pageMeta2: PageMeta
|
||||
) => Promise<void>
|
||||
): Promise<number> {
|
||||
// Awesome practice: adding sleeps to fix issues!
|
||||
await sleep(2);
|
||||
let n = await sync.syncPages(conflictResolver);
|
||||
await sleep(2);
|
||||
return n;
|
||||
}
|
||||
});
|
||||
|
||||
function sleep(ms: number = 5): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
setTimeout(resolve, ms);
|
||||
});
|
||||
}
|
||||
@@ -1,208 +0,0 @@
|
||||
import { Space } from "./space";
|
||||
import { PageMeta } from "../types";
|
||||
import { SpacePrimitives } from "./space_primitives";
|
||||
|
||||
export class SpaceSync {
|
||||
constructor(
|
||||
private primary: Space,
|
||||
private secondary: Space,
|
||||
public primaryLastSync: number,
|
||||
public secondaryLastSync: number,
|
||||
private trashPrefix: string
|
||||
) {}
|
||||
|
||||
// Strategy: Primary wins
|
||||
public static primaryConflictResolver(
|
||||
primary: Space,
|
||||
secondary: Space
|
||||
): (pageMeta1: PageMeta, pageMeta2: PageMeta) => Promise<void> {
|
||||
return async (pageMeta1, pageMeta2) => {
|
||||
const pageName = pageMeta1.name;
|
||||
const revisionPageName = `${pageName}.conflicted.${pageMeta2.lastModified}`;
|
||||
// Copy secondary to conflict copy
|
||||
let oldPageData = await secondary.readPage(pageName);
|
||||
await secondary.writePage(revisionPageName, oldPageData.text);
|
||||
|
||||
// Write replacement on top
|
||||
let newPageData = await primary.readPage(pageName);
|
||||
await secondary.writePage(
|
||||
pageName,
|
||||
newPageData.text,
|
||||
true,
|
||||
newPageData.meta.lastModified
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
async syncablePages(
|
||||
space: Space
|
||||
): Promise<{ pages: PageMeta[]; nowTimestamp: number }> {
|
||||
let fetchResult = await space.fetchPageList();
|
||||
return {
|
||||
pages: [...fetchResult.pages].filter(
|
||||
(pageMeta) => !pageMeta.name.startsWith(this.trashPrefix)
|
||||
),
|
||||
nowTimestamp: fetchResult.nowTimestamp,
|
||||
};
|
||||
}
|
||||
|
||||
async trashPages(space: SpacePrimitives): Promise<PageMeta[]> {
|
||||
return [...(await space.fetchPageList()).pages]
|
||||
.filter((pageMeta) => pageMeta.name.startsWith(this.trashPrefix))
|
||||
.map((pageMeta) => ({
|
||||
...pageMeta,
|
||||
name: pageMeta.name.substring(this.trashPrefix.length),
|
||||
}));
|
||||
}
|
||||
|
||||
async syncPages(
|
||||
conflictResolver?: (
|
||||
pageMeta1: PageMeta,
|
||||
pageMeta2: PageMeta
|
||||
) => Promise<void>
|
||||
): Promise<number> {
|
||||
let syncOps = 0;
|
||||
|
||||
let { pages: primaryAllPagesSet, nowTimestamp: primarySyncTimestamp } =
|
||||
await this.syncablePages(this.primary);
|
||||
let allPagesPrimary = new Map(primaryAllPagesSet.map((p) => [p.name, p]));
|
||||
let { pages: secondaryAllPagesSet, nowTimestamp: secondarySyncTimestamp } =
|
||||
await this.syncablePages(this.secondary);
|
||||
let allPagesSecondary = new Map(
|
||||
secondaryAllPagesSet.map((p) => [p.name, p])
|
||||
);
|
||||
|
||||
let allTrashPrimary = new Map(
|
||||
(await this.trashPages(this.primary))
|
||||
// Filter out old trash
|
||||
.filter((p) => p.lastModified > this.primaryLastSync)
|
||||
.map((p) => [p.name, p])
|
||||
);
|
||||
let allTrashSecondary = new Map(
|
||||
(await this.trashPages(this.secondary))
|
||||
// Filter out old trash
|
||||
.filter((p) => p.lastModified > this.secondaryLastSync)
|
||||
.map((p) => [p.name, p])
|
||||
);
|
||||
|
||||
// Iterate over all pages on the primary first
|
||||
for (let [name, pageMetaPrimary] of allPagesPrimary.entries()) {
|
||||
let pageMetaSecondary = allPagesSecondary.get(pageMetaPrimary.name);
|
||||
if (!pageMetaSecondary) {
|
||||
// New page on primary
|
||||
// Let's check it's not on the deleted list
|
||||
if (allTrashSecondary.has(name)) {
|
||||
// Explicitly deleted, let's skip
|
||||
continue;
|
||||
}
|
||||
|
||||
// Push from primary to secondary
|
||||
console.log("New page on primary", name, "syncing to secondary");
|
||||
let pageData = await this.primary.readPage(name);
|
||||
await this.secondary.writePage(
|
||||
name,
|
||||
pageData.text,
|
||||
true,
|
||||
secondarySyncTimestamp // The reason for this is to not include it in the next sync cycle, we cannot blindly use the lastModified date due to time skew
|
||||
);
|
||||
syncOps++;
|
||||
} else {
|
||||
// Existing page
|
||||
if (pageMetaPrimary.lastModified > this.primaryLastSync) {
|
||||
// Primary updated since last sync
|
||||
if (pageMetaSecondary.lastModified > this.secondaryLastSync) {
|
||||
// Secondary also updated! CONFLICT
|
||||
if (conflictResolver) {
|
||||
await conflictResolver(pageMetaPrimary, pageMetaSecondary);
|
||||
} else {
|
||||
throw Error(
|
||||
`Sync conflict for ${name} with no conflict resolver specified`
|
||||
);
|
||||
}
|
||||
} else {
|
||||
// Ok, not changed on secondary, push it secondary
|
||||
console.log(
|
||||
"Changed page on primary",
|
||||
name,
|
||||
"syncing to secondary"
|
||||
);
|
||||
let pageData = await this.primary.readPage(name);
|
||||
await this.secondary.writePage(
|
||||
name,
|
||||
pageData.text,
|
||||
false,
|
||||
secondarySyncTimestamp
|
||||
);
|
||||
syncOps++;
|
||||
}
|
||||
} else if (pageMetaSecondary.lastModified > this.secondaryLastSync) {
|
||||
// Secondary updated, but not primary (checked above)
|
||||
// Push from secondary to primary
|
||||
console.log("Changed page on secondary", name, "syncing to primary");
|
||||
let pageData = await this.secondary.readPage(name);
|
||||
await this.primary.writePage(
|
||||
name,
|
||||
pageData.text,
|
||||
false,
|
||||
primarySyncTimestamp
|
||||
);
|
||||
syncOps++;
|
||||
} else {
|
||||
// Neither updated, no-op
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Now do a simplified version in reverse, only detecting new pages
|
||||
for (let [name, pageMetaSecondary] of allPagesSecondary.entries()) {
|
||||
if (!allPagesPrimary.has(pageMetaSecondary.name)) {
|
||||
// New page on secondary
|
||||
// Let's check it's not on the deleted list
|
||||
if (allTrashPrimary.has(name)) {
|
||||
// Explicitly deleted, let's skip
|
||||
continue;
|
||||
}
|
||||
// Push from secondary to primary
|
||||
console.log("New page on secondary", name, "pushing to primary");
|
||||
let pageData = await this.secondary.readPage(name);
|
||||
await this.primary.writePage(
|
||||
name,
|
||||
pageData.text,
|
||||
false,
|
||||
primarySyncTimestamp
|
||||
);
|
||||
syncOps++;
|
||||
}
|
||||
}
|
||||
|
||||
// And finally, let's trash some pages
|
||||
for (let pageToDelete of allTrashPrimary.values()) {
|
||||
console.log("Deleting", pageToDelete.name, "on secondary");
|
||||
try {
|
||||
await this.secondary.deletePage(
|
||||
pageToDelete.name,
|
||||
secondarySyncTimestamp
|
||||
);
|
||||
syncOps++;
|
||||
} catch (e: any) {
|
||||
console.log("Page already gone", e.message);
|
||||
}
|
||||
}
|
||||
|
||||
for (let pageToDelete of allTrashSecondary.values()) {
|
||||
console.log("Deleting", pageToDelete.name, "on primary");
|
||||
try {
|
||||
await this.primary.deletePage(pageToDelete.name, primarySyncTimestamp);
|
||||
syncOps++;
|
||||
} catch (e: any) {
|
||||
console.log("Page already gone", e.message);
|
||||
}
|
||||
}
|
||||
|
||||
// Setting last sync time to the timestamps we got back when fetching the page lists on each end
|
||||
this.primaryLastSync = primarySyncTimestamp;
|
||||
this.secondaryLastSync = secondarySyncTimestamp;
|
||||
|
||||
return syncOps;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user