1
0
silverbullet/common/spaces/sync.ts
2023-01-13 15:41:29 +01:00

272 lines
9.1 KiB
TypeScript

import type { FileMeta } from "../types.ts";
import { SpacePrimitives } from "./space_primitives.ts";
type SyncHash = number;
// Tuple where the first value represents a lastModified timestamp for the primary space
// and the second item the lastModified value of the secondary space
export type SyncStatusItem = [SyncHash, SyncHash];
// Implementation of this algorithm https://unterwaditzer.net/2016/sync-algorithm.html
export class SpaceSync {
constructor(
private primary: SpacePrimitives,
private secondary: SpacePrimitives,
readonly snapshot: Map<string, SyncStatusItem>,
) {}
async syncFiles(
conflictResolver?: (
name: string,
snapshot: Map<string, SyncStatusItem>,
primarySpace: SpacePrimitives,
secondarySpace: SpacePrimitives,
) => Promise<void>,
): Promise<number> {
let operations = 0;
console.log("Fetching snapshot from primary");
const primaryAllPages = this.syncCandidates(
await this.primary.fetchFileList(),
);
console.log("Fetching snapshot from secondary");
try {
const secondaryAllPages = this.syncCandidates(
await this.secondary.fetchFileList(),
);
const primaryFileMap = new Map<string, SyncHash>(
primaryAllPages.map((m) => [m.name, m.lastModified]),
);
const secondaryFileMap = new Map<string, SyncHash>(
secondaryAllPages.map((m) => [m.name, m.lastModified]),
);
const allFilesToProcess = new Set([
...this.snapshot.keys(),
...primaryFileMap.keys(),
...secondaryFileMap.keys(),
]);
console.log("Iterating over all files");
for (const name of allFilesToProcess) {
if (
primaryFileMap.has(name) && !secondaryFileMap.has(name) &&
!this.snapshot.has(name)
) {
// New file, created on primary, copy from primary to secondary
console.log(
"New file created on primary, copying to secondary",
name,
);
const { data } = await this.primary.readFile(name, "arraybuffer");
const writtenMeta = await this.secondary.writeFile(
name,
"arraybuffer",
data,
);
this.snapshot.set(name, [
primaryFileMap.get(name)!,
writtenMeta.lastModified,
]);
operations++;
} else if (
secondaryFileMap.has(name) && !primaryFileMap.has(name) &&
!this.snapshot.has(name)
) {
// New file, created on secondary, copy from secondary to primary
console.log(
"New file created on secondary, copying from secondary to primary",
name,
);
const { data } = await this.secondary.readFile(name, "arraybuffer");
const writtenMeta = await this.primary.writeFile(
name,
"arraybuffer",
data,
);
this.snapshot.set(name, [
writtenMeta.lastModified,
secondaryFileMap.get(name)!,
]);
operations++;
} else if (
primaryFileMap.has(name) && this.snapshot.has(name) &&
!secondaryFileMap.has(name)
) {
// File deleted on B
console.log("File deleted on secondary, deleting from primary", name);
await this.primary.deleteFile(name);
this.snapshot.delete(name);
operations++;
} else if (
secondaryFileMap.has(name) && this.snapshot.has(name) &&
!primaryFileMap.has(name)
) {
// File deleted on A
console.log("File deleted on primary, deleting from secondary", name);
await this.secondary.deleteFile(name);
this.snapshot.delete(name);
operations++;
} else if (
this.snapshot.has(name) && !primaryFileMap.has(name) &&
!secondaryFileMap.has(name)
) {
// File deleted on both sides, :shrug:
console.log("File deleted on both ends, deleting from status", name);
this.snapshot.delete(name);
operations++;
} else if (
primaryFileMap.has(name) && secondaryFileMap.has(name) &&
this.snapshot.get(name) &&
primaryFileMap.get(name) !== this.snapshot.get(name)![0] &&
secondaryFileMap.get(name) === this.snapshot.get(name)![1]
) {
// File has changed on primary, but not secondary: copy from primary to secondary
console.log("File changed on primary, copying to secondary", name);
const { data } = await this.primary.readFile(name, "arraybuffer");
const writtenMeta = await this.secondary.writeFile(
name,
"arraybuffer",
data,
);
this.snapshot.set(name, [
primaryFileMap.get(name)!,
writtenMeta.lastModified,
]);
operations++;
} else if (
primaryFileMap.has(name) && secondaryFileMap.has(name) &&
this.snapshot.get(name) &&
secondaryFileMap.get(name) !== this.snapshot.get(name)![1] &&
primaryFileMap.get(name) === this.snapshot.get(name)![0]
) {
// File has changed on secondary, but not primary: copy from secondary to primary
const { data } = await this.secondary.readFile(name, "arraybuffer");
const writtenMeta = await this.primary.writeFile(
name,
"arraybuffer",
data,
);
this.snapshot.set(name, [
writtenMeta.lastModified,
secondaryFileMap.get(name)!,
]);
operations++;
} else if (
( // File changed on both ends, but we don't have any info in the snapshot (resync scenario?): have to run through conflict handling
primaryFileMap.has(name) && secondaryFileMap.has(name) &&
!this.snapshot.has(name)
) ||
( // File changed on both ends, CONFLICT!
primaryFileMap.has(name) && secondaryFileMap.has(name) &&
this.snapshot.get(name) &&
secondaryFileMap.get(name) !== this.snapshot.get(name)![1] &&
primaryFileMap.get(name) !== this.snapshot.get(name)![0]
)
) {
console.log("File changed on both ends, conflict!", name);
if (conflictResolver) {
await conflictResolver(
name,
this.snapshot,
this.primary,
this.secondary,
);
} else {
throw Error(
`Sync conflict for ${name} with no conflict resolver specified`,
);
}
operations++;
} else {
// Nothing needs to happen
}
}
} catch (e: any) {
console.error("Boom", e.message);
throw e;
}
return operations;
}
// Strategy: Primary wins
public static async primaryConflictResolver(
name: string,
snapshot: Map<string, SyncStatusItem>,
primary: SpacePrimitives,
secondary: SpacePrimitives,
): Promise<void> {
console.log("Hit a conflict for", name);
const filePieces = name.split(".");
const fileNameBase = filePieces.slice(0, -1).join(".");
const fileNameExt = filePieces[filePieces.length - 1];
const pageData1 = await primary.readFile(name, "arraybuffer");
const pageData2 = await secondary.readFile(name, "arraybuffer");
let byteWiseMatch = true;
const arrayBuffer1 = new Uint8Array(pageData1.data as ArrayBuffer);
const arrayBuffer2 = new Uint8Array(pageData2.data as ArrayBuffer);
if (arrayBuffer1.byteLength !== arrayBuffer2.byteLength) {
byteWiseMatch = false;
}
if (byteWiseMatch) {
// Byte-wise comparison
for (let i = 0; i < arrayBuffer1.byteLength; i++) {
if (arrayBuffer1[i] !== arrayBuffer2[i]) {
byteWiseMatch = false;
break;
}
}
// Byte wise they're still the same, so no confict
if (byteWiseMatch) {
snapshot.set(name, [
pageData1.meta.lastModified,
pageData2.meta.lastModified,
]);
return;
}
}
const revisionFileName = filePieces.length === 1
? `${name}.conflicted.${pageData2.meta.lastModified}`
: `${fileNameBase}.conflicted.${pageData2.meta.lastModified}.${fileNameExt}`;
console.log(
"Going to create conflicting copy",
revisionFileName,
);
// Copy secondary to conflict copy
const localConflictMeta = await primary.writeFile(
revisionFileName,
"arraybuffer",
pageData2.data,
);
const remoteConflictMeta = await secondary.writeFile(
revisionFileName,
"arraybuffer",
pageData2.data,
);
// Updating snapshot
snapshot.set(revisionFileName, [
localConflictMeta.lastModified,
remoteConflictMeta.lastModified,
]);
// Write replacement on top
const writeMeta = await secondary.writeFile(
name,
"arraybuffer",
pageData1.data,
true,
);
snapshot.set(name, [pageData1.meta.lastModified, writeMeta.lastModified]);
}
syncCandidates(files: FileMeta[]): FileMeta[] {
return files.filter((f) => !f.name.startsWith("_plug/"));
}
}