Zef Hemel
@@ -0,0 +1,271 @@
|
||||
import type { FileMeta } from "../types.ts";
|
||||
import { SpacePrimitives } from "./space_primitives.ts";
|
||||
|
||||
type SyncHash = number;
|
||||
|
||||
// Tuple where the first value represents a lastModified timestamp for the primary space
|
||||
// and the second item the lastModified value of the secondary space
|
||||
export type SyncStatusItem = [SyncHash, SyncHash];
|
||||
|
||||
// Implementation of this algorithm https://unterwaditzer.net/2016/sync-algorithm.html
|
||||
export class SpaceSync {
|
||||
constructor(
|
||||
private primary: SpacePrimitives,
|
||||
private secondary: SpacePrimitives,
|
||||
readonly snapshot: Map<string, SyncStatusItem>,
|
||||
) {}
|
||||
|
||||
async syncFiles(
|
||||
conflictResolver?: (
|
||||
name: string,
|
||||
snapshot: Map<string, SyncStatusItem>,
|
||||
primarySpace: SpacePrimitives,
|
||||
secondarySpace: SpacePrimitives,
|
||||
) => Promise<void>,
|
||||
): Promise<number> {
|
||||
let operations = 0;
|
||||
console.log("Fetching snapshot from primary");
|
||||
const primaryAllPages = this.syncCandidates(
|
||||
await this.primary.fetchFileList(),
|
||||
);
|
||||
|
||||
console.log("Fetching snapshot from secondary");
|
||||
try {
|
||||
const secondaryAllPages = this.syncCandidates(
|
||||
await this.secondary.fetchFileList(),
|
||||
);
|
||||
|
||||
const primaryFileMap = new Map<string, SyncHash>(
|
||||
primaryAllPages.map((m) => [m.name, m.lastModified]),
|
||||
);
|
||||
const secondaryFileMap = new Map<string, SyncHash>(
|
||||
secondaryAllPages.map((m) => [m.name, m.lastModified]),
|
||||
);
|
||||
|
||||
const allFilesToProcess = new Set([
|
||||
...this.snapshot.keys(),
|
||||
...primaryFileMap.keys(),
|
||||
...secondaryFileMap.keys(),
|
||||
]);
|
||||
|
||||
console.log("Iterating over all files");
|
||||
for (const name of allFilesToProcess) {
|
||||
if (
|
||||
primaryFileMap.has(name) && !secondaryFileMap.has(name) &&
|
||||
!this.snapshot.has(name)
|
||||
) {
|
||||
// New file, created on primary, copy from primary to secondary
|
||||
console.log(
|
||||
"New file created on primary, copying to secondary",
|
||||
name,
|
||||
);
|
||||
const { data } = await this.primary.readFile(name, "arraybuffer");
|
||||
const writtenMeta = await this.secondary.writeFile(
|
||||
name,
|
||||
"arraybuffer",
|
||||
data,
|
||||
);
|
||||
this.snapshot.set(name, [
|
||||
primaryFileMap.get(name)!,
|
||||
writtenMeta.lastModified,
|
||||
]);
|
||||
operations++;
|
||||
} else if (
|
||||
secondaryFileMap.has(name) && !primaryFileMap.has(name) &&
|
||||
!this.snapshot.has(name)
|
||||
) {
|
||||
// New file, created on secondary, copy from secondary to primary
|
||||
console.log(
|
||||
"New file created on secondary, copying from secondary to primary",
|
||||
name,
|
||||
);
|
||||
const { data } = await this.secondary.readFile(name, "arraybuffer");
|
||||
const writtenMeta = await this.primary.writeFile(
|
||||
name,
|
||||
"arraybuffer",
|
||||
data,
|
||||
);
|
||||
this.snapshot.set(name, [
|
||||
writtenMeta.lastModified,
|
||||
secondaryFileMap.get(name)!,
|
||||
]);
|
||||
operations++;
|
||||
} else if (
|
||||
primaryFileMap.has(name) && this.snapshot.has(name) &&
|
||||
!secondaryFileMap.has(name)
|
||||
) {
|
||||
// File deleted on B
|
||||
console.log("File deleted on secondary, deleting from primary", name);
|
||||
await this.primary.deleteFile(name);
|
||||
this.snapshot.delete(name);
|
||||
operations++;
|
||||
} else if (
|
||||
secondaryFileMap.has(name) && this.snapshot.has(name) &&
|
||||
!primaryFileMap.has(name)
|
||||
) {
|
||||
// File deleted on A
|
||||
console.log("File deleted on primary, deleting from secondary", name);
|
||||
await this.secondary.deleteFile(name);
|
||||
this.snapshot.delete(name);
|
||||
operations++;
|
||||
} else if (
|
||||
this.snapshot.has(name) && !primaryFileMap.has(name) &&
|
||||
!secondaryFileMap.has(name)
|
||||
) {
|
||||
// File deleted on both sides, :shrug:
|
||||
console.log("File deleted on both ends, deleting from status", name);
|
||||
this.snapshot.delete(name);
|
||||
operations++;
|
||||
} else if (
|
||||
primaryFileMap.has(name) && secondaryFileMap.has(name) &&
|
||||
this.snapshot.get(name) &&
|
||||
primaryFileMap.get(name) !== this.snapshot.get(name)![0] &&
|
||||
secondaryFileMap.get(name) === this.snapshot.get(name)![1]
|
||||
) {
|
||||
// File has changed on primary, but not secondary: copy from primary to secondary
|
||||
console.log("File changed on primary, copying to secondary", name);
|
||||
const { data } = await this.primary.readFile(name, "arraybuffer");
|
||||
const writtenMeta = await this.secondary.writeFile(
|
||||
name,
|
||||
"arraybuffer",
|
||||
data,
|
||||
);
|
||||
this.snapshot.set(name, [
|
||||
primaryFileMap.get(name)!,
|
||||
writtenMeta.lastModified,
|
||||
]);
|
||||
operations++;
|
||||
} else if (
|
||||
primaryFileMap.has(name) && secondaryFileMap.has(name) &&
|
||||
this.snapshot.get(name) &&
|
||||
secondaryFileMap.get(name) !== this.snapshot.get(name)![1] &&
|
||||
primaryFileMap.get(name) === this.snapshot.get(name)![0]
|
||||
) {
|
||||
// File has changed on secondary, but not primary: copy from secondary to primary
|
||||
const { data } = await this.secondary.readFile(name, "arraybuffer");
|
||||
const writtenMeta = await this.primary.writeFile(
|
||||
name,
|
||||
"arraybuffer",
|
||||
data,
|
||||
);
|
||||
this.snapshot.set(name, [
|
||||
writtenMeta.lastModified,
|
||||
secondaryFileMap.get(name)!,
|
||||
]);
|
||||
operations++;
|
||||
} else if (
|
||||
( // File changed on both ends, but we don't have any info in the snapshot (resync scenario?): have to run through conflict handling
|
||||
primaryFileMap.has(name) && secondaryFileMap.has(name) &&
|
||||
!this.snapshot.has(name)
|
||||
) ||
|
||||
( // File changed on both ends, CONFLICT!
|
||||
primaryFileMap.has(name) && secondaryFileMap.has(name) &&
|
||||
this.snapshot.get(name) &&
|
||||
secondaryFileMap.get(name) !== this.snapshot.get(name)![1] &&
|
||||
primaryFileMap.get(name) !== this.snapshot.get(name)![0]
|
||||
)
|
||||
) {
|
||||
console.log("File changed on both ends, conflict!", name);
|
||||
if (conflictResolver) {
|
||||
await conflictResolver(
|
||||
name,
|
||||
this.snapshot,
|
||||
this.primary,
|
||||
this.secondary,
|
||||
);
|
||||
} else {
|
||||
throw Error(
|
||||
`Sync conflict for ${name} with no conflict resolver specified`,
|
||||
);
|
||||
}
|
||||
operations++;
|
||||
} else {
|
||||
// Nothing needs to happen
|
||||
}
|
||||
}
|
||||
} catch (e: any) {
|
||||
console.error("Boom", e.message);
|
||||
throw e;
|
||||
}
|
||||
|
||||
return operations;
|
||||
}
|
||||
|
||||
// Strategy: Primary wins
|
||||
public static async primaryConflictResolver(
|
||||
name: string,
|
||||
snapshot: Map<string, SyncStatusItem>,
|
||||
primary: SpacePrimitives,
|
||||
secondary: SpacePrimitives,
|
||||
): Promise<void> {
|
||||
console.log("Hit a conflict for", name);
|
||||
const filePieces = name.split(".");
|
||||
const fileNameBase = filePieces.slice(0, -1).join(".");
|
||||
const fileNameExt = filePieces[filePieces.length - 1];
|
||||
const pageData1 = await primary.readFile(name, "arraybuffer");
|
||||
const pageData2 = await secondary.readFile(name, "arraybuffer");
|
||||
|
||||
let byteWiseMatch = true;
|
||||
const arrayBuffer1 = new Uint8Array(pageData1.data as ArrayBuffer);
|
||||
const arrayBuffer2 = new Uint8Array(pageData2.data as ArrayBuffer);
|
||||
if (arrayBuffer1.byteLength !== arrayBuffer2.byteLength) {
|
||||
byteWiseMatch = false;
|
||||
}
|
||||
if (byteWiseMatch) {
|
||||
// Byte-wise comparison
|
||||
for (let i = 0; i < arrayBuffer1.byteLength; i++) {
|
||||
if (arrayBuffer1[i] !== arrayBuffer2[i]) {
|
||||
byteWiseMatch = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Byte wise they're still the same, so no confict
|
||||
if (byteWiseMatch) {
|
||||
snapshot.set(name, [
|
||||
pageData1.meta.lastModified,
|
||||
pageData2.meta.lastModified,
|
||||
]);
|
||||
return;
|
||||
}
|
||||
}
|
||||
const revisionFileName = filePieces.length === 1
|
||||
? `${name}.conflicted.${pageData2.meta.lastModified}`
|
||||
: `${fileNameBase}.conflicted.${pageData2.meta.lastModified}.${fileNameExt}`;
|
||||
console.log(
|
||||
"Going to create conflicting copy",
|
||||
revisionFileName,
|
||||
);
|
||||
|
||||
// Copy secondary to conflict copy
|
||||
const localConflictMeta = await primary.writeFile(
|
||||
revisionFileName,
|
||||
"arraybuffer",
|
||||
pageData2.data,
|
||||
);
|
||||
const remoteConflictMeta = await secondary.writeFile(
|
||||
revisionFileName,
|
||||
"arraybuffer",
|
||||
pageData2.data,
|
||||
);
|
||||
|
||||
// Updating snapshot
|
||||
snapshot.set(revisionFileName, [
|
||||
localConflictMeta.lastModified,
|
||||
remoteConflictMeta.lastModified,
|
||||
]);
|
||||
|
||||
// Write replacement on top
|
||||
const writeMeta = await secondary.writeFile(
|
||||
name,
|
||||
"arraybuffer",
|
||||
pageData1.data,
|
||||
true,
|
||||
);
|
||||
|
||||
snapshot.set(name, [pageData1.meta.lastModified, writeMeta.lastModified]);
|
||||
}
|
||||
|
||||
syncCandidates(files: FileMeta[]): FileMeta[] {
|
||||
return files.filter((f) => !f.name.startsWith("_plug/"));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user