neil/silverbullet
1
0
Fork 0
Code Activity

Continuous sync (#320)

Browse Source 
* Continuous sync

* Mobile dep upgrade
This commit is contained in:
Zef Hemel
2023-01-20 16:08:01 +01:00
committed by GitHub
parent 3fb393baf3
commit 2577a2db32
17 changed files with 416 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/spaces/sync.test.ts
+5 -2
View File
@@ -130,9 +130,12 @@ Deno.test("Test store", async () => {
ternary,
new Map<string, SyncStatusItem>(),
);
console.log("N ops", await sync2.syncFiles());
console.log(
"N ops",
await sync2.syncFiles(SpaceSync.primaryConflictResolver),
);
await sleep(2);
assertEquals(await sync2.syncFiles(), 0);
assertEquals(await sync2.syncFiles(SpaceSync.primaryConflictResolver), 0);
await Deno.remove(primaryPath, { recursive: true });
await Deno.remove(secondaryPath, { recursive: true });
common/spaces/sync.ts
+172 -155
View File
@@ -30,7 +30,7 @@ export class SpaceSync {
) {}
async syncFiles(
conflictResolver?: (
conflictResolver: (
name: string,
snapshot: Map<string, SyncStatusItem>,
primarySpace: SpacePrimitives,
@@ -65,160 +65,12 @@ export class SpaceSync {
this.logger.log("info", "Iterating over all files");
for (const name of allFilesToProcess) {
if (
primaryFileMap.has(name) && !secondaryFileMap.has(name) &&
!this.snapshot.has(name)
) {
// New file, created on primary, copy from primary to secondary
this.logger.log(
"info",
"New file created on primary, copying to secondary",
name,
);
const { data } = await this.primary.readFile(name, "arraybuffer");
const writtenMeta = await this.secondary.writeFile(
name,
"arraybuffer",
data,
);
this.snapshot.set(name, [
primaryFileMap.get(name)!,
writtenMeta.lastModified,
]);
operations++;
} else if (
secondaryFileMap.has(name) && !primaryFileMap.has(name) &&
!this.snapshot.has(name)
) {
// New file, created on secondary, copy from secondary to primary
this.logger.log(
"info",
"New file created on secondary, copying from secondary to primary",
name,
);
const { data } = await this.secondary.readFile(name, "arraybuffer");
const writtenMeta = await this.primary.writeFile(
name,
"arraybuffer",
data,
);
this.snapshot.set(name, [
writtenMeta.lastModified,
secondaryFileMap.get(name)!,
]);
operations++;
} else if (
primaryFileMap.has(name) && this.snapshot.has(name) &&
!secondaryFileMap.has(name)
) {
// File deleted on B
this.logger.log(
"info",
"File deleted on secondary, deleting from primary",
name,
);
await this.primary.deleteFile(name);
this.snapshot.delete(name);
operations++;
} else if (
secondaryFileMap.has(name) && this.snapshot.has(name) &&
!primaryFileMap.has(name)
) {
// File deleted on A
this.logger.log(
"info",
"File deleted on primary, deleting from secondary",
name,
);
await this.secondary.deleteFile(name);
this.snapshot.delete(name);
operations++;
} else if (
this.snapshot.has(name) && !primaryFileMap.has(name) &&
!secondaryFileMap.has(name)
) {
// File deleted on both sides, :shrug:
this.logger.log(
"info",
"File deleted on both ends, deleting from status",
name,
);
this.snapshot.delete(name);
operations++;
} else if (
primaryFileMap.has(name) && secondaryFileMap.has(name) &&
this.snapshot.get(name) &&
primaryFileMap.get(name) !== this.snapshot.get(name)![0] &&
secondaryFileMap.get(name) === this.snapshot.get(name)![1]
) {
// File has changed on primary, but not secondary: copy from primary to secondary
this.logger.log(
"info",
"File changed on primary, copying to secondary",
name,
);
const { data } = await this.primary.readFile(name, "arraybuffer");
const writtenMeta = await this.secondary.writeFile(
name,
"arraybuffer",
data,
);
this.snapshot.set(name, [
primaryFileMap.get(name)!,
writtenMeta.lastModified,
]);
operations++;
} else if (
primaryFileMap.has(name) && secondaryFileMap.has(name) &&
this.snapshot.get(name) &&
secondaryFileMap.get(name) !== this.snapshot.get(name)![1] &&
primaryFileMap.get(name) === this.snapshot.get(name)![0]
) {
// File has changed on secondary, but not primary: copy from secondary to primary
const { data } = await this.secondary.readFile(name, "arraybuffer");
const writtenMeta = await this.primary.writeFile(
name,
"arraybuffer",
data,
);
this.snapshot.set(name, [
writtenMeta.lastModified,
secondaryFileMap.get(name)!,
]);
operations++;
} else if (
( // File changed on both ends, but we don't have any info in the snapshot (resync scenario?): have to run through conflict handling
primaryFileMap.has(name) && secondaryFileMap.has(name) &&
!this.snapshot.has(name)
) ||
( // File changed on both ends, CONFLICT!
primaryFileMap.has(name) && secondaryFileMap.has(name) &&
this.snapshot.get(name) &&
secondaryFileMap.get(name) !== this.snapshot.get(name)![1] &&
primaryFileMap.get(name) !== this.snapshot.get(name)![0]
)
) {
this.logger.log(
"info",
"File changed on both ends, potential conflict",
name,
);
if (conflictResolver) {
operations += await conflictResolver(
name,
this.snapshot,
this.primary,
this.secondary,
this.logger,
);
} else {
throw Error(
`Sync conflict for ${name} with no conflict resolver specified`,
);
}
} else {
// Nothing needs to happen
}
operations += await this.syncFile(
name,
primaryFileMap.get(name),
secondaryFileMap.get(name),
conflictResolver,
);
}
} catch (e: any) {
this.logger.log("error", "Sync error:", e.message);
@@ -229,6 +81,171 @@ export class SpaceSync {
return operations;
}
async syncFile(
name: string,
primaryHash: SyncHash | undefined,
secondaryHash: SyncHash | undefined,
conflictResolver: (
name: string,
snapshot: Map<string, SyncStatusItem>,
primarySpace: SpacePrimitives,
secondarySpace: SpacePrimitives,
logger: Logger,
) => Promise<number>,
): Promise<number> {
let operations = 0;
if (
primaryHash && !secondaryHash &&
!this.snapshot.has(name)
) {
// New file, created on primary, copy from primary to secondary
this.logger.log(
"info",
"New file created on primary, copying to secondary",
name,
);
const { data } = await this.primary.readFile(name, "arraybuffer");
const writtenMeta = await this.secondary.writeFile(
name,
"arraybuffer",
data,
);
this.snapshot.set(name, [
primaryHash,
writtenMeta.lastModified,
]);
operations++;
} else if (
secondaryHash && !primaryHash &&
!this.snapshot.has(name)
) {
// New file, created on secondary, copy from secondary to primary
this.logger.log(
"info",
"New file created on secondary, copying from secondary to primary",
name,
);
const { data } = await this.secondary.readFile(name, "arraybuffer");
const writtenMeta = await this.primary.writeFile(
name,
"arraybuffer",
data,
);
this.snapshot.set(name, [
writtenMeta.lastModified,
secondaryHash,
]);
operations++;
} else if (
primaryHash && this.snapshot.has(name) &&
!secondaryHash
) {
// File deleted on B
this.logger.log(
"info",
"File deleted on secondary, deleting from primary",
name,
);
await this.primary.deleteFile(name);
this.snapshot.delete(name);
operations++;
} else if (
secondaryHash && this.snapshot.has(name) &&
!primaryHash
) {
// File deleted on A
this.logger.log(
"info",
"File deleted on primary, deleting from secondary",
name,
);
await this.secondary.deleteFile(name);
this.snapshot.delete(name);
operations++;
} else if (
this.snapshot.has(name) && !primaryHash &&
!secondaryHash
) {
// File deleted on both sides, :shrug:
this.logger.log(
"info",
"File deleted on both ends, deleting from status",
name,
);
this.snapshot.delete(name);
operations++;
} else if (
primaryHash && secondaryHash &&
this.snapshot.get(name) &&
primaryHash !== this.snapshot.get(name)![0] &&
secondaryHash === this.snapshot.get(name)![1]
) {
// File has changed on primary, but not secondary: copy from primary to secondary
this.logger.log(
"info",
"File changed on primary, copying to secondary",
name,
);
const { data } = await this.primary.readFile(name, "arraybuffer");
const writtenMeta = await this.secondary.writeFile(
name,
"arraybuffer",
data,
);
this.snapshot.set(name, [
primaryHash,
writtenMeta.lastModified,
]);
operations++;
} else if (
primaryHash && secondaryHash &&
this.snapshot.get(name) &&
secondaryHash !== this.snapshot.get(name)![1] &&
primaryHash === this.snapshot.get(name)![0]
) {
// File has changed on secondary, but not primary: copy from secondary to primary
const { data } = await this.secondary.readFile(name, "arraybuffer");
const writtenMeta = await this.primary.writeFile(
name,
"arraybuffer",
data,
);
this.snapshot.set(name, [
writtenMeta.lastModified,
secondaryHash,
]);
operations++;
} else if (
( // File changed on both ends, but we don't have any info in the snapshot (resync scenario?): have to run through conflict handling
primaryHash && secondaryHash &&
!this.snapshot.has(name)
) ||
( // File changed on both ends, CONFLICT!
primaryHash && secondaryHash &&
this.snapshot.get(name) &&
secondaryHash !== this.snapshot.get(name)![1] &&
primaryHash !== this.snapshot.get(name)![0]
)
) {
this.logger.log(
"info",
"File changed on both ends, potential conflict",
name,
);
operations += await conflictResolver(
name,
this.snapshot,
this.primary,
this.secondary,
this.logger,
);
} else {
// Nothing needs to happen
}
return operations;
}
// Strategy: Primary wins
public static async primaryConflictResolver(
name: string,