neil/silverbullet
1
0
Fork 0
Code Activity

SilverBullet pivot to become an offline-first PWA (#403)

Browse Source
This commit is contained in:
Zef Hemel
2023-05-23 20:53:53 +02:00
committed by GitHub
parent b256269897
commit 5f484bed57
389 changed files with 4484 additions and 291129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/spaces/sync.ts
+131 -126
View File
@@ -10,52 +10,41 @@ type SyncHash = number;
// and the second item the lastModified value of the secondary space
export type SyncStatusItem = [SyncHash, SyncHash];
export interface Logger {
log(level: string, ...messageBits: any[]): void;
}
class ConsoleLogger implements Logger {
log(_level: string, ...messageBits: any[]) {
console.log(...messageBits);
}
}
export type SyncStatus = {
filesProcessed: number;
totalFiles: number;
snapshot: Map<string, SyncStatusItem>;
};
export type SyncOptions = {
logger?: Logger;
excludePrefixes?: string[];
conflictResolver: (
name: string,
snapshot: Map<string, SyncStatusItem>,
primarySpace: SpacePrimitives,
secondarySpace: SpacePrimitives,
) => Promise<number>;
isSyncCandidate?: (path: string) => boolean;
// Used to track progress, may want to pass more specific info later
onSyncProgress?: (syncStatus: SyncStatus) => void;
};
// Implementation of this algorithm https://unterwaditzer.net/2016/sync-algorithm.html
export class SpaceSync {
logger: ConsoleLogger;
excludePrefixes: string[];
constructor(
private primary: SpacePrimitives,
private secondary: SpacePrimitives,
readonly snapshot: Map<string, SyncStatusItem>,
readonly options: SyncOptions,
) {
this.logger = options.logger || new ConsoleLogger();
this.excludePrefixes = options.excludePrefixes || [];
}
async syncFiles(
conflictResolver: (
name: string,
snapshot: Map<string, SyncStatusItem>,
primarySpace: SpacePrimitives,
secondarySpace: SpacePrimitives,
logger: Logger,
) => Promise<number>,
): Promise<number> {
async syncFiles(snapshot: Map<string, SyncStatusItem>): Promise<number> {
let operations = 0;
this.logger.log("info", "Fetching snapshot from primary");
console.log("[sync]", "Fetching snapshot from primary");
const primaryAllPages = this.syncCandidates(
await this.primary.fetchFileList(),
);
this.logger.log("info", "Fetching snapshot from secondary");
console.log("[sync]", "Fetching snapshot from secondary");
try {
const secondaryAllPages = this.syncCandidates(
await this.secondary.fetchFileList(),
@@ -69,177 +58,188 @@ export class SpaceSync {
);
const allFilesToProcess = new Set([
...this.snapshot.keys(),
...snapshot.keys(),
...primaryFileMap.keys(),
...secondaryFileMap.keys(),
]);
this.logger.log("info", "Iterating over all files");
for (const name of allFilesToProcess) {
const sortedFilenames = [...allFilesToProcess];
sortedFilenames.sort((a) => {
// Just make sure that _plug/ files appear first
// This is important for the initial sync: plugs are loaded the moment they are pulled into the space,
// which would activate e.g. any indexing logic for the remaining space content
return a.startsWith("_plug/") ? -1 : 1;
});
// console.log("[sync]", "Iterating over all files");
let filesProcessed = 0;
for (const name of sortedFilenames) {
try {
operations += await this.syncFile(
snapshot,
name,
primaryFileMap.get(name),
secondaryFileMap.get(name),
conflictResolver,
);
filesProcessed++;
// Only report something significant
if (operations > 1 && this.options.onSyncProgress) {
this.options.onSyncProgress({
filesProcessed,
totalFiles: sortedFilenames.length,
snapshot,
});
}
} catch (e: any) {
this.logger.log("error", "Error syncing file", name, e.message);
console.log("error", "Error syncing file", name, e.message);
}
}
} catch (e: any) {
this.logger.log("error", "General sync error:", e.message);
console.log("error", "General sync error:", e.message);
throw e;
}
this.logger.log("info", "Sync complete, operations performed", operations);
console.log("[sync]", "Sync complete, operations performed", operations);
return operations;
}
async syncFile(
snapshot: Map<string, SyncStatusItem>,
name: string,
primaryHash: SyncHash | undefined,
secondaryHash: SyncHash | undefined,
conflictResolver: (
name: string,
snapshot: Map<string, SyncStatusItem>,
primarySpace: SpacePrimitives,
secondarySpace: SpacePrimitives,
logger: Logger,
) => Promise<number>,
): Promise<number> {
if (this.options.isSyncCandidate && !this.options.isSyncCandidate(name)) {
return 0;
}
// console.log("Syncing", name, primaryHash, secondaryHash);
let operations = 0;
// Check if not matching one of the excluded prefixes
for (const prefix of this.excludePrefixes) {
if (name.startsWith(prefix)) {
return operations;
}
}
if (
primaryHash !== undefined && secondaryHash === undefined &&
!this.snapshot.has(name)
!snapshot.has(name)
) {
// New file, created on primary, copy from primary to secondary
this.logger.log(
"info",
console.log(
"[sync]",
"New file created on primary, copying to secondary",
name,
);
const { data } = await this.primary.readFile(name, "arraybuffer");
const { data, meta } = await this.primary.readFile(name);
const writtenMeta = await this.secondary.writeFile(
name,
"arraybuffer",
data,
false,
meta.lastModified,
);
this.snapshot.set(name, [
snapshot.set(name, [
primaryHash,
writtenMeta.lastModified,
]);
operations++;
} else if (
secondaryHash !== undefined && primaryHash === undefined &&
!this.snapshot.has(name)
!snapshot.has(name)
) {
// New file, created on secondary, copy from secondary to primary
this.logger.log(
"info",
console.log(
"[sync]",
"New file created on secondary, copying from secondary to primary",
name,
);
const { data } = await this.secondary.readFile(name, "arraybuffer");
const { data, meta } = await this.secondary.readFile(name);
const writtenMeta = await this.primary.writeFile(
name,
"arraybuffer",
data,
false,
meta.lastModified,
);
this.snapshot.set(name, [
snapshot.set(name, [
writtenMeta.lastModified,
secondaryHash,
]);
operations++;
} else if (
primaryHash !== undefined && this.snapshot.has(name) &&
primaryHash !== undefined && snapshot.has(name) &&
secondaryHash === undefined
) {
// File deleted on B
this.logger.log(
"info",
console.log(
"[sync]",
"File deleted on secondary, deleting from primary",
name,
);
await this.primary.deleteFile(name);
this.snapshot.delete(name);
snapshot.delete(name);
operations++;
} else if (
secondaryHash !== undefined && this.snapshot.has(name) &&
secondaryHash !== undefined && snapshot.has(name) &&
primaryHash === undefined
) {
// File deleted on A
this.logger.log(
"info",
console.log(
"[sync]",
"File deleted on primary, deleting from secondary",
name,
);
await this.secondary.deleteFile(name);
this.snapshot.delete(name);
snapshot.delete(name);
operations++;
} else if (
this.snapshot.has(name) && primaryHash === undefined &&
snapshot.has(name) && primaryHash === undefined &&
secondaryHash === undefined
) {
// File deleted on both sides, :shrug:
this.logger.log(
"info",
console.log(
"[sync]",
"File deleted on both ends, deleting from status",
name,
);
this.snapshot.delete(name);
snapshot.delete(name);
operations++;
} else if (
primaryHash !== undefined && secondaryHash !== undefined &&
this.snapshot.get(name) &&
primaryHash !== this.snapshot.get(name)![0] &&
secondaryHash === this.snapshot.get(name)![1]
snapshot.get(name) &&
primaryHash !== snapshot.get(name)![0] &&
secondaryHash === snapshot.get(name)![1]
) {
// File has changed on primary, but not secondary: copy from primary to secondary
this.logger.log(
"info",
console.log(
"[sync]",
"File changed on primary, copying to secondary",
name,
);
const { data } = await this.primary.readFile(name, "arraybuffer");
const { data, meta } = await this.primary.readFile(name);
const writtenMeta = await this.secondary.writeFile(
name,
"arraybuffer",
data,
false,
meta.lastModified,
);
this.snapshot.set(name, [
snapshot.set(name, [
primaryHash,
writtenMeta.lastModified,
]);
operations++;
} else if (
primaryHash !== undefined && secondaryHash !== undefined &&
this.snapshot.get(name) &&
secondaryHash !== this.snapshot.get(name)![1] &&
primaryHash === this.snapshot.get(name)![0]
snapshot.get(name) &&
secondaryHash !== snapshot.get(name)![1] &&
primaryHash === snapshot.get(name)![0]
) {
// File has changed on secondary, but not primary: copy from secondary to primary
this.logger.log(
"info",
console.log(
"[sync]",
"File has changed on secondary, but not primary: copy from secondary to primary",
name,
);
const { data } = await this.secondary.readFile(name, "arraybuffer");
const { data, meta } = await this.secondary.readFile(name);
const writtenMeta = await this.primary.writeFile(
name,
"arraybuffer",
data,
false,
meta.lastModified,
);
this.snapshot.set(name, [
snapshot.set(name, [
writtenMeta.lastModified,
secondaryHash,
]);
@@ -247,26 +247,25 @@ export class SpaceSync {
} else if (
( // File changed on both ends, but we don't have any info in the snapshot (resync scenario?): have to run through conflict handling
primaryHash !== undefined && secondaryHash !== undefined &&
!this.snapshot.has(name)
!snapshot.has(name)
) ||
( // File changed on both ends, CONFLICT!
primaryHash && secondaryHash &&
this.snapshot.get(name) &&
secondaryHash !== this.snapshot.get(name)![1] &&
primaryHash !== this.snapshot.get(name)![0]
snapshot.get(name) &&
secondaryHash !== snapshot.get(name)![1] &&
primaryHash !== snapshot.get(name)![0]
)
) {
this.logger.log(
"info",
console.log(
"[sync]",
"File changed on both ends, potential conflict",
name,
);
operations += await conflictResolver(
operations += await this.options.conflictResolver!(
name,
this.snapshot,
snapshot,
this.primary,
this.secondary,
this.logger,
);
} else {
// Nothing needs to happen
@@ -280,27 +279,29 @@ export class SpaceSync {
snapshot: Map<string, SyncStatusItem>,
primary: SpacePrimitives,
secondary: SpacePrimitives,
logger: Logger,
): Promise<number> {
logger.log("info", "Starting conflict resolution for", name);
console.log("[sync]", "Starting conflict resolution for", name);
const filePieces = name.split(".");
const fileNameBase = filePieces.slice(0, -1).join(".");
const fileNameExt = filePieces[filePieces.length - 1];
const pageData1 = await primary.readFile(name, "arraybuffer");
const pageData2 = await secondary.readFile(name, "arraybuffer");
const pageData1 = await primary.readFile(name);
const pageData2 = await secondary.readFile(name);
if (name.endsWith(".md")) {
logger.log("info", "File is markdown, using smart conflict resolution");
console.log(
"[sync]",
"File is markdown, using smart conflict resolution",
);
// Let's use a smartert check for markdown files, ignoring directive bodies
const pageText1 = removeDirectiveBody(
new TextDecoder().decode(pageData1.data as Uint8Array),
new TextDecoder().decode(pageData1.data),
);
const pageText2 = removeDirectiveBody(
new TextDecoder().decode(pageData2.data as Uint8Array),
new TextDecoder().decode(pageData2.data),
);
if (pageText1 === pageText2) {
logger.log(
"info",
console.log(
"[sync]",
"Files are the same (eliminating the directive bodies), no conflict",
);
snapshot.set(name, [
@@ -311,8 +312,8 @@ export class SpaceSync {
}
} else {
let byteWiseMatch = true;
const arrayBuffer1 = new Uint8Array(pageData1.data as ArrayBuffer);
const arrayBuffer2 = new Uint8Array(pageData2.data as ArrayBuffer);
const arrayBuffer1 = pageData1.data;
const arrayBuffer2 = pageData2.data;
if (arrayBuffer1.byteLength !== arrayBuffer2.byteLength) {
byteWiseMatch = false;
}
@@ -326,7 +327,8 @@ export class SpaceSync {
}
// Byte wise they're still the same, so no confict
if (byteWiseMatch) {
logger.log("info", "Files are the same, no conflict");
console.log("[sync]", "Files are the same, no conflict");
snapshot.set(name, [
pageData1.meta.lastModified,
pageData2.meta.lastModified,
@@ -335,11 +337,12 @@ export class SpaceSync {
}
}
}
let operations = 0;
const revisionFileName = filePieces.length === 1
? `${name}.conflicted.${pageData2.meta.lastModified}`
: `${fileNameBase}.conflicted.${pageData2.meta.lastModified}.${fileNameExt}`;
logger.log(
"info",
console.log(
"[sync]",
"Going to create conflicting copy",
revisionFileName,
);
@@ -347,14 +350,22 @@ export class SpaceSync {
// Copy secondary to conflict copy
const localConflictMeta = await primary.writeFile(
revisionFileName,
"arraybuffer",
pageData2.data,
);
operations++;
const remoteConflictMeta = await secondary.writeFile(
revisionFileName,
"arraybuffer",
pageData2.data,
);
operations++;
// Write replacement on top
const writeMeta = await secondary.writeFile(
name,
pageData1.data,
true,
);
operations++;
// Updating snapshot
snapshot.set(revisionFileName, [
@@ -362,22 +373,16 @@ export class SpaceSync {
remoteConflictMeta.lastModified,
]);
// Write replacement on top
const writeMeta = await secondary.writeFile(
name,
"arraybuffer",
pageData1.data,
true,
);
snapshot.set(name, [pageData1.meta.lastModified, writeMeta.lastModified]);
return 1;
return operations;
}
syncCandidates(files: FileMeta[]): FileMeta[] {
return files.filter((f) =>
!f.name.startsWith("_plug/") && f.lastModified > 0
);
if (this.options.isSyncCandidate) {
return files.filter((meta) => this.options.isSyncCandidate!(meta.name));
} else {
return files;
}
}
}