1
0
silverbullet/web/sync_service.ts
2023-06-13 20:47:05 +02:00

385 lines
12 KiB
TypeScript

import { sleep } from "../common/async_util.ts";
import type { SpacePrimitives } from "../common/spaces/space_primitives.ts";
import {
SpaceSync,
SyncStatus,
SyncStatusItem,
} from "../common/spaces/sync.ts";
import { EventHook } from "../plugos/hooks/event.ts";
import { KVStore } from "../plugos/lib/kv_store.ts";
// Keeps the current sync snapshot
const syncSnapshotKey = "syncSnapshot";
// Keeps the start time of an ongoing sync, is reset once the sync is done
const syncStartTimeKey = "syncStartTime";
// Keeps the last time an activity was registered, used to detect if a sync is still alive and whether a new one should be started already
const syncLastActivityKey = "syncLastActivity";
const syncExcludePrefix = "syncExclude:";
// maximum time between two activities before we consider a sync crashed
const syncMaxIdleTimeout = 1000 * 20; // 20s
// How often to sync the whole space
const syncInterval = 10 * 1000; // Every 10s
/**
* The SyncService primarily wraps the SpaceSync engine but also coordinates sync between
* different browser tabs. It is using the KVStore to keep track of sync state.
*/
export class SyncService {
spaceSync: SpaceSync;
lastReportedSyncStatus = Date.now();
constructor(
readonly localSpacePrimitives: SpacePrimitives,
readonly remoteSpace: SpacePrimitives,
private kvStore: KVStore,
private eventHook: EventHook,
private isSyncCandidate: (path: string) => boolean,
) {
this.spaceSync = new SpaceSync(
this.localSpacePrimitives,
this.remoteSpace!,
{
conflictResolver: this.plugAwareConflictResolver.bind(this),
isSyncCandidate: this.isSyncCandidate,
onSyncProgress: (status) => {
this.registerSyncProgress(status).catch(console.error);
},
},
);
eventHook.addLocalListener("editor:pageLoaded", async (name) => {
await this.syncFile(`${name}.md`);
});
eventHook.addLocalListener("editor:pageSaved", async (name, meta) => {
const path = `${name}.md`;
await this.syncFile(path);
if (await this.isExcludedFromSync(path)) {
// So we're editing a page and just saved it, but it's excluded from sync
// Assumption: we're in collab mode for this file, so we're going to constantly update our local hash
// console.log(
// "Locally updating last modified in snapshot because we're in collab mode",
// );
await this.updateLocalLastModified(path, meta.lastModified);
}
});
}
async isSyncing(): Promise<boolean> {
const startTime = await this.kvStore.get(syncStartTimeKey);
if (!startTime) {
return false;
}
// Sync is running, but is it still alive?
const lastActivity = await this.kvStore.get(syncLastActivityKey)!;
if (Date.now() - lastActivity > syncMaxIdleTimeout) {
// It's been too long since the last activity, let's consider this one crashed and
// reset the sync start state
await this.kvStore.del(syncStartTimeKey);
return false;
}
return true;
}
async hasInitialSyncCompleted(): Promise<boolean> {
// Initial sync has happened when sync progress has been reported at least once, but the syncStartTime has been reset (which happens after sync finishes)
return !!(!(await this.kvStore.get(syncStartTimeKey)) &&
(await this.kvStore.get(syncLastActivityKey)));
}
async registerSyncStart(): Promise<void> {
// Assumption: this is called after an isSyncing() check
await this.kvStore.batchSet([{
key: syncStartTimeKey,
value: Date.now(),
}, {
key: syncLastActivityKey,
value: Date.now(),
}]);
}
async registerSyncProgress(status?: SyncStatus): Promise<void> {
// Emit a sync event at most every 10s
if (status && this.lastReportedSyncStatus < Date.now() - 10000) {
this.eventHook.dispatchEvent("sync:progress", status);
this.lastReportedSyncStatus = Date.now();
await this.saveSnapshot(status.snapshot);
}
await this.kvStore.set(syncLastActivityKey, Date.now());
}
async registerSyncStop(): Promise<void> {
await this.registerSyncProgress();
await this.kvStore.del(syncStartTimeKey);
}
// Temporarily exclude a specific file from sync (e.g. when in collab mode)
excludeFromSync(path: string): Promise<void> {
return this.kvStore.set(syncExcludePrefix + path, Date.now());
}
unExcludeFromSync(path: string): Promise<void> {
return this.kvStore.del(syncExcludePrefix + path);
}
async isExcludedFromSync(path: string): Promise<boolean> {
const lastExcluded = await this.kvStore.get(syncExcludePrefix + path);
return lastExcluded && Date.now() - lastExcluded < syncMaxIdleTimeout;
}
async fetchAllExcludedFromSync(): Promise<string[]> {
const entries = await this.kvStore.queryPrefix(syncExcludePrefix);
const expiredPaths: string[] = [];
const now = Date.now();
const result = entries.filter(({ key, value }) => {
if (now - value > syncMaxIdleTimeout) {
expiredPaths.push(key);
return false;
}
return true;
}).map(({ key }) => key.slice(syncExcludePrefix.length));
if (expiredPaths.length > 0) {
console.log("Purging expired sync exclusions: ", expiredPaths);
await this.kvStore.batchDelete(expiredPaths);
}
return result;
}
async getSnapshot(): Promise<Map<string, SyncStatusItem>> {
const snapshot = (await this.kvStore.get(syncSnapshotKey)) || {};
return new Map<string, SyncStatusItem>(
Object.entries(snapshot),
);
}
// Await a moment when the sync is no longer running
async noOngoingSync(): Promise<void> {
// Not completely safe, could have race condition on setting the syncStartTimeKey
while (await this.isSyncing()) {
await sleep(100);
}
}
// When in collab mode, we delegate the sync to the CDRT engine, to avoid conflicts, we try to keep the lastModified time in sync with the remote
async updateRemoteLastModified(path: string, lastModified: number) {
await this.noOngoingSync();
await this.registerSyncStart();
const snapshot = await this.getSnapshot();
const entry = snapshot.get(path);
if (entry) {
snapshot.set(path, [entry[0], lastModified]);
} else {
// In the unlikely scenario that a space first openen on a collab page before every being synced
try {
console.log(
"Received lastModified time for file not in snapshot",
path,
lastModified,
);
snapshot.set(path, [
(await this.localSpacePrimitives.getFileMeta(path)).lastModified,
lastModified,
]);
} catch (e) {
console.warn(
"Received lastModified time for non-existing file not in snapshot",
path,
lastModified,
e,
);
}
}
await this.saveSnapshot(snapshot);
await this.registerSyncStop();
}
// Reach out out to remote space, fetch the latest lastModified time and update the local snapshot
// This is used when exiting collab mode
async fetchAndPersistRemoteLastModified(path: string) {
const meta = await this.remoteSpace.getFileMeta(path);
await this.updateRemoteLastModified(
path,
meta.lastModified,
);
}
// When in collab mode, we delegate the sync to the CDRT engine, to avoid conflicts, we try to keep the lastModified time in sync when local changes happen
async updateLocalLastModified(path: string, lastModified: number) {
await this.noOngoingSync();
await this.registerSyncStart();
const snapshot = await this.getSnapshot();
const entry = snapshot.get(path);
if (entry) {
snapshot.set(path, [lastModified, entry[1]]);
} else {
// In the unlikely scenario that a space first openen on a collab page before every being synced
try {
console.log(
"Setting lastModified time for file not in snapshot",
path,
lastModified,
);
snapshot.set(path, [
lastModified,
(await this.localSpacePrimitives.getFileMeta(path)).lastModified,
]);
} catch (e) {
console.warn(
"Received lastModified time for non-existing file not in snapshot",
path,
lastModified,
e,
);
}
}
await this.saveSnapshot(snapshot);
await this.registerSyncStop();
// console.log("All done!");
}
start() {
this.syncSpace().catch(
console.error,
);
setInterval(async () => {
try {
const lastActivity = (await this.kvStore.get(syncLastActivityKey)) || 0;
if (lastActivity && Date.now() - lastActivity > syncInterval) {
// It's been a while since the last activity, let's sync the whole space
// The reason to do this check is that there may be multiple tabs open each with their sync cycle
await this.syncSpace();
}
} catch (e: any) {
console.error(e);
}
}, syncInterval / 2); // check every half the sync cycle because actually running the sync takes some time therefore we don't want to wait for the full cycle
}
async syncSpace(): Promise<number> {
if (await this.isSyncing()) {
console.log("Already syncing");
return 0;
}
await this.registerSyncStart();
let operations = 0;
const snapshot = await this.getSnapshot();
// Fetch the list of files that are excluded from sync (e.g. because they're in collab mode)
const excludedFromSync = await this.fetchAllExcludedFromSync();
// console.log("Excluded from sync", excludedFromSync);
try {
operations = await this.spaceSync!.syncFiles(
snapshot,
(path) =>
this.isSyncCandidate(path) && !excludedFromSync.includes(path),
);
this.eventHook.dispatchEvent("sync:success", operations);
} catch (e: any) {
this.eventHook.dispatchEvent("sync:error", e.message);
console.error("Sync error", e);
}
await this.saveSnapshot(snapshot);
await this.registerSyncStop();
return operations;
}
async syncFile(name: string) {
if (await this.isSyncing()) {
// console.log("Already syncing");
return;
}
if (!this.isSyncCandidate(name) || (await this.isExcludedFromSync(name))) {
return;
}
await this.registerSyncStart();
console.log("Syncing file", name);
const snapshot = await this.getSnapshot();
try {
let localHash: number | undefined;
let remoteHash: number | undefined;
try {
localHash =
(await this.localSpacePrimitives.getFileMeta(name)).lastModified;
} catch {
// Not present
}
try {
// This is wasteful, but Netlify (silverbullet.md) doesn't support OPTIONS call (404s) so we'll just fetch the whole file
remoteHash = (await this.remoteSpace!.readFile(name)).meta.lastModified;
} catch (e: any) {
if (e.message === "Not found") {
// File doesn't exist remotely, that's ok
} else {
throw e;
}
}
await this.spaceSync!.syncFile(snapshot, name, localHash, remoteHash);
this.eventHook.dispatchEvent("sync:success");
} catch (e: any) {
this.eventHook.dispatchEvent("sync:error", e.message);
console.error("Sync error", e);
}
await this.saveSnapshot(snapshot);
await this.registerSyncStop();
}
async saveSnapshot(snapshot: Map<string, SyncStatusItem>) {
await this.kvStore.set(syncSnapshotKey, Object.fromEntries(snapshot));
}
public async plugAwareConflictResolver(
name: string,
snapshot: Map<string, SyncStatusItem>,
primary: SpacePrimitives,
secondary: SpacePrimitives,
): Promise<number> {
if (!name.startsWith("_plug/")) {
const operations = await SpaceSync.primaryConflictResolver(
name,
snapshot,
primary,
secondary,
);
if (operations > 0) {
// Something happened -> conflict copy generated, let's report it
await this.eventHook.dispatchEvent("sync:conflict", name);
}
return operations;
}
console.log(
"[sync]",
"Conflict in plug",
name,
"will pick the version from secondary and be done with it.",
);
// Read file from secondary
const { data, meta } = await secondary.readFile(
name,
);
// Write file to primary
const newMeta = await primary.writeFile(
name,
data,
false,
meta.lastModified,
);
// Update snapshot
snapshot.set(name, [
newMeta.lastModified,
meta.lastModified,
]);
return 1;
}
}