From d23846cdbf237725e7e9cb782044e5c3dd4b7b62 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Mon, 16 Jan 2023 18:55:35 +0100 Subject: [PATCH] Sync: no conflict when only directive bodies differ --- common/spaces/sync.test.ts | 38 +++++++++++++++-- common/spaces/sync.ts | 85 +++++++++++++++++++++++++++----------- plugs/core/core.plug.yaml | 1 - plugs/sync/sync.plug.yaml | 1 + 4 files changed, 98 insertions(+), 27 deletions(-) diff --git a/common/spaces/sync.test.ts b/common/spaces/sync.test.ts index b7bd102..a12281f 100644 --- a/common/spaces/sync.test.ts +++ b/common/spaces/sync.test.ts @@ -1,4 +1,4 @@ -import { SpaceSync, SyncStatusItem } from "./sync.ts"; +import { removeDirectiveBody, SpaceSync, SyncStatusItem } from "./sync.ts"; import { DiskSpacePrimitives } from "./disk_space_primitives.ts"; import { assertEquals } from "../../test_deps.ts"; @@ -100,11 +100,23 @@ Deno.test("Test store", async () => { await primary.writeFile("index", "utf8", "Hello 1"); await secondary.writeFile("index", "utf8", "Hello 1"); + // And two more files with different bodies, but only within a query directive — shouldn't conflict + await primary.writeFile( + "index.md", + "utf8", + "Hello\n\nHello 1\n", + ); + await secondary.writeFile( + "index.md", + "utf8", + "Hello\n\nHello 2\n", + ); + await doSync(); await doSync(); - // test + index + previous index.conflicting copy but nothing more - assertEquals((await primary.fetchFileList()).length, 3); + // test + index + index.md + previous index.conflicting copy but nothing more + assertEquals((await primary.fetchFileList()).length, 4); console.log("Bringing a third device in the mix"); @@ -141,3 +153,23 @@ function sleep(ms = 10): Promise { setTimeout(resolve, ms); }); } + +Deno.test("Remove directive bodies", () => { + assertEquals( + removeDirectiveBody(` +This is a body +bla bla bla + +Hello + +This is a body + +`), + ` + +Hello + + +`, + ); +}); diff --git a/common/spaces/sync.ts b/common/spaces/sync.ts index c5601ef..bc9d364 100644 --- a/common/spaces/sync.ts +++ b/common/spaces/sync.ts @@ -1,4 +1,6 @@ -import { LogEntry } from "../../plugos/sandbox.ts"; +import { renderToText, replaceNodesMatching } from "../../plug-api/lib/tree.ts"; +import buildMarkdown from "../markdown_parser/parser.ts"; +import { parse } from "../markdown_parser/parse_tree.ts"; import type { FileMeta } from "../types.ts"; import { SpacePrimitives } from "./space_primitives.ts"; @@ -34,7 +36,7 @@ export class SpaceSync { primarySpace: SpacePrimitives, secondarySpace: SpacePrimitives, logger: Logger, - ) => Promise, + ) => Promise, ): Promise { let operations = 0; this.logger.log("info", "Fetching snapshot from primary"); @@ -202,7 +204,7 @@ export class SpaceSync { name, ); if (conflictResolver) { - await conflictResolver( + operations += await conflictResolver( name, this.snapshot, this.primary, @@ -214,7 +216,6 @@ export class SpaceSync { `Sync conflict for ${name} with no conflict resolver specified`, ); } - operations++; } else { // Nothing needs to happen } @@ -235,7 +236,7 @@ export class SpaceSync { primary: SpacePrimitives, secondary: SpacePrimitives, logger: Logger, - ): Promise { + ): Promise { logger.log("info", "Starting conflict resolution for", name); const filePieces = name.split("."); const fileNameBase = filePieces.slice(0, -1).join("."); @@ -243,28 +244,50 @@ export class SpaceSync { const pageData1 = await primary.readFile(name, "arraybuffer"); const pageData2 = await secondary.readFile(name, "arraybuffer"); - let byteWiseMatch = true; - const arrayBuffer1 = new Uint8Array(pageData1.data as ArrayBuffer); - const arrayBuffer2 = new Uint8Array(pageData2.data as ArrayBuffer); - if (arrayBuffer1.byteLength !== arrayBuffer2.byteLength) { - byteWiseMatch = false; - } - if (byteWiseMatch) { - // Byte-wise comparison - for (let i = 0; i < arrayBuffer1.byteLength; i++) { - if (arrayBuffer1[i] !== arrayBuffer2[i]) { - byteWiseMatch = false; - break; - } - } - // Byte wise they're still the same, so no confict - if (byteWiseMatch) { - logger.log("info", "Files are the same, no conflict"); + if (name.endsWith(".md")) { + logger.log("info", "File is markdown, using smart conflict resolution"); + // Let's use a smartert check for markdown files, ignoring directive bodies + const pageText1 = removeDirectiveBody( + new TextDecoder().decode(pageData1.data as Uint8Array), + ); + const pageText2 = removeDirectiveBody( + new TextDecoder().decode(pageData2.data as Uint8Array), + ); + if (pageText1 === pageText2) { + logger.log( + "info", + "Files are the same (eliminating the directive bodies), no conflict", + ); snapshot.set(name, [ pageData1.meta.lastModified, pageData2.meta.lastModified, ]); - return; + return 0; + } + } else { + let byteWiseMatch = true; + const arrayBuffer1 = new Uint8Array(pageData1.data as ArrayBuffer); + const arrayBuffer2 = new Uint8Array(pageData2.data as ArrayBuffer); + if (arrayBuffer1.byteLength !== arrayBuffer2.byteLength) { + byteWiseMatch = false; + } + if (byteWiseMatch) { + // Byte-wise comparison + for (let i = 0; i < arrayBuffer1.byteLength; i++) { + if (arrayBuffer1[i] !== arrayBuffer2[i]) { + byteWiseMatch = false; + break; + } + } + // Byte wise they're still the same, so no confict + if (byteWiseMatch) { + logger.log("info", "Files are the same, no conflict"); + snapshot.set(name, [ + pageData1.meta.lastModified, + pageData2.meta.lastModified, + ]); + return 0; + } } } const revisionFileName = filePieces.length === 1 @@ -303,9 +326,25 @@ export class SpaceSync { ); snapshot.set(name, [pageData1.meta.lastModified, writeMeta.lastModified]); + return 1; } syncCandidates(files: FileMeta[]): FileMeta[] { return files.filter((f) => !f.name.startsWith("_plug/")); } } + +const markdownLanguage = buildMarkdown([]); + +export function removeDirectiveBody(text: string): string { + // Parse + const tree = parse(markdownLanguage, text); + // Remove bodies + replaceNodesMatching(tree, (node) => { + if (node.type === "DirectiveBody") { + return null; + } + }); + // Turn back into text + return renderToText(tree); +} diff --git a/plugs/core/core.plug.yaml b/plugs/core/core.plug.yaml index 574ab5c..04f7c6f 100644 --- a/plugs/core/core.plug.yaml +++ b/plugs/core/core.plug.yaml @@ -414,7 +414,6 @@ functions: path: ./stats.ts:statsCommand command: name: "Stats: Show" - key: "Shift-Alt-s" # Cloud pages readPageCloud: diff --git a/plugs/sync/sync.plug.yaml b/plugs/sync/sync.plug.yaml index 7f9c94c..c2fbfaf 100644 --- a/plugs/sync/sync.plug.yaml +++ b/plugs/sync/sync.plug.yaml @@ -9,6 +9,7 @@ functions: path: sync.ts:syncCommand command: name: "Sync: Sync" + key: "Shift-Alt-s" wipeAndSyncCommand: path: sync.ts:localWipeAndSyncCommand