Sync: no conflict when only directive bodies differ

common/spaces/sync.test.ts

@@ -1,4 +1,4 @@
-import { SpaceSync, SyncStatusItem } from "./sync.ts";
+import { removeDirectiveBody, SpaceSync, SyncStatusItem } from "./sync.ts";
 import { DiskSpacePrimitives } from "./disk_space_primitives.ts";
 import { assertEquals } from "../../test_deps.ts";
 
@@ -100,11 +100,23 @@ Deno.test("Test store", async () => {
   await primary.writeFile("index", "utf8", "Hello 1");
   await secondary.writeFile("index", "utf8", "Hello 1");
 
+  // And two more files with different bodies, but only within a query directive — shouldn't conflict
+  await primary.writeFile(
+    "index.md",
+    "utf8",
+    "Hello\n<!-- #query page -->\nHello 1\n<!-- /query -->",
+  );
+  await secondary.writeFile(
+    "index.md",
+    "utf8",
+    "Hello\n<!-- #query page -->\nHello 2\n<!-- /query -->",
+  );
+
   await doSync();
   await doSync();
 
-  // test + index + previous index.conflicting copy but nothing more
+  // test + index + index.md + previous index.conflicting copy but nothing more
-  assertEquals((await primary.fetchFileList()).length, 3);
+  assertEquals((await primary.fetchFileList()).length, 4);
 
   console.log("Bringing a third device in the mix");
 
@@ -141,3 +153,23 @@ function sleep(ms = 10): Promise<void> {
     setTimeout(resolve, ms);
   });
 }
+
+Deno.test("Remove directive bodies", () => {
+  assertEquals(
+    removeDirectiveBody(`<!-- #query page -->
+This is a body
+bla bla bla
+<!-- /query -->
+Hello
+<!-- #include [[test]] -->
+This is a body
+<!-- /include -->
+`),
+    `<!-- #query page -->
+<!-- /query -->
+Hello
+<!-- #include [[test]] -->
+<!-- /include -->
+`,
+  );
+});

common/spaces/sync.ts

@@ -1,4 +1,6 @@
-import { LogEntry } from "../../plugos/sandbox.ts";
+import { renderToText, replaceNodesMatching } from "../../plug-api/lib/tree.ts";
+import buildMarkdown from "../markdown_parser/parser.ts";
+import { parse } from "../markdown_parser/parse_tree.ts";
 import type { FileMeta } from "../types.ts";
 import { SpacePrimitives } from "./space_primitives.ts";
 
@@ -34,7 +36,7 @@ export class SpaceSync {
       primarySpace: SpacePrimitives,
       secondarySpace: SpacePrimitives,
       logger: Logger,
-    ) => Promise<void>,
+    ) => Promise<number>,
   ): Promise<number> {
     let operations = 0;
     this.logger.log("info", "Fetching snapshot from primary");
@@ -202,7 +204,7 @@ export class SpaceSync {
             name,
           );
           if (conflictResolver) {
-            await conflictResolver(
+            operations += await conflictResolver(
               name,
               this.snapshot,
               this.primary,
@@ -214,7 +216,6 @@ export class SpaceSync {
               `Sync conflict for ${name} with no conflict resolver specified`,
             );
           }
-          operations++;
         } else {
           // Nothing needs to happen
         }
@@ -235,7 +236,7 @@ export class SpaceSync {
     primary: SpacePrimitives,
     secondary: SpacePrimitives,
     logger: Logger,
-  ): Promise<void> {
+  ): Promise<number> {
     logger.log("info", "Starting conflict resolution for", name);
     const filePieces = name.split(".");
     const fileNameBase = filePieces.slice(0, -1).join(".");
@@ -243,28 +244,50 @@ export class SpaceSync {
     const pageData1 = await primary.readFile(name, "arraybuffer");
     const pageData2 = await secondary.readFile(name, "arraybuffer");
 
-    let byteWiseMatch = true;
+    if (name.endsWith(".md")) {
-    const arrayBuffer1 = new Uint8Array(pageData1.data as ArrayBuffer);
+      logger.log("info", "File is markdown, using smart conflict resolution");
-    const arrayBuffer2 = new Uint8Array(pageData2.data as ArrayBuffer);
+      // Let's use a smartert check for markdown files, ignoring directive bodies
-    if (arrayBuffer1.byteLength !== arrayBuffer2.byteLength) {
+      const pageText1 = removeDirectiveBody(
-      byteWiseMatch = false;
+        new TextDecoder().decode(pageData1.data as Uint8Array),
-    }
+      );
-    if (byteWiseMatch) {
+      const pageText2 = removeDirectiveBody(
-      // Byte-wise comparison
+        new TextDecoder().decode(pageData2.data as Uint8Array),
-      for (let i = 0; i < arrayBuffer1.byteLength; i++) {
+      );
-        if (arrayBuffer1[i] !== arrayBuffer2[i]) {
+      if (pageText1 === pageText2) {
-          byteWiseMatch = false;
+        logger.log(
-          break;
+          "info",
-        }
+          "Files are the same (eliminating the directive bodies), no conflict",
-      }
+        );
-      // Byte wise they're still the same, so no confict
-      if (byteWiseMatch) {
-        logger.log("info", "Files are the same, no conflict");
         snapshot.set(name, [
           pageData1.meta.lastModified,
           pageData2.meta.lastModified,
         ]);
-        return;
+        return 0;
+      }
+    } else {
+      let byteWiseMatch = true;
+      const arrayBuffer1 = new Uint8Array(pageData1.data as ArrayBuffer);
+      const arrayBuffer2 = new Uint8Array(pageData2.data as ArrayBuffer);
+      if (arrayBuffer1.byteLength !== arrayBuffer2.byteLength) {
+        byteWiseMatch = false;
+      }
+      if (byteWiseMatch) {
+        // Byte-wise comparison
+        for (let i = 0; i < arrayBuffer1.byteLength; i++) {
+          if (arrayBuffer1[i] !== arrayBuffer2[i]) {
+            byteWiseMatch = false;
+            break;
+          }
+        }
+        // Byte wise they're still the same, so no confict
+        if (byteWiseMatch) {
+          logger.log("info", "Files are the same, no conflict");
+          snapshot.set(name, [
+            pageData1.meta.lastModified,
+            pageData2.meta.lastModified,
+          ]);
+          return 0;
+        }
       }
     }
     const revisionFileName = filePieces.length === 1
@@ -303,9 +326,25 @@ export class SpaceSync {
     );
 
     snapshot.set(name, [pageData1.meta.lastModified, writeMeta.lastModified]);
+    return 1;
   }
 
   syncCandidates(files: FileMeta[]): FileMeta[] {
     return files.filter((f) => !f.name.startsWith("_plug/"));
   }
 }
+
+const markdownLanguage = buildMarkdown([]);
+
+export function removeDirectiveBody(text: string): string {
+  // Parse
+  const tree = parse(markdownLanguage, text);
+  // Remove bodies
+  replaceNodesMatching(tree, (node) => {
+    if (node.type === "DirectiveBody") {
+      return null;
+    }
+  });
+  // Turn back into text
+  return renderToText(tree);
+}

plugs/core/core.plug.yaml

@@ -414,7 +414,6 @@ functions:
     path: ./stats.ts:statsCommand
     command:
       name: "Stats: Show"
-      key: "Shift-Alt-s"
 
   # Cloud pages
   readPageCloud:

plugs/sync/sync.plug.yaml

@@ -9,6 +9,7 @@ functions:
     path: sync.ts:syncCommand
     command:
       name: "Sync: Sync"
+      key: "Shift-Alt-s"
 
   wipeAndSyncCommand:
     path: sync.ts:localWipeAndSyncCommand