Timeouts for sync config

common/async_util.ts

@@ -0,0 +1,32 @@
+export function throttle(func: () => void, limit: number) {
+  let timer: any = null;
+  return function () {
+    if (!timer) {
+      timer = setTimeout(() => {
+        func();
+        timer = null;
+      }, limit);
+    }
+  };
+}
+
+// race for promises returns first promise that resolves
+export function race<T>(promises: Promise<T>[]): Promise<T> {
+  return new Promise((resolve, reject) => {
+    for (const p of promises) {
+      p.then(resolve, reject);
+    }
+  });
+}
+
+export function timeout(ms: number): Promise<never> {
+  return new Promise((_resolve, reject) =>
+    setTimeout(() => {
+      reject(new Error("timeout"));
+    }, ms)
+  );
+}
+
+export function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

common/spaces/sync.ts

@@ -1,3 +1,4 @@
+import { LogEntry } from "../../plugos/sandbox.ts";
 import type { FileMeta } from "../types.ts";
 import { SpacePrimitives } from "./space_primitives.ts";
 
@@ -7,12 +8,23 @@ type SyncHash = number;
 // and the second item the lastModified value of the secondary space
 export type SyncStatusItem = [SyncHash, SyncHash];
 
+export interface Logger {
+  log(level: string, ...messageBits: any[]): void;
+}
+
+class ConsoleLogger implements Logger {
+  log(_level: string, ...messageBits: any[]) {
+    console.log(...messageBits);
+  }
+}
+
 // Implementation of this algorithm https://unterwaditzer.net/2016/sync-algorithm.html
 export class SpaceSync {
   constructor(
     private primary: SpacePrimitives,
     private secondary: SpacePrimitives,
     readonly snapshot: Map<string, SyncStatusItem>,
+    readonly logger: Logger = new ConsoleLogger(),
   ) {}
 
   async syncFiles(
@@ -21,15 +33,16 @@ export class SpaceSync {
       snapshot: Map<string, SyncStatusItem>,
       primarySpace: SpacePrimitives,
       secondarySpace: SpacePrimitives,
+      logger: Logger,
     ) => Promise<void>,
   ): Promise<number> {
     let operations = 0;
-    console.log("Fetching snapshot from primary");
+    this.logger.log("info", "Fetching snapshot from primary");
     const primaryAllPages = this.syncCandidates(
       await this.primary.fetchFileList(),
     );
 
-    console.log("Fetching snapshot from secondary");
+    this.logger.log("info", "Fetching snapshot from secondary");
     try {
       const secondaryAllPages = this.syncCandidates(
         await this.secondary.fetchFileList(),
@@ -48,14 +61,15 @@ export class SpaceSync {
         ...secondaryFileMap.keys(),
       ]);
 
-      console.log("Iterating over all files");
+      this.logger.log("info", "Iterating over all files");
       for (const name of allFilesToProcess) {
         if (
           primaryFileMap.has(name) && !secondaryFileMap.has(name) &&
           !this.snapshot.has(name)
         ) {
           // New file, created on primary, copy from primary to secondary
-          console.log(
+          this.logger.log(
+            "info",
             "New file created on primary, copying to secondary",
             name,
           );
@@ -75,7 +89,8 @@ export class SpaceSync {
           !this.snapshot.has(name)
         ) {
           // New file, created on secondary, copy from secondary to primary
-          console.log(
+          this.logger.log(
+            "info",
             "New file created on secondary, copying from secondary to primary",
             name,
           );
@@ -95,7 +110,11 @@ export class SpaceSync {
           !secondaryFileMap.has(name)
         ) {
           // File deleted on B
-          console.log("File deleted on secondary, deleting from primary", name);
+          this.logger.log(
+            "info",
+            "File deleted on secondary, deleting from primary",
+            name,
+          );
           await this.primary.deleteFile(name);
           this.snapshot.delete(name);
           operations++;
@@ -104,7 +123,11 @@ export class SpaceSync {
           !primaryFileMap.has(name)
         ) {
           // File deleted on A
-          console.log("File deleted on primary, deleting from secondary", name);
+          this.logger.log(
+            "info",
+            "File deleted on primary, deleting from secondary",
+            name,
+          );
           await this.secondary.deleteFile(name);
           this.snapshot.delete(name);
           operations++;
@@ -113,7 +136,11 @@ export class SpaceSync {
           !secondaryFileMap.has(name)
         ) {
           // File deleted on both sides, :shrug:
-          console.log("File deleted on both ends, deleting from status", name);
+          this.logger.log(
+            "info",
+            "File deleted on both ends, deleting from status",
+            name,
+          );
           this.snapshot.delete(name);
           operations++;
         } else if (
@@ -123,7 +150,11 @@ export class SpaceSync {
           secondaryFileMap.get(name) === this.snapshot.get(name)![1]
         ) {
           // File has changed on primary, but not secondary: copy from primary to secondary
-          console.log("File changed on primary, copying to secondary", name);
+          this.logger.log(
+            "info",
+            "File changed on primary, copying to secondary",
+            name,
+          );
           const { data } = await this.primary.readFile(name, "arraybuffer");
           const writtenMeta = await this.secondary.writeFile(
             name,
@@ -165,13 +196,18 @@ export class SpaceSync {
             primaryFileMap.get(name) !== this.snapshot.get(name)![0]
           )
         ) {
-          console.log("File changed on both ends, conflict!", name);
+          this.logger.log(
+            "info",
+            "File changed on both ends, potential conflict",
+            name,
+          );
           if (conflictResolver) {
             await conflictResolver(
               name,
               this.snapshot,
               this.primary,
               this.secondary,
+              this.logger,
             );
           } else {
             throw Error(
@@ -184,9 +220,10 @@ export class SpaceSync {
         }
       }
     } catch (e: any) {
-      console.error("Boom", e.message);
+      this.logger.log("error", "Sync error:", e.message);
       throw e;
     }
+    this.logger.log("info", "Sync complete, operations performed", operations);
 
     return operations;
   }
@@ -197,8 +234,9 @@ export class SpaceSync {
     snapshot: Map<string, SyncStatusItem>,
     primary: SpacePrimitives,
     secondary: SpacePrimitives,
+    logger: Logger,
   ): Promise<void> {
-    console.log("Hit a conflict for", name);
+    logger.log("info", "Starting conflict resolution for", name);
     const filePieces = name.split(".");
     const fileNameBase = filePieces.slice(0, -1).join(".");
     const fileNameExt = filePieces[filePieces.length - 1];
@@ -221,6 +259,7 @@ export class SpaceSync {
       }
       // Byte wise they're still the same, so no confict
       if (byteWiseMatch) {
+        logger.log("info", "Files are the same, no conflict");
         snapshot.set(name, [
           pageData1.meta.lastModified,
           pageData2.meta.lastModified,
@@ -231,7 +270,8 @@ export class SpaceSync {
     const revisionFileName = filePieces.length === 1
       ? `${name}.conflicted.${pageData2.meta.lastModified}`
       : `${fileNameBase}.conflicted.${pageData2.meta.lastModified}.${fileNameExt}`;
-    console.log(
+    logger.log(
+      "info",
       "Going to create conflicting copy",
       revisionFileName,
     );

common/syscalls/sync.ts

@@ -1,10 +1,14 @@
-import { SysCallMapping } from "../../plugos/system.ts";
+import { SysCallMapping, System } from "../../plugos/system.ts";
 import type { SyncEndpoint } from "../../plug-api/silverbullet-syscall/sync.ts";
 import { SpaceSync, SyncStatusItem } from "../spaces/sync.ts";
 import { HttpSpacePrimitives } from "../spaces/http_space_primitives.ts";
 import { SpacePrimitives } from "../spaces/space_primitives.ts";
+import { race, timeout } from "../async_util.ts";
 
-export function syncSyscalls(localSpace: SpacePrimitives): SysCallMapping {
+export function syncSyscalls(
+  localSpace: SpacePrimitives,
+  system: System<any>,
+): SysCallMapping {
   return {
     "sync.sync": async (
       _ctx,
@@ -33,6 +37,8 @@ export function syncSyscalls(localSpace: SpacePrimitives): SysCallMapping {
         localSpace,
         syncSpace,
         syncStatusMap,
+        // Log to the "sync" plug sandbox
+        system.loadedPlugs.get("sync")!.sandbox!,
       );
 
       try {
@@ -58,8 +64,13 @@ export function syncSyscalls(localSpace: SpacePrimitives): SysCallMapping {
         endpoint.user,
         endpoint.password,
       );
-      // Let's just fetch the file list to see if it works
+      // Let's just fetch the file list to see if it works with a timeout of 5s
-      await syncSpace.fetchFileList();
+      try {
+        await race([syncSpace.fetchFileList(), timeout(5000)]);
+      } catch (e: any) {
+        console.error("Sync check failure", e.message);
+        throw e;
+      }
     },
   };
 }

common/util.ts

@@ -1,7 +1,6 @@
 import { SETTINGS_TEMPLATE } from "./settings_template.ts";
 import { YAML } from "./deps.ts";
 import { Space } from "./spaces/space.ts";
-import { BuiltinSettings } from "../web/types.ts";
 
 export function safeRun(fn: () => Promise<void>) {
   fn().catch((e) => {
@@ -13,18 +12,6 @@ export function isMacLike() {
   return /(Mac|iPhone|iPod|iPad)/i.test(navigator.platform);
 }
 
-export function throttle(func: () => void, limit: number) {
-  let timer: any = null;
-  return function () {
-    if (!timer) {
-      timer = setTimeout(() => {
-        func();
-        timer = null;
-      }, limit);
-    }
-  };
-}
-
 // TODO: This is naive, may be better to use a proper parser
 const yamlSettingsRegex = /```yaml([^`]+)```/;
 

deno.jsonc

@@ -21,7 +21,8 @@
     "desktop:build": "deno task build && deno task bundle && cd desktop && npm run make",
     // Mobile
     "mobile:deps": "cd mobile && npm install && npx cap sync",
-    "mobile:build": "deno task clean && deno task plugs && deno run -A --unstable --check build_mobile.ts && cd mobile && npx cap copy && npx cap open ios"
+    "mobile:clean-build": "deno task clean && deno task plugs && deno run -A --unstable --check build_mobile.ts && cd mobile && npx cap copy && npx cap open ios",
+    "mobile:build": "deno run -A --unstable --check build_mobile.ts && cd mobile && npx cap copy && npx cap open ios"
   },
 
   "compilerOptions": {

mobile/boot.ts

@@ -30,6 +30,7 @@ import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitive
 import { EventHook } from "../plugos/hooks/event.ts";
 import { clientStoreSyscalls } from "./syscalls/clientStore.ts";
 import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts";
+import { syncSyscalls } from "../common/syscalls/sync.ts";
 
 safeRun(async () => {
   // Instantiate a PlugOS system for the client
@@ -86,6 +87,7 @@ safeRun(async () => {
     storeSyscalls(db, "store"),
     indexSyscalls,
     clientStoreSyscalls(db),
+    syncSyscalls(spacePrimitives, system),
     fullTextSearchSyscalls(db, "fts"),
     sandboxFetchSyscalls(),
   );

mobile/ios/App/App.xcodeproj/project.pbxproj

@@ -350,7 +350,7 @@
 			buildSettings = {
 				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 1;
+				CURRENT_PROJECT_VERSION = 2;
 				DEVELOPMENT_TEAM = Z92J6WM6X8;
 				INFOPLIST_FILE = App/Info.plist;
 				INFOPLIST_KEY_CFBundleDisplayName = SilverBullet;
@@ -376,7 +376,7 @@
 			buildSettings = {
 				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 1;
+				CURRENT_PROJECT_VERSION = 2;
 				DEVELOPMENT_TEAM = Z92J6WM6X8;
 				INFOPLIST_FILE = App/Info.plist;
 				INFOPLIST_KEY_CFBundleDisplayName = SilverBullet;

plug-api/silverbullet-syscall/system.ts

@@ -22,3 +22,8 @@ export function listCommands(): Promise<{ [key: string]: CommandDef }> {
 export function reloadPlugs() {
   syscall("system.reloadPlugs");
 }
+
+// Returns what runtime environment this plug is run in, e.g. "server" or "client" can be undefined, which would mean a hybrid environment (such as mobile)
+export function getEnv(): Promise<string | undefined> {
+  return syscall("system.getEnv");
+}

plugos/sandbox.ts

@@ -126,15 +126,7 @@ export class Sandbox {
         break;
       }
       case "log": {
-        this.logBuffer.push({
+        this.log(data.level!, data.message!);
-          level: data.level!,
-          message: data.message!,
-          date: Date.now(),
-        });
-        if (this.logBuffer.length > this.maxLogBufferSize) {
-          this.logBuffer.shift();
-        }
-        console.log(`[Sandbox ${data.level}]`, data.message);
         break;
       }
       default:
@@ -142,6 +134,19 @@ export class Sandbox {
     }
   }
 
+  log(level: string, ...messageBits: any[]) {
+    const message = messageBits.map((a) => "" + a).join(" ");
+    this.logBuffer.push({
+      message,
+      level: level as LogLevel,
+      date: Date.now(),
+    });
+    if (this.logBuffer.length > this.maxLogBufferSize) {
+      this.logBuffer.shift();
+    }
+    console.log(`[Sandbox ${level}]`, message);
+  }
+
   invoke(name: string, args: any[]): Promise<any> {
     this.reqId++;
     this.worker.postMessage({

plugs/core/core.plug.yaml

@@ -385,6 +385,8 @@ functions:
       name: "UI: Hide BHS"
       key: "Ctrl-Alt-b"
       mac: "Cmd-Alt-b"
+    events:
+      - log:hide
 
   # Link unfurl infrastructure
   unfurlLink:

plugs/core/debug.ts

@@ -3,6 +3,7 @@ import {
   editor,
   markdown,
   sandbox as serverSandbox,
+  system,
 } from "$sb/silverbullet-syscall/mod.ts";
 
 export async function parsePageCommand() {
@@ -17,29 +18,38 @@ export async function parsePageCommand() {
 }
 
 export async function showLogsCommand() {
-  const clientLogs = await sandbox.getLogs();
+  // Running in client/server mode?
-  const serverLogs = await serverSandbox.getServerLogs();
+  const clientServer = !!(await system.getEnv());
 
-  await editor.showPanel(
+  if (clientServer) {
-    "bhs",
+    const clientLogs = await sandbox.getLogs();
-    1,
+    const serverLogs = await serverSandbox.getServerLogs();
-    `
+    await editor.showPanel(
+      "bhs",
+      1,
+      `
     <style>
+    #reload {
+      width: 75%;
+    }
+    #close {
+      width: 20%;
+    }
     #client-log-header {
         position: absolute;
         left: 0;
-        top: 5px;
+        top: 35px;
     }
     #server-log-header {
         position: absolute;
         right: 0;
-        top: 5px;
+        top: 35px;
         width: 50%;
     }
     #client-log {
         position: absolute;
         left: 0;
-        top: 30px;
+        top: 60px;
         bottom: 0;
         width: 50%;
         overflow: scroll;
@@ -47,41 +57,93 @@ export async function showLogsCommand() {
     #server-log {
         position: absolute;
         right: 0;
-        top: 30px;
+        top: 60px;
         bottom: 0;
         width: 50%;
         overflow: scroll;
     }
     </style>
+    <button onclick="self.reloadLogs()" id="reload">Reload</button>
+    <button onclick="self.close()" id="close">Close</button>
     <div id="client-log-header">Client logs (max 100)</div>
     <div id="client-log">
       <pre>${
-      clientLogs
+        clientLogs
-        .map((le) => `[${le.level}] ${le.message}`)
+          .map((le) => `[${le.level}] ${le.message}`)
-        .join("\n")
+          .join("\n")
-    }</pre>
+      }</pre>
     </div>
     <div id="server-log-header">Server logs (max 100)</div>
     <div id="server-log">
       <pre>${
-      serverLogs
+        serverLogs
-        .map((le) => `[${le.level}] ${le.message}`)
+          .map((le) => `[${le.level}] ${le.message}`)
-        .join("\n")
+          .join("\n")
-    }</pre>
+      }</pre>
     </div>`,
-    `
+      `
       var clientDiv = document.getElementById("client-log");
       clientDiv.scrollTop = clientDiv.scrollHeight;
       var serverDiv = document.getElementById("server-log");
       serverDiv.scrollTop = serverDiv.scrollHeight;
-      if(window.reloadInterval) {
+
-        clearInterval(window.reloadInterval);
+      self.reloadLogs = () => {
-      }
-      window.reloadInterval = setInterval(() => {
         sendEvent("log:reload");
-      }, 1000);
+      };
+      self.close = () => {
+        sendEvent("log:hide");
+      };
       `,
-  );
+    );
+  } else {
+    const logs = await sandbox.getLogs();
+    await editor.showPanel(
+      "bhs",
+      1,
+      `
+        <style>
+        #reload {
+          width: 75%;
+        }
+        #close {
+          width: 20%;
+        }
+        #log-header {
+          position: absolute;
+          left: 0;
+          top: 35px;
+        }
+        #log {
+          position: absolute;
+          left: 0;
+          top: 60px;
+          bottom: 0;
+          width: 100%;
+          overflow: scroll;
+        }
+        </style>
+        <button onclick="self.reloadLogs()" id="reload">Reload</button>
+        <button onclick="self.close()" id="close">Close</button>
+            <div id="log-header">Logs (max 100)</div>
+        <div id="log">
+          <pre>${
+        logs
+          .map((le) => `[${le.level}] ${le.message}`)
+          .join("\n")
+      }</pre>
+        </div>`,
+      `
+          var clientDiv = document.getElementById("log");
+          clientDiv.scrollTop = clientDiv.scrollHeight;
+          self.reloadLogs = () => {
+            sendEvent("log:reload");
+          };
+          self.close = () => {
+            sendEvent("log:hide");
+          };
+          `,
+    );
+  }
 }
 
 export async function hideBhsCommand() {

plugs/sync/sync.ts

@@ -54,6 +54,8 @@ export async function syncCommand() {
   }
   await editor.flashNotification("Starting sync...");
   try {
+    await system.invokeFunction("server", "check", config);
+
     const operations = await system.invokeFunction("server", "performSync");
     await editor.flashNotification(
       `Sync complete. Performed ${operations} operations.`,

server/space_system.ts

@@ -112,7 +112,7 @@ export class SpaceSystem {
       storeSyscalls(this.db, "store"),
       fullTextSearchSyscalls(this.db, "fts"),
       spaceSyscalls(this.space),
-      syncSyscalls(this.spacePrimitives),
+      syncSyscalls(this.spacePrimitives, this.system),
       eventSyscalls(this.eventHook),
       markdownSyscalls(buildMarkdown([])),
       esbuildSyscalls([globalModules]),

server/syscalls/system.ts

@@ -31,5 +31,8 @@ export function systemSyscalls(
     "system.reloadPlugs": () => {
       return plugReloader();
     },
+    "system.getEnv": () => {
+      return system.env;
+    },
   };
 }

web/editor.tsx

@@ -43,7 +43,7 @@ import buildMarkdown from "../common/markdown_parser/parser.ts";
 import { Space } from "../common/spaces/space.ts";
 import { markdownSyscalls } from "../common/syscalls/markdown.ts";
 import { FilterOption, PageMeta } from "../common/types.ts";
-import { isMacLike, safeRun, throttle } from "../common/util.ts";
+import { isMacLike, safeRun } from "../common/util.ts";
 import { createSandbox } from "../plugos/environments/webworker_sandbox.ts";
 import { EventHook } from "../plugos/hooks/event.ts";
 import assetSyscalls from "../plugos/syscalls/asset.ts";
@@ -98,6 +98,7 @@ import type {
 import { CodeWidgetHook } from "./hooks/code_widget.ts";
 import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts";
 import { syncSyscalls } from "../common/syscalls/sync.ts";
+import { throttle } from "../common/async_util.ts";
 
 const frontMatterRegex = /^---\n(.*?)---\n/ms;
 
@@ -196,7 +197,6 @@ export class Editor {
       markdownSyscalls(buildMarkdown(this.mdExtensions)),
       sandboxSyscalls(this.system),
       assetSyscalls(this.system),
-      syncSyscalls(this.space.spacePrimitives),
       collabSyscalls(this),
     );
 

web/syscalls/system.ts

@@ -50,5 +50,8 @@ export function systemSyscalls(
     "sandbox.getServerLogs": (ctx) => {
       return editor.space.proxySyscall(ctx.plug, "sandbox.getLogs", []);
     },
+    "system.getEnv": () => {
+      return system.env;
+    },
   };
 }