From e5276319e03bab07fbe64904c45d6e038e15b40b Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Sat, 14 Jan 2023 18:51:00 +0100 Subject: [PATCH] Timeouts for sync config --- common/async_util.ts | 32 ++++++ common/spaces/sync.ts | 66 ++++++++--- common/syscalls/sync.ts | 19 +++- common/util.ts | 13 --- deno.jsonc | 3 +- mobile/boot.ts | 2 + mobile/ios/App/App.xcodeproj/project.pbxproj | 4 +- plug-api/silverbullet-syscall/system.ts | 5 + plugos/sandbox.ts | 23 ++-- plugs/core/core.plug.yaml | 2 + plugs/core/debug.ts | 112 ++++++++++++++----- plugs/sync/sync.ts | 2 + server/space_system.ts | 2 +- server/syscalls/system.ts | 3 + web/editor.tsx | 4 +- web/syscalls/system.ts | 3 + 16 files changed, 225 insertions(+), 70 deletions(-) create mode 100644 common/async_util.ts diff --git a/common/async_util.ts b/common/async_util.ts new file mode 100644 index 0000000..c76a801 --- /dev/null +++ b/common/async_util.ts @@ -0,0 +1,32 @@ +export function throttle(func: () => void, limit: number) { + let timer: any = null; + return function () { + if (!timer) { + timer = setTimeout(() => { + func(); + timer = null; + }, limit); + } + }; +} + +// race for promises returns first promise that resolves +export function race(promises: Promise[]): Promise { + return new Promise((resolve, reject) => { + for (const p of promises) { + p.then(resolve, reject); + } + }); +} + +export function timeout(ms: number): Promise { + return new Promise((_resolve, reject) => + setTimeout(() => { + reject(new Error("timeout")); + }, ms) + ); +} + +export function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/common/spaces/sync.ts b/common/spaces/sync.ts index c5df889..c5601ef 100644 --- a/common/spaces/sync.ts +++ b/common/spaces/sync.ts @@ -1,3 +1,4 @@ +import { LogEntry } from "../../plugos/sandbox.ts"; import type { FileMeta } from "../types.ts"; import { SpacePrimitives } from "./space_primitives.ts"; @@ -7,12 +8,23 @@ type SyncHash = number; // and the second item the lastModified value of the secondary space export type SyncStatusItem = [SyncHash, SyncHash]; +export interface Logger { + log(level: string, ...messageBits: any[]): void; +} + +class ConsoleLogger implements Logger { + log(_level: string, ...messageBits: any[]) { + console.log(...messageBits); + } +} + // Implementation of this algorithm https://unterwaditzer.net/2016/sync-algorithm.html export class SpaceSync { constructor( private primary: SpacePrimitives, private secondary: SpacePrimitives, readonly snapshot: Map, + readonly logger: Logger = new ConsoleLogger(), ) {} async syncFiles( @@ -21,15 +33,16 @@ export class SpaceSync { snapshot: Map, primarySpace: SpacePrimitives, secondarySpace: SpacePrimitives, + logger: Logger, ) => Promise, ): Promise { let operations = 0; - console.log("Fetching snapshot from primary"); + this.logger.log("info", "Fetching snapshot from primary"); const primaryAllPages = this.syncCandidates( await this.primary.fetchFileList(), ); - console.log("Fetching snapshot from secondary"); + this.logger.log("info", "Fetching snapshot from secondary"); try { const secondaryAllPages = this.syncCandidates( await this.secondary.fetchFileList(), @@ -48,14 +61,15 @@ export class SpaceSync { ...secondaryFileMap.keys(), ]); - console.log("Iterating over all files"); + this.logger.log("info", "Iterating over all files"); for (const name of allFilesToProcess) { if ( primaryFileMap.has(name) && !secondaryFileMap.has(name) && !this.snapshot.has(name) ) { // New file, created on primary, copy from primary to secondary - console.log( + this.logger.log( + "info", "New file created on primary, copying to secondary", name, ); @@ -75,7 +89,8 @@ export class SpaceSync { !this.snapshot.has(name) ) { // New file, created on secondary, copy from secondary to primary - console.log( + this.logger.log( + "info", "New file created on secondary, copying from secondary to primary", name, ); @@ -95,7 +110,11 @@ export class SpaceSync { !secondaryFileMap.has(name) ) { // File deleted on B - console.log("File deleted on secondary, deleting from primary", name); + this.logger.log( + "info", + "File deleted on secondary, deleting from primary", + name, + ); await this.primary.deleteFile(name); this.snapshot.delete(name); operations++; @@ -104,7 +123,11 @@ export class SpaceSync { !primaryFileMap.has(name) ) { // File deleted on A - console.log("File deleted on primary, deleting from secondary", name); + this.logger.log( + "info", + "File deleted on primary, deleting from secondary", + name, + ); await this.secondary.deleteFile(name); this.snapshot.delete(name); operations++; @@ -113,7 +136,11 @@ export class SpaceSync { !secondaryFileMap.has(name) ) { // File deleted on both sides, :shrug: - console.log("File deleted on both ends, deleting from status", name); + this.logger.log( + "info", + "File deleted on both ends, deleting from status", + name, + ); this.snapshot.delete(name); operations++; } else if ( @@ -123,7 +150,11 @@ export class SpaceSync { secondaryFileMap.get(name) === this.snapshot.get(name)![1] ) { // File has changed on primary, but not secondary: copy from primary to secondary - console.log("File changed on primary, copying to secondary", name); + this.logger.log( + "info", + "File changed on primary, copying to secondary", + name, + ); const { data } = await this.primary.readFile(name, "arraybuffer"); const writtenMeta = await this.secondary.writeFile( name, @@ -165,13 +196,18 @@ export class SpaceSync { primaryFileMap.get(name) !== this.snapshot.get(name)![0] ) ) { - console.log("File changed on both ends, conflict!", name); + this.logger.log( + "info", + "File changed on both ends, potential conflict", + name, + ); if (conflictResolver) { await conflictResolver( name, this.snapshot, this.primary, this.secondary, + this.logger, ); } else { throw Error( @@ -184,9 +220,10 @@ export class SpaceSync { } } } catch (e: any) { - console.error("Boom", e.message); + this.logger.log("error", "Sync error:", e.message); throw e; } + this.logger.log("info", "Sync complete, operations performed", operations); return operations; } @@ -197,8 +234,9 @@ export class SpaceSync { snapshot: Map, primary: SpacePrimitives, secondary: SpacePrimitives, + logger: Logger, ): Promise { - console.log("Hit a conflict for", name); + logger.log("info", "Starting conflict resolution for", name); const filePieces = name.split("."); const fileNameBase = filePieces.slice(0, -1).join("."); const fileNameExt = filePieces[filePieces.length - 1]; @@ -221,6 +259,7 @@ export class SpaceSync { } // Byte wise they're still the same, so no confict if (byteWiseMatch) { + logger.log("info", "Files are the same, no conflict"); snapshot.set(name, [ pageData1.meta.lastModified, pageData2.meta.lastModified, @@ -231,7 +270,8 @@ export class SpaceSync { const revisionFileName = filePieces.length === 1 ? `${name}.conflicted.${pageData2.meta.lastModified}` : `${fileNameBase}.conflicted.${pageData2.meta.lastModified}.${fileNameExt}`; - console.log( + logger.log( + "info", "Going to create conflicting copy", revisionFileName, ); diff --git a/common/syscalls/sync.ts b/common/syscalls/sync.ts index 89ce9c1..dec830b 100644 --- a/common/syscalls/sync.ts +++ b/common/syscalls/sync.ts @@ -1,10 +1,14 @@ -import { SysCallMapping } from "../../plugos/system.ts"; +import { SysCallMapping, System } from "../../plugos/system.ts"; import type { SyncEndpoint } from "../../plug-api/silverbullet-syscall/sync.ts"; import { SpaceSync, SyncStatusItem } from "../spaces/sync.ts"; import { HttpSpacePrimitives } from "../spaces/http_space_primitives.ts"; import { SpacePrimitives } from "../spaces/space_primitives.ts"; +import { race, timeout } from "../async_util.ts"; -export function syncSyscalls(localSpace: SpacePrimitives): SysCallMapping { +export function syncSyscalls( + localSpace: SpacePrimitives, + system: System, +): SysCallMapping { return { "sync.sync": async ( _ctx, @@ -33,6 +37,8 @@ export function syncSyscalls(localSpace: SpacePrimitives): SysCallMapping { localSpace, syncSpace, syncStatusMap, + // Log to the "sync" plug sandbox + system.loadedPlugs.get("sync")!.sandbox!, ); try { @@ -58,8 +64,13 @@ export function syncSyscalls(localSpace: SpacePrimitives): SysCallMapping { endpoint.user, endpoint.password, ); - // Let's just fetch the file list to see if it works - await syncSpace.fetchFileList(); + // Let's just fetch the file list to see if it works with a timeout of 5s + try { + await race([syncSpace.fetchFileList(), timeout(5000)]); + } catch (e: any) { + console.error("Sync check failure", e.message); + throw e; + } }, }; } diff --git a/common/util.ts b/common/util.ts index 5b7a8eb..237694c 100644 --- a/common/util.ts +++ b/common/util.ts @@ -1,7 +1,6 @@ import { SETTINGS_TEMPLATE } from "./settings_template.ts"; import { YAML } from "./deps.ts"; import { Space } from "./spaces/space.ts"; -import { BuiltinSettings } from "../web/types.ts"; export function safeRun(fn: () => Promise) { fn().catch((e) => { @@ -13,18 +12,6 @@ export function isMacLike() { return /(Mac|iPhone|iPod|iPad)/i.test(navigator.platform); } -export function throttle(func: () => void, limit: number) { - let timer: any = null; - return function () { - if (!timer) { - timer = setTimeout(() => { - func(); - timer = null; - }, limit); - } - }; -} - // TODO: This is naive, may be better to use a proper parser const yamlSettingsRegex = /```yaml([^`]+)```/; diff --git a/deno.jsonc b/deno.jsonc index 7f7a52c..e2cce72 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -21,7 +21,8 @@ "desktop:build": "deno task build && deno task bundle && cd desktop && npm run make", // Mobile "mobile:deps": "cd mobile && npm install && npx cap sync", - "mobile:build": "deno task clean && deno task plugs && deno run -A --unstable --check build_mobile.ts && cd mobile && npx cap copy && npx cap open ios" + "mobile:clean-build": "deno task clean && deno task plugs && deno run -A --unstable --check build_mobile.ts && cd mobile && npx cap copy && npx cap open ios", + "mobile:build": "deno run -A --unstable --check build_mobile.ts && cd mobile && npx cap copy && npx cap open ios" }, "compilerOptions": { diff --git a/mobile/boot.ts b/mobile/boot.ts index 2c4242e..4ea517a 100644 --- a/mobile/boot.ts +++ b/mobile/boot.ts @@ -30,6 +30,7 @@ import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitive import { EventHook } from "../plugos/hooks/event.ts"; import { clientStoreSyscalls } from "./syscalls/clientStore.ts"; import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts"; +import { syncSyscalls } from "../common/syscalls/sync.ts"; safeRun(async () => { // Instantiate a PlugOS system for the client @@ -86,6 +87,7 @@ safeRun(async () => { storeSyscalls(db, "store"), indexSyscalls, clientStoreSyscalls(db), + syncSyscalls(spacePrimitives, system), fullTextSearchSyscalls(db, "fts"), sandboxFetchSyscalls(), ); diff --git a/mobile/ios/App/App.xcodeproj/project.pbxproj b/mobile/ios/App/App.xcodeproj/project.pbxproj index 051536c..ced4b7b 100644 --- a/mobile/ios/App/App.xcodeproj/project.pbxproj +++ b/mobile/ios/App/App.xcodeproj/project.pbxproj @@ -350,7 +350,7 @@ buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 1; + CURRENT_PROJECT_VERSION = 2; DEVELOPMENT_TEAM = Z92J6WM6X8; INFOPLIST_FILE = App/Info.plist; INFOPLIST_KEY_CFBundleDisplayName = SilverBullet; @@ -376,7 +376,7 @@ buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 1; + CURRENT_PROJECT_VERSION = 2; DEVELOPMENT_TEAM = Z92J6WM6X8; INFOPLIST_FILE = App/Info.plist; INFOPLIST_KEY_CFBundleDisplayName = SilverBullet; diff --git a/plug-api/silverbullet-syscall/system.ts b/plug-api/silverbullet-syscall/system.ts index 20d42ee..be37e40 100644 --- a/plug-api/silverbullet-syscall/system.ts +++ b/plug-api/silverbullet-syscall/system.ts @@ -22,3 +22,8 @@ export function listCommands(): Promise<{ [key: string]: CommandDef }> { export function reloadPlugs() { syscall("system.reloadPlugs"); } + +// Returns what runtime environment this plug is run in, e.g. "server" or "client" can be undefined, which would mean a hybrid environment (such as mobile) +export function getEnv(): Promise { + return syscall("system.getEnv"); +} diff --git a/plugos/sandbox.ts b/plugos/sandbox.ts index ba402fc..5ededfb 100644 --- a/plugos/sandbox.ts +++ b/plugos/sandbox.ts @@ -126,15 +126,7 @@ export class Sandbox { break; } case "log": { - this.logBuffer.push({ - level: data.level!, - message: data.message!, - date: Date.now(), - }); - if (this.logBuffer.length > this.maxLogBufferSize) { - this.logBuffer.shift(); - } - console.log(`[Sandbox ${data.level}]`, data.message); + this.log(data.level!, data.message!); break; } default: @@ -142,6 +134,19 @@ export class Sandbox { } } + log(level: string, ...messageBits: any[]) { + const message = messageBits.map((a) => "" + a).join(" "); + this.logBuffer.push({ + message, + level: level as LogLevel, + date: Date.now(), + }); + if (this.logBuffer.length > this.maxLogBufferSize) { + this.logBuffer.shift(); + } + console.log(`[Sandbox ${level}]`, message); + } + invoke(name: string, args: any[]): Promise { this.reqId++; this.worker.postMessage({ diff --git a/plugs/core/core.plug.yaml b/plugs/core/core.plug.yaml index abffd8e..617d147 100644 --- a/plugs/core/core.plug.yaml +++ b/plugs/core/core.plug.yaml @@ -385,6 +385,8 @@ functions: name: "UI: Hide BHS" key: "Ctrl-Alt-b" mac: "Cmd-Alt-b" + events: + - log:hide # Link unfurl infrastructure unfurlLink: diff --git a/plugs/core/debug.ts b/plugs/core/debug.ts index 806f0d6..93d2eed 100644 --- a/plugs/core/debug.ts +++ b/plugs/core/debug.ts @@ -3,6 +3,7 @@ import { editor, markdown, sandbox as serverSandbox, + system, } from "$sb/silverbullet-syscall/mod.ts"; export async function parsePageCommand() { @@ -17,29 +18,38 @@ export async function parsePageCommand() { } export async function showLogsCommand() { - const clientLogs = await sandbox.getLogs(); - const serverLogs = await serverSandbox.getServerLogs(); + // Running in client/server mode? + const clientServer = !!(await system.getEnv()); - await editor.showPanel( - "bhs", - 1, - ` + if (clientServer) { + const clientLogs = await sandbox.getLogs(); + const serverLogs = await serverSandbox.getServerLogs(); + await editor.showPanel( + "bhs", + 1, + ` + +
Client logs (max 100)
${
-      clientLogs
-        .map((le) => `[${le.level}] ${le.message}`)
-        .join("\n")
-    }
+ clientLogs + .map((le) => `[${le.level}] ${le.message}`) + .join("\n") + }
Server logs (max 100)
${
-      serverLogs
-        .map((le) => `[${le.level}] ${le.message}`)
-        .join("\n")
-    }
+ serverLogs + .map((le) => `[${le.level}] ${le.message}`) + .join("\n") + }
`, - ` + ` var clientDiv = document.getElementById("client-log"); clientDiv.scrollTop = clientDiv.scrollHeight; var serverDiv = document.getElementById("server-log"); serverDiv.scrollTop = serverDiv.scrollHeight; - if(window.reloadInterval) { - clearInterval(window.reloadInterval); - } - window.reloadInterval = setInterval(() => { + + self.reloadLogs = () => { sendEvent("log:reload"); - }, 1000); + }; + self.close = () => { + sendEvent("log:hide"); + }; `, - ); + ); + } else { + const logs = await sandbox.getLogs(); + await editor.showPanel( + "bhs", + 1, + ` + + + +
Logs (max 100)
+
+ 
${
+        logs
+          .map((le) => `[${le.level}] ${le.message}`)
+          .join("\n")
+      }
+
`, + ` + var clientDiv = document.getElementById("log"); + clientDiv.scrollTop = clientDiv.scrollHeight; + self.reloadLogs = () => { + sendEvent("log:reload"); + }; + self.close = () => { + sendEvent("log:hide"); + }; + `, + ); + } } export async function hideBhsCommand() { diff --git a/plugs/sync/sync.ts b/plugs/sync/sync.ts index 47cce72..44715d2 100644 --- a/plugs/sync/sync.ts +++ b/plugs/sync/sync.ts @@ -54,6 +54,8 @@ export async function syncCommand() { } await editor.flashNotification("Starting sync..."); try { + await system.invokeFunction("server", "check", config); + const operations = await system.invokeFunction("server", "performSync"); await editor.flashNotification( `Sync complete. Performed ${operations} operations.`, diff --git a/server/space_system.ts b/server/space_system.ts index d880ba3..3c9c4f8 100644 --- a/server/space_system.ts +++ b/server/space_system.ts @@ -112,7 +112,7 @@ export class SpaceSystem { storeSyscalls(this.db, "store"), fullTextSearchSyscalls(this.db, "fts"), spaceSyscalls(this.space), - syncSyscalls(this.spacePrimitives), + syncSyscalls(this.spacePrimitives, this.system), eventSyscalls(this.eventHook), markdownSyscalls(buildMarkdown([])), esbuildSyscalls([globalModules]), diff --git a/server/syscalls/system.ts b/server/syscalls/system.ts index 710f395..6505419 100644 --- a/server/syscalls/system.ts +++ b/server/syscalls/system.ts @@ -31,5 +31,8 @@ export function systemSyscalls( "system.reloadPlugs": () => { return plugReloader(); }, + "system.getEnv": () => { + return system.env; + }, }; } diff --git a/web/editor.tsx b/web/editor.tsx index 0cd0ef7..2f8f111 100644 --- a/web/editor.tsx +++ b/web/editor.tsx @@ -43,7 +43,7 @@ import buildMarkdown from "../common/markdown_parser/parser.ts"; import { Space } from "../common/spaces/space.ts"; import { markdownSyscalls } from "../common/syscalls/markdown.ts"; import { FilterOption, PageMeta } from "../common/types.ts"; -import { isMacLike, safeRun, throttle } from "../common/util.ts"; +import { isMacLike, safeRun } from "../common/util.ts"; import { createSandbox } from "../plugos/environments/webworker_sandbox.ts"; import { EventHook } from "../plugos/hooks/event.ts"; import assetSyscalls from "../plugos/syscalls/asset.ts"; @@ -98,6 +98,7 @@ import type { import { CodeWidgetHook } from "./hooks/code_widget.ts"; import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts"; import { syncSyscalls } from "../common/syscalls/sync.ts"; +import { throttle } from "../common/async_util.ts"; const frontMatterRegex = /^---\n(.*?)---\n/ms; @@ -196,7 +197,6 @@ export class Editor { markdownSyscalls(buildMarkdown(this.mdExtensions)), sandboxSyscalls(this.system), assetSyscalls(this.system), - syncSyscalls(this.space.spacePrimitives), collabSyscalls(this), ); diff --git a/web/syscalls/system.ts b/web/syscalls/system.ts index 93c8ea6..6f7bc57 100644 --- a/web/syscalls/system.ts +++ b/web/syscalls/system.ts @@ -50,5 +50,8 @@ export function systemSyscalls( "sandbox.getServerLogs": (ctx) => { return editor.space.proxySyscall(ctx.plug, "sandbox.getLogs", []); }, + "system.getEnv": () => { + return system.env; + }, }; }