Real-time collaboration within space (#411)

.github/workflows/docker-s3.yml

@@ -7,7 +7,7 @@ on:
     tags:
       - "*"
 env:
-  DENO_VERSION: v1.33
+  DENO_VERSION: v1.34
   # Docker & Registries
   ARCHITECTURES: linux/amd64,linux/arm64
   IMAGE_NAME: silverbullet-s3

.github/workflows/docker.yml

@@ -7,7 +7,7 @@ on:
     tags:
       - "*"
 env:
-  DENO_VERSION: v1.33
+  DENO_VERSION: v1.34
   # Docker & Registries
   ARCHITECTURES: linux/amd64,linux/arm64
   IMAGE_NAME: silverbullet

.github/workflows/release.yml

@@ -13,7 +13,7 @@ jobs:
       - name: Setup Deno
         uses: denoland/setup-deno@v1
         with:
-          deno-version: v1.33
+          deno-version: v1.34
       - name: Run build
         run: deno task build
       - name: Bundle

.github/workflows/test.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Setup Deno
         uses: denoland/setup-deno@v1
         with:
-          deno-version: v1.33
+          deno-version: v1.34
 
       - name: Run build
         run: deno task build

.gitignore

@@ -9,4 +9,6 @@ website_build
 deno.lock
 fly.toml
 env.sh
-test_space
+node_modules
+*.db
+test_space

build_bundle.ts

@@ -13,12 +13,23 @@ await esbuild.build({
   sourcemap: false,
   minify: false,
   plugins: [
+    // ESBuild plugin to make npm modules external
+    {
+      name: "npm-external",
+      setup(build: any) {
+        build.onResolve({ filter: /^npm:/ }, (args: any) => {
+          return {
+            path: args.path,
+            external: true,
+          };
+        });
+      },
+    },
     {
       name: "json",
       setup: (build) =>
         build.onLoad({ filter: /\.json$/ }, () => ({ loader: "json" })),
     },
-
     ...denoPlugins({
       importMapURL: new URL("./import_map.json", import.meta.url)
         .toString(),

build_web.ts

@@ -1,4 +1,3 @@
-import { denoPlugins } from "https://deno.land/x/esbuild_deno_loader@0.7.0/mod.ts";
 import { copy } from "https://deno.land/std@0.165.0/fs/copy.ts";
 
 import sass from "https://deno.land/x/denosass@1.0.4/mod.ts";
@@ -6,7 +5,7 @@ import { bundleFolder } from "./plugos/asset_bundle/builder.ts";
 
 import * as flags from "https://deno.land/std@0.165.0/flags/mod.ts";
 import { patchDenoLibJS } from "./plugos/compile.ts";
-import { esbuild } from "./plugos/deps.ts";
+import { denoPlugins, esbuild } from "./plugos/deps.ts";
 
 export async function bundleAll(
   watch: boolean,
@@ -43,7 +42,7 @@ export async function copyAssets(dist: string) {
   await copy("web/auth.html", `${dist}/auth.html`, {
     overwrite: true,
   });
-  await copy("web/reset.html", `${dist}/reset.html`, {
+  await copy("web/logout.html", `${dist}/logout.html`, {
     overwrite: true,
   });
   await copy("web/images/favicon.png", `${dist}/favicon.png`, {

cmd/server.ts

@@ -11,8 +11,10 @@ import { AssetBundlePlugSpacePrimitives } from "../common/spaces/asset_bundle_sp
 import { DiskSpacePrimitives } from "../common/spaces/disk_space_primitives.ts";
 import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
 import { S3SpacePrimitives } from "../server/spaces/s3_space_primitives.ts";
+import { Authenticator } from "../server/auth.ts";
+import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts";
 
-export function serveCommand(
+export async function serveCommand(
   options: any,
   folder?: string,
 ) {
@@ -61,12 +63,42 @@ To allow outside connections, pass -L 0.0.0.0 as a flag, and put a TLS terminato
     new AssetBundle(plugAssetBundle as AssetJson),
   );
 
+  const authStore = new JSONKVStore();
+  const authenticator = new Authenticator(authStore);
+
+  const flagUser = options.user ?? Deno.env.get("SB_USER");
+  if (flagUser) {
+    // If explicitly added via env/parameter, add on the fly
+    const [username, password] = flagUser.split(":");
+    await authenticator.register(username, password, ["admin"], "");
+  }
+
+  if (options.auth) {
+    // Load auth file
+    const authFile: string = options.auth;
+    console.log("Loading authentication credentials from", authFile);
+    await authStore.load(authFile);
+    (async () => {
+      // Asynchronously kick off file watcher
+      for await (const _event of Deno.watchFs(options.auth)) {
+        console.log("Authentication file changed, reloading...");
+        await authStore.load(authFile);
+      }
+    })().catch(console.error);
+  }
+
+  const envAuth = Deno.env.get("SB_AUTH");
+  if (envAuth) {
+    console.log("Loading authentication from SB_AUTH");
+    authStore.loadString(envAuth);
+  }
+
   const httpServer = new HttpServer(spacePrimitives!, {
     hostname,
     port: port,
     pagesPath: folder!,
     clientAssetBundle: new AssetBundle(clientAssetBundle as AssetJson),
-    user: options.user ?? Deno.env.get("SB_USER"),
+    authenticator,
     keyFile: options.key,
     certFile: options.cert,
     maxFileSizeMB: +maxFileSizeMB,

cmd/user_add.ts

@@ -0,0 +1,35 @@
+import getpass from "https://deno.land/x/getpass@0.3.1/mod.ts";
+import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts";
+import { Authenticator } from "../server/auth.ts";
+
+export async function userAdd(
+  options: any,
+  username?: string,
+) {
+  const authFile = options.auth || ".auth.json";
+  console.log("Using auth file", authFile);
+  if (!username) {
+    username = prompt("Username:")!;
+  }
+  if (!username) {
+    return;
+  }
+  const pw = getpass("Password: ");
+  if (!pw) {
+    return;
+  }
+
+  console.log("Adding user to groups", options.group);
+
+  const store = new JSONKVStore();
+  try {
+    await store.load(authFile);
+  } catch (e: any) {
+    if (e instanceof Deno.errors.NotFound) {
+      console.log("Creating new auth database because it didn't exist.");
+    }
+  }
+  const auth = new Authenticator(store);
+  await auth.register(username!, pw!, options.group);
+  await store.save(authFile);
+}

cmd/user_chgrp.ts

@@ -0,0 +1,30 @@
+import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts";
+import { Authenticator } from "../server/auth.ts";
+
+export async function userChgrp(
+  options: any,
+  username?: string,
+) {
+  const authFile = options.auth || ".auth.json";
+  console.log("Using auth file", authFile);
+  if (!username) {
+    username = prompt("Username:")!;
+  }
+  if (!username) {
+    return;
+  }
+
+  console.log("Setting groups for user:", options.group);
+
+  const store = new JSONKVStore();
+  try {
+    await store.load(authFile);
+  } catch (e: any) {
+    if (e instanceof Deno.errors.NotFound) {
+      console.log("Creating new auth database because it didn't exist.");
+    }
+  }
+  const auth = new Authenticator(store);
+  await auth.setGroups(username!, options.group);
+  await store.save(authFile);
+}

cmd/user_delete.ts

@@ -0,0 +1,37 @@
+import getpass from "https://deno.land/x/getpass@0.3.1/mod.ts";
+import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts";
+import { Authenticator } from "../server/auth.ts";
+
+export async function userDelete(
+  options: any,
+  username?: string,
+) {
+  const authFile = options.auth || ".auth.json";
+  console.log("Using auth file", authFile);
+  if (!username) {
+    username = prompt("Username:")!;
+  }
+  if (!username) {
+    return;
+  }
+
+  const store = new JSONKVStore();
+  try {
+    await store.load(authFile);
+  } catch (e: any) {
+    if (e instanceof Deno.errors.NotFound) {
+      console.log("Creating new auth database because it didn't exist.");
+    }
+  }
+  const auth = new Authenticator(store);
+
+  const user = await auth.getUser(username);
+
+  if (!user) {
+    console.error("User", username, "not found.");
+    Deno.exit(1);
+  }
+
+  await auth.deleteUser(username!);
+  await store.save(authFile);
+}

cmd/user_passwd.ts

@@ -0,0 +1,42 @@
+import getpass from "https://deno.land/x/getpass@0.3.1/mod.ts";
+import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts";
+import { Authenticator } from "../server/auth.ts";
+
+export async function userPasswd(
+  options: any,
+  username?: string,
+) {
+  const authFile = options.auth || ".auth.json";
+  console.log("Using auth file", authFile);
+  if (!username) {
+    username = prompt("Username:")!;
+  }
+  if (!username) {
+    return;
+  }
+
+  const store = new JSONKVStore();
+  try {
+    await store.load(authFile);
+  } catch (e: any) {
+    if (e instanceof Deno.errors.NotFound) {
+      console.log("Creating new auth database because it didn't exist.");
+    }
+  }
+  const auth = new Authenticator(store);
+
+  const user = await auth.getUser(username);
+
+  if (!user) {
+    console.error("User", username, "not found.");
+    Deno.exit(1);
+  }
+
+  const pw = getpass("New password: ");
+  if (!pw) {
+    return;
+  }
+
+  await auth.setPassword(username!, pw!);
+  await store.save(authFile);
+}

common/deps.ts

@@ -61,7 +61,7 @@ export {
 } from "@codemirror/view";
 export type { DecorationSet, KeyBinding } from "@codemirror/view";
 
-export { markdown } from "https://esm.sh/@codemirror/lang-markdown@6.1.1?external=@codemirror/state,@lezer/common,@codemirror/language,@lezer/markdown,@codemirror/view,@lezer/highlight,@@codemirror/lang-html";
+export { markdown } from "https://esm.sh/@codemirror/lang-markdown@6.1.1?external=@codemirror/state,@lezer/common,@codemirror/language,@lezer/markdown,@codemirror/view,@lezer/highlight,@codemirror/lang-html";
 
 export {
   EditorSelection,

common/spaces/asset_bundle_space_primitives.ts

@@ -1,9 +1,7 @@
 import { FileMeta } from "../types.ts";
 import { SpacePrimitives } from "./space_primitives.ts";
 import { AssetBundle } from "../../plugos/asset_bundle/bundle.ts";
-import { mime } from "../deps.ts";
 
-const bootTime = Date.now();
 export class AssetBundlePlugSpacePrimitives implements SpacePrimitives {
   constructor(
     private wrapped: SpacePrimitives,
@@ -16,8 +14,8 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives {
     return this.assetBundle.listFiles()
       .map((p) => ({
         name: p,
-        contentType: mime.getType(p) || "application/octet-stream",
+        contentType: this.assetBundle.getMimeType(p),
-        lastModified: bootTime,
+        lastModified: this.assetBundle.getMtime(p),
         perm: "ro",
         size: -1,
       } as FileMeta)).concat(files);
@@ -32,10 +30,10 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives {
       return Promise.resolve({
         data,
         meta: {
-          lastModified: bootTime,
+          contentType: this.assetBundle.getMimeType(name),
+          lastModified: this.assetBundle.getMtime(name),
           size: data.byteLength,
           perm: "ro",
-          contentType: this.assetBundle.getMimeType(name),
         } as FileMeta,
       });
     }
@@ -46,10 +44,10 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives {
     if (this.assetBundle.has(name)) {
       const data = this.assetBundle.readFileSync(name);
       return Promise.resolve({
-        lastModified: bootTime,
+        contentType: this.assetBundle.getMimeType(name),
+        lastModified: this.assetBundle.getMtime(name),
         size: data.byteLength,
         perm: "ro",
-        contentType: this.assetBundle.getMimeType(name),
       } as FileMeta);
     }
     return this.wrapped.getFileMeta(name);

common/spaces/disk_space_primitives.ts

@@ -63,7 +63,7 @@ export class DiskSpacePrimitives implements SpacePrimitives {
       };
     } catch {
       // console.error("Error while reading file", name, e);
-      throw Error(`Could not read file ${name}`);
+      throw Error("Not found");
     }
   }
 

common/spaces/evented_space_primitives.ts

@@ -36,7 +36,7 @@ export class EventedSpacePrimitives implements SpacePrimitives {
       text = decoder.decode(data);
 
       this.eventHook
-        .dispatchEvent("page:saved", pageName)
+        .dispatchEvent("page:saved", pageName, newMeta)
         .then(() => {
           return this.eventHook.dispatchEvent("page:index_text", {
             name: pageName,

common/spaces/sync.ts

@@ -37,7 +37,10 @@ export class SpaceSync {
   ) {
   }
 
-  async syncFiles(snapshot: Map<string, SyncStatusItem>): Promise<number> {
+  async syncFiles(
+    snapshot: Map<string, SyncStatusItem>,
+    isSyncCandidate = this.options.isSyncCandidate,
+  ): Promise<number> {
     let operations = 0;
     console.log("[sync]", "Fetching snapshot from primary");
     const primaryAllPages = this.syncCandidates(
@@ -73,6 +76,9 @@ export class SpaceSync {
       // console.log("[sync]", "Iterating over all files");
       let filesProcessed = 0;
       for (const name of sortedFilenames) {
+        if (isSyncCandidate && !isSyncCandidate(name)) {
+          continue;
+        }
         try {
           operations += await this.syncFile(
             snapshot,

common/util.ts

@@ -41,14 +41,19 @@ export async function ensureSettingsAndIndex(
     settingsText = new TextDecoder().decode(
       (await space.readFile("SETTINGS.md")).data,
     );
-  } catch {
+  } catch (e: any) {
-    await space.writeFile(
+    if (e.message === "Not found") {
-      "SETTINGS.md",
+      await space.writeFile(
-      new TextEncoder().encode(SETTINGS_TEMPLATE),
+        "SETTINGS.md",
-      true,
+        new TextEncoder().encode(SETTINGS_TEMPLATE),
-    );
+        true,
+      );
+    } else {
+      console.error("Error reading settings", e.message);
+      console.error("Falling back to default settings");
+    }
     settingsText = SETTINGS_TEMPLATE;
-    // Ok, then let's also write the index page
+    // Ok, then let's also check the index page
     try {
       await space.getFileMeta("index.md");
     } catch {

import_map.json

@@ -1,10 +1,11 @@
 {
   "imports": {
     "@lezer/common": "https://esm.sh/@lezer/common@1.0.2",
-    "@lezer/lr": "https://esm.sh/@lezer/lr@1.3.5?external=@lezer/common",
+    "@lezer/lr": "https://esm.sh/@lezer/lr@1.3.5?external=@lezer/common&target=deno",
-    "@lezer/markdown": "https://esm.sh/@lezer/markdown@1.0.2?external=@lezer/common,@codemirror/language,@lezer/highlight",
+    "@lezer/markdown": "https://esm.sh/@lezer/markdown@1.0.2?external=@lezer/common,@codemirror/language,@lezer/highlight,@lezer/lr",
-    "@lezer/javascript": "https://esm.sh/@lezer/javascript@1.4.3?external=@lezer/common,@codemirror/language,@lezer/highlight",
+    "@lezer/javascript": "https://esm.sh/@lezer/javascript@1.4.3?external=@lezer/common,@codemirror/language,@lezer/highlight,@lezer/lr",
-    "@lezer/highlight": "https://esm.sh/@lezer/highlight@1.1.6?external=@lezer/common",
+    "@lezer/highlight": "https://esm.sh/@lezer/highlight@1.1.6?external=@lezer/common,@lezer/lr",
+    "@lezer/html": "https://esm.sh/@lezer/html@1.3.4?external=@lezer/common,@lezer/lr",
 
     "@codemirror/state": "https://esm.sh/@codemirror/state@6.2.1",
     "@codemirror/language": "https://esm.sh/@codemirror/language@6.7.0?external=@codemirror/state,@lezer/common,@lezer/lr,@codemirror/view,@lezer/highlight",
@@ -12,11 +13,12 @@
     "@codemirror/view": "https://esm.sh/@codemirror/view@6.9.0?external=@codemirror/state,@lezer/common",
     "@codemirror/autocomplete": "https://esm.sh/@codemirror/autocomplete@6.7.1?external=@codemirror/state,@codemirror/commands,@lezer/common,@codemirror/view",
     "@codemirror/lint": "https://esm.sh/@codemirror/lint@6.2.1?external=@codemirror/state,@lezer/common",
-    "@codemirror/lang-html": "https://esm.sh/@codemirror/lang-html@6.4.3?external=@codemirror/language,@codemirror/autocomplete,@codemirror/lang-css,@codemirror/state",
+    "@codemirror/lang-css": "https://esm.sh/@codemirror/lang-css@6.2.0?external=@codemirror/language,@codemirror/autocomplete,@codemirror/state,@lezer/lr,@lezer/html",
+    "@codemirror/lang-html": "https://esm.sh/@codemirror/lang-html@6.4.3?external=@codemirror/language,@codemirror/autocomplete,@codemirror/lang-css,@codemirror/state,@lezer/lr,@lezer/html",
     "@codemirror/search": "https://esm.sh/@codemirror/search@6.4.0?external=@codemirror/state,@codemirror/view",
 
     "preact": "https://esm.sh/preact@10.11.1",
-    "yjs": "https://esm.sh/yjs@13.5.42",
+    "yjs": "https://esm.sh/yjs@13.5.42?target=es2022",
     "$sb/": "./plug-api/",
     "handlebars": "https://esm.sh/handlebars@4.7.7?target=es2022",
     "dexie": "https://esm.sh/dexie@3.2.2"

plug-api/app_event.ts

@@ -9,6 +9,7 @@ export type AppEvent =
   | "editor:init"
   | "editor:pageLoaded"
   | "editor:pageReloaded"
+  | "editor:pageSaved"
   | "editor:modeswitch"
   | "plugs:loaded";
 

plug-api/silverbullet-syscall/editor.ts

@@ -42,8 +42,8 @@ export function reloadPage(): Promise<void> {
   return syscall("editor.reloadPage");
 }
 
-export function openUrl(url: string): Promise<void> {
+export function openUrl(url: string, existingWindow = false): Promise<void> {
-  return syscall("editor.openUrl", url);
+  return syscall("editor.openUrl", url, existingWindow);
 }
 
 // Force the client to download the file in dataUrl with filename as file name

plugos/deps.ts

@@ -6,5 +6,5 @@ export { mime } from "https://deno.land/x/mimetypes@v1.0.0/mod.ts";
 export { default as cacheDir } from "https://deno.land/x/cache_dir@0.2.0/mod.ts";
 export * as flags from "https://deno.land/std@0.165.0/flags/mod.ts";
 export * as esbuild from "https://deno.land/x/esbuild@v0.17.18/mod.js";
-export { denoPlugins } from "https://deno.land/x/esbuild_deno_loader@0.7.0/mod.ts";
+export { denoPlugins } from "https://deno.land/x/esbuild_deno_loader@0.8.1/mod.ts";
 export * as YAML from "https://deno.land/std@0.184.0/yaml/mod.ts";

plugos/hooks/event.ts

@@ -10,9 +10,9 @@ export type EventHookT = {
 
 export class EventHook implements Hook<EventHookT> {
   private system?: System<EventHookT>;
-  public localListeners: Map<string, ((data: any) => any)[]> = new Map();
+  public localListeners: Map<string, ((...args: any[]) => any)[]> = new Map();
 
-  addLocalListener(eventName: string, callback: (data: any) => any) {
+  addLocalListener(eventName: string, callback: (...args: any[]) => any) {
     if (!this.localListeners.has(eventName)) {
       this.localListeners.set(eventName, []);
     }
@@ -41,13 +41,13 @@ export class EventHook implements Hook<EventHookT> {
     return [...eventNames];
   }
 
-  async dispatchEvent(eventName: string, data?: any): Promise<any[]> {
+  async dispatchEvent(eventName: string, ...args: any[]): Promise<any[]> {
     if (!this.system) {
       throw new Error("Event hook is not initialized");
     }
     const responses: any[] = [];
     for (const plug of this.system.loadedPlugs.values()) {
-      const manifest = await plug.manifest;
+      const manifest = plug.manifest;
       for (
         const [name, functionDef] of Object.entries(
           manifest!.functions,
@@ -56,7 +56,7 @@ export class EventHook implements Hook<EventHookT> {
         if (functionDef.events && functionDef.events.includes(eventName)) {
           // Only dispatch functions that can run in this environment
           if (await plug.canInvoke(name)) {
-            const result = await plug.invoke(name, [data]);
+            const result = await plug.invoke(name, args);
             if (result !== undefined) {
               responses.push(result);
             }
@@ -67,7 +67,7 @@ export class EventHook implements Hook<EventHookT> {
     const localListeners = this.localListeners.get(eventName);
     if (localListeners) {
       for (const localListener of localListeners) {
-        const result = await Promise.resolve(localListener(data));
+        const result = await Promise.resolve(localListener(...args));
         if (result) {
           responses.push(result);
         }

plugos/lib/kv_store.json_file.ts

@@ -0,0 +1,71 @@
+import { KV, KVStore } from "./kv_store.ts";
+
+export class JSONKVStore implements KVStore {
+  private data: { [key: string]: any } = {};
+
+  async load(path: string) {
+    this.loadString(await Deno.readTextFile(path));
+  }
+
+  loadString(jsonString: string) {
+    this.data = JSON.parse(jsonString);
+  }
+
+  async save(path: string) {
+    await Deno.writeTextFile(path, JSON.stringify(this.data));
+  }
+
+  del(key: string): Promise<void> {
+    delete this.data[key];
+    return Promise.resolve();
+  }
+
+  deletePrefix(prefix: string): Promise<void> {
+    for (const key in this.data) {
+      if (key.startsWith(prefix)) {
+        delete this.data[key];
+      }
+    }
+    return Promise.resolve();
+  }
+
+  deleteAll(): Promise<void> {
+    this.data = {};
+    return Promise.resolve();
+  }
+
+  set(key: string, value: any): Promise<void> {
+    this.data[key] = value;
+    return Promise.resolve();
+  }
+  batchSet(kvs: KV[]): Promise<void> {
+    for (const kv of kvs) {
+      this.data[kv.key] = kv.value;
+    }
+    return Promise.resolve();
+  }
+  batchDelete(keys: string[]): Promise<void> {
+    for (const key of keys) {
+      delete this.data[key];
+    }
+    return Promise.resolve();
+  }
+  batchGet(keys: string[]): Promise<any[]> {
+    return Promise.resolve(keys.map((key) => this.data[key]));
+  }
+  get(key: string): Promise<any> {
+    return Promise.resolve(this.data[key]);
+  }
+  has(key: string): Promise<boolean> {
+    return Promise.resolve(key in this.data);
+  }
+  queryPrefix(keyPrefix: string): Promise<{ key: string; value: any }[]> {
+    const results: { key: string; value: any }[] = [];
+    for (const key in this.data) {
+      if (key.startsWith(keyPrefix)) {
+        results.push({ key, value: this.data[key] });
+      }
+    }
+    return Promise.resolve(results);
+  }
+}

plugs/collab/constants.ts

@@ -0,0 +1 @@
+export const collabPingInterval = 2500;

plugs/core/account.ts

@@ -0,0 +1,5 @@
+import { editor } from "$sb/silverbullet-syscall/mod.ts";
+
+export async function accountLogoutCommand() {
+  await editor.openUrl("/.client/logout.html", true);
+}

plugs/core/core.plug.yaml

@@ -326,22 +326,21 @@ functions:
     path: ./debug.ts:parsePageCommand
     command:
       name: "Debug: Parse Document"
-  resetClientCommand:
-    path: ./debug.ts:resetClientCommand
-    command:
-      name: "Debug: Reset Client"
-
 
   versionCommand:
     path: ./help.ts:versionCommand
     command:
       name: "Help: Version"
-
   gettingStartedCommand:
     path: ./help.ts:gettingStartedCommand
     command:
       name: "Help: Getting Started"
 
+  accountLogoutCommand:
+    path: ./account.ts:accountLogoutCommand
+    command:
+      name: "Account: Logout"
+
   # Link unfurl infrastructure
   unfurlLink:
     path: ./link.ts:unfurlCommand

plugs/core/debug.ts

@@ -10,7 +10,3 @@ export async function parsePageCommand() {
     ),
   );
 }
-
-export async function resetClientCommand() {
-  editor.openUrl("/.client/reset.html");
-}

plugs/core/page.ts

@@ -9,7 +9,6 @@ import {
   index,
   markdown,
   space,
-  system,
 } from "$sb/silverbullet-syscall/mod.ts";
 
 import { events } from "$sb/plugos-syscall/mod.ts";

plugs/core/plugmanager.ts

@@ -1,5 +1,4 @@
 import { events } from "$sb/plugos-syscall/mod.ts";
-import type { Manifest } from "../../common/manifest.ts";
 import { editor, space, system } from "$sb/silverbullet-syscall/mod.ts";
 import { readYamlPage } from "$sb/lib/yaml_page.ts";
 import { builtinPlugNames } from "../builtin_plugs.ts";
@@ -21,7 +20,7 @@ export async function updatePlugsCommand() {
         );
       }
     } catch (e: any) {
-      if (e.message.includes("Could not read file")) {
+      if (e.message.includes("Not found")) {
         console.warn("No PLUGS page found, not loading anything");
         return;
       }

server/auth.ts

@@ -0,0 +1,120 @@
+import { KVStore } from "../plugos/lib/kv_store.ts";
+
+export type User = {
+  username: string;
+  passwordHash: string; // hashed password
+  salt: string;
+  groups: string[]; // special "admin"
+};
+
+async function createUser(
+  username: string,
+  password: string,
+  groups: string[],
+  salt = generateSalt(16),
+): Promise<User> {
+  return {
+    username,
+    passwordHash: await hashSHA256(`${salt}${password}`),
+    salt,
+    groups,
+  };
+}
+
+const userPrefix = `u:`;
+
+export class Authenticator {
+  constructor(private store: KVStore) {
+  }
+
+  async register(
+    username: string,
+    password: string,
+    groups: string[],
+    salt?: string,
+  ): Promise<void> {
+    await this.store.set(
+      `${userPrefix}${username}`,
+      await createUser(username, password, groups, salt),
+    );
+  }
+
+  async authenticateHashed(
+    username: string,
+    hashedPassword: string,
+  ): Promise<boolean> {
+    const user = await this.store.get(`${userPrefix}${username}`) as User;
+    if (!user) {
+      return false;
+    }
+    return user.passwordHash === hashedPassword;
+  }
+
+  async authenticate(
+    username: string,
+    password: string,
+  ): Promise<string | undefined> {
+    const user = await this.store.get(`${userPrefix}${username}`) as User;
+    if (!user) {
+      return undefined;
+    }
+    const hashedPassword = await hashSHA256(`${user.salt}${password}`);
+    return user.passwordHash === hashedPassword ? hashedPassword : undefined;
+  }
+
+  async getAllUsers(): Promise<User[]> {
+    return (await this.store.queryPrefix(userPrefix)).map((item) => item.value);
+  }
+
+  getUser(username: string): Promise<User | undefined> {
+    return this.store.get(`${userPrefix}${username}`);
+  }
+
+  async setPassword(username: string, password: string): Promise<void> {
+    const user = await this.getUser(username);
+    if (!user) {
+      throw new Error(`User does not exist`);
+    }
+    user.passwordHash = await hashSHA256(`${user.salt}${password}`);
+    await this.store.set(`${userPrefix}${username}`, user);
+  }
+
+  async deleteUser(username: string): Promise<void> {
+    const user = await this.getUser(username);
+    if (!user) {
+      throw new Error(`User does not exist`);
+    }
+    await this.store.del(`${userPrefix}${username}`);
+  }
+
+  async setGroups(username: string, groups: string[]): Promise<void> {
+    const user = await this.getUser(username);
+    if (!user) {
+      throw new Error(`User does not exist`);
+    }
+    user.groups = groups;
+    await this.store.set(`${userPrefix}${username}`, user);
+  }
+}
+
+async function hashSHA256(message: string): Promise<string> {
+  // Transform the string into an ArrayBuffer
+  const encoder = new TextEncoder();
+  const data = encoder.encode(message);
+
+  // Generate the hash
+  const hashBuffer = await window.crypto.subtle.digest("SHA-256", data);
+
+  // Transform the hash into a hex string
+  return Array.from(new Uint8Array(hashBuffer)).map((b) =>
+    b.toString(16).padStart(2, "0")
+  ).join("");
+}
+
+function generateSalt(length: number): string {
+  const array = new Uint8Array(length / 2); // because two characters represent one byte in hex
+  crypto.getRandomValues(array);
+  return Array.from(array, (byte) => ("00" + byte.toString(16)).slice(-2)).join(
+    "",
+  );
+}

server/collab.test.ts

@@ -0,0 +1,46 @@
+import { assert, assertEquals } from "../test_deps.ts";
+import { CollabServer } from "./collab.ts";
+
+Deno.test("Collab server", async () => {
+  const collabServer = new CollabServer(null as any);
+  console.log("Client 1 joins page 1");
+  assertEquals(collabServer.updatePresence("client1", "page1"), {});
+  assertEquals(collabServer.pages.size, 1);
+  console.log("CLient 1 leaves page 1");
+  assertEquals(collabServer.updatePresence("client1", undefined, "page1"), {});
+  assertEquals(collabServer.pages.size, 0);
+  assertEquals(collabServer.updatePresence("client1", "page1"), {});
+  console.log("Client 1 joins page 2");
+  assertEquals(collabServer.updatePresence("client1", "page2", "page1"), {});
+  assertEquals(collabServer.pages.size, 1);
+  console.log("Client 2 joins to page 2, collab id created, but not exposed");
+  assertEquals(
+    collabServer.updatePresence("client2", "page2").collabId,
+    undefined,
+  );
+  assert(
+    collabServer.updatePresence("client1", "page2").collabId !== undefined,
+  );
+  console.log("Client 2 moves to page 1, collab id destroyed");
+  assertEquals(collabServer.updatePresence("client2", "page1", "page2"), {});
+  assertEquals(collabServer.updatePresence("client1", "page2", "page2"), {});
+  assertEquals(collabServer.pages.get("page2")!.collabId, undefined);
+  assertEquals(collabServer.pages.get("page1")!.collabId, undefined);
+  console.log("Going to cleanup, which should have no effect");
+  collabServer.cleanup(50);
+  assertEquals(collabServer.pages.size, 2);
+  collabServer.updatePresence("client2", "page2", "page1");
+  console.log("Going to sleep 20ms");
+  await sleep(20);
+  console.log("Then client 1 pings, but client 2 does not");
+  collabServer.updatePresence("client1", "page2", "page2");
+  await sleep(20);
+  console.log("Going to cleanup, which should clean client 2");
+  collabServer.cleanup(35);
+  assertEquals(collabServer.pages.get("page2")!.collabId, undefined);
+  console.log(collabServer);
+});
+
+function sleep(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

server/collab.ts

@@ -0,0 +1,231 @@
+import { getAvailablePortSync } from "https://deno.land/x/port@1.0.0/mod.ts";
+import { nanoid } from "https://esm.sh/nanoid@4.0.0";
+import { race, timeout } from "../common/async_util.ts";
+import { Application } from "./deps.ts";
+import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
+import { collabPingInterval } from "../plugs/collab/constants.ts";
+import { Hocuspocus } from "./deps.ts";
+
+type CollabPage = {
+  clients: Map<string, number>; // clientId -> lastPing
+  collabId?: string;
+  // The currently elected provider of the initial document
+  masterClientId: string;
+};
+
+export class CollabServer {
+  // clients: Map<string, { openPage: string; lastPing: number }> = new Map();
+  pages: Map<string, CollabPage> = new Map();
+  yCollabServer?: Hocuspocus;
+
+  constructor(private spacePrimitives: SpacePrimitives) {
+  }
+
+  start() {
+    setInterval(() => {
+      this.cleanup(3 * collabPingInterval);
+    }, collabPingInterval);
+  }
+
+  updatePresence(
+    clientId: string,
+    currentPage?: string,
+    previousPage?: string,
+  ): { collabId?: string } {
+    if (previousPage && currentPage !== previousPage) {
+      // Client switched pages
+      // Update last page record
+      const lastCollabPage = this.pages.get(previousPage);
+      if (lastCollabPage) {
+        lastCollabPage.clients.delete(clientId);
+        if (lastCollabPage.clients.size === 1) {
+          delete lastCollabPage.collabId;
+        }
+
+        if (lastCollabPage.clients.size === 0) {
+          this.pages.delete(previousPage);
+        } else {
+          // Elect a new master client
+          if (lastCollabPage.masterClientId === clientId) {
+            // Any is fine, really
+            lastCollabPage.masterClientId =
+              [...lastCollabPage.clients.keys()][0];
+          }
+        }
+      }
+    }
+
+    if (currentPage) {
+      // Update new page
+      let nextCollabPage = this.pages.get(currentPage);
+      if (!nextCollabPage) {
+        // Newly opened page (no other clients on this page right now)
+        nextCollabPage = {
+          clients: new Map(),
+          masterClientId: clientId,
+        };
+        this.pages.set(currentPage, nextCollabPage);
+      }
+      // Register last ping from us
+      nextCollabPage.clients.set(clientId, Date.now());
+
+      if (nextCollabPage.clients.size > 1 && !nextCollabPage.collabId) {
+        // Create a new collabId
+        nextCollabPage.collabId = nanoid();
+      }
+      // console.log("State", this.pages);
+      if (nextCollabPage.collabId) {
+        // We will now expose this collabId, except when we're just starting this session
+        // in which case we'll wait for the original client to publish the document
+        const existingyCollabSession = this.yCollabServer?.documents.get(
+          buildCollabId(nextCollabPage.collabId, `${currentPage}.md`),
+        );
+        if (existingyCollabSession) {
+          // console.log("Found an existing collab session already, let's join!");
+          return { collabId: nextCollabPage.collabId };
+        } else if (clientId === nextCollabPage.masterClientId) {
+          // console.log("We're the master, so we should connect");
+          return { collabId: nextCollabPage.collabId };
+        } else {
+          // We're not the first client, so we need to wait for the first client to connect
+          // console.log("We're not the master, so we should wait");
+          return {};
+        }
+      } else {
+        return {};
+      }
+    } else {
+      return {};
+    }
+  }
+
+  cleanup(timeout: number) {
+    // Clean up pages and their clients that haven't pinged for some time
+    for (const [pageName, page] of this.pages) {
+      for (const [clientId, lastPing] of page.clients) {
+        if (Date.now() - lastPing > timeout) {
+          // Eject client
+          page.clients.delete(clientId);
+          // Elect a new master client
+          if (page.masterClientId === clientId && page.clients.size > 0) {
+            page.masterClientId = [...page.clients.keys()][0];
+          }
+        }
+      }
+      if (page.clients.size === 1) {
+        // If there's only one client left, we don't need to keep the collabId around anymore
+        delete page.collabId;
+      }
+      if (page.clients.size === 0) {
+        // And if we have no clients left, well...
+        this.pages.delete(pageName);
+      }
+    }
+  }
+
+  route(app: Application) {
+    // The way this works is that we spin up a separate WS server locally and then proxy requests to it
+    // This is the only way I could get Hocuspocus to work with Deno
+    const internalPort = getAvailablePortSync();
+    this.yCollabServer = new Hocuspocus({
+      port: internalPort,
+      address: "127.0.0.1",
+      quiet: true,
+      onStoreDocument: async (data) => {
+        const [_, path] = splitCollabId(data.documentName);
+        const text = data.document.getText("codemirror").toString();
+        console.log(
+          "[Hocuspocus]",
+          "Persisting",
+          path,
+          "to space on server",
+        );
+        const meta = await this.spacePrimitives.writeFile(
+          path,
+          new TextEncoder().encode(text),
+        );
+        // Broadcast new persisted lastModified date
+        data.document.broadcastStateless(
+          JSON.stringify({
+            type: "persisted",
+            path,
+            lastModified: meta.lastModified,
+          }),
+        );
+        return;
+      },
+      onDisconnect: (client) => {
+        console.log("[Hocuspocus]", "Client disconnected", client.clientsCount);
+        if (client.clientsCount === 0) {
+          console.log(
+            "[Hocuspocus]",
+            "Last client disconnected from",
+            client.documentName,
+            "purging from memory",
+          );
+          this.yCollabServer!.documents.delete(client.documentName);
+        }
+        return Promise.resolve();
+      },
+    });
+
+    this.yCollabServer.listen();
+
+    app.use((ctx) => {
+      // if (ctx.request.url.pathname === "/.ws") {
+      //   const sock = ctx.upgrade();
+      //   sock.onmessage = (e) => {
+      //     console.log("WS: Got message", e.data);
+      //   };
+      // }
+      // Websocket proxy to hocuspocus
+      if (ctx.request.url.pathname === "/.ws-collab") {
+        const sock = ctx.upgrade();
+
+        const ws = new WebSocket(`ws://localhost:${internalPort}`);
+        const wsReady = race([
+          new Promise<void>((resolve) => {
+            ws.onopen = () => {
+              resolve();
+            };
+          }),
+          timeout(1000),
+        ]).catch(() => {
+          console.error("Timeout waiting for collab to open websocket");
+          sock.close();
+        });
+        sock.onmessage = (e) => {
+          // console.log("Got message", e);
+          wsReady.then(() => ws.send(e.data)).catch(console.error);
+        };
+        sock.onclose = () => {
+          if (ws.OPEN) {
+            ws.close();
+          }
+        };
+        ws.onmessage = (e) => {
+          if (sock.OPEN) {
+            sock.send(e.data);
+          } else {
+            console.error("Got message from websocket but socket is not open");
+          }
+        };
+        ws.onclose = () => {
+          if (sock.OPEN) {
+            sock.close();
+          }
+        };
+      }
+    });
+  }
+}
+
+function splitCollabId(documentName: string): [string, string] {
+  const [collabId, ...pathPieces] = documentName.split("/");
+  const path = pathPieces.join("/");
+  return [collabId, path];
+}
+
+function buildCollabId(collabId: string, path: string): string {
+  return `${collabId}/${path}`;
+}

server/deps.ts

@@ -1,3 +1,5 @@
 export * from "../common/deps.ts";
 export { Application, Router } from "https://deno.land/x/oak@v12.4.0/mod.ts";
 export * as etag from "https://deno.land/x/oak@v12.4.0/etag.ts";
+
+export { Hocuspocus } from "npm:@hocuspocus/server@2.1.0";

server/http_server.ts

@@ -7,13 +7,15 @@ import { performLocalFetch } from "../common/proxy_fetch.ts";
 import { BuiltinSettings } from "../web/types.ts";
 import { gitIgnoreCompiler } from "./deps.ts";
 import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts";
+import { CollabServer } from "./collab.ts";
+import { Authenticator } from "./auth.ts";
 
 export type ServerOptions = {
   hostname: string;
   port: number;
   pagesPath: string;
   clientAssetBundle: AssetBundle;
-  user?: string;
+  authenticator: Authenticator;
   pass?: string;
   certFile?: string;
   keyFile?: string;
@@ -24,11 +26,12 @@ export class HttpServer {
   app: Application;
   private hostname: string;
   private port: number;
-  user?: string;
   abortController?: AbortController;
   clientAssetBundle: AssetBundle;
   settings?: BuiltinSettings;
   spacePrimitives: SpacePrimitives;
+  collab: CollabServer;
+  authenticator: Authenticator;
 
   constructor(
     spacePrimitives: SpacePrimitives,
@@ -37,7 +40,7 @@ export class HttpServer {
     this.hostname = options.hostname;
     this.port = options.port;
     this.app = new Application();
-    this.user = options.user;
+    this.authenticator = options.authenticator;
     this.clientAssetBundle = options.clientAssetBundle;
 
     let fileFilterFn: (s: string) => boolean = () => true;
@@ -62,6 +65,8 @@ export class HttpServer {
         }
       },
     );
+    this.collab = new CollabServer(this.spacePrimitives);
+    this.collab.start();
   }
 
   // Replaces some template variables in index.html in a rather ad-hoc manner, but YOLO
@@ -123,7 +128,8 @@ export class HttpServer {
     this.app.use(({ request, response }, next) => {
       if (
         !request.url.pathname.startsWith("/.fs") &&
-        request.url.pathname !== "/.auth"
+        request.url.pathname !== "/.auth" &&
+        !request.url.pathname.startsWith("/.ws")
       ) {
         response.headers.set("Content-type", "text/html");
         response.body = this.renderIndexHtml();
@@ -134,10 +140,12 @@ export class HttpServer {
 
     // Pages API
     const fsRouter = this.buildFsRouter(this.spacePrimitives);
-    this.addPasswordAuth(this.app);
+    await this.addPasswordAuth(this.app);
     this.app.use(fsRouter.routes());
     this.app.use(fsRouter.allowedMethods());
 
+    this.collab.route(this.app);
+
     this.abortController = new AbortController();
     const listenOptions: any = {
       hostname: this.hostname,
@@ -168,25 +176,40 @@ export class HttpServer {
     this.settings = await ensureSettingsAndIndex(this.spacePrimitives);
   }
 
-  private addPasswordAuth(app: Application) {
+  private async addPasswordAuth(app: Application) {
     const excludedPaths = [
       "/manifest.json",
       "/favicon.png",
       "/logo.png",
       "/.auth",
     ];
-    if (this.user) {
+    if ((await this.authenticator.getAllUsers()).length > 0) {
-      const b64User = btoa(this.user);
       app.use(async ({ request, response, cookies }, next) => {
         if (!excludedPaths.includes(request.url.pathname)) {
           const authCookie = await cookies.get("auth");
-          if (!authCookie || authCookie !== b64User) {
+          if (!authCookie) {
             response.status = 401;
             response.body = "Unauthorized, please authenticate";
             return;
           }
+          const [username, hashedPassword] = authCookie.split(":");
+          if (
+            !await this.authenticator.authenticateHashed(
+              username,
+              hashedPassword,
+            )
+          ) {
+            response.status = 401;
+            response.body = "Invalid username/password, please reauthenticate";
+            return;
+          }
         }
+
         if (request.url.pathname === "/.auth") {
+          if (request.url.search === "?logout") {
+            await cookies.delete("auth");
+            // Implicit fallthrough to login page
+          }
           if (request.method === "GET") {
             response.headers.set("Content-type", "text/html");
             response.body = this.clientAssetBundle.readTextFileSync(
@@ -195,11 +218,15 @@ export class HttpServer {
             return;
           } else if (request.method === "POST") {
             const values = await request.body({ type: "form" }).value;
-            const username = values.get("username"),
+            const username = values.get("username")!,
-              password = values.get("password"),
+              password = values.get("password")!,
               refer = values.get("refer");
-            if (this.user === `${username}:${password}`) {
+            const hashedPassword = await this.authenticator.authenticate(
-              await cookies.set("auth", b64User, {
+              username,
+              password,
+            );
+            if (hashedPassword) {
+              await cookies.set("auth", `${username}:${hashedPassword}`, {
                 expires: new Date(Date.now() + 1000 * 60 * 60 * 24 * 7), // in a week
                 sameSite: "strict",
               });
@@ -273,6 +300,19 @@ export class HttpServer {
             });
             return;
           }
+          case "presence": {
+            // RPC to check (for collab purposes) which client has what page open
+            response.headers.set("Content-Type", "application/json");
+            console.log("Got presence update", body);
+            response.body = JSON.stringify(
+              this.collab.updatePresence(
+                body.clientId,
+                body.currentPage,
+                body.previousPage,
+              ),
+            );
+            return;
+          }
           default:
             response.headers.set("Content-Type", "text/plain");
             response.status = 400;
@@ -290,6 +330,11 @@ export class HttpServer {
       .get("\/(.+)", async ({ params, response, request }) => {
         const name = params[0];
         console.log("Loading file", name);
+        if (name.startsWith(".")) {
+          // Don't expose hidden files
+          response.status = 404;
+          return;
+        }
         try {
           const attachmentData = await spacePrimitives.readFile(
             name,
@@ -322,6 +367,11 @@ export class HttpServer {
       .put("\/(.+)", async ({ request, response, params }) => {
         const name = params[0];
         console.log("Saving file", name);
+        if (name.startsWith(".")) {
+          // Don't expose hidden files
+          response.status = 403;
+          return;
+        }
 
         let body: Uint8Array;
         if (

silverbullet.ts

@@ -7,6 +7,10 @@ import { upgradeCommand } from "./cmd/upgrade.ts";
 import { versionCommand } from "./cmd/version.ts";
 import { serveCommand } from "./cmd/server.ts";
 import { plugCompileCommand } from "./cmd/plug_compile.ts";
+import { userAdd } from "./cmd/user_add.ts";
+import { userPasswd } from "./cmd/user_passwd.ts";
+import { userDelete } from "./cmd/user_delete.ts";
+import { userChgrp } from "./cmd/user_chgrp.ts";
 
 await new Command()
   .name("silverbullet")
@@ -27,6 +31,10 @@ await new Command()
     "--user <user:string>",
     "'username:password' combo for BasicAuth authentication",
   )
+  .option(
+    "--auth <auth.json:string>",
+    "User authentication file to use for authentication",
+  )
   .option(
     "--cert <certFile:string>",
     "Path to TLS certificate",
@@ -58,6 +66,42 @@ await new Command()
   .option("--importmap <path:string>", "Path to import map file to use")
   .option("--runtimeUrl <url:string>", "URL to worker_runtime.ts to use")
   .action(plugCompileCommand)
+  .command("user:add", "Add a new user to an authentication file")
+  .arguments("[username:string]")
+  .option(
+    "--auth <auth.json:string>",
+    "User authentication file to use",
+  )
+  .option("-G, --group <name:string>", "Add user to group", {
+    collect: true,
+    default: [] as string[],
+  })
+  .action(userAdd)
+  .command("user:delete", "Delete an existing user")
+  .arguments("[username:string]")
+  .option(
+    "--auth <auth.json:string>",
+    "User authentication file to use",
+  )
+  .action(userDelete)
+  .command("user:chgrp", "Update user groups")
+  .arguments("[username:string]")
+  .option(
+    "--auth <auth.json:string>",
+    "User authentication file to use",
+  )
+  .option("-G, --group <name:string>", "Groups to put user into", {
+    collect: true,
+    default: [] as string[],
+  })
+  .action(userChgrp)
+  .command("user:passwd", "Set the password for an existing user")
+  .arguments("[username:string]")
+  .option(
+    "--auth <auth.json:string>",
+    "User authentication file to use",
+  )
+  .action(userPasswd)
   // upgrade
   .command("upgrade", "Upgrade SilverBullet")
   .action(upgradeCommand)

web/auth.html

@@ -56,8 +56,14 @@
 <body>
   <header>
     <h1>Login to <img src="/.client/logo.png" style="height: 1ch;" /> SilverBullet</h1>
+    <script>
+      function saveUsername() {
+        localStorage.setItem("username", document.getElementsByName("username")[0].value);
+        return true;
+      }
+    </script>
   </header>
-  <form action="/.auth" method="POST">
+  <form action="/.auth" method="POST" onsubmit="saveUsername()">
     <input type="hidden" name="refer" value="" />
     <div class="error-message"></div>
     <div>

web/cm_plugins/collab.ts

@@ -1,4 +1,6 @@
-import { Extension, WebsocketProvider, Y, yCollab } from "../deps.ts";
+import { safeRun } from "../../common/util.ts";
+import { Extension, HocuspocusProvider, Y, yCollab } from "../deps.ts";
+import { SyncService } from "../sync_service.ts";
 
 const userColors = [
   { color: "#30bced", light: "#30bced33" },
@@ -12,27 +14,45 @@ const userColors = [
 ];
 
 export class CollabState {
-  ydoc: Y.Doc;
+  public ytext: Y.Text;
-  collabProvider: WebsocketProvider;
+  collabProvider: HocuspocusProvider;
-  ytext: Y.Text;
+  private yundoManager: Y.UndoManager;
-  yundoManager: Y.UndoManager;
+  interval?: number;
 
-  constructor(serverUrl: string, token: string, username: string) {
+  constructor(
-    this.ydoc = new Y.Doc();
+    serverUrl: string,
-    this.collabProvider = new WebsocketProvider(
+    readonly path: string,
-      serverUrl,
+    readonly token: string,
-      token,
+    username: string,
-      this.ydoc,
+    private syncService: SyncService,
-    );
+    public isLocalCollab: boolean,
+  ) {
+    this.collabProvider = new HocuspocusProvider({
+      url: serverUrl,
+      name: token,
+
+      // Receive broadcasted messages from the server (right now only "page has been persisted" notifications)
+      onStateless: (
+        { payload },
+      ) => {
+        const message = JSON.parse(payload);
+        switch (message.type) {
+          case "persisted": {
+            // Received remote persist notification, updating snapshot
+            syncService.updateRemoteLastModified(
+              message.path,
+              message.lastModified,
+            ).catch(console.error);
+          }
+        }
+      },
+    });
 
     this.collabProvider.on("status", (e: any) => {
       console.log("Collab status change", e);
     });
-    this.collabProvider.on("sync", (e: any) => {
-      console.log("Sync status", e);
-    });
 
-    this.ytext = this.ydoc.getText("codemirror");
+    this.ytext = this.collabProvider.document.getText("codemirror");
     this.yundoManager = new Y.UndoManager(this.ytext);
 
     const randomColor =
@@ -43,10 +63,32 @@ export class CollabState {
       color: randomColor.color,
       colorLight: randomColor.light,
     });
+    if (isLocalCollab) {
+      syncService.excludeFromSync(path).catch(console.error);
+
+      this.interval = setInterval(() => {
+        // Ping the store to make sure the file remains in exclusion
+        syncService.excludeFromSync(path).catch(console.error);
+      }, 1000);
+    }
   }
 
   stop() {
+    console.log("[COLLAB] Destroying collab provider");
+    if (this.interval) {
+      clearInterval(this.interval);
+    }
     this.collabProvider.destroy();
+    // For whatever reason, destroy() doesn't properly clean up everything so we need to help a bit
+    this.collabProvider.configuration.websocketProvider.webSocket = null;
+    this.collabProvider.configuration.websocketProvider.destroy();
+
+    // When stopping collaboration, we're going back to sync mode. Make sure we got the latest and greatest remote timestamp to avoid
+    // conflicts
+    safeRun(async () => {
+      await this.syncService.unExcludeFromSync(this.path);
+      await this.syncService.fetchAndPersistRemoteLastModified(this.path);
+    });
   }
 
   collabExtension(): Extension {

web/collab_manager.ts

@@ -0,0 +1,88 @@
+import { nanoid } from "https://esm.sh/nanoid@4.0.0";
+import type { Editor } from "./editor.tsx";
+
+const collabPingInterval = 2500;
+
+export class CollabManager {
+  clientId = nanoid();
+  localCollabServer: string;
+
+  constructor(private editor: Editor) {
+    this.localCollabServer = location.protocol === "http:"
+      ? `ws://${location.host}/.ws-collab`
+      : `wss://${location.host}/.ws-collab`;
+    editor.eventHook.addLocalListener(
+      "editor:pageLoaded",
+      (pageName, previousPage) => {
+        console.log("Page loaded", pageName, previousPage);
+        this.updatePresence(pageName, previousPage).catch(console.error);
+      },
+    );
+  }
+
+  start() {
+    setInterval(() => {
+      this.updatePresence(this.editor.currentPage!).catch(console.error);
+    }, collabPingInterval);
+  }
+
+  async updatePresence(currentPage?: string, previousPage?: string) {
+    try {
+      const resp = await this.editor.remoteSpacePrimitives.authenticatedFetch(
+        this.editor.remoteSpacePrimitives.url,
+        {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+          },
+          body: JSON.stringify({
+            operation: "presence",
+            clientId: this.clientId,
+            previousPage,
+            currentPage,
+          }),
+          keepalive: true, // important for beforeunload event
+        },
+      );
+      const { collabId } = await resp.json();
+
+      if (this.editor.collabState && !this.editor.collabState.isLocalCollab) {
+        // We're in a remote collab mode, don't do anything
+        return;
+      }
+
+      // console.log("Collab ID", collabId);
+      const previousCollabId = this.editor.collabState?.token.split("/")[0];
+      if (!collabId && this.editor.collabState) {
+        // Stop collab
+        console.log("Stopping collab");
+        if (this.editor.collabState.path === `${currentPage}.md`) {
+          this.editor.flashNotification(
+            "Other users have left this page, switched back to single-user mode.",
+          );
+        }
+        this.editor.stopCollab();
+      } else if (collabId && collabId !== previousCollabId) {
+        // Start collab
+        console.log("Starting collab");
+        this.editor.flashNotification(
+          "Opening page in multi-user mode.",
+        );
+        this.editor.startCollab(
+          this.localCollabServer,
+          `${collabId}/${currentPage}.md`,
+          this.editor.getUsername(),
+          true,
+        );
+      }
+    } catch (e: any) {
+      // console.error("Ping error", e);
+      if (
+        e.message.toLowerCase().includes("failed") && this.editor.collabState
+      ) {
+        console.log("Offline, stopping collab");
+        this.editor.stopCollab();
+      }
+    }
+  }
+}

web/deps.ts

@@ -21,7 +21,7 @@ export {
   yCollab,
   yUndoManagerKeymap,
 } from "https://esm.sh/y-codemirror.next@0.3.2?external=yjs,@codemirror/state,@codemirror/commands,@codemirror/history,@codemirror/view";
-export { WebsocketProvider } from "https://esm.sh/y-websocket@1.4.5?external=yjs";
+export { HocuspocusProvider } from "https://esm.sh/@hocuspocus/provider@2.1.0?external=yjs,ws&target=es2022";
 
 // Vim mode
 export {

web/editor.tsx

@@ -136,7 +136,7 @@ import { HttpSpacePrimitives } from "../common/spaces/http_space_primitives.ts";
 import { FallbackSpacePrimitives } from "../common/spaces/fallback_space_primitives.ts";
 import { syncSyscalls } from "./syscalls/sync.ts";
 import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts";
-import { globToRegExp } from "https://deno.land/std@0.189.0/path/glob.ts";
+import { CollabManager } from "./collab_manager.ts";
 
 const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/;
 
@@ -193,6 +193,7 @@ export class Editor {
   syncService: SyncService;
   settings?: BuiltinSettings;
   kvStore: DexieKVStore;
+  collabManager: CollabManager;
 
   constructor(
     parent: Element,
@@ -214,6 +215,8 @@ export class Editor {
     this.eventHook = new EventHook();
     system.addHook(this.eventHook);
 
+    this.collabManager = new CollabManager(this);
+
     // Cron hook
     const cronHook = new CronHook(system);
     system.addHook(cronHook);
@@ -368,6 +371,11 @@ export class Editor {
       }
     });
 
+    globalThis.addEventListener("beforeunload", (e) => {
+      console.log("Pinging with with undefined page name");
+      this.collabManager.updatePresence(undefined, this.currentPage);
+    });
+
     this.eventHook.addLocalListener("plug:changed", async (fileName) => {
       console.log("Plug updated, reloading:", fileName);
       system.unload(fileName);
@@ -389,7 +397,8 @@ export class Editor {
 
     this.space.on({
       pageChanged: (meta) => {
-        if (this.currentPage === meta.name) {
+        // Only reload when watching the current page (to avoid reloading when switching pages and in collab mode)
+        if (this.space.watchInterval && this.currentPage === meta.name) {
           console.log("Page changed elsewhere, reloading");
           this.flashNotification("Page changed elsewhere, reloading");
           this.reloadPage();
@@ -471,6 +480,7 @@ export class Editor {
 
     // Kick off background sync
     this.syncService.start();
+    this.collabManager.start();
 
     this.eventHook.addLocalListener("sync:success", async (operations) => {
       // console.log("Operations", operations);
@@ -557,8 +567,13 @@ export class Editor {
                 this.editorView!.state.sliceDoc(0),
                 true,
               )
-              .then(() => {
+              .then(async (meta) => {
                 this.viewDispatch({ type: "page-saved" });
+                await this.dispatchAppEvent(
+                  "editor:pageSaved",
+                  this.currentPage,
+                  meta,
+                );
                 resolve();
               })
               .catch((e) => {
@@ -656,8 +671,8 @@ export class Editor {
     });
   }
 
-  dispatchAppEvent(name: AppEvent, data?: any): Promise<any[]> {
+  dispatchAppEvent(name: AppEvent, ...args: any[]): Promise<any[]> {
-    return this.eventHook.dispatchEvent(name, data);
+    return this.eventHook.dispatchEvent(name, ...args);
   }
 
   createEditorState(
@@ -950,38 +965,42 @@ export class Editor {
             touchCount = 0;
           },
           mousedown: (event: MouseEvent, view: EditorView) => {
-            // Make sure <a> tags are clicked without moving the cursor there
-            if (!event.altKey && event.target instanceof Element) {
-              const parentA = event.target.closest("a");
-              if (parentA) {
-                event.stopPropagation();
-                event.preventDefault();
-                const clickEvent: ClickEvent = {
-                  page: pageName,
-                  ctrlKey: event.ctrlKey,
-                  metaKey: event.metaKey,
-                  altKey: event.altKey,
-                  pos: view.posAtCoords({
-                    x: event.x,
-                    y: event.y,
-                  })!,
-                };
-                this.dispatchAppEvent("page:click", clickEvent).catch(
-                  console.error,
-                );
-              }
-            }
-          },
-          click: (event: MouseEvent, view: EditorView) => {
             safeRun(async () => {
-              const clickEvent: ClickEvent = {
+              const pos = view.posAtCoords(event);
+              if (!pos) {
+                return;
+              }
+              const potentialClickEvent: ClickEvent = {
                 page: pageName,
                 ctrlKey: event.ctrlKey,
                 metaKey: event.metaKey,
                 altKey: event.altKey,
-                pos: view.posAtCoords(event)!,
+                pos: view.posAtCoords({
+                  x: event.x,
+                  y: event.y,
+                })!,
               };
-              await this.dispatchAppEvent("page:click", clickEvent);
+              // Make sure <a> tags are clicked without moving the cursor there
+              if (!event.altKey && event.target instanceof Element) {
+                const parentA = event.target.closest("a");
+                if (parentA) {
+                  event.stopPropagation();
+                  event.preventDefault();
+                  await this.dispatchAppEvent(
+                    "page:click",
+                    potentialClickEvent,
+                  );
+                  return;
+                }
+              }
+
+              const distanceX = event.x - view.coordsAtPos(pos)!.left;
+              // What we're trying to determine here is if the click occured anywhere near the looked up position
+              // this may not be the case with locations that expand signifcantly based on live preview (such as links), we don't want any accidental clicks
+              // Fixes #357
+              if (distanceX <= view.defaultCharacterWidth) {
+                await this.dispatchAppEvent("page:click", potentialClickEvent);
+              }
             });
           },
         }),
@@ -1107,6 +1126,10 @@ export class Editor {
     this.editorView!.focus();
   }
 
+  getUsername(): string {
+    return localStorage.getItem("username") || "you";
+  }
+
   async navigate(
     name: string,
     pos?: number | string,
@@ -1144,8 +1167,7 @@ export class Editor {
         await this.save(true);
         // And stop the collab session
         if (this.collabState) {
-          this.collabState.stop();
+          this.stopCollab();
-          this.collabState = undefined;
         }
       }
     }
@@ -1187,9 +1209,10 @@ export class Editor {
 
     // Note: these events are dispatched asynchronously deliberately (not waiting for results)
     if (loadingDifferentPage) {
-      this.eventHook.dispatchEvent("editor:pageLoaded", pageName).catch(
+      this.eventHook.dispatchEvent("editor:pageLoaded", pageName, previousPage)
-        console.error,
+        .catch(
-      );
+          console.error,
+        );
     } else {
       this.eventHook.dispatchEvent("editor:pageReloaded", pageName).catch(
         console.error,
@@ -1226,10 +1249,18 @@ export class Editor {
     if (pageState) {
       // Restore state
       editorView.scrollDOM.scrollTop = pageState!.scrollTop;
-      editorView.dispatch({
+      try {
-        selection: pageState.selection,
+        editorView.dispatch({
-        scrollIntoView: true,
+          selection: pageState.selection,
-      });
+          scrollIntoView: true,
+        });
+      } catch {
+        // This is fine, just go to the top
+        editorView.dispatch({
+          selection: { anchor: 0 },
+          scrollIntoView: true,
+        });
+      }
     } else {
       editorView.scrollDOM.scrollTop = 0;
       editorView.dispatch({
@@ -1502,19 +1533,49 @@ export class Editor {
     return;
   }
 
-  startCollab(serverUrl: string, token: string, username: string) {
+  startCollab(
+    serverUrl: string,
+    token: string,
+    username: string,
+    isLocalCollab = false,
+  ) {
     if (this.collabState) {
       // Clean up old collab state
       this.collabState.stop();
     }
     const initialText = this.editorView!.state.sliceDoc();
-    this.collabState = new CollabState(serverUrl, token, username);
+    this.collabState = new CollabState(
-    this.collabState.collabProvider.once("sync", (synced: boolean) => {
+      serverUrl,
-      if (this.collabState?.ytext.toString() === "") {
+      `${this.currentPage!}.md`,
-        console.log("Synced value is empty, putting back original text");
+      token,
-        this.collabState?.ytext.insert(0, initialText);
+      username,
+      this.syncService,
+      isLocalCollab,
+    );
+
+    this.collabState.collabProvider.on("synced", () => {
+      if (this.collabState!.ytext.toString() === "") {
+        console.log(
+          "[Collab]",
+          "Synced value is empty (new collab session), inserting local copy",
+        );
+        this.collabState!.ytext.insert(0, initialText);
       }
     });
+
     this.rebuildEditorState();
+
+    // Don't watch for local changes in this mode
+    this.space.unwatch();
+  }
+
+  stopCollab() {
+    if (this.collabState) {
+      this.collabState.stop();
+      this.collabState = undefined;
+      this.rebuildEditorState();
+    }
+    // Start file watching again
+    this.space.watch();
   }
 }

web/logout.html

@@ -55,13 +55,16 @@
 
 <body>
     <header>
-        <h1>Reset page</h1>
+        <h1>Logout</h1>
     </header>
-    <button onclick="resetAll()">Flush everything</button>
+    <button onclick="resetAll()">Logout</button>
-    <button onclick="javascript:location='/'">Back</button>
+    <button onclick="javascript:location='/'">Cancel</button>
 
     <script>
         function resetAll() {
+            // Reset local storage username
+            localStorage.removeItem("username");
+
             if (indexedDB.databases) {
                 // get a list of all existing IndexedDB databases
                 indexedDB.databases().then((databases) => {
@@ -75,8 +78,12 @@
                         })
                     );
                 }).then(() => {
-                    alert("All IndexedDB databases deleted");
+                    alert("Flushed local data, you're now logged out");
+                    location.href = "/.auth?logout";
                 });
+            } else {
+                alert("Cannot flush local data (Firefox user?), will now log you out");
+                location.href = "/.auth?logout";
             }
 
             if (navigator.serviceWorker) {
@@ -90,7 +97,6 @@
                         navigator.serviceWorker.getRegistrations().then((registrations) => {
                             for (const registration of registrations) {
                                 registration.unregister();
-                                alert("Service worker unregistered");
                             }
                         });
 

web/navigator.ts

@@ -31,7 +31,7 @@ export class PathPageNavigator {
         `${this.root}/${encodedPage}`,
       );
     }
-    window.dispatchEvent(
+    globalThis.dispatchEvent(
       new PopStateEvent("popstate", {
         state: { page, pos },
       }),
@@ -60,12 +60,12 @@ export class PathPageNavigator {
         }
       });
     };
-    window.addEventListener("popstate", cb);
+    globalThis.addEventListener("popstate", cb);
     cb();
   }
 
   decodeURI(): [string, number | string] {
-    let [page, pos] = decodeURI(
+    const [page, pos] = decodeURI(
       location.pathname.substring(this.root.length + 1),
     ).split("@");
     if (pos) {

web/reducer.ts

@@ -43,6 +43,8 @@ export default function reducer(
       return {
         ...state,
         showPageNavigator: true,
+        showCommandPalette: false,
+        showFilterBox: false,
       };
     case "stop-navigate":
       return {
@@ -69,6 +71,8 @@ export default function reducer(
       return {
         ...state,
         showCommandPalette: true,
+        showPageNavigator: false,
+        showFilterBox: false,
         showCommandPaletteContext: action.context,
       };
     }

web/service_worker.ts

@@ -8,7 +8,7 @@ const CACHE_NAME = "{{CACHE_NAME}}";
 
 const precacheFiles = Object.fromEntries([
   "/",
-  "/.client/reset.html",
+  "/.client/logout.html",
   "/.client/client.js",
   "/.client/favicon.png",
   "/.client/iAWriterMonoS-Bold.woff2",
@@ -83,13 +83,14 @@ self.addEventListener("fetch", (event: any) => {
 
         const requestUrl = new URL(event.request.url);
         const pathname = requestUrl.pathname;
+        // console.log("In service worker, pathname is", pathname);
         // If this is a /.fs request, this can either be a plug worker load or an attachment load
         if (pathname.startsWith("/.fs")) {
           if (fileContentTable && !event.request.headers.has("x-sync-mode")) {
-            console.log(
+            // console.log(
-              "Attempting to serve file from locally synced space:",
+            //   "Attempting to serve file from locally synced space:",
-              pathname,
+            //   pathname,
-            );
+            // );
             // Don't fetch from DB when in sync mode (because then updates won't sync)
             const path = decodeURIComponent(
               requestUrl.pathname.slice("/.fs/".length),
@@ -97,7 +98,7 @@ self.addEventListener("fetch", (event: any) => {
             return fileContentTable.get(path).then(
               (data) => {
                 if (data) {
-                  console.log("Serving from space", path);
+                  // console.log("Serving from space", path);
                   return new Response(data.data, {
                     headers: {
                       "Content-type": mime.getType(path) ||

web/space.ts

@@ -51,7 +51,7 @@ export class Space extends EventEmitter<SpaceEvents> {
     super();
     this.kvStore.get("imageHeightCache").then((cache) => {
       if (cache) {
-        console.log("Loaded image height cache from KV store", cache);
+        // console.log("Loaded image height cache from KV store", cache);
         this.imageHeightCache = cache;
       }
     });
@@ -200,13 +200,9 @@ export class Space extends EventEmitter<SpaceEvents> {
   writeAttachment(
     name: string,
     data: Uint8Array,
-    selfUpdate?: boolean | undefined,
+    selfUpdate?: boolean,
   ): Promise<AttachmentMeta> {
-    return this.spacePrimitives.writeFile(
+    return this.spacePrimitives.writeFile(name, data, selfUpdate);
-      name,
-      data as Uint8Array,
-      selfUpdate,
-    );
   }
 
   deleteAttachment(name: string): Promise<void> {

web/sync_service.ts

@@ -1,3 +1,4 @@
+import { sleep } from "../common/async_util.ts";
 import type { SpacePrimitives } from "../common/spaces/space_primitives.ts";
 import {
   SpaceSync,
@@ -16,6 +17,8 @@ const syncStartTimeKey = "syncStartTime";
 // Keeps the last time an activity was registered, used to detect if a sync is still alive and whether a new one should be started already
 const syncLastActivityKey = "syncLastActivity";
 
+const syncExcludePrefix = "syncExclude:";
+
 // maximum time between two activities before we consider a sync crashed
 const syncMaxIdleTimeout = 1000 * 20; // 20s
 
@@ -31,8 +34,8 @@ export class SyncService {
   lastReportedSyncStatus = Date.now();
 
   constructor(
-    private localSpacePrimitives: SpacePrimitives,
+    readonly localSpacePrimitives: SpacePrimitives,
-    private remoteSpace: SpacePrimitives,
+    readonly remoteSpace: SpacePrimitives,
     private kvStore: KVStore,
     private eventHook: EventHook,
     private isSyncCandidate: (path: string) => boolean,
@@ -53,8 +56,17 @@ export class SyncService {
       await this.syncFile(`${name}.md`);
     });
 
-    eventHook.addLocalListener("page:saved", async (name) => {
+    eventHook.addLocalListener("editor:pageSaved", async (name, meta) => {
-      await this.syncFile(`${name}.md`);
+      const path = `${name}.md`;
+      await this.syncFile(path);
+      if (await this.isExcludedFromSync(path)) {
+        // So we're editing a page and just saved it, but it's excluded from sync
+        // Assumption: we're in collab mode for this file, so we're going to constantly update our local hash
+        // console.log(
+        //   "Locally updating last modified in snapshot because we're in collab mode",
+        // );
+        await this.updateLocalLastModified(path, meta.lastModified);
+      }
     });
   }
 
@@ -82,8 +94,13 @@ export class SyncService {
 
   async registerSyncStart(): Promise<void> {
     // Assumption: this is called after an isSyncing() check
-    await this.kvStore.set(syncStartTimeKey, Date.now());
+    await this.kvStore.batchSet([{
-    await this.kvStore.set(syncLastActivityKey, Date.now());
+      key: syncStartTimeKey,
+      value: Date.now(),
+    }, {
+      key: syncLastActivityKey,
+      value: Date.now(),
+    }]);
   }
 
   async registerSyncProgress(status?: SyncStatus): Promise<void> {
@@ -101,6 +118,40 @@ export class SyncService {
     await this.kvStore.del(syncStartTimeKey);
   }
 
+  // Temporarily exclude a specific file from sync (e.g. when in collab mode)
+  excludeFromSync(path: string): Promise<void> {
+    return this.kvStore.set(syncExcludePrefix + path, Date.now());
+  }
+
+  unExcludeFromSync(path: string): Promise<void> {
+    return this.kvStore.del(syncExcludePrefix + path);
+  }
+
+  async isExcludedFromSync(path: string): Promise<boolean> {
+    const lastExcluded = await this.kvStore.get(syncExcludePrefix + path);
+    return lastExcluded && Date.now() - lastExcluded < syncMaxIdleTimeout;
+  }
+
+  async fetchAllExcludedFromSync(): Promise<string[]> {
+    const entries = await this.kvStore.queryPrefix(syncExcludePrefix);
+    const expiredPaths: string[] = [];
+    const now = Date.now();
+    const result = entries.filter(({ key, value }) => {
+      if (now - value > syncMaxIdleTimeout) {
+        expiredPaths.push(key);
+        return false;
+      }
+      return true;
+    }).map(({ key }) => key.slice(syncExcludePrefix.length));
+
+    if (expiredPaths.length > 0) {
+      console.log("Purging expired sync exclusions: ", expiredPaths);
+      await this.kvStore.batchDelete(expiredPaths);
+    }
+
+    return result;
+  }
+
   async getSnapshot(): Promise<Map<string, SyncStatusItem>> {
     const snapshot = (await this.kvStore.get(syncSnapshotKey)) || {};
     return new Map<string, SyncStatusItem>(
@@ -108,6 +159,91 @@ export class SyncService {
     );
   }
 
+  // Await a moment when the sync is no longer running
+  async noOngoingSync(): Promise<void> {
+    // Not completely safe, could have race condition on setting the syncStartTimeKey
+    while (await this.isSyncing()) {
+      await sleep(100);
+    }
+  }
+
+  // When in collab mode, we delegate the sync to the CDRT engine, to avoid conflicts, we try to keep the lastModified time in sync with the remote
+  async updateRemoteLastModified(path: string, lastModified: number) {
+    await this.noOngoingSync();
+    await this.registerSyncStart();
+    const snapshot = await this.getSnapshot();
+    const entry = snapshot.get(path);
+    if (entry) {
+      snapshot.set(path, [entry[0], lastModified]);
+    } else {
+      // In the unlikely scenario that a space first openen on a collab page before every being synced
+      try {
+        console.log(
+          "Received lastModified time for file not in snapshot",
+          path,
+          lastModified,
+        );
+        snapshot.set(path, [
+          (await this.localSpacePrimitives.getFileMeta(path)).lastModified,
+          lastModified,
+        ]);
+      } catch (e) {
+        console.warn(
+          "Received lastModified time for non-existing file not in snapshot",
+          path,
+          lastModified,
+          e,
+        );
+      }
+    }
+    await this.saveSnapshot(snapshot);
+    await this.registerSyncStop();
+  }
+
+  // Reach out out to remote space, fetch the latest lastModified time and update the local snapshot
+  // This is used when exiting collab mode
+  async fetchAndPersistRemoteLastModified(path: string) {
+    const meta = await this.remoteSpace.getFileMeta(path);
+    await this.updateRemoteLastModified(
+      path,
+      meta.lastModified,
+    );
+  }
+
+  // When in collab mode, we delegate the sync to the CDRT engine, to avoid conflicts, we try to keep the lastModified time in sync when local changes happen
+  async updateLocalLastModified(path: string, lastModified: number) {
+    await this.noOngoingSync();
+    await this.registerSyncStart();
+    const snapshot = await this.getSnapshot();
+    const entry = snapshot.get(path);
+    if (entry) {
+      snapshot.set(path, [lastModified, entry[1]]);
+    } else {
+      // In the unlikely scenario that a space first openen on a collab page before every being synced
+      try {
+        console.log(
+          "Setting lastModified time for file not in snapshot",
+          path,
+          lastModified,
+        );
+        snapshot.set(path, [
+          lastModified,
+          (await this.localSpacePrimitives.getFileMeta(path)).lastModified,
+        ]);
+      } catch (e) {
+        console.warn(
+          "Received lastModified time for non-existing file not in snapshot",
+          path,
+          lastModified,
+          e,
+        );
+      }
+    }
+    await this.saveSnapshot(snapshot);
+    await this.registerSyncStop();
+    // console.log("All done!");
+  }
+
   start() {
     this.syncSpace().catch(
       console.error,
@@ -135,8 +271,15 @@ export class SyncService {
     await this.registerSyncStart();
     let operations = 0;
     const snapshot = await this.getSnapshot();
+    // Fetch the list of files that are excluded from sync (e.g. because they're in collab mode)
+    const excludedFromSync = await this.fetchAllExcludedFromSync();
+    // console.log("Excluded from sync", excludedFromSync);
     try {
-      operations = await this.spaceSync!.syncFiles(snapshot);
+      operations = await this.spaceSync!.syncFiles(
+        snapshot,
+        (path) =>
+          this.isSyncCandidate(path) && !excludedFromSync.includes(path),
+      );
       this.eventHook.dispatchEvent("sync:success", operations);
     } catch (e: any) {
       this.eventHook.dispatchEvent("sync:error", e.message);
@@ -152,15 +295,15 @@ export class SyncService {
       // console.log("Already syncing");
       return;
     }
-    if (!this.isSyncCandidate(name)) {
+    if (!this.isSyncCandidate(name) || (await this.isExcludedFromSync(name))) {
       return;
     }
     await this.registerSyncStart();
     console.log("Syncing file", name);
     const snapshot = await this.getSnapshot();
     try {
-      let localHash: number | undefined = undefined;
+      let localHash: number | undefined;
-      let remoteHash: number | undefined = undefined;
+      let remoteHash: number | undefined;
       try {
         localHash =
           (await this.localSpacePrimitives.getFileMeta(name)).lastModified;
@@ -169,8 +312,7 @@ export class SyncService {
       }
       try {
         // This is wasteful, but Netlify (silverbullet.md) doesn't support OPTIONS call (404s) so we'll just fetch the whole file
-        const { meta } = await this.remoteSpace!.readFile(name);
+        remoteHash = (await this.remoteSpace!.readFile(name)).meta.lastModified;
-        remoteHash = meta.lastModified;
       } catch (e: any) {
         if (e.message === "Not found") {
           // File doesn't exist remotely, that's ok
@@ -220,10 +362,8 @@ export class SyncService {
       name,
       "will pick the version from secondary and be done with it.",
     );
-    const fileMeta = await primary.getFileMeta(name);
-
     // Read file from secondary
-    const { data } = await secondary.readFile(
+    const { data, meta } = await secondary.readFile(
       name,
     );
     // Write file to primary
@@ -231,13 +371,14 @@ export class SyncService {
       name,
       data,
       false,
-      fileMeta.lastModified,
+      meta.lastModified,
     );
     // Update snapshot
     snapshot.set(name, [
       newMeta.lastModified,
-      fileMeta.lastModified,
+      meta.lastModified,
     ]);
+
     return 1;
   }
 }

web/syscalls/collab.ts

@@ -14,7 +14,7 @@ export function collabSyscalls(editor: Editor): SysCallMapping {
     "collab.stop": (
       _ctx,
     ) => {
-      editor.collabState?.stop();
+      editor.stopCollab();
     },
   };
 }

web/syscalls/editor.ts

@@ -32,10 +32,14 @@ export function editorSyscalls(editor: Editor): SysCallMapping {
     "editor.reloadPage": async () => {
       await editor.reloadPage();
     },
-    "editor.openUrl": (_ctx, url: string) => {
+    "editor.openUrl": (_ctx, url: string, existingWindow = false) => {
-      const win = window.open(url, "_blank");
+      if (!existingWindow) {
-      if (win) {
+        const win = window.open(url, "_blank");
-        win.focus();
+        if (win) {
+          win.focus();
+        }
+      } else {
+        location.href = url;
       }
     },
     "editor.downloadFile": (_ctx, filename: string, dataUrl: string) => {

website/Authentication.md

@@ -0,0 +1,46 @@
+SilverBullet supports simple authentication for one or many users.
+
+**Note**: This feature is experimental and will likely change significantly over time.
+
+## Single User
+By simply passing the `--user` flag with a username:password combination, you enable authentication for a single user. For instance:
+
+```shell
+silverbullet --user pete:1234 .
+```
+
+Will let `pete` authenticate with password `1234`. 
+
+## Multiple users
+Although multi-user support is still rudimentary, it is possible to have multiple users authenticate. These users can be configured using an JSON authentication file that SB can generate for you. It is usually named `.auth.json`. 
+
+You can enable authentication as follows:
+
+```shell
+silverbullet --auth /path/to/.auth.json
+```
+
+To create and manage an `.auth.json` file you can use the following commands:
+
+* `silverbullet user:add --auth /path/to/.auth.json [username]` to add a user
+* `silverbullet user:delete --auth /path/to/.auth.json [username]` to delete a user
+* `silverbullet user:passwd --auth /path/to/.auth.json [username]` to update a password
+
+If the `.auth.json` file does not yet exist, it will be created.
+
+When SB is run with a `--auth` flag, this fill will automatically be reloaded upon change.
+
+### Group management
+While this functionality is not yet used, users can also be added to groups, which can be arbitrarily named. Likely the `admin` group will have special meaning down the line. 
+
+When adding a user, you can add one more `-G` or `--group` flags:
+
+```shell
+silverbullet user:add --auth /path/to/.auth.json -G admin pete
+```
+
+And you can update these groups later with `silverbullet user:chgrp`:
+
+```shell
+silverbullet user:chgrp --auth /path/to/.auth.json -G admin pete
+```

website/CHANGELOG.md

@@ -3,8 +3,12 @@ release.
 
 ## Next
 
+* **Real-time collaboration support** between clients: Open the same page in multiple windows (browser tabs, mobile devices) and within a few seconds you should get kicked into real-time collaboration mode, showing other participants cursors, selections and edits in real time (Google doc style). This only works when a connection with the server can be established.
+  * This **breaks** existing [[🔌 Collab]] links, since we switched real-time collaboration libraries. We’re still looking at the best way to keep supporting this feature.
+* [[Authentication|Multi-user authentication]]: you can now allow multiple user accounts authenticate, which makes the real-time collaboration support actually useful. This feature is still experimental and will likely evolve over time.
 * Added `spaceIgnore` setting to not sync specific folders or file patterns to the client, see [[SETTINGS]] for documentation
-
+* Much improved image loading behavior on page (previously scroll bars would jump up and down like a mad person)
+* Various bug fixes and quality of life improvements
 
 ---
 
@@ -23,7 +27,7 @@ A detailed description of what happened [can be found in this PR](https://github
 * From a UI perspective little changes, except for a few things related to sync:
   * While SB is in an out-of-sync state, the title bar will appear yellow. This will also happen when it cannot reach the server. SB is still fully functional in this state. Once the connection is restored, all changes while offline are synced back to the server.
   * Upon initial load, a full sync will take place, which — depending on the size of your space — may take some time. Or even blow up completely, if you have a big amount of data there.
-* To reset your browser state (flush out your entire space, caches and data stores) visit the `/.client/reset.html` page, e.g. at http://localhost:3000/.client/reset.html and push the button. Note that any unsynced changes will be wiped.
+* To reset your browser state (flush out your entire space, caches and data stores) visit the `/.client/logout.html` page, e.g. at http://localhost:3000/.client/logout.html and push the button. Note that any unsynced changes will be wiped.
 
 Besides these architectural changes, a few other breaking changes were made to seize the moment:
 * **In plugs**:

website/Install.md

@@ -32,7 +32,7 @@ silverbullet <pages-path>
 
 By default, SilverBullet will bind to port `3000`; to use a different port, use the `-p` flag.
 
-For security reasons, by default, SilverBullet only allows connections via `localhost` (or `127.0.0.1`). To also allow connections from the network, pass a `-L 0.0.0.0` flag (0.0.0.0 for all connections, or insert a specific address to limit the host), ideally combined with `--user username:password` to add BasicAuth password protection. Credentials can also be specified with the `SB_USER` environment variable, `SB_USER=username:password`. If both are specified, the `--user` flag takes precedence.
+For security reasons, by default, SilverBullet only allows connections via `localhost` (or `127.0.0.1`). To also allow connections from the network, pass a `-L 0.0.0.0` flag (0.0.0.0 for all connections, or insert a specific address to limit the host), ideally combined with `--user username:password` to add BasicAuth password protection.
 
 Once downloaded and booted, SilverBullet will print out a URL to open SB in your browser. Please make note of [[@tls|the use of HTTPs]].
 
@@ -103,4 +103,12 @@ Caddy can automatically find these certificates once provisioned, so you can jus
 $ sudo caddy reverse-proxy --to :3000 --from yourserver.yourtsdomain.ts.net:443
 ```
 
-If you access SilverBullet via plain HTTP (outside of localhost) everything _should_ still mostly work, except offline mode.
+If you access SilverBullet via plain HTTP (outside of localhost) everything _should_ still mostly work, except offline mode.
+
+## Environment variables
+You can configure SB with environment variables instead of flags as well. The following environment variables are supported:
+
+* `SB_USER`: Sets single-user credentials (like `--user`), e.g. `SB_USER=pete:1234`
+* `SB_PORT`: Sets the port to listen to, e.g. `SB_PORT=1234`
+* `SB_FOLDER`: Sets the folder to expose, e.g. `SB_FOLDER=/space`
+* `SB_AUTH`: Loads an [[Authentication]] database from a (JSON encoded) string, e.g. `SB_AUTH=$(cat /path/to/.auth.json)`

website/🔌 Collab.md

@@ -4,7 +4,7 @@ repo: https://github.com/silverbulletmd/silverbullet
 share-support: true
 ---
 
-The Collab plug implements real-time “Google Doc” style collaboration with other SilverBullet users using the [Yjs](https://yjs.dev) library. It supports:
+The Collab plug implements real-time “Google Doc” style collaboration with other SilverBullet users using the [Hocuspocus](https://hocuspocus.dev/) library. It supports:
 
 * Real-time editing
 * Showing other participant’s cursors and selections
@@ -27,14 +27,15 @@ To use it:
 5. If the collaborator wants to keep a persistent copy of the page collaborated page, they can simply _rename_ the page to something not prefixed with `collab:`. Everything will keep working for as long as the `collab:` will appear in the `$share` attribute of [[Frontmatter]]
 
 ## How it works
-The Collab plug uses Yjs for real-time collaboration via a WebSocket. A random ID is assigned to every shared page, and a copy of this page (as well as its history) will be stored on the collaboration server. Therefore, be cautious about what you share, especially when using a public collab server like `collab.silverbullet.md`. For “production use” we recommend deploying your own collab server.
+The Collab plug uses Hocuspocus for real-time collaboration via a WebSocket. A random ID is assigned to every shared page, and a copy of this page (as well as its history) will be stored on the collaboration server. Therefore, be cautious about what you share, especially when using a public collab server like `collab.silverbullet.md`. For “production use” we recommend deploying your own collab server.
 
 ## Deploying your own collab server
 $deploy
-A detailed description of how to deploy your own collab server [can be found here](https://github.com/yjs/y-websocket). The short version is:
+
+Collaboration uses the excellent Hocuspocus library. You can easily deploy your own collaboration server as follows (requires node.js and npm):
 
 ```shell
-HOST=0.0.0.0 PORT=1337 YPERSISTENCE=./store npx y-websocket
+npx @hocuspocus/cli@2.0.6 --sqlite documents.db --port 1337
 ```
 
-This will run the `y-websocket` server on port 1337, and store page data persistently in `./store`. You can connect to this server via `ws://ip:1337`. To use SSL, put a TLS server in front of it, in which case you can use `wss://` instead.
+This will run the hocuspocus server on port 1337, and store page data persistently in a SQLite database `documents.db`. You can connect to this server via `ws://ip:1337`. To use SSL, put a TLS terminator in front of it, in which case you can use `wss://` instead.