No More Collab. Fixes #449

* Fully removes real-time collaboration
* URL scheme rewrite

.gitignore

@@ -13,3 +13,5 @@ node_modules
 *.db
 test_space
 silverbullet
+# Local Netlify folder
+.netlify

common/hooks/page_namespace.ts

@@ -69,7 +69,7 @@ export class PageNamespaceHook implements Hook<PageNamespaceHookT> {
     if (!manifest.functions) {
       return [];
     }
-    for (let [funcName, funcDef] of Object.entries(manifest.functions)) {
+    for (const [funcName, funcDef] of Object.entries(manifest.functions)) {
       if (funcDef.pageNamespace) {
         if (!funcDef.pageNamespace.pattern) {
           errors.push(`Function ${funcName} has a namespace but no pattern`);

common/spaces/http_space_primitives.ts

@@ -26,30 +26,31 @@ export class HttpSpacePrimitives implements SpacePrimitives {
     });
     if (result.redirected) {
       // Got a redirect, we'll assume this is due to invalid credentials and redirecting to an auth page
-      console.log("Got a redirect via the API so will redirect to URL", url);
+      console.log(
+        "Got a redirect via the API so will redirect to URL",
+        result.url,
+      );
       location.href = result.url;
       throw new Error("Invalid credentials");
     }
     return result;
   }
 
-  getRealStatus(r: Response) {
-    if (r.headers.get("X-Status")) {
-      return +r.headers.get("X-Status")!;
-    }
-    return r.status;
-  }
-
   async fetchFileList(): Promise<FileMeta[]> {
-    const resp = await this.authenticatedFetch(this.url, {
+    const resp = await this.authenticatedFetch(`${this.url}/index.json`, {
       method: "GET",
+      headers: {
+        Accept: "application/json",
+      },
     });
 
     if (
-      this.getRealStatus(resp) === 200 &&
+      resp.status === 200 &&
       this.expectedSpacePath &&
       resp.headers.get("X-Space-Path") !== this.expectedSpacePath
     ) {
+      console.log("Expected space path", this.expectedSpacePath);
+      console.log("Got space path", resp.headers.get("X-Space-Path"));
       await flushCachesAndUnregisterServiceWorker();
       alert("Space folder path different on server, reloading the page");
       location.reload();
@@ -67,7 +68,7 @@ export class HttpSpacePrimitives implements SpacePrimitives {
         method: "GET",
       },
     );
-    if (this.getRealStatus(res) === 404) {
+    if (res.status === 404) {
       throw new Error(`Not found`);
     }
     return {
@@ -109,7 +110,7 @@ export class HttpSpacePrimitives implements SpacePrimitives {
         method: "DELETE",
       },
     );
-    if (this.getRealStatus(req) !== 200) {
+    if (req.status !== 200) {
       throw Error(`Failed to delete file: ${req.statusText}`);
     }
   }
@@ -118,10 +119,10 @@ export class HttpSpacePrimitives implements SpacePrimitives {
     const res = await this.authenticatedFetch(
       `${this.url}/${encodeURI(name)}`,
       {
-        method: "OPTIONS",
+        method: "HEAD",
       },
     );
-    if (this.getRealStatus(res) === 404) {
+    if (res.status === 404) {
       throw new Error(`Not found`);
     }
     return this.responseToMeta(name, res);
@@ -130,7 +131,7 @@ export class HttpSpacePrimitives implements SpacePrimitives {
   private responseToMeta(name: string, res: Response): FileMeta {
     return {
       name,
-      size: +res.headers.get("X-Content-Length")!,
+      size: +res.headers.get("Content-Length")!,
       contentType: res.headers.get("Content-type")!,
       lastModified: +(res.headers.get("X-Last-Modified") || "0"),
       perm: (res.headers.get("X-Permission") as "rw" | "ro") || "rw",

import_map.json

@@ -18,7 +18,6 @@
     "@codemirror/search": "https://esm.sh/@codemirror/search@6.4.0?external=@codemirror/state,@codemirror/view&target=es2022",
 
     "preact": "https://esm.sh/preact@10.11.1",
-    "yjs": "https://esm.sh/yjs@13.5.42?deps=lib0@0.2.70&target=es2022",
     "$sb/": "./plug-api/",
     "handlebars": "https://esm.sh/handlebars@4.7.7?target=es2022",
     "dexie": "https://esm.sh/dexie@3.2.2"

plug-api/lib/page.ts

@@ -0,0 +1,4 @@
+export function isValidPageName(name: string): boolean {
+  // Page can not be empty and not end with a file extension (e.g. "bla.md")
+  return name !== "" && !/\.[a-zA-Z]+$/.test(name);
+}

plug-api/lib/settings_page.ts

@@ -1,8 +1,7 @@
-import { readYamlPage } from "./yaml_page.ts";
+import { readYamlPage } from "$sb/lib/yaml_page.ts";
-import { notifyUser } from "./util.ts";
 import { YAML } from "$sb/plugos-syscall/mod.ts";
 
-import { space } from "$sb/silverbullet-syscall/mod.ts";
+import { editor, space } from "$sb/silverbullet-syscall/mod.ts";
 
 /**
  * Convenience function to read a specific set of settings from the `SETTINGS` page as well as default values
@@ -65,7 +64,7 @@ export async function writeSettings<T extends object>(settings: T) {
   try {
     readSettings = (await readYamlPage(SETTINGS_PAGE, ["yaml"])) || {};
   } catch {
-    await notifyUser("Creating a new SETTINGS page...", "info");
+    await editor.flashNotification("Creating a new SETTINGS page...", "info");
   }
   const writeSettings: any = { ...readSettings, ...settings };
   // const doc = new YAML.Document();

plug-api/lib/util.ts

@@ -1,36 +0,0 @@
-import { editor } from "$sb/silverbullet-syscall/mod.ts";
-
-export async function replaceAsync(
-  str: string,
-  regex: RegExp,
-  asyncFn: (match: string, ...args: any[]) => Promise<string>,
-) {
-  const promises: Promise<string>[] = [];
-  str.replace(regex, (match: string, ...args: any[]): string => {
-    const promise = asyncFn(match, ...args);
-    promises.push(promise);
-    return "";
-  });
-  const data = await Promise.all(promises);
-  return str.replace(regex, () => data.shift()!);
-}
-
-export function isServer() {
-  return (
-    typeof window === "undefined" || typeof window.document === "undefined"
-  ); // if something defines window the same way as the browser, this will fail.
-}
-
-// this helps keep if's condition as positive
-export function isBrowser() {
-  return !isServer();
-}
-
-export function notifyUser(message: string, type?: "info" | "error") {
-  if (isBrowser()) {
-    return editor.flashNotification(message, type);
-  }
-  const log = type === "error" ? console.error : console.log;
-  log(message); // we should end up sending the message to the user, users dont read logs.
-  return;
-}

plug-api/silverbullet-syscall/collab.ts

@@ -1,9 +0,0 @@
-import { syscall } from "./syscall.ts";
-
-export function start(serverUrl: string, token: string, username: string) {
-  return syscall("collab.start", serverUrl, token, username);
-}
-
-export function stop() {
-  return syscall("collab.stop");
-}

plug-api/silverbullet-syscall/mod.ts

@@ -3,7 +3,6 @@ export * as index from "./index.ts";
 export * as markdown from "./markdown.ts";
 export * as space from "./space.ts";
 export * as system from "./system.ts";
-export * as collab from "./collab.ts";
 // Legacy redirect, use "store" in $sb/plugos-syscall/mod.ts instead
 export * as clientStore from "./store.ts";
 export * as sync from "./sync.ts";

plugos/asset_bundle/builder.ts

@@ -38,6 +38,7 @@ export async function bundleFolder(
   bundlePath: string,
 ) {
   const bundle = new AssetBundle();
+
   await Deno.mkdir(path.dirname(bundlePath), { recursive: true });
   for await (
     const { path: filePath } of walk(rootPath, { includeDirs: false })

plugs/builtin_plugs.ts

@@ -1,6 +1,5 @@
 // TODO: Figure out how to keep this up-to-date automatically
 export const builtinPlugNames = [
-  "collab",
   "core",
   "directive",
   "emoji",

plugs/collab/collab.plug.yaml

@@ -1,36 +0,0 @@
-name: collab
-functions:
-  detectCollabPage:
-    path: "./collab.ts:detectPage"
-    events:
-      - editor:pageLoaded
-      - plugs:loaded
-  joinCommand:
-    path: "./collab.ts:joinCommand"
-    command:
-      name: "Share: Join Collab"
-  shareCommand:
-    path: "./collab.ts:shareCommand"
-    command:
-      name: "Share: Collab"
-  shareNoop:
-    path: "./collab.ts:shareNoop"
-    events:
-    - share:collab
-  
-  # Space extension
-  readPageCollab:
-    path: ./collab.ts:readFileCollab
-    pageNamespace:
-      pattern: "collab:.+"
-      operation: readFile
-  writePageCollab:
-    path: ./collab.ts:writeFileCollab
-    pageNamespace:
-      pattern: "collab:.+"
-      operation: writeFile
-  getPageMetaCollab:
-    path: ./collab.ts:getFileMetaCollab
-    pageNamespace:
-      pattern: "collab:.+"
-      operation: getFileMeta

plugs/collab/collab.ts

@@ -1,158 +0,0 @@
-import {
-  findNodeOfType,
-  removeParentPointers,
-  renderToText,
-} from "$sb/lib/tree.ts";
-import { getText } from "$sb/silverbullet-syscall/editor.ts";
-import { parseMarkdown } from "$sb/silverbullet-syscall/markdown.ts";
-import {
-  extractFrontmatter,
-  prepareFrontmatterDispatch,
-} from "$sb/lib/frontmatter.ts";
-import { store, YAML } from "$sb/plugos-syscall/mod.ts";
-import { collab, editor, markdown } from "$sb/silverbullet-syscall/mod.ts";
-
-import { nanoid } from "https://esm.sh/nanoid@4.0.0";
-import { FileMeta } from "../../common/types.ts";
-
-const defaultServer = "wss://collab.silverbullet.md";
-
-async function ensureUsername(): Promise<string> {
-  let username = await store.get("collabUsername");
-  if (!username) {
-    username = await editor.prompt(
-      "Please enter a publicly visible user name (or cancel for 'anonymous'):",
-    );
-    if (!username) {
-      return "anonymous";
-    } else {
-      await store.set("collabUsername", username);
-    }
-  }
-  return username;
-}
-
-export async function joinCommand() {
-  let collabUri = await editor.prompt(
-    "Collab share URI:",
-  );
-  if (!collabUri) {
-    return;
-  }
-  if (!collabUri.startsWith("collab:")) {
-    collabUri = "collab:" + collabUri;
-  }
-  await editor.navigate(collabUri);
-}
-
-export async function shareCommand() {
-  const serverUrl = await editor.prompt(
-    "Please enter the URL of the collab server to use:",
-    defaultServer,
-  );
-  if (!serverUrl) {
-    return;
-  }
-  const roomId = nanoid().replaceAll("_", "-");
-  await editor.save();
-  const text = await editor.getText();
-  const tree = await markdown.parseMarkdown(text);
-  let { $share } = await extractFrontmatter(tree);
-  if (!$share) {
-    $share = [];
-  }
-  if (!Array.isArray($share)) {
-    $share = [$share];
-  }
-
-  removeParentPointers(tree);
-  const dispatchData = await prepareFrontmatterDispatch(tree, {
-    $share: [...$share, `collab:${serverUrl}/${roomId}`],
-  });
-
-  await editor.dispatch(dispatchData);
-
-  collab.start(
-    serverUrl,
-    roomId,
-    await ensureUsername(),
-  );
-}
-
-export async function detectPage() {
-  const tree = await parseMarkdown(await getText());
-  const frontMatter = findNodeOfType(tree, "FrontMatter");
-  if (frontMatter) {
-    const yamlText = renderToText(frontMatter.children![1].children![0]);
-    try {
-      let { $share } = await YAML.parse(yamlText) as any;
-      if (!$share) {
-        return;
-      }
-      if (!Array.isArray($share)) {
-        $share = [$share];
-      }
-      for (const uri of $share) {
-        if (uri.startsWith("collab:")) {
-          console.log("Going to enable collab");
-          const uriPieces = uri.substring("collab:".length).split("/");
-          await collab.start(
-            // All parts except the last one
-            uriPieces.slice(0, uriPieces.length - 1).join("/"),
-            // because the last one is the room ID
-            uriPieces[uriPieces.length - 1],
-            await ensureUsername(),
-          );
-        }
-      }
-    } catch (e) {
-      console.error("Error parsing YAML", e);
-    }
-  }
-}
-
-export function shareNoop() {
-  return true;
-}
-
-export function readFileCollab(
-  name: string,
-): { data: Uint8Array; meta: FileMeta } {
-  if (!name.endsWith(".md")) {
-    throw new Error("Not found");
-  }
-  const collabUri = name.substring(0, name.length - ".md".length);
-  const text = `---\n$share: ${collabUri}\n---\n`;
-
-  return {
-    // encoding === "arraybuffer" is not an option, so either it's "utf8" or "dataurl"
-    data: new TextEncoder().encode(text),
-    meta: {
-      name,
-      contentType: "text/markdown",
-      size: text.length,
-      lastModified: 0,
-      perm: "rw",
-    },
-  };
-}
-
-export function getFileMetaCollab(name: string): FileMeta {
-  return {
-    name,
-    contentType: "text/markdown",
-    size: -1,
-    lastModified: 0,
-    perm: "rw",
-  };
-}
-
-export function writeFileCollab(name: string): FileMeta {
-  return {
-    name,
-    contentType: "text/markdown",
-    size: -1,
-    lastModified: 0,
-    perm: "rw",
-  };
-}

plugs/collab/constants.ts

@@ -1 +0,0 @@
-export const collabPingInterval = 2500;

plugs/core/navigate.ts

@@ -77,7 +77,7 @@ async function actionClickOrActionEnter(
         return editor.flashNotification("Empty link, ignoring", "error");
       }
       if (url.indexOf("://") === -1 && !url.startsWith("mailto:")) {
-        return editor.openUrl(`/.fs/${decodeURI(url)}`);
+        return editor.openUrl(decodeURI(url));
       } else {
         await editor.openUrl(url);
       }

plugs/core/page.ts

@@ -23,6 +23,7 @@ import {
 import { applyQuery, removeQueries } from "$sb/lib/query.ts";
 import { extractFrontmatter } from "$sb/lib/frontmatter.ts";
 import { invokeFunction } from "$sb/silverbullet-syscall/system.ts";
+import { isValidPageName } from "$sb/lib/page.ts";
 
 // Key space:
 //   pl:toPage:pos => pageName
@@ -136,6 +137,13 @@ export async function renamePage(cmdDef: any) {
     return;
   }
 
+  if (!isValidPageName(newName)) {
+    return editor.flashNotification(
+      "Invalid page name: page names cannot end with a file extension",
+      "error",
+    );
+  }
+
   console.log("New name", newName);
 
   if (newName.trim() === oldName.trim()) {

plugs/directive/command.ts

@@ -20,27 +20,6 @@ export async function updateDirectivesOnPageCommand(arg: any) {
     return;
   }
 
-  // if (!(await sync.hasInitialSyncCompleted())) {
-  //   console.info("Initial sync hasn't completed yet, not updating directives.");
-  //   return;
-  // }
-
-  // If this page is shared ($share) via collab: disable directives as well
-  // due to security concerns
-  if (metaData.$share) {
-    for (const uri of metaData.$share) {
-      if (uri.startsWith("collab:")) {
-        if (explicitCall) {
-          await editor.flashNotification(
-            "Directives are disabled for 'collab' pages (safety reasons).",
-            "error",
-          );
-        }
-        return;
-      }
-    }
-  }
-
   // Collect all directives and their body replacements
   const replacements: { fullMatch: string; textPromise: Promise<string> }[] =
     [];

plugs/directive/template_directive.ts

@@ -1,6 +1,5 @@
 import { queryRegex } from "$sb/lib/query.ts";
 import { ParseTree, renderToText } from "$sb/lib/tree.ts";
-import { replaceAsync } from "$sb/lib/util.ts";
 import { markdown, space } from "$sb/silverbullet-syscall/mod.ts";
 import Handlebars from "handlebars";
 
@@ -68,32 +67,28 @@ export async function templateDirectiveRenderer(
   return newBody.trim();
 }
 
-export function cleanTemplateInstantiations(text: string): Promise<string> {
+export function cleanTemplateInstantiations(text: string) {
-  return replaceAsync(
+  return text.replaceAll(directiveRegex, (
-    text,
+    _fullMatch,
-    directiveRegex,
+    startInst,
-    (
+    type,
-      _fullMatch,
+    _args,
-      startInst,
+    body,
-      type,
+    endInst,
-      _args,
+  ): string => {
-      body,
+    if (type === "use") {
-      endInst,
+      body = body.replaceAll(
-    ): Promise<string> => {
+        queryRegex,
-      if (type === "use") {
+        (
-        body = body.replaceAll(
+          _fullMatch: string,
-          queryRegex,
+          _startQuery: string,
-          (
+          _query: string,
-            _fullMatch: string,
+          body: string,
-            _startQuery: string,
+        ) => {
-            _query: string,
+          return body.trim();
-            body: string,
+        },
-          ) => {
+      );
-            return body.trim();
+    }
-          },
+    return `${startInst}${body}${endInst}`;
-        );
+  });
-      }
-      return Promise.resolve(`${startInst}${body}${endInst}`);
-    },
-  );
 }

plugs/federation/federation.ts

@@ -5,9 +5,6 @@ import { readSetting } from "$sb/lib/settings_page.ts";
 function resolveFederated(pageName: string): string {
   // URL without the prefix "!""
   let url = pageName.substring(1);
-  const pieces = url.split("/");
-  pieces.splice(1, 0, ".fs");
-  url = pieces.join("/");
   if (!url.startsWith("127.0.0.1") && !url.startsWith("localhost")) {
     url = `https://${url}`;
   } else {
@@ -153,7 +150,7 @@ export async function deleteFile(
 export async function getFileMeta(name: string): Promise<FileMeta> {
   const url = resolveFederated(name);
   console.log("Fetching federation file meta", url);
-  const r = await nativeFetch(url, { method: "OPTIONS" });
+  const r = await nativeFetch(url, { method: "HEAD" });
   const fileMeta = await responseToFileMeta(r, name);
   if (!r.ok) {
     throw new Error("Not found");

plugs/markdown/preview.ts

@@ -7,7 +7,6 @@ export async function updateMarkdownPreview() {
   if (!(await store.get("enableMarkdownPreview"))) {
     return;
   }
-  const pageName = await editor.getCurrentPage();
   const text = await editor.getText();
   const mdTree = await parseMarkdown(text);
   // const cleanMd = await cleanMarkdown(text);
@@ -18,7 +17,7 @@ export async function updateMarkdownPreview() {
     annotationPositions: true,
     translateUrls: (url) => {
       if (!url.includes("://")) {
-        return `/.fs/${url}`;
+        return decodeURI(url);
       }
       return url;
     },

scripts/build_website.sh

@@ -10,33 +10,29 @@ if [ "$1" != "local" ]; then
 
 fi
 
+deno task clean
+mkdir -p website_build/_plug website_build/_client
+
+echo "Copying website content"
+cp -r website/* website_build/
+#rm website_build/{_redirects,_headers}
 
 echo "Building silver bullet"
-rm -rf website_build
-deno task clean
 deno task build
-echo "Cleaning website build dir"
+
-rm -rf website_build
-mkdir -p website_build/_fs/_plug website_build/_client
 echo "Copying silverbullet runtime files"
 cp dist_client_bundle/* website_build/
 cp -r dist_client_bundle/.client/* website_build/_client/
 
 echo "And all plugs"
-cp -r dist_plug_bundle/_plug/* website_build/_fs/_plug/
+cp -r dist_plug_bundle/_plug/* website_build/_plug/
 #echo "And additional ones"
-curl https://raw.githubusercontent.com/silverbulletmd/silverbullet-mermaid/main/mermaid.plug.js > website_build/_fs/_plug/mermaid.plug.js
+curl https://raw.githubusercontent.com/silverbulletmd/silverbullet-mermaid/main/mermaid.plug.js > website_build/_plug/mermaid.plug.js
 echo "But remove some plugs"
-rm -rf website_build/_fs/_plug/{plugmd}.plug.js
+rm -rf website_build/_plug/{plugmd}.plug.js
 
-echo "Copying website content into fs/"
-cp -r website/* website_build/_fs/
-rm website_build/_fs/{_redirects,_headers}
 
-echo "Copy website files another time into the root"
+# Generate random modified date, and replace in _headers too
-cp -r website/* website_build/
-
-# Genereate random modified date, and replace in _headers too
 export LAST_MODIFIED_TIMESTAMP=$RANDOM
 
 cat website/_headers | sed "s/12345/$LAST_MODIFIED_TIMESTAMP/g" > website_build/_headers

scripts/generate_fs_list.ts

@@ -4,7 +4,7 @@ import { walk } from "https://deno.land/std@0.165.0/fs/mod.ts";
 import { resolve } from "https://deno.land/std@0.165.0/path/mod.ts";
 import { mime } from "https://deno.land/x/mimetypes@v1.0.0/mod.ts";
 
-const rootDir = resolve("website_build/_fs");
+const rootDir = resolve("website_build");
 
 const lastModifiedTimestamp = +Deno.env.get("LAST_MODIFIED_TIMESTAMP")! ||
   Date.now();
@@ -14,7 +14,7 @@ for await (
   const file of walk(rootDir, {
     includeDirs: false,
     // Exclude hidden files
-    skip: [/^.*\/\..+$/],
+    skip: [/^.*\/(\..+|_redirects|_headers|_client\/.*)$/],
   })
 ) {
   const fullPath = file.path;

server/collab.test.ts

@@ -1,42 +0,0 @@
-import { assert, assertEquals } from "../test_deps.ts";
-import { CollabServer } from "./collab.ts";
-
-Deno.test("Collab server", async () => {
-  const collabServer = new CollabServer(null as any);
-  console.log("Client 1 joins page 1");
-  assertEquals(collabServer.updatePresence("client1", "page1"), {});
-  assertEquals(collabServer.pages.size, 1);
-  assertEquals(collabServer.updatePresence("client1", "page2"), {});
-  assertEquals(collabServer.pages.size, 1);
-  console.log("Client 2 joins to page 2, collab id created, but not exposed");
-  assertEquals(
-    collabServer.updatePresence("client2", "page2").collabId,
-    undefined,
-  );
-  assert(
-    collabServer.updatePresence("client1", "page2").collabId !== undefined,
-  );
-  console.log("Client 2 moves to page 1, collab id destroyed");
-  assertEquals(collabServer.updatePresence("client2", "page1"), {});
-  assertEquals(collabServer.updatePresence("client1", "page2"), {});
-  assertEquals(collabServer.pages.get("page2")!.collabId, undefined);
-  assertEquals(collabServer.pages.get("page1")!.collabId, undefined);
-  console.log("Going to cleanup, which should have no effect");
-  collabServer.cleanup(50);
-  assertEquals(collabServer.pages.size, 2);
-  collabServer.updatePresence("client2", "page2");
-  console.log("Going to sleep 20ms");
-  await sleep(20);
-  console.log("Then client 1 pings, but client 2 does not");
-  collabServer.updatePresence("client1", "page2");
-  await sleep(20);
-  console.log("Going to cleanup, which should clean client 2");
-  collabServer.cleanup(35);
-  assertEquals(collabServer.pages.get("page2")!.collabId, undefined);
-  assertEquals(collabServer.clients.size, 1);
-  console.log(collabServer);
-});
-
-function sleep(ms: number) {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}

server/collab.ts

@@ -1,241 +0,0 @@
-import { getAvailablePortSync } from "https://deno.land/x/port@1.0.0/mod.ts";
-import { nanoid } from "https://esm.sh/nanoid@4.0.0";
-import { race, timeout } from "../common/async_util.ts";
-import { Application } from "./deps.ts";
-import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
-import { collabPingInterval } from "../plugs/collab/constants.ts";
-import { Hocuspocus } from "./deps.ts";
-
-type CollabPage = {
-  clients: Map<string, number>; // clientId -> lastPing
-  collabId?: string;
-  // The currently elected provider of the initial document
-  masterClientId: string;
-};
-
-export class CollabServer {
-  clients: Map<string, { openPage: string; lastUpdate: number }> = new Map(); // clientId -> openPage
-  pages: Map<string, CollabPage> = new Map();
-  yCollabServer?: Hocuspocus;
-
-  constructor(private spacePrimitives: SpacePrimitives) {
-  }
-
-  start() {
-    setInterval(() => {
-      this.cleanup(3 * collabPingInterval);
-    }, collabPingInterval);
-  }
-
-  updatePresence(
-    clientId: string,
-    currentPage: string,
-  ): { collabId?: string } {
-    let client = this.clients.get(clientId);
-    if (!client) {
-      client = { openPage: "", lastUpdate: 0 };
-      this.clients.set(clientId, client);
-    }
-    client.lastUpdate = Date.now();
-    if (currentPage !== client.openPage) {
-      // Client switched pages
-      // Update last page record
-      const lastCollabPage = this.pages.get(client.openPage);
-      if (lastCollabPage) {
-        lastCollabPage.clients.delete(clientId);
-        if (lastCollabPage.clients.size === 1) {
-          delete lastCollabPage.collabId;
-        }
-
-        if (lastCollabPage.clients.size === 0) {
-          this.pages.delete(client.openPage);
-        } else {
-          // Elect a new master client
-          if (lastCollabPage.masterClientId === clientId) {
-            // Any is fine, really
-            lastCollabPage.masterClientId =
-              [...lastCollabPage.clients.keys()][0];
-          }
-        }
-      }
-      // Ok, let's update our records now
-      client.openPage = currentPage;
-    }
-
-    // Update new page
-    let nextCollabPage = this.pages.get(currentPage);
-    if (!nextCollabPage) {
-      // Newly opened page (no other clients on this page right now)
-      nextCollabPage = {
-        clients: new Map(),
-        masterClientId: clientId,
-      };
-      this.pages.set(currentPage, nextCollabPage);
-    }
-    // Register last ping from us
-    nextCollabPage.clients.set(clientId, Date.now());
-
-    if (nextCollabPage.clients.size > 1 && !nextCollabPage.collabId) {
-      // Create a new collabId
-      nextCollabPage.collabId = nanoid();
-    }
-    // console.log("State", this.pages);
-    if (nextCollabPage.collabId) {
-      // We will now expose this collabId, except when we're just starting this session
-      // in which case we'll wait for the original client to publish the document
-      const existingyCollabSession = this.yCollabServer?.documents.get(
-        buildCollabId(nextCollabPage.collabId, `${currentPage}.md`),
-      );
-      if (existingyCollabSession) {
-        // console.log("Found an existing collab session already, let's join!");
-        return { collabId: nextCollabPage.collabId };
-      } else if (clientId === nextCollabPage.masterClientId) {
-        // console.log("We're the master, so we should connect");
-        return { collabId: nextCollabPage.collabId };
-      } else {
-        // We're not the first client, so we need to wait for the first client to connect
-        // console.log("We're not the master, so we should wait");
-        return {};
-      }
-    } else {
-      return {};
-    }
-  }
-
-  cleanup(timeout: number) {
-    // Clean up pages and their clients that haven't pinged for some time
-    for (const [pageName, page] of this.pages) {
-      for (const [clientId, lastPing] of page.clients) {
-        if (Date.now() - lastPing > timeout) {
-          // Eject client
-          page.clients.delete(clientId);
-          // Elect a new master client
-          if (page.masterClientId === clientId && page.clients.size > 0) {
-            page.masterClientId = [...page.clients.keys()][0];
-          }
-        }
-      }
-      if (page.clients.size === 1) {
-        // If there's only one client left, we don't need to keep the collabId around anymore
-        delete page.collabId;
-      }
-      if (page.clients.size === 0) {
-        // And if we have no clients left, well...
-        this.pages.delete(pageName);
-      }
-    }
-
-    for (const [clientId, { lastUpdate }] of this.clients) {
-      if (Date.now() - lastUpdate > timeout) {
-        // Eject client
-        this.clients.delete(clientId);
-      }
-    }
-  }
-
-  route(app: Application) {
-    // The way this works is that we spin up a separate WS server locally and then proxy requests to it
-    // This is the only way I could get Hocuspocus to work with Deno
-    const internalPort = getAvailablePortSync();
-    this.yCollabServer = new Hocuspocus({
-      port: internalPort,
-      address: "127.0.0.1",
-      quiet: true,
-      onStoreDocument: async (data) => {
-        const [_, path] = splitCollabId(data.documentName);
-        const text = data.document.getText("codemirror").toString();
-        console.log(
-          "[Hocuspocus]",
-          "Persisting",
-          path,
-          "to space on server",
-        );
-        const meta = await this.spacePrimitives.writeFile(
-          path,
-          new TextEncoder().encode(text),
-        );
-        // Broadcast new persisted lastModified date
-        data.document.broadcastStateless(
-          JSON.stringify({
-            type: "persisted",
-            path,
-            lastModified: meta.lastModified,
-          }),
-        );
-        return;
-      },
-      onDisconnect: (client) => {
-        console.log("[Hocuspocus]", "Client disconnected", client.clientsCount);
-        if (client.clientsCount === 0) {
-          console.log(
-            "[Hocuspocus]",
-            "Last client disconnected from",
-            client.documentName,
-            "purging from memory",
-          );
-          this.yCollabServer!.documents.delete(client.documentName);
-        }
-        return Promise.resolve();
-      },
-    });
-
-    this.yCollabServer.listen();
-
-    app.use((ctx) => {
-      // if (ctx.request.url.pathname === "/.ws") {
-      //   const sock = ctx.upgrade();
-      //   sock.onmessage = (e) => {
-      //     console.log("WS: Got message", e.data);
-      //   };
-      // }
-      // Websocket proxy to hocuspocus
-      if (ctx.request.url.pathname === "/.ws-collab") {
-        const sock = ctx.upgrade();
-
-        const ws = new WebSocket(`ws://localhost:${internalPort}`);
-        const wsReady = race([
-          new Promise<void>((resolve) => {
-            ws.onopen = () => {
-              resolve();
-            };
-          }),
-          timeout(1000),
-        ]).catch(() => {
-          console.error("Timeout waiting for collab to open websocket");
-          sock.close();
-        });
-        sock.onmessage = (e) => {
-          // console.log("Got message", e);
-          wsReady.then(() => ws.send(e.data)).catch(console.error);
-        };
-        sock.onclose = () => {
-          if (ws.OPEN) {
-            ws.close();
-          }
-        };
-        ws.onmessage = (e) => {
-          if (sock.OPEN) {
-            sock.send(e.data);
-          } else {
-            console.error("Got message from websocket but socket is not open");
-          }
-        };
-        ws.onclose = () => {
-          if (sock.OPEN) {
-            sock.close();
-          }
-        };
-      }
-    });
-  }
-}
-
-function splitCollabId(documentName: string): [string, string] {
-  const [collabId, ...pathPieces] = documentName.split("/");
-  const path = pathPieces.join("/");
-  return [collabId, path];
-}
-
-function buildCollabId(collabId: string, path: string): string {
-  return `${collabId}/${path}`;
-}

server/deps.ts

@@ -1,5 +1,9 @@
 export * from "../common/deps.ts";
-export { Application, Router } from "https://deno.land/x/oak@v12.4.0/mod.ts";
+export type { Next } from "https://deno.land/x/oak@v12.4.0/mod.ts";
+export {
+  Application,
+  Context,
+  Router,
+} from "https://deno.land/x/oak@v12.4.0/mod.ts";
 export * as etag from "https://deno.land/x/oak@v12.4.0/etag.ts";
-
+export { oakCors } from "https://deno.land/x/cors@v1.2.2/mod.ts";
-export { Hocuspocus } from "npm:@hocuspocus/server@2.1.0";

server/http_server.ts

@@ -1,15 +1,13 @@
-import { Application, Router } from "./deps.ts";
+import { Application, Context, Next, oakCors, Router } from "./deps.ts";
 import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
 import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
-import { base64Decode } from "../plugos/asset_bundle/base64.ts";
 import { ensureSettingsAndIndex } from "../common/util.ts";
 import { performLocalFetch } from "../common/proxy_fetch.ts";
 import { BuiltinSettings } from "../web/types.ts";
 import { gitIgnoreCompiler } from "./deps.ts";
 import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts";
-import { CollabServer } from "./collab.ts";
 import { Authenticator } from "./auth.ts";
-import { oakCors } from "https://deno.land/x/cors@v1.2.2/mod.ts";
+import { FileMeta } from "../common/types.ts";
 
 export type ServerOptions = {
   hostname: string;
@@ -31,7 +29,6 @@ export class HttpServer {
   clientAssetBundle: AssetBundle;
   settings?: BuiltinSettings;
   spacePrimitives: SpacePrimitives;
-  collab: CollabServer;
   authenticator: Authenticator;
 
   constructor(
@@ -66,8 +63,6 @@ export class HttpServer {
         }
       },
     );
-    this.collab = new CollabServer(this.spacePrimitives);
-    this.collab.start();
   }
 
   // Replaces some template variables in index.html in a rather ad-hoc manner, but YOLO
@@ -76,76 +71,25 @@ export class HttpServer {
       .replaceAll(
         "{{SPACE_PATH}}",
         this.options.pagesPath.replaceAll("\\", "\\\\"),
-      ).replaceAll(
-        "{{SYNC_ENDPOINT}}",
-        "/.fs",
       );
   }
 
   async start() {
     await this.reloadSettings();
+
     // Serve static files (javascript, css, html)
-    this.app.use(async ({ request, response }, next) => {
+    this.app.use(this.serveStatic.bind(this));
-      if (request.url.pathname === "/") {
-        // Note: we're explicitly not setting Last-Modified and If-Modified-Since header here because this page is dynamic
-        response.headers.set("Content-type", "text/html");
-        response.body = this.renderIndexHtml();
-        return;
-      }
-      try {
-        const assetName = request.url.pathname.slice(1);
-        if (
-          this.clientAssetBundle.has(assetName) &&
-          request.headers.get("If-Modified-Since") ===
-            utcDateString(this.clientAssetBundle.getMtime(assetName))
-        ) {
-          response.status = 304;
-          return;
-        }
-        response.status = 200;
-        response.headers.set(
-          "Content-type",
-          this.clientAssetBundle.getMimeType(assetName),
-        );
-        const data = this.clientAssetBundle.readFileSync(
-          assetName,
-        );
-        response.headers.set("Cache-Control", "no-cache");
-        response.headers.set("Content-length", "" + data.length);
-        response.headers.set(
-          "Last-Modified",
-          utcDateString(this.clientAssetBundle.getMtime(assetName)),
-        );
 
-        if (request.method === "GET") {
-          response.body = data;
-        }
-      } catch {
-        await next();
-      }
-    });
-
-    // Fallback, serve index.html
-    this.app.use(({ request, response }, next) => {
-      if (
-        !request.url.pathname.startsWith("/.fs") &&
-        request.url.pathname !== "/.auth" &&
-        !request.url.pathname.startsWith("/.ws")
-      ) {
-        response.headers.set("Content-type", "text/html");
-        response.body = this.renderIndexHtml();
-      } else {
-        return next();
-      }
-    });
-
-    // Pages API
-    const fsRouter = this.buildFsRouter(this.spacePrimitives);
     await this.addPasswordAuth(this.app);
+    const fsRouter = this.addFsRoutes(this.spacePrimitives);
     this.app.use(fsRouter.routes());
     this.app.use(fsRouter.allowedMethods());
 
-    this.collab.route(this.app);
+    // Fallback, serve the UI index.html
+    this.app.use(({ response }) => {
+      response.headers.set("Content-type", "text/html");
+      response.body = this.renderIndexHtml();
+    });
 
     this.abortController = new AbortController();
     const listenOptions: any = {
@@ -172,6 +116,52 @@ export class HttpServer {
     );
   }
 
+  serveStatic(
+    { request, response }: Context<Record<string, any>, Record<string, any>>,
+    next: Next,
+  ) {
+    if (
+      request.url.pathname === "/"
+    ) {
+      // Serve the UI (index.html)
+      // Note: we're explicitly not setting Last-Modified and If-Modified-Since header here because this page is dynamic
+      response.headers.set("Content-type", "text/html");
+      response.body = this.renderIndexHtml();
+      return;
+    }
+    try {
+      const assetName = request.url.pathname.slice(1);
+      if (
+        this.clientAssetBundle.has(assetName) &&
+        request.headers.get("If-Modified-Since") ===
+          utcDateString(this.clientAssetBundle.getMtime(assetName))
+      ) {
+        response.status = 304;
+        return;
+      }
+      response.status = 200;
+      response.headers.set(
+        "Content-type",
+        this.clientAssetBundle.getMimeType(assetName),
+      );
+      const data = this.clientAssetBundle.readFileSync(
+        assetName,
+      );
+      response.headers.set("Cache-Control", "no-cache");
+      response.headers.set("Content-length", "" + data.length);
+      response.headers.set(
+        "Last-Modified",
+        utcDateString(this.clientAssetBundle.getMtime(assetName)),
+      );
+
+      if (request.method === "GET") {
+        response.body = data;
+      }
+    } catch {
+      return next();
+    }
+  }
+
   async reloadSettings() {
     // TODO: Throttle this?
     this.settings = await ensureSettingsAndIndex(this.spacePrimitives);
@@ -185,6 +175,7 @@ export class HttpServer {
       "/.auth",
     ];
 
+    // Middleware handling the /.auth page and flow
     app.use(async ({ request, response, cookies }, next) => {
       if (request.url.pathname === "/.auth") {
         if (request.url.search === "?logout") {
@@ -227,6 +218,7 @@ export class HttpServer {
     });
 
     if ((await this.authenticator.getAllUsers()).length > 0) {
+      // Users defined, so enabling auth
       app.use(async ({ request, response, cookies }, next) => {
         if (!excludedPaths.includes(request.url.pathname)) {
           const authCookie = await cookies.get("auth");
@@ -250,23 +242,37 @@ export class HttpServer {
     }
   }
 
-  private buildFsRouter(spacePrimitives: SpacePrimitives): Router {
+  private addFsRoutes(spacePrimitives: SpacePrimitives): Router {
     const fsRouter = new Router();
     const corsMiddleware = oakCors({
       allowedHeaders: "*",
       exposedHeaders: "*",
-      methods: ["GET", "POST", "PUT", "DELETE", "OPTIONS"],
+      methods: ["GET", "POST", "PUT", "DELETE", "HEAD", "OPTIONS"],
-    });
-    // File list
-    fsRouter.get("/", corsMiddleware, async ({ response }) => {
-      response.headers.set("Content-type", "application/json");
-      response.headers.set("X-Space-Path", this.options.pagesPath);
-      const files = await spacePrimitives.fetchFileList();
-      response.body = JSON.stringify(files);
     });
 
+    fsRouter.use(corsMiddleware);
+
+    // File list
+    fsRouter.get(
+      "/index.json",
+      // corsMiddleware,
+      async ({ request, response }) => {
+        if (request.headers.get("Accept") === "application/json") {
+          // Only handle direct requests for a JSON representation of the file list
+          response.headers.set("Content-type", "application/json");
+          response.headers.set("X-Space-Path", this.options.pagesPath);
+          const files = await spacePrimitives.fetchFileList();
+          response.body = JSON.stringify(files);
+        } else {
+          // Otherwise, redirect to the UI
+          // The reason to do this is to handle authentication systems like Authelia nicely
+          response.redirect("/");
+        }
+      },
+    );
+
     // RPC
-    fsRouter.post("/", corsMiddleware, async ({ request, response }) => {
+    fsRouter.post("/.rpc", async ({ request, response }) => {
       const body = await request.body({ type: "json" }).value;
       try {
         switch (body.operation) {
@@ -288,6 +294,7 @@ export class HttpServer {
               });
               return;
             }
+            console.log("Running shell command:", body.cmd, body.args);
             const p = new Deno.Command(body.cmd, {
               args: body.args,
               cwd: this.options.pagesPath,
@@ -304,6 +311,9 @@ export class HttpServer {
               stderr,
               code: output.code,
             });
+            if (output.code !== 0) {
+              console.error("Error running shell command", stdout, stderr);
+            }
             return;
           }
           default:
@@ -319,129 +329,104 @@ export class HttpServer {
       }
     });
 
+    const filePathRegex = "\/(.+\\.[a-zA-Z]+)";
+
     fsRouter
-      .get("\/(.+)", corsMiddleware, async ({ params, response, request }) => {
+      .get(
-        const name = params[0];
+        filePathRegex,
-        console.log("Loading file", name);
+        // corsMiddleware,
-        if (name.startsWith(".")) {
+        async ({ params, response, request }) => {
-          // Don't expose hidden files
+          const name = params[0];
-          response.status = 404;
+          console.log("Requested file", name);
-          return;
+          if (name.startsWith(".")) {
-        }
+            // Don't expose hidden files
-        try {
+            response.status = 404;
-          const attachmentData = await spacePrimitives.readFile(
+            response.body = "Not exposed";
-            name,
-          );
-          const lastModifiedHeader = new Date(attachmentData.meta.lastModified)
-            .toUTCString();
-          if (request.headers.get("If-Modified-Since") === lastModifiedHeader) {
-            response.status = 304;
             return;
           }
-          response.status = 200;
+          try {
-          response.headers.set(
+            const fileData = await spacePrimitives.readFile(
-            "X-Last-Modified",
+              name,
-            "" + attachmentData.meta.lastModified,
+            );
-          );
+            const lastModifiedHeader = new Date(fileData.meta.lastModified)
-          response.headers.set("Cache-Control", "no-cache");
+              .toUTCString();
-          response.headers.set("X-Permission", attachmentData.meta.perm);
+            if (
-          response.headers.set(
+              request.headers.get("If-Modified-Since") === lastModifiedHeader
-            "Last-Modified",
+            ) {
-            lastModifiedHeader,
+              response.status = 304;
-          );
+              return;
-          response.headers.set("Content-Type", attachmentData.meta.contentType);
+            }
-          response.body = attachmentData.data;
+            response.status = 200;
-        } catch {
+            this.fileMetaToHeaders(response.headers, fileData.meta);
-          // console.error("Error in main router", e);
+            response.headers.set("Last-Modified", lastModifiedHeader);
-          response.status = 404;
+
-          response.body = "";
+            response.body = fileData.data;
-        }
+          } catch {
-      })
+            // console.error("Error GETting of file", name, e);
-      .put("\/(.+)", corsMiddleware, async ({ request, response, params }) => {
+            response.status = 404;
+            response.body = "Not found";
+          }
+        },
+      )
+      .put(
+        filePathRegex,
+        // corsMiddleware,
+        async ({ request, response, params }) => {
+          const name = params[0];
+          console.log("Saving file", name);
+          if (name.startsWith(".")) {
+            // Don't expose hidden files
+            response.status = 403;
+            return;
+          }
+
+          const body = await request.body({ type: "bytes" }).value;
+
+          try {
+            const meta = await spacePrimitives.writeFile(
+              name,
+              body,
+            );
+            response.status = 200;
+            this.fileMetaToHeaders(response.headers, meta);
+            response.body = "OK";
+          } catch (err) {
+            console.error("Write failed", err);
+            response.status = 500;
+            response.body = "Write failed";
+          }
+        },
+      )
+      .delete(filePathRegex, async ({ response, params }) => {
         const name = params[0];
-        console.log("Saving file", name);
+        console.log("Deleting file", name);
         if (name.startsWith(".")) {
           // Don't expose hidden files
           response.status = 403;
           return;
         }
-
-        let body: Uint8Array;
-        if (
-          request.headers.get("X-Content-Base64")
-        ) {
-          const content = await request.body({ type: "text" }).value;
-          body = base64Decode(content);
-        } else {
-          body = await request.body({ type: "bytes" }).value;
-        }
-
-        try {
-          const meta = await spacePrimitives.writeFile(
-            name,
-            body,
-          );
-          response.status = 200;
-          response.headers.set("Content-Type", meta.contentType);
-          response.headers.set("X-Last-Modified", "" + meta.lastModified);
-          response.headers.set("X-Content-Length", "" + meta.size);
-          response.headers.set("X-Permission", meta.perm);
-          response.body = "OK";
-        } catch (err) {
-          response.status = 500;
-          response.body = "Write failed";
-          console.error("Pipeline failed", err);
-        }
-      })
-      .options("\/(.+)", async ({ request, response, params }) => {
-        const name = params[0];
-        // Manually set CORS headers
-        response.headers.set("access-control-allow-headers", "*");
-        response.headers.set(
-          "access-control-allow-methods",
-          "GET,POST,PUT,DELETE,OPTIONS",
-        );
-        response.headers.set("access-control-allow-origin", "*");
-        response.headers.set("access-control-expose-headers", "*");
-        try {
-          const meta = await spacePrimitives.getFileMeta(name);
-          response.status = 200;
-          response.headers.set("Content-Type", meta.contentType);
-          response.headers.set("X-Last-Modified", "" + meta.lastModified);
-          response.headers.set("X-Content-Length", "" + meta.size);
-          response.headers.set("X-Permission", meta.perm);
-
-          const clientId = request.headers.get("X-Client-Id");
-          if (name.endsWith(".md") && clientId) {
-            const pageName = name.substring(0, name.length - ".md".length);
-            console.log(`Got presence update from ${clientId}: ${pageName}`);
-            const { collabId } = this.collab.updatePresence(clientId, pageName);
-            if (collabId) {
-              response.headers.set("X-Collab-Id", collabId);
-            }
-          }
-        } catch {
-          // Have to do this because of CORS
-          response.status = 200;
-          response.headers.set("X-Status", "404");
-          response.body = "Not found";
-          // console.error("Options failed", err);
-        }
-      })
-      .delete("\/(.+)", corsMiddleware, async ({ response, params }) => {
-        const name = params[0];
-        console.log("Deleting file", name);
         try {
           await spacePrimitives.deleteFile(name);
           response.status = 200;
           response.body = "OK";
         } catch (e: any) {
           console.error("Error deleting attachment", e);
-          response.status = 200;
+          response.status = 500;
           response.body = e.message;
         }
-      });
+      })
-    return new Router().use("/.fs", fsRouter.routes());
+      .options(filePathRegex, corsMiddleware);
+    return fsRouter;
+  }
+
+  private fileMetaToHeaders(headers: Headers, fileMeta: FileMeta) {
+    headers.set("Content-Type", fileMeta.contentType);
+    headers.set(
+      "X-Last-Modified",
+      "" + fileMeta.lastModified,
+    );
+    headers.set("Cache-Control", "no-cache");
+    headers.set("X-Permission", fileMeta.perm);
   }
 
   stop() {

web/cm_plugins/collab.ts

@@ -1,99 +0,0 @@
-import { safeRun } from "../../common/util.ts";
-import { Extension, HocuspocusProvider, Y, yCollab } from "../deps.ts";
-import { SyncService } from "../sync_service.ts";
-
-const userColors = [
-  { color: "#30bced", light: "#30bced33" },
-  { color: "#6eeb83", light: "#6eeb8333" },
-  { color: "#ffbc42", light: "#ffbc4233" },
-  { color: "#ecd444", light: "#ecd44433" },
-  { color: "#ee6352", light: "#ee635233" },
-  { color: "#9ac2c9", light: "#9ac2c933" },
-  { color: "#8acb88", light: "#8acb8833" },
-  { color: "#1be7ff", light: "#1be7ff33" },
-];
-
-export class CollabState {
-  public ytext: Y.Text;
-  collabProvider: HocuspocusProvider;
-  private yundoManager: Y.UndoManager;
-  interval?: number;
-
-  constructor(
-    serverUrl: string,
-    readonly path: string,
-    readonly token: string,
-    username: string,
-    private syncService: SyncService,
-    public isLocalCollab: boolean,
-  ) {
-    this.collabProvider = new HocuspocusProvider({
-      url: serverUrl,
-      name: token,
-
-      // Receive broadcasted messages from the server (right now only "page has been persisted" notifications)
-      onStateless: (
-        { payload },
-      ) => {
-        const message = JSON.parse(payload);
-        switch (message.type) {
-          case "persisted": {
-            // Received remote persist notification, updating snapshot
-            syncService.updateRemoteLastModified(
-              message.path,
-              message.lastModified,
-            ).catch(console.error);
-          }
-        }
-      },
-    });
-
-    this.collabProvider.on("status", (e: any) => {
-      console.log("Collab status change", e);
-    });
-
-    this.ytext = this.collabProvider.document.getText("codemirror");
-    this.yundoManager = new Y.UndoManager(this.ytext);
-
-    const randomColor =
-      userColors[Math.floor(Math.random() * userColors.length)];
-
-    this.collabProvider.awareness.setLocalStateField("user", {
-      name: username,
-      color: randomColor.color,
-      colorLight: randomColor.light,
-    });
-    if (isLocalCollab) {
-      syncService.excludeFromSync(path).catch(console.error);
-
-      this.interval = setInterval(() => {
-        // Ping the store to make sure the file remains in exclusion
-        syncService.excludeFromSync(path).catch(console.error);
-      }, 1000);
-    }
-  }
-
-  stop() {
-    console.log("[COLLAB] Destroying collab provider");
-    if (this.interval) {
-      clearInterval(this.interval);
-    }
-    this.collabProvider.destroy();
-    // For whatever reason, destroy() doesn't properly clean up everything so we need to help a bit
-    this.collabProvider.configuration.websocketProvider.webSocket = null;
-    this.collabProvider.configuration.websocketProvider.destroy();
-
-    // When stopping collaboration, we're going back to sync mode. Make sure we got the latest and greatest remote timestamp to avoid
-    // conflicts
-    safeRun(async () => {
-      await this.syncService.unExcludeFromSync(this.path);
-      await this.syncService.fetchAndPersistRemoteLastModified(this.path);
-    });
-  }
-
-  collabExtension(): Extension {
-    return yCollab(this.ytext, this.collabProvider.awareness, {
-      undoManager: this.yundoManager,
-    });
-  }
-}

web/cm_plugins/fenced_code.ts

@@ -21,7 +21,6 @@ class IFrameWidget extends WidgetType {
   }
 
   toDOM(): HTMLElement {
-    console.log("toDOM");
     const iframe = document.createElement("iframe");
     iframe.srcdoc = panelHtml;
     // iframe.style.height = "0";

web/cm_plugins/inline_image.ts

@@ -74,7 +74,7 @@ export function inlineImagesPlugin(editor: Editor) {
         let url = imageRexexResult.groups.url;
         const title = imageRexexResult.groups.title;
         if (url.indexOf("://") === -1) {
-          url = `/.fs/${url}`;
+          url = decodeURI(url);
         }
         widgets.push(
           Decoration.widget({

web/cm_plugins/table.ts

@@ -39,7 +39,7 @@ class TableViewWidget extends WidgetType {
       annotationPositions: true,
       translateUrls: (url) => {
         if (!url.includes("://")) {
-          return `/.fs/${url}`;
+          return `/${url}`;
         }
         return url;
       },

web/collab_manager.ts

@@ -1,83 +0,0 @@
-import { nanoid } from "https://esm.sh/nanoid@4.0.0";
-import type { Editor } from "./editor.tsx";
-
-const collabPingInterval = 2500;
-
-export class CollabManager {
-  clientId = nanoid();
-  localCollabServer: string;
-
-  constructor(private editor: Editor) {
-    this.localCollabServer = location.protocol === "http:"
-      ? `ws://${location.host}/.ws-collab`
-      : `wss://${location.host}/.ws-collab`;
-    editor.eventHook.addLocalListener(
-      "editor:pageLoaded",
-      (pageName, previousPage) => {
-        console.log("Page loaded", pageName, previousPage);
-        this.updatePresence(pageName).catch(console.error);
-      },
-    );
-  }
-
-  start() {
-    setInterval(() => {
-      this.updatePresence(this.editor.currentPage!).catch(console.error);
-    }, collabPingInterval);
-  }
-
-  async updatePresence(currentPage: string) {
-    try {
-      // This is signaled through an OPTIONS call on the file we have open
-      const resp = await this.editor.remoteSpacePrimitives.authenticatedFetch(
-        `${this.editor.remoteSpacePrimitives.url}/${currentPage}.md`,
-        {
-          method: "OPTIONS",
-          headers: {
-            "X-Client-Id": this.clientId,
-          },
-        },
-      );
-      const collabId = resp.headers.get("X-Collab-Id");
-      // Not reading body at all, is that a problem?
-
-      if (this.editor.collabState && !this.editor.collabState.isLocalCollab) {
-        // We're in a remote collab mode, don't do anything
-        return;
-      }
-
-      // console.log("Collab ID", collabId);
-      const previousCollabId = this.editor.collabState?.token.split("/")[0];
-      if (!collabId && this.editor.collabState) {
-        // Stop collab
-        console.log("Stopping collab");
-        if (this.editor.collabState.path === `${currentPage}.md`) {
-          this.editor.flashNotification(
-            "Other users have left this page, switched back to single-user mode.",
-          );
-        }
-        this.editor.stopCollab();
-      } else if (collabId && collabId !== previousCollabId) {
-        // Start collab
-        console.log("Starting collab");
-        this.editor.flashNotification(
-          "Opening page in multi-user mode.",
-        );
-        this.editor.startCollab(
-          this.localCollabServer,
-          `${collabId}/${currentPage}.md`,
-          this.editor.getUsername(),
-          true,
-        );
-      }
-    } catch (e: any) {
-      // console.error("Ping error", e);
-      if (
-        e.message.toLowerCase().includes("failed") && this.editor.collabState
-      ) {
-        console.log("Offline, stopping collab");
-        this.editor.stopCollab();
-      }
-    }
-  }
-}

web/deps.ts

@@ -15,14 +15,6 @@ export {
   Terminal as TerminalIcon,
 } from "https://esm.sh/preact-feather@4.2.1?external=preact";
 
-// Y collab
-export * as Y from "yjs";
-export {
-  yCollab,
-  yUndoManagerKeymap,
-} from "https://esm.sh/y-codemirror.next@0.3.2?external=yjs,@codemirror/state,@codemirror/commands,@codemirror/history,@codemirror/view";
-export { HocuspocusProvider } from "https://esm.sh/@hocuspocus/provider@2.2.0?deps=lib0@0.2.70&external=yjs,ws&target=es2022";
-
 // Vim mode
 export {
   getCM as vimGetCm,

web/editor.tsx

@@ -68,7 +68,6 @@ import assetSyscalls from "../plugos/syscalls/asset.ts";
 import { eventSyscalls } from "../plugos/syscalls/event.ts";
 import { System } from "../plugos/system.ts";
 import { cleanModePlugins } from "./cm_plugins/clean.ts";
-import { CollabState } from "./cm_plugins/collab.ts";
 import {
   attachmentExtension,
   pasteLinkExtension,
@@ -91,14 +90,12 @@ import {
   useEffect,
   useReducer,
   vim,
-  yUndoManagerKeymap,
 } from "./deps.ts";
 import { AppCommand, CommandHook } from "./hooks/command.ts";
 import { SlashCommandHook } from "./hooks/slash_command.ts";
 import { PathPageNavigator } from "./navigator.ts";
 import reducer from "./reducer.ts";
 import customMarkdownStyle from "./style.ts";
-import { collabSyscalls } from "./syscalls/collab.ts";
 import { editorSyscalls } from "./syscalls/editor.ts";
 import { spaceSyscalls } from "./syscalls/space.ts";
 import { systemSyscalls } from "./syscalls/system.ts";
@@ -137,7 +134,8 @@ import { HttpSpacePrimitives } from "../common/spaces/http_space_primitives.ts";
 import { FallbackSpacePrimitives } from "../common/spaces/fallback_space_primitives.ts";
 import { syncSyscalls } from "./syscalls/sync.ts";
 import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts";
-import { CollabManager } from "./collab_manager.ts";
+import { run } from "../plug-api/plugos-syscall/shell.ts";
+import { isValidPageName } from "$sb/lib/page.ts";
 
 const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/;
 
@@ -155,7 +153,6 @@ declare global {
     // Injected via index.html
     silverBulletConfig: {
       spaceFolderPath: string;
-      syncEndpoint: string;
     };
     editor: Editor;
   }
@@ -190,11 +187,9 @@ export class Editor {
   fullSyncCompleted = false;
 
   // Runtime state (that doesn't make sense in viewState)
-  collabState?: CollabState;
   syncService: SyncService;
   settings?: BuiltinSettings;
   kvStore: DexieKVStore;
-  collabManager: CollabManager;
 
   constructor(
     parent: Element,
@@ -216,8 +211,6 @@ export class Editor {
     this.eventHook = new EventHook();
     system.addHook(this.eventHook);
 
-    this.collabManager = new CollabManager(this);
-
     // Cron hook
     const cronHook = new CronHook(system);
     system.addHook(cronHook);
@@ -237,7 +230,7 @@ export class Editor {
 
     // Setup space
     this.remoteSpacePrimitives = new HttpSpacePrimitives(
-      runtimeConfig.syncEndpoint,
+      location.origin,
       runtimeConfig.spaceFolderPath,
       true,
     );
@@ -328,7 +321,6 @@ export class Editor {
       systemSyscalls(this, this.system),
       markdownSyscalls(buildMarkdown(this.mdExtensions)),
       assetSyscalls(this.system),
-      collabSyscalls(this),
       yamlSyscalls(),
       storeCalls,
       indexSyscalls,
@@ -376,17 +368,12 @@ export class Editor {
       }
     });
 
-    // globalThis.addEventListener("beforeunload", (e) => {
-    //   console.log("Pinging with with undefined page name");
-    //   this.collabManager.updatePresence(undefined, this.currentPage);
-    // });
-
     this.eventHook.addLocalListener("plug:changed", async (fileName) => {
       console.log("Plug updated, reloading:", fileName);
       system.unload(fileName);
       await system.load(
         // await this.space.readFile(fileName, "utf8"),
-        new URL(`/.fs/${fileName}`, location.href),
+        new URL(`/${fileName}`, location.href),
         createSandbox,
       );
       this.plugsUpdated = true;
@@ -402,7 +389,7 @@ export class Editor {
 
     this.space.on({
       pageChanged: (meta) => {
-        // Only reload when watching the current page (to avoid reloading when switching pages and in collab mode)
+        // Only reload when watching the current page (to avoid reloading when switching pages)
         if (this.space.watchInterval && this.currentPage === meta.name) {
           console.log("Page changed elsewhere, reloading");
           this.flashNotification("Page changed elsewhere, reloading");
@@ -485,7 +472,6 @@ export class Editor {
 
     // Kick off background sync
     this.syncService.start();
-    this.collabManager.start();
 
     this.eventHook.addLocalListener("sync:success", async (operations) => {
       // console.log("Operations", operations);
@@ -738,7 +724,7 @@ export class Editor {
     let touchCount = 0;
 
     return EditorState.create({
-      doc: this.collabState ? this.collabState.ytext.toString() : text,
+      doc: text,
       extensions: [
         // Not using CM theming right now, but some extensions depend on the "dark" thing
         EditorView.theme({}, { dark: this.viewState.uiOptions.darkMode }),
@@ -934,7 +920,6 @@ export class Editor {
           ...searchKeymap,
           ...historyKeymap,
           ...completionKeymap,
-          ...(this.collabState ? yUndoManagerKeymap : []),
           indentWithTab,
           ...commandKeyBindings,
           {
@@ -1057,7 +1042,6 @@ export class Editor {
         pasteLinkExtension,
         attachmentExtension(this),
         closeBrackets(),
-        ...[this.collabState ? this.collabState.collabExtension() : []],
       ],
     });
   }
@@ -1070,7 +1054,7 @@ export class Editor {
     await Promise.all((await this.space.listPlugs()).map(async (plugName) => {
       try {
         await this.system.load(
-          new URL(`/.fs/${plugName}`, location.href),
+          new URL(plugName, location.origin),
           createSandbox,
         );
       } catch (e: any) {
@@ -1181,6 +1165,13 @@ export class Editor {
       name = this.settings!.indexPage;
     }
 
+    if (!isValidPageName(name)) {
+      return this.flashNotification(
+        "Invalid page name: page names cannot end with a file extension",
+        "error",
+      );
+    }
+
     if (newWindow) {
       const win = window.open(`${location.origin}/${name}`, "_blank");
       if (win) {
@@ -1206,10 +1197,6 @@ export class Editor {
       this.space.unwatchPage(previousPage);
       if (previousPage !== pageName) {
         await this.save(true);
-        // And stop the collab session
-        if (this.collabState) {
-          this.stopCollab();
-        }
       }
     }
 
@@ -1222,6 +1209,9 @@ export class Editor {
     let doc;
     try {
       doc = await this.space.readPage(pageName);
+      if (doc.meta.contentType.startsWith("text/html")) {
+        throw new Error("Got HTML page, not markdown");
+      }
     } catch (e: any) {
       // Not found, new page
       console.log("Creating new page", pageName);
@@ -1578,50 +1568,4 @@ export class Editor {
     }
     return;
   }
-
-  startCollab(
-    serverUrl: string,
-    token: string,
-    username: string,
-    isLocalCollab = false,
-  ) {
-    if (this.collabState) {
-      // Clean up old collab state
-      this.collabState.stop();
-    }
-    const initialText = this.editorView!.state.sliceDoc();
-    this.collabState = new CollabState(
-      serverUrl,
-      `${this.currentPage!}.md`,
-      token,
-      username,
-      this.syncService,
-      isLocalCollab,
-    );
-
-    this.collabState.collabProvider.on("synced", () => {
-      if (this.collabState!.ytext.toString() === "") {
-        console.log(
-          "[Collab]",
-          "Synced value is empty (new collab session), inserting local copy",
-        );
-        this.collabState!.ytext.insert(0, initialText);
-      }
-    });
-
-    this.rebuildEditorState();
-
-    // Don't watch for local changes in this mode
-    this.space.unwatch();
-  }
-
-  stopCollab() {
-    if (this.collabState) {
-      this.collabState.stop();
-      this.collabState = undefined;
-      this.rebuildEditorState();
-    }
-    // Start file watching again
-    this.space.watch();
-  }
 }

web/index.html

@@ -35,13 +35,11 @@
     window.silverBulletConfig = {
       // These {{VARIABLES}} are replaced by http_server.ts
       spaceFolderPath: "{{SPACE_PATH}}",
-      syncEndpoint: "{{SYNC_ENDPOINT}}",
     };
     // But in case these variables aren't replaced by the server, fall back fully static mode (no sync)
     if (window.silverBulletConfig.spaceFolderPath.includes("{{")) {
       window.silverBulletConfig = {
         spaceFolderPath: "",
-        syncEndpoint: "/.fs"
       };
     }
   </script>

web/service_worker.ts

@@ -24,42 +24,40 @@ const precacheFiles = Object.fromEntries([
 self.addEventListener("install", (event: any) => {
   console.log("[Service worker]", "Installing service worker...");
   event.waitUntil(
-    caches.open(CACHE_NAME)
+    (async () => {
-      .then((cache) => {
+      const cache = await caches.open(CACHE_NAME);
-        console.log(
+      console.log(
-          "[Service worker]",
+        "[Service worker]",
-          "Now pre-caching client files",
+        "Now pre-caching client files",
-        );
+      );
-        return cache.addAll(Object.values(precacheFiles)).then(() => {
+      await cache.addAll(Object.values(precacheFiles));
-          console.log(
+      console.log(
-            "[Service worker]",
+        "[Service worker]",
-            Object.keys(precacheFiles).length,
+        Object.keys(precacheFiles).length,
-            "client files cached",
+        "client files cached",
-          );
+      );
-          // @ts-ignore: No need to wait
+      // @ts-ignore: No need to wait
-          self.skipWaiting();
+      self.skipWaiting();
-        });
+    })(),
-      }),
   );
 });
 
 self.addEventListener("activate", (event: any) => {
   console.log("[Service worker]", "Activating new service worker!!!");
   event.waitUntil(
-    caches.keys().then((cacheNames) => {
+    (async () => {
-      return Promise.all(
+      const cacheNames = await caches.keys();
+      await Promise.all(
         cacheNames.map((cacheName) => {
           if (cacheName !== CACHE_NAME) {
             console.log("[Service worker]", "Removing old cache", cacheName);
             return caches.delete(cacheName);
           }
         }),
-      ).then(() => {
+      );
-        // Let's activate ourselves for all existing clients
+      // @ts-ignore: No need to wait
-        // @ts-ignore: No need to wait, clients is a serviceworker thing
+      return clients.claim();
-        return clients.claim();
+    })(),
-      });
-    }),
   );
 });
 
@@ -75,6 +73,15 @@ self.addEventListener("fetch", (event: any) => {
 
   event.respondWith(
     (async () => {
+      const request = event.request;
+
+      // console.log("Getting request", request, [...request.headers.entries()]);
+
+      // Any request with the X-Sync-Mode header originates from the sync engine: pass it on to the server
+      if (request.headers.has("x-sync-mode")) {
+        return fetch(request);
+      }
+
       // Try the static (client) file cache first
       const cachedResponse = await caches.match(cacheKey);
       // Return the cached response if found
@@ -82,67 +89,65 @@ self.addEventListener("fetch", (event: any) => {
         return cachedResponse;
       }
 
-      const requestUrl = new URL(event.request.url);
+      const requestUrl = new URL(request.url);
-
       const pathname = requestUrl.pathname;
-      // console.log("In service worker, pathname is", pathname);
-      // Are we fetching a URL from the same origin as the app? If not, we don't handle it here
-      const fetchingLocal = location.host === requestUrl.host;
 
-      if (!fetchingLocal) {
+      // Are we fetching a URL from the same origin as the app? If not, we don't handle it and pass it on
-        return fetch(event.request);
+      if (location.host !== requestUrl.host) {
+        return fetch(request);
       }
 
-      // If this is a /.fs request, this can either be a plug worker load or an attachment load
+      // If this is a /*.* request, this can either be a plug worker load or an attachment load
-      if (pathname.startsWith("/.fs")) {
+      if (/\/.+\.[a-zA-Z]+$/.test(pathname)) {
-        if (!fileContentTable || event.request.headers.has("x-sync-mode")) {
+        return handleLocalFileRequest(request, pathname);
-          // Not initialzed yet, or explicitly in sync mode (so direct server communication requested)
-          return fetch(event.request);
-        }
-        // console.log(
-        //   "Attempting to serve file from locally synced space:",
-        //   pathname,
-        // );
-        const path = decodeURIComponent(
-          requestUrl.pathname.slice("/.fs/".length),
-        );
-        const data = await fileContentTable.get(path);
-        if (data) {
-          // console.log("Serving from space", path);
-          if (!data.meta) {
-            // Legacy database not fully synced yet
-            data.meta = (await fileMetatable!.get(path))!;
-          }
-          return new Response(
-            data.data,
-            {
-              headers: {
-                "Content-type": data.meta.contentType,
-                "Content-Length": "" + data.meta.size,
-                "X-Permission": data.meta.perm,
-                "X-Last-Modified": "" + data.meta.lastModified,
-              },
-            },
-          );
-        } else {
-          console.error(
-            "Did not find file in locally synced space",
-            path,
-          );
-          return new Response("Not found", {
-            status: 404,
-          });
-        }
       } else if (pathname === "/.auth") {
-        return fetch(event.request);
+        return fetch(request);
       } else {
         // Must be a page URL, let's serve index.html which will handle it
-        return (await caches.match(precacheFiles["/"])) || fetch(event.request);
+        return (await caches.match(precacheFiles["/"])) || fetch(request);
       }
     })(),
   );
 });
 
+async function handleLocalFileRequest(
+  request: Request,
+  pathname: string,
+): Promise<Response> {
+  if (!fileContentTable) {
+    // Not initialzed yet, or explicitly in sync mode (so direct server communication requested)
+    return fetch(request);
+  }
+  const path = decodeURIComponent(pathname.slice(1));
+  const data = await fileContentTable.get(path);
+  if (data) {
+    // console.log("Serving from space", path);
+    if (!data.meta) {
+      // Legacy database not fully synced yet
+      data.meta = (await fileMetatable!.get(path))!;
+    }
+    return new Response(
+      data.data,
+      {
+        headers: {
+          "Content-type": data.meta.contentType,
+          "Content-Length": "" + data.meta.size,
+          "X-Permission": data.meta.perm,
+          "X-Last-Modified": "" + data.meta.lastModified,
+        },
+      },
+    );
+  } else {
+    console.error(
+      "Did not find file in locally synced space",
+      path,
+    );
+    return new Response("Not found", {
+      status: 404,
+    });
+  }
+}
+
 self.addEventListener("message", (event: any) => {
   if (event.data.type === "flushCache") {
     caches.delete(CACHE_NAME)

web/sync_service.ts

@@ -17,7 +17,7 @@ const syncStartTimeKey = "syncStartTime";
 // Keeps the last time an activity was registered, used to detect if a sync is still alive and whether a new one should be started already
 const syncLastActivityKey = "syncLastActivity";
 
-const syncExcludePrefix = "syncExclude:";
+const syncInitialFullSyncCompletedKey = "syncInitialFullSyncCompleted";
 
 // maximum time between two activities before we consider a sync crashed
 const syncMaxIdleTimeout = 1000 * 20; // 20s
@@ -56,17 +56,9 @@ export class SyncService {
       await this.syncFile(`${name}.md`);
     });
 
-    eventHook.addLocalListener("editor:pageSaved", async (name, meta) => {
+    eventHook.addLocalListener("editor:pageSaved", async (name) => {
       const path = `${name}.md`;
       await this.syncFile(path);
-      if (await this.isExcludedFromSync(path)) {
-        // So we're editing a page and just saved it, but it's excluded from sync
-        // Assumption: we're in collab mode for this file, so we're going to constantly update our local hash
-        // console.log(
-        //   "Locally updating last modified in snapshot because we're in collab mode",
-        // );
-        await this.updateLocalLastModified(path, meta.lastModified);
-      }
     });
   }
 
@@ -86,10 +78,9 @@ export class SyncService {
     return true;
   }
 
-  async hasInitialSyncCompleted(): Promise<boolean> {
+  hasInitialSyncCompleted(): Promise<boolean> {
     // Initial sync has happened when sync progress has been reported at least once, but the syncStartTime has been reset (which happens after sync finishes)
-    return !(await this.kvStore.has(syncStartTimeKey)) &&
+    return this.kvStore.has(syncInitialFullSyncCompletedKey);
-      (await this.kvStore.has(syncLastActivityKey));
   }
 
   async registerSyncStart(): Promise<void> {
@@ -116,40 +107,7 @@ export class SyncService {
   async registerSyncStop(): Promise<void> {
     await this.registerSyncProgress();
     await this.kvStore.del(syncStartTimeKey);
-  }
+    await this.kvStore.set(syncInitialFullSyncCompletedKey, true);
-
-  // Temporarily exclude a specific file from sync (e.g. when in collab mode)
-  excludeFromSync(path: string): Promise<void> {
-    return this.kvStore.set(syncExcludePrefix + path, Date.now());
-  }
-
-  unExcludeFromSync(path: string): Promise<void> {
-    return this.kvStore.del(syncExcludePrefix + path);
-  }
-
-  async isExcludedFromSync(path: string): Promise<boolean> {
-    const lastExcluded = await this.kvStore.get(syncExcludePrefix + path);
-    return lastExcluded && Date.now() - lastExcluded < syncMaxIdleTimeout;
-  }
-
-  async fetchAllExcludedFromSync(): Promise<string[]> {
-    const entries = await this.kvStore.queryPrefix(syncExcludePrefix);
-    const expiredPaths: string[] = [];
-    const now = Date.now();
-    const result = entries.filter(({ key, value }) => {
-      if (now - value > syncMaxIdleTimeout) {
-        expiredPaths.push(key);
-        return false;
-      }
-      return true;
-    }).map(({ key }) => key.slice(syncExcludePrefix.length));
-
-    if (expiredPaths.length > 0) {
-      console.log("Purging expired sync exclusions: ", expiredPaths);
-      await this.kvStore.batchDelete(expiredPaths);
-    }
-
-    return result;
   }
 
   async getSnapshot(): Promise<Map<string, SyncStatusItem>> {
@@ -167,83 +125,6 @@ export class SyncService {
     }
   }
 
-  // When in collab mode, we delegate the sync to the CDRT engine, to avoid conflicts, we try to keep the lastModified time in sync with the remote
-  async updateRemoteLastModified(path: string, lastModified: number) {
-    await this.noOngoingSync();
-    await this.registerSyncStart();
-    const snapshot = await this.getSnapshot();
-    const entry = snapshot.get(path);
-    if (entry) {
-      snapshot.set(path, [entry[0], lastModified]);
-    } else {
-      // In the unlikely scenario that a space first openen on a collab page before every being synced
-      try {
-        console.log(
-          "Received lastModified time for file not in snapshot",
-          path,
-          lastModified,
-        );
-        snapshot.set(path, [
-          (await this.localSpacePrimitives.getFileMeta(path)).lastModified,
-          lastModified,
-        ]);
-      } catch (e) {
-        console.warn(
-          "Received lastModified time for non-existing file not in snapshot",
-          path,
-          lastModified,
-          e,
-        );
-      }
-    }
-    await this.saveSnapshot(snapshot);
-    await this.registerSyncStop();
-  }
-
-  // Reach out out to remote space, fetch the latest lastModified time and update the local snapshot
-  // This is used when exiting collab mode
-  async fetchAndPersistRemoteLastModified(path: string) {
-    const meta = await this.remoteSpace.getFileMeta(path);
-    await this.updateRemoteLastModified(
-      path,
-      meta.lastModified,
-    );
-  }
-
-  // When in collab mode, we delegate the sync to the CDRT engine, to avoid conflicts, we try to keep the lastModified time in sync when local changes happen
-  async updateLocalLastModified(path: string, lastModified: number) {
-    await this.noOngoingSync();
-    await this.registerSyncStart();
-    const snapshot = await this.getSnapshot();
-    const entry = snapshot.get(path);
-    if (entry) {
-      snapshot.set(path, [lastModified, entry[1]]);
-    } else {
-      // In the unlikely scenario that a space first openen on a collab page before every being synced
-      try {
-        console.log(
-          "Setting lastModified time for file not in snapshot",
-          path,
-          lastModified,
-        );
-        snapshot.set(path, [
-          lastModified,
-          (await this.localSpacePrimitives.getFileMeta(path)).lastModified,
-        ]);
-      } catch (e) {
-        console.warn(
-          "Received lastModified time for non-existing file not in snapshot",
-          path,
-          lastModified,
-          e,
-        );
-      }
-    }
-    await this.saveSnapshot(snapshot);
-    await this.registerSyncStop();
-    // console.log("All done!");
-  }
-
   start() {
     this.syncSpace().catch(
       console.error,
@@ -271,14 +152,11 @@ export class SyncService {
     await this.registerSyncStart();
     let operations = 0;
     const snapshot = await this.getSnapshot();
-    // Fetch the list of files that are excluded from sync (e.g. because they're in collab mode)
-    const excludedFromSync = await this.fetchAllExcludedFromSync();
     // console.log("Excluded from sync", excludedFromSync);
     try {
       operations = await this.spaceSync!.syncFiles(
         snapshot,
-        (path) =>
+        (path) => this.isSyncCandidate(path),
-          this.isSyncCandidate(path) && !excludedFromSync.includes(path),
       );
       this.eventHook.dispatchEvent("sync:success", operations);
     } catch (e: any) {
@@ -295,7 +173,7 @@ export class SyncService {
       // console.log("Already syncing");
       return;
     }
-    if (!this.isSyncCandidate(name) || (await this.isExcludedFromSync(name))) {
+    if (!this.isSyncCandidate(name)) {
       return;
     }
     await this.registerSyncStart();

web/syscalls/collab.ts

@@ -1,20 +0,0 @@
-import { SysCallMapping } from "../../plugos/system.ts";
-import type { Editor } from "../editor.tsx";
-
-export function collabSyscalls(editor: Editor): SysCallMapping {
-  return {
-    "collab.start": (
-      _ctx,
-      serverUrl: string,
-      token: string,
-      username: string,
-    ) => {
-      editor.startCollab(serverUrl, token, username);
-    },
-    "collab.stop": (
-      _ctx,
-    ) => {
-      editor.stopCollab();
-    },
-  };
-}

web/syscalls/fetch.ts

@@ -21,7 +21,7 @@ export function sandboxFetchSyscalls(
         return performLocalFetch(url, options);
       }
       const resp = httpSpacePrimitives.authenticatedFetch(
-        httpSpacePrimitives.url,
+        `${httpSpacePrimitives.url}/.rpc`,
         {
           method: "POST",
           body: JSON.stringify({

web/syscalls/shell.ts

@@ -14,7 +14,7 @@ export function shellSyscalls(
         throw new Error("Not supported in fully local mode");
       }
       const resp = httpSpacePrimitives.authenticatedFetch(
-        httpSpacePrimitives.url,
+        `${httpSpacePrimitives.url}/.rpc`,
         {
           method: "POST",
           body: JSON.stringify({

website/API.md

@@ -0,0 +1,12 @@
+The server API is relatively small. The client primarily communicates with the server for file “CRUD” (Create, Read, Update, Delete) style operations.
+
+Here’s an attempt to document this API:
+
+* `GET /index.json` (when sent with an `Accept: application/json` request header): will return a full listing of all files in your space including meta data like when the file was last modified, as well as permissions. This is primarily for sync purposes with the client. A request sent without the mentioned `Accept` header will redirect to `/` (to better support authentication layers like [Authelia](https://www.authelia.com/)).
+* `GET /*.*`: _Reads_ and returns the content of the file at the given path. This means that if you `GET /index.md` you will receive the content of your `index` page. The `GET` response will have a few additional SB-specific headers:
+  * `X-Last-Modified` as a UNIX timestamp in ms (as coming from `Data.now()`)
+  * `X-Permission`: either `rw` or `ro` which will change whether the editor opens in read-only or regular mode.
+* `PUT /*.*`: The same as `GET` except that it takes the body of the request and _writes_ it a file.
+* `DELETE /*.*`: Again the same, except this will _delete_ the given file.
+* `GET /.client/*`: Retrieve files implementing the client
+* `GET /*` and `GET /`: Anything else (any path without a file extension) will serve the SilverBullet UI HTML.

website/CHANGELOG.md

@@ -3,6 +3,12 @@ release.
 
 ---
 
+## Next
+* **Removal of all real-time collaboration features**: this was causing too many edge cases, and complicated the code too much. To simplify the product as well as the code, we completely removed all real-time collaboration features for now. We may introduce this at some point in the future when the demand and focus is there.
+* **Change of APIs**: This is mostly internal, but will likely have effects on the first load after the upgrade: you may see errors or a message around “the path has changed”, or your page may not properly load. Don’t freak out, just reload once or twice and all should resync and be fine. There’s a beginning of documenting the server [[API]] now.
+
+---
+
 ## 0.3.4
 
 * **Breaking change (for some templates):** Template in various places allowed you to use `{{variables}}` and various handlebars functions. There also used to be a magic `{{page}}` variable that you could use in various places, but not everywhere. This has now been unified. And the magical `{{page}}` now has been replaced with the global `@page` which does not just expose the page’s name, but any page meta data. More information here: [[🔌 Core/Templates@vars]]. You will now get completion for built-in handlebars helpers after typing `{{`.
@@ -18,8 +24,7 @@ release.
 
 ## 0.3.2
 
-* **Real-time collaboration support** between clients: Open the same page in multiple windows (browser tabs, mobile devices) and within a few seconds you should get kicked into real-time collaboration mode, showing other participants cursors, selections and edits in real time (Google doc style). This only works when a connection with the server can be established.
+* REMOVED:  **Real-time collaboration support** between clients: Open the same page in multiple windows (browser tabs, mobile devices) and within a few seconds you should get kicked into real-time collaboration mode, showing other participants cursors, selections and edits in real time (Google doc style). This only works when a connection with the server can be established.
-  * This **breaks** existing [[🔌 Collab]] links, since we switched real-time collaboration libraries. We’re still looking at the best way to keep supporting this feature.
 * [[Authentication|Multi-user authentication]]: you can now allow multiple user accounts authenticate, which makes the real-time collaboration support actually useful. This feature is still experimental and will likely evolve over time.
 * Added `spaceIgnore` setting to not sync specific folders or file patterns to the client, see [[SETTINGS]] for documentation
 * Much improved image loading behavior on page (previously scroll bars would jump up and down like a mad person)
@@ -209,7 +214,7 @@ Besides these architectural changes, a few other breaking changes were made to s
 * New `Plugs: Add` command to quickly add a new plug (will create a `PLUGS` page if you don't have one yet).
 * **Paste without formatting**: holding `Shift` while pasting will disable "rich text paste."
 * **New core plug:** [[🔌 Share]] for sharing your pages with the outside work (such as collab, see below).
-* **New plug:** [[🔌 Collab]] for real-time collaboration on your pages.
+* **New plug:** 🔌 Collab for real-time collaboration on your pages.
 
 ---
 

website/SilverBullet.md

@@ -15,7 +15,6 @@ Now that we got that out of the way, let’s have a look at some of SilverBullet
   * Run commands via their keyboard shortcuts, or the **command palette** (triggered with `Cmd-/` or `Ctrl-/` on Linux and Windows).
   * Use [[🔌 Core/Slash Commands|slash commands]] to perform common text editing operations.
 * Provides a platform for [end-user programming](https://www.inkandswitch.com/end-user-programming/) through its support for annotating pages with [[Frontmatter]] and [[🔌 Directive|directives]] (such as [[🔌 Directive/Query|#query]]), making parts of pages _dynamic_.
-* Experimental [[🔌 Collab|real-time collaboration support]].
 * Robust extension mechanism using [[🔌 Plugs]].
 * **Self-hosted**: you own your data. All content is stored as plain files in a folder on disk. Back up, sync, edit, publish, script with any additional tools you like.
 * SilverBullet is [open source, MIT licensed](https://github.com/silverbulletmd/silverbullet) software.

website/_headers

@@ -1,14 +1,7 @@
-/.fs/_plug/*
+/*
-  X-Last-Modified: 12345
-  X-Permission: ro
-  access-control-allow-headers: *
-  access-control-allow-methods: GET,POST,PUT,DELETE,OPTIONS
-  access-control-allow-origin: *
-  access-control-expose-headers: *
-/.fs/*
   X-Last-Modified: 12345
   X-Permission: rw
   access-control-allow-headers: *
-  access-control-allow-methods: GET,POST,PUT,DELETE,OPTIONS
+  access-control-allow-methods: GET,POST,PUT,DELETE,OPTIONS,HEAD
   access-control-allow-origin: *
   access-control-expose-headers: *

website/_redirects

@@ -1,5 +1,3 @@
-/.fs   /index.json 200
+# /.fs     /index.json 200
-/.fs/* /_fs/:splat 200!
-/.fs/* /empty.md   200
 /.client/* /_client/:splat 200!
-/*    /_client/index.html 200
+/*         /_client/index.html 200

website/🔌 Collab.md

@@ -1,41 +0,0 @@
----
-type: plug
-repo: https://github.com/silverbulletmd/silverbullet
-share-support: true
----
-
-The Collab plug implements real-time “Google Doc” style collaboration with other SilverBullet users using the [Hocuspocus](https://hocuspocus.dev/) library. It supports:
-
-* Real-time editing
-* Showing other participant’s cursors and selections
-
-The philosophy behind this plug is that by default your data is private and not shared with others. However, there are cases where you would like to collaborate on individual pages. 
-
-Some example use cases:
-
-* Sharing a meeting agenda or meeting notes
-* Writing or editing an article with others
-
-The collab plug allows you to share individual pages. All collaborators will keep their own local copy on disk (which they can back up, and you probably should), but the “source of truth” moves to a central collaboration server. There is one deployed at `wss://collab.silverbullet.md`, but you can also run your own, see [[@deploy|the instructions below]]. The collab plugin leverages SB’s [[🔌 Share]] infrastructure.
-
-To use it:
-
-1. Open a page you would like to collaborate on
-2. Run the {[Share: Collab]} command and select the collab server to use (an open one runs at `wss://collab.silverbullet.md`)
-3. Copy & paste the `collab:...` URI that is injected into the `$share` [[Frontmatter]] and send to a collaborator **or** if your collaborator is not (yet) a SilverBullet user, you can use the silverbullet.md website (which is an SB instance) directly via the `https://silverbullet.md/collab:...` URL scheme.
-4. If your collaborator is an SB user, have them use the {[Share: Join Collab]} command, or directly open the `collab:...` URI as a page in SilverBullet (both do the same).
-5. If the collaborator wants to keep a persistent copy of the page collaborated page, they can simply _rename_ the page to something not prefixed with `collab:`. Everything will keep working for as long as the `collab:` will appear in the `$share` attribute of [[Frontmatter]]
-
-## How it works
-The Collab plug uses Hocuspocus for real-time collaboration via a WebSocket. A random ID is assigned to every shared page, and a copy of this page (as well as its history) will be stored on the collaboration server. Therefore, be cautious about what you share, especially when using a public collab server like `collab.silverbullet.md`. For “production use” we recommend deploying your own collab server.
-
-## Deploying your own collab server
-$deploy
-
-Collaboration uses the excellent Hocuspocus library. You can easily deploy your own collaboration server as follows (requires node.js and npm):
-
-```shell
-npx @hocuspocus/cli@2.0.6 --sqlite documents.db --port 1337
-```
-
-This will run the hocuspocus server on port 1337, and store page data persistently in a SQLite database `documents.db`. You can connect to this server via `ws://ip:1337`. To use SSL, put a TLS terminator in front of it, in which case you can use `wss://` instead.