Something works

plugs/core/page.ts

@@ -20,18 +20,17 @@ export async function indexLinks({ name, text }: IndexEvent) {
 }
 
 export async function deletePage() {
-  let pageMeta = await syscall("editor.getCurrentPage");
+  let pageName = await syscall("editor.getCurrentPage");
   console.log("Navigating to start page");
   await syscall("editor.navigate", "start");
   console.log("Deleting page from space");
-  await syscall("space.deletePage", pageMeta.name);
+  await syscall("space.deletePage", pageName);
   console.log("Reloading page list");
   await syscall("space.reloadPageList");
 }
 
 export async function renamePage() {
-  const pageMeta = await syscall("editor.getCurrentPage");
+  const oldName = await syscall("editor.getCurrentPage");
-  const oldName = pageMeta.name;
   console.log("Old name is", oldName);
   const newName = await syscall(
     "editor.prompt",
@@ -98,8 +97,8 @@ async function getBackLinks(pageName: string): Promise<BackLink[]> {
 }
 
 export async function showBackLinks() {
-  const pageMeta = await syscall("editor.getCurrentPage");
+  const pageName = await syscall("editor.getCurrentPage");
-  let backLinks = await getBackLinks(pageMeta.name);
+  let backLinks = await getBackLinks(pageName);
 
   console.log("Backlinks", backLinks);
 }

server/.vscode/settings.json

@@ -1,4 +0,0 @@
-{
-    "deno.enable": false,
-    "deno.unstable": true
-}

server/src/server.ts

@@ -5,12 +5,21 @@ import { readdir, readFile, stat, unlink } from "fs/promises";
 import path from "path";
 import stream from "stream";
 import { promisify } from "util";
+import { debounce } from "lodash";
 
 import { ChangeSet, Text } from "@codemirror/state";
 import { Update } from "@codemirror/collab";
 import http from "http";
 import { Server } from "socket.io";
 
+import { cursorEffect } from "../../webapp/src/cursorEffect";
+
+function safeRun(fn: () => Promise<void>) {
+  fn().catch((e) => {
+    console.error(e);
+  });
+}
+
 const app = express();
 const server = http.createServer(app);
 const io = new Server(server, {
@@ -82,7 +91,7 @@ class DiskFS {
   async writePage(pageName: string, body: any): Promise<PageMeta> {
     let localPath = path.join(pagesPath, pageName + ".md");
     await pipeline(body, fs.createWriteStream(localPath));
-    console.log(`Wrote to ${localPath}`);
+    // console.log(`Wrote to ${localPath}`);
     const s = await stat(localPath);
     return {
       name: pageName,
@@ -105,14 +114,245 @@ class DiskFS {
   }
 }
 
+import { Socket } from "socket.io";
+
+class Page {
+  text: Text;
+  updates: Update[];
+  sockets: Set<Socket>;
+  meta: PageMeta;
+
+  pending: ((value: any) => void)[] = [];
+
+  saveTimer: NodeJS.Timeout | undefined;
+
+  constructor(text: string, meta: PageMeta) {
+    this.updates = [];
+    this.text = Text.of(text.split("\n"));
+    this.meta = meta;
+    this.sockets = new Set<Socket>();
+  }
+}
+
+class RealtimeEditFS extends DiskFS {
+  openPages = new Map<string, Page>();
+
+  disconnectSocket(socket: Socket, pageName: string) {
+    let page = this.openPages.get(pageName);
+    if (page) {
+      page.sockets.delete(socket);
+      if (page.sockets.size === 0) {
+        console.log("No more sockets for", pageName, "flushing");
+        this.flushPageToDisk(pageName, page);
+        this.openPages.delete(pageName);
+      }
+    }
+  }
+
+  flushPageToDisk(name: string, page: Page) {
+    super
+      .writePage(name, page.text.sliceString(0))
+      .then((meta) => {
+        console.log(`Wrote page ${name} to disk`);
+        page.meta = meta;
+      })
+      .catch((e) => {
+        console.log(`Could not write ${name} to disk:`, e);
+      });
+  }
+
+  // Override
+  async readPage(pageName: string): Promise<{ text: string; meta: PageMeta }> {
+    let page = this.openPages.get(pageName);
+    if (page) {
+      console.log("Serving page from memory", pageName);
+      return {
+        text: page.text.sliceString(0),
+        meta: page.meta,
+      };
+    } else {
+      return super.readPage(pageName);
+    }
+  }
+
+  async writePage(pageName: string, body: any): Promise<PageMeta> {
+    let page = this.openPages.get(pageName);
+    if (page) {
+      for (let socket of page.sockets) {
+        socket.emit("reload", pageName);
+      }
+      this.openPages.delete(pageName);
+    }
+    return super.writePage(pageName, body);
+  }
+
+  constructor(rootPath: string, io: Server) {
+    super(rootPath);
+
+    setInterval(() => {
+      console.log("Currently open pages:", this.openPages.keys());
+    }, 10000);
+
+    // Disk watcher
+    fs.watch(
+      rootPath,
+      {
+        recursive: true,
+        persistent: false,
+      },
+      (eventType, filename) => {
+        safeRun(async () => {
+          if (path.extname(filename) !== ".md") {
+            return;
+          }
+          let localPath = path.join(rootPath, filename);
+          let pageName = filename.substring(0, filename.length - 3);
+          let s = await stat(localPath);
+          // console.log("Edit in", pageName);
+          const openPage = this.openPages.get(pageName);
+          if (openPage) {
+            if (openPage.meta.lastModified < s.mtime.getTime()) {
+              console.log("Page changed on disk outside of editor, reloading");
+              for (let socket of openPage.sockets) {
+                socket.emit("reload", pageName);
+              }
+              this.openPages.delete(pageName);
+            }
+          }
+        });
+      }
+    );
+
+    io.on("connection", (socket) => {
+      console.log("Connected", socket.id);
+      let socketOpenPages = new Set<string>();
+
+      function onCall(eventName: string, cb: (...args: any[]) => Promise<any>) {
+        socket.on(eventName, (reqId: number, ...args) => {
+          cb(...args).then((result) => {
+            socket.emit(`${eventName}Resp${reqId}`, result);
+          });
+        });
+      }
+
+      onCall("openPage", async (pageName: string) => {
+        let page = this.openPages.get(pageName);
+        if (!page) {
+          try {
+            let { text, meta } = await super.readPage(pageName);
+            page = new Page(text, meta);
+          } catch (e) {
+            // console.log(`Could not open ${pageName}:`, e);
+            // Page does not exist, let's create a new one
+            console.log("Creating new page", pageName);
+            page = new Page("", { name: pageName, lastModified: 0 });
+          }
+          this.openPages.set(pageName, page);
+        }
+        page.sockets.add(socket);
+        socketOpenPages.add(pageName);
+        console.log("Opened page", pageName);
+        return [page.updates.length, page.text.toJSON()];
+      });
+
+      socket.on("closePage", (pageName: string) => {
+        console.log("Closing page", pageName);
+        this.disconnectSocket(socket, pageName);
+        socketOpenPages.delete(pageName);
+      });
+
+      onCall(
+        "pushUpdates",
+        async (
+          pageName: string,
+          version: number,
+          updates: any[]
+        ): Promise<boolean> => {
+          let page = this.openPages.get(pageName);
+
+          if (!page) {
+            console.error(
+              "Received updates for not open page",
+              pageName,
+              this.openPages.keys()
+            );
+            return;
+          }
+          if (version !== page.updates.length) {
+            console.error("Invalid version", version, page.updates.length);
+            return false;
+          } else {
+            console.log("Applying", updates.length, "updates");
+            let transformedUpdates = [];
+            for (let update of updates) {
+              let changes = ChangeSet.fromJSON(update.changes);
+              console.log("Got effect", update);
+              let transformedUpdate = {
+                changes,
+                clientID: update.clientID,
+                effects: update.cursors?.map((c) => {
+                  return cursorEffect.of(c);
+                }),
+              };
+              page.updates.push(transformedUpdate);
+              transformedUpdates.push(transformedUpdate);
+              // TODO: save cursors locally as well
+              page.text = changes.apply(page.text);
+            }
+
+            if (page.saveTimer) {
+              clearTimeout(page.saveTimer);
+            }
+
+            page.saveTimer = setTimeout(() => {
+              this.flushPageToDisk(pageName, page);
+            }, 1000);
+            while (page.pending.length) {
+              page.pending.pop()!(transformedUpdates);
+            }
+            return true;
+          }
+        }
+      );
+
+      onCall(
+        "pullUpdates",
+        async (pageName: string, version: number): Promise<Update[]> => {
+          let page = this.openPages.get(pageName);
+          // console.log("Pulling updates for", pageName);
+          if (!page) {
+            console.error("Fetching updates for not open page");
+            return [];
+          }
+          if (version < page.updates.length) {
+            return page.updates.slice(version);
+          } else {
+            return new Promise((resolve) => {
+              page.pending.push(resolve);
+            });
+          }
+        }
+      );
+
+      socket.on("disconnect", () => {
+        console.log("Disconnected", socket.id);
+        socketOpenPages.forEach((page) => {
+          this.disconnectSocket(socket, page);
+        });
+      });
+    });
+  }
+}
+
 app.use("/", express.static(distDir));
 
 let fsRouter = express.Router();
-let diskFS = new DiskFS(pagesPath);
+// let diskFS = new DiskFS(pagesPath);
+let filesystem = new RealtimeEditFS(pagesPath, io);
 
 // Page list
 fsRouter.route("/").get(async (req, res) => {
-  res.json(await diskFS.listPages());
+  res.json(await filesystem.listPages());
 });
 
 fsRouter
@@ -121,7 +361,7 @@ fsRouter
     let reqPath = req.params[0];
     console.log("Getting", reqPath);
     try {
-      let { text, meta } = await diskFS.readPage(reqPath);
+      let { text, meta } = await filesystem.readPage(reqPath);
       res.status(200);
       res.header("Last-Modified", "" + meta.lastModified);
       res.header("Content-Type", "text/markdown");
@@ -135,7 +375,7 @@ fsRouter
     let reqPath = req.params[0];
 
     try {
-      let meta = await diskFS.writePage(reqPath, req);
+      let meta = await filesystem.writePage(reqPath, req);
       res.status(200);
       res.header("Last-Modified", "" + meta.lastModified);
       res.send("OK");
@@ -148,7 +388,7 @@ fsRouter
   .options(async (req, res) => {
     let reqPath = req.params[0];
     try {
-      const meta = await diskFS.getPageMeta(reqPath);
+      const meta = await filesystem.getPageMeta(reqPath);
       res.status(200);
       res.header("Last-Modified", "" + meta.lastModified);
       res.header("Content-Type", "text/markdown");
@@ -161,7 +401,7 @@ fsRouter
   .delete(async (req, res) => {
     let reqPath = req.params[0];
     try {
-      await diskFS.deletePage(reqPath);
+      await filesystem.deletePage(reqPath);
       res.status(200);
       res.send("OK");
     } catch (e) {
@@ -189,127 +429,6 @@ app.get("/*", async (req, res) => {
   res.status(200).header("Content-Type", "text/html").send(cachedIndex);
 });
 
-import { Socket } from "socket.io";
-
-class Page {
-  text: Text;
-  updates: Update[];
-  sockets: Map<string, Socket>;
-  meta: PageMeta;
-
-  pending: ((value: any) => void)[] = [];
-
-  constructor(text: string, meta: PageMeta) {
-    this.updates = [];
-    this.text = Text.of(text.split("\n"));
-    this.meta = meta;
-    this.sockets = new Map<string, Socket>();
-  }
-}
-
-let openPages = new Map<string, Page>();
-
-io.on("connection", (socket) => {
-  function disconnectSocket(pageName: string) {
-    let page = openPages.get(pageName);
-    if (page) {
-      page.sockets.delete(socket.id);
-      if (page.sockets.size === 0) {
-        console.log("No more sockets for", pageName, "flushing");
-        openPages.delete(pageName);
-      }
-    }
-  }
-
-  console.log("Connected", socket.id);
-  let socketOpenPages = new Set<string>();
-
-  function onCall(eventName: string, cb: (...args: any[]) => Promise<any>) {
-    socket.on(eventName, (reqId: number, ...args) => {
-      cb(...args).then((result) => {
-        socket.emit(`${eventName}Resp${reqId}`, result);
-      });
-    });
-  }
-
-  onCall("openPage", async (pageName: string) => {
-    let page = openPages.get(pageName);
-    if (!page) {
-      let { text, meta } = await diskFS.readPage(pageName);
-      page = new Page(text, meta);
-      openPages.set(pageName, page);
-    }
-    page.sockets.set(socket.id, socket);
-    socketOpenPages.add(pageName);
-    console.log("Sending document text");
-    let enhancedMeta = { ...page.meta, version: page.updates.length };
-    return [enhancedMeta, page.text.toJSON()];
-  });
-
-  socket.on("closePage", (pageName: string) => {
-    console.log("Closing page", pageName);
-    disconnectSocket(pageName);
-    socketOpenPages.delete(pageName);
-  });
-
-  onCall(
-    "pushUpdates",
-    async (
-      pageName: string,
-      version: number,
-      updates: any[]
-    ): Promise<boolean> => {
-      let page = openPages.get(pageName);
-
-      if (!page) {
-        console.error("Received updates for not open page");
-        return;
-      }
-      if (version !== page.updates.length) {
-        console.error("Invalid version", version, page.updates.length);
-        return false;
-      } else {
-        console.log("Applying", updates.length, "updates");
-        for (let update of updates) {
-          let changes = ChangeSet.fromJSON(update.changes);
-          page.updates.push({ changes, clientID: update.clientID });
-          page.text = changes.apply(page.text);
-        }
-        while (page.pending.length) {
-          page.pending.pop()!(updates);
-        }
-        return true;
-      }
-    }
-  );
-
-  onCall(
-    "pullUpdates",
-    async (pageName: string, version: number): Promise<Update[]> => {
-      let page = openPages.get(pageName);
-      console.log("Pulling updates for", pageName);
-      if (!page) {
-        console.error("Received updates for not open page");
-        return;
-      }
-      console.log(`Let's get real: ${version} < ${page.updates.length}`);
-      if (version < page.updates.length) {
-        console.log("Yes");
-        return page.updates.slice(version);
-      } else {
-        console.log("No");
-        return new Promise((resolve) => {
-          page.pending.push(resolve);
-        });
-      }
-    }
-  );
-
-  socket.on("disconnect", () => {
-    console.log("Disconnected", socket.id);
-    socketOpenPages.forEach(disconnectSocket);
-  });
-});
 //sup
 server.listen(port, () => {
   console.log(`Server istening on port ${port}`);

server/yarn.lock

@@ -670,6 +670,11 @@
     "@types/qs" "*"
     "@types/serve-static" "*"
 
+"@types/lodash@^4.14.179":
+  version "4.14.179"
+  resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.179.tgz#490ec3288088c91295780237d2497a3aa9dfb5c5"
+  integrity sha512-uwc1x90yCKqGcIOAT6DwOSuxnrAbpkdPsUOZtwrXb4D/6wZs+6qG7QnIawDuZWg0sWpxl+ltIKCaLoMlna678w==
+
 "@types/mime@^1":
   version "1.3.2"
   resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a"
@@ -1735,6 +1740,11 @@ lodash.uniq@^4.5.0:
   resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773"
   integrity sha1-0CJTc662Uq3BvILklFM5qEJ1R3M=
 
+lodash@^4.17.21:
+  version "4.17.21"
+  resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
+  integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
+
 lowercase-keys@^1.0.0, lowercase-keys@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-1.0.1.tgz#6f9e30b47084d971a7c820ff15a6c5167b74c26f"

webapp/package.json

@@ -18,6 +18,7 @@
     "@parcel/transformer-webmanifest": "2.3.2",
     "@parcel/validator-typescript": "^2.3.2",
     "@types/events": "^3.0.0",
+    "@types/lodash": "^4.14.179",
     "@types/node": "^17.0.21",
     "@types/react": "^17.0.39",
     "@types/react-dom": "^17.0.11",
@@ -52,6 +53,7 @@
     "@parcel/service-worker": "^2.3.2",
     "dexie": "^3.2.1",
     "idb": "^7.0.0",
+    "lodash": "^4.17.21",
     "react": "^17.0.2",
     "react-dom": "^17.0.2",
     "socket.io-client": "^4.4.1"

webapp/src/collab.ts

@@ -1,35 +1,129 @@
-import { EditorView, ViewPlugin, ViewUpdate } from "@codemirror/view";
+// TODO:
-import { HttpRemoteSpace, Space } from "./space";
+// Send state to client
+// Shape of editor.editorView.state.toJSON({"cursors": cursorField})
+// From there import it
+// EditorState.fromJSON(js, {extensions: cursorField}, {cursors: cursorField})
+
 import {
-  Update,
-  receiveUpdates,
-  sendableUpdates,
   collab,
   getSyncedVersion,
+  receiveUpdates,
+  sendableUpdates,
 } from "@codemirror/collab";
-import { PageMeta } from "./types";
+import { EditorState, StateEffect, StateField, Text } from "@codemirror/state";
-import { Text } from "@codemirror/state";
+import {
+  Decoration,
+  DecorationSet,
+  EditorView,
+  ViewPlugin,
+  ViewUpdate,
+  WidgetType,
+} from "@codemirror/view";
+import { cursorEffect } from "./cursorEffect";
+import { HttpRemoteSpace } from "./space";
 
 export class Document {
   text: Text;
-  meta: PageMeta;
+  version: number;
 
-  constructor(text: Text, meta: PageMeta) {
+  constructor(text: Text, version: number) {
     this.text = text;
-    this.meta = meta;
+    this.version = version;
+  }
+}
+
+let meId = "";
+
+const cursorField = StateField.define<DecorationSet>({
+  create() {
+    return Decoration.none;
+  },
+  update(cursors, tr) {
+    cursors = cursors.map(tr.changes);
+    for (let e of tr.effects) {
+      if (e.is(cursorEffect)) {
+        const newCursorDecoration = Decoration.widget({
+          widget: new CursorWidget(e.value.userId, e.value.color, e.value.pos),
+          side: 1,
+        });
+        cursors = cursors.update({
+          filter: (from, to, d) => !d.eq(newCursorDecoration),
+          // add: [newCursorDecoration.range(e.value.pos)],
+          sort: true,
+        });
+      }
+    }
+    // console.log("New cursors", cursors.size);
+    return cursors;
+  },
+  provide: (f) => EditorView.decorations.from(f),
+  fromJSON(cursorJSONs) {
+    let cursors = [];
+    for (let cursorJSON of cursorJSONs) {
+      cursors.push(
+        Decoration.widget({
+          widget: new CursorWidget(
+            cursorJSON.userId,
+            cursorJSON.color,
+            cursorJSON.pos
+          ),
+          side: 1,
+        }).range(cursorJSON.pos)
+      );
+    }
+    return Decoration.set(cursors);
+  },
+  toJSON(cursors) {
+    let cursor = cursors.iter();
+    let results = [];
+    while (cursor.value) {
+      results.push({ ...cursor.value.spec.widget });
+      cursor.next();
+    }
+    return results;
+  },
+});
+
+class CursorWidget extends WidgetType {
+  userId: string;
+  color: string;
+  pos: number;
+
+  constructor(userId: string, color: string, pos: number) {
+    super();
+    this.userId = userId;
+    this.color = color;
+    this.pos = pos;
+  }
+
+  eq(other: CursorWidget) {
+    return other.userId == this.userId;
+  }
+
+  toDOM() {
+    let el = document.createElement("span");
+    el.className = "other-cursor";
+    el.style.backgroundColor = this.color;
+    if (this.userId == meId) {
+      el.style.display = "none";
+    }
+    return el;
   }
 }
 
 export function collabExtension(
   pageName: string,
+  clientID: string,
   startVersion: number,
   space: HttpRemoteSpace,
   reloadCallback: () => void
 ) {
+  meId = clientID;
   let plugin = ViewPlugin.fromClass(
     class {
       private pushing = false;
       private done = false;
+      private failedPushes = 0;
 
       constructor(private view: EditorView) {
         if (pageName) {
@@ -38,19 +132,47 @@ export function collabExtension(
       }
 
       update(update: ViewUpdate) {
-        if (update.docChanged) this.push();
+        if (update.selectionSet) {
+          let pos = update.state.selection.main.head;
+          console.log("New pos", pos);
+          // return;
+          setTimeout(() => {
+            update.view.dispatch({
+              effects: [
+                cursorEffect.of({ pos: pos, userId: clientID, color: "red" }),
+              ],
+            });
+          });
+        }
+        let foundEffect = false;
+        for (let tx of update.transactions) {
+          if (tx.effects.some((e) => e.is(cursorEffect))) {
+            foundEffect = true;
+          }
+        }
+        if (update.docChanged || foundEffect) this.push();
       }
 
       async push() {
         let updates = sendableUpdates(this.view.state);
         if (this.pushing || !updates.length) return;
+        console.log("Updates", updates);
         this.pushing = true;
         let version = getSyncedVersion(this.view.state);
         let success = await space.pushUpdates(pageName, version, updates);
         this.pushing = false;
 
         if (!success) {
-          reloadCallback();
+          this.failedPushes++;
+          if (this.failedPushes > 10) {
+            // Not sure if 10 is a good number, but YOLO
+            console.log("10 pushes failed, reloading");
+            reloadCallback();
+            return this.destroy();
+          }
+          console.log("Push failed temporarily, but will try again");
+        } else {
+          this.failedPushes = 0;
         }
 
         // Regardless of whether the push failed or new updates came in
@@ -64,7 +186,9 @@ export function collabExtension(
         while (!this.done) {
           let version = getSyncedVersion(this.view.state);
           let updates = await space.pullUpdates(pageName, version);
-          this.view.dispatch(receiveUpdates(this.view.state, updates));
+          let d = receiveUpdates(this.view.state, updates);
+          console.log("Received", d);
+          this.view.dispatch(d);
         }
       }
 
@@ -73,5 +197,16 @@ export function collabExtension(
       }
     }
   );
-  return [collab({ startVersion }), plugin];
+
+  return [
+    collab({
+      startVersion,
+      clientID,
+      sharedEffects: (tr) => {
+        return tr.effects.filter((e) => e.is(cursorEffect));
+      },
+    }),
+    cursorField,
+    plugin,
+  ];
 }

webapp/src/components/page_navigator.tsx

@@ -9,11 +9,11 @@ export function PageNavigator({
 }: {
   allPages: PageMeta[];
   onNavigate: (page: string | undefined) => void;
-  currentPage?: PageMeta;
+  currentPage?: string;
 }) {
   let options: Option[] = [];
   for (let pageMeta of allPages) {
-    if (currentPage && currentPage.name == pageMeta.name) {
+    if (currentPage && currentPage === pageMeta.name) {
       continue;
     }
     // Order by last modified date in descending order

webapp/src/components/status_bar.tsx

@@ -1,13 +1,7 @@
 import { EditorView } from "@codemirror/view";
 import * as util from "../util";
 
-export function StatusBar({
+export function StatusBar({ editorView }: { editorView?: EditorView }) {
-  isSaved,
-  editorView,
-}: {
-  isSaved: boolean;
-  editorView?: EditorView;
-}) {
   let wordCount = 0,
     readingTime = 0;
   if (editorView) {
@@ -17,7 +11,7 @@ export function StatusBar({
   }
   return (
     <div id="bottom">
-      {wordCount} words | {readingTime} min | {isSaved ? "Saved" : "Edited"}
+      {wordCount} words | {readingTime} min
     </div>
   );
 }

webapp/src/components/top_bar.tsx

@@ -13,7 +13,7 @@ export function TopBar({
   currentPage,
   onClick,
 }: {
-  currentPage?: PageMeta;
+  currentPage?: string;
   onClick: () => void;
 }) {
   return (
@@ -22,7 +22,7 @@ export function TopBar({
         <span className="icon">
           <FontAwesomeIcon icon={faFileLines} />
         </span>
-        <span className="current-page">{prettyName(currentPage?.name)}</span>
+        <span className="current-page">{prettyName(currentPage)}</span>
       </div>
     </div>
   );

webapp/src/cursorEffect.ts

@@ -0,0 +1,11 @@
+import { StateEffect } from "@codemirror/state";
+
+export const cursorEffect = StateEffect.define<{
+  pos: number;
+  userId: string;
+  color: string;
+}>({
+  map({ pos, userId, color }, changes) {
+    return { pos: changes.mapPos(pos), userId, color };
+  },
+});

webapp/src/editor.tsx

@@ -20,6 +20,8 @@ import {
   keymap,
 } from "@codemirror/view";
 
+import { debounce } from "lodash";
+
 import React, { useEffect, useReducer } from "react";
 import ReactDOM from "react-dom";
 import coreManifest from "./generated/core.plug.json";
@@ -62,16 +64,15 @@ import { safeRun } from "./util";
 import { collabExtension } from "./collab";
 
 import { Document } from "./collab";
+import { EditorSelection } from "@codemirror/state";
 
 class PageState {
-  editorState: EditorState;
   scrollTop: number;
-  meta: PageMeta;
+  selection: EditorSelection;
 
-  constructor(editorState: EditorState, scrollTop: number, meta: PageMeta) {
+  constructor(scrollTop: number, selection: EditorSelection) {
-    this.editorState = editorState;
     this.scrollTop = scrollTop;
-    this.meta = meta;
+    this.selection = selection;
   }
 }
 
@@ -88,6 +89,7 @@ export class Editor implements AppEventDispatcher {
   indexer: Indexer;
   navigationResolve?: (val: undefined) => void;
   pageNavigator: IPageNavigator;
+  indexCurrentPageDebounced: () => any;
 
   constructor(space: HttpRemoteSpace, parent: Element) {
     this.editorCommands = new Map();
@@ -98,18 +100,13 @@ export class Editor implements AppEventDispatcher {
     this.viewDispatch = () => {};
     this.render(parent);
     this.editorView = new EditorView({
-      state: this.createEditorState(
+      state: this.createEditorState("", new Document(Text.of([""]), 0)),
-        new Document(Text.of([""]), {
-          name: "",
-          lastModified: new Date(),
-          version: 0,
-        })
-      ),
       parent: document.getElementById("editor")!,
     });
     this.pageNavigator = new PathPageNavigator();
     this.indexer = new Indexer("page-index", space);
-    // this.watch();
+
+    this.indexCurrentPageDebounced = debounce(this.indexCurrentPage, 2000);
   }
 
   async init() {
@@ -118,16 +115,38 @@ export class Editor implements AppEventDispatcher {
     this.focus();
 
     this.pageNavigator.subscribe(async (pageName) => {
-      await this.save();
       console.log("Now navigating to", pageName);
 
       if (!this.editorView) {
         return;
       }
 
+      if (this.currentPage) {
+        let pageState = this.openPages.get(this.currentPage)!;
+        pageState.selection = this.editorView!.state.selection;
+        pageState.scrollTop = this.editorView!.scrollDOM.scrollTop;
+
+        this.space.closePage(this.currentPage);
+      }
+
       await this.loadPage(pageName);
     });
 
+    this.space.addEventListener("connect", () => {
+      if (this.currentPage) {
+        console.log("Connected to socket, fetch fresh?");
+        this.reloadPage();
+      }
+    });
+
+    this.space.addEventListener("reload", (e) => {
+      let pageName = (e as CustomEvent).detail;
+      if (this.currentPage === pageName) {
+        console.log("Was told to reload the page");
+        this.reloadPage();
+      }
+    });
+
     if (this.pageNavigator.getCurrentPage() === "") {
       this.pageNavigator.navigate("start");
     }
@@ -182,11 +201,11 @@ export class Editor implements AppEventDispatcher {
     return results;
   }
 
-  get currentPage(): PageMeta | undefined {
+  get currentPage(): string | undefined {
     return this.viewState.currentPage;
   }
 
-  createEditorState(doc: Document): EditorState {
+  createEditorState(pageName: string, doc: Document): EditorState {
     const editor = this;
     let commandKeyBindings: KeyBinding[] = [];
     for (let def of this.editorCommands.values()) {
@@ -217,8 +236,9 @@ export class Editor implements AppEventDispatcher {
         bracketMatching(),
         closeBrackets(),
         collabExtension(
-          doc.meta.name,
+          pageName,
-          doc.meta.version!,
+          this.space.socket.id,
+          doc.version,
           this.space,
           this.reloadPage.bind(this)
         ),
@@ -277,14 +297,6 @@ export class Editor implements AppEventDispatcher {
               return true;
             },
           },
-          {
-            key: "Ctrl-s",
-            mac: "Cmd-s",
-            run: (target): boolean => {
-              this.save();
-              return true;
-            },
-          },
           {
             key: "Ctrl-.",
             mac: "Cmd-.",
@@ -310,17 +322,6 @@ export class Editor implements AppEventDispatcher {
               await this.dispatchAppEvent("page:click", clickEvent);
             });
           },
-          // focus: (event: FocusEvent, view: EditorView) => {
-          //   console.log("Got focus");
-          //   document.body.classList.add("keyboard");
-          // },
-          // blur: (event: FocusEvent, view: EditorView) => {
-          //   console.log("Lost focus");
-          //   document.body.classList.remove("keyboard");
-          // },
-          // focusout: (event: FocusEvent, view: EditorView) => {
-          //   window.scrollTo(0, 0);
-          // },
         }),
         markdown({
           base: customMarkDown,
@@ -333,7 +334,10 @@ export class Editor implements AppEventDispatcher {
     });
   }
 
-  reloadPage() {}
+  reloadPage() {
+    console.log("Reloading page");
+    this.loadPage(this.currentPage!);
+  }
 
   async plugCompleter(
     ctx: CompletionContext
@@ -385,51 +389,22 @@ export class Editor implements AppEventDispatcher {
 
   update(value: null, transaction: Transaction): null {
     if (transaction.docChanged) {
-      this.viewDispatch({
+      this.indexCurrentPageDebounced();
-        type: "page-updated",
-      });
     }
 
     return null;
   }
 
-  async save() {
+  private async indexCurrentPage() {
-    const editorState = this.editorView!.state;
+    if (this.currentPage) {
-
+      console.log("Indexing page", this.currentPage);
-    if (!this.currentPage) {
+      await this.indexer.indexPage(
-      return;
+        this,
+        this.currentPage,
+        this.editorView!.state.sliceDoc(),
+        true
+      );
     }
-
-    if (this.viewState.isSaved) {
-      console.log("Page not modified, skipping saving");
-      return;
-    }
-    // Write to the space
-    const pageName = this.currentPage.name;
-    const text = editorState.sliceDoc();
-    let pageMeta = await this.space.writePage(pageName, text);
-
-    // Update in open page cache
-    this.openPages.set(
-      pageName,
-      new PageState(editorState, this.editorView!.scrollDOM.scrollTop, pageMeta)
-    );
-
-    // Dispatch update to view
-    this.viewDispatch({ type: "page-saved", meta: pageMeta });
-
-    // If a new page was created, let's refresh the page list
-    if (pageMeta.created) {
-      await this.loadPageList();
-    }
-
-    // Reindex page
-    await this.indexPage(text, pageMeta);
-  }
-
-  private async indexPage(text: string, pageMeta: PageMeta) {
-    console.log("Indexing page", pageMeta.name);
-    this.indexer.indexPage(this, pageMeta, text, true);
   }
 
   async loadPageList() {
@@ -440,32 +415,6 @@ export class Editor implements AppEventDispatcher {
     });
   }
 
-  watch() {
-    setInterval(() => {
-      safeRun(async () => {
-        if (this.currentPage && this.viewState.isSaved) {
-          await this.checkForNewVersion(this.currentPage);
-        }
-      });
-    }, watchInterval);
-  }
-
-  async checkForNewVersion(cachedMeta: PageMeta) {
-    const currentPageName = cachedMeta.name;
-    let newPageMeta = await this.space.getPageMeta(currentPageName);
-    if (
-      cachedMeta.lastModified.getTime() !== newPageMeta.lastModified.getTime()
-    ) {
-      console.log("File changed on disk, reloading");
-      let doc = await this.space.openPage(currentPageName);
-      this.openPages.set(
-        currentPageName,
-        new PageState(this.createEditorState(doc), 0, doc.meta)
-      );
-      await this.loadPage(currentPageName, false);
-    }
-  }
-
   focus() {
     this.editorView!.focus();
   }
@@ -474,39 +423,41 @@ export class Editor implements AppEventDispatcher {
     this.pageNavigator.navigate(name);
   }
 
-  async loadPage(pageName: string, checkNewVersion: boolean = true) {
+  async loadPage(pageName: string) {
+    let doc = await this.space.openPage(pageName);
+    let editorState = this.createEditorState(pageName, doc);
     let pageState = this.openPages.get(pageName);
-    if (!pageState) {
+    const editorView = this.editorView;
-      let doc = await this.space.openPage(pageName);
+    if (!editorView) {
-      pageState = new PageState(this.createEditorState(doc), 0, doc.meta);
+      return;
-      this.openPages.set(pageName, pageState!);
+    }
-      // Freshly loaded, no need to check for a new version either way
+    editorView.setState(editorState);
-      checkNewVersion = false;
+    if (!pageState) {
+      pageState = new PageState(0, editorState.selection);
+      this.openPages.set(pageName, pageState!);
+    } else {
+      // Restore state
+      console.log("Restoring selection state");
+      editorView.dispatch({
+        selection: pageState.selection,
+      });
+      editorView.scrollDOM.scrollTop = pageState!.scrollTop;
     }
-    this.editorView!.setState(pageState!.editorState);
-    this.editorView!.scrollDOM.scrollTop = pageState!.scrollTop;
 
     this.viewDispatch({
       type: "page-loaded",
-      meta: pageState.meta,
+      name: pageName,
     });
 
-    let indexerPageMeta = await this.indexer.getPageIndexPageMeta(pageName);
+    // let indexerPageMeta = await this.indexer.getPageIndexPageMeta(pageName);
-    if (
+    // if (
-      (indexerPageMeta &&
+    //   (indexerPageMeta &&
-        pageState.meta.lastModified.getTime() !==
+    //     doc.meta.lastModified.getTime() !==
-          indexerPageMeta.lastModified.getTime()) ||
+    //       indexerPageMeta.lastModified.getTime()) ||
-      !indexerPageMeta
+    //   !indexerPageMeta
-    ) {
+    // ) {
-      await this.indexPage(pageState.editorState.sliceDoc(), pageState.meta);
+    await this.indexCurrentPage();
-    }
+    // }
-
-    if (checkNewVersion) {
-      // Loaded page from in-memory cache, let's async see if this page hasn't been updated
-      this.checkForNewVersion(pageState.meta).catch((e) => {
-        console.error("Failed to check for new version");
-      });
-    }
   }
 
   ViewComponent(): React.ReactElement {
@@ -514,23 +465,11 @@ export class Editor implements AppEventDispatcher {
     this.viewState = viewState;
     this.viewDispatch = dispatch;
 
-    // Auto save
-    useEffect(() => {
-      const id = setTimeout(() => {
-        if (!viewState.isSaved) {
-          this.save();
-        }
-      }, 2000);
-      return () => {
-        clearTimeout(id);
-      };
-    }, [viewState.isSaved]);
-
     let editor = this;
 
     useEffect(() => {
       if (viewState.currentPage) {
-        document.title = viewState.currentPage.name;
+        document.title = viewState.currentPage;
       }
     }, [viewState.currentPage]);
 
@@ -573,7 +512,7 @@ export class Editor implements AppEventDispatcher {
           }}
         />
         <div id="editor"></div>
-        <StatusBar isSaved={viewState.isSaved} editorView={this.editorView} />
+        <StatusBar editorView={this.editorView} />
       </>
     );
   }

webapp/src/indexer.ts

@@ -38,39 +38,39 @@ export class Indexer {
     await this.pageIndex.clear();
   }
 
-  async setPageIndexPageMeta(pageName: string, meta: PageMeta) {
+  // async setPageIndexPageMeta(pageName: string, meta: PageMeta) {
-    await this.set(pageName, "$meta", {
+  //   await this.set(pageName, "$meta", {
-      lastModified: meta.lastModified.getTime(),
+  //     lastModified: meta.lastModified.getTime(),
-    });
+  //   });
-  }
+  // }
 
-  async getPageIndexPageMeta(pageName: string): Promise<PageMeta | null> {
+  // async getPageIndexPageMeta(pageName: string): Promise<PageMeta | null> {
-    let meta = await this.get(pageName, "$meta");
+  //   let meta = await this.get(pageName, "$meta");
-    if (meta) {
+  //   if (meta) {
-      return {
+  //     return {
-        name: pageName,
+  //       name: pageName,
-        lastModified: new Date(meta.lastModified),
+  //       lastModified: new Date(meta.lastModified),
-      };
+  //     };
-    } else {
+  //   } else {
-      return null;
+  //     return null;
-    }
+  //   }
-  }
+  // }
 
   async indexPage(
     appEventDispatcher: AppEventDispatcher,
-    pageMeta: PageMeta,
+    pageName: string,
     text: string,
     withFlush: boolean
   ) {
     if (withFlush) {
-      await this.clearPageIndexForPage(pageMeta.name);
+      await this.clearPageIndexForPage(pageName);
     }
     let indexEvent: IndexEvent = {
-      name: pageMeta.name,
+      name: pageName,
       text,
     };
     await appEventDispatcher.dispatchAppEvent("page:index", indexEvent);
-    await this.setPageIndexPageMeta(pageMeta.name, pageMeta);
+    // await this.setPageIndexPageMeta(pageMeta.name, pageMeta);
   }
 
   async reindexSpace(space: Space, appEventDispatcher: AppEventDispatcher) {
@@ -79,12 +79,7 @@ export class Indexer {
     // TODO: Parallelize?
     for (let page of allPages) {
       let pageData = await space.readPage(page.name);
-      await this.indexPage(
+      await this.indexPage(appEventDispatcher, page.name, pageData.text, false);
-        appEventDispatcher,
-        pageData.meta,
-        pageData.text,
-        false
-      );
     }
   }
 

webapp/src/reducer.ts

@@ -10,27 +10,11 @@ export default function reducer(
       return {
         ...state,
         allPages: state.allPages.map((pageMeta) =>
-          pageMeta.name === action.meta.name
+          pageMeta.name === action.name
             ? { ...pageMeta, lastOpened: new Date() }
             : pageMeta
         ),
-        currentPage: action.meta,
+        currentPage: action.name,
-        isSaved: true,
-      };
-    case "page-saved":
-      return {
-        ...state,
-        currentPage: action.meta,
-        isSaved: true,
-      };
-    case "page-updated":
-      // Minor rerender optimization, this is triggered a lot
-      if (!state.isSaved) {
-        return state;
-      }
-      return {
-        ...state,
-        isSaved: false,
       };
     case "start-navigate":
       return {

webapp/src/space.ts

@@ -4,6 +4,7 @@ import { Update } from "@codemirror/collab";
 import { Transaction, Text, ChangeSet } from "@codemirror/state";
 
 import { Document } from "./collab";
+import { cursorEffect } from "./cursorEffect";
 
 export interface Space {
   listPages(): Promise<PageMeta[]>;
@@ -13,38 +14,48 @@ export interface Space {
   getPageMeta(name: string): Promise<PageMeta>;
 }
 
-export class HttpRemoteSpace implements Space {
+export class HttpRemoteSpace extends EventTarget implements Space {
   url: string;
   socket: Socket;
   reqId = 0;
 
   constructor(url: string, socket: Socket) {
+    super();
     this.url = url;
     this.socket = socket;
 
     socket.on("connect", () => {
-      console.log("connected via SocketIO");
+      console.log("connected to socket");
+      this.dispatchEvent(new Event("connect"));
+    });
+
+    socket.on("reload", (pageName: string) => {
+      this.dispatchEvent(new CustomEvent("reload", { detail: pageName }));
     });
   }
 
-  pushUpdates(
+  private wsCall(eventName: string, ...args: any[]): Promise<any> {
+    return new Promise((resolve) => {
+      this.reqId++;
+      this.socket!.once(`${eventName}Resp${this.reqId}`, resolve);
+      this.socket!.emit(eventName, this.reqId, ...args);
+    });
+  }
+
+  async pushUpdates(
     pageName: string,
     version: number,
     fullUpdates: readonly (Update & { origin: Transaction })[]
   ): Promise<boolean> {
-    return new Promise((resolve) => {
+    if (this.socket) {
-      if (this.socket) {
+      let updates = fullUpdates.map((u) => ({
-        let updates = fullUpdates.map((u) => ({
+        clientID: u.clientID,
-          clientID: u.clientID,
+        changes: u.changes.toJSON(),
-          changes: u.changes.toJSON(),
+        cursors: u.effects?.map((e) => e.value),
-        }));
+      }));
-        this.reqId++;
+      return this.wsCall("pushUpdates", pageName, version, updates);
-        this.socket.emit("pushUpdates", this.reqId, pageName, version, updates);
+    }
-        this.socket.once("pushUpdatesResp" + this.reqId, (result) => {
+    return false;
-          resolve(result);
-        });
-      }
-    });
   }
 
   async pullUpdates(
@@ -52,11 +63,13 @@ export class HttpRemoteSpace implements Space {
     version: number
   ): Promise<readonly Update[]> {
     let updates: Update[] = await this.wsCall("pullUpdates", pageName, version);
-    console.log("Got updates", updates);
+    let ups = updates.map((u) => ({
-    return updates.map((u) => ({
       changes: ChangeSet.fromJSON(u.changes),
+      effects: u.effects?.map((e) => cursorEffect.of(e.value)),
       clientID: u.clientID,
     }));
+    console.log("Got updates", ups);
+    return ups;
   }
 
   async listPages(): Promise<PageMeta[]> {
@@ -70,20 +83,10 @@ export class HttpRemoteSpace implements Space {
     }));
   }
 
-  wsCall(eventName: string, ...args: any[]): Promise<any> {
-    return new Promise((resolve) => {
-      this.reqId++;
-      this.socket!.once(`${eventName}Resp${this.reqId}`, resolve);
-      this.socket!.emit(eventName, this.reqId, ...args);
-    });
-  }
-
   async openPage(name: string): Promise<Document> {
     this.reqId++;
-    let [meta, text] = await this.wsCall("openPage", name);
+    let [version, text] = await this.wsCall("openPage", name);
-    console.log("Got this", meta, text);
+    return new Document(Text.of(text), version);
-    meta.lastModified = new Date(meta.lastModified);
-    return new Document(Text.of(text), meta);
   }
 
   async closePage(name: string): Promise<void> {

webapp/src/styles/editor.scss

@@ -9,6 +9,13 @@
     max-width: 800px;
   }
 
+  .other-cursor {
+    display: inline-block;
+    width: 1px;
+    margin-right: -1px;
+    height: 1em;
+  }
+
   .cm-selectionBackground {
     background-color: #d7e1f6 !important;
   }

webapp/src/syscalls/editor.browser.ts

@@ -27,7 +27,7 @@ function ensureAnchor(expr: any, start: boolean) {
 }
 
 export default (editor: Editor) => ({
-  "editor.getCurrentPage": (): PageMeta => {
+  "editor.getCurrentPage": (): string => {
     return editor.currentPage!;
   },
   "editor.getText": () => {

webapp/src/syscalls/space.native.ts

@@ -23,7 +23,7 @@ export default (editor: Editor) => ({
     console.log("Clearing page index", name);
     await editor.indexer.clearPageIndexForPage(name);
     // If we're deleting the current page, navigate to the start page
-    if (editor.currentPage?.name === name) {
+    if (editor.currentPage === name) {
       await editor.navigate("start");
     }
     // Remove page from open pages in editor

webapp/src/types.ts

@@ -37,8 +37,7 @@ export interface CommandDef {
 }
 
 export type AppViewState = {
-  currentPage?: PageMeta;
+  currentPage?: string;
-  isSaved: boolean;
   showPageNavigator: boolean;
   showCommandPalette: boolean;
   allPages: PageMeta[];
@@ -46,7 +45,6 @@ export type AppViewState = {
 };
 
 export const initialViewState: AppViewState = {
-  isSaved: false,
   showPageNavigator: false,
   showCommandPalette: false,
   allPages: [],
@@ -54,9 +52,7 @@ export const initialViewState: AppViewState = {
 };
 
 export type Action =
-  | { type: "page-loaded"; meta: PageMeta }
+  | { type: "page-loaded"; name: string }
-  | { type: "page-saved"; meta: PageMeta }
-  | { type: "page-updated" }
   | { type: "pages-listed"; pages: PageMeta[] }
   | { type: "start-navigate" }
   | { type: "stop-navigate" }

webapp/yarn.lock

@@ -979,6 +979,11 @@
   resolved "https://registry.yarnpkg.com/@types/events/-/events-3.0.0.tgz#2862f3f58a9a7f7c3e78d79f130dd4d71c25c2a7"
   integrity sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g==
 
+"@types/lodash@^4.14.179":
+  version "4.14.179"
+  resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.179.tgz#490ec3288088c91295780237d2497a3aa9dfb5c5"
+  integrity sha512-uwc1x90yCKqGcIOAT6DwOSuxnrAbpkdPsUOZtwrXb4D/6wZs+6qG7QnIawDuZWg0sWpxl+ltIKCaLoMlna678w==
+
 "@types/node@^17.0.21":
   version "17.0.21"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.21.tgz#864b987c0c68d07b4345845c3e63b75edd143644"
@@ -2051,6 +2056,11 @@ lodash.uniq@^4.5.0:
   resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773"
   integrity sha1-0CJTc662Uq3BvILklFM5qEJ1R3M=
 
+lodash@^4.17.21:
+  version "4.17.21"
+  resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
+  integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
+
 loose-envify@^1.1.0, loose-envify@^1.4.0:
   version "1.4.0"
   resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf"