Big refactors and fixes

* Query regen * Fix anchor completion * Dependency fixes * Changelog update
2023-07-02 11:25:32 +02:00
parent fee2c5928e
commit 7c825348b2
74 changed files with 935 additions and 567 deletions
@@ -6,12 +6,7 @@ Deno.test("Collab server", async () => {
  console.log("Client 1 joins page 1");
  assertEquals(collabServer.updatePresence("client1", "page1"), {});
  assertEquals(collabServer.pages.size, 1);
-  console.log("CLient 1 leaves page 1");
-  assertEquals(collabServer.updatePresence("client1", undefined, "page1"), {});
-  assertEquals(collabServer.pages.size, 0);
-  assertEquals(collabServer.updatePresence("client1", "page1"), {});
-  console.log("Client 1 joins page 2");
-  assertEquals(collabServer.updatePresence("client1", "page2", "page1"), {});
+  assertEquals(collabServer.updatePresence("client1", "page2"), {});
  assertEquals(collabServer.pages.size, 1);
  console.log("Client 2 joins to page 2, collab id created, but not exposed");
  assertEquals(
@@ -22,22 +17,23 @@ Deno.test("Collab server", async () => {
    collabServer.updatePresence("client1", "page2").collabId !== undefined,
  );
  console.log("Client 2 moves to page 1, collab id destroyed");
-  assertEquals(collabServer.updatePresence("client2", "page1", "page2"), {});
-  assertEquals(collabServer.updatePresence("client1", "page2", "page2"), {});
+  assertEquals(collabServer.updatePresence("client2", "page1"), {});
+  assertEquals(collabServer.updatePresence("client1", "page2"), {});
  assertEquals(collabServer.pages.get("page2")!.collabId, undefined);
  assertEquals(collabServer.pages.get("page1")!.collabId, undefined);
  console.log("Going to cleanup, which should have no effect");
  collabServer.cleanup(50);
  assertEquals(collabServer.pages.size, 2);
-  collabServer.updatePresence("client2", "page2", "page1");
+  collabServer.updatePresence("client2", "page2");
  console.log("Going to sleep 20ms");
  await sleep(20);
  console.log("Then client 1 pings, but client 2 does not");
-  collabServer.updatePresence("client1", "page2", "page2");
+  collabServer.updatePresence("client1", "page2");
  await sleep(20);
  console.log("Going to cleanup, which should clean client 2");
  collabServer.cleanup(35);
  assertEquals(collabServer.pages.get("page2")!.collabId, undefined);
+  assertEquals(collabServer.clients.size, 1);
  console.log(collabServer);
 });

@@ -14,7 +14,7 @@ type CollabPage = {
 };

 export class CollabServer {
-  // clients: Map<string, { openPage: string; lastPing: number }> = new Map();
+  clients: Map<string, { openPage: string; lastUpdate: number }> = new Map(); // clientId -> openPage
  pages: Map<string, CollabPage> = new Map();
  yCollabServer?: Hocuspocus;

@@ -29,13 +29,18 @@ export class CollabServer {

  updatePresence(
    clientId: string,
-    currentPage?: string,
-    previousPage?: string,
+    currentPage: string,
  ): { collabId?: string } {
-    if (previousPage && currentPage !== previousPage) {
+    let client = this.clients.get(clientId);
+    if (!client) {
+      client = { openPage: "", lastUpdate: 0 };
+      this.clients.set(clientId, client);
+    }
+    client.lastUpdate = Date.now();
+    if (currentPage !== client.openPage) {
      // Client switched pages
      // Update last page record
-      const lastCollabPage = this.pages.get(previousPage);
+      const lastCollabPage = this.pages.get(client.openPage);
      if (lastCollabPage) {
        lastCollabPage.clients.delete(clientId);
        if (lastCollabPage.clients.size === 1) {
@@ -43,7 +48,7 @@ export class CollabServer {
        }

        if (lastCollabPage.clients.size === 0) {
-          this.pages.delete(previousPage);
+          this.pages.delete(client.openPage);
        } else {
          // Elect a new master client
          if (lastCollabPage.masterClientId === clientId) {
@@ -53,45 +58,43 @@ export class CollabServer {
          }
        }
      }
+      // Ok, let's update our records now
+      client.openPage = currentPage;
    }

-    if (currentPage) {
-      // Update new page
-      let nextCollabPage = this.pages.get(currentPage);
-      if (!nextCollabPage) {
-        // Newly opened page (no other clients on this page right now)
-        nextCollabPage = {
-          clients: new Map(),
-          masterClientId: clientId,
-        };
-        this.pages.set(currentPage, nextCollabPage);
-      }
-      // Register last ping from us
-      nextCollabPage.clients.set(clientId, Date.now());
+    // Update new page
+    let nextCollabPage = this.pages.get(currentPage);
+    if (!nextCollabPage) {
+      // Newly opened page (no other clients on this page right now)
+      nextCollabPage = {
+        clients: new Map(),
+        masterClientId: clientId,
+      };
+      this.pages.set(currentPage, nextCollabPage);
+    }
+    // Register last ping from us
+    nextCollabPage.clients.set(clientId, Date.now());

-      if (nextCollabPage.clients.size > 1 && !nextCollabPage.collabId) {
-        // Create a new collabId
-        nextCollabPage.collabId = nanoid();
-      }
-      // console.log("State", this.pages);
-      if (nextCollabPage.collabId) {
-        // We will now expose this collabId, except when we're just starting this session
-        // in which case we'll wait for the original client to publish the document
-        const existingyCollabSession = this.yCollabServer?.documents.get(
-          buildCollabId(nextCollabPage.collabId, `${currentPage}.md`),
-        );
-        if (existingyCollabSession) {
-          // console.log("Found an existing collab session already, let's join!");
-          return { collabId: nextCollabPage.collabId };
-        } else if (clientId === nextCollabPage.masterClientId) {
-          // console.log("We're the master, so we should connect");
-          return { collabId: nextCollabPage.collabId };
-        } else {
-          // We're not the first client, so we need to wait for the first client to connect
-          // console.log("We're not the master, so we should wait");
-          return {};
-        }
+    if (nextCollabPage.clients.size > 1 && !nextCollabPage.collabId) {
+      // Create a new collabId
+      nextCollabPage.collabId = nanoid();
+    }
+    // console.log("State", this.pages);
+    if (nextCollabPage.collabId) {
+      // We will now expose this collabId, except when we're just starting this session
+      // in which case we'll wait for the original client to publish the document
+      const existingyCollabSession = this.yCollabServer?.documents.get(
+        buildCollabId(nextCollabPage.collabId, `${currentPage}.md`),
+      );
+      if (existingyCollabSession) {
+        // console.log("Found an existing collab session already, let's join!");
+        return { collabId: nextCollabPage.collabId };
+      } else if (clientId === nextCollabPage.masterClientId) {
+        // console.log("We're the master, so we should connect");
+        return { collabId: nextCollabPage.collabId };
      } else {
+        // We're not the first client, so we need to wait for the first client to connect
+        // console.log("We're not the master, so we should wait");
        return {};
      }
    } else {
@@ -121,6 +124,13 @@ export class CollabServer {
        this.pages.delete(pageName);
      }
    }
+
+    for (const [clientId, { lastUpdate }] of this.clients) {
+      if (Date.now() - lastUpdate > timeout) {
+        // Eject client
+        this.clients.delete(clientId);
+      }
+    }
  }

  route(app: Application) {
@@ -9,6 +9,7 @@ import { gitIgnoreCompiler } from "./deps.ts";
 import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts";
 import { CollabServer } from "./collab.ts";
 import { Authenticator } from "./auth.ts";
+import { oakCors } from "https://deno.land/x/cors@v1.2.2/mod.ts";

 export type ServerOptions = {
  hostname: string;
@@ -183,6 +184,49 @@ export class HttpServer {
      "/logo.png",
      "/.auth",
    ];
+
+    app.use(async ({ request, response, cookies }, next) => {
+      if (request.url.pathname === "/.auth") {
+        if (request.url.search === "?logout") {
+          await cookies.delete("auth");
+          // Implicit fallthrough to login page
+        }
+        if (request.method === "GET") {
+          response.headers.set("Content-type", "text/html");
+          response.body = this.clientAssetBundle.readTextFileSync(
+            ".client/auth.html",
+          );
+          return;
+        } else if (request.method === "POST") {
+          const values = await request.body({ type: "form" }).value;
+          const username = values.get("username")!,
+            password = values.get("password")!,
+            refer = values.get("refer");
+          const hashedPassword = await this.authenticator.authenticate(
+            username,
+            password,
+          );
+          if (hashedPassword) {
+            await cookies.set("auth", `${username}:${hashedPassword}`, {
+              expires: new Date(Date.now() + 1000 * 60 * 60 * 24 * 7), // in a week
+              sameSite: "strict",
+            });
+            response.redirect(refer || "/");
+            // console.log("All headers", request.headers);
+          } else {
+            response.redirect("/.auth?error=1");
+          }
+          return;
+        } else {
+          response.status = 401;
+          response.body = "Unauthorized";
+          return;
+        }
+      } else {
+        await next();
+      }
+    });
+
    if ((await this.authenticator.getAllUsers()).length > 0) {
      app.use(async ({ request, response, cookies }, next) => {
        if (!excludedPaths.includes(request.url.pathname)) {
@@ -204,55 +248,20 @@ export class HttpServer {
            return;
          }
        }
-
-        if (request.url.pathname === "/.auth") {
-          if (request.url.search === "?logout") {
-            await cookies.delete("auth");
-            // Implicit fallthrough to login page
-          }
-          if (request.method === "GET") {
-            response.headers.set("Content-type", "text/html");
-            response.body = this.clientAssetBundle.readTextFileSync(
-              ".client/auth.html",
-            );
-            return;
-          } else if (request.method === "POST") {
-            const values = await request.body({ type: "form" }).value;
-            const username = values.get("username")!,
-              password = values.get("password")!,
-              refer = values.get("refer");
-            const hashedPassword = await this.authenticator.authenticate(
-              username,
-              password,
-            );
-            if (hashedPassword) {
-              await cookies.set("auth", `${username}:${hashedPassword}`, {
-                expires: new Date(Date.now() + 1000 * 60 * 60 * 24 * 7), // in a week
-                sameSite: "strict",
-              });
-              response.redirect(refer || "/");
-              // console.log("All headers", request.headers);
-            } else {
-              response.redirect("/.auth?error=1");
-            }
-            return;
-          } else {
-            response.status = 401;
-            response.body = "Unauthorized";
-            return;
-          }
-        } else {
-          // Unauthenticated access to excluded paths
-          await next();
-        }
+        await next();
      });
    }
  }

  private buildFsRouter(spacePrimitives: SpacePrimitives): Router {
    const fsRouter = new Router();
+    const corsMiddleware = oakCors({
+      allowedHeaders: "*",
+      exposedHeaders: "*",
+      methods: ["GET", "POST", "PUT", "DELETE", "OPTIONS"],
+    });
    // File list
-    fsRouter.get("/", async ({ response }) => {
+    fsRouter.get("/", corsMiddleware, async ({ response }) => {
      response.headers.set("Content-type", "application/json");
      response.headers.set("X-Space-Path", this.options.pagesPath);
      const files = await spacePrimitives.fetchFileList();
@@ -260,7 +269,7 @@ export class HttpServer {
    });

    // RPC
-    fsRouter.post("/", async ({ request, response }) => {
+    fsRouter.post("/", corsMiddleware, async ({ request, response }) => {
      const body = await request.body({ type: "json" }).value;
      try {
        switch (body.operation) {
@@ -300,19 +309,6 @@ export class HttpServer {
            });
            return;
          }
-          case "presence": {
-            // RPC to check (for collab purposes) which client has what page open
-            response.headers.set("Content-Type", "application/json");
-            console.log("Got presence update", body);
-            response.body = JSON.stringify(
-              this.collab.updatePresence(
-                body.clientId,
-                body.currentPage,
-                body.previousPage,
-              ),
-            );
-            return;
-          }
          default:
            response.headers.set("Content-Type", "text/plain");
            response.status = 400;
@@ -327,7 +323,7 @@ export class HttpServer {
    });

    fsRouter
-      .get("\/(.+)", async ({ params, response, request }) => {
+      .get("\/(.+)", corsMiddleware, async ({ params, response, request }) => {
        const name = params[0];
        console.log("Loading file", name);
        if (name.startsWith(".")) {
@@ -364,7 +360,7 @@ export class HttpServer {
          response.body = "";
        }
      })
-      .put("\/(.+)", async ({ request, response, params }) => {
+      .put("\/(.+)", corsMiddleware, async ({ request, response, params }) => {
        const name = params[0];
        console.log("Saving file", name);
        if (name.startsWith(".")) {
@@ -400,8 +396,16 @@ export class HttpServer {
          console.error("Pipeline failed", err);
        }
      })
-      .options("\/(.+)", async ({ response, params }) => {
+      .options("\/(.+)", async ({ request, response, params }) => {
        const name = params[0];
+        // Manually set CORS headers
+        response.headers.set("access-control-allow-headers", "*");
+        response.headers.set(
+          "access-control-allow-methods",
+          "GET,POST,PUT,DELETE,OPTIONS",
+        );
+        response.headers.set("access-control-allow-origin", "*");
+        response.headers.set("access-control-expose-headers", "*");
        try {
          const meta = await spacePrimitives.getFileMeta(name);
          response.status = 200;
@@ -409,13 +413,25 @@ export class HttpServer {
          response.headers.set("X-Last-Modified", "" + meta.lastModified);
          response.headers.set("X-Content-Length", "" + meta.size);
          response.headers.set("X-Permission", meta.perm);
+
+          const clientId = request.headers.get("X-Client-Id");
+          if (name.endsWith(".md") && clientId) {
+            const pageName = name.substring(0, name.length - ".md".length);
+            console.log(`Got presence update from ${clientId}: ${pageName}`);
+            const { collabId } = this.collab.updatePresence(clientId, pageName);
+            if (collabId) {
+              response.headers.set("X-Collab-Id", collabId);
+            }
+          }
        } catch {
-          response.status = 404;
+          // Have to do this because of CORS
+          response.status = 200;
+          response.headers.set("X-Status", "404");
          response.body = "Not found";
          // console.error("Options failed", err);
        }
      })
-      .delete("\/(.+)", async ({ response, params }) => {
+      .delete("\/(.+)", corsMiddleware, async ({ response, params }) => {
        const name = params[0];
        console.log("Deleting file", name);
        try {