Initial pass of supporting attachments #71

packages/common/spaces/disk_space_primitives.ts

@@ -8,10 +8,11 @@ import {
   writeFile,
 } from "fs/promises";
 import * as path from "path";
-import { PageMeta } from "../types";
+import { AttachmentMeta, PageMeta } from "../types";
 import { SpacePrimitives } from "./space_primitives";
 import { Plug } from "@plugos/plugos/plug";
 import { realpathSync } from "fs";
+import mime from "mime-types";
 
 export class DiskSpacePrimitives implements SpacePrimitives {
   rootPath: string;
@@ -47,6 +48,7 @@ export class DiskSpacePrimitives implements SpacePrimitives {
     );
   }
 
+  // Pages
   async readPage(pageName: string): Promise<{ text: string; meta: PageMeta }> {
     const localPath = this.pageNameToPath(pageName);
     try {
@@ -148,6 +150,136 @@ export class DiskSpacePrimitives implements SpacePrimitives {
     };
   }
 
+  // Attachments
+  attachmentNameToPath(name: string) {
+    return this.safePath(path.join(this.rootPath, name));
+  }
+
+  pathToAttachmentName(fullPath: string): string {
+    return fullPath.substring(this.rootPath.length + 1);
+  }
+
+  async fetchAttachmentList(): Promise<{
+    attachments: Set<AttachmentMeta>;
+    nowTimestamp: number;
+  }> {
+    let attachments = new Set<AttachmentMeta>();
+
+    const walkPath = async (dir: string) => {
+      let files = await readdir(dir);
+      for (let file of files) {
+        const fullPath = path.join(dir, file);
+        let s = await stat(fullPath);
+        if (s.isDirectory()) {
+          if (!file.startsWith(".")) {
+            await walkPath(fullPath);
+          }
+        } else {
+          if (
+            !file.startsWith(".") &&
+            !file.endsWith(".md") &&
+            !file.endsWith(".json")
+          ) {
+            attachments.add({
+              name: this.pathToAttachmentName(fullPath),
+              lastModified: s.mtime.getTime(),
+              size: s.size,
+              contentType: mime.lookup(file) || "application/octet-stream",
+              perm: "rw",
+            } as AttachmentMeta);
+          }
+        }
+      }
+    };
+    await walkPath(this.rootPath);
+    return {
+      attachments,
+      nowTimestamp: Date.now(),
+    };
+  }
+
+  async readAttachment(
+    name: string
+  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> {
+    const localPath = this.attachmentNameToPath(name);
+    let fileBuffer = await readFile(localPath);
+
+    try {
+      const s = await stat(localPath);
+      return {
+        buffer: fileBuffer.buffer,
+        meta: {
+          name: name,
+          lastModified: s.mtime.getTime(),
+          size: s.size,
+          contentType: mime.lookup(name) || "application/octet-stream",
+          perm: "rw",
+        },
+      };
+    } catch (e) {
+      // console.error("Error while reading attachment", name, e);
+      throw Error(`Could not read attachment ${name}`);
+    }
+  }
+
+  async getAttachmentMeta(name: string): Promise<AttachmentMeta> {
+    const localPath = this.attachmentNameToPath(name);
+    try {
+      const s = await stat(localPath);
+      return {
+        name: name,
+        lastModified: s.mtime.getTime(),
+        size: s.size,
+        contentType: mime.lookup(name) || "application/octet-stream",
+        perm: "rw",
+      };
+    } catch (e) {
+      // console.error("Error while getting attachment meta", name, e);
+      throw Error(`Could not get meta for ${name}`);
+    }
+  }
+
+  async writeAttachment(
+    name: string,
+    blob: ArrayBuffer,
+    selfUpdate?: boolean,
+    lastModified?: number
+  ): Promise<AttachmentMeta> {
+    let localPath = this.attachmentNameToPath(name);
+    try {
+      // Ensure parent folder exists
+      await mkdir(path.dirname(localPath), { recursive: true });
+
+      // Actually write the file
+      await writeFile(localPath, Buffer.from(blob));
+
+      if (lastModified) {
+        let d = new Date(lastModified);
+        console.log("Going to set the modified time", d);
+        await utimes(localPath, d, d);
+      }
+
+      // Fetch new metadata
+      const s = await stat(localPath);
+      return {
+        name: name,
+        lastModified: s.mtime.getTime(),
+        size: s.size,
+        contentType: mime.lookup(name) || "application/octet-stream",
+        perm: "rw",
+      };
+    } catch (e) {
+      console.error("Error while writing attachment", name, e);
+      throw Error(`Could not write ${name}`);
+    }
+  }
+
+  async deleteAttachment(name: string): Promise<void> {
+    let localPath = this.attachmentNameToPath(name);
+    await unlink(localPath);
+  }
+
+  // Plugs
   invokeFunction(
     plug: Plug<any>,
     env: string,

packages/common/spaces/evented_space_primitives.ts

@@ -1,7 +1,7 @@
 import { EventHook } from "@plugos/plugos/hooks/event";
 import { Plug } from "@plugos/plugos/plug";
 
-import { PageMeta } from "../types";
+import { AttachmentMeta, PageMeta } from "../types";
 import { plugPrefix, trashPrefix } from "./constants";
 import { SpacePrimitives } from "./space_primitives";
 
@@ -66,4 +66,42 @@ export class EventedSpacePrimitives implements SpacePrimitives {
     await this.eventHook.dispatchEvent("page:deleted", pageName);
     return this.wrapped.deletePage(pageName);
   }
+
+  fetchAttachmentList(): Promise<{
+    attachments: Set<AttachmentMeta>;
+    nowTimestamp: number;
+  }> {
+    return this.wrapped.fetchAttachmentList();
+  }
+
+  readAttachment(
+    name: string
+  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> {
+    return this.wrapped.readAttachment(name);
+  }
+
+  getAttachmentMeta(name: string): Promise<AttachmentMeta> {
+    return this.wrapped.getAttachmentMeta(name);
+  }
+
+  async writeAttachment(
+    name: string,
+    blob: ArrayBuffer,
+    selfUpdate?: boolean | undefined,
+    lastModified?: number | undefined
+  ): Promise<AttachmentMeta> {
+    let meta = await this.wrapped.writeAttachment(
+      name,
+      blob,
+      selfUpdate,
+      lastModified
+    );
+    await this.eventHook.dispatchEvent("attachment:saved", name);
+    return meta;
+  }
+
+  async deleteAttachment(name: string): Promise<void> {
+    await this.eventHook.dispatchEvent("attachment:deleted", name);
+    return this.wrapped.deleteAttachment(name);
+  }
 }

packages/common/spaces/http_space_primitives.ts

@@ -1,14 +1,16 @@
-import { PageMeta } from "../types";
+import { AttachmentMeta, PageMeta } from "../types";
 import { Plug } from "@plugos/plugos/plug";
 import { SpacePrimitives } from "./space_primitives";
 
 export class HttpSpacePrimitives implements SpacePrimitives {
-  pageUrl: string;
+  fsUrl: string;
+  fsaUrl: string;
   private plugUrl: string;
   token?: string;
 
   constructor(url: string, token?: string) {
-    this.pageUrl = url + "/fs";
+    this.fsUrl = url + "/page";
+    this.fsaUrl = url + "/attachment";
     this.plugUrl = url + "/plug";
     this.token = token;
   }
@@ -32,7 +34,7 @@ export class HttpSpacePrimitives implements SpacePrimitives {
     pages: Set<PageMeta>;
     nowTimestamp: number;
   }> {
-    let req = await this.authenticatedFetch(this.pageUrl, {
+    let req = await this.authenticatedFetch(this.fsUrl, {
       method: "GET",
     });
 
@@ -53,7 +55,7 @@ export class HttpSpacePrimitives implements SpacePrimitives {
   }
 
   async readPage(name: string): Promise<{ text: string; meta: PageMeta }> {
-    let res = await this.authenticatedFetch(`${this.pageUrl}/${name}`, {
+    let res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
       method: "GET",
     });
     if (res.headers.get("X-Status") === "404") {
@@ -61,7 +63,7 @@ export class HttpSpacePrimitives implements SpacePrimitives {
     }
     return {
       text: await res.text(),
-      meta: this.responseToMeta(name, res),
+      meta: this.responseToPageMeta(name, res),
     };
   }
 
@@ -72,7 +74,7 @@ export class HttpSpacePrimitives implements SpacePrimitives {
     lastModified?: number
   ): Promise<PageMeta> {
     // TODO: lastModified ignored for now
-    let res = await this.authenticatedFetch(`${this.pageUrl}/${name}`, {
+    let res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
       method: "PUT",
       body: text,
       headers: lastModified
@@ -81,12 +83,12 @@ export class HttpSpacePrimitives implements SpacePrimitives {
           }
         : undefined,
     });
-    const newMeta = this.responseToMeta(name, res);
+    const newMeta = this.responseToPageMeta(name, res);
     return newMeta;
   }
 
   async deletePage(name: string): Promise<void> {
-    let req = await this.authenticatedFetch(`${this.pageUrl}/${name}`, {
+    let req = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
       method: "DELETE",
     });
     if (req.status !== 200) {
@@ -115,6 +117,90 @@ export class HttpSpacePrimitives implements SpacePrimitives {
     return await req.json();
   }
 
+  // Attachments
+  public async fetchAttachmentList(): Promise<{
+    attachments: Set<AttachmentMeta>;
+    nowTimestamp: number;
+  }> {
+    let req = await this.authenticatedFetch(this.fsaUrl, {
+      method: "GET",
+    });
+
+    let result = new Set<AttachmentMeta>();
+    ((await req.json()) as any[]).forEach((meta: any) => {
+      const pageName = meta.name;
+      result.add({
+        name: pageName,
+        size: meta.size,
+        lastModified: meta.lastModified,
+        contentType: meta.contentType,
+        perm: "rw",
+      });
+    });
+
+    return {
+      attachments: result,
+      nowTimestamp: +req.headers.get("Now-Timestamp")!,
+    };
+  }
+
+  async readAttachment(
+    name: string
+  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> {
+    let res = await this.authenticatedFetch(`${this.fsaUrl}/${name}`, {
+      method: "GET",
+    });
+    if (res.headers.get("X-Status") === "404") {
+      throw new Error(`Page not found`);
+    }
+    let blob = await res.blob();
+    return {
+      buffer: await blob.arrayBuffer(),
+      meta: this.responseToAttachmentMeta(name, res),
+    };
+  }
+
+  async writeAttachment(
+    name: string,
+    buffer: ArrayBuffer,
+    selfUpdate?: boolean,
+    lastModified?: number
+  ): Promise<AttachmentMeta> {
+    // TODO: lastModified ignored for now
+    let res = await this.authenticatedFetch(`${this.fsaUrl}/${name}`, {
+      method: "PUT",
+      body: buffer,
+      headers: {
+        "Last-Modified": lastModified ? "" + lastModified : undefined,
+        "Content-type": "application/octet-stream",
+        "Content-length": "" + buffer.byteLength,
+      },
+    });
+    const newMeta = this.responseToAttachmentMeta(name, res);
+    return newMeta;
+  }
+
+  async getAttachmentMeta(name: string): Promise<AttachmentMeta> {
+    let res = await this.authenticatedFetch(`${this.fsaUrl}/${name}`, {
+      method: "OPTIONS",
+    });
+    if (res.headers.get("X-Status") === "404") {
+      throw new Error(`Page not found`);
+    }
+    return this.responseToAttachmentMeta(name, res);
+  }
+
+  async deleteAttachment(name: string): Promise<void> {
+    let req = await this.authenticatedFetch(`${this.fsaUrl}/${name}`, {
+      method: "DELETE",
+    });
+    if (req.status !== 200) {
+      throw Error(`Failed to delete attachment: ${req.statusText}`);
+    }
+  }
+
+  // Plugs
+
   async invokeFunction(
     plug: Plug<any>,
     env: string,
@@ -151,20 +237,34 @@ export class HttpSpacePrimitives implements SpacePrimitives {
   }
 
   async getPageMeta(name: string): Promise<PageMeta> {
-    let res = await this.authenticatedFetch(`${this.pageUrl}/${name}`, {
+    let res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
       method: "OPTIONS",
     });
     if (res.headers.get("X-Status") === "404") {
       throw new Error(`Page not found`);
     }
-    return this.responseToMeta(name, res);
+    return this.responseToPageMeta(name, res);
   }
 
-  private responseToMeta(name: string, res: Response): PageMeta {
+  private responseToPageMeta(name: string, res: Response): PageMeta {
     return {
       name,
       lastModified: +(res.headers.get("Last-Modified") || "0"),
       perm: (res.headers.get("X-Permission") as "rw" | "ro") || "rw",
     };
   }
+
+  private responseToAttachmentMeta(
+    name: string,
+    res: Response
+  ): AttachmentMeta {
+    return {
+      name,
+      lastModified: +(res.headers.get("Last-Modified") || "0"),
+      size: +(res.headers.get("Content-Length") || "0"),
+      contentType:
+        res.headers.get("Content-Type") || "application/octet-stream",
+      perm: (res.headers.get("X-Permission") as "rw" | "ro") || "rw",
+    };
+  }
 }

packages/common/spaces/indexeddb_space_primitives.ts

@@ -1,5 +1,5 @@
 import { SpacePrimitives } from "./space_primitives";
-import { PageMeta } from "../types";
+import { AttachmentMeta, PageMeta } from "../types";
 import Dexie, { Table } from "dexie";
 import { Plug } from "@plugos/plugos/plug";
 
@@ -19,6 +19,31 @@ export class IndexedDBSpacePrimitives implements SpacePrimitives {
     });
     this.pageTable = db.table("page");
   }
+  fetchAttachmentList(): Promise<{
+    attachments: Set<AttachmentMeta>;
+    nowTimestamp: number;
+  }> {
+    throw new Error("Method not implemented.");
+  }
+  readAttachment(
+    name: string
+  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> {
+    throw new Error("Method not implemented.");
+  }
+  getAttachmentMeta(name: string): Promise<AttachmentMeta> {
+    throw new Error("Method not implemented.");
+  }
+  writeAttachment(
+    name: string,
+    blob: ArrayBuffer,
+    selfUpdate?: boolean | undefined,
+    lastModified?: number | undefined
+  ): Promise<AttachmentMeta> {
+    throw new Error("Method not implemented.");
+  }
+  deleteAttachment(name: string): Promise<void> {
+    throw new Error("Method not implemented.");
+  }
 
   async deletePage(name: string): Promise<void> {
     return this.pageTable.delete(name);

packages/common/spaces/space.ts

@@ -1,5 +1,5 @@
 import { SpacePrimitives } from "./space_primitives";
-import { PageMeta } from "../types";
+import { AttachmentMeta, PageMeta } from "../types";
 import { EventEmitter } from "@plugos/plugos/event";
 import { Plug } from "@plugos/plugos/plug";
 import { Manifest } from "../manifest";
@@ -15,7 +15,10 @@ export type SpaceEvents = {
   pageListUpdated: (pages: Set<PageMeta>) => void;
 };
 
-export class Space extends EventEmitter<SpaceEvents> {
+export class Space
+  extends EventEmitter<SpaceEvents>
+  implements SpacePrimitives
+{
   pageMetaCache = new Map<string, PageMeta>();
   watchedPages = new Set<string>();
   private initialPageListLoad = true;
@@ -215,6 +218,32 @@ export class Space extends EventEmitter<SpaceEvents> {
     return this.space.fetchPageList();
   }
 
+  fetchAttachmentList(): Promise<{
+    attachments: Set<AttachmentMeta>;
+    nowTimestamp: number;
+  }> {
+    return this.space.fetchAttachmentList();
+  }
+  readAttachment(
+    name: string
+  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> {
+    return this.space.readAttachment(name);
+  }
+  getAttachmentMeta(name: string): Promise<AttachmentMeta> {
+    return this.space.getAttachmentMeta(name);
+  }
+  writeAttachment(
+    name: string,
+    blob: ArrayBuffer,
+    selfUpdate?: boolean | undefined,
+    lastModified?: number | undefined
+  ): Promise<AttachmentMeta> {
+    return this.space.writeAttachment(name, blob, selfUpdate, lastModified);
+  }
+  deleteAttachment(name: string): Promise<void> {
+    return this.space.deleteAttachment(name);
+  }
+
   private metaCacher(name: string, pageMeta: PageMeta): PageMeta {
     this.pageMetaCache.set(name, pageMeta);
     return pageMeta;

packages/common/spaces/space_primitives.ts

@@ -1,5 +1,5 @@
 import { Plug } from "@plugos/plugos/plug";
-import { PageMeta } from "../types";
+import { AttachmentMeta, PageMeta } from "../types";
 
 export interface SpacePrimitives {
   // Pages
@@ -14,6 +14,23 @@ export interface SpacePrimitives {
   ): Promise<PageMeta>;
   deletePage(name: string): Promise<void>;
 
+  // Attachments
+  fetchAttachmentList(): Promise<{
+    attachments: Set<AttachmentMeta>;
+    nowTimestamp: number;
+  }>;
+  readAttachment(
+    name: string
+  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }>;
+  getAttachmentMeta(name: string): Promise<AttachmentMeta>;
+  writeAttachment(
+    name: string,
+    blob: ArrayBuffer,
+    selfUpdate?: boolean,
+    lastModified?: number
+  ): Promise<AttachmentMeta>;
+  deleteAttachment(name: string): Promise<void>;
+
   // Plugs
   proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any>;
   invokeFunction(