Initial pass of supporting attachments #71

mattermost-plugin/server/plugin.go

@@ -26,7 +26,7 @@ type Plugin struct {
 // ServeHTTP demonstrates a plugin that handles HTTP requests by greeting the world.
 func (p *Plugin) ServeHTTP(c *plugin.Context, w http.ResponseWriter, r *http.Request) {
 
-	if strings.HasPrefix(r.URL.Path, "/fs") || strings.HasPrefix(r.URL.Path, "/plug/") {
+	if strings.HasPrefix(r.URL.Path, "/page") || strings.HasPrefix(r.URL.Path, "/plug/") {
 		p.httpProxy(w, r)
 		return
 	}

package-lock.json

@@ -3094,6 +3094,12 @@
       "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz",
       "integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw=="
     },
+    "node_modules/@types/mime-types": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/@types/mime-types/-/mime-types-2.1.1.tgz",
+      "integrity": "sha512-vXOTGVSLR2jMw440moWTC7H19iUyLtP3Z1YTj7cSsubOICinjMxFeb/V57v9QdyyPGbbWolUFSSmSiRSn94tFw==",
+      "dev": true
+    },
     "node_modules/@types/node": {
       "version": "18.6.1",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-18.6.1.tgz",
@@ -9793,7 +9799,11 @@
         "@lezer/common": "1.0.0",
         "@lezer/highlight": "1.0.0",
         "@lezer/markdown": "1.0.1",
+        "mime-types": "^2.1.35",
         "yaml": "^1.10.2"
+      },
+      "devDependencies": {
+        "@types/mime-types": "^2.1.1"
       }
     },
     "packages/common/node_modules/@lezer/common": {
@@ -18298,6 +18308,8 @@
         "@lezer/common": "1.0.0",
         "@lezer/highlight": "1.0.0",
         "@lezer/markdown": "1.0.1",
+        "@types/mime-types": "*",
+        "mime-types": "^2.1.35",
         "yaml": "^1.10.2"
       },
       "dependencies": {
@@ -20031,6 +20043,12 @@
       "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz",
       "integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw=="
     },
+    "@types/mime-types": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/@types/mime-types/-/mime-types-2.1.1.tgz",
+      "integrity": "sha512-vXOTGVSLR2jMw440moWTC7H19iUyLtP3Z1YTj7cSsubOICinjMxFeb/V57v9QdyyPGbbWolUFSSmSiRSn94tFw==",
+      "dev": true
+    },
     "@types/node": {
       "version": "18.6.1",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-18.6.1.tgz",

packages/common/package.json

@@ -19,6 +19,10 @@
     "@lezer/common": "1.0.0",
     "@lezer/highlight": "1.0.0",
     "@lezer/markdown": "1.0.1",
+    "mime-types": "^2.1.35",
     "yaml": "^1.10.2"
+  },
+  "devDependencies": {
+    "@types/mime-types": "^2.1.1"
   }
 }

packages/common/spaces/disk_space_primitives.ts

@@ -8,10 +8,11 @@ import {
   writeFile,
 } from "fs/promises";
 import * as path from "path";
-import { PageMeta } from "../types";
+import { AttachmentMeta, PageMeta } from "../types";
 import { SpacePrimitives } from "./space_primitives";
 import { Plug } from "@plugos/plugos/plug";
 import { realpathSync } from "fs";
+import mime from "mime-types";
 
 export class DiskSpacePrimitives implements SpacePrimitives {
   rootPath: string;
@@ -47,6 +48,7 @@ export class DiskSpacePrimitives implements SpacePrimitives {
     );
   }
 
+  // Pages
   async readPage(pageName: string): Promise<{ text: string; meta: PageMeta }> {
     const localPath = this.pageNameToPath(pageName);
     try {
@@ -148,6 +150,136 @@ export class DiskSpacePrimitives implements SpacePrimitives {
     };
   }
 
+  // Attachments
+  attachmentNameToPath(name: string) {
+    return this.safePath(path.join(this.rootPath, name));
+  }
+
+  pathToAttachmentName(fullPath: string): string {
+    return fullPath.substring(this.rootPath.length + 1);
+  }
+
+  async fetchAttachmentList(): Promise<{
+    attachments: Set<AttachmentMeta>;
+    nowTimestamp: number;
+  }> {
+    let attachments = new Set<AttachmentMeta>();
+
+    const walkPath = async (dir: string) => {
+      let files = await readdir(dir);
+      for (let file of files) {
+        const fullPath = path.join(dir, file);
+        let s = await stat(fullPath);
+        if (s.isDirectory()) {
+          if (!file.startsWith(".")) {
+            await walkPath(fullPath);
+          }
+        } else {
+          if (
+            !file.startsWith(".") &&
+            !file.endsWith(".md") &&
+            !file.endsWith(".json")
+          ) {
+            attachments.add({
+              name: this.pathToAttachmentName(fullPath),
+              lastModified: s.mtime.getTime(),
+              size: s.size,
+              contentType: mime.lookup(file) || "application/octet-stream",
+              perm: "rw",
+            } as AttachmentMeta);
+          }
+        }
+      }
+    };
+    await walkPath(this.rootPath);
+    return {
+      attachments,
+      nowTimestamp: Date.now(),
+    };
+  }
+
+  async readAttachment(
+    name: string
+  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> {
+    const localPath = this.attachmentNameToPath(name);
+    let fileBuffer = await readFile(localPath);
+
+    try {
+      const s = await stat(localPath);
+      return {
+        buffer: fileBuffer.buffer,
+        meta: {
+          name: name,
+          lastModified: s.mtime.getTime(),
+          size: s.size,
+          contentType: mime.lookup(name) || "application/octet-stream",
+          perm: "rw",
+        },
+      };
+    } catch (e) {
+      // console.error("Error while reading attachment", name, e);
+      throw Error(`Could not read attachment ${name}`);
+    }
+  }
+
+  async getAttachmentMeta(name: string): Promise<AttachmentMeta> {
+    const localPath = this.attachmentNameToPath(name);
+    try {
+      const s = await stat(localPath);
+      return {
+        name: name,
+        lastModified: s.mtime.getTime(),
+        size: s.size,
+        contentType: mime.lookup(name) || "application/octet-stream",
+        perm: "rw",
+      };
+    } catch (e) {
+      // console.error("Error while getting attachment meta", name, e);
+      throw Error(`Could not get meta for ${name}`);
+    }
+  }
+
+  async writeAttachment(
+    name: string,
+    blob: ArrayBuffer,
+    selfUpdate?: boolean,
+    lastModified?: number
+  ): Promise<AttachmentMeta> {
+    let localPath = this.attachmentNameToPath(name);
+    try {
+      // Ensure parent folder exists
+      await mkdir(path.dirname(localPath), { recursive: true });
+
+      // Actually write the file
+      await writeFile(localPath, Buffer.from(blob));
+
+      if (lastModified) {
+        let d = new Date(lastModified);
+        console.log("Going to set the modified time", d);
+        await utimes(localPath, d, d);
+      }
+
+      // Fetch new metadata
+      const s = await stat(localPath);
+      return {
+        name: name,
+        lastModified: s.mtime.getTime(),
+        size: s.size,
+        contentType: mime.lookup(name) || "application/octet-stream",
+        perm: "rw",
+      };
+    } catch (e) {
+      console.error("Error while writing attachment", name, e);
+      throw Error(`Could not write ${name}`);
+    }
+  }
+
+  async deleteAttachment(name: string): Promise<void> {
+    let localPath = this.attachmentNameToPath(name);
+    await unlink(localPath);
+  }
+
+  // Plugs
   invokeFunction(
     plug: Plug<any>,
     env: string,

packages/common/spaces/evented_space_primitives.ts

@@ -1,7 +1,7 @@
 import { EventHook } from "@plugos/plugos/hooks/event";
 import { Plug } from "@plugos/plugos/plug";
 
-import { PageMeta } from "../types";
+import { AttachmentMeta, PageMeta } from "../types";
 import { plugPrefix, trashPrefix } from "./constants";
 import { SpacePrimitives } from "./space_primitives";
 
@@ -66,4 +66,42 @@ export class EventedSpacePrimitives implements SpacePrimitives {
     await this.eventHook.dispatchEvent("page:deleted", pageName);
     return this.wrapped.deletePage(pageName);
   }
+
+  fetchAttachmentList(): Promise<{
+    attachments: Set<AttachmentMeta>;
+    nowTimestamp: number;
+  }> {
+    return this.wrapped.fetchAttachmentList();
+  }
+
+  readAttachment(
+    name: string
+  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> {
+    return this.wrapped.readAttachment(name);
+  }
+
+  getAttachmentMeta(name: string): Promise<AttachmentMeta> {
+    return this.wrapped.getAttachmentMeta(name);
+  }
+
+  async writeAttachment(
+    name: string,
+    blob: ArrayBuffer,
+    selfUpdate?: boolean | undefined,
+    lastModified?: number | undefined
+  ): Promise<AttachmentMeta> {
+    let meta = await this.wrapped.writeAttachment(
+      name,
+      blob,
+      selfUpdate,
+      lastModified
+    );
+    await this.eventHook.dispatchEvent("attachment:saved", name);
+    return meta;
+  }
+
+  async deleteAttachment(name: string): Promise<void> {
+    await this.eventHook.dispatchEvent("attachment:deleted", name);
+    return this.wrapped.deleteAttachment(name);
+  }
 }

packages/common/spaces/http_space_primitives.ts

@@ -1,14 +1,16 @@
-import { PageMeta } from "../types";
+import { AttachmentMeta, PageMeta } from "../types";
 import { Plug } from "@plugos/plugos/plug";
 import { SpacePrimitives } from "./space_primitives";
 
 export class HttpSpacePrimitives implements SpacePrimitives {
-  pageUrl: string;
+  fsUrl: string;
+  fsaUrl: string;
   private plugUrl: string;
   token?: string;
 
   constructor(url: string, token?: string) {
-    this.pageUrl = url + "/fs";
+    this.fsUrl = url + "/page";
+    this.fsaUrl = url + "/attachment";
     this.plugUrl = url + "/plug";
     this.token = token;
   }
@@ -32,7 +34,7 @@ export class HttpSpacePrimitives implements SpacePrimitives {
     pages: Set<PageMeta>;
     nowTimestamp: number;
   }> {
-    let req = await this.authenticatedFetch(this.pageUrl, {
+    let req = await this.authenticatedFetch(this.fsUrl, {
       method: "GET",
     });
 
@@ -53,7 +55,7 @@ export class HttpSpacePrimitives implements SpacePrimitives {
   }
 
   async readPage(name: string): Promise<{ text: string; meta: PageMeta }> {
-    let res = await this.authenticatedFetch(`${this.pageUrl}/${name}`, {
+    let res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
       method: "GET",
     });
     if (res.headers.get("X-Status") === "404") {
@@ -61,7 +63,7 @@ export class HttpSpacePrimitives implements SpacePrimitives {
     }
     return {
       text: await res.text(),
-      meta: this.responseToMeta(name, res),
+      meta: this.responseToPageMeta(name, res),
     };
   }
 
@@ -72,7 +74,7 @@ export class HttpSpacePrimitives implements SpacePrimitives {
     lastModified?: number
   ): Promise<PageMeta> {
     // TODO: lastModified ignored for now
-    let res = await this.authenticatedFetch(`${this.pageUrl}/${name}`, {
+    let res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
       method: "PUT",
       body: text,
       headers: lastModified
@@ -81,12 +83,12 @@ export class HttpSpacePrimitives implements SpacePrimitives {
           }
         : undefined,
     });
-    const newMeta = this.responseToMeta(name, res);
+    const newMeta = this.responseToPageMeta(name, res);
     return newMeta;
   }
 
   async deletePage(name: string): Promise<void> {
-    let req = await this.authenticatedFetch(`${this.pageUrl}/${name}`, {
+    let req = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
       method: "DELETE",
     });
     if (req.status !== 200) {
@@ -115,6 +117,90 @@ export class HttpSpacePrimitives implements SpacePrimitives {
     return await req.json();
   }
 
+  // Attachments
+  public async fetchAttachmentList(): Promise<{
+    attachments: Set<AttachmentMeta>;
+    nowTimestamp: number;
+  }> {
+    let req = await this.authenticatedFetch(this.fsaUrl, {
+      method: "GET",
+    });
+
+    let result = new Set<AttachmentMeta>();
+    ((await req.json()) as any[]).forEach((meta: any) => {
+      const pageName = meta.name;
+      result.add({
+        name: pageName,
+        size: meta.size,
+        lastModified: meta.lastModified,
+        contentType: meta.contentType,
+        perm: "rw",
+      });
+    });
+
+    return {
+      attachments: result,
+      nowTimestamp: +req.headers.get("Now-Timestamp")!,
+    };
+  }
+
+  async readAttachment(
+    name: string
+  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> {
+    let res = await this.authenticatedFetch(`${this.fsaUrl}/${name}`, {
+      method: "GET",
+    });
+    if (res.headers.get("X-Status") === "404") {
+      throw new Error(`Page not found`);
+    }
+    let blob = await res.blob();
+    return {
+      buffer: await blob.arrayBuffer(),
+      meta: this.responseToAttachmentMeta(name, res),
+    };
+  }
+
+  async writeAttachment(
+    name: string,
+    buffer: ArrayBuffer,
+    selfUpdate?: boolean,
+    lastModified?: number
+  ): Promise<AttachmentMeta> {
+    // TODO: lastModified ignored for now
+    let res = await this.authenticatedFetch(`${this.fsaUrl}/${name}`, {
+      method: "PUT",
+      body: buffer,
+      headers: {
+        "Last-Modified": lastModified ? "" + lastModified : undefined,
+        "Content-type": "application/octet-stream",
+        "Content-length": "" + buffer.byteLength,
+      },
+    });
+    const newMeta = this.responseToAttachmentMeta(name, res);
+    return newMeta;
+  }
+
+  async getAttachmentMeta(name: string): Promise<AttachmentMeta> {
+    let res = await this.authenticatedFetch(`${this.fsaUrl}/${name}`, {
+      method: "OPTIONS",
+    });
+    if (res.headers.get("X-Status") === "404") {
+      throw new Error(`Page not found`);
+    }
+    return this.responseToAttachmentMeta(name, res);
+  }
+
+  async deleteAttachment(name: string): Promise<void> {
+    let req = await this.authenticatedFetch(`${this.fsaUrl}/${name}`, {
+      method: "DELETE",
+    });
+    if (req.status !== 200) {
+      throw Error(`Failed to delete attachment: ${req.statusText}`);
+    }
+  }
+
+  // Plugs
+
   async invokeFunction(
     plug: Plug<any>,
     env: string,
@@ -151,20 +237,34 @@ export class HttpSpacePrimitives implements SpacePrimitives {
   }
 
   async getPageMeta(name: string): Promise<PageMeta> {
-    let res = await this.authenticatedFetch(`${this.pageUrl}/${name}`, {
+    let res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
       method: "OPTIONS",
     });
     if (res.headers.get("X-Status") === "404") {
       throw new Error(`Page not found`);
     }
-    return this.responseToMeta(name, res);
+    return this.responseToPageMeta(name, res);
   }
 
-  private responseToMeta(name: string, res: Response): PageMeta {
+  private responseToPageMeta(name: string, res: Response): PageMeta {
     return {
       name,
       lastModified: +(res.headers.get("Last-Modified") || "0"),
       perm: (res.headers.get("X-Permission") as "rw" | "ro") || "rw",
     };
   }
+
+  private responseToAttachmentMeta(
+    name: string,
+    res: Response
+  ): AttachmentMeta {
+    return {
+      name,
+      lastModified: +(res.headers.get("Last-Modified") || "0"),
+      size: +(res.headers.get("Content-Length") || "0"),
+      contentType:
+        res.headers.get("Content-Type") || "application/octet-stream",
+      perm: (res.headers.get("X-Permission") as "rw" | "ro") || "rw",
+    };
+  }
 }

packages/common/spaces/indexeddb_space_primitives.ts

@@ -1,5 +1,5 @@
 import { SpacePrimitives } from "./space_primitives";
-import { PageMeta } from "../types";
+import { AttachmentMeta, PageMeta } from "../types";
 import Dexie, { Table } from "dexie";
 import { Plug } from "@plugos/plugos/plug";
 
@@ -19,6 +19,31 @@ export class IndexedDBSpacePrimitives implements SpacePrimitives {
     });
     this.pageTable = db.table("page");
   }
+  fetchAttachmentList(): Promise<{
+    attachments: Set<AttachmentMeta>;
+    nowTimestamp: number;
+  }> {
+    throw new Error("Method not implemented.");
+  }
+  readAttachment(
+    name: string
+  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> {
+    throw new Error("Method not implemented.");
+  }
+  getAttachmentMeta(name: string): Promise<AttachmentMeta> {
+    throw new Error("Method not implemented.");
+  }
+  writeAttachment(
+    name: string,
+    blob: ArrayBuffer,
+    selfUpdate?: boolean | undefined,
+    lastModified?: number | undefined
+  ): Promise<AttachmentMeta> {
+    throw new Error("Method not implemented.");
+  }
+  deleteAttachment(name: string): Promise<void> {
+    throw new Error("Method not implemented.");
+  }
 
   async deletePage(name: string): Promise<void> {
     return this.pageTable.delete(name);

packages/common/spaces/space.ts

@@ -1,5 +1,5 @@
 import { SpacePrimitives } from "./space_primitives";
-import { PageMeta } from "../types";
+import { AttachmentMeta, PageMeta } from "../types";
 import { EventEmitter } from "@plugos/plugos/event";
 import { Plug } from "@plugos/plugos/plug";
 import { Manifest } from "../manifest";
@@ -15,7 +15,10 @@ export type SpaceEvents = {
   pageListUpdated: (pages: Set<PageMeta>) => void;
 };
 
-export class Space extends EventEmitter<SpaceEvents> {
+export class Space
+  extends EventEmitter<SpaceEvents>
+  implements SpacePrimitives
+{
   pageMetaCache = new Map<string, PageMeta>();
   watchedPages = new Set<string>();
   private initialPageListLoad = true;
@@ -215,6 +218,32 @@ export class Space extends EventEmitter<SpaceEvents> {
     return this.space.fetchPageList();
   }
 
+  fetchAttachmentList(): Promise<{
+    attachments: Set<AttachmentMeta>;
+    nowTimestamp: number;
+  }> {
+    return this.space.fetchAttachmentList();
+  }
+  readAttachment(
+    name: string
+  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> {
+    return this.space.readAttachment(name);
+  }
+  getAttachmentMeta(name: string): Promise<AttachmentMeta> {
+    return this.space.getAttachmentMeta(name);
+  }
+  writeAttachment(
+    name: string,
+    blob: ArrayBuffer,
+    selfUpdate?: boolean | undefined,
+    lastModified?: number | undefined
+  ): Promise<AttachmentMeta> {
+    return this.space.writeAttachment(name, blob, selfUpdate, lastModified);
+  }
+  deleteAttachment(name: string): Promise<void> {
+    return this.space.deleteAttachment(name);
+  }
+
   private metaCacher(name: string, pageMeta: PageMeta): PageMeta {
     this.pageMetaCache.set(name, pageMeta);
     return pageMeta;

packages/common/spaces/space_primitives.ts

@@ -1,5 +1,5 @@
 import { Plug } from "@plugos/plugos/plug";
-import { PageMeta } from "../types";
+import { AttachmentMeta, PageMeta } from "../types";
 
 export interface SpacePrimitives {
   // Pages
@@ -14,6 +14,23 @@ export interface SpacePrimitives {
   ): Promise<PageMeta>;
   deletePage(name: string): Promise<void>;
 
+  // Attachments
+  fetchAttachmentList(): Promise<{
+    attachments: Set<AttachmentMeta>;
+    nowTimestamp: number;
+  }>;
+  readAttachment(
+    name: string
+  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }>;
+  getAttachmentMeta(name: string): Promise<AttachmentMeta>;
+  writeAttachment(
+    name: string,
+    blob: ArrayBuffer,
+    selfUpdate?: boolean,
+    lastModified?: number
+  ): Promise<AttachmentMeta>;
+  deleteAttachment(name: string): Promise<void>;
+
   // Plugs
   proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any>;
   invokeFunction(

packages/common/types.ts

@@ -1,3 +1,5 @@
+export const reservedPageNames = ["page", "attachment", "plug"];
+
 export type PageMeta = {
   name: string;
   lastModified: number;
@@ -5,6 +7,14 @@ export type PageMeta = {
   perm: "ro" | "rw";
 };
 
+export type AttachmentMeta = {
+  name: string;
+  contentType: string;
+  lastModified: number;
+  size: number;
+  perm: "ro" | "rw";
+};
+
 // Used by FilterBox
 export type FilterOption = {
   name: string;

packages/server/express_server.ts

@@ -281,6 +281,113 @@ export class ExpressServer {
     // Serve static files (javascript, css, html)
     this.app.use("/", express.static(this.distDir));
 
+    // Pages API
+    this.app.use(
+      "/page",
+      passwordMiddleware,
+      cors({
+        methods: "GET,HEAD,PUT,OPTIONS,POST,DELETE",
+        preflightContinue: true,
+      }),
+      this.buildFsRouter()
+    );
+
+    // Attachment API
+    this.app.use(
+      "/attachment",
+      passwordMiddleware,
+      cors({
+        methods: "GET,HEAD,PUT,OPTIONS,POST,DELETE",
+        preflightContinue: true,
+      }),
+      this.buildAttachmentRouter()
+    );
+
+    // Plug API
+    this.app.use(
+      "/plug",
+      passwordMiddleware,
+      cors({
+        methods: "GET,HEAD,PUT,OPTIONS,POST,DELETE",
+        preflightContinue: true,
+      }),
+      this.buildPlugRouter()
+    );
+
+    // Fallback, serve index.html
+    this.app.get("/*", async (req, res) => {
+      res.sendFile(`${this.distDir}/index.html`, {});
+    });
+
+    this.server = http.createServer(this.app);
+    this.server.listen(this.port, () => {
+      console.log(
+        `Silver Bullet is now running: http://localhost:${this.port}`
+      );
+      console.log("--------------");
+    });
+  }
+
+  private buildPlugRouter() {
+    let plugRouter = express.Router();
+
+    plugRouter.post(
+      "/:plug/syscall/:name",
+      bodyParser.json(),
+      async (req, res) => {
+        const name = req.params.name;
+        const plugName = req.params.plug;
+        const args = req.body as any;
+        const plug = this.system.loadedPlugs.get(plugName);
+        if (!plug) {
+          res.status(404);
+          return res.send(`Plug ${plugName} not found`);
+        }
+        try {
+          const result = await this.system.syscallWithContext(
+            { plug },
+            name,
+            args
+          );
+          res.status(200);
+          res.header("Content-Type", "application/json");
+          res.send(JSON.stringify(result));
+        } catch (e: any) {
+          res.status(500);
+          return res.send(e.message);
+        }
+      }
+    );
+
+    plugRouter.post(
+      "/:plug/function/:name",
+      bodyParser.json(),
+      async (req, res) => {
+        const name = req.params.name;
+        const plugName = req.params.plug;
+        const args = req.body as any[];
+        const plug = this.system.loadedPlugs.get(plugName);
+        if (!plug) {
+          res.status(404);
+          return res.send(`Plug ${plugName} not found`);
+        }
+        try {
+          const result = await plug.invoke(name, args);
+          res.status(200);
+          res.header("Content-Type", "application/json");
+          res.send(JSON.stringify(result));
+        } catch (e: any) {
+          res.status(500);
+          // console.log("Error invoking function", e);
+          return res.send(e.message);
+        }
+      }
+    );
+
+    return plugRouter;
+  }
+
+  private buildFsRouter() {
     let fsRouter = express.Router();
 
     // Page list
@@ -360,96 +467,120 @@ export class ExpressServer {
           res.send("OK");
         }
       });
+    return fsRouter;
+  }
 
-    this.app.use(
+  // Build attachment router
-      "/fs",
+  private buildAttachmentRouter() {
-      passwordMiddleware,
+    let fsaRouter = express.Router();
-      cors({
-        methods: "GET,HEAD,PUT,OPTIONS,POST,DELETE",
-        preflightContinue: true,
-      }),
-      fsRouter
-    );
 
-    let plugRouter = express.Router();
+    // Page list
-
+    fsaRouter.route("/").get(async (req, res) => {
-    // TODO: This is currently only used for the indexer calls, it's potentially dangerous
+      let { nowTimestamp, attachments } =
-    // do we need a better solution?
+        await this.space.fetchAttachmentList();
-    plugRouter.post(
+      res.header("Now-Timestamp", "" + nowTimestamp);
-      "/:plug/syscall/:name",
+      res.json([...attachments]);
-      bodyParser.json(),
-      async (req, res) => {
-        const name = req.params.name;
-        const plugName = req.params.plug;
-        const args = req.body as any;
-        const plug = this.system.loadedPlugs.get(plugName);
-        if (!plug) {
-          res.status(404);
-          return res.send(`Plug ${plugName} not found`);
-        }
-        try {
-          const result = await this.system.syscallWithContext(
-            { plug },
-            name,
-            args
-          );
-          res.status(200);
-          res.header("Content-Type", "application/json");
-          res.send(JSON.stringify(result));
-        } catch (e: any) {
-          res.status(500);
-          return res.send(e.message);
-        }
-      }
-    );
-
-    plugRouter.post(
-      "/:plug/function/:name",
-      bodyParser.json(),
-      async (req, res) => {
-        const name = req.params.name;
-        const plugName = req.params.plug;
-        const args = req.body as any[];
-        const plug = this.system.loadedPlugs.get(plugName);
-        if (!plug) {
-          res.status(404);
-          return res.send(`Plug ${plugName} not found`);
-        }
-        try {
-          const result = await plug.invoke(name, args);
-          res.status(200);
-          res.header("Content-Type", "application/json");
-          res.send(JSON.stringify(result));
-        } catch (e: any) {
-          res.status(500);
-          // console.log("Error invoking function", e);
-          return res.send(e.message);
-        }
-      }
-    );
-
-    this.app.use(
-      "/plug",
-      passwordMiddleware,
-      cors({
-        methods: "GET,HEAD,PUT,OPTIONS,POST,DELETE",
-        preflightContinue: true,
-      }),
-      plugRouter
-    );
-
-    // Fallback, serve index.html
-    this.app.get("/*", async (req, res) => {
-      res.sendFile(`${this.distDir}/index.html`, {});
     });
 
-    this.server = http.createServer(this.app);
+    fsaRouter
-    this.server.listen(this.port, () => {
+      .route(/\/(.+)/)
-      console.log(
+      .get(async (req, res) => {
-        `Silver Bullet is now running: http://localhost:${this.port}`
+        let attachmentName = req.params[0];
-      );
+        if (!this.attachmentCheck(attachmentName, res)) {
-      console.log("--------------");
+          return;
-    });
+        }
+        console.log("Getting", attachmentName);
+        try {
+          let attachmentData = await this.space.readAttachment(attachmentName);
+          res.status(200);
+          res.header("Last-Modified", "" + attachmentData.meta.lastModified);
+          res.header("X-Permission", attachmentData.meta.perm);
+          res.header("Content-Type", attachmentData.meta.contentType);
+          // res.header("X-Content-Length", "" + attachmentData.meta.size);
+          res.send(Buffer.from(attachmentData.buffer));
+        } catch (e) {
+          // CORS
+          res.status(200);
+          res.header("X-Status", "404");
+          res.send("");
+        }
+      })
+      .put(
+        bodyParser.raw({ type: "*/*", limit: "100mb" }),
+        async (req, res) => {
+          let attachmentName = req.params[0];
+          if (!this.attachmentCheck(attachmentName, res)) {
+            return;
+          }
+          console.log("Saving attachment", attachmentName);
+
+          try {
+            let meta = await this.space.writeAttachment(
+              attachmentName,
+              req.body,
+              false,
+              req.header("Last-Modified")
+                ? +req.header("Last-Modified")!
+                : undefined
+            );
+            res.status(200);
+            res.header("Last-Modified", "" + meta.lastModified);
+            res.header("Content-Type", meta.contentType);
+            res.header("Content-Length", "" + meta.size);
+            res.header("X-Permission", meta.perm);
+            res.send("OK");
+          } catch (err) {
+            res.status(500);
+            res.send("Write failed");
+            console.error("Pipeline failed", err);
+          }
+        }
+      )
+      .options(async (req, res) => {
+        let attachmentName = req.params[0];
+        if (!this.attachmentCheck(attachmentName, res)) {
+          return;
+        }
+        try {
+          const meta = await this.space.getAttachmentMeta(attachmentName);
+          res.status(200);
+          res.header("Last-Modified", "" + meta.lastModified);
+          res.header("X-Permission", meta.perm);
+          res.header("Content-Length", "" + meta.size);
+          res.header("Content-Type", meta.contentType);
+          res.send("");
+        } catch (e) {
+          // CORS
+          res.status(200);
+          res.header("X-Status", "404");
+          res.send("Not found");
+        }
+      })
+      .delete(async (req, res) => {
+        let attachmentName = req.params[0];
+        if (!this.attachmentCheck(attachmentName, res)) {
+          return;
+        }
+        try {
+          await this.space.deleteAttachment(attachmentName);
+          res.status(200);
+          res.send("OK");
+        } catch (e) {
+          console.error("Error deleting attachment", e);
+          res.status(500);
+          res.send("OK");
+        }
+      });
+    return fsaRouter;
+  }
+
+  attachmentCheck(attachmentName: string, res: express.Response): boolean {
+    if (attachmentName.endsWith(".md")) {
+      res.status(405);
+      res.send("No markdown files allowed through the attachment API");
+      return false;
+    }
+    return true;
   }
 
   async ensureAndLoadSettings() {

packages/server/hooks/plug_space_primitives.ts

@@ -1,6 +1,6 @@
 import { Plug } from "@plugos/plugos/plug";
 import { SpacePrimitives } from "@silverbulletmd/common/spaces/space_primitives";
-import { PageMeta } from "@silverbulletmd/common/types";
+import { AttachmentMeta, PageMeta } from "@silverbulletmd/common/types";
 import { PageNamespaceHook, PageNamespaceOperation } from "./page_namespace";
 
 export class PlugSpacePrimitives implements SpacePrimitives {
@@ -92,6 +92,32 @@ export class PlugSpacePrimitives implements SpacePrimitives {
     return this.wrapped.deletePage(name);
   }
 
+  fetchAttachmentList(): Promise<{
+    attachments: Set<AttachmentMeta>;
+    nowTimestamp: number;
+  }> {
+    return this.wrapped.fetchAttachmentList();
+  }
+  readAttachment(
+    name: string
+  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> {
+    return this.wrapped.readAttachment(name);
+  }
+  getAttachmentMeta(name: string): Promise<AttachmentMeta> {
+    return this.wrapped.getAttachmentMeta(name);
+  }
+  writeAttachment(
+    name: string,
+    blob: ArrayBuffer,
+    selfUpdate?: boolean | undefined,
+    lastModified?: number | undefined
+  ): Promise<AttachmentMeta> {
+    return this.wrapped.writeAttachment(name, blob, selfUpdate, lastModified);
+  }
+  deleteAttachment(name: string): Promise<void> {
+    return this.wrapped.deleteAttachment(name);
+  }
+
   proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
     return this.wrapped.proxySyscall(plug, name, args);
   }

packages/web/editor.tsx

@@ -47,7 +47,7 @@ import { systemSyscalls } from "./syscalls/system";
 import { Panel } from "./components/panel";
 import { CommandHook } from "./hooks/command";
 import { SlashCommandHook } from "./hooks/slash_command";
-import { pasteLinkExtension } from "./editor_paste";
+import { pasteAttachmentExtension, pasteLinkExtension } from "./editor_paste";
 import { markdownSyscalls } from "@silverbulletmd/common/syscalls/markdown";
 import { clientStoreSyscalls } from "./syscalls/clientStore";
 import {
@@ -55,7 +55,11 @@ import {
   MDExt,
 } from "@silverbulletmd/common/markdown_ext";
 import { FilterList } from "./components/filter";
-import { FilterOption, PageMeta } from "@silverbulletmd/common/types";
+import {
+  FilterOption,
+  PageMeta,
+  reservedPageNames,
+} from "@silverbulletmd/common/types";
 import { syntaxTree } from "@codemirror/language";
 import sandboxSyscalls from "@plugos/plugos/syscalls/sandbox";
 import { eventSyscalls } from "@plugos/plugos/syscalls/event";
@@ -204,6 +208,13 @@ export class Editor {
     this.focus();
 
     this.pageNavigator.subscribe(async (pageName, pos: number | string) => {
+      if (reservedPageNames.includes(pageName)) {
+        this.flashNotification(
+          `"${pageName}" is a reserved page name. It cannot be used.`,
+          "error"
+        );
+        return;
+      }
       console.log("Now navigating to", pageName, pos);
 
       if (!this.editorView) {
@@ -507,6 +518,7 @@ export class Editor {
           }
         ),
         pasteLinkExtension,
+        pasteAttachmentExtension(this.space),
         closeBrackets(),
       ],
     });

packages/web/editor_paste.ts

@@ -1,4 +1,5 @@
-import { ViewPlugin, ViewUpdate } from "@codemirror/view";
+import { EditorView, ViewPlugin, ViewUpdate } from "@codemirror/view";
+import { Space } from "@silverbulletmd/common/spaces/space";
 import { createImportSpecifier } from "typescript";
 
 const urlRegexp =
@@ -43,3 +44,51 @@ export const pasteLinkExtension = ViewPlugin.fromClass(
     }
   }
 );
+
+export function pasteAttachmentExtension(space: Space) {
+  return EditorView.domEventHandlers({
+    paste: (event: ClipboardEvent, editor) => {
+      let payload = [...event.clipboardData!.items];
+
+      if (!payload.length || payload.length === 0) {
+        return false;
+      }
+      let file = payload.find((item) => item.kind === "file");
+      if (!file) {
+        return false;
+      }
+      const fileType = file.type;
+      Promise.resolve()
+        .then(async () => {
+          let data = await file!.getAsFile()?.arrayBuffer();
+          let ext = fileType.split("/")[1];
+          let fileName = new Date()
+            .toISOString()
+            .split(".")[0]
+            .replace("T", "_")
+            .replaceAll(":", "-");
+          let finalFileName = prompt(
+            "File name for pasted attachment",
+            `${fileName}.${ext}`
+          );
+          if (!finalFileName) {
+            return;
+          }
+          await space.writeAttachment(finalFileName, data!);
+          let attachmentMarkdown = `[${finalFileName}](attachment/${finalFileName})`;
+          if (fileType.startsWith("image/")) {
+            attachmentMarkdown = `![](attachment/${finalFileName})`;
+          }
+          editor.dispatch({
+            changes: [
+              {
+                insert: attachmentMarkdown,
+                from: editor.state.selection.main.from,
+              },
+            ],
+          });
+        })
+        .catch(console.error);
+    },
+  });
+}

packages/web/index.html

@@ -2,6 +2,9 @@
 <html lang="en">
   <head>
     <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width" />
+    <base href="/" />
+
     <title>Silver Bullet</title>
     <style>
       html,
@@ -20,8 +23,6 @@
     <script type="module" src="boot.ts"></script>
     <link rel="manifest" href="manifest.json" />
     <link rel="icon" type="image/x-icon" href="images/favicon.gif" />
-    <meta charset="UTF-8" />
-    <meta name="viewport" content="width=device-width" />
   </head>
   <body>
     <div id="sb-root"></div>

packages/web/service_worker.ts

@@ -35,8 +35,10 @@ self.addEventListener("fetch", (event: any) => {
         return response;
       } else {
         if (
-          parsedUrl.pathname !== "/fs" &&
+          parsedUrl.pathname !== "/page" &&
-          !parsedUrl.pathname.startsWith("/fs/") &&
+          !parsedUrl.pathname.startsWith("/page/") &&
+          parsedUrl.pathname !== "/attachment" &&
+          !parsedUrl.pathname.startsWith("/attachment/") &&
           !parsedUrl.pathname.startsWith("/plug/")
         ) {
           return cache.match("/index.html");

scripts/build_website.sh

@@ -4,22 +4,22 @@ echo "Building silver bullet"
 npm run clean-build
 echo "Cleaning website build dir"
 rm -rf website_build
-mkdir -p website_build/fs/_plug
+mkdir -p website_build/page/_plug
 echo "Copying silverbullet runtime files"
 cp -r packages/web/dist/* website_build/
 echo "Copying netlify config files"
 cp website/{_redirects,_headers} website_build/
 
 echo "Copying website markdown files"
-cp -r website/* website_build/fs/
+cp -r website/* website_build/page/
-rm website_build/fs/{_redirects,_headers}
+rm website_build/page/{_redirects,_headers}
 
 echo "Copying standard set of plugs"
-cp packages/plugs/dist/* website_build/fs/_plug/
+cp packages/plugs/dist/* website_build/page/_plug/
 
 echo "Applying rename magic"
-find website_build/fs/ -depth -name "*.md" -exec sh -c 'mv "$1" "${1%.md}"' _ {} \;
+find website_build/page/ -depth -name "*.md" -exec sh -c 'mv "$1" "${1%.md}"' _ {} \;
-find website_build/fs/ -depth -name "*.plug.json" -exec sh -c 'mv "$1" "${1%.plug.json}"' _ {} \;
+find website_build/page/ -depth -name "*.plug.json" -exec sh -c 'mv "$1" "${1%.plug.json}"' _ {} \;
 
 echo "Generating file listing"
 node scripts/generate_fs_list.js > website_build/index.json

scripts/generate_fs_list.js

@@ -12,7 +12,7 @@ async function getFiles(dir) {
   return Array.prototype.concat(...files);
 }
 
-const rootDir = resolve("website_build/fs");
+const rootDir = resolve("website_build/page");
 
 getFiles(rootDir).then((files) => {
   files = files

website/CHANGELOG.md

@@ -3,6 +3,7 @@ An attempt at documenting of the changes/new features introduced in each (pre) r
 ---
 ## 0.0.33
 * Changed full-text search page prefix from `@search/` to `🔍` for the {[Search Space]} command.
+* `page`, `plug` and `attachment` are now _reserved page names_, you cannot name your pages these (you will get an error when explicitly navigating to them).
 
 ---
 

website/_headers

@@ -1,8 +1,8 @@
-/fs/_plug/*
+/page/_plug/*
   Content-Type: application/json
   Last-Modified: 0
   X-Permission: ro
-/fs/*
+/page/*
   Content-Type: text/markdown
   Last-Modified: 0
   X-Permission: rw

website/_redirects

@@ -1,3 +1,3 @@
-/fs   /index.json 200
+/page   /index.json 200
-/fs/* /empty.md   200
+/page/* /empty.md   200
 /*    /index.html 200