FS API encoding support

packages/common/spaces/disk_space_primitives.ts

@@ -9,7 +9,11 @@ import {
 } from "fs/promises";
 import * as path from "path";
 import { AttachmentMeta, PageMeta } from "../types";
-import { SpacePrimitives } from "./space_primitives";
+import {
+  AttachmentData,
+  AttachmentEncoding,
+  SpacePrimitives,
+} from "./space_primitives";
 import { Plug } from "@plugos/plugos/plug";
 import { realpathSync } from "fs";
 import mime from "mime-types";
@@ -199,20 +203,27 @@ export class DiskSpacePrimitives implements SpacePrimitives {
   }
 
   async readAttachment(
-    name: string
-  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> {
+    name: string,
+    encoding: AttachmentEncoding
+  ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> {
     const localPath = this.attachmentNameToPath(name);
-    let fileBuffer = await readFile(localPath);
+    let fileBuffer = await readFile(localPath, {
+      encoding: encoding === "dataurl" ? "base64" : null,
+    });
 
     try {
       const s = await stat(localPath);
+      let contentType = mime.lookup(name) || "application/octet-stream";
       return {
-        buffer: fileBuffer.buffer,
+        data:
+          encoding === "dataurl"
+            ? `data:${contentType};base64,${fileBuffer}`
+            : (fileBuffer as Buffer).buffer,
         meta: {
           name: name,
           lastModified: s.mtime.getTime(),
           size: s.size,
-          contentType: mime.lookup(name) || "application/octet-stream",
+          contentType: contentType,
           perm: "rw",
         },
       };
@@ -241,7 +252,7 @@ export class DiskSpacePrimitives implements SpacePrimitives {
 
   async writeAttachment(
     name: string,
-    blob: ArrayBuffer,
+    data: AttachmentData,
     selfUpdate?: boolean,
     lastModified?: number
   ): Promise<AttachmentMeta> {
@@ -251,7 +262,11 @@ export class DiskSpacePrimitives implements SpacePrimitives {
       await mkdir(path.dirname(localPath), { recursive: true });
 
       // Actually write the file
-      await writeFile(localPath, Buffer.from(blob));
+      if (typeof data === "string") {
+        await writeFile(localPath, data.split(",")[1], { encoding: "base64" });
+      } else {
+        await writeFile(localPath, Buffer.from(data));
+      }
 
       if (lastModified) {
         let d = new Date(lastModified);

packages/common/spaces/evented_space_primitives.ts

@@ -3,7 +3,11 @@ import { Plug } from "@plugos/plugos/plug";
 
 import { AttachmentMeta, PageMeta } from "../types";
 import { plugPrefix, trashPrefix } from "./constants";
-import { SpacePrimitives } from "./space_primitives";
+import {
+  AttachmentData,
+  AttachmentEncoding,
+  SpacePrimitives,
+} from "./space_primitives";
 
 export class EventedSpacePrimitives implements SpacePrimitives {
   constructor(private wrapped: SpacePrimitives, private eventHook: EventHook) {}
@@ -75,9 +79,10 @@ export class EventedSpacePrimitives implements SpacePrimitives {
   }
 
   readAttachment(
-    name: string
-  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> {
-    return this.wrapped.readAttachment(name);
+    name: string,
+    encoding: AttachmentEncoding
+  ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> {
+    return this.wrapped.readAttachment(name, encoding);
   }
 
   getAttachmentMeta(name: string): Promise<AttachmentMeta> {

packages/common/spaces/http_space_primitives.ts

@@ -1,6 +1,10 @@
 import { AttachmentMeta, PageMeta } from "../types";
 import { Plug } from "@plugos/plugos/plug";
-import { SpacePrimitives } from "./space_primitives";
+import {
+  AttachmentData,
+  AttachmentEncoding,
+  SpacePrimitives,
+} from "./space_primitives";
 
 export class HttpSpacePrimitives implements SpacePrimitives {
   fsUrl: string;
@@ -145,8 +149,9 @@ export class HttpSpacePrimitives implements SpacePrimitives {
   }
 
   async readAttachment(
-    name: string
-  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> {
+    name: string,
+    encoding: AttachmentEncoding
+  ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> {
     let res = await this.authenticatedFetch(`${this.fsaUrl}/${name}`, {
       method: "GET",
     });
@@ -155,25 +160,29 @@ export class HttpSpacePrimitives implements SpacePrimitives {
     }
     let blob = await res.blob();
     return {
-      buffer: await blob.arrayBuffer(),
+      data:
+        encoding === "arraybuffer"
+          ? await blob.arrayBuffer()
+          : arrayBufferToDataUrl(await blob.arrayBuffer()),
       meta: this.responseToAttachmentMeta(name, res),
     };
   }
 
   async writeAttachment(
     name: string,
-    buffer: ArrayBuffer,
+    data: AttachmentData,
     selfUpdate?: boolean,
     lastModified?: number
   ): Promise<AttachmentMeta> {
-    // TODO: lastModified ignored for now
+    if (typeof data === "string") {
+      data = dataUrlToArrayBuffer(data);
+    }
     let res = await this.authenticatedFetch(`${this.fsaUrl}/${name}`, {
       method: "PUT",
-      body: buffer,
+      body: data,
       headers: {
         "Last-Modified": lastModified ? "" + lastModified : undefined,
         "Content-type": "application/octet-stream",
-        "Content-length": "" + buffer.byteLength,
       },
     });
     const newMeta = this.responseToAttachmentMeta(name, res);
@@ -268,3 +277,23 @@ export class HttpSpacePrimitives implements SpacePrimitives {
     };
   }
 }
+
+function dataUrlToArrayBuffer(dataUrl: string): ArrayBuffer {
+  var binary_string = window.atob(dataUrl.split(",")[1]);
+  var len = binary_string.length;
+  var bytes = new Uint8Array(len);
+  for (var i = 0; i < len; i++) {
+    bytes[i] = binary_string.charCodeAt(i);
+  }
+  return bytes.buffer;
+}
+
+function arrayBufferToDataUrl(buffer: ArrayBuffer): string {
+  var binary = "";
+  var bytes = new Uint8Array(buffer);
+  var len = bytes.byteLength;
+  for (var i = 0; i < len; i++) {
+    binary += String.fromCharCode(bytes[i]);
+  }
+  return `data:application/octet-stream,${window.btoa(binary)}`;
+}

packages/common/spaces/indexeddb_space_primitives.ts

@@ -1,4 +1,8 @@
-import { SpacePrimitives } from "./space_primitives";
+import {
+  AttachmentData,
+  AttachmentEncoding,
+  SpacePrimitives,
+} from "./space_primitives";
 import { AttachmentMeta, PageMeta } from "../types";
 import Dexie, { Table } from "dexie";
 import { Plug } from "@plugos/plugos/plug";
@@ -26,8 +30,9 @@ export class IndexedDBSpacePrimitives implements SpacePrimitives {
     throw new Error("Method not implemented.");
   }
   readAttachment(
-    name: string
-  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> {
+    name: string,
+    encoding: AttachmentEncoding
+  ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> {
     throw new Error("Method not implemented.");
   }
   getAttachmentMeta(name: string): Promise<AttachmentMeta> {

packages/common/spaces/space.ts

@@ -1,4 +1,8 @@
-import { SpacePrimitives } from "./space_primitives";
+import {
+  AttachmentData,
+  AttachmentEncoding,
+  SpacePrimitives,
+} from "./space_primitives";
 import { AttachmentMeta, PageMeta } from "../types";
 import { EventEmitter } from "@plugos/plugos/event";
 import { Plug } from "@plugos/plugos/plug";
@@ -225,20 +229,21 @@ export class Space
     return this.space.fetchAttachmentList();
   }
   readAttachment(
-    name: string
-  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> {
-    return this.space.readAttachment(name);
+    name: string,
+    encoding: AttachmentEncoding
+  ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> {
+    return this.space.readAttachment(name, encoding);
   }
   getAttachmentMeta(name: string): Promise<AttachmentMeta> {
     return this.space.getAttachmentMeta(name);
   }
   writeAttachment(
     name: string,
-    blob: ArrayBuffer,
+    data: AttachmentData,
     selfUpdate?: boolean | undefined,
     lastModified?: number | undefined
   ): Promise<AttachmentMeta> {
-    return this.space.writeAttachment(name, blob, selfUpdate, lastModified);
+    return this.space.writeAttachment(name, data, selfUpdate, lastModified);
   }
   deleteAttachment(name: string): Promise<void> {
     return this.space.deleteAttachment(name);

packages/common/spaces/space_primitives.ts

@@ -1,6 +1,8 @@
 import { Plug } from "@plugos/plugos/plug";
 import { AttachmentMeta, PageMeta } from "../types";
 
+export type AttachmentEncoding = "arraybuffer" | "dataurl";
+export type AttachmentData = ArrayBuffer | string;
 export interface SpacePrimitives {
   // Pages
   fetchPageList(): Promise<{ pages: Set<PageMeta>; nowTimestamp: number }>;
@@ -20,12 +22,13 @@ export interface SpacePrimitives {
     nowTimestamp: number;
   }>;
   readAttachment(
-    name: string
-  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }>;
+    name: string,
+    encoding: AttachmentEncoding
+  ): Promise<{ data: AttachmentData; meta: AttachmentMeta }>;
   getAttachmentMeta(name: string): Promise<AttachmentMeta>;
   writeAttachment(
     name: string,
-    blob: ArrayBuffer,
+    data: AttachmentData,
     selfUpdate?: boolean,
     lastModified?: number
   ): Promise<AttachmentMeta>;

packages/plugos-syscall/fs.ts

@@ -6,17 +6,22 @@ export type FileMeta = {
 };
 
 export async function readFile(
-  path: string
+  path: string,
+  encoding: "utf8" | "dataurl" = "utf8"
 ): Promise<{ text: string; meta: FileMeta }> {
-  return syscall("fs.readFile", path);
+  return syscall("fs.readFile", path, encoding);
 }
 
 export async function getFileMeta(path: string): Promise<FileMeta> {
   return syscall("fs.getFileMeta", path);
 }
 
-export async function writeFile(path: string, text: string): Promise<FileMeta> {
-  return syscall("fs.writeFile", path, text);
+export async function writeFile(
+  path: string,
+  text: string,
+  encoding: "utf8" | "dataurl" = "utf8"
+): Promise<FileMeta> {
+  return syscall("fs.writeFile", path, text, encoding);
 }
 
 export async function deleteFile(path: string): Promise<void> {

packages/plugos/syscalls/fs.node.ts

@@ -19,10 +19,16 @@ export default function fileSystemSyscalls(root: string = "/"): SysCallMapping {
   return {
     "fs.readFile": async (
       ctx,
-      filePath: string
+      filePath: string,
+      encoding: "utf8" | "dataurl" = "utf8"
     ): Promise<{ text: string; meta: FileMeta }> => {
       let p = resolvedPath(filePath);
-      let text = await readFile(p, "utf8");
+      let text = "";
+      if (encoding === "utf8") {
+        text = await readFile(p, "utf8");
+      } else {
+        text = `data:application/octet-stream,${await readFile(p, "base64")}`;
+      }
       let s = await stat(p);
       return {
         text,
@@ -43,11 +49,18 @@ export default function fileSystemSyscalls(root: string = "/"): SysCallMapping {
     "fs.writeFile": async (
       ctx,
       filePath: string,
-      text: string
+      text: string,
+      encoding: "utf8" | "dataurl" = "utf8"
     ): Promise<FileMeta> => {
       let p = resolvedPath(filePath);
       await mkdir(path.dirname(p), { recursive: true });
-      await writeFile(p, text);
+      if (encoding === "utf8") {
+        await writeFile(p, text);
+      } else {
+        await writeFile(p, text.split(",")[1], {
+          encoding: "base64",
+        });
+      }
       let s = await stat(p);
       return {
         name: filePath,

packages/server/express_server.ts

@@ -491,13 +491,16 @@ export class ExpressServer {
         }
         console.log("Getting", attachmentName);
         try {
-          let attachmentData = await this.space.readAttachment(attachmentName);
+          let attachmentData = await this.space.readAttachment(
+            attachmentName,
+            "arraybuffer"
+          );
           res.status(200);
           res.header("Last-Modified", "" + attachmentData.meta.lastModified);
           res.header("X-Permission", attachmentData.meta.perm);
           res.header("Content-Type", attachmentData.meta.contentType);
           // res.header("X-Content-Length", "" + attachmentData.meta.size);
-          res.send(Buffer.from(attachmentData.buffer));
+          res.send(Buffer.from(attachmentData.data as ArrayBuffer));
         } catch (e) {
           // CORS
           res.status(200);

packages/server/hooks/plug_space_primitives.ts

@@ -1,5 +1,9 @@
 import { Plug } from "@plugos/plugos/plug";
-import { SpacePrimitives } from "@silverbulletmd/common/spaces/space_primitives";
+import {
+  AttachmentData,
+  AttachmentEncoding,
+  SpacePrimitives,
+} from "@silverbulletmd/common/spaces/space_primitives";
 import { AttachmentMeta, PageMeta } from "@silverbulletmd/common/types";
 import { PageNamespaceHook, PageNamespaceOperation } from "./page_namespace";
 
@@ -99,9 +103,10 @@ export class PlugSpacePrimitives implements SpacePrimitives {
     return this.wrapped.fetchAttachmentList();
   }
   readAttachment(
-    name: string
-  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> {
-    return this.wrapped.readAttachment(name);
+    name: string,
+    encoding: AttachmentEncoding
+  ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> {
+    return this.wrapped.readAttachment(name, encoding);
   }
   getAttachmentMeta(name: string): Promise<AttachmentMeta> {
     return this.wrapped.getAttachmentMeta(name);

packages/server/syscalls/space.ts

@@ -1,6 +1,7 @@
 import { AttachmentMeta, PageMeta } from "@silverbulletmd/common/types";
 import { SysCallMapping } from "@plugos/plugos/system";
 import { Space } from "@silverbulletmd/common/spaces/space";
+import { AttachmentData } from "@silverbulletmd/common/spaces/space_primitives";
 
 export default (space: Space): SysCallMapping => {
   return {
@@ -32,8 +33,8 @@ export default (space: Space): SysCallMapping => {
     "space.readAttachment": async (
       ctx,
       name: string
-    ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> => {
-      return await space.readAttachment(name);
+    ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> => {
+      return await space.readAttachment(name, "dataurl");
     },
     "space.getAttachmentMeta": async (
       ctx,
@@ -44,9 +45,9 @@ export default (space: Space): SysCallMapping => {
     "space.writeAttachment": async (
       ctx,
       name: string,
-      buffer: ArrayBuffer
+      data: string
     ): Promise<AttachmentMeta> => {
-      return await space.writeAttachment(name, buffer);
+      return await space.writeAttachment(name, data);
     },
     "space.deleteAttachment": async (ctx, name: string) => {
       await space.deleteAttachment(name);

packages/web/syscalls/space.ts

@@ -1,6 +1,7 @@
 import { Editor } from "../editor";
 import { SysCallMapping } from "@plugos/plugos/system";
 import { AttachmentMeta, PageMeta } from "@silverbulletmd/common/types";
+import { AttachmentData } from "@silverbulletmd/common/spaces/space_primitives";
 
 export function spaceSyscalls(editor: Editor): SysCallMapping {
   return {
@@ -39,8 +40,8 @@ export function spaceSyscalls(editor: Editor): SysCallMapping {
     "space.readAttachment": async (
       ctx,
       name: string
-    ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> => {
-      return await editor.space.readAttachment(name);
+    ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> => {
+      return await editor.space.readAttachment(name, "dataurl");
     },
     "space.getAttachmentMeta": async (
       ctx,