FS API encoding support

packages/common/spaces/disk_space_primitives.ts

@@ -9,7 +9,11 @@ import {
 } from "fs/promises";
 import * as path from "path";
 import { AttachmentMeta, PageMeta } from "../types";
-import { SpacePrimitives } from "./space_primitives";
+import {
+  AttachmentData,
+  AttachmentEncoding,
+  SpacePrimitives,
+} from "./space_primitives";
 import { Plug } from "@plugos/plugos/plug";
 import { realpathSync } from "fs";
 import mime from "mime-types";
@@ -199,20 +203,27 @@ export class DiskSpacePrimitives implements SpacePrimitives {
   }
 
   async readAttachment(
-    name: string
-  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> {
+    name: string,
+    encoding: AttachmentEncoding
+  ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> {
     const localPath = this.attachmentNameToPath(name);
-    let fileBuffer = await readFile(localPath);
+    let fileBuffer = await readFile(localPath, {
+      encoding: encoding === "dataurl" ? "base64" : null,
+    });
 
     try {
       const s = await stat(localPath);
+      let contentType = mime.lookup(name) || "application/octet-stream";
       return {
-        buffer: fileBuffer.buffer,
+        data:
+          encoding === "dataurl"
+            ? `data:${contentType};base64,${fileBuffer}`
+            : (fileBuffer as Buffer).buffer,
         meta: {
           name: name,
           lastModified: s.mtime.getTime(),
           size: s.size,
-          contentType: mime.lookup(name) || "application/octet-stream",
+          contentType: contentType,
           perm: "rw",
         },
       };
@@ -241,7 +252,7 @@ export class DiskSpacePrimitives implements SpacePrimitives {
 
   async writeAttachment(
     name: string,
-    blob: ArrayBuffer,
+    data: AttachmentData,
     selfUpdate?: boolean,
     lastModified?: number
   ): Promise<AttachmentMeta> {
@@ -251,7 +262,11 @@ export class DiskSpacePrimitives implements SpacePrimitives {
       await mkdir(path.dirname(localPath), { recursive: true });
 
       // Actually write the file
-      await writeFile(localPath, Buffer.from(blob));
+      if (typeof data === "string") {
+        await writeFile(localPath, data.split(",")[1], { encoding: "base64" });
+      } else {
+        await writeFile(localPath, Buffer.from(data));
+      }
 
       if (lastModified) {
         let d = new Date(lastModified);

packages/common/spaces/evented_space_primitives.ts

@@ -3,7 +3,11 @@ import { Plug } from "@plugos/plugos/plug";
 
 import { AttachmentMeta, PageMeta } from "../types";
 import { plugPrefix, trashPrefix } from "./constants";
-import { SpacePrimitives } from "./space_primitives";
+import {
+  AttachmentData,
+  AttachmentEncoding,
+  SpacePrimitives,
+} from "./space_primitives";
 
 export class EventedSpacePrimitives implements SpacePrimitives {
   constructor(private wrapped: SpacePrimitives, private eventHook: EventHook) {}
@@ -75,9 +79,10 @@ export class EventedSpacePrimitives implements SpacePrimitives {
   }
 
   readAttachment(
-    name: string
-  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> {
-    return this.wrapped.readAttachment(name);
+    name: string,
+    encoding: AttachmentEncoding
+  ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> {
+    return this.wrapped.readAttachment(name, encoding);
   }
 
   getAttachmentMeta(name: string): Promise<AttachmentMeta> {

packages/common/spaces/http_space_primitives.ts

@@ -1,6 +1,10 @@
 import { AttachmentMeta, PageMeta } from "../types";
 import { Plug } from "@plugos/plugos/plug";
-import { SpacePrimitives } from "./space_primitives";
+import {
+  AttachmentData,
+  AttachmentEncoding,
+  SpacePrimitives,
+} from "./space_primitives";
 
 export class HttpSpacePrimitives implements SpacePrimitives {
   fsUrl: string;
@@ -145,8 +149,9 @@ export class HttpSpacePrimitives implements SpacePrimitives {
   }
 
   async readAttachment(
-    name: string
-  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> {
+    name: string,
+    encoding: AttachmentEncoding
+  ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> {
     let res = await this.authenticatedFetch(`${this.fsaUrl}/${name}`, {
       method: "GET",
     });
@@ -155,25 +160,29 @@ export class HttpSpacePrimitives implements SpacePrimitives {
     }
     let blob = await res.blob();
     return {
-      buffer: await blob.arrayBuffer(),
+      data:
+        encoding === "arraybuffer"
+          ? await blob.arrayBuffer()
+          : arrayBufferToDataUrl(await blob.arrayBuffer()),
       meta: this.responseToAttachmentMeta(name, res),
     };
   }
 
   async writeAttachment(
     name: string,
-    buffer: ArrayBuffer,
+    data: AttachmentData,
     selfUpdate?: boolean,
     lastModified?: number
   ): Promise<AttachmentMeta> {
-    // TODO: lastModified ignored for now
+    if (typeof data === "string") {
+      data = dataUrlToArrayBuffer(data);
+    }
     let res = await this.authenticatedFetch(`${this.fsaUrl}/${name}`, {
       method: "PUT",
-      body: buffer,
+      body: data,
       headers: {
         "Last-Modified": lastModified ? "" + lastModified : undefined,
         "Content-type": "application/octet-stream",
-        "Content-length": "" + buffer.byteLength,
       },
     });
     const newMeta = this.responseToAttachmentMeta(name, res);
@@ -268,3 +277,23 @@ export class HttpSpacePrimitives implements SpacePrimitives {
     };
   }
 }
+
+function dataUrlToArrayBuffer(dataUrl: string): ArrayBuffer {
+  var binary_string = window.atob(dataUrl.split(",")[1]);
+  var len = binary_string.length;
+  var bytes = new Uint8Array(len);
+  for (var i = 0; i < len; i++) {
+    bytes[i] = binary_string.charCodeAt(i);
+  }
+  return bytes.buffer;
+}
+
+function arrayBufferToDataUrl(buffer: ArrayBuffer): string {
+  var binary = "";
+  var bytes = new Uint8Array(buffer);
+  var len = bytes.byteLength;
+  for (var i = 0; i < len; i++) {
+    binary += String.fromCharCode(bytes[i]);
+  }
+  return `data:application/octet-stream,${window.btoa(binary)}`;
+}

packages/common/spaces/indexeddb_space_primitives.ts

@@ -1,4 +1,8 @@
-import { SpacePrimitives } from "./space_primitives";
+import {
+  AttachmentData,
+  AttachmentEncoding,
+  SpacePrimitives,
+} from "./space_primitives";
 import { AttachmentMeta, PageMeta } from "../types";
 import Dexie, { Table } from "dexie";
 import { Plug } from "@plugos/plugos/plug";
@@ -26,8 +30,9 @@ export class IndexedDBSpacePrimitives implements SpacePrimitives {
     throw new Error("Method not implemented.");
   }
   readAttachment(
-    name: string
-  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> {
+    name: string,
+    encoding: AttachmentEncoding
+  ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> {
     throw new Error("Method not implemented.");
   }
   getAttachmentMeta(name: string): Promise<AttachmentMeta> {

packages/common/spaces/space.ts

@@ -1,4 +1,8 @@
-import { SpacePrimitives } from "./space_primitives";
+import {
+  AttachmentData,
+  AttachmentEncoding,
+  SpacePrimitives,
+} from "./space_primitives";
 import { AttachmentMeta, PageMeta } from "../types";
 import { EventEmitter } from "@plugos/plugos/event";
 import { Plug } from "@plugos/plugos/plug";
@@ -225,20 +229,21 @@ export class Space
     return this.space.fetchAttachmentList();
   }
   readAttachment(
-    name: string
-  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> {
-    return this.space.readAttachment(name);
+    name: string,
+    encoding: AttachmentEncoding
+  ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> {
+    return this.space.readAttachment(name, encoding);
   }
   getAttachmentMeta(name: string): Promise<AttachmentMeta> {
     return this.space.getAttachmentMeta(name);
   }
   writeAttachment(
     name: string,
-    blob: ArrayBuffer,
+    data: AttachmentData,
     selfUpdate?: boolean | undefined,
     lastModified?: number | undefined
   ): Promise<AttachmentMeta> {
-    return this.space.writeAttachment(name, blob, selfUpdate, lastModified);
+    return this.space.writeAttachment(name, data, selfUpdate, lastModified);
   }
   deleteAttachment(name: string): Promise<void> {
     return this.space.deleteAttachment(name);

packages/common/spaces/space_primitives.ts

@@ -1,6 +1,8 @@
 import { Plug } from "@plugos/plugos/plug";
 import { AttachmentMeta, PageMeta } from "../types";
 
+export type AttachmentEncoding = "arraybuffer" | "dataurl";
+export type AttachmentData = ArrayBuffer | string;
 export interface SpacePrimitives {
   // Pages
   fetchPageList(): Promise<{ pages: Set<PageMeta>; nowTimestamp: number }>;
@@ -20,12 +22,13 @@ export interface SpacePrimitives {
     nowTimestamp: number;
   }>;
   readAttachment(
-    name: string
-  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }>;
+    name: string,
+    encoding: AttachmentEncoding
+  ): Promise<{ data: AttachmentData; meta: AttachmentMeta }>;
   getAttachmentMeta(name: string): Promise<AttachmentMeta>;
   writeAttachment(
     name: string,
-    blob: ArrayBuffer,
+    data: AttachmentData,
     selfUpdate?: boolean,
     lastModified?: number
   ): Promise<AttachmentMeta>;

packages/plugos-syscall/fs.ts

@@ -6,17 +6,22 @@ export type FileMeta = {
 };
 
 export async function readFile(
-  path: string
+  path: string,
+  encoding: "utf8" | "dataurl" = "utf8"
 ): Promise<{ text: string; meta: FileMeta }> {
-  return syscall("fs.readFile", path);
+  return syscall("fs.readFile", path, encoding);
 }
 
 export async function getFileMeta(path: string): Promise<FileMeta> {
   return syscall("fs.getFileMeta", path);
 }
 
-export async function writeFile(path: string, text: string): Promise<FileMeta> {
-  return syscall("fs.writeFile", path, text);
+export async function writeFile(
+  path: string,
+  text: string,
+  encoding: "utf8" | "dataurl" = "utf8"
+): Promise<FileMeta> {
+  return syscall("fs.writeFile", path, text, encoding);
 }
 
 export async function deleteFile(path: string): Promise<void> {

packages/plugos/syscalls/fs.node.ts

@@ -19,10 +19,16 @@ export default function fileSystemSyscalls(root: string = "/"): SysCallMapping {
   return {
     "fs.readFile": async (
       ctx,
-      filePath: string
+      filePath: string,
+      encoding: "utf8" | "dataurl" = "utf8"
     ): Promise<{ text: string; meta: FileMeta }> => {
       let p = resolvedPath(filePath);
-      let text = await readFile(p, "utf8");
+      let text = "";
+      if (encoding === "utf8") {
+        text = await readFile(p, "utf8");
+      } else {
+        text = `data:application/octet-stream,${await readFile(p, "base64")}`;
+      }
       let s = await stat(p);
       return {
         text,
@@ -43,11 +49,18 @@ export default function fileSystemSyscalls(root: string = "/"): SysCallMapping {
     "fs.writeFile": async (
       ctx,
       filePath: string,
-      text: string
+      text: string,
+      encoding: "utf8" | "dataurl" = "utf8"
     ): Promise<FileMeta> => {
       let p = resolvedPath(filePath);
       await mkdir(path.dirname(p), { recursive: true });
-      await writeFile(p, text);
+      if (encoding === "utf8") {
+        await writeFile(p, text);
+      } else {
+        await writeFile(p, text.split(",")[1], {
+          encoding: "base64",
+        });
+      }
       let s = await stat(p);
       return {
         name: filePath,

packages/server/express_server.ts

@@ -491,13 +491,16 @@ export class ExpressServer {
         }
         console.log("Getting", attachmentName);
         try {
-          let attachmentData = await this.space.readAttachment(attachmentName);
+          let attachmentData = await this.space.readAttachment(
+            attachmentName,
+            "arraybuffer"
+          );
           res.status(200);
           res.header("Last-Modified", "" + attachmentData.meta.lastModified);
           res.header("X-Permission", attachmentData.meta.perm);
           res.header("Content-Type", attachmentData.meta.contentType);
           // res.header("X-Content-Length", "" + attachmentData.meta.size);
-          res.send(Buffer.from(attachmentData.buffer));
+          res.send(Buffer.from(attachmentData.data as ArrayBuffer));
         } catch (e) {
           // CORS
           res.status(200);

packages/server/hooks/plug_space_primitives.ts

@@ -1,5 +1,9 @@
 import { Plug } from "@plugos/plugos/plug";
-import { SpacePrimitives } from "@silverbulletmd/common/spaces/space_primitives";
+import {
+  AttachmentData,
+  AttachmentEncoding,
+  SpacePrimitives,
+} from "@silverbulletmd/common/spaces/space_primitives";
 import { AttachmentMeta, PageMeta } from "@silverbulletmd/common/types";
 import { PageNamespaceHook, PageNamespaceOperation } from "./page_namespace";
 
@@ -99,9 +103,10 @@ export class PlugSpacePrimitives implements SpacePrimitives {
     return this.wrapped.fetchAttachmentList();
   }
   readAttachment(
-    name: string
-  ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> {
-    return this.wrapped.readAttachment(name);
+    name: string,
+    encoding: AttachmentEncoding
+  ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> {
+    return this.wrapped.readAttachment(name, encoding);
   }
   getAttachmentMeta(name: string): Promise<AttachmentMeta> {
     return this.wrapped.getAttachmentMeta(name);

packages/server/syscalls/space.ts

@@ -1,6 +1,7 @@
 import { AttachmentMeta, PageMeta } from "@silverbulletmd/common/types";
 import { SysCallMapping } from "@plugos/plugos/system";
 import { Space } from "@silverbulletmd/common/spaces/space";
+import { AttachmentData } from "@silverbulletmd/common/spaces/space_primitives";
 
 export default (space: Space): SysCallMapping => {
   return {
@@ -32,8 +33,8 @@ export default (space: Space): SysCallMapping => {
     "space.readAttachment": async (
       ctx,
       name: string
-    ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> => {
-      return await space.readAttachment(name);
+    ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> => {
+      return await space.readAttachment(name, "dataurl");
     },
     "space.getAttachmentMeta": async (
       ctx,
@@ -44,9 +45,9 @@ export default (space: Space): SysCallMapping => {
     "space.writeAttachment": async (
       ctx,
       name: string,
-      buffer: ArrayBuffer
+      data: string
     ): Promise<AttachmentMeta> => {
-      return await space.writeAttachment(name, buffer);
+      return await space.writeAttachment(name, data);
     },
     "space.deleteAttachment": async (ctx, name: string) => {
       await space.deleteAttachment(name);

packages/web/syscalls/space.ts

@@ -1,6 +1,7 @@
 import { Editor } from "../editor";
 import { SysCallMapping } from "@plugos/plugos/system";
 import { AttachmentMeta, PageMeta } from "@silverbulletmd/common/types";
+import { AttachmentData } from "@silverbulletmd/common/spaces/space_primitives";
 
 export function spaceSyscalls(editor: Editor): SysCallMapping {
   return {
@@ -39,8 +40,8 @@ export function spaceSyscalls(editor: Editor): SysCallMapping {
     "space.readAttachment": async (
       ctx,
       name: string
-    ): Promise<{ buffer: ArrayBuffer; meta: AttachmentMeta }> => {
-      return await editor.space.readAttachment(name);
+    ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> => {
+      return await editor.space.readAttachment(name, "dataurl");
     },
     "space.getAttachmentMeta": async (
       ctx,

website/CHANGELOG.md

@@ -1,9 +1,9 @@
-An attempt at documenting of the changes/new features introduced in each 
+An attempt at documenting of the changes/new features introduced in each release.
 
 ---
 
 ## 0.0.33
-* **Attachments**: you can now copy & paste or drag & drop files (images, PDF, whatever you like) into a page and it will be uploaded and appropriately linked from your page. Attachment size is currently limited to 100mb.
+* **Attachments**: you can now copy & paste, or drag & drop files (images, PDF, whatever you like) into a page and it will be uploaded and appropriately linked from your page. Attachment size is currently limited to 100mb.
 * Changed full-text search page prefix from `@search/` to `🔍` for the {[Search Space]} command.
 * `page`, `plug` and `attachment` are now _reserved page names_, you cannot name your pages these (you will get an error when explicitly navigating to them).
 

website/PUBLISH.md

@@ -0,0 +1,7 @@
+```yaml
+title: Silver Bullet
+publishAll: true
+indexPage: Silver Bullet
+footerPage: website-footer
+```
+

website/Sandbox.md

@@ -0,0 +1,27 @@
+What does Silver Bullet look like? Well, have a look around. **You’re looking at it at this very moment!** 🤯
+
+Note that what you’re looking at is not a fully functional version, because the _back-end is read-only_. That said, it should give you some feel for what it’s like to use SB before making the commitment of running a single `npx` command (see below) to download and run it locally in its fully functioning mode.
+
+## Start playing
+So, feel free to make some edits in this space. Don’t worry, you won’t break anything, nothing is saved (just reload the page to see).
+
+Here are some things to try:
+
+* Click on the page title (`index` for this particular one) at the top, or hit `Cmd-k` (Mac) or `Ctrl-k` (Linux and Windows) to open the **page switcher**. Type the a name of a non-existing page to create it (although it won’t save in this environment).
+* Click on the run button (top right) or hit `Cmd-/` (Mac) or `Ctrl-/` (Linux and Windows) to open the **command palette** (note not all commands will work in this quasi read-only mode).
+* Select some text and hit `Alt-m` to ==highlight== it, or `Cmd-b` (Mac) or `Ctrl-b` to make it **bold**.
+* Click a link somewhere in this page to navigate there.
+* Start typing `[[` somewhere to insert a page link (with completion).
+* [ ] Tap this box 👈 to mark this task as done.
+* Start typing `:party` to trigger the emoji picker 🎉
+* Type `/` somewhere in the text to invoke a **slash command**.
+* Hit `Cmd-p` (Mac) or `Ctrl-p` (Windows, Linux) to show a live preview for the current page on the side, if your brain doesn’t speak native Markdown yet.
+* Open this site on your phone or tablet and… it just works!
+* Are you using a browser with **PWA support** (e.g. any Chromium-based browser)? Click on that little icon to the right of your location bar that says “Install Silver Bullet” to give SB its own window frame and desktop icon, like it is a stand-alone app (not particularly useful on silverbullet.md, but definitely do this once you install it yourself).
+
+There are a few features you don’t get to fully experience in this environment, because they rely on a working back-end, such as:
+
+* Using SB’s powerful page indexing and **query mechanism** where part of pages are automatically rendered and kept up to date by querying various data sources (such as pages and their metadata, back links, tasks embedded in pages, and list items) with an SQL like syntax, rendered with handlebars templates.
+* Intelligent **page renaming**, automatically updating any pages that link to it.
+* **Full text search**.
+* **Extending** and updating SB’s functionality by installing additional [[🔌 Plugs]] (SB parlance for plug-ins) and writing your own.

website/Silver Bullet.md

@@ -1,37 +1,12 @@
-# Silver Bullet
 ## Markdown as a platform
 Silver Bullet (SB) is highly-extensible, [open source](https://github.com/silverbulletmd/silverbullet) **personal knowledge management** software. Indeed, that’s fancy language for “a note taking app with links.”
 
+Here is a screenshot:
+
+![Silver Bullet PWA screenshot](attachment/silverbullet-pwa.png)
+
 At its core, SB is a Markdown editor that stores _pages_ (notes) as plain markdown files in a folder referred to as a _space_. Pages can be cross-linked using the `[[link to other page]]` syntax. However, once you leverage its various extensions (called _plugs_) it can feel more like a _knowledge platform_, allowing you to annotate, combine and query your accumulated knowledge in creative ways, specific to you. To get a good feel for it, [watch this video](https://youtu.be/RYdc3UF9gok).
 
-What does Silver Bullet look like? Well, have a look around. **You’re looking at it at this very moment!** 🤯
-
-Note that what you’re looking at is not a fully functional version, because the _back-end is read-only_. That said, it should give you some feel for what it’s like to use SB before making the commitment of running a single `npx` command (see below) to download and run it locally in its fully functioning mode.
-
-## Start playing
-So, feel free to make some edits in this space. Don’t worry, you won’t break anything, nothing is saved (just reload the page to see).
-
-Here are some things to try:
-
-* Click on the page title (`index` for this particular one) at the top, or hit `Cmd-k` (Mac) or `Ctrl-k` (Linux and Windows) to open the **page switcher**. Type the a name of a non-existing page to create it (although it won’t save in this environment).
-* Click on the run button (top right) or hit `Cmd-/` (Mac) or `Ctrl-/` (Linux and Windows) to open the **command palette** (note not all commands will work in this quasi read-only mode).
-* Select some text and hit `Alt-m` to ==highlight== it, or `Cmd-b` (Mac) or `Ctrl-b` to make it **bold**.
-* Click a link somewhere in this page to navigate there.
-* Start typing `[[` somewhere to insert a page link (with completion).
-* [ ] Tap this box 👈 to mark this task as done.
-* Start typing `:party` to trigger the emoji picker 🎉
-* Type `/` somewhere in the text to invoke a **slash command**.
-* Hit `Cmd-p` (Mac) or `Ctrl-p` (Windows, Linux) to show a live preview for the current page on the side, if your brain doesn’t speak native Markdown yet.
-* Open this site on your phone or tablet and… it just works!
-* Are you using a browser with **PWA support** (e.g. any Chromium-based browser)? Click on that little icon to the right of your location bar that says “Install Silver Bullet” to give SB its own window frame and desktop icon, like it is a stand-alone app (not particularly useful on silverbullet.md, but definitely do this once you install it yourself).
-
-There are a few features you don’t get to fully experience in this environment, because they rely on a working back-end, such as:
-
-* Using SB’s powerful page indexing and **query mechanism** where part of pages are automatically rendered and kept up to date by querying various data sources (such as pages and their metadata, back links, tasks embedded in pages, and list items) with an SQL like syntax, rendered with handlebars templates.
-* Intelligent **page renaming**, automatically updating any pages that link to it.
-* **Full text search**.
-* **Extending** and updating SB’s functionality by installing additional [[🔌 Plugs]] (SB parlance for plug-ins) and writing your own.
-
 ## Extensions
 What type of extensions, you ask? Let us demonstrate this in a very meta way: by querying a list of plugs and injecting it into this page!
 
@@ -53,11 +28,11 @@ In a regular SB installation, the body of this query 👆 (in between the placeh
 ## Explore more
 Click on the links below to explore various aspects of Silver Bullet more in-depth:
 
-[[CHANGELOG]]
-[[🤯 Features]]
-[[💡 Inspiration]]
-[[🔌 Plugs]]
-[[🔨 Development]]
+* [[CHANGELOG]]
+* [[🤯 Features]]
+* [[💡 Inspiration]]
+* [[🔌 Plugs]]
+* [[🔨 Development]]
 
 More of a video person? Here’s two to get you started:
 

website/website-footer.md

@@ -0,0 +1,3 @@
+---
+©️ [[Silver Bullet]] authors.  
+_Published with [Silver Bullet Publish](https://github.com/silverbulletmd/silverbullet-publish)._