E2E encryption (prototype) (#601)

Prototype E2E encryption

cmd/server.ts

@@ -22,6 +22,7 @@ export async function serveCommand(
     key?: string;
     reindex?: boolean;
     syncOnly?: boolean;
+    clientEncryption?: boolean;
   },
   folder?: string,
 ) {
@@ -29,7 +30,22 @@ export async function serveCommand(
     "127.0.0.1";
   const port = options.port ||
     (Deno.env.get("SB_PORT") && +Deno.env.get("SB_PORT")!) || 3000;
+
+  const clientEncryption = options.clientEncryption ||
+    !!Deno.env.get("SB_CLIENT_ENCRYPTION");
+
+  if (clientEncryption) {
+    console.log(
+      "Running in client encryption mode, this will implicitly enable sync-only mode",
+    );
+  }
+
   const syncOnly = options.syncOnly || !!Deno.env.get("SB_SYNC_ONLY");
+
+  if (syncOnly) {
+    console.log("Running in sync-only mode (no backend processing)");
+  }
+
   const app = new Application();
 
   if (!folder) {
@@ -69,6 +85,8 @@ To allow outside connections, pass -L 0.0.0.0 as a flag, and put a TLS terminato
     namespace: "*",
     auth: userCredentials,
     authToken: Deno.env.get("SB_AUTH_TOKEN"),
+    syncOnly,
+    clientEncryption,
     pagesPath: folder,
   });
 
@@ -79,7 +97,6 @@ To allow outside connections, pass -L 0.0.0.0 as a flag, and put a TLS terminato
     clientAssetBundle: new AssetBundle(clientAssetBundle as AssetJson),
     plugAssetBundle: new AssetBundle(plugAssetBundle as AssetJson),
     baseKvPrimitives,
-    syncOnly,
     keyFile: options.key,
     certFile: options.cert,
     configs,

common/crypto/aes.test.ts

@@ -1,33 +0,0 @@
-import { assertEquals } from "../../test_deps.ts";
-import {
-  decryptAES,
-  decryptPath,
-  deriveKeyFromPassword,
-  encryptAES,
-  encryptPath,
-} from "./aes.ts";
-
-Deno.test("AES encryption and decryption", async () => {
-  const password = "YourPassword";
-  const salt = "UniquePerUserSalt";
-  const message = "Hello, World!";
-
-  const key = await deriveKeyFromPassword(password, salt);
-  const encrypted = await encryptAES(key, message);
-
-  const decrypted = await decryptAES(key, encrypted);
-  assertEquals(decrypted, message);
-
-  // Test that checks if a path is encrypted the same way every time and can be unencrypted
-  const path =
-    "this/is/a/long/path/that/needs/to/be/encrypted because that's what we do.md";
-  const encryptedPath = await encryptPath(key, path);
-  const encryptedPath2 = await encryptPath(key, path);
-  // Assure two runs give the same result
-  assertEquals(encryptedPath, encryptedPath2);
-
-  // Ensure decryption works
-  const decryptedPath = await decryptPath(key, encryptedPath);
-  console.log(encryptedPath);
-  assertEquals(decryptedPath, path);
-});

common/crypto/aes.ts

@@ -1,111 +0,0 @@
-import {
-  base64Decode,
-  base64Encode,
-} from "../../plugos/asset_bundle/base64.ts";
-
-const encoder = new TextEncoder();
-const decoder = new TextDecoder();
-
-export async function deriveKeyFromPassword(
-  password: string,
-  salt: string,
-): Promise<CryptoKey> {
-  const baseKey = encoder.encode(password);
-  const importedKey = await window.crypto.subtle.importKey(
-    "raw",
-    baseKey,
-    { name: "PBKDF2" },
-    false,
-    ["deriveKey"],
-  );
-  return crypto.subtle.deriveKey(
-    {
-      name: "PBKDF2",
-      salt: encoder.encode(salt),
-      iterations: 10000,
-      hash: "SHA-256",
-    },
-    importedKey,
-    {
-      name: "AES-GCM",
-      length: 256,
-    },
-    true,
-    ["encrypt", "decrypt"],
-  );
-}
-
-export async function encryptAES(
-  key: CryptoKey,
-  message: string,
-): Promise<ArrayBuffer> {
-  const iv = crypto.getRandomValues(new Uint8Array(12));
-  const encodedMessage = encoder.encode(message);
-  const ciphertext = await window.crypto.subtle.encrypt(
-    {
-      name: "AES-GCM",
-      iv: iv,
-    },
-    key,
-    encodedMessage,
-  );
-  return appendBuffer(iv, ciphertext);
-}
-
-export async function decryptAES(
-  key: CryptoKey,
-  data: ArrayBuffer,
-): Promise<string> {
-  const iv = data.slice(0, 12);
-  const ciphertext = data.slice(12);
-  const decrypted = await window.crypto.subtle.decrypt(
-    {
-      name: "AES-GCM",
-      iv: iv,
-    },
-    key,
-    ciphertext,
-  );
-  return decoder.decode(decrypted);
-}
-
-function appendBuffer(buffer1: ArrayBuffer, buffer2: ArrayBuffer): ArrayBuffer {
-  const tmp = new Uint8Array(buffer1.byteLength + buffer2.byteLength);
-  tmp.set(new Uint8Array(buffer1), 0);
-  tmp.set(new Uint8Array(buffer2), buffer1.byteLength);
-  return tmp.buffer;
-}
-
-// This is against security recommendations, but we need a way to always generate the same encrypted path for the same path and password
-const pathIv = new Uint8Array(12); // 12 bytes of 0
-
-export async function encryptPath(
-  key: CryptoKey,
-  path: string,
-): Promise<string> {
-  const encodedMessage = encoder.encode(path);
-  const ciphertext = await crypto.subtle.encrypt(
-    {
-      name: "AES-GCM",
-      iv: pathIv,
-    },
-    key,
-    encodedMessage,
-  );
-  return base64Encode(new Uint8Array(ciphertext));
-}
-
-export async function decryptPath(
-  key: CryptoKey,
-  data: string,
-): Promise<string> {
-  const decrypted = await crypto.subtle.decrypt(
-    {
-      name: "AES-GCM",
-      iv: pathIv,
-    },
-    key,
-    base64Decode(data),
-  );
-  return decoder.decode(decrypted);
-}

common/spaces/encrypted_space_primitives.test.ts

@@ -0,0 +1,62 @@
+import { MemoryKvPrimitives } from "../../plugos/lib/memory_kv_primitives.ts";
+import { assert, assertEquals } from "../../test_deps.ts";
+import { ChunkedKvStoreSpacePrimitives } from "./chunked_datastore_space_primitives.ts";
+import { EncryptedSpacePrimitives } from "./encrypted_space_primitives.ts";
+import { testSpacePrimitives } from "./space_primitives.test.ts";
+
+Deno.test("Encrypted Space Primitives", async () => {
+  // Using an in-memory store for testing
+  const memoryKv = new MemoryKvPrimitives();
+  const spacePrimitives = new EncryptedSpacePrimitives(
+    new ChunkedKvStoreSpacePrimitives(
+      memoryKv,
+      1024 * 1024,
+    ),
+  );
+  assertEquals(false, await spacePrimitives.init());
+  await spacePrimitives.setup("password");
+  assertEquals(await spacePrimitives.fetchFileList(), []);
+  await testSpacePrimitives(spacePrimitives);
+
+  // Let's try an incorrect password
+  try {
+    await spacePrimitives.login("wronk");
+    assert(false);
+  } catch (e: any) {
+    assertEquals(e.message, "Incorrect password");
+  }
+
+  // Now let's update the password
+  await spacePrimitives.updatePassword("password", "password2");
+
+  try {
+    await spacePrimitives.updatePassword("password", "password2");
+    assert(false);
+  } catch (e: any) {
+    assertEquals(e.message, "Incorrect password");
+  }
+
+  await spacePrimitives.writeFile(
+    "test.txt",
+    new TextEncoder().encode("Hello World"),
+  );
+
+  // Let's do this again with the new password
+
+  const spacePrimitives2 = new EncryptedSpacePrimitives(
+    new ChunkedKvStoreSpacePrimitives(
+      memoryKv,
+      1024 * 1024,
+    ),
+  );
+  assertEquals(true, await spacePrimitives2.init());
+  await spacePrimitives2.login("password2");
+  assertEquals(
+    new TextDecoder().decode(
+      (await spacePrimitives2.readFile("test.txt")).data,
+    ),
+    "Hello World",
+  );
+  await spacePrimitives2.deleteFile("test.txt");
+  await testSpacePrimitives(spacePrimitives2);
+});

common/spaces/encrypted_space_primitives.ts

@@ -0,0 +1,455 @@
+import { FileMeta } from "../../plug-api/types.ts";
+import { SpacePrimitives } from "./space_primitives.ts";
+
+export const encryptedFileExt = ".crypt";
+export const keyPath = "KEY";
+export const saltFile = "salt.crypt";
+
+/**
+ * This class adds an (AES) based encryption layer on top of another SpacePrimitives implementation.
+ * It encrypts all file names and file contents.
+ * It uses a key file (default named _KEY) to store the encryption key, this file is encrypted with a key derived from the user's password.
+ * The reason to keep the actualy encryption key in a file is to allow the user to change their password without having to re-encrypt all files.
+ * Important note: FileMeta's size will reflect the underlying encrypted size, not the original size
+ */
+export class EncryptedSpacePrimitives implements SpacePrimitives {
+  private masterKey?: CryptoKey;
+  private encryptedKeyFileName?: string;
+  spaceSalt?: Uint8Array;
+
+  constructor(
+    private wrapped: SpacePrimitives,
+  ) {
+  }
+
+  /**
+   * Checks if the space is initialized by loading the salt file.
+   * @returns true if the space was initialized, false if it was not initialized yet
+   */
+  async init(salt?: Uint8Array | undefined | null): Promise<boolean> {
+    if (salt) {
+      this.spaceSalt = salt;
+      return true;
+    }
+    try {
+      this.spaceSalt = (await this.wrapped.readFile(saltFile)).data;
+      return true;
+    } catch (e: any) {
+      if (e.message === "Not found") {
+        console.warn("Space not initialized");
+        return false;
+      }
+      throw e;
+    }
+  }
+
+  /**
+   * Setup a fresh space with a new salt and master encryption key derived from a password
+   * @param password
+   */
+  async setup(password: string): Promise<void> {
+    if (this.spaceSalt) {
+      throw new Error("Space already initialized");
+    }
+    this.spaceSalt = this.generateSalt();
+    await this.wrapped.writeFile(saltFile, this.spaceSalt);
+    await this.createKey(password);
+  }
+
+  /**
+   * Loads the encryption key from the master key based on the user's password
+   * @param password the user's password
+   */
+  async login(password: string): Promise<void> {
+    if (!this.spaceSalt) {
+      throw new Error("Space not initialized");
+    }
+    // First derive an encryption key solely used for encrypting the key file from the user's password
+    const keyEncryptionKey = await this.deriveKeyFromPassword(password);
+    const encryptedKeyFileName = await this.encryptPath(
+      keyEncryptionKey,
+      keyPath,
+    );
+
+    try {
+      this.masterKey = await this.importKey(
+        await this.decryptAES(
+          keyEncryptionKey,
+          (await this.wrapped.readFile(
+            encryptedKeyFileName,
+          )).data,
+        ),
+      );
+      this.encryptedKeyFileName = encryptedKeyFileName;
+    } catch (e: any) {
+      if (e.message === "Not found") {
+        throw new Error("Incorrect password");
+      }
+      console.trace();
+      throw e;
+    }
+  }
+
+  private generateKey(): Promise<CryptoKey> {
+    return window.crypto.subtle.generateKey(
+      {
+        name: "AES-GCM",
+        length: 256,
+      },
+      true,
+      ["encrypt", "decrypt"],
+    );
+  }
+
+  private async createKey(password: string): Promise<void> {
+    const keyEncryptionKey = await this.deriveKeyFromPassword(password);
+    this.encryptedKeyFileName = await this.encryptPath(
+      keyEncryptionKey,
+      keyPath,
+    );
+    this.masterKey = await this.generateKey();
+    // And write it
+    await this.wrapped.writeFile(
+      this.encryptedKeyFileName,
+      await this.encryptAES(
+        keyEncryptionKey,
+        await this.exportKey(this.masterKey),
+      ),
+    );
+  }
+
+  async updatePassword(oldPassword: string, newPasword: string): Promise<void> {
+    if (!this.masterKey) {
+      throw new Error("No key loaded");
+    }
+    const oldPasswordKeyFileName = await this.encryptPath(
+      await this.deriveKeyFromPassword(oldPassword),
+      keyPath,
+    );
+
+    // Check if the old password is correct
+    try {
+      await this.wrapped.getFileMeta(oldPasswordKeyFileName);
+    } catch (e: any) {
+      if (e.message === "Not found") {
+        throw new Error("Incorrect password");
+      } else {
+        throw e;
+      }
+    }
+
+    // First derive an encryption key solely used for encrypting the key file from the user's password
+    const keyEncryptionKey = await this.deriveKeyFromPassword(newPasword);
+
+    this.encryptedKeyFileName = await this.encryptPath(
+      keyEncryptionKey,
+      keyPath,
+    );
+    // And write it
+    await this.wrapped.writeFile(
+      this.encryptedKeyFileName,
+      await this.encryptAES(
+        keyEncryptionKey,
+        await this.exportKey(this.masterKey),
+      ),
+    );
+
+    // Then delete the old key file based on the old password
+    await this.wrapped.deleteFile(oldPasswordKeyFileName);
+  }
+
+  isUnencryptedPath(name: string) {
+    return name.startsWith("_plug/");
+  }
+
+  private generateSalt(): Uint8Array {
+    return crypto.getRandomValues(new Uint8Array(16));
+  }
+
+  private async exportKey(key: CryptoKey): Promise<Uint8Array> {
+    const arrayBuffer = await window.crypto.subtle.exportKey("raw", key);
+    return new Uint8Array(arrayBuffer);
+  }
+
+  private importKey(key: Uint8Array): Promise<CryptoKey> {
+    return window.crypto.subtle.importKey(
+      "raw",
+      key,
+      { name: "AES-GCM" },
+      true,
+      ["encrypt", "decrypt"],
+    );
+  }
+
+  private async deriveKeyFromPassword(
+    password: string,
+  ): Promise<CryptoKey> {
+    const baseKey = new TextEncoder().encode(password);
+    const importedKey = await window.crypto.subtle.importKey(
+      "raw",
+      baseKey,
+      { name: "PBKDF2" },
+      false,
+      ["deriveKey"],
+    );
+    return crypto.subtle.deriveKey(
+      {
+        name: "PBKDF2",
+        salt: this.spaceSalt!,
+        iterations: 10000,
+        hash: "SHA-256",
+      },
+      importedKey,
+      {
+        name: "AES-GCM",
+        length: 256,
+      },
+      true,
+      ["encrypt", "decrypt"],
+    );
+  }
+
+  /**
+   * Encrypts using AES-GCM and prepends the IV to the ciphertext
+   * @param key
+   * @param message
+   * @returns
+   */
+  private async encryptAES(
+    key: CryptoKey,
+    message: Uint8Array,
+  ): Promise<Uint8Array> {
+    const iv = crypto.getRandomValues(new Uint8Array(12));
+    const ciphertext = await window.crypto.subtle.encrypt(
+      {
+        name: "AES-GCM",
+        iv: iv,
+      },
+      key,
+      message,
+    );
+    return appendBuffer(iv, new Uint8Array(ciphertext));
+  }
+
+  /**
+   * Decrypts using AES-GCM and expects the IV to be prepended to the ciphertext
+   * @param key
+   * @param data
+   * @returns
+   */
+  async decryptAES(
+    key: CryptoKey,
+    data: Uint8Array,
+  ): Promise<Uint8Array> {
+    const iv = data.slice(0, 12);
+    const ciphertext = data.slice(12);
+    const decrypted = await window.crypto.subtle.decrypt(
+      {
+        name: "AES-GCM",
+        iv: iv,
+      },
+      key,
+      ciphertext,
+    );
+    return new Uint8Array(decrypted);
+  }
+
+  /**
+   * Left pads a string with zeros to a length of 32, encrypts it using AES-GCM and returns the base32 encoded ciphertext
+   * @param key
+   * @param path
+   * @returns
+   */
+  async encryptPath(
+    key: CryptoKey,
+    path: string,
+  ): Promise<string> {
+    if (!this.spaceSalt) {
+      throw new Error("Space not initialized");
+    }
+    if (this.isUnencryptedPath(path)) {
+      return path;
+    }
+
+    path = path.padEnd(32, "\0");
+    const encodedMessage = new TextEncoder().encode(path);
+
+    const ciphertext = await crypto.subtle.encrypt(
+      {
+        name: "AES-GCM",
+        iv: this.spaceSalt,
+      },
+      key,
+      encodedMessage,
+    );
+    const encodedPath = base32Encode(new Uint8Array(ciphertext));
+    // console.log(new TextDecoder().decode(ciphertext));
+    return encodedPath.slice(0, 3) + "/" + encodedPath.slice(3) +
+      encryptedFileExt;
+  }
+
+  private async decryptPath(
+    key: CryptoKey,
+    encryptedPath: string,
+  ): Promise<string> {
+    if (!this.spaceSalt) {
+      throw new Error("Space not initialized");
+    }
+    if (this.isUnencryptedPath(encryptedPath)) {
+      return encryptedPath;
+    }
+
+    if (!encryptedPath.endsWith(encryptedFileExt)) {
+      throw new Error("Invalid encrypted path");
+    }
+    // Remove the extension and slashes
+    encryptedPath = encryptedPath.slice(0, -encryptedFileExt.length).replaceAll(
+      "/",
+      "",
+    );
+
+    // console.log("To decrypt", encryptedPath);
+    const decrypted = await crypto.subtle.decrypt(
+      {
+        name: "AES-GCM",
+        iv: this.spaceSalt,
+      },
+      key,
+      base32Decode(encryptedPath),
+    );
+    // Decode the buffer and remove the padding
+    return removePadding(new TextDecoder().decode(decrypted), "\0");
+  }
+
+  async fetchFileList(): Promise<FileMeta[]> {
+    const files = await this.wrapped.fetchFileList();
+    // console.log(files);
+    return Promise.all(
+      files.filter((fileMeta) =>
+        fileMeta.name !== this.encryptedKeyFileName &&
+        fileMeta.name !== saltFile
+      )
+        .map(async (fileMeta) => {
+          return {
+            ...fileMeta,
+            name: await this.decryptPath(this.masterKey!, fileMeta.name),
+          };
+        }),
+    );
+  }
+
+  async getFileMeta(name: string): Promise<FileMeta> {
+    if (this.isUnencryptedPath(name)) {
+      return this.wrapped.getFileMeta(name);
+    }
+    const fileMeta = await this.wrapped.getFileMeta(
+      await this.encryptPath(this.masterKey!, name),
+    );
+    return {
+      ...fileMeta,
+      name,
+    };
+  }
+
+  async readFile(name: string): Promise<{ data: Uint8Array; meta: FileMeta }> {
+    if (this.isUnencryptedPath(name)) {
+      return this.wrapped.readFile(name);
+    }
+    const { data, meta } = await this.wrapped.readFile(
+      await this.encryptPath(this.masterKey!, name),
+    );
+    return {
+      data: await this.decryptAES(this.masterKey!, data),
+      meta: {
+        ...meta,
+        name,
+      },
+    };
+  }
+
+  async writeFile(
+    name: string,
+    data: Uint8Array,
+    selfUpdate?: boolean | undefined,
+    meta?: FileMeta | undefined,
+  ): Promise<FileMeta> {
+    if (this.isUnencryptedPath(name)) {
+      return this.wrapped.writeFile(name, data, selfUpdate, meta);
+    }
+    const newMeta = await this.wrapped.writeFile(
+      await this.encryptPath(this.masterKey!, name),
+      await this.encryptAES(this.masterKey!, data),
+      selfUpdate,
+      meta,
+    );
+    return {
+      ...newMeta,
+      name,
+    };
+  }
+
+  async deleteFile(name: string): Promise<void> {
+    if (this.isUnencryptedPath(name)) {
+      return this.wrapped.deleteFile(name);
+    }
+    return this.wrapped.deleteFile(
+      await this.encryptPath(this.masterKey!, name),
+    );
+  }
+}
+
+function removePadding(str: string, paddingChar: string): string {
+  // let startIndex = 0;
+  // while (startIndex < str.length && str[startIndex] === paddingChar) {
+  //   startIndex++;
+  // }
+  // return str.substring(startIndex);
+  let endIndex = str.length - 1;
+  while (endIndex >= 0 && str[endIndex] === paddingChar) {
+    endIndex--;
+  }
+  return str.substring(0, endIndex + 1);
+}
+
+function base32Encode(data: Uint8Array): string {
+  const alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
+  let result = "";
+  let bits = 0;
+  let value = 0;
+  for (const byte of data) {
+    value = (value << 8) | byte;
+    bits += 8;
+    while (bits >= 5) {
+      result += alphabet[(value >>> (bits - 5)) & 31];
+      bits -= 5;
+    }
+  }
+  if (bits > 0) {
+    result += alphabet[(value << (5 - bits)) & 31];
+  }
+  return result;
+}
+
+function base32Decode(data: string): Uint8Array {
+  const alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
+  const result = new Uint8Array(Math.floor(data.length * 5 / 8));
+  let bits = 0;
+  let value = 0;
+  let index = 0;
+  for (const char of data) {
+    value = (value << 5) | alphabet.indexOf(char);
+    bits += 5;
+    if (bits >= 8) {
+      result[index++] = (value >>> (bits - 8)) & 255;
+      bits -= 8;
+    }
+  }
+  return result;
+}
+
+function appendBuffer(buffer1: Uint8Array, buffer2: Uint8Array): Uint8Array {
+  const tmp = new Uint8Array(buffer1.byteLength + buffer2.byteLength);
+  tmp.set(new Uint8Array(buffer1), 0);
+  tmp.set(new Uint8Array(buffer2), buffer1.byteLength);
+  return tmp;
+}

common/spaces/space_primitives.test.ts

@@ -35,8 +35,8 @@ export async function testSpacePrimitives(spacePrimitives: SpacePrimitives) {
   buf.set([1, 2, 3, 4, 5]);
   // Write binary file
   await spacePrimitives.writeFile("test.bin", buf);
-  const fMeta = await spacePrimitives.getFileMeta("test.bin");
+  const fileData = await spacePrimitives.readFile("test.bin");
-  assertEquals(fMeta.size, 1024 * 1024);
+  assertEquals(fileData.data.length, 1024 * 1024);
   assertEquals((await spacePrimitives.fetchFileList()).length, 2);
   //   console.log(spacePrimitives);
 

common/util.ts

@@ -1,6 +1,7 @@
 import { SETTINGS_TEMPLATE } from "./settings_template.ts";
 import { YAML } from "./deps.ts";
 import { SpacePrimitives } from "./spaces/space_primitives.ts";
+import { expandPropertyNames } from "$sb/lib/json.ts";
 
 export function safeRun(fn: () => Promise<void>) {
   fn().catch((e) => {
@@ -43,6 +44,7 @@ export async function ensureSettingsAndIndex(
     );
   } catch (e: any) {
     if (e.message === "Not found") {
+      console.log("No settings found, creating default settings");
       await space.writeFile(
         "SETTINGS.md",
         new TextEncoder().encode(SETTINGS_TEMPLATE),
@@ -56,7 +58,11 @@ export async function ensureSettingsAndIndex(
     // Ok, then let's also check the index page
     try {
       await space.getFileMeta("index.md");
-    } catch {
+    } catch (e: any) {
+      console.log(
+        "No index page found, creating default index page",
+        e.message,
+      );
       await space.writeFile(
         "index.md",
         new TextEncoder().encode(
@@ -71,5 +77,7 @@ page: "[[!silverbullet.md/Getting Started]]"
     }
   }
 
-  return parseYamlSettings(settingsText);
+  const settings = parseYamlSettings(settingsText);
+  expandPropertyNames(settings);
+  return settings;
 }

server/http_server.ts

@@ -14,6 +14,7 @@ import { SpaceServer, SpaceServerConfig } from "./instance.ts";
 import { KvPrimitives } from "../plugos/lib/kv_primitives.ts";
 import { EndpointHook } from "../plugos/hooks/endpoint.ts";
 import { PrefixedKvPrimitives } from "../plugos/lib/prefixed_kv_primitives.ts";
+import { base64Encode } from "../plugos/asset_bundle/base64.ts";
 
 const authenticationExpirySeconds = 60 * 60 * 24 * 7; // 1 week
 
@@ -24,7 +25,6 @@ export type ServerOptions = {
   clientAssetBundle: AssetBundle;
   plugAssetBundle: AssetBundle;
   baseKvPrimitives: KvPrimitives;
-  syncOnly: boolean;
   certFile?: string;
   keyFile?: string;
 
@@ -42,7 +42,6 @@ export class HttpServer {
   certFile: string | undefined;
 
   spaceServers = new Map<string, Promise<SpaceServer>>();
-  syncOnly: boolean;
   baseKvPrimitives: KvPrimitives;
   configs: Map<string, SpaceServerConfig>;
 
@@ -54,7 +53,6 @@ export class HttpServer {
     this.app = options.app;
     this.keyFile = options.keyFile;
     this.certFile = options.certFile;
-    this.syncOnly = options.syncOnly;
     this.baseKvPrimitives = options.baseKvPrimitives;
     this.configs = options.configs;
   }
@@ -67,7 +65,6 @@ export class HttpServer {
       new PrefixedKvPrimitives(this.baseKvPrimitives, [
         config.namespace,
       ]),
-      this.syncOnly,
     );
     await spaceServer.init();
 
@@ -114,15 +111,18 @@ export class HttpServer {
   }
 
   // Replaces some template variables in index.html in a rather ad-hoc manner, but YOLO
-  renderIndexHtml(pagesPath: string) {
+  renderIndexHtml(spaceServer: SpaceServer) {
     return this.clientAssetBundle.readTextFileSync(".client/index.html")
       .replaceAll(
         "{{SPACE_PATH}}",
-        pagesPath.replaceAll("\\", "\\\\"),
+        spaceServer.pagesPath.replaceAll("\\", "\\\\"),
         // );
       ).replaceAll(
         "{{SYNC_ONLY}}",
-        this.syncOnly ? "true" : "false",
+        spaceServer.syncOnly ? "true" : "false",
+      ).replaceAll(
+        "{{CLIENT_ENCRYPTION}}",
+        spaceServer.clientEncryption ? "true" : "false",
       );
   }
 
@@ -149,7 +149,7 @@ export class HttpServer {
       response.headers.set("Content-type", "text/html");
       response.headers.set("Cache-Control", "no-cache");
       const spaceServer = await this.ensureSpaceServer(request);
-      response.body = this.renderIndexHtml(spaceServer.pagesPath);
+      response.body = this.renderIndexHtml(spaceServer);
     });
 
     this.abortController = new AbortController();
@@ -181,6 +181,7 @@ export class HttpServer {
     { request, response }: Context<Record<string, any>, Record<string, any>>,
     next: Next,
   ) {
+    const spaceServer = await this.ensureSpaceServer(request);
     if (
       request.url.pathname === "/"
     ) {
@@ -188,8 +189,7 @@ export class HttpServer {
       // Note: we're explicitly not setting Last-Modified and If-Modified-Since header here because this page is dynamic
       response.headers.set("Content-type", "text/html");
       response.headers.set("Cache-Control", "no-cache");
-      const spaceServer = await this.ensureSpaceServer(request);
+      response.body = this.renderIndexHtml(spaceServer);
-      response.body = this.renderIndexHtml(spaceServer.pagesPath);
       return;
     }
     try {
@@ -197,7 +197,8 @@ export class HttpServer {
       if (
         this.clientAssetBundle.has(assetName) &&
         request.headers.get("If-Modified-Since") ===
-          utcDateString(this.clientAssetBundle.getMtime(assetName))
+          utcDateString(this.clientAssetBundle.getMtime(assetName)) &&
+        assetName !== "service_worker.js"
       ) {
         response.status = 304;
         return;
@@ -207,17 +208,34 @@ export class HttpServer {
         "Content-type",
         this.clientAssetBundle.getMimeType(assetName),
       );
-      const data = this.clientAssetBundle.readFileSync(
+      let data: Uint8Array | string = this.clientAssetBundle.readFileSync(
         assetName,
       );
       response.headers.set("Cache-Control", "no-cache");
       response.headers.set("Content-length", "" + data.length);
+      if (assetName !== "service_worker.js") {
         response.headers.set(
           "Last-Modified",
           utcDateString(this.clientAssetBundle.getMtime(assetName)),
         );
+      }
 
       if (request.method === "GET") {
+        if (assetName === "service_worker.js") {
+          const textData = new TextDecoder().decode(data);
+          // console.log(
+          //   "Swapping out config hash in service worker",
+          // );
+          data = textData.replaceAll(
+            "{{CONFIG_HASH}}",
+            base64Encode(
+              JSON.stringify([
+                spaceServer.clientEncryption,
+                spaceServer.syncOnly,
+              ]),
+            ),
+          );
+        }
         response.body = data;
       }
     } catch {
@@ -384,7 +402,7 @@ export class HttpServer {
             return;
           }
           case "syscall": {
-            if (this.syncOnly) {
+            if (spaceServer.syncOnly) {
               response.headers.set("Content-Type", "text/plain");
               response.status = 400;
               response.body = "Unknown operation";

server/instance.ts

@@ -21,6 +21,8 @@ export type SpaceServerConfig = {
   // Additional API auth token
   authToken?: string;
   pagesPath: string;
+  syncOnly?: boolean;
+  clientEncryption?: boolean;
 };
 
 export class SpaceServer {
@@ -37,18 +39,26 @@ export class SpaceServer {
   // Only set when syncOnly == false
   private serverSystem?: ServerSystem;
   system?: System<SilverBulletHooks>;
+  clientEncryption: boolean;
+  syncOnly: boolean;
 
   constructor(
     config: SpaceServerConfig,
     public shellBackend: ShellBackend,
     private plugAssetBundle: AssetBundle,
     private kvPrimitives: KvPrimitives,
-    private syncOnly: boolean,
   ) {
     this.pagesPath = config.pagesPath;
     this.hostname = config.hostname;
     this.auth = config.auth;
     this.authToken = config.authToken;
+    this.clientEncryption = !!config.clientEncryption;
+    this.syncOnly = !!config.syncOnly;
+    if (this.clientEncryption) {
+      // Sync only will forced on when encryption is enabled
+      this.syncOnly = true;
+    }
+
     this.jwtIssuer = new JWTIssuer(kvPrimitives);
   }
 
@@ -100,7 +110,13 @@ export class SpaceServer {
   }
 
   async reloadSettings() {
-    // TODO: Throttle this?
+    if (!this.clientEncryption) {
+      // Only attempt this when the space is not encrypted
       this.settings = await ensureSettingsAndIndex(this.spacePrimitives);
+    } else {
+      this.settings = {
+        indexPage: "index",
+      };
+    }
   }
 }

silverbullet.ts

@@ -41,6 +41,10 @@ await new Command()
     "--sync-only",
     "Run the server as a pure space (file) store only without any backend processing (this disables 'online mode' in the client)",
   )
+  .option(
+    "--client-encryption",
+    "Enable client-side encryption for spaces",
+  )
   .option(
     "--reindex",
     "Reindex space on startup",

web/client.ts

@@ -9,7 +9,7 @@ import {
 } from "../common/deps.ts";
 import { fileMetaToPageMeta, Space } from "./space.ts";
 import { FilterOption } from "./types.ts";
-import { parseYamlSettings } from "../common/util.ts";
+import { ensureSettingsAndIndex } from "../common/util.ts";
 import { EventHook } from "../plugos/hooks/event.ts";
 import { AppCommand } from "./hooks/command.ts";
 import { PathPageNavigator } from "./navigator.ts";
@@ -37,13 +37,16 @@ import { createEditorState } from "./editor_state.ts";
 import { OpenPages } from "./open_pages.ts";
 import { MainUI } from "./editor_ui.tsx";
 import { cleanPageRef } from "$sb/lib/resolve.ts";
-import { expandPropertyNames } from "$sb/lib/json.ts";
 import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
 import { FileMeta, PageMeta } from "$sb/types.ts";
 import { DataStore } from "../plugos/lib/datastore.ts";
 import { IndexedDBKvPrimitives } from "../plugos/lib/indexeddb_kv_primitives.ts";
 import { DataStoreMQ } from "../plugos/lib/mq.datastore.ts";
 import { DataStoreSpacePrimitives } from "../common/spaces/datastore_space_primitives.ts";
+import {
+  encryptedFileExt,
+  EncryptedSpacePrimitives,
+} from "../common/spaces/encrypted_space_primitives.ts";
 const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/;
 
 const autoSaveInterval = 1000;
@@ -54,6 +57,7 @@ declare global {
     silverBulletConfig: {
       spaceFolderPath: string;
       syncOnly: boolean;
+      clientEncryption: boolean;
     };
     client: Client;
   }
@@ -69,7 +73,7 @@ export class Client {
 
   plugSpaceRemotePrimitives!: PlugSpacePrimitives;
   // localSpacePrimitives!: FilteredSpacePrimitives;
-  remoteSpacePrimitives!: HttpSpacePrimitives;
+  httpSpacePrimitives!: HttpSpacePrimitives;
   space!: Space;
 
   saveTimeout?: number;
@@ -177,15 +181,14 @@ export class Client {
 
     this.focus();
 
-    // This constructor will always be followed by an (async) invocatition of init()
     await this.system.init();
 
     // Load settings
-    this.settings = await this.loadSettings();
+    this.settings = await ensureSettingsAndIndex(localSpacePrimitives);
 
     // Pinging a remote space to ensure we're authenticated properly, if not will result in a redirect to auth page
     try {
-      await this.remoteSpacePrimitives.ping();
+      await this.httpSpacePrimitives.ping();
     } catch (e: any) {
       if (e.message === "Not authenticated") {
         console.warn("Not authenticated, redirecting to auth page");
@@ -346,13 +349,83 @@ export class Client {
   }
 
   async initSpace(): Promise<SpacePrimitives> {
-    this.remoteSpacePrimitives = new HttpSpacePrimitives(
+    this.httpSpacePrimitives = new HttpSpacePrimitives(
       location.origin,
       window.silverBulletConfig.spaceFolderPath,
     );
 
+    let remoteSpacePrimitives: SpacePrimitives = this.httpSpacePrimitives;
+
+    if (window.silverBulletConfig.clientEncryption) {
+      console.log("Enabling encryption");
+
+      const encryptedSpacePrimitives = new EncryptedSpacePrimitives(
+        this.httpSpacePrimitives,
+      );
+      remoteSpacePrimitives = encryptedSpacePrimitives;
+      let loggedIn = false;
+      // First figure out if we're online & if the key file exists, if not we need to initialize the space
+      try {
+        if (!await encryptedSpacePrimitives.init()) {
+          console.log(
+            "Space not initialized, will ask for password to initialize",
+          );
+          alert(
+            "You appear to be accessing a new space with encryption enabled, you will now be asked to create a password",
+          );
+          const password = prompt("Choose a password");
+          if (!password) {
+            alert("Cannot do anything without a password, reloading");
+            location.reload();
+            throw new Error("Not initialized");
+          }
+          const password2 = prompt("Confirm password");
+          if (password !== password2) {
+            alert("Passwords don't match, reloading");
+            location.reload();
+            throw new Error("Not initialized");
+          }
+          await encryptedSpacePrimitives.setup(password);
+          // this.stateDataStore.set(["encryptionKey"], password);
+          await this.stateDataStore.set(
+            ["spaceSalt"],
+            encryptedSpacePrimitives.spaceSalt,
+          );
+          loggedIn = true;
+        }
+      } catch (e: any) {
+        if (e.message === "Offline") {
+          console.log(
+            "Offline, will assume encryption space is initialized, fetching salt from data store",
+          );
+          await encryptedSpacePrimitives.init(
+            await this.stateDataStore.get(["spaceSalt"]),
+          );
+        }
+      }
+      if (!loggedIn) {
+        // Let's ask for the password
+        try {
+          await encryptedSpacePrimitives.login(
+            prompt("Password")!,
+          );
+          await this.stateDataStore.set(
+            ["spaceSalt"],
+            encryptedSpacePrimitives.spaceSalt,
+          );
+        } catch (e: any) {
+          console.log("Got this error", e);
+          if (e.message === "Incorrect password") {
+            alert("Incorrect password");
+            location.reload();
+          }
+          throw e;
+        }
+      }
+    }
+
     this.plugSpaceRemotePrimitives = new PlugSpacePrimitives(
-      this.remoteSpacePrimitives,
+      remoteSpacePrimitives,
       this.system.namespaceHook,
       this.syncMode ? undefined : "client",
     );
@@ -380,7 +453,10 @@ export class Client {
         (meta) => fileFilterFn(meta.name),
         // Run when a list of files has been retrieved
         async () => {
-          await this.loadSettings();
+          if (!this.settings) {
+            this.settings = await ensureSettingsAndIndex(localSpacePrimitives!);
+          }
+
           if (typeof this.settings?.spaceIgnore === "string") {
             fileFilterFn = gitIgnoreCompiler(this.settings.spaceIgnore).accepts;
           } else {
@@ -439,27 +515,6 @@ export class Client {
     return localSpacePrimitives;
   }
 
-  async loadSettings(): Promise<BuiltinSettings> {
-    let settingsText: string | undefined;
-
-    try {
-      settingsText = (await this.space.readPage("SETTINGS")).text;
-    } catch (e: any) {
-      console.info("No SETTINGS page, falling back to default", e);
-      settingsText = '```yaml\nindexPage: "[[index]]"\n```\n';
-    }
-    let settings = parseYamlSettings(settingsText!) as BuiltinSettings;
-
-    settings = expandPropertyNames(settings);
-
-    // console.log("Settings", settings);
-
-    if (!settings.indexPage) {
-      settings.indexPage = "[[index]]";
-    }
-    return settings;
-  }
-
   get currentPage(): string | undefined {
     return this.ui.viewState.currentPage;
   }

web/editor_state.ts

@@ -166,6 +166,7 @@ export function createEditorState(
         { selector: "FrontMatter", class: "sb-frontmatter" },
       ]),
       keymap.of([
+        ...commandKeyBindings,
         ...smartQuoteKeymap,
         ...closeBracketsKeymap,
         ...standardKeymap,
@@ -173,7 +174,6 @@ export function createEditorState(
         ...historyKeymap,
         ...completionKeymap,
         indentWithTab,
-        ...commandKeyBindings,
         {
           key: "Ctrl-k",
           mac: "Cmd-k",

web/index.html

@@ -39,12 +39,14 @@
       // These {{VARIABLES}} are replaced by http_server.ts
       spaceFolderPath: "{{SPACE_PATH}}",
       syncOnly: "{{SYNC_ONLY}}" === "true",
+      clientEncryption: "{{CLIENT_ENCRYPTION}}" === "true",
     };
     // But in case these variables aren't replaced by the server, fall back sync only mode
     if (window.silverBulletConfig.spaceFolderPath.includes("{{")) {
       window.silverBulletConfig = {
         spaceFolderPath: "",
         syncOnly: true,
+        clientEncryption: false,
       };
     }
   </script>

web/service_worker.ts

@@ -3,7 +3,7 @@ import { simpleHash } from "../common/crypto.ts";
 import { DataStore } from "../plugos/lib/datastore.ts";
 import { IndexedDBKvPrimitives } from "../plugos/lib/indexeddb_kv_primitives.ts";
 
-const CACHE_NAME = "{{CACHE_NAME}}";
+const CACHE_NAME = "{{CACHE_NAME}}_{{CONFIG_HASH}}";
 
 const precacheFiles = Object.fromEntries([
   "/",
@@ -61,7 +61,6 @@ self.addEventListener("activate", (event: any) => {
 });
 
 let ds: DataStore | undefined;
-const filesMetaPrefix = ["file", "meta"];
 const filesContentPrefix = ["file", "content"];
 
 self.addEventListener("fetch", (event: any) => {

web/syscalls/fetch.ts

@@ -28,15 +28,15 @@ export function sandboxFetchSyscalls(
           body: options.base64Body && base64Decode(options.base64Body),
         }
         : {};
-      if (!client.remoteSpacePrimitives) {
+      if (!client.httpSpacePrimitives) {
         // No SB server to proxy the fetch available so let's execute the request directly
         return performLocalFetch(url, fetchOptions);
       }
       fetchOptions.headers = fetchOptions.headers
         ? { ...fetchOptions.headers, "X-Proxy-Request": "true" }
         : { "X-Proxy-Request": "true" };
-      const resp = await client.remoteSpacePrimitives.authenticatedFetch(
+      const resp = await client.httpSpacePrimitives.authenticatedFetch(
-        `${client.remoteSpacePrimitives.url}/!${url}`,
+        `${client.httpSpacePrimitives.url}/!${url}`,
         fetchOptions,
       );
       const body = await resp.arrayBuffer();

web/syscalls/shell.ts

@@ -10,11 +10,11 @@ export function shellSyscalls(
       cmd: string,
       args: string[],
     ): Promise<{ stdout: string; stderr: string; code: number }> => {
-      if (!client.remoteSpacePrimitives) {
+      if (!client.httpSpacePrimitives) {
         throw new Error("Not supported in fully local mode");
       }
-      const resp = client.remoteSpacePrimitives.authenticatedFetch(
+      const resp = client.httpSpacePrimitives.authenticatedFetch(
-        `${client.remoteSpacePrimitives.url}/.rpc`,
+        `${client.httpSpacePrimitives.url}/.rpc`,
         {
           method: "POST",
           body: JSON.stringify({

web/syscalls/system.ts

@@ -43,7 +43,7 @@ export function systemSyscalls(
         // Proxy to another environment
         return proxySyscall(
           ctx,
-          client.remoteSpacePrimitives,
+          client.httpSpacePrimitives,
           "system.invokeFunction",
           [fullName, ...args],
         );

web/syscalls/util.ts

@@ -7,7 +7,7 @@ export function proxySyscalls(client: Client, names: string[]): SysCallMapping {
   const syscalls: SysCallMapping = {};
   for (const name of names) {
     syscalls[name] = (ctx, ...args: any[]) => {
-      return proxySyscall(ctx, client.remoteSpacePrimitives, name, args);
+      return proxySyscall(ctx, client.httpSpacePrimitives, name, args);
     };
   }
   return syscalls;