Refactoring work to support multi-tenancy and multiple storage, database backends (#598)

* Backend infrastructure
* New backend configuration work
* Factor out KV prefixing
* Don't put assets in the manifest cache
* Removed fancy authentication stuff
* Documentation updates

plugos/lib/datastore.test.ts

@@ -2,11 +2,11 @@ import "https://esm.sh/fake-indexeddb@4.0.2/auto";
 import { IndexedDBKvPrimitives } from "./indexeddb_kv_primitives.ts";
 import { DataStore } from "./datastore.ts";
 import { DenoKvPrimitives } from "./deno_kv_primitives.ts";
-import { KvPrimitives } from "./kv_primitives.ts";
+import { KvPrimitives, PrefixedKvPrimitives } from "./kv_primitives.ts";
 import { assertEquals } from "https://deno.land/std@0.165.0/testing/asserts.ts";
 
 async function test(db: KvPrimitives) {
-  const datastore = new DataStore(db, ["ds"], {
+  const datastore = new DataStore(new PrefixedKvPrimitives(db, ["ds"]), {
     count: (arr: any[]) => arr.length,
   });
   await datastore.set(["user", "peter"], { name: "Peter" });

plugos/lib/datastore.ts

@@ -10,25 +10,16 @@ import { KvPrimitives } from "./kv_primitives.ts";
 export class DataStore {
   constructor(
     readonly kv: KvPrimitives,
-    private prefix: KvKey = [],
     private functionMap: FunctionMap = builtinFunctions,
   ) {
   }
 
-  prefixed(prefix: KvKey): DataStore {
-    return new DataStore(
-      this.kv,
-      [...this.prefix, ...prefix],
-      this.functionMap,
-    );
-  }
-
   async get<T = any>(key: KvKey): Promise<T | null> {
     return (await this.batchGet([key]))[0];
   }
 
   batchGet<T = any>(keys: KvKey[]): Promise<(T | null)[]> {
-    return this.kv.batchGet(keys.map((key) => this.applyPrefix(key)));
+    return this.kv.batchGet(keys);
   }
 
   set(key: KvKey, value: any): Promise<void> {
@@ -44,7 +35,7 @@ export class DataStore {
         console.warn(`Duplicate key ${keyString} in batchSet, skipping`);
       } else {
         allKeyStrings.add(keyString);
-        uniqueEntries.push({ key: this.applyPrefix(key), value });
+        uniqueEntries.push({ key, value });
       }
     }
     return this.kv.batchSet(uniqueEntries);
@@ -55,7 +46,7 @@ export class DataStore {
   }
 
   batchDelete(keys: KvKey[]): Promise<void> {
-    return this.kv.batchDelete(keys.map((key) => this.applyPrefix(key)));
+    return this.kv.batchDelete(keys);
   }
 
   async query<T = any>(query: KvQuery): Promise<KV<T>[]> {
@@ -63,15 +54,11 @@ export class DataStore {
     let itemCount = 0;
     // Accumulate results
     let limit = Infinity;
-    const prefixedQuery: KvQuery = {
-      ...query,
-      prefix: query.prefix ? this.applyPrefix(query.prefix) : undefined,
-    };
     if (query.limit) {
       limit = evalQueryExpression(query.limit, {}, this.functionMap);
     }
     for await (
-      const entry of this.kv.query(prefixedQuery)
+      const entry of this.kv.query(query)
     ) {
       // Filter
       if (
@@ -89,29 +76,16 @@ export class DataStore {
       }
     }
     // Apply order by, limit, and select
-    return applyQueryNoFilterKV(prefixedQuery, results, this.functionMap).map((
-      { key, value },
-    ) => ({ key: this.stripPrefix(key), value }));
+    return applyQueryNoFilterKV(query, results, this.functionMap);
   }
 
   async queryDelete(query: KvQuery): Promise<void> {
     const keys: KvKey[] = [];
     for (
-      const { key } of await this.query({
-        ...query,
-        prefix: query.prefix ? this.applyPrefix(query.prefix) : undefined,
-      })
+      const { key } of await this.query(query)
     ) {
       keys.push(key);
     }
     return this.batchDelete(keys);
   }
-
-  private applyPrefix(key: KvKey): KvKey {
-    return [...this.prefix, ...(key ? key : [])];
-  }
-
-  private stripPrefix(key: KvKey): KvKey {
-    return key.slice(this.prefix.length);
-  }
 }

plugos/lib/kv_primitives.ts

@@ -9,4 +9,48 @@ export interface KvPrimitives {
   batchSet(entries: KV[]): Promise<void>;
   batchDelete(keys: KvKey[]): Promise<void>;
   query(options: KvQueryOptions): AsyncIterableIterator<KV>;
+  close(): void;
+}
+
+/**
+ * Turns any KvPrimitives into a KvPrimitives that automatically prefixes all keys (and removes them again when reading)
+ */
+export class PrefixedKvPrimitives implements KvPrimitives {
+  constructor(private wrapped: KvPrimitives, private prefix: KvKey) {
+  }
+
+  batchGet(keys: KvKey[]): Promise<any[]> {
+    return this.wrapped.batchGet(keys.map((key) => this.applyPrefix(key)));
+  }
+
+  batchSet(entries: KV[]): Promise<void> {
+    return this.wrapped.batchSet(
+      entries.map(({ key, value }) => ({ key: this.applyPrefix(key), value })),
+    );
+  }
+
+  batchDelete(keys: KvKey[]): Promise<void> {
+    return this.wrapped.batchDelete(keys.map((key) => this.applyPrefix(key)));
+  }
+
+  async *query(options: KvQueryOptions): AsyncIterableIterator<KV> {
+    for await (
+      const result of this.wrapped.query({
+        prefix: this.applyPrefix(options.prefix),
+      })
+    ) {
+      yield { key: this.stripPrefix(result.key), value: result.value };
+    }
+  }
+  close(): void {
+    this.wrapped.close();
+  }
+
+  private applyPrefix(key?: KvKey): KvKey {
+    return [...this.prefix, ...(key ? key : [])];
+  }
+
+  private stripPrefix(key: KvKey): KvKey {
+    return key.slice(this.prefix.length);
+  }
 }

plugos/lib/kv_store.json_file.ts

@@ -1,56 +0,0 @@
-import { KV } from "$sb/types.ts";
-
-export class JSONKVStore {
-  private data: { [key: string]: any } = {};
-
-  async load(path: string) {
-    this.loadString(await Deno.readTextFile(path));
-  }
-
-  loadString(jsonString: string) {
-    this.data = JSON.parse(jsonString);
-  }
-
-  async save(path: string) {
-    await Deno.writeTextFile(path, JSON.stringify(this.data));
-  }
-
-  del(key: string): Promise<void> {
-    delete this.data[key];
-    return Promise.resolve();
-  }
-
-  deletePrefix(prefix: string): Promise<void> {
-    for (const key in this.data) {
-      if (key.startsWith(prefix)) {
-        delete this.data[key];
-      }
-    }
-    return Promise.resolve();
-  }
-
-  deleteAll(): Promise<void> {
-    this.data = {};
-    return Promise.resolve();
-  }
-
-  set(key: string, value: any): Promise<void> {
-    this.data[key] = value;
-    return Promise.resolve();
-  }
-  get(key: string): Promise<any> {
-    return Promise.resolve(this.data[key]);
-  }
-  has(key: string): Promise<boolean> {
-    return Promise.resolve(key in this.data);
-  }
-  queryPrefix(keyPrefix: string): Promise<{ key: string; value: any }[]> {
-    const results: { key: string; value: any }[] = [];
-    for (const key in this.data) {
-      if (key.startsWith(keyPrefix)) {
-        results.push({ key, value: this.data[key] });
-      }
-    }
-    return Promise.resolve(results);
-  }
-}

plugos/lib/mq.datastore.test.ts

@@ -3,12 +3,15 @@ import { assertEquals } from "../../test_deps.ts";
 import { sleep } from "$sb/lib/async.ts";
 import { DenoKvPrimitives } from "./deno_kv_primitives.ts";
 import { DataStore } from "./datastore.ts";
+import { PrefixedKvPrimitives } from "./kv_primitives.ts";
 
 Deno.test("DataStore MQ", async () => {
   const tmpFile = await Deno.makeTempFile();
   const db = new DenoKvPrimitives(await Deno.openKv(tmpFile));
 
-  const mq = new DataStoreMQ(new DataStore(db, ["mq"]));
+  const mq = new DataStoreMQ(
+    new DataStore(new PrefixedKvPrimitives(db, ["mq"])),
+  );
   await mq.send("test", "Hello World");
   let messages = await mq.poll("test", 10);
   assertEquals(messages.length, 1);