Work on #508 (thin client)

plugos/lib/kv_store.deno_kv.test.ts

@@ -15,6 +15,25 @@ Deno.test("Test KV index", async () => {
     { key: "page:hello2", value: "Hello 2" },
     { key: "page:hello3", value: "Hello 3" },
     { key: "something", value: "Something" },
+    { key: "something1", value: "Something" },
+    { key: "something2", value: "Something" },
+    { key: "something3", value: "Something" },
+    { key: "something4", value: "Something" },
+    { key: "something5", value: "Something" },
+    { key: "something6", value: "Something" },
+    { key: "something7", value: "Something" },
+    { key: "something8", value: "Something" },
+    { key: "something9", value: "Something" },
+    { key: "something10", value: "Something" },
+    { key: "something11", value: "Something" },
+    { key: "something12", value: "Something" },
+    { key: "something13", value: "Something" },
+    { key: "something14", value: "Something" },
+    { key: "something15", value: "Something" },
+    { key: "something16", value: "Something" },
+    { key: "something17", value: "Something" },
+    { key: "something18", value: "Something" },
+    { key: "something19", value: "Something" },
   ]);
 
   const results = await kv.queryPrefix("page:");
@@ -25,5 +44,13 @@ Deno.test("Test KV index", async () => {
     "Hello 3",
   ]);
 
+  await kv.deletePrefix("page:");
+
+  assertEquals(await kv.queryPrefix("page:"), []);
+  assertEquals((await kv.queryPrefix("")).length, 20);
+
+  await kv.deletePrefix("");
+  assertEquals(await kv.queryPrefix(""), []);
+
   await kv.delete();
 });

plugos/lib/kv_store.deno_kv.ts

@@ -2,6 +2,8 @@
 
 import { KV, KVStore } from "./kv_store.ts";
 
+const kvBatchSize = 10;
+
 export class DenoKVStore implements KVStore {
   kv!: Deno.Kv;
   path: string | undefined;
@@ -22,55 +24,75 @@ export class DenoKVStore implements KVStore {
     }
   }
 
-  async del(key: string): Promise<void> {
-    const res = await this.kv.atomic()
-      .delete([key])
-      .commit();
-    if (!res.ok) {
-      throw res;
-    }
+  del(key: string): Promise<void> {
+    return this.batchDelete([key]);
   }
   async deletePrefix(prefix: string): Promise<void> {
+    const allKeys: string[] = [];
     for await (
-      const result of this.kv.list({
-        start: [prefix],
-        end: [endRange(prefix)],
-      })
+      const result of this.kv.list(
+        prefix
+          ? {
+            start: [prefix],
+            end: [endRange(prefix)],
+          }
+          : { prefix: [] },
+      )
     ) {
-      await this.del(result.key[0] as string);
+      allKeys.push(result.key[0] as string);
     }
+    return this.batchDelete(allKeys);
   }
-  async deleteAll(): Promise<void> {
-    for await (
-      const result of this.kv.list({ prefix: [] })
-    ) {
-      await this.del(result.key[0] as string);
-    }
+  deleteAll(): Promise<void> {
+    return this.deletePrefix("");
   }
-  async set(key: string, value: any): Promise<void> {
-    const res = await this.kv.atomic()
-      .set([key], value)
-      .commit();
-    if (!res.ok) {
-      throw res;
-    }
+  set(key: string, value: any): Promise<void> {
+    return this.batchSet([{ key, value }]);
   }
   async batchSet(kvs: KV[]): Promise<void> {
-    for (const { key, value } of kvs) {
-      await this.set(key, value);
+    // Split into batches of kvBatchSize
+    const batches: KV[][] = [];
+    for (let i = 0; i < kvs.length; i += kvBatchSize) {
+      batches.push(kvs.slice(i, i + kvBatchSize));
+    }
+    for (const batch of batches) {
+      let batchOp = this.kv.atomic();
+      for (const { key, value } of batch) {
+        batchOp = batchOp.set([key], value);
+      }
+      const res = await batchOp.commit();
+      if (!res.ok) {
+        throw res;
+      }
     }
   }
   async batchDelete(keys: string[]): Promise<void> {
-    for (const key of keys) {
-      await this.del(key);
+    const batches: string[][] = [];
+    for (let i = 0; i < keys.length; i += kvBatchSize) {
+      batches.push(keys.slice(i, i + kvBatchSize));
+    }
+    for (const batch of batches) {
+      let batchOp = this.kv.atomic();
+      for (const key of batch) {
+        batchOp = batchOp.delete([key]);
+      }
+      const res = await batchOp.commit();
+      if (!res.ok) {
+        throw res;
+      }
     }
   }
-  batchGet(keys: string[]): Promise<any[]> {
-    const results: Promise<any>[] = [];
-    for (const key of keys) {
-      results.push(this.get(key));
+  async batchGet(keys: string[]): Promise<any[]> {
+    const results: any[] = [];
+    const batches: Deno.KvKey[][] = [];
+    for (let i = 0; i < keys.length; i += kvBatchSize) {
+      batches.push(keys.slice(i, i + kvBatchSize).map((k) => [k]));
     }
-    return Promise.all(results);
+    for (const batch of batches) {
+      const res = await this.kv.getMany(batch);
+      results.push(...res.map((r) => r.value));
+    }
+    return results;
   }
   async get(key: string): Promise<any> {
     return (await this.kv.get([key])).value;
@@ -81,10 +103,14 @@ export class DenoKVStore implements KVStore {
   async queryPrefix(keyPrefix: string): Promise<{ key: string; value: any }[]> {
     const results: { key: string; value: any }[] = [];
     for await (
-      const result of (this.kv).list({
-        start: [keyPrefix],
-        end: [endRange(keyPrefix)],
-      })
+      const result of this.kv.list(
+        keyPrefix
+          ? {
+            start: [keyPrefix],
+            end: [endRange(keyPrefix)],
+          }
+          : { prefix: [] },
+      )
     ) {
       results.push({
         key: result.key[0] as string,