Complete redo of content indexing and querying (#517)

Complete redo of data store
Introduces live queries and live templates

server/syscalls/index.test.ts

@@ -1,38 +0,0 @@
-import { DenoKVStore } from "../../plugos/lib/kv_store.deno_kv.ts";
-import { assertEquals } from "../../test_deps.ts";
-import { pageIndexSyscalls } from "./index.ts";
-
-Deno.test("Test KV index", async () => {
-  const ctx: any = {};
-  const denoKv = await Deno.openKv("test.db");
-  const kv = new DenoKVStore(denoKv);
-  const calls = pageIndexSyscalls(kv);
-  await calls["index.set"](ctx, "page", "test", "value");
-  assertEquals(await calls["index.get"](ctx, "page", "test"), "value");
-  await calls["index.delete"](ctx, "page", "test");
-  assertEquals(await calls["index.get"](ctx, "page", "test"), null);
-  await calls["index.batchSet"](ctx, "page", [{
-    key: "attr:test",
-    value: "value",
-  }, {
-    key: "attr:test2",
-    value: "value2",
-  }, { key: "random", value: "value3" }]);
-  await calls["index.batchSet"](ctx, "page2", [{
-    key: "attr:test",
-    value: "value",
-  }, {
-    key: "attr:test2",
-    value: "value2",
-  }, { key: "random", value: "value3" }]);
-  let results = await calls["index.queryPrefix"](ctx, "attr:");
-  assertEquals(results.length, 4);
-  await calls["index.clearPageIndexForPage"](ctx, "page");
-  results = await calls["index.queryPrefix"](ctx, "attr:");
-  assertEquals(results.length, 2);
-  await calls["index.clearPageIndex"](ctx);
-  results = await calls["index.queryPrefix"](ctx, "");
-  assertEquals(results.length, 0);
-  denoKv.close();
-  await Deno.remove("test.db");
-});

server/syscalls/index.ts

@@ -1,100 +0,0 @@
-import { KVStore } from "../../plugos/lib/kv_store.ts";
-import type { SysCallMapping } from "../../plugos/system.ts";
-
-export type KV = {
-  key: string;
-  value: any;
-};
-
-// Keyspace:
-// ["index", page, key] -> value
-// ["indexByKey", key, page] -> value
-
-const sep = "!";
-
-/**
- * Implements the index syscalls using Deno's KV store.
- * @param dbFile
- * @returns
- */
-export function pageIndexSyscalls(kv: KVStore): SysCallMapping {
-  const apiObj: SysCallMapping = {
-    "index.set": (_ctx, page: string, key: string, value: any) => {
-      return kv.batchSet(
-        [{
-          key: `index${sep}${page}${sep}${key}`,
-          value,
-        }, {
-          key: `indexByKey${sep}${key}${sep}${page}`,
-          value,
-        }],
-      );
-    },
-    "index.batchSet": (_ctx, page: string, kvs: KV[]) => {
-      const batch: KV[] = [];
-      for (const { key, value } of kvs) {
-        batch.push({
-          key: `index${sep}${page}${sep}${key}`,
-          value,
-        }, {
-          key: `indexByKey${sep}${key}${sep}${page}`,
-          value,
-        });
-      }
-      return kv.batchSet(batch);
-    },
-    "index.delete": (_ctx, page: string, key: string) => {
-      return kv.batchDelete([
-        `index${sep}${page}${sep}${key}`,
-        `indexByKey${sep}${key}${sep}${page}`,
-      ]);
-    },
-    "index.get": (_ctx, page: string, key: string) => {
-      return kv.get(`index${sep}${page}${sep}${key}`);
-    },
-    "index.queryPrefix": async (_ctx, prefix: string) => {
-      const results: { key: string; page: string; value: any }[] = [];
-      for (
-        const result of await kv.queryPrefix(`indexByKey!${prefix}`)
-      ) {
-        const [_ns, key, page] = result.key.split(sep);
-        results.push({
-          key,
-          page,
-          value: result.value,
-        });
-      }
-      return results;
-    },
-    "index.clearPageIndexForPage": async (ctx, page: string) => {
-      await apiObj["index.deletePrefixForPage"](ctx, page, "");
-    },
-    "index.deletePrefixForPage": async (_ctx, page: string, prefix: string) => {
-      const allKeys: string[] = [];
-      for (
-        const result of await kv.queryPrefix(
-          `index${sep}${page}${sep}${prefix}`,
-        )
-      ) {
-        const [_ns, page, key] = result.key.split(sep);
-        allKeys.push(
-          `index${sep}${page}${sep}${key}`,
-          `indexByKey${sep}${key}${sep}${page}`,
-        );
-      }
-      return kv.batchDelete(allKeys);
-    },
-    "index.clearPageIndex": async () => {
-      const allKeys: string[] = [];
-      for (const result of await kv.queryPrefix(`index${sep}`)) {
-        const [_ns, page, key] = result.key.split(sep);
-        allKeys.push(
-          `index${sep}${page}${sep}${key}`,
-          `indexByKey${sep}${key}${sep}${page}`,
-        );
-      }
-      return kv.batchDelete(allKeys);
-    },
-  };
-  return apiObj;
-}

server/syscalls/space.ts

@@ -1,7 +1,6 @@
-import { FileMeta } from "$sb/types.ts";
+import { AttachmentMeta, FileMeta, PageMeta } from "$sb/types.ts";
 import { SysCallMapping } from "../../plugos/system.ts";
 import type { Space } from "../../web/space.ts";
-import { AttachmentMeta, PageMeta } from "../../web/types.ts";
 
 /**
  * Almost the same as web/syscalls/space.ts except leaving out client-specific stuff