Refactor all the things

2023-08-28 17:12:15 +02:00
parent 54d2deea15
commit 5ff1a8bae3
87 changed files with 930 additions and 896 deletions
@@ -11,7 +11,6 @@ import { EndpointHook } from "../plugos/hooks/endpoint.ts";
 import { EventHook } from "../plugos/hooks/event.ts";
 import { MQHook } from "../plugos/hooks/mq.ts";
 import { DenoKVStore } from "../plugos/lib/kv_store.deno_kv.ts";
-import { DexieMQ } from "../plugos/lib/mq.dexie.ts";
 import assetSyscalls from "../plugos/syscalls/asset.ts";
 import { eventSyscalls } from "../plugos/syscalls/event.ts";
 import { mqSyscalls } from "../plugos/syscalls/mq.dexie.ts";
@@ -19,16 +18,16 @@ import { storeSyscalls } from "../plugos/syscalls/store.ts";
 import { System } from "../plugos/system.ts";
 import { Space } from "../web/space.ts";
 import { debugSyscalls } from "../web/syscalls/debug.ts";
-import { pageIndexSyscalls } from "../cli/syscalls/index.ts";
+import { pageIndexSyscalls } from "./syscalls/index.ts";
 import { markdownSyscalls } from "../web/syscalls/markdown.ts";
-import { spaceSyscalls } from "../cli/syscalls/space.ts";
+import { spaceSyscalls } from "./syscalls/space.ts";
 import { systemSyscalls } from "../web/syscalls/system.ts";
 import { yamlSyscalls } from "../web/syscalls/yaml.ts";
 import { Application, path } from "./deps.ts";
 import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts";
 import { shellSyscalls } from "../plugos/syscalls/shell.deno.ts";
-import { IDBKeyRange, indexedDB } from "https://esm.sh/fake-indexeddb@4.0.2";
 import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
+import { DenoKvMQ } from "../plugos/lib/mq.deno_kv.ts";

 const fileListInterval = 30 * 1000; // 30s

@@ -38,6 +37,7 @@ export class ServerSystem {
  private requeueInterval?: number;
  kvStore?: DenoKVStore;
  listInterval?: number;
+  denoKv!: Deno.Kv;

  constructor(
    private baseSpacePrimitives: SpacePrimitives,
@@ -56,19 +56,15 @@ export class ServerSystem {
    const cronHook = new CronHook(this.system);
    this.system.addHook(cronHook);

-    this.kvStore = new DenoKVStore();
-    await this.kvStore.init(this.dbPath);
+    this.denoKv = await Deno.openKv(this.dbPath);
+
+    this.kvStore = new DenoKVStore(this.denoKv);

    // Endpoint hook
    this.system.addHook(new EndpointHook(this.app, "/_/"));

    // Use DexieMQ for this, in memory
-    const mq = new DexieMQ("mq", indexedDB, IDBKeyRange);
-
-    this.requeueInterval = setInterval(() => {
-      // Timeout after 5s, retries 3 times, otherwise drops the message (no DLQ)
-      mq.requeueTimeouts(5000, 3, true).catch(console.error);
-    }, 20000); // Look to requeue every 20s
+    const mq = new DenoKvMQ(this.denoKv);

    const pageIndexCalls = pageIndexSyscalls(this.kvStore);

@@ -148,9 +144,7 @@ export class ServerSystem {
    const tempDir = await Deno.makeTempDir();
    try {
      for (const { name } of await this.spacePrimitives.fetchFileList()) {
-        if (
-          name.endsWith(".plug.js") // && !filePath.includes("search.plug.js")
-        ) {
+        if (name.endsWith(".plug.js")) {
          const plugPath = path.join(tempDir, name);
          await Deno.mkdir(path.dirname(plugPath), { recursive: true });
          await Deno.writeFile(
@@ -0,0 +1,38 @@
+import { DenoKVStore } from "../../plugos/lib/kv_store.deno_kv.ts";
+import { assertEquals } from "../../test_deps.ts";
+import { pageIndexSyscalls } from "./index.ts";
+
+Deno.test("Test KV index", async () => {
+  const ctx: any = {};
+  const denoKv = await Deno.openKv("test.db");
+  const kv = new DenoKVStore(denoKv);
+  const calls = pageIndexSyscalls(kv);
+  await calls["index.set"](ctx, "page", "test", "value");
+  assertEquals(await calls["index.get"](ctx, "page", "test"), "value");
+  await calls["index.delete"](ctx, "page", "test");
+  assertEquals(await calls["index.get"](ctx, "page", "test"), null);
+  await calls["index.batchSet"](ctx, "page", [{
+    key: "attr:test",
+    value: "value",
+  }, {
+    key: "attr:test2",
+    value: "value2",
+  }, { key: "random", value: "value3" }]);
+  await calls["index.batchSet"](ctx, "page2", [{
+    key: "attr:test",
+    value: "value",
+  }, {
+    key: "attr:test2",
+    value: "value2",
+  }, { key: "random", value: "value3" }]);
+  let results = await calls["index.queryPrefix"](ctx, "attr:");
+  assertEquals(results.length, 4);
+  await calls["index.clearPageIndexForPage"](ctx, "page");
+  results = await calls["index.queryPrefix"](ctx, "attr:");
+  assertEquals(results.length, 2);
+  await calls["index.clearPageIndex"](ctx);
+  results = await calls["index.queryPrefix"](ctx, "");
+  assertEquals(results.length, 0);
+  denoKv.close();
+  await Deno.remove("test.db");
+});
@@ -0,0 +1,100 @@
+import { KVStore } from "../../plugos/lib/kv_store.ts";
+import type { SysCallMapping } from "../../plugos/system.ts";
+
+export type KV = {
+  key: string;
+  value: any;
+};
+
+// Keyspace:
+// ["index", page, key] -> value
+// ["indexByKey", key, page] -> value
+
+const sep = "!";
+
+/**
+ * Implements the index syscalls using Deno's KV store.
+ * @param dbFile
+ * @returns
+ */
+export function pageIndexSyscalls(kv: KVStore): SysCallMapping {
+  const apiObj: SysCallMapping = {
+    "index.set": (_ctx, page: string, key: string, value: any) => {
+      return kv.batchSet(
+        [{
+          key: `index${sep}${page}${sep}${key}`,
+          value,
+        }, {
+          key: `indexByKey${sep}${key}${sep}${page}`,
+          value,
+        }],
+      );
+    },
+    "index.batchSet": (_ctx, page: string, kvs: KV[]) => {
+      const batch: KV[] = [];
+      for (const { key, value } of kvs) {
+        batch.push({
+          key: `index${sep}${page}${sep}${key}`,
+          value,
+        }, {
+          key: `indexByKey${sep}${key}${sep}${page}`,
+          value,
+        });
+      }
+      return kv.batchSet(batch);
+    },
+    "index.delete": (_ctx, page: string, key: string) => {
+      return kv.batchDelete([
+        `index${sep}${page}${sep}${key}`,
+        `indexByKey${sep}${key}${sep}${page}`,
+      ]);
+    },
+    "index.get": (_ctx, page: string, key: string) => {
+      return kv.get(`index${sep}${page}${sep}${key}`);
+    },
+    "index.queryPrefix": async (_ctx, prefix: string) => {
+      const results: { key: string; page: string; value: any }[] = [];
+      for (
+        const result of await kv.queryPrefix(`indexByKey!${prefix}`)
+      ) {
+        const [_ns, key, page] = result.key.split(sep);
+        results.push({
+          key,
+          page,
+          value: result.value,
+        });
+      }
+      return results;
+    },
+    "index.clearPageIndexForPage": async (ctx, page: string) => {
+      await apiObj["index.deletePrefixForPage"](ctx, page, "");
+    },
+    "index.deletePrefixForPage": async (_ctx, page: string, prefix: string) => {
+      const allKeys: string[] = [];
+      for (
+        const result of await kv.queryPrefix(
+          `index${sep}${page}${sep}${prefix}`,
+        )
+      ) {
+        const [_ns, page, key] = result.key.split(sep);
+        allKeys.push(
+          `index${sep}${page}${sep}${key}`,
+          `indexByKey${sep}${key}${sep}${page}`,
+        );
+      }
+      return kv.batchDelete(allKeys);
+    },
+    "index.clearPageIndex": async () => {
+      const allKeys: string[] = [];
+      for (const result of await kv.queryPrefix(`index${sep}`)) {
+        const [_ns, page, key] = result.key.split(sep);
+        allKeys.push(
+          `index${sep}${page}${sep}${key}`,
+          `indexByKey${sep}${key}${sep}${page}`,
+        );
+      }
+      return kv.batchDelete(allKeys);
+    },
+  };
+  return apiObj;
+}
@@ -0,0 +1,83 @@
+import { FileMeta } from "$sb/types.ts";
+import { SysCallMapping } from "../../plugos/system.ts";
+import type { Space } from "../../web/space.ts";
+import { AttachmentMeta, PageMeta } from "../../web/types.ts";
+
+/**
+ * Almost the same as web/syscalls/space.ts except leaving out client-specific stuff
+ */
+export function spaceSyscalls(space: Space): SysCallMapping {
+  return {
+    "space.listPages": (): Promise<PageMeta[]> => {
+      return space.fetchPageList();
+    },
+    "space.readPage": async (
+      _ctx,
+      name: string,
+    ): Promise<string> => {
+      return (await space.readPage(name)).text;
+    },
+    "space.getPageMeta": (_ctx, name: string): Promise<PageMeta> => {
+      return space.getPageMeta(name);
+    },
+    "space.writePage": (
+      _ctx,
+      name: string,
+      text: string,
+    ): Promise<PageMeta> => {
+      return space.writePage(name, text);
+    },
+    "space.deletePage": async (_ctx, name: string) => {
+      await space.deletePage(name);
+    },
+    "space.listPlugs": (): Promise<string[]> => {
+      return space.listPlugs();
+    },
+    "space.listAttachments": async (): Promise<AttachmentMeta[]> => {
+      return await space.fetchAttachmentList();
+    },
+    "space.readAttachment": async (
+      _ctx,
+      name: string,
+    ): Promise<Uint8Array> => {
+      return (await space.readAttachment(name)).data;
+    },
+    "space.getAttachmentMeta": async (
+      _ctx,
+      name: string,
+    ): Promise<AttachmentMeta> => {
+      return await space.getAttachmentMeta(name);
+    },
+    "space.writeAttachment": (
+      _ctx,
+      name: string,
+      data: Uint8Array,
+    ): Promise<AttachmentMeta> => {
+      return space.writeAttachment(name, data);
+    },
+    "space.deleteAttachment": async (_ctx, name: string) => {
+      await space.deleteAttachment(name);
+    },
+
+    // FS
+    "space.listFiles": (): Promise<FileMeta[]> => {
+      return space.spacePrimitives.fetchFileList();
+    },
+    "space.getFileMeta": (_ctx, name: string): Promise<FileMeta> => {
+      return space.spacePrimitives.getFileMeta(name);
+    },
+    "space.readFile": async (_ctx, name: string): Promise<Uint8Array> => {
+      return (await space.spacePrimitives.readFile(name)).data;
+    },
+    "space.writeFile": (
+      _ctx,
+      name: string,
+      data: Uint8Array,
+    ): Promise<FileMeta> => {
+      return space.spacePrimitives.writeFile(name, data);
+    },
+    "space.deleteFile": (_ctx, name: string) => {
+      return space.spacePrimitives.deleteFile(name);
+    },
+  };
+}