Complete redo of content indexing and querying (#517)

Complete redo of data store
Introduces live queries and live templates

server/server_system.ts

@@ -3,33 +3,35 @@ import { SilverBulletHooks } from "../common/manifest.ts";
 import { loadMarkdownExtensions } from "../common/markdown_parser/markdown_ext.ts";
 import buildMarkdown from "../common/markdown_parser/parser.ts";
 import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives.ts";
-import { FileMetaSpacePrimitives } from "../common/spaces/file_meta_space_primitives.ts";
 import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts";
 import { createSandbox } from "../plugos/environments/webworker_sandbox.ts";
 import { CronHook } from "../plugos/hooks/cron.ts";
 import { EndpointHook } from "../plugos/hooks/endpoint.ts";
 import { EventHook } from "../plugos/hooks/event.ts";
 import { MQHook } from "../plugos/hooks/mq.ts";
-import { DenoKVStore } from "../plugos/lib/kv_store.deno_kv.ts";
 import assetSyscalls from "../plugos/syscalls/asset.ts";
 import { eventSyscalls } from "../plugos/syscalls/event.ts";
-import { mqSyscalls } from "../plugos/syscalls/mq.dexie.ts";
-import { storeSyscalls } from "../plugos/syscalls/store.ts";
+import { mqSyscalls } from "../plugos/syscalls/mq.ts";
 import { System } from "../plugos/system.ts";
 import { Space } from "../web/space.ts";
 import { debugSyscalls } from "../web/syscalls/debug.ts";
-import { pageIndexSyscalls } from "./syscalls/index.ts";
-import { markdownSyscalls } from "../web/syscalls/markdown.ts";
+import { markdownSyscalls } from "../common/syscalls/markdown.ts";
 import { spaceSyscalls } from "./syscalls/space.ts";
 import { systemSyscalls } from "../web/syscalls/system.ts";
-import { yamlSyscalls } from "../web/syscalls/yaml.ts";
+import { yamlSyscalls } from "../common/syscalls/yaml.ts";
 import { Application } from "./deps.ts";
 import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts";
 import { shellSyscalls } from "../plugos/syscalls/shell.deno.ts";
 import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
-import { DenoKvMQ } from "../plugos/lib/mq.deno_kv.ts";
 import { base64EncodedDataUrl } from "../plugos/asset_bundle/base64.ts";
 import { Plug } from "../plugos/plug.ts";
+import { DenoKvPrimitives } from "../plugos/lib/deno_kv_primitives.ts";
+import { DataStore } from "../plugos/lib/datastore.ts";
+import { dataStoreSyscalls } from "../plugos/syscalls/datastore.ts";
+import { DataStoreMQ } from "../plugos/lib/mq.datastore.ts";
+import { language } from "@codemirror/language";
+import { languageSyscalls } from "../common/syscalls/language.ts";
+import { handlebarsSyscalls } from "../common/syscalls/handlebars.ts";
 
 const fileListInterval = 30 * 1000; // 30s
 
@@ -37,8 +39,8 @@ export class ServerSystem {
   system: System<SilverBulletHooks> = new System("server");
   spacePrimitives!: SpacePrimitives;
   denoKv!: Deno.Kv;
-  kvStore!: DenoKVStore;
   listInterval?: number;
+  ds!: DataStore;
 
   constructor(
     private baseSpacePrimitives: SpacePrimitives,
@@ -58,33 +60,26 @@ export class ServerSystem {
     this.system.addHook(cronHook);
 
     this.denoKv = await Deno.openKv(this.dbPath);
-
-    this.kvStore = new DenoKVStore(this.denoKv);
+    this.ds = new DataStore(new DenoKvPrimitives(this.denoKv));
 
     // Endpoint hook
     this.system.addHook(new EndpointHook(this.app, "/_/"));
 
-    // Use DexieMQ for this, in memory
-    const mq = new DenoKvMQ(this.denoKv);
-
-    const pageIndexCalls = pageIndexSyscalls(this.kvStore);
+    const mq = new DataStoreMQ(this.ds);
 
     const plugNamespaceHook = new PlugNamespaceHook();
     this.system.addHook(plugNamespaceHook);
 
     this.system.addHook(new MQHook(this.system, mq));
 
-    this.spacePrimitives = new FileMetaSpacePrimitives(
-      new EventedSpacePrimitives(
-        new PlugSpacePrimitives(
-          this.baseSpacePrimitives,
-          plugNamespaceHook,
-        ),
-        eventHook,
+    this.spacePrimitives = new EventedSpacePrimitives(
+      new PlugSpacePrimitives(
+        this.baseSpacePrimitives,
+        plugNamespaceHook,
       ),
-      pageIndexCalls,
+      eventHook,
     );
-    const space = new Space(this.spacePrimitives, this.kvStore, eventHook);
+    const space = new Space(this.spacePrimitives, this.ds, eventHook);
 
     // Add syscalls
     this.system.registerSyscalls(
@@ -93,10 +88,11 @@ export class ServerSystem {
       spaceSyscalls(space),
       assetSyscalls(this.system),
       yamlSyscalls(),
-      storeSyscalls(this.kvStore),
       systemSyscalls(this.system),
       mqSyscalls(mq),
-      pageIndexCalls,
+      languageSyscalls(),
+      handlebarsSyscalls(),
+      dataStoreSyscalls(this.ds),
       debugSyscalls(),
       markdownSyscalls(buildMarkdown([])), // Will later be replaced with markdown extensions
     );
@@ -147,15 +143,17 @@ export class ServerSystem {
     });
 
     // Check if this space was ever indexed before
-    if (!await this.kvStore.has("$initialIndexCompleted")) {
+    if (!await this.ds.get(["$initialIndexDone"])) {
       console.log("Indexing space for the first time (in the background)");
       this.system.loadedPlugs.get("index")!.invoke(
         "reindexSpace",
         [],
       ).then(() => {
-        this.kvStore.set("$initialIndexCompleted", true);
+        this.ds.set(["$initialIndexDone"], true);
       }).catch(console.error);
     }
+
+    await eventHook.dispatchEvent("system:ready");
   }
 
   async loadPlugs() {