Complete redo of content indexing and querying (#517)
Complete redo of data store Introduces live queries and live templates
This commit is contained in:
Zef Hemel
+2
-2
@@ -1,4 +1,4 @@
|
||||
import { KVStore } from "../plugos/lib/kv_store.ts";
|
||||
import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts";
|
||||
|
||||
export type User = {
|
||||
username: string;
|
||||
@@ -24,7 +24,7 @@ async function createUser(
|
||||
const userPrefix = `u:`;
|
||||
|
||||
export class Authenticator {
|
||||
constructor(private store: KVStore) {
|
||||
constructor(private store: JSONKVStore) {
|
||||
}
|
||||
|
||||
async register(
|
||||
|
||||
@@ -330,8 +330,11 @@ export class HttpServer {
|
||||
}
|
||||
const syscallCommand: SyscallRequest = body;
|
||||
try {
|
||||
const result = await this.system.localSyscall(
|
||||
syscallCommand.ctx,
|
||||
const plug = this.system.loadedPlugs.get(syscallCommand.ctx);
|
||||
if (!plug) {
|
||||
throw new Error(`Plug ${syscallCommand.ctx} not found`);
|
||||
}
|
||||
const result = await plug.syscall(
|
||||
syscallCommand.name,
|
||||
syscallCommand.args,
|
||||
);
|
||||
|
||||
+26
-28
@@ -3,33 +3,35 @@ import { SilverBulletHooks } from "../common/manifest.ts";
|
||||
import { loadMarkdownExtensions } from "../common/markdown_parser/markdown_ext.ts";
|
||||
import buildMarkdown from "../common/markdown_parser/parser.ts";
|
||||
import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives.ts";
|
||||
import { FileMetaSpacePrimitives } from "../common/spaces/file_meta_space_primitives.ts";
|
||||
import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts";
|
||||
import { createSandbox } from "../plugos/environments/webworker_sandbox.ts";
|
||||
import { CronHook } from "../plugos/hooks/cron.ts";
|
||||
import { EndpointHook } from "../plugos/hooks/endpoint.ts";
|
||||
import { EventHook } from "../plugos/hooks/event.ts";
|
||||
import { MQHook } from "../plugos/hooks/mq.ts";
|
||||
import { DenoKVStore } from "../plugos/lib/kv_store.deno_kv.ts";
|
||||
import assetSyscalls from "../plugos/syscalls/asset.ts";
|
||||
import { eventSyscalls } from "../plugos/syscalls/event.ts";
|
||||
import { mqSyscalls } from "../plugos/syscalls/mq.dexie.ts";
|
||||
import { storeSyscalls } from "../plugos/syscalls/store.ts";
|
||||
import { mqSyscalls } from "../plugos/syscalls/mq.ts";
|
||||
import { System } from "../plugos/system.ts";
|
||||
import { Space } from "../web/space.ts";
|
||||
import { debugSyscalls } from "../web/syscalls/debug.ts";
|
||||
import { pageIndexSyscalls } from "./syscalls/index.ts";
|
||||
import { markdownSyscalls } from "../web/syscalls/markdown.ts";
|
||||
import { markdownSyscalls } from "../common/syscalls/markdown.ts";
|
||||
import { spaceSyscalls } from "./syscalls/space.ts";
|
||||
import { systemSyscalls } from "../web/syscalls/system.ts";
|
||||
import { yamlSyscalls } from "../web/syscalls/yaml.ts";
|
||||
import { yamlSyscalls } from "../common/syscalls/yaml.ts";
|
||||
import { Application } from "./deps.ts";
|
||||
import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts";
|
||||
import { shellSyscalls } from "../plugos/syscalls/shell.deno.ts";
|
||||
import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
|
||||
import { DenoKvMQ } from "../plugos/lib/mq.deno_kv.ts";
|
||||
import { base64EncodedDataUrl } from "../plugos/asset_bundle/base64.ts";
|
||||
import { Plug } from "../plugos/plug.ts";
|
||||
import { DenoKvPrimitives } from "../plugos/lib/deno_kv_primitives.ts";
|
||||
import { DataStore } from "../plugos/lib/datastore.ts";
|
||||
import { dataStoreSyscalls } from "../plugos/syscalls/datastore.ts";
|
||||
import { DataStoreMQ } from "../plugos/lib/mq.datastore.ts";
|
||||
import { language } from "@codemirror/language";
|
||||
import { languageSyscalls } from "../common/syscalls/language.ts";
|
||||
import { handlebarsSyscalls } from "../common/syscalls/handlebars.ts";
|
||||
|
||||
const fileListInterval = 30 * 1000; // 30s
|
||||
|
||||
@@ -37,8 +39,8 @@ export class ServerSystem {
|
||||
system: System<SilverBulletHooks> = new System("server");
|
||||
spacePrimitives!: SpacePrimitives;
|
||||
denoKv!: Deno.Kv;
|
||||
kvStore!: DenoKVStore;
|
||||
listInterval?: number;
|
||||
ds!: DataStore;
|
||||
|
||||
constructor(
|
||||
private baseSpacePrimitives: SpacePrimitives,
|
||||
@@ -58,33 +60,26 @@ export class ServerSystem {
|
||||
this.system.addHook(cronHook);
|
||||
|
||||
this.denoKv = await Deno.openKv(this.dbPath);
|
||||
|
||||
this.kvStore = new DenoKVStore(this.denoKv);
|
||||
this.ds = new DataStore(new DenoKvPrimitives(this.denoKv));
|
||||
|
||||
// Endpoint hook
|
||||
this.system.addHook(new EndpointHook(this.app, "/_/"));
|
||||
|
||||
// Use DexieMQ for this, in memory
|
||||
const mq = new DenoKvMQ(this.denoKv);
|
||||
|
||||
const pageIndexCalls = pageIndexSyscalls(this.kvStore);
|
||||
const mq = new DataStoreMQ(this.ds);
|
||||
|
||||
const plugNamespaceHook = new PlugNamespaceHook();
|
||||
this.system.addHook(plugNamespaceHook);
|
||||
|
||||
this.system.addHook(new MQHook(this.system, mq));
|
||||
|
||||
this.spacePrimitives = new FileMetaSpacePrimitives(
|
||||
new EventedSpacePrimitives(
|
||||
new PlugSpacePrimitives(
|
||||
this.baseSpacePrimitives,
|
||||
plugNamespaceHook,
|
||||
),
|
||||
eventHook,
|
||||
this.spacePrimitives = new EventedSpacePrimitives(
|
||||
new PlugSpacePrimitives(
|
||||
this.baseSpacePrimitives,
|
||||
plugNamespaceHook,
|
||||
),
|
||||
pageIndexCalls,
|
||||
eventHook,
|
||||
);
|
||||
const space = new Space(this.spacePrimitives, this.kvStore, eventHook);
|
||||
const space = new Space(this.spacePrimitives, this.ds, eventHook);
|
||||
|
||||
// Add syscalls
|
||||
this.system.registerSyscalls(
|
||||
@@ -93,10 +88,11 @@ export class ServerSystem {
|
||||
spaceSyscalls(space),
|
||||
assetSyscalls(this.system),
|
||||
yamlSyscalls(),
|
||||
storeSyscalls(this.kvStore),
|
||||
systemSyscalls(this.system),
|
||||
mqSyscalls(mq),
|
||||
pageIndexCalls,
|
||||
languageSyscalls(),
|
||||
handlebarsSyscalls(),
|
||||
dataStoreSyscalls(this.ds),
|
||||
debugSyscalls(),
|
||||
markdownSyscalls(buildMarkdown([])), // Will later be replaced with markdown extensions
|
||||
);
|
||||
@@ -147,15 +143,17 @@ export class ServerSystem {
|
||||
});
|
||||
|
||||
// Check if this space was ever indexed before
|
||||
if (!await this.kvStore.has("$initialIndexCompleted")) {
|
||||
if (!await this.ds.get(["$initialIndexDone"])) {
|
||||
console.log("Indexing space for the first time (in the background)");
|
||||
this.system.loadedPlugs.get("index")!.invoke(
|
||||
"reindexSpace",
|
||||
[],
|
||||
).then(() => {
|
||||
this.kvStore.set("$initialIndexCompleted", true);
|
||||
this.ds.set(["$initialIndexDone"], true);
|
||||
}).catch(console.error);
|
||||
}
|
||||
|
||||
await eventHook.dispatchEvent("system:ready");
|
||||
}
|
||||
|
||||
async loadPlugs() {
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
import { DenoKVStore } from "../../plugos/lib/kv_store.deno_kv.ts";
|
||||
import { assertEquals } from "../../test_deps.ts";
|
||||
import { pageIndexSyscalls } from "./index.ts";
|
||||
|
||||
Deno.test("Test KV index", async () => {
|
||||
const ctx: any = {};
|
||||
const denoKv = await Deno.openKv("test.db");
|
||||
const kv = new DenoKVStore(denoKv);
|
||||
const calls = pageIndexSyscalls(kv);
|
||||
await calls["index.set"](ctx, "page", "test", "value");
|
||||
assertEquals(await calls["index.get"](ctx, "page", "test"), "value");
|
||||
await calls["index.delete"](ctx, "page", "test");
|
||||
assertEquals(await calls["index.get"](ctx, "page", "test"), null);
|
||||
await calls["index.batchSet"](ctx, "page", [{
|
||||
key: "attr:test",
|
||||
value: "value",
|
||||
}, {
|
||||
key: "attr:test2",
|
||||
value: "value2",
|
||||
}, { key: "random", value: "value3" }]);
|
||||
await calls["index.batchSet"](ctx, "page2", [{
|
||||
key: "attr:test",
|
||||
value: "value",
|
||||
}, {
|
||||
key: "attr:test2",
|
||||
value: "value2",
|
||||
}, { key: "random", value: "value3" }]);
|
||||
let results = await calls["index.queryPrefix"](ctx, "attr:");
|
||||
assertEquals(results.length, 4);
|
||||
await calls["index.clearPageIndexForPage"](ctx, "page");
|
||||
results = await calls["index.queryPrefix"](ctx, "attr:");
|
||||
assertEquals(results.length, 2);
|
||||
await calls["index.clearPageIndex"](ctx);
|
||||
results = await calls["index.queryPrefix"](ctx, "");
|
||||
assertEquals(results.length, 0);
|
||||
denoKv.close();
|
||||
await Deno.remove("test.db");
|
||||
});
|
||||
@@ -1,100 +0,0 @@
|
||||
import { KVStore } from "../../plugos/lib/kv_store.ts";
|
||||
import type { SysCallMapping } from "../../plugos/system.ts";
|
||||
|
||||
export type KV = {
|
||||
key: string;
|
||||
value: any;
|
||||
};
|
||||
|
||||
// Keyspace:
|
||||
// ["index", page, key] -> value
|
||||
// ["indexByKey", key, page] -> value
|
||||
|
||||
const sep = "!";
|
||||
|
||||
/**
|
||||
* Implements the index syscalls using Deno's KV store.
|
||||
* @param dbFile
|
||||
* @returns
|
||||
*/
|
||||
export function pageIndexSyscalls(kv: KVStore): SysCallMapping {
|
||||
const apiObj: SysCallMapping = {
|
||||
"index.set": (_ctx, page: string, key: string, value: any) => {
|
||||
return kv.batchSet(
|
||||
[{
|
||||
key: `index${sep}${page}${sep}${key}`,
|
||||
value,
|
||||
}, {
|
||||
key: `indexByKey${sep}${key}${sep}${page}`,
|
||||
value,
|
||||
}],
|
||||
);
|
||||
},
|
||||
"index.batchSet": (_ctx, page: string, kvs: KV[]) => {
|
||||
const batch: KV[] = [];
|
||||
for (const { key, value } of kvs) {
|
||||
batch.push({
|
||||
key: `index${sep}${page}${sep}${key}`,
|
||||
value,
|
||||
}, {
|
||||
key: `indexByKey${sep}${key}${sep}${page}`,
|
||||
value,
|
||||
});
|
||||
}
|
||||
return kv.batchSet(batch);
|
||||
},
|
||||
"index.delete": (_ctx, page: string, key: string) => {
|
||||
return kv.batchDelete([
|
||||
`index${sep}${page}${sep}${key}`,
|
||||
`indexByKey${sep}${key}${sep}${page}`,
|
||||
]);
|
||||
},
|
||||
"index.get": (_ctx, page: string, key: string) => {
|
||||
return kv.get(`index${sep}${page}${sep}${key}`);
|
||||
},
|
||||
"index.queryPrefix": async (_ctx, prefix: string) => {
|
||||
const results: { key: string; page: string; value: any }[] = [];
|
||||
for (
|
||||
const result of await kv.queryPrefix(`indexByKey!${prefix}`)
|
||||
) {
|
||||
const [_ns, key, page] = result.key.split(sep);
|
||||
results.push({
|
||||
key,
|
||||
page,
|
||||
value: result.value,
|
||||
});
|
||||
}
|
||||
return results;
|
||||
},
|
||||
"index.clearPageIndexForPage": async (ctx, page: string) => {
|
||||
await apiObj["index.deletePrefixForPage"](ctx, page, "");
|
||||
},
|
||||
"index.deletePrefixForPage": async (_ctx, page: string, prefix: string) => {
|
||||
const allKeys: string[] = [];
|
||||
for (
|
||||
const result of await kv.queryPrefix(
|
||||
`index${sep}${page}${sep}${prefix}`,
|
||||
)
|
||||
) {
|
||||
const [_ns, page, key] = result.key.split(sep);
|
||||
allKeys.push(
|
||||
`index${sep}${page}${sep}${key}`,
|
||||
`indexByKey${sep}${key}${sep}${page}`,
|
||||
);
|
||||
}
|
||||
return kv.batchDelete(allKeys);
|
||||
},
|
||||
"index.clearPageIndex": async () => {
|
||||
const allKeys: string[] = [];
|
||||
for (const result of await kv.queryPrefix(`index${sep}`)) {
|
||||
const [_ns, page, key] = result.key.split(sep);
|
||||
allKeys.push(
|
||||
`index${sep}${page}${sep}${key}`,
|
||||
`indexByKey${sep}${key}${sep}${page}`,
|
||||
);
|
||||
}
|
||||
return kv.batchDelete(allKeys);
|
||||
},
|
||||
};
|
||||
return apiObj;
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
import { FileMeta } from "$sb/types.ts";
|
||||
import { AttachmentMeta, FileMeta, PageMeta } from "$sb/types.ts";
|
||||
import { SysCallMapping } from "../../plugos/system.ts";
|
||||
import type { Space } from "../../web/space.ts";
|
||||
import { AttachmentMeta, PageMeta } from "../../web/types.ts";
|
||||
|
||||
/**
|
||||
* Almost the same as web/syscalls/space.ts except leaving out client-specific stuff
|
||||
|
||||
Reference in New Issue
Block a user