neil/silverbullet
1
0
Fork 0
Code Activity

Complete redo of content indexing and querying (#517)

Browse Source 
Complete redo of data store
Introduces live queries and live templates
This commit is contained in:
Zef Hemel
2023-10-03 14:16:33 +02:00
committed by GitHub
parent 7af98e7c7b
commit 0313565610
200 changed files with 4675 additions and 4363 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/spaces/indexeddb_space_primitives.test.ts → common/spaces/datastore_space_primitives.test.ts
+14 -4
View File
@@ -1,9 +1,17 @@
import { indexedDB } from "https://deno.land/x/indexeddb@1.3.5/ponyfill_memory.ts";
import { IndexedDBSpacePrimitives } from "./indexeddb_space_primitives.ts";
import "https://esm.sh/fake-indexeddb@4.0.2/auto";
import { assertEquals } from "../../test_deps.ts";
import { DataStore } from "../../plugos/lib/datastore.ts";
import { IndexedDBKvPrimitives } from "../../plugos/lib/indexeddb_kv_primitives.ts";
import { DataStoreSpacePrimitives } from "./datastore_space_primitives.ts";
Deno.test("IndexedDBSpacePrimitives", async () => {
const space = new IndexedDBSpacePrimitives("test", indexedDB);
Deno.test("DataStoreSpacePrimitives", {
sanitizeResources: false,
sanitizeOps: false,
}, async () => {
const db = new IndexedDBKvPrimitives("test");
await db.init();
const space = new DataStoreSpacePrimitives(new DataStore(db));
const files = await space.fetchFileList();
assertEquals(files, []);
// Write text file
@@ -28,6 +36,8 @@ Deno.test("IndexedDBSpacePrimitives", async () => {
await space.deleteFile("test.bin");
assertEquals(await space.fetchFileList(), [fileMeta]);
db.close();
});
function stringToBytes(str: string): Uint8Array {
common/spaces/indexeddb_space_primitives.ts → common/spaces/datastore_space_primitives.ts
+31 -25
View File
@@ -1,7 +1,7 @@
import type { SpacePrimitives } from "./space_primitives.ts";
import Dexie, { Table } from "dexie";
import { mime } from "../deps.ts";
import { FileMeta } from "$sb/types.ts";
import { DataStore } from "../../plugos/lib/datastore.ts";
export type FileContent = {
name: string;
@@ -9,34 +9,27 @@ export type FileContent = {
data: Uint8Array;
};
export class IndexedDBSpacePrimitives implements SpacePrimitives {
private db: Dexie;
filesMetaTable: Table<FileMeta, string>;
filesContentTable: Table<FileContent, string>;
const filesMetaPrefix = ["file", "meta"];
const filesContentPrefix = ["file", "content"];
export class DataStoreSpacePrimitives implements SpacePrimitives {
constructor(
dbName: string,
indexedDB?: any,
private ds: DataStore,
) {
this.db = new Dexie(dbName, {
indexedDB,
});
this.db.version(1).stores({
fileMeta: "name",
fileContent: "name",
});
this.filesMetaTable = this.db.table("fileMeta");
this.filesContentTable = this.db.table<FileContent, string>("fileContent");
}
fetchFileList(): Promise<FileMeta[]> {
return this.filesMetaTable.toArray();
async fetchFileList(): Promise<FileMeta[]> {
return (await this.ds.query<FileMeta>({ prefix: filesMetaPrefix }))
.map((kv) => kv.value);
}
async readFile(
name: string,
): Promise<{ data: Uint8Array; meta: FileMeta }> {
const fileContent = await this.filesContentTable.get(name);
const fileContent = await this.ds.get<FileContent>([
...filesContentPrefix,
name,
]);
if (!fileContent) {
throw new Error("Not found");
}
@@ -60,22 +53,35 @@ export class IndexedDBSpacePrimitives implements SpacePrimitives {
size: data.byteLength,
perm: suggestedMeta?.perm || "rw",
};
await this.filesContentTable.put({ name, data, meta });
await this.filesMetaTable.put(meta);
await this.ds.batchSet<FileMeta | FileContent>([
{
key: [...filesContentPrefix, name],
value: { name, data, meta },
},
{
key: [...filesMetaPrefix, name],
value: meta,
},
]);
return meta;
}
async deleteFile(name: string): Promise<void> {
const fileMeta = await this.filesMetaTable.get(name);
const fileMeta = await this.ds.get<FileMeta>([
...filesMetaPrefix,
name,
]);
if (!fileMeta) {
throw new Error("Not found");
}
await this.filesMetaTable.delete(name);
await this.filesContentTable.delete(name);
return this.ds.batchDelete([
[...filesMetaPrefix, name],
[...filesContentPrefix, name],
]);
}
async getFileMeta(name: string): Promise<FileMeta> {
const fileMeta = await this.filesMetaTable.get(name);
const fileMeta = await this.ds.get([...filesMetaPrefix, name]);
if (!fileMeta) {
throw new Error("Not found");
}
common/spaces/disk_space_primitives.ts
+1 -8
View File
@@ -93,14 +93,7 @@ export class DiskSpacePrimitives implements SpacePrimitives {
file.close();
// Fetch new metadata
const s = await Deno.stat(localPath);
return {
name: name,
size: s.size,
contentType: lookupContentType(name),
lastModified: s.mtime!.getTime(),
perm: "rw",
};
return this.getFileMeta(name);
} catch (e) {
console.error("Error while writing file", name, e);
throw Error(`Could not write ${name}`);
common/spaces/evented_space_primitives.ts
+12 -17
View File
@@ -47,7 +47,7 @@ export class EventedSpacePrimitives implements SpacePrimitives {
oldHash !== newHash
)
) {
this.dispatchEvent("file:changed", meta.name);
await this.dispatchEvent("file:changed", meta.name);
}
// Page found, not deleted
deletedFiles.delete(meta.name);
@@ -58,7 +58,7 @@ export class EventedSpacePrimitives implements SpacePrimitives {
for (const deletedFile of deletedFiles) {
delete this.spaceSnapshot[deletedFile];
this.dispatchEvent("file:deleted", deletedFile);
await this.dispatchEvent("file:deleted", deletedFile);
if (deletedFile.endsWith(".md")) {
const pageName = deletedFile.substring(0, deletedFile.length - 3);
@@ -66,7 +66,7 @@ export class EventedSpacePrimitives implements SpacePrimitives {
}
}
this.dispatchEvent("file:listed", newFileList);
await this.dispatchEvent("file:listed", newFileList);
this.alreadyFetching = false;
this.initialFileListLoad = false;
return newFileList;
@@ -93,7 +93,7 @@ export class EventedSpacePrimitives implements SpacePrimitives {
meta,
);
if (!selfUpdate) {
this.dispatchEvent("file:changed", name, true);
await this.dispatchEvent("file:changed", name, true);
}
this.spaceSnapshot[name] = newMeta.lastModified;
@@ -104,16 +104,11 @@ export class EventedSpacePrimitives implements SpacePrimitives {
const decoder = new TextDecoder("utf-8");
text = decoder.decode(data);
this.dispatchEvent("page:saved", pageName, newMeta)
.then(() => {
return this.dispatchEvent("page:index_text", {
name: pageName,
text,
});
})
.catch((e) => {
console.error("Error dispatching page:saved event", e);
});
await this.dispatchEvent("page:saved", pageName, newMeta);
await this.dispatchEvent("page:index_text", {
name: pageName,
text,
});
}
return newMeta;
}
@@ -134,9 +129,9 @@ export class EventedSpacePrimitives implements SpacePrimitives {
this.triggerEventsAndCache(name, newMeta.lastModified);
return newMeta;
} catch (e: any) {
console.log("Checking error", e, name);
// console.log("Checking error", e, name);
if (e.message === "Not found") {
this.dispatchEvent("file:deleted", name);
await this.dispatchEvent("file:deleted", name);
if (name.endsWith(".md")) {
const pageName = name.substring(0, name.length - 3);
await this.dispatchEvent("page:deleted", pageName);
@@ -154,6 +149,6 @@ export class EventedSpacePrimitives implements SpacePrimitives {
// await this.getPageMeta(name); // Check if page exists, if not throws Error
await this.wrapped.deleteFile(name);
delete this.spaceSnapshot[name];
this.dispatchEvent("file:deleted", name);
await this.dispatchEvent("file:deleted", name);
}
}
common/spaces/file_meta_space_primitives.ts
-86
View File
@@ -1,86 +0,0 @@
import { SpacePrimitives } from "./space_primitives.ts";
import type { SysCallMapping } from "../../plugos/system.ts";
import { FileMeta } from "$sb/types.ts";
// Enriches the file list listing with custom metadata from the page index
export class FileMetaSpacePrimitives implements SpacePrimitives {
constructor(
private wrapped: SpacePrimitives,
private indexSyscalls: SysCallMapping,
) {
}
async fetchFileList(): Promise<FileMeta[]> {
const files = await this.wrapped.fetchFileList();
// Enrich the file list with custom meta data (for pages)
const allFilesMap: Map<string, any> = new Map(
files.map((fm) => [fm.name, fm]),
);
for (
const { page, value } of await this.indexSyscalls["index.queryPrefix"](
{} as any,
"meta:",
)
) {
const p = allFilesMap.get(`${page}.md`);
if (p) {
for (const [k, v] of Object.entries(value)) {
if (
["name", "lastModified", "size", "perm", "contentType"].includes(k)
) {
continue;
}
p[k] = v;
}
}
}
return [...allFilesMap.values()];
}
readFile(
name: string,
): Promise<{ data: Uint8Array; meta: FileMeta }> {
return this.wrapped.readFile(name);
}
async getFileMeta(name: string): Promise<FileMeta> {
const meta = await this.wrapped.getFileMeta(name);
if (name.endsWith(".md")) {
const pageName = name.slice(0, -3);
const additionalMeta = await this.indexSyscalls["index.get"](
{} as any,
pageName,
"meta:",
);
if (additionalMeta) {
for (const [k, v] of Object.entries(additionalMeta)) {
if (
["name", "lastModified", "size", "perm", "contentType"].includes(k)
) {
continue;
}
meta[k] = v;
}
}
}
return meta;
}
writeFile(
name: string,
data: Uint8Array,
selfUpdate?: boolean,
meta?: FileMeta,
): Promise<FileMeta> {
return this.wrapped.writeFile(
name,
data,
selfUpdate,
meta,
);
}
deleteFile(name: string): Promise<void> {
return this.wrapped.deleteFile(name);
}
}