Complete redo of content indexing and querying (#517)
Complete redo of data store Introduces live queries and live templates
This commit is contained in:
Zef Hemel
+80
-147
@@ -1,183 +1,116 @@
|
||||
import { KvKey, KvPrimitives } from "./kv_primitives.ts";
|
||||
|
||||
export type { KvKey };
|
||||
|
||||
export type KvValue = any;
|
||||
|
||||
export type KV = {
|
||||
key: KvKey;
|
||||
value: KvValue;
|
||||
};
|
||||
|
||||
export type KvOrderBy = {
|
||||
attribute: string;
|
||||
desc: boolean;
|
||||
};
|
||||
|
||||
export type KvQuery = {
|
||||
prefix: KvKey;
|
||||
filter?: KvQueryFilter;
|
||||
orderBy?: KvOrderBy[];
|
||||
limit?: number;
|
||||
select?: string[];
|
||||
};
|
||||
|
||||
export type KvQueryFilter =
|
||||
| ["=", string, any]
|
||||
| ["!=", string, any]
|
||||
| ["=~", string, RegExp]
|
||||
| ["!=~", string, RegExp]
|
||||
| ["prefix", string, string]
|
||||
| ["<", string, any]
|
||||
| ["<=", string, any]
|
||||
| [">", string, any]
|
||||
| [">=", string, any]
|
||||
| ["in", string, any[]]
|
||||
| ["and", KvQueryFilter, KvQueryFilter]
|
||||
| ["or", KvQueryFilter, KvQueryFilter];
|
||||
|
||||
function filterKvQuery(kvQuery: KvQueryFilter, obj: KvValue): boolean {
|
||||
const [op, op1, op2] = kvQuery;
|
||||
|
||||
if (op === "and") {
|
||||
return filterKvQuery(op1, obj) &&
|
||||
filterKvQuery(op2, obj);
|
||||
} else if (op === "or") {
|
||||
return filterKvQuery(op1, obj) || filterKvQuery(op2, obj);
|
||||
}
|
||||
|
||||
// Look up the value of the attribute, supporting nested attributes via `attr.attr2.attr3`, and empty attribute value signifies the root object
|
||||
let attributeVal = obj;
|
||||
for (const part of op1.split(".")) {
|
||||
if (!part) {
|
||||
continue;
|
||||
}
|
||||
if (attributeVal === undefined) {
|
||||
return false;
|
||||
}
|
||||
attributeVal = attributeVal[part];
|
||||
}
|
||||
|
||||
// And apply the operator
|
||||
switch (op) {
|
||||
case "=": {
|
||||
if (Array.isArray(attributeVal) && !Array.isArray(op2)) {
|
||||
// Record property is an array, and value is a scalar: find the value in the array
|
||||
if (attributeVal.includes(op2)) {
|
||||
return true;
|
||||
}
|
||||
} else if (Array.isArray(attributeVal) && Array.isArray(obj)) {
|
||||
// Record property is an array, and value is an array: find the value in the array
|
||||
if (attributeVal.some((v) => obj.includes(v))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return attributeVal === op2;
|
||||
}
|
||||
case "!=":
|
||||
return attributeVal !== op2;
|
||||
case "=~":
|
||||
return op2.test(attributeVal);
|
||||
case "!=~":
|
||||
return !op2.test(attributeVal);
|
||||
case "prefix":
|
||||
return attributeVal.startsWith(op2);
|
||||
case "<":
|
||||
return attributeVal < op2;
|
||||
case "<=":
|
||||
return attributeVal <= op2;
|
||||
case ">":
|
||||
return attributeVal > op2;
|
||||
case ">=":
|
||||
return attributeVal >= op2;
|
||||
case "in":
|
||||
return op2.includes(attributeVal);
|
||||
default:
|
||||
throw new Error(`Unupported operator: ${op}`);
|
||||
}
|
||||
}
|
||||
import { applyQueryNoFilterKV, evalQueryExpression } from "$sb/lib/query.ts";
|
||||
import { FunctionMap, KV, KvKey, KvQuery } from "$sb/types.ts";
|
||||
import { builtinFunctions } from "$sb/lib/builtin_query_functions.ts";
|
||||
import { KvPrimitives } from "./kv_primitives.ts";
|
||||
|
||||
/**
|
||||
* This is the data store class you'll actually want to use, wrapping the primitives
|
||||
* in a more user-friendly way
|
||||
*/
|
||||
export class DataStore {
|
||||
constructor(private kv: KvPrimitives) {
|
||||
constructor(
|
||||
private kv: KvPrimitives,
|
||||
private prefix: KvKey = [],
|
||||
private functionMap: FunctionMap = builtinFunctions,
|
||||
) {
|
||||
}
|
||||
|
||||
async get(key: KvKey): Promise<KvValue> {
|
||||
return (await this.kv.batchGet([key]))[0];
|
||||
prefixed(prefix: KvKey): DataStore {
|
||||
return new DataStore(
|
||||
this.kv,
|
||||
[...this.prefix, ...prefix],
|
||||
this.functionMap,
|
||||
);
|
||||
}
|
||||
|
||||
batchGet(keys: KvKey[]): Promise<KvValue[]> {
|
||||
return this.kv.batchGet(keys);
|
||||
async get<T = any>(key: KvKey): Promise<T | null> {
|
||||
return (await this.batchGet([key]))[0];
|
||||
}
|
||||
|
||||
set(key: KvKey, value: KvValue): Promise<void> {
|
||||
return this.kv.batchSet([{ key, value }]);
|
||||
batchGet<T = any>(keys: KvKey[]): Promise<(T | null)[]> {
|
||||
return this.kv.batchGet(keys.map((key) => this.applyPrefix(key)));
|
||||
}
|
||||
|
||||
batchSet(entries: KV[]): Promise<void> {
|
||||
return this.kv.batchSet(entries);
|
||||
set(key: KvKey, value: any): Promise<void> {
|
||||
return this.batchSet([{ key, value }]);
|
||||
}
|
||||
|
||||
batchSet<T = any>(entries: KV<T>[]): Promise<void> {
|
||||
const allKeyStrings = new Set<string>();
|
||||
const uniqueEntries: KV[] = [];
|
||||
for (const { key, value } of entries) {
|
||||
const keyString = JSON.stringify(key);
|
||||
if (allKeyStrings.has(keyString)) {
|
||||
console.warn(`Duplicate key ${keyString} in batchSet, skipping`);
|
||||
} else {
|
||||
allKeyStrings.add(keyString);
|
||||
uniqueEntries.push({ key: this.applyPrefix(key), value });
|
||||
}
|
||||
}
|
||||
return this.kv.batchSet(uniqueEntries);
|
||||
}
|
||||
|
||||
delete(key: KvKey): Promise<void> {
|
||||
return this.kv.batchDelete([key]);
|
||||
return this.batchDelete([key]);
|
||||
}
|
||||
|
||||
batchDelete(keys: KvKey[]): Promise<void> {
|
||||
return this.kv.batchDelete(keys);
|
||||
return this.kv.batchDelete(keys.map((key) => this.applyPrefix(key)));
|
||||
}
|
||||
|
||||
async query(query: KvQuery): Promise<KV[]> {
|
||||
const results: KV[] = [];
|
||||
async query<T = any>(query: KvQuery): Promise<KV<T>[]> {
|
||||
const results: KV<T>[] = [];
|
||||
let itemCount = 0;
|
||||
// Accumuliate results
|
||||
for await (const entry of this.kv.query({ prefix: query.prefix })) {
|
||||
// Accumulate results
|
||||
let limit = Infinity;
|
||||
const prefixedQuery: KvQuery = {
|
||||
...query,
|
||||
prefix: query.prefix ? this.applyPrefix(query.prefix) : undefined,
|
||||
};
|
||||
if (query.limit) {
|
||||
limit = evalQueryExpression(query.limit, {}, this.functionMap);
|
||||
}
|
||||
for await (
|
||||
const entry of this.kv.query(prefixedQuery)
|
||||
) {
|
||||
// Filter
|
||||
if (query.filter && !filterKvQuery(query.filter, entry.value)) {
|
||||
if (
|
||||
query.filter &&
|
||||
!evalQueryExpression(query.filter, entry.value, this.functionMap)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
results.push(entry);
|
||||
itemCount++;
|
||||
// Stop when the limit has been reached
|
||||
if (itemCount === query.limit) {
|
||||
if (itemCount === limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Order by
|
||||
if (query.orderBy) {
|
||||
results.sort((a, b) => {
|
||||
const aVal = a.value;
|
||||
const bVal = b.value;
|
||||
for (const { attribute, desc } of query.orderBy!) {
|
||||
if (
|
||||
aVal[attribute] < bVal[attribute] || aVal[attribute] === undefined
|
||||
) {
|
||||
return desc ? 1 : -1;
|
||||
}
|
||||
if (
|
||||
aVal[attribute] > bVal[attribute] || bVal[attribute] === undefined
|
||||
) {
|
||||
return desc ? -1 : 1;
|
||||
}
|
||||
}
|
||||
// Consider them equal. This helps with comparing arrays (like tags)
|
||||
return 0;
|
||||
});
|
||||
}
|
||||
// Apply order by, limit, and select
|
||||
return applyQueryNoFilterKV(prefixedQuery, results, this.functionMap).map((
|
||||
{ key, value },
|
||||
) => ({ key: this.stripPrefix(key), value }));
|
||||
}
|
||||
|
||||
if (query.select) {
|
||||
for (let i = 0; i < results.length; i++) {
|
||||
const rec = results[i].value;
|
||||
const newRec: any = {};
|
||||
for (const k of query.select) {
|
||||
newRec[k] = rec[k];
|
||||
}
|
||||
results[i].value = newRec;
|
||||
}
|
||||
async queryDelete(query: KvQuery): Promise<void> {
|
||||
const keys: KvKey[] = [];
|
||||
for (
|
||||
const { key } of await this.query({
|
||||
...query,
|
||||
prefix: query.prefix ? this.applyPrefix(query.prefix) : undefined,
|
||||
})
|
||||
) {
|
||||
keys.push(key);
|
||||
}
|
||||
return results;
|
||||
return this.batchDelete(keys);
|
||||
}
|
||||
|
||||
private applyPrefix(key: KvKey): KvKey {
|
||||
return [...this.prefix, ...(key ? key : [])];
|
||||
}
|
||||
|
||||
private stripPrefix(key: KvKey): KvKey {
|
||||
return key.slice(this.prefix.length);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user