neil/silverbullet
1
0
Fork 0
Code Activity

Refactoring work to support multi-tenancy and multiple storage, database backends (#598)

Browse Source 
* Backend infrastructure
* New backend configuration work
* Factor out KV prefixing
* Don't put assets in the manifest cache
* Removed fancy authentication stuff
* Documentation updates
This commit is contained in:
Zef Hemel
2023-12-10 13:23:42 +01:00
committed by GitHub
parent 573eca3676
commit 30ba3fcca7
33 changed files with 647 additions and 781 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugos/hooks/endpoint.test.ts
+5 -1
View File
@@ -26,7 +26,11 @@ Deno.test("Run a plugos endpoint server", async () => {
const app = new Application();
const port = 3123;
system.addHook(new EndpointHook(app, "/_/"));
const endpointHook = new EndpointHook("/_/");
app.use((context, next) => {
return endpointHook.handleRequest(system, context, next);
});
const controller = new AbortController();
app.listen({ port: port, signal: controller.signal });
plugos/hooks/endpoint.ts
+71 -68
View File
@@ -1,6 +1,6 @@
import { Hook, Manifest } from "../types.ts";
import { System } from "../system.ts";
import { Application } from "../../server/deps.ts";
import { Application, Context, Next } from "../../server/deps.ts";
export type EndpointRequest = {
method: string;
@@ -26,87 +26,90 @@ export type EndPointDef = {
};
export class EndpointHook implements Hook<EndpointHookT> {
private app: Application;
readonly prefix: string;
constructor(app: Application, prefix: string) {
this.app = app;
constructor(prefix: string) {
this.prefix = prefix;
}
apply(system: System<EndpointHookT>): void {
this.app.use(async (ctx, next) => {
const req = ctx.request;
const requestPath = ctx.request.url.pathname;
if (!requestPath.startsWith(this.prefix)) {
return next();
public async handleRequest(
system: System<EndpointHookT>,
ctx: Context,
next: Next,
) {
const req = ctx.request;
const requestPath = ctx.request.url.pathname;
if (!requestPath.startsWith(this.prefix)) {
return next();
}
console.log("Endpoint request", requestPath);
// Iterate over all loaded plugins
for (const [plugName, plug] of system.loadedPlugs.entries()) {
const manifest = plug.manifest;
if (!manifest) {
continue;
}
console.log("Endpoint request", requestPath);
// Iterate over all loaded plugins
for (const [plugName, plug] of system.loadedPlugs.entries()) {
const manifest = plug.manifest;
if (!manifest) {
const functions = manifest.functions;
// console.log("Checking plug", plugName);
const prefix = `${this.prefix}${plugName}`;
if (!requestPath.startsWith(prefix)) {
continue;
}
for (const [name, functionDef] of Object.entries(functions)) {
if (!functionDef.http) {
continue;
}
const functions = manifest.functions;
// console.log("Checking plug", plugName);
const prefix = `${this.prefix}${plugName}`;
if (!requestPath.startsWith(prefix)) {
continue;
}
for (const [name, functionDef] of Object.entries(functions)) {
if (!functionDef.http) {
continue;
}
// console.log("Got config", functionDef);
const endpoints = Array.isArray(functionDef.http)
? functionDef.http
: [functionDef.http];
// console.log(endpoints);
for (const { path, method } of endpoints) {
const prefixedPath = `${prefix}${path}`;
if (
prefixedPath === requestPath &&
((method || "GET") === req.method || method === "ANY")
) {
try {
const response: EndpointResponse = await plug.invoke(name, [
{
path: req.url.pathname,
method: req.method,
body: req.body(),
query: Object.fromEntries(
req.url.searchParams.entries(),
),
headers: Object.fromEntries(req.headers.entries()),
} as EndpointRequest,
]);
if (response.headers) {
for (
const [key, value] of Object.entries(
response.headers,
)
) {
ctx.response.headers.set(key, value);
}
// console.log("Got config", functionDef);
const endpoints = Array.isArray(functionDef.http)
? functionDef.http
: [functionDef.http];
// console.log(endpoints);
for (const { path, method } of endpoints) {
const prefixedPath = `${prefix}${path}`;
if (
prefixedPath === requestPath &&
((method || "GET") === req.method || method === "ANY")
) {
try {
const response: EndpointResponse = await plug.invoke(name, [
{
path: req.url.pathname,
method: req.method,
body: req.body(),
query: Object.fromEntries(
req.url.searchParams.entries(),
),
headers: Object.fromEntries(req.headers.entries()),
} as EndpointRequest,
]);
if (response.headers) {
for (
const [key, value] of Object.entries(
response.headers,
)
) {
ctx.response.headers.set(key, value);
}
ctx.response.status = response.status;
ctx.response.body = response.body;
// console.log("Sent result");
return;
} catch (e: any) {
console.error("Error executing function", e);
ctx.response.status = 500;
ctx.response.body = e.message;
return;
}
ctx.response.status = response.status;
ctx.response.body = response.body;
// console.log("Sent result");
return;
} catch (e: any) {
console.error("Error executing function", e);
ctx.response.status = 500;
ctx.response.body = e.message;
return;
}
}
}
}
// console.log("Shouldn't get here");
await next();
});
}
// console.log("Shouldn't get here");
await next();
}
apply(): void {
}
validateManifest(manifest: Manifest<EndpointHookT>): string[] {
plugos/lib/datastore.test.ts
+2 -2
View File
@@ -2,11 +2,11 @@ import "https://esm.sh/fake-indexeddb@4.0.2/auto";
import { IndexedDBKvPrimitives } from "./indexeddb_kv_primitives.ts";
import { DataStore } from "./datastore.ts";
import { DenoKvPrimitives } from "./deno_kv_primitives.ts";
import { KvPrimitives } from "./kv_primitives.ts";
import { KvPrimitives, PrefixedKvPrimitives } from "./kv_primitives.ts";
import { assertEquals } from "https://deno.land/std@0.165.0/testing/asserts.ts";
async function test(db: KvPrimitives) {
const datastore = new DataStore(db, ["ds"], {
const datastore = new DataStore(new PrefixedKvPrimitives(db, ["ds"]), {
count: (arr: any[]) => arr.length,
});
await datastore.set(["user", "peter"], { name: "Peter" });
plugos/lib/datastore.ts
+6 -32
View File
@@ -10,25 +10,16 @@ import { KvPrimitives } from "./kv_primitives.ts";
export class DataStore {
constructor(
readonly kv: KvPrimitives,
private prefix: KvKey = [],
private functionMap: FunctionMap = builtinFunctions,
) {
}
prefixed(prefix: KvKey): DataStore {
return new DataStore(
this.kv,
[...this.prefix, ...prefix],
this.functionMap,
);
}
async get<T = any>(key: KvKey): Promise<T | null> {
return (await this.batchGet([key]))[0];
}
batchGet<T = any>(keys: KvKey[]): Promise<(T | null)[]> {
return this.kv.batchGet(keys.map((key) => this.applyPrefix(key)));
return this.kv.batchGet(keys);
}
set(key: KvKey, value: any): Promise<void> {
@@ -44,7 +35,7 @@ export class DataStore {
console.warn(`Duplicate key ${keyString} in batchSet, skipping`);
} else {
allKeyStrings.add(keyString);
uniqueEntries.push({ key: this.applyPrefix(key), value });
uniqueEntries.push({ key, value });
}
}
return this.kv.batchSet(uniqueEntries);
@@ -55,7 +46,7 @@ export class DataStore {
}
batchDelete(keys: KvKey[]): Promise<void> {
return this.kv.batchDelete(keys.map((key) => this.applyPrefix(key)));
return this.kv.batchDelete(keys);
}
async query<T = any>(query: KvQuery): Promise<KV<T>[]> {
@@ -63,15 +54,11 @@ export class DataStore {
let itemCount = 0;
// Accumulate results
let limit = Infinity;
const prefixedQuery: KvQuery = {
...query,
prefix: query.prefix ? this.applyPrefix(query.prefix) : undefined,
};
if (query.limit) {
limit = evalQueryExpression(query.limit, {}, this.functionMap);
}
for await (
const entry of this.kv.query(prefixedQuery)
const entry of this.kv.query(query)
) {
// Filter
if (
@@ -89,29 +76,16 @@ export class DataStore {
}
}
// Apply order by, limit, and select
return applyQueryNoFilterKV(prefixedQuery, results, this.functionMap).map((
{ key, value },
) => ({ key: this.stripPrefix(key), value }));
return applyQueryNoFilterKV(query, results, this.functionMap);
}
async queryDelete(query: KvQuery): Promise<void> {
const keys: KvKey[] = [];
for (
const { key } of await this.query({
...query,
prefix: query.prefix ? this.applyPrefix(query.prefix) : undefined,
})
const { key } of await this.query(query)
) {
keys.push(key);
}
return this.batchDelete(keys);
}
private applyPrefix(key: KvKey): KvKey {
return [...this.prefix, ...(key ? key : [])];
}
private stripPrefix(key: KvKey): KvKey {
return key.slice(this.prefix.length);
}
}
plugos/lib/kv_primitives.ts
+44
View File
@@ -9,4 +9,48 @@ export interface KvPrimitives {
batchSet(entries: KV[]): Promise<void>;
batchDelete(keys: KvKey[]): Promise<void>;
query(options: KvQueryOptions): AsyncIterableIterator<KV>;
close(): void;
}
/**
* Turns any KvPrimitives into a KvPrimitives that automatically prefixes all keys (and removes them again when reading)
*/
export class PrefixedKvPrimitives implements KvPrimitives {
constructor(private wrapped: KvPrimitives, private prefix: KvKey) {
}
batchGet(keys: KvKey[]): Promise<any[]> {
return this.wrapped.batchGet(keys.map((key) => this.applyPrefix(key)));
}
batchSet(entries: KV[]): Promise<void> {
return this.wrapped.batchSet(
entries.map(({ key, value }) => ({ key: this.applyPrefix(key), value })),
);
}
batchDelete(keys: KvKey[]): Promise<void> {
return this.wrapped.batchDelete(keys.map((key) => this.applyPrefix(key)));
}
async *query(options: KvQueryOptions): AsyncIterableIterator<KV> {
for await (
const result of this.wrapped.query({
prefix: this.applyPrefix(options.prefix),
})
) {
yield { key: this.stripPrefix(result.key), value: result.value };
}
}
close(): void {
this.wrapped.close();
}
private applyPrefix(key?: KvKey): KvKey {
return [...this.prefix, ...(key ? key : [])];
}
private stripPrefix(key: KvKey): KvKey {
return key.slice(this.prefix.length);
}
}
plugos/lib/kv_store.json_file.ts
-56
View File
@@ -1,56 +0,0 @@
import { KV } from "$sb/types.ts";
export class JSONKVStore {
private data: { [key: string]: any } = {};
async load(path: string) {
this.loadString(await Deno.readTextFile(path));
}
loadString(jsonString: string) {
this.data = JSON.parse(jsonString);
}
async save(path: string) {
await Deno.writeTextFile(path, JSON.stringify(this.data));
}
del(key: string): Promise<void> {
delete this.data[key];
return Promise.resolve();
}
deletePrefix(prefix: string): Promise<void> {
for (const key in this.data) {
if (key.startsWith(prefix)) {
delete this.data[key];
}
}
return Promise.resolve();
}
deleteAll(): Promise<void> {
this.data = {};
return Promise.resolve();
}
set(key: string, value: any): Promise<void> {
this.data[key] = value;
return Promise.resolve();
}
get(key: string): Promise<any> {
return Promise.resolve(this.data[key]);
}
has(key: string): Promise<boolean> {
return Promise.resolve(key in this.data);
}
queryPrefix(keyPrefix: string): Promise<{ key: string; value: any }[]> {
const results: { key: string; value: any }[] = [];
for (const key in this.data) {
if (key.startsWith(keyPrefix)) {
results.push({ key, value: this.data[key] });
}
}
return Promise.resolve(results);
}
}
plugos/lib/mq.datastore.test.ts
+4 -1
View File
@@ -3,12 +3,15 @@ import { assertEquals } from "../../test_deps.ts";
import { sleep } from "$sb/lib/async.ts";
import { DenoKvPrimitives } from "./deno_kv_primitives.ts";
import { DataStore } from "./datastore.ts";
import { PrefixedKvPrimitives } from "./kv_primitives.ts";
Deno.test("DataStore MQ", async () => {
const tmpFile = await Deno.makeTempFile();
const db = new DenoKvPrimitives(await Deno.openKv(tmpFile));
const mq = new DataStoreMQ(new DataStore(db, ["mq"]));
const mq = new DataStoreMQ(
new DataStore(new PrefixedKvPrimitives(db, ["mq"])),
);
await mq.send("test", "Hello World");
let messages = await mq.poll("test", 10);
assertEquals(messages.length, 1);
plugos/system.ts
+1 -1
View File
@@ -141,7 +141,7 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
if (this.plugs.has(manifest.name)) {
this.unload(manifest.name);
}
console.log("Loaded plug", manifest.name);
console.log("Activated plug", manifest.name);
this.plugs.set(manifest.name, plug);
await this.emit("plugLoaded", plug);