neil/silverbullet
1
0
Fork 0
Code Activity

Major backend refactor (#599)

Browse Source 
Backend refactor
This commit is contained in:
Zef Hemel
2023-12-13 17:52:56 +01:00
committed by GitHub
parent 60d7cc704a
commit 9f082c83a9
42 changed files with 959 additions and 503 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/crypto.ts
+1
View File
@@ -13,6 +13,7 @@ export class JWTIssuer {
constructor(readonly kv: KvPrimitives) {
}
// authString is only used to compare hashes to see if the auth has changed
async init(authString: string) {
const [secret] = await this.kv.batchGet([[jwtSecretKey]]);
if (!secret) {
server/db_backend.ts
+8 -11
View File
@@ -1,18 +1,17 @@
import { DenoKvPrimitives } from "../plugos/lib/deno_kv_primitives.ts";
import { KvPrimitives } from "../plugos/lib/kv_primitives.ts";
import { MemoryKvPrimitives } from "../plugos/lib/memory_kv_primitives.ts";
import { path } from "./deps.ts";
/**
* Environment variables:
* - SB_DB_BACKEND: "denokv" or "off" (default: denokv)
* - SB_KV_DB (denokv only): path to the database file (default .silverbullet.db) or ":cloud:" for cloud storage
* - SB_DB_BACKEND: "denokv" or "memory" (default: denokv)
* - SB_KV_DB (denokv only): path to the database file (default .silverbullet.db)
*/
export async function determineDatabaseBackend(
singleTenantFolder?: string,
): Promise<
KvPrimitives | undefined
> {
): Promise<KvPrimitives> {
const backendConfig = Deno.env.get("SB_DB_BACKEND") || "denokv";
switch (backendConfig) {
case "denokv": {
@@ -24,21 +23,19 @@ export async function determineDatabaseBackend(
dbFile = path.resolve(singleTenantFolder, dbFile);
}
if (dbFile === ":cloud:") {
if (Deno.env.get("DENO_DEPLOYMENT_ID") !== undefined) { // We're running in Deno Deploy
dbFile = undefined; // Deno Deploy will use the default KV store
}
const denoDb = await Deno.openKv(dbFile);
console.info(
`Using DenoKV as a database backend (${
dbFile || "cloud"
}), running in server-processing mode.`,
`Using DenoKV as a database backend (${dbFile || "cloud"}.`,
);
return new DenoKvPrimitives(denoDb);
}
default:
console.info(
"Running in databaseless mode: no server-side indexing and state keeping (beyond space files) will happen.",
"Running in in-memory database mode: index data will be flushed on every restart. Not recommended, but to each their own.",
);
return;
return new MemoryKvPrimitives();
}
}
server/http_server.ts
+34 -29
View File
@@ -23,7 +23,7 @@ export type ServerOptions = {
port: number;
clientAssetBundle: AssetBundle;
plugAssetBundle: AssetBundle;
baseKvPrimitives?: KvPrimitives;
baseKvPrimitives: KvPrimitives;
syncOnly: boolean;
certFile?: string;
keyFile?: string;
@@ -43,7 +43,7 @@ export class HttpServer {
spaceServers = new Map<string, Promise<SpaceServer>>();
syncOnly: boolean;
baseKvPrimitives?: KvPrimitives;
baseKvPrimitives: KvPrimitives;
configs: Map<string, SpaceServerConfig>;
constructor(options: ServerOptions) {
@@ -64,11 +64,10 @@ export class HttpServer {
config,
determineShellBackend(config.pagesPath),
this.plugAssetBundle,
this.baseKvPrimitives
? new PrefixedKvPrimitives(this.baseKvPrimitives, [
config.namespace,
])
: undefined,
new PrefixedKvPrimitives(this.baseKvPrimitives, [
config.namespace,
]),
this.syncOnly,
);
await spaceServer.init();
@@ -140,7 +139,7 @@ export class HttpServer {
return endpointHook.handleRequest(spaceServer.system!, context, next);
});
this.addPasswordAuth(this.app);
this.addAuth(this.app);
const fsRouter = this.addFsRoutes();
this.app.use(fsRouter.routes());
this.app.use(fsRouter.allowedMethods());
@@ -226,7 +225,7 @@ export class HttpServer {
}
}
private addPasswordAuth(app: Application) {
private addAuth(app: Application) {
const excludedPaths = [
"/manifest.json",
"/favicon.png",
@@ -252,20 +251,9 @@ export class HttpServer {
const values = await request.body({ type: "form" }).value;
const username = values.get("username")!;
const password = values.get("password")!;
const formCSRF = values.get("csrf");
const cookieCSRF = await cookies.get("csrf_token");
if (formCSRF !== cookieCSRF) {
response.redirect("/.auth?error=2");
console.log("CSRF mismatch", formCSRF, cookieCSRF);
return;
}
await cookies.delete("csrf_token");
const spaceServer = await this.ensureSpaceServer(request);
const [expectedUser, expectedPassword] = spaceServer.auth!.split(":");
const { user: expectedUser, pass: expectedPassword } = spaceServer
.auth!;
if (username === expectedUser && password === expectedPassword) {
// Generate a JWT and set it as a cookie
const jwt = await spaceServer.jwtIssuer.createJWT(
@@ -305,18 +293,35 @@ export class HttpServer {
}
const host = request.url.host;
if (!excludedPaths.includes(request.url.pathname)) {
const authCookie = await cookies.get(authCookieName(host));
if (!authCookie) {
const authToken = await cookies.get(authCookieName(host));
if (!authToken && spaceServer.authToken) {
// Attempt Bearer Authorization based authentication
const authHeader = request.headers.get("Authorization");
if (authHeader && authHeader.startsWith("Bearer ")) {
const authToken = authHeader.slice("Bearer ".length);
if (authToken === spaceServer.authToken) {
// All good, let's proceed
return next();
} else {
console.log(
"Unauthorized token access, redirecting to auth page",
);
response.status = 401;
response.body = "Unauthorized";
return;
}
}
}
if (!authToken) {
console.log("Unauthorized access, redirecting to auth page");
return response.redirect("/.auth");
}
const [expectedUser] = spaceServer.auth!.split(
":",
);
const { user: expectedUser } = spaceServer.auth!;
try {
const verifiedJwt = await spaceServer.jwtIssuer.verifyAndDecodeJWT(
authCookie,
authToken,
);
if (verifiedJwt.username !== expectedUser) {
throw new Error("Username mismatch");
@@ -329,7 +334,7 @@ export class HttpServer {
return response.redirect("/.auth");
}
}
await next();
return next();
});
}
server/instance.ts
+23 -14
View File
@@ -5,7 +5,6 @@ import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
import { ensureSettingsAndIndex } from "../common/util.ts";
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
import { KvPrimitives } from "../plugos/lib/kv_primitives.ts";
import { MemoryKvPrimitives } from "../plugos/lib/memory_kv_primitives.ts";
import { System } from "../plugos/system.ts";
import { BuiltinSettings } from "../web/types.ts";
import { JWTIssuer } from "./crypto.ts";
@@ -17,17 +16,21 @@ import { determineStorageBackend } from "./storage_backend.ts";
export type SpaceServerConfig = {
hostname: string;
namespace: string;
auth?: string; // username:password
// Enable username/password auth
auth?: { user: string; pass: string };
// Additional API auth token
authToken?: string;
pagesPath: string;
};
export class SpaceServer {
public pagesPath: string;
auth?: string;
auth?: { user: string; pass: string };
authToken?: string;
hostname: string;
private settings?: BuiltinSettings;
spacePrimitives: SpacePrimitives;
spacePrimitives!: SpacePrimitives;
jwtIssuer: JWTIssuer;
@@ -38,20 +41,24 @@ export class SpaceServer {
constructor(
config: SpaceServerConfig,
public shellBackend: ShellBackend,
plugAssetBundle: AssetBundle,
private kvPrimitives?: KvPrimitives,
private plugAssetBundle: AssetBundle,
private kvPrimitives: KvPrimitives,
private syncOnly: boolean,
) {
this.pagesPath = config.pagesPath;
this.hostname = config.hostname;
this.auth = config.auth;
this.jwtIssuer = new JWTIssuer(kvPrimitives || new MemoryKvPrimitives());
this.authToken = config.authToken;
this.jwtIssuer = new JWTIssuer(kvPrimitives);
}
async init() {
let fileFilterFn: (s: string) => boolean = () => true;
this.spacePrimitives = new FilteredSpacePrimitives(
new AssetBundlePlugSpacePrimitives(
determineStorageBackend(this.pagesPath),
plugAssetBundle,
await determineStorageBackend(this.kvPrimitives, this.pagesPath),
this.plugAssetBundle,
),
(meta) => fileFilterFn(meta.name),
async () => {
@@ -65,25 +72,27 @@ export class SpaceServer {
);
// system = undefined in databaseless mode (no PlugOS instance on the server and no DB)
if (kvPrimitives) {
if (!this.syncOnly) {
// Enable server-side processing
const serverSystem = new ServerSystem(
this.spacePrimitives,
kvPrimitives,
this.kvPrimitives,
);
this.serverSystem = serverSystem;
}
}
async init() {
if (this.auth) {
// Initialize JWT issuer
await this.jwtIssuer.init(this.auth);
await this.jwtIssuer.init(
JSON.stringify({ auth: this.auth, authToken: this.authToken }),
);
}
if (this.serverSystem) {
await this.serverSystem.init();
this.system = this.serverSystem.system;
// Swap in the space primitives from the server system
this.spacePrimitives = this.serverSystem.spacePrimitives;
}
await this.reloadSettings();
server/server_system.ts
+1 -1
View File
@@ -39,7 +39,7 @@ const plugNameExtractRegex = /\/(.+)\.plug\.js$/;
export class ServerSystem {
system!: System<SilverBulletHooks>;
spacePrimitives!: SpacePrimitives;
public spacePrimitives!: SpacePrimitives;
// denoKv!: Deno.Kv;
listInterval?: number;
ds!: DataStore;
server/spaces/s3_space_primitives.test.ts
-38
View File
@@ -1,38 +0,0 @@
import { S3SpacePrimitives } from "./s3_space_primitives.ts";
import { assert, assertEquals } from "../../test_deps.ts";
Deno.test("s3_space_primitives", async () => {
return;
const options = {
accessKey: Deno.env.get("AWS_ACCESS_KEY_ID")!,
secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!,
endPoint: "s3.eu-central-1.amazonaws.com",
region: "eu-central-1",
bucket: "zef-sb-space",
prefix: "test",
};
const primitives = new S3SpacePrimitives(options);
console.log(await primitives.fetchFileList());
console.log(
await primitives.writeFile("test+'s.txt", stringToBytes("Hello world!")),
);
assertEquals(
stringToBytes("Hello world!"),
(await primitives.readFile("test+'s.txt")).data,
);
await primitives.deleteFile("test+'s.txt");
try {
await primitives.getFileMeta("test+'s.txt");
assert(false);
} catch (e: any) {
assertEquals(e.message, "Not found");
}
// console.log(await primitives.readFile("SETTINGS.md", "utf8"));
});
function stringToBytes(str: string): Uint8Array {
return new TextEncoder().encode(str);
}
server/spaces/s3_space_primitives.ts
-128
View File
@@ -1,128 +0,0 @@
// We're explicitly using 0.4.0 to be able to hijack the path encoding, which is inconsisently broken in 0.5.0
import { S3Client } from "https://deno.land/x/s3_lite_client@0.4.0/mod.ts";
import type { ClientOptions } from "https://deno.land/x/s3_lite_client@0.4.0/client.ts";
import { SpacePrimitives } from "../../common/spaces/space_primitives.ts";
import { mime } from "../deps.ts";
import { FileMeta } from "$sb/types.ts";
// TODO: IMPORTANT: This needs a different way to keep meta data (last modified and created dates)
export type S3SpacePrimitivesOptions = ClientOptions & { prefix: string };
export class S3SpacePrimitives implements SpacePrimitives {
client: S3Client;
prefix: string;
constructor(options: S3SpacePrimitivesOptions) {
this.client = new S3Client(options);
// TODO: Use this
this.prefix = options.prefix;
}
private encodePath(name: string): string {
return uriEscapePath(name);
}
private decodePath(encoded: string): string {
// AWS only returns ' replace with &apos;
return encoded.replaceAll("&apos;", "'");
}
async fetchFileList(): Promise<FileMeta[]> {
const allFiles: FileMeta[] = [];
for await (const obj of this.client.listObjects({ prefix: "" })) {
allFiles.push({
name: this.decodePath(obj.key),
perm: "rw",
created: 0,
lastModified: obj.lastModified.getTime(),
contentType: mime.getType(obj.key) || "application/octet-stream",
size: obj.size,
});
}
return allFiles;
}
async readFile(
name: string,
): Promise<{ data: Uint8Array; meta: FileMeta }> {
try {
// console.log("Fetching object", encodeURI(name));
const obj = await this.client.getObject(this.encodePath(name));
const contentType = mime.getType(name) || "application/octet-stream";
const meta: FileMeta = {
name,
perm: "rw",
created: 0,
lastModified: new Date(obj.headers.get("Last-Modified")!).getTime(),
contentType,
size: parseInt(obj.headers.get("Content-Length")!),
};
return {
data: new Uint8Array(await obj.arrayBuffer()),
meta,
};
} catch (e: any) {
console.log("GOt error", e.message);
if (e.message.includes("does not exist")) {
throw new Error(`Not found`);
}
throw e;
}
}
async getFileMeta(name: string): Promise<FileMeta> {
try {
const stat = await this.client.statObject(this.encodePath(name));
return {
name,
perm: "rw",
// TODO: Created is not accurate
created: 0,
lastModified: new Date(stat.lastModified).getTime(),
size: stat.size,
contentType: mime.getType(name) || "application/octet-stream",
};
} catch (e: any) {
if (e.message.includes("404")) {
throw new Error(`Not found`);
}
throw e;
}
}
async writeFile(
name: string,
data: Uint8Array,
): Promise<FileMeta> {
if (data.byteLength === 0) {
// S3 doesn't like empty files, so we'll put a space in it. Not ideal, but it works. I hope.
data = new TextEncoder().encode(" ");
}
await this.client.putObject(this.encodePath(name), data);
// TODO: Dangerous due to eventual consistency? maybe check with etag or versionid?
return this.getFileMeta(name);
}
async deleteFile(name: string): Promise<void> {
await this.client.deleteObject(this.encodePath(name));
}
}
// Stolen from https://github.com/aws/aws-sdk-js/blob/master/lib/util.js
export function uriEscapePath(string: string): string {
return string.split("/").map(uriEscape).join("/");
}
function uriEscape(string: string): string {
let output = encodeURIComponent(string);
output = output.replace(/[^A-Za-z0-9_.~\-%]+/g, escape);
// AWS percent-encodes some extra non-standard characters in a URI
output = output.replace(/[*]/g, function (ch) {
return "%" + ch.charCodeAt(0).toString(16).toUpperCase();
});
return output;
}
server/storage_backend.ts
+44 -11
View File
@@ -1,19 +1,52 @@
import { DiskSpacePrimitives } from "../common/spaces/disk_space_primitives.ts";
import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
import { path } from "./deps.ts";
import { S3SpacePrimitives } from "./spaces/s3_space_primitives.ts";
import { S3SpacePrimitives } from "../common/spaces/s3_space_primitives.ts";
import { KvPrimitives } from "../plugos/lib/kv_primitives.ts";
import { ChunkedKvStoreSpacePrimitives } from "../common/spaces/chunked_datastore_space_primitives.ts";
import { HttpSpacePrimitives } from "../common/spaces/http_space_primitives.ts";
export function determineStorageBackend(folder: string): SpacePrimitives {
if (folder === "s3://") {
export async function determineStorageBackend(
kvPrimitives: KvPrimitives,
folder: string,
): Promise<SpacePrimitives> {
if (folder.startsWith("s3://")) {
console.info("Using S3 as a storage backend");
return new S3SpacePrimitives({
accessKey: Deno.env.get("AWS_ACCESS_KEY_ID")!,
secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!,
endPoint: Deno.env.get("AWS_ENDPOINT")!,
region: Deno.env.get("AWS_REGION")!,
bucket: Deno.env.get("AWS_BUCKET")!,
prefix: folder.slice(5),
});
let objectPrefix = folder.slice("s3://".length);
if (objectPrefix !== "") {
// Add a suffix /
objectPrefix += "/";
}
const spacePrimitives = new S3SpacePrimitives(
kvPrimitives,
["meta"],
objectPrefix,
{
accessKey: Deno.env.get("AWS_ACCESS_KEY_ID")!,
secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!,
endPoint: Deno.env.get("AWS_ENDPOINT")!,
region: Deno.env.get("AWS_REGION")!,
bucket: Deno.env.get("AWS_BUCKET")!,
},
);
if (Deno.env.get("SB_S3_PERFORM_SYNC") === "true") {
console.log("Performing S3 file list sync");
await spacePrimitives.syncFileList();
console.info("S3 file list sync complete");
}
return spacePrimitives;
} else if (folder === "db://") {
console.info(`Using the database as a storage backend`);
return new ChunkedKvStoreSpacePrimitives(
kvPrimitives,
65536, // For DenoKV, this is the maximum size of a single value
);
} else if (folder.startsWith("http://") || folder.startsWith("https://")) {
return new HttpSpacePrimitives(
folder,
undefined,
Deno.env.get("SB_AUTH_TOKEN"),
);
} else {
folder = path.resolve(Deno.cwd(), folder);
console.info(`Using local disk as a storage backend: ${folder}`);