neil/silverbullet
1
0
Fork 0
Code Activity

Refactoring work to support multi-tenancy and multiple storage, database backends (#598)

Browse Source 
* Backend infrastructure
* New backend configuration work
* Factor out KV prefixing
* Don't put assets in the manifest cache
* Removed fancy authentication stuff
* Documentation updates
This commit is contained in:
Zef Hemel
2023-12-10 13:23:42 +01:00
committed by GitHub
parent 573eca3676
commit 30ba3fcca7
33 changed files with 647 additions and 781 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/http_server.ts
+187 -142
View File
@@ -1,95 +1,155 @@
import { Application, Context, Next, oakCors, Router } from "./deps.ts";
import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
import { ensureSettingsAndIndex } from "../common/util.ts";
import { BuiltinSettings } from "../web/types.ts";
import { gitIgnoreCompiler } from "./deps.ts";
import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts";
import { Authenticator } from "./auth.ts";
import { FileMeta } from "$sb/types.ts";
import {
ShellRequest,
ShellResponse,
SyscallRequest,
SyscallResponse,
} from "./rpc.ts";
import { SilverBulletHooks } from "../common/manifest.ts";
import { System } from "../plugos/system.ts";
Application,
Context,
Next,
oakCors,
Request,
Router,
} from "./deps.ts";
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
import { FileMeta } from "$sb/types.ts";
import { ShellRequest, SyscallRequest, SyscallResponse } from "./rpc.ts";
import { determineShellBackend } from "./shell_backend.ts";
import { SpaceServer, SpaceServerConfig } from "./instance.ts";
import {
KvPrimitives,
PrefixedKvPrimitives,
} from "../plugos/lib/kv_primitives.ts";
import { EndpointHook } from "../plugos/hooks/endpoint.ts";
import { hashSHA256 } from "./crypto.ts";
export type ServerOptions = {
app: Application;
hostname: string;
port: number;
pagesPath: string;
clientAssetBundle: AssetBundle;
authenticator: Authenticator;
pass?: string;
plugAssetBundle: AssetBundle;
baseKvPrimitives?: KvPrimitives;
syncOnly: boolean;
certFile?: string;
keyFile?: string;
configs: Map<string, SpaceServerConfig>;
};
export class HttpServer {
private hostname: string;
private port: number;
abortController?: AbortController;
clientAssetBundle: AssetBundle;
settings?: BuiltinSettings;
spacePrimitives: SpacePrimitives;
authenticator: Authenticator;
plugAssetBundle: AssetBundle;
hostname: string;
port: number;
app: Application<Record<string, any>>;
keyFile: string | undefined;
certFile: string | undefined;
constructor(
spacePrimitives: SpacePrimitives,
private app: Application,
private system: System<SilverBulletHooks> | undefined,
private options: ServerOptions,
) {
spaceServers = new Map<string, Promise<SpaceServer>>();
syncOnly: boolean;
baseKvPrimitives?: KvPrimitives;
configs: Map<string, SpaceServerConfig>;
constructor(options: ServerOptions) {
this.clientAssetBundle = options.clientAssetBundle;
this.plugAssetBundle = options.plugAssetBundle;
this.hostname = options.hostname;
this.port = options.port;
this.authenticator = options.authenticator;
this.clientAssetBundle = options.clientAssetBundle;
this.app = options.app;
this.keyFile = options.keyFile;
this.certFile = options.certFile;
this.syncOnly = options.syncOnly;
this.baseKvPrimitives = options.baseKvPrimitives;
this.configs = options.configs;
}
let fileFilterFn: (s: string) => boolean = () => true;
this.spacePrimitives = new FilteredSpacePrimitives(
spacePrimitives,
(meta) => fileFilterFn(meta.name),
async () => {
await this.reloadSettings();
if (typeof this.settings?.spaceIgnore === "string") {
fileFilterFn = gitIgnoreCompiler(this.settings.spaceIgnore).accepts;
} else {
fileFilterFn = () => true;
}
},
async bootSpaceServer(config: SpaceServerConfig): Promise<SpaceServer> {
const spaceServer = new SpaceServer(
config,
determineShellBackend(config.pagesPath),
this.plugAssetBundle,
this.baseKvPrimitives
? new PrefixedKvPrimitives(this.baseKvPrimitives, [
config.namespace,
])
: undefined,
);
await spaceServer.init();
return spaceServer;
}
determineConfig(req: Request): [string, SpaceServerConfig] {
let hostname = req.url.host; // hostname:port
// First try a full match
let config = this.configs.get(hostname);
if (config) {
return [hostname, config];
}
// Then rip off the port and try again
hostname = hostname.split(":")[0];
config = this.configs.get(hostname);
if (config) {
return [hostname, config];
}
// If all else fails, try the wildcard
config = this.configs.get("*");
if (config) {
return ["*", config];
}
throw new Error(`No space server config found for hostname ${hostname}`);
}
ensureSpaceServer(req: Request): Promise<SpaceServer> {
const [matchedHostname, config] = this.determineConfig(req);
const spaceServer = this.spaceServers.get(matchedHostname);
if (spaceServer) {
return spaceServer;
}
// And then boot the thing, async
const spaceServerPromise = this.bootSpaceServer(config);
// But immediately write the promise to the map so that we don't boot it twice
this.spaceServers.set(matchedHostname, spaceServerPromise);
return spaceServerPromise;
}
// Replaces some template variables in index.html in a rather ad-hoc manner, but YOLO
renderIndexHtml() {
renderIndexHtml(pagesPath: string) {
return this.clientAssetBundle.readTextFileSync(".client/index.html")
.replaceAll(
"{{SPACE_PATH}}",
this.options.pagesPath.replaceAll("\\", "\\\\"),
pagesPath.replaceAll("\\", "\\\\"),
// );
).replaceAll(
"{{SYNC_ONLY}}",
this.system ? "false" : "true",
this.syncOnly ? "true" : "false",
);
}
async start() {
await this.reloadSettings();
start() {
// Serve static files (javascript, css, html)
this.app.use(this.serveStatic.bind(this));
await this.addPasswordAuth(this.app);
const fsRouter = this.addFsRoutes(this.spacePrimitives);
const endpointHook = new EndpointHook("/_/");
this.app.use(async (context, next) => {
const spaceServer = await this.ensureSpaceServer(context.request);
return endpointHook.handleRequest(spaceServer.system!, context, next);
});
this.addPasswordAuth(this.app);
const fsRouter = this.addFsRoutes();
this.app.use(fsRouter.routes());
this.app.use(fsRouter.allowedMethods());
// Fallback, serve the UI index.html
this.app.use(({ response }) => {
this.app.use(async ({ request, response }) => {
response.headers.set("Content-type", "text/html");
response.body = this.renderIndexHtml();
response.headers.set("Cache-Control", "no-cache");
const spaceServer = await this.ensureSpaceServer(request);
response.body = this.renderIndexHtml(spaceServer.pagesPath);
});
this.abortController = new AbortController();
@@ -98,11 +158,11 @@ export class HttpServer {
port: this.port,
signal: this.abortController.signal,
};
if (this.options.keyFile) {
listenOptions.key = Deno.readTextFileSync(this.options.keyFile);
if (this.keyFile) {
listenOptions.key = Deno.readTextFileSync(this.keyFile);
}
if (this.options.certFile) {
listenOptions.cert = Deno.readTextFileSync(this.options.certFile);
if (this.certFile) {
listenOptions.cert = Deno.readTextFileSync(this.certFile);
}
this.app.listen(listenOptions)
.catch((e: any) => {
@@ -117,7 +177,7 @@ export class HttpServer {
);
}
serveStatic(
async serveStatic(
{ request, response }: Context<Record<string, any>, Record<string, any>>,
next: Next,
) {
@@ -127,7 +187,9 @@ export class HttpServer {
// Serve the UI (index.html)
// Note: we're explicitly not setting Last-Modified and If-Modified-Since header here because this page is dynamic
response.headers.set("Content-type", "text/html");
response.body = this.renderIndexHtml();
response.headers.set("Cache-Control", "no-cache");
const spaceServer = await this.ensureSpaceServer(request);
response.body = this.renderIndexHtml(spaceServer.pagesPath);
return;
}
try {
@@ -163,12 +225,7 @@ export class HttpServer {
}
}
async reloadSettings() {
// TODO: Throttle this?
this.settings = await ensureSettingsAndIndex(this.spacePrimitives);
}
private async addPasswordAuth(app: Application) {
private addPasswordAuth(app: Application) {
const excludedPaths = [
"/manifest.json",
"/favicon.png",
@@ -192,14 +249,17 @@ export class HttpServer {
return;
} else if (request.method === "POST") {
const values = await request.body({ type: "form" }).value;
const username = values.get("username")!,
password = values.get("password")!,
refer = values.get("refer");
const hashedPassword = await this.authenticator.authenticate(
username,
password,
);
if (hashedPassword) {
const username = values.get("username")!;
const password = values.get("password")!;
const refer = values.get("refer");
const spaceServer = await this.ensureSpaceServer(request);
const hashedPassword = await hashSHA256(password);
const [expectedUser, expectedPassword] = spaceServer.auth!.split(":");
if (
username === expectedUser &&
hashedPassword === await hashSHA256(expectedPassword)
) {
await cookies.set(
authCookieName(host),
`${username}:${hashedPassword}`,
@@ -223,33 +283,38 @@ export class HttpServer {
}
});
if ((await this.authenticator.getAllUsers()).length > 0) {
// Users defined, so enabling auth
app.use(async ({ request, response, cookies }, next) => {
const host = request.url.host;
if (!excludedPaths.includes(request.url.pathname)) {
const authCookie = await cookies.get(authCookieName(host));
if (!authCookie) {
response.redirect("/.auth");
return;
}
const [username, hashedPassword] = authCookie.split(":");
if (
!await this.authenticator.authenticateHashed(
username,
hashedPassword,
)
) {
response.redirect("/.auth");
return;
}
// Check auth
app.use(async ({ request, response, cookies }, next) => {
const spaceServer = await this.ensureSpaceServer(request);
if (!spaceServer.auth) {
// Auth disabled in this config, skip
return next();
}
const host = request.url.host;
if (!excludedPaths.includes(request.url.pathname)) {
const authCookie = await cookies.get(authCookieName(host));
if (!authCookie) {
response.redirect("/.auth");
return;
}
await next();
});
}
const spaceServer = await this.ensureSpaceServer(request);
const [username, hashedPassword] = authCookie.split(":");
const [expectedUser, expectedPassword] = spaceServer.auth!.split(
":",
);
if (
username !== expectedUser ||
hashedPassword !== await hashSHA256(expectedPassword)
) {
response.redirect("/.auth");
return;
}
}
await next();
});
}
private addFsRoutes(spacePrimitives: SpacePrimitives): Router {
private addFsRoutes(): Router {
const fsRouter = new Router();
const corsMiddleware = oakCors({
allowedHeaders: "*",
@@ -264,11 +329,12 @@ export class HttpServer {
"/index.json",
// corsMiddleware,
async ({ request, response }) => {
const spaceServer = await this.ensureSpaceServer(request);
if (request.headers.has("X-Sync-Mode")) {
// Only handle direct requests for a JSON representation of the file list
response.headers.set("Content-type", "application/json");
response.headers.set("X-Space-Path", this.options.pagesPath);
const files = await spacePrimitives.fetchFileList();
response.headers.set("X-Space-Path", spaceServer.pagesPath);
const files = await spaceServer.spacePrimitives.fetchFileList();
response.body = JSON.stringify(files);
} else {
// Otherwise, redirect to the UI
@@ -280,49 +346,29 @@ export class HttpServer {
// RPC
fsRouter.post("/.rpc", async ({ request, response }) => {
const spaceServer = await this.ensureSpaceServer(request);
const body = await request.body({ type: "json" }).value;
try {
switch (body.operation) {
case "shell": {
// TODO: Have a nicer way to do this
if (this.options.pagesPath.startsWith("s3://")) {
response.status = 500;
response.body = JSON.stringify({
stdout: "",
stderr: "Cannot run shell commands with S3 backend",
code: 500,
});
return;
}
const shellCommand: ShellRequest = body;
console.log(
"Running shell command:",
shellCommand.cmd,
shellCommand.args,
);
const p = new Deno.Command(shellCommand.cmd, {
args: shellCommand.args,
cwd: this.options.pagesPath,
stdout: "piped",
stderr: "piped",
});
const output = await p.output();
const stdout = new TextDecoder().decode(output.stdout);
const stderr = new TextDecoder().decode(output.stderr);
const shellResponse = await spaceServer.shellBackend.handle(
shellCommand,
);
response.headers.set("Content-Type", "application/json");
response.body = JSON.stringify({
stdout,
stderr,
code: output.code,
} as ShellResponse);
if (output.code !== 0) {
console.error("Error running shell command", stdout, stderr);
response.body = JSON.stringify(shellResponse);
if (shellResponse.code !== 0) {
console.error("Error running shell command", shellResponse);
}
return;
}
case "syscall": {
if (!this.system) {
if (this.syncOnly) {
response.headers.set("Content-Type", "text/plain");
response.status = 400;
response.body = "Unknown operation";
@@ -330,7 +376,9 @@ export class HttpServer {
}
const syscallCommand: SyscallRequest = body;
try {
const plug = this.system.loadedPlugs.get(syscallCommand.ctx);
const plug = spaceServer.system!.loadedPlugs.get(
syscallCommand.ctx,
);
if (!plug) {
throw new Error(`Plug ${syscallCommand.ctx} not found`);
}
@@ -372,6 +420,7 @@ export class HttpServer {
filePathRegex,
async ({ params, response, request }) => {
const name = params[0];
const spaceServer = await this.ensureSpaceServer(request);
console.log("Requested file", name);
if (!request.headers.has("X-Sync-Mode") && name.endsWith(".md")) {
// It can happen that during a sync, authentication expires, this may result in a redirect to the login page and then back to this particular file. This particular file may be an .md file, which isn't great to show so we're redirecting to the associated SB UI page.
@@ -415,13 +464,15 @@ export class HttpServer {
try {
if (request.headers.has("X-Get-Meta")) {
// Getting meta via GET request
const fileData = await spacePrimitives.getFileMeta(name);
const fileData = await spaceServer.spacePrimitives.getFileMeta(
name,
);
response.status = 200;
this.fileMetaToHeaders(response.headers, fileData);
response.body = "";
return;
}
const fileData = await spacePrimitives.readFile(name);
const fileData = await spaceServer.spacePrimitives.readFile(name);
const lastModifiedHeader = new Date(fileData.meta.lastModified)
.toUTCString();
if (
@@ -447,6 +498,7 @@ export class HttpServer {
filePathRegex,
async ({ request, response, params }) => {
const name = params[0];
const spaceServer = await this.ensureSpaceServer(request);
console.log("Saving file", name);
if (name.startsWith(".")) {
// Don't expose hidden files
@@ -457,7 +509,7 @@ export class HttpServer {
const body = await request.body({ type: "bytes" }).value;
try {
const meta = await spacePrimitives.writeFile(
const meta = await spaceServer.spacePrimitives.writeFile(
name,
body,
);
@@ -471,8 +523,9 @@ export class HttpServer {
}
},
)
.delete(filePathRegex, async ({ response, params }) => {
.delete(filePathRegex, async ({ request, response, params }) => {
const name = params[0];
const spaceServer = await this.ensureSpaceServer(request);
console.log("Deleting file", name);
if (name.startsWith(".")) {
// Don't expose hidden files
@@ -480,7 +533,7 @@ export class HttpServer {
return;
}
try {
await spacePrimitives.deleteFile(name);
await spaceServer.spacePrimitives.deleteFile(name);
response.status = 200;
response.body = "OK";
} catch (e: any) {
@@ -573,11 +626,3 @@ function utcDateString(mtime: number): string {
function authCookieName(host: string) {
return `auth:${host}`;
}
function headersToJson(headers: Headers) {
let headersObj: any = {};
for (const [key, value] of headers.entries()) {
headersObj[key] = value;
}
return JSON.stringify(headersObj);
}