Refactoring work to support multi-tenancy and multiple storage, database backends (#598)

* Backend infrastructure
* New backend configuration work
* Factor out KV prefixing
* Don't put assets in the manifest cache
* Removed fancy authentication stuff
* Documentation updates

server/auth.ts

@@ -1,120 +0,0 @@
-import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts";
-
-export type User = {
-  username: string;
-  passwordHash: string; // hashed password
-  salt: string;
-  groups: string[]; // special "admin"
-};
-
-async function createUser(
-  username: string,
-  password: string,
-  groups: string[],
-  salt = generateSalt(16),
-): Promise<User> {
-  return {
-    username,
-    passwordHash: await hashSHA256(`${salt}${password}`),
-    salt,
-    groups,
-  };
-}
-
-const userPrefix = `u:`;
-
-export class Authenticator {
-  constructor(private store: JSONKVStore) {
-  }
-
-  async register(
-    username: string,
-    password: string,
-    groups: string[],
-    salt?: string,
-  ): Promise<void> {
-    await this.store.set(
-      `${userPrefix}${username}`,
-      await createUser(username, password, groups, salt),
-    );
-  }
-
-  async authenticateHashed(
-    username: string,
-    hashedPassword: string,
-  ): Promise<boolean> {
-    const user = await this.store.get(`${userPrefix}${username}`) as User;
-    if (!user) {
-      return false;
-    }
-    return user.passwordHash === hashedPassword;
-  }
-
-  async authenticate(
-    username: string,
-    password: string,
-  ): Promise<string | undefined> {
-    const user = await this.store.get(`${userPrefix}${username}`) as User;
-    if (!user) {
-      return undefined;
-    }
-    const hashedPassword = await hashSHA256(`${user.salt}${password}`);
-    return user.passwordHash === hashedPassword ? hashedPassword : undefined;
-  }
-
-  async getAllUsers(): Promise<User[]> {
-    return (await this.store.queryPrefix(userPrefix)).map((item) => item.value);
-  }
-
-  getUser(username: string): Promise<User | undefined> {
-    return this.store.get(`${userPrefix}${username}`);
-  }
-
-  async setPassword(username: string, password: string): Promise<void> {
-    const user = await this.getUser(username);
-    if (!user) {
-      throw new Error(`User does not exist`);
-    }
-    user.passwordHash = await hashSHA256(`${user.salt}${password}`);
-    await this.store.set(`${userPrefix}${username}`, user);
-  }
-
-  async deleteUser(username: string): Promise<void> {
-    const user = await this.getUser(username);
-    if (!user) {
-      throw new Error(`User does not exist`);
-    }
-    await this.store.del(`${userPrefix}${username}`);
-  }
-
-  async setGroups(username: string, groups: string[]): Promise<void> {
-    const user = await this.getUser(username);
-    if (!user) {
-      throw new Error(`User does not exist`);
-    }
-    user.groups = groups;
-    await this.store.set(`${userPrefix}${username}`, user);
-  }
-}
-
-async function hashSHA256(message: string): Promise<string> {
-  // Transform the string into an ArrayBuffer
-  const encoder = new TextEncoder();
-  const data = encoder.encode(message);
-
-  // Generate the hash
-  const hashBuffer = await window.crypto.subtle.digest("SHA-256", data);
-
-  // Transform the hash into a hex string
-  return Array.from(new Uint8Array(hashBuffer)).map((b) =>
-    b.toString(16).padStart(2, "0")
-  ).join("");
-}
-
-function generateSalt(length: number): string {
-  const array = new Uint8Array(length / 2); // because two characters represent one byte in hex
-  crypto.getRandomValues(array);
-  return Array.from(array, (byte) => ("00" + byte.toString(16)).slice(-2)).join(
-    "",
-  );
-}

server/crypto.ts

@@ -0,0 +1,13 @@
+export async function hashSHA256(message: string): Promise<string> {
+  // Transform the string into an ArrayBuffer
+  const encoder = new TextEncoder();
+  const data = encoder.encode(message);
+
+  // Generate the hash
+  const hashBuffer = await window.crypto.subtle.digest("SHA-256", data);
+
+  // Transform the hash into a hex string
+  return Array.from(new Uint8Array(hashBuffer)).map((b) =>
+    b.toString(16).padStart(2, "0")
+  ).join("");
+}

server/db_backend.ts

@@ -0,0 +1,44 @@
+import { DenoKvPrimitives } from "../plugos/lib/deno_kv_primitives.ts";
+import { KvPrimitives } from "../plugos/lib/kv_primitives.ts";
+import { path } from "./deps.ts";
+
+/**
+ * Environment variables:
+ * - SB_DB_BACKEND: "denokv" or "off" (default: denokv)
+ * - SB_KV_DB (denokv only): path to the database file (default .silverbullet.db) or ":cloud:" for cloud storage
+ */
+
+export async function determineDatabaseBackend(
+  singleTenantFolder?: string,
+): Promise<
+  KvPrimitives | undefined
+> {
+  const backendConfig = Deno.env.get("SB_DB_BACKEND") || "denokv";
+  switch (backendConfig) {
+    case "denokv": {
+      let dbFile: string | undefined = Deno.env.get("SB_KV_DB") ||
+        ".silverbullet.db";
+
+      if (singleTenantFolder) {
+        // If we're running in single tenant mode, we may as well use the tenant's space folder to keep the database
+        dbFile = path.resolve(singleTenantFolder, dbFile);
+      }
+
+      if (dbFile === ":cloud:") {
+        dbFile = undefined; // Deno Deploy will use the default KV store
+      }
+      const denoDb = await Deno.openKv(dbFile);
+      console.info(
+        `Using DenoKV as a database backend (${
+          dbFile || "cloud"
+        }), running in server-processing mode.`,
+      );
+      return new DenoKvPrimitives(denoDb);
+    }
+    default:
+      console.info(
+        "Running in databaseless mode: no server-side indexing and state keeping (beyond space files) will happen.",
+      );
+      return;
+  }
+}

server/deps.ts

@@ -3,6 +3,8 @@ export type { Next } from "https://deno.land/x/oak@v12.4.0/mod.ts";
 export {
   Application,
   Context,
+  Request,
+  Response,
   Router,
 } from "https://deno.land/x/oak@v12.4.0/mod.ts";
 export * as etag from "https://deno.land/x/oak@v12.4.0/etag.ts";

server/http_server.ts

@@ -1,95 +1,155 @@
-import { Application, Context, Next, oakCors, Router } from "./deps.ts";
-import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
-import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
-import { ensureSettingsAndIndex } from "../common/util.ts";
-import { BuiltinSettings } from "../web/types.ts";
-import { gitIgnoreCompiler } from "./deps.ts";
-import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts";
-import { Authenticator } from "./auth.ts";
-import { FileMeta } from "$sb/types.ts";
 import {
-  ShellRequest,
-  ShellResponse,
-  SyscallRequest,
-  SyscallResponse,
-} from "./rpc.ts";
-import { SilverBulletHooks } from "../common/manifest.ts";
-import { System } from "../plugos/system.ts";
+  Application,
+  Context,
+  Next,
+  oakCors,
+  Request,
+  Router,
+} from "./deps.ts";
+import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
+import { FileMeta } from "$sb/types.ts";
+import { ShellRequest, SyscallRequest, SyscallResponse } from "./rpc.ts";
+import { determineShellBackend } from "./shell_backend.ts";
+import { SpaceServer, SpaceServerConfig } from "./instance.ts";
+import {
+  KvPrimitives,
+  PrefixedKvPrimitives,
+} from "../plugos/lib/kv_primitives.ts";
+import { EndpointHook } from "../plugos/hooks/endpoint.ts";
+import { hashSHA256 } from "./crypto.ts";
 
 export type ServerOptions = {
+  app: Application;
   hostname: string;
   port: number;
-  pagesPath: string;
   clientAssetBundle: AssetBundle;
-  authenticator: Authenticator;
-  pass?: string;
+  plugAssetBundle: AssetBundle;
+  baseKvPrimitives?: KvPrimitives;
+  syncOnly: boolean;
   certFile?: string;
   keyFile?: string;
+
+  configs: Map<string, SpaceServerConfig>;
 };
 
 export class HttpServer {
-  private hostname: string;
-  private port: number;
   abortController?: AbortController;
   clientAssetBundle: AssetBundle;
-  settings?: BuiltinSettings;
-  spacePrimitives: SpacePrimitives;
-  authenticator: Authenticator;
+  plugAssetBundle: AssetBundle;
+  hostname: string;
+  port: number;
+  app: Application<Record<string, any>>;
+  keyFile: string | undefined;
+  certFile: string | undefined;
 
-  constructor(
-    spacePrimitives: SpacePrimitives,
-    private app: Application,
-    private system: System<SilverBulletHooks> | undefined,
-    private options: ServerOptions,
-  ) {
+  spaceServers = new Map<string, Promise<SpaceServer>>();
+  syncOnly: boolean;
+  baseKvPrimitives?: KvPrimitives;
+  configs: Map<string, SpaceServerConfig>;
+
+  constructor(options: ServerOptions) {
+    this.clientAssetBundle = options.clientAssetBundle;
+    this.plugAssetBundle = options.plugAssetBundle;
     this.hostname = options.hostname;
     this.port = options.port;
-    this.authenticator = options.authenticator;
-    this.clientAssetBundle = options.clientAssetBundle;
+    this.app = options.app;
+    this.keyFile = options.keyFile;
+    this.certFile = options.certFile;
+    this.syncOnly = options.syncOnly;
+    this.baseKvPrimitives = options.baseKvPrimitives;
+    this.configs = options.configs;
+  }
 
-    let fileFilterFn: (s: string) => boolean = () => true;
-    this.spacePrimitives = new FilteredSpacePrimitives(
-      spacePrimitives,
-      (meta) => fileFilterFn(meta.name),
-      async () => {
-        await this.reloadSettings();
-        if (typeof this.settings?.spaceIgnore === "string") {
-          fileFilterFn = gitIgnoreCompiler(this.settings.spaceIgnore).accepts;
-        } else {
-          fileFilterFn = () => true;
-        }
-      },
+  async bootSpaceServer(config: SpaceServerConfig): Promise<SpaceServer> {
+    const spaceServer = new SpaceServer(
+      config,
+      determineShellBackend(config.pagesPath),
+      this.plugAssetBundle,
+      this.baseKvPrimitives
+        ? new PrefixedKvPrimitives(this.baseKvPrimitives, [
+          config.namespace,
+        ])
+        : undefined,
     );
+    await spaceServer.init();
+
+    return spaceServer;
+  }
+
+  determineConfig(req: Request): [string, SpaceServerConfig] {
+    let hostname = req.url.host; // hostname:port
+
+    // First try a full match
+    let config = this.configs.get(hostname);
+    if (config) {
+      return [hostname, config];
+    }
+
+    // Then rip off the port and try again
+    hostname = hostname.split(":")[0];
+    config = this.configs.get(hostname);
+    if (config) {
+      return [hostname, config];
+    }
+
+    // If all else fails, try the wildcard
+    config = this.configs.get("*");
+
+    if (config) {
+      return ["*", config];
+    }
+
+    throw new Error(`No space server config found for hostname ${hostname}`);
+  }
+
+  ensureSpaceServer(req: Request): Promise<SpaceServer> {
+    const [matchedHostname, config] = this.determineConfig(req);
+    const spaceServer = this.spaceServers.get(matchedHostname);
+    if (spaceServer) {
+      return spaceServer;
+    }
+    // And then boot the thing, async
+    const spaceServerPromise = this.bootSpaceServer(config);
+    // But immediately write the promise to the map so that we don't boot it twice
+    this.spaceServers.set(matchedHostname, spaceServerPromise);
+    return spaceServerPromise;
   }
 
   // Replaces some template variables in index.html in a rather ad-hoc manner, but YOLO
-  renderIndexHtml() {
+  renderIndexHtml(pagesPath: string) {
     return this.clientAssetBundle.readTextFileSync(".client/index.html")
       .replaceAll(
         "{{SPACE_PATH}}",
-        this.options.pagesPath.replaceAll("\\", "\\\\"),
+        pagesPath.replaceAll("\\", "\\\\"),
         // );
       ).replaceAll(
         "{{SYNC_ONLY}}",
-        this.system ? "false" : "true",
+        this.syncOnly ? "true" : "false",
       );
   }
 
-  async start() {
-    await this.reloadSettings();
-
+  start() {
     // Serve static files (javascript, css, html)
     this.app.use(this.serveStatic.bind(this));
 
-    await this.addPasswordAuth(this.app);
-    const fsRouter = this.addFsRoutes(this.spacePrimitives);
+    const endpointHook = new EndpointHook("/_/");
+
+    this.app.use(async (context, next) => {
+      const spaceServer = await this.ensureSpaceServer(context.request);
+      return endpointHook.handleRequest(spaceServer.system!, context, next);
+    });
+
+    this.addPasswordAuth(this.app);
+    const fsRouter = this.addFsRoutes();
     this.app.use(fsRouter.routes());
     this.app.use(fsRouter.allowedMethods());
 
     // Fallback, serve the UI index.html
-    this.app.use(({ response }) => {
+    this.app.use(async ({ request, response }) => {
       response.headers.set("Content-type", "text/html");
-      response.body = this.renderIndexHtml();
+      response.headers.set("Cache-Control", "no-cache");
+      const spaceServer = await this.ensureSpaceServer(request);
+      response.body = this.renderIndexHtml(spaceServer.pagesPath);
     });
 
     this.abortController = new AbortController();
@@ -98,11 +158,11 @@ export class HttpServer {
       port: this.port,
       signal: this.abortController.signal,
     };
-    if (this.options.keyFile) {
-      listenOptions.key = Deno.readTextFileSync(this.options.keyFile);
+    if (this.keyFile) {
+      listenOptions.key = Deno.readTextFileSync(this.keyFile);
     }
-    if (this.options.certFile) {
-      listenOptions.cert = Deno.readTextFileSync(this.options.certFile);
+    if (this.certFile) {
+      listenOptions.cert = Deno.readTextFileSync(this.certFile);
     }
     this.app.listen(listenOptions)
       .catch((e: any) => {
@@ -117,7 +177,7 @@ export class HttpServer {
     );
   }
 
-  serveStatic(
+  async serveStatic(
     { request, response }: Context<Record<string, any>, Record<string, any>>,
     next: Next,
   ) {
@@ -127,7 +187,9 @@ export class HttpServer {
       // Serve the UI (index.html)
       // Note: we're explicitly not setting Last-Modified and If-Modified-Since header here because this page is dynamic
       response.headers.set("Content-type", "text/html");
-      response.body = this.renderIndexHtml();
+      response.headers.set("Cache-Control", "no-cache");
+      const spaceServer = await this.ensureSpaceServer(request);
+      response.body = this.renderIndexHtml(spaceServer.pagesPath);
       return;
     }
     try {
@@ -163,12 +225,7 @@ export class HttpServer {
     }
   }
 
-  async reloadSettings() {
-    // TODO: Throttle this?
-    this.settings = await ensureSettingsAndIndex(this.spacePrimitives);
-  }
-
-  private async addPasswordAuth(app: Application) {
+  private addPasswordAuth(app: Application) {
     const excludedPaths = [
       "/manifest.json",
       "/favicon.png",
@@ -192,14 +249,17 @@ export class HttpServer {
           return;
         } else if (request.method === "POST") {
           const values = await request.body({ type: "form" }).value;
-          const username = values.get("username")!,
-            password = values.get("password")!,
-            refer = values.get("refer");
-          const hashedPassword = await this.authenticator.authenticate(
-            username,
-            password,
-          );
-          if (hashedPassword) {
+          const username = values.get("username")!;
+          const password = values.get("password")!;
+          const refer = values.get("refer");
+
+          const spaceServer = await this.ensureSpaceServer(request);
+          const hashedPassword = await hashSHA256(password);
+          const [expectedUser, expectedPassword] = spaceServer.auth!.split(":");
+          if (
+            username === expectedUser &&
+            hashedPassword === await hashSHA256(expectedPassword)
+          ) {
             await cookies.set(
               authCookieName(host),
               `${username}:${hashedPassword}`,
@@ -223,33 +283,38 @@ export class HttpServer {
       }
     });
 
-    if ((await this.authenticator.getAllUsers()).length > 0) {
-      // Users defined, so enabling auth
-      app.use(async ({ request, response, cookies }, next) => {
-        const host = request.url.host;
-        if (!excludedPaths.includes(request.url.pathname)) {
-          const authCookie = await cookies.get(authCookieName(host));
-          if (!authCookie) {
-            response.redirect("/.auth");
-            return;
-          }
-          const [username, hashedPassword] = authCookie.split(":");
-          if (
-            !await this.authenticator.authenticateHashed(
-              username,
-              hashedPassword,
-            )
-          ) {
-            response.redirect("/.auth");
-            return;
-          }
+    // Check auth
+    app.use(async ({ request, response, cookies }, next) => {
+      const spaceServer = await this.ensureSpaceServer(request);
+      if (!spaceServer.auth) {
+        // Auth disabled in this config, skip
+        return next();
+      }
+      const host = request.url.host;
+      if (!excludedPaths.includes(request.url.pathname)) {
+        const authCookie = await cookies.get(authCookieName(host));
+        if (!authCookie) {
+          response.redirect("/.auth");
+          return;
         }
-        await next();
-      });
-    }
+        const spaceServer = await this.ensureSpaceServer(request);
+        const [username, hashedPassword] = authCookie.split(":");
+        const [expectedUser, expectedPassword] = spaceServer.auth!.split(
+          ":",
+        );
+        if (
+          username !== expectedUser ||
+          hashedPassword !== await hashSHA256(expectedPassword)
+        ) {
+          response.redirect("/.auth");
+          return;
+        }
+      }
+      await next();
+    });
   }
 
-  private addFsRoutes(spacePrimitives: SpacePrimitives): Router {
+  private addFsRoutes(): Router {
     const fsRouter = new Router();
     const corsMiddleware = oakCors({
       allowedHeaders: "*",
@@ -264,11 +329,12 @@ export class HttpServer {
       "/index.json",
       // corsMiddleware,
       async ({ request, response }) => {
+        const spaceServer = await this.ensureSpaceServer(request);
         if (request.headers.has("X-Sync-Mode")) {
           // Only handle direct requests for a JSON representation of the file list
           response.headers.set("Content-type", "application/json");
-          response.headers.set("X-Space-Path", this.options.pagesPath);
-          const files = await spacePrimitives.fetchFileList();
+          response.headers.set("X-Space-Path", spaceServer.pagesPath);
+          const files = await spaceServer.spacePrimitives.fetchFileList();
           response.body = JSON.stringify(files);
         } else {
           // Otherwise, redirect to the UI
@@ -280,49 +346,29 @@ export class HttpServer {
 
     // RPC
     fsRouter.post("/.rpc", async ({ request, response }) => {
+      const spaceServer = await this.ensureSpaceServer(request);
       const body = await request.body({ type: "json" }).value;
       try {
         switch (body.operation) {
           case "shell": {
-            // TODO: Have a nicer way to do this
-            if (this.options.pagesPath.startsWith("s3://")) {
-              response.status = 500;
-              response.body = JSON.stringify({
-                stdout: "",
-                stderr: "Cannot run shell commands with S3 backend",
-                code: 500,
-              });
-              return;
-            }
             const shellCommand: ShellRequest = body;
             console.log(
               "Running shell command:",
               shellCommand.cmd,
               shellCommand.args,
             );
-            const p = new Deno.Command(shellCommand.cmd, {
-              args: shellCommand.args,
-              cwd: this.options.pagesPath,
-              stdout: "piped",
-              stderr: "piped",
-            });
-            const output = await p.output();
-            const stdout = new TextDecoder().decode(output.stdout);
-            const stderr = new TextDecoder().decode(output.stderr);
-
+            const shellResponse = await spaceServer.shellBackend.handle(
+              shellCommand,
+            );
             response.headers.set("Content-Type", "application/json");
-            response.body = JSON.stringify({
-              stdout,
-              stderr,
-              code: output.code,
-            } as ShellResponse);
-            if (output.code !== 0) {
-              console.error("Error running shell command", stdout, stderr);
+            response.body = JSON.stringify(shellResponse);
+            if (shellResponse.code !== 0) {
+              console.error("Error running shell command", shellResponse);
             }
             return;
           }
           case "syscall": {
-            if (!this.system) {
+            if (this.syncOnly) {
               response.headers.set("Content-Type", "text/plain");
               response.status = 400;
               response.body = "Unknown operation";
@@ -330,7 +376,9 @@ export class HttpServer {
             }
             const syscallCommand: SyscallRequest = body;
             try {
-              const plug = this.system.loadedPlugs.get(syscallCommand.ctx);
+              const plug = spaceServer.system!.loadedPlugs.get(
+                syscallCommand.ctx,
+              );
               if (!plug) {
                 throw new Error(`Plug ${syscallCommand.ctx} not found`);
               }
@@ -372,6 +420,7 @@ export class HttpServer {
         filePathRegex,
         async ({ params, response, request }) => {
           const name = params[0];
+          const spaceServer = await this.ensureSpaceServer(request);
           console.log("Requested file", name);
           if (!request.headers.has("X-Sync-Mode") && name.endsWith(".md")) {
             // It can happen that during a sync, authentication expires, this may result in a redirect to the login page and then back to this particular file. This particular file may be an .md file, which isn't great to show so we're redirecting to the associated SB UI page.
@@ -415,13 +464,15 @@ export class HttpServer {
           try {
             if (request.headers.has("X-Get-Meta")) {
               // Getting meta via GET request
-              const fileData = await spacePrimitives.getFileMeta(name);
+              const fileData = await spaceServer.spacePrimitives.getFileMeta(
+                name,
+              );
               response.status = 200;
               this.fileMetaToHeaders(response.headers, fileData);
               response.body = "";
               return;
             }
-            const fileData = await spacePrimitives.readFile(name);
+            const fileData = await spaceServer.spacePrimitives.readFile(name);
             const lastModifiedHeader = new Date(fileData.meta.lastModified)
               .toUTCString();
             if (
@@ -447,6 +498,7 @@ export class HttpServer {
         filePathRegex,
         async ({ request, response, params }) => {
           const name = params[0];
+          const spaceServer = await this.ensureSpaceServer(request);
           console.log("Saving file", name);
           if (name.startsWith(".")) {
             // Don't expose hidden files
@@ -457,7 +509,7 @@ export class HttpServer {
           const body = await request.body({ type: "bytes" }).value;
 
           try {
-            const meta = await spacePrimitives.writeFile(
+            const meta = await spaceServer.spacePrimitives.writeFile(
               name,
               body,
             );
@@ -471,8 +523,9 @@ export class HttpServer {
           }
         },
       )
-      .delete(filePathRegex, async ({ response, params }) => {
+      .delete(filePathRegex, async ({ request, response, params }) => {
         const name = params[0];
+        const spaceServer = await this.ensureSpaceServer(request);
         console.log("Deleting file", name);
         if (name.startsWith(".")) {
           // Don't expose hidden files
@@ -480,7 +533,7 @@ export class HttpServer {
           return;
         }
         try {
-          await spacePrimitives.deleteFile(name);
+          await spaceServer.spacePrimitives.deleteFile(name);
           response.status = 200;
           response.body = "OK";
         } catch (e: any) {
@@ -573,11 +626,3 @@ function utcDateString(mtime: number): string {
 function authCookieName(host: string) {
   return `auth:${host}`;
 }
-
-function headersToJson(headers: Headers) {
-  let headersObj: any = {};
-  for (const [key, value] of headers.entries()) {
-    headersObj[key] = value;
-  }
-  return JSON.stringify(headersObj);
-}

server/instance.ts

@@ -0,0 +1,87 @@
+import { SilverBulletHooks } from "../common/manifest.ts";
+import { AssetBundlePlugSpacePrimitives } from "../common/spaces/asset_bundle_space_primitives.ts";
+import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts";
+import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
+import { ensureSettingsAndIndex } from "../common/util.ts";
+import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
+import { KvPrimitives } from "../plugos/lib/kv_primitives.ts";
+import { System } from "../plugos/system.ts";
+import { BuiltinSettings } from "../web/types.ts";
+import { gitIgnoreCompiler } from "./deps.ts";
+import { ServerSystem } from "./server_system.ts";
+import { ShellBackend } from "./shell_backend.ts";
+import { determineStorageBackend } from "./storage_backend.ts";
+
+export type SpaceServerConfig = {
+  hostname: string;
+  namespace: string;
+  auth?: string; // username:password
+  pagesPath: string;
+};
+
+export class SpaceServer {
+  public pagesPath: string;
+  auth?: string;
+  hostname: string;
+
+  private settings?: BuiltinSettings;
+  spacePrimitives: SpacePrimitives;
+
+  // Only set when syncOnly == false
+  private serverSystem?: ServerSystem;
+  system?: System<SilverBulletHooks>;
+
+  constructor(
+    config: SpaceServerConfig,
+    public shellBackend: ShellBackend,
+    plugAssetBundle: AssetBundle,
+    kvPrimitives?: KvPrimitives,
+  ) {
+    this.pagesPath = config.pagesPath;
+    this.hostname = config.hostname;
+    this.auth = config.auth;
+
+    let fileFilterFn: (s: string) => boolean = () => true;
+
+    this.spacePrimitives = new FilteredSpacePrimitives(
+      new AssetBundlePlugSpacePrimitives(
+        determineStorageBackend(this.pagesPath),
+        plugAssetBundle,
+      ),
+      (meta) => fileFilterFn(meta.name),
+      async () => {
+        await this.reloadSettings();
+        if (typeof this.settings?.spaceIgnore === "string") {
+          fileFilterFn = gitIgnoreCompiler(this.settings.spaceIgnore).accepts;
+        } else {
+          fileFilterFn = () => true;
+        }
+      },
+    );
+
+    // system = undefined in databaseless mode (no PlugOS instance on the server and no DB)
+    if (kvPrimitives) {
+      // Enable server-side processing
+      const serverSystem = new ServerSystem(
+        this.spacePrimitives,
+        kvPrimitives,
+      );
+      this.serverSystem = serverSystem;
+    }
+  }
+
+  async init() {
+    if (this.serverSystem) {
+      await this.serverSystem.init();
+      this.system = this.serverSystem.system;
+    }
+
+    await this.reloadSettings();
+    console.log("Booted server with hostname", this.hostname);
+  }
+
+  async reloadSettings() {
+    // TODO: Throttle this?
+    this.settings = await ensureSettingsAndIndex(this.spacePrimitives);
+  }
+}

server/server_system.ts

@@ -25,7 +25,6 @@ import { shellSyscalls } from "../plugos/syscalls/shell.deno.ts";
 import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
 import { base64EncodedDataUrl } from "../plugos/asset_bundle/base64.ts";
 import { Plug } from "../plugos/plug.ts";
-import { DenoKvPrimitives } from "../plugos/lib/deno_kv_primitives.ts";
 import { DataStore } from "../plugos/lib/datastore.ts";
 import { dataStoreSyscalls } from "../plugos/syscalls/datastore.ts";
 import { DataStoreMQ } from "../plugos/lib/mq.datastore.ts";
@@ -34,6 +33,7 @@ import { handlebarsSyscalls } from "../common/syscalls/handlebars.ts";
 import { codeWidgetSyscalls } from "../web/syscalls/code_widget.ts";
 import { CodeWidgetHook } from "../web/hooks/code_widget.ts";
 import { KVPrimitivesManifestCache } from "../plugos/manifest_cache.ts";
+import { KvPrimitives } from "../plugos/lib/kv_primitives.ts";
 
 const fileListInterval = 30 * 1000; // 30s
 
@@ -42,27 +42,27 @@ const plugNameExtractRegex = /\/(.+)\.plug\.js$/;
 export class ServerSystem {
   system!: System<SilverBulletHooks>;
   spacePrimitives!: SpacePrimitives;
-  denoKv!: Deno.Kv;
+  // denoKv!: Deno.Kv;
   listInterval?: number;
   ds!: DataStore;
 
   constructor(
     private baseSpacePrimitives: SpacePrimitives,
-    private dbPath: string,
-    private app: Application,
+    readonly kvPrimitives: KvPrimitives,
   ) {
   }
 
   // Always needs to be invoked right after construction
   async init(awaitIndex = false) {
-    this.denoKv = await Deno.openKv(this.dbPath);
-    const kvPrimitives = new DenoKvPrimitives(this.denoKv);
-    this.ds = new DataStore(kvPrimitives);
+    this.ds = new DataStore(this.kvPrimitives);
 
     this.system = new System(
       "server",
       {
-        manifestCache: new KVPrimitivesManifestCache(kvPrimitives, "manifest"),
+        manifestCache: new KVPrimitivesManifestCache(
+          this.kvPrimitives,
+          "manifest",
+        ),
         plugFlushTimeout: 5 * 60 * 1000, // 5 minutes
       },
     );
@@ -75,9 +75,6 @@ export class ServerSystem {
     const cronHook = new CronHook(this.system);
     this.system.addHook(cronHook);
 
-    // Endpoint hook
-    this.system.addHook(new EndpointHook(this.app, "/_/"));
-
     const mq = new DataStoreMQ(this.ds);
 
     setInterval(() => {

server/shell_backend.ts

@@ -0,0 +1,64 @@
+import { ShellRequest, ShellResponse } from "./rpc.ts";
+
+/**
+ * Configuration via environment variables:
+ * - SB_SHELL_BACKEND: "local" or "off"
+ */
+
+export function determineShellBackend(path: string): ShellBackend {
+  const backendConfig = Deno.env.get("SB_SHELL_BACKEND") || "local";
+  switch (backendConfig) {
+    case "local":
+      return new LocalShell(path);
+    default:
+      console.info(
+        "Running in shellless mode, meaning shell commands are disabled",
+      );
+      return new NotSupportedShell();
+  }
+}
+
+export interface ShellBackend {
+  handle(shellRequest: ShellRequest): Promise<ShellResponse>;
+}
+
+class NotSupportedShell implements ShellBackend {
+  handle(): Promise<ShellResponse> {
+    return Promise.resolve({
+      stdout: "",
+      stderr: "Not supported",
+      code: 1,
+    });
+  }
+}
+
+class LocalShell implements ShellBackend {
+  constructor(private cwd: string) {
+  }
+
+  async handle(shellRequest: ShellRequest): Promise<ShellResponse> {
+    console.log(
+      "Running shell command:",
+      shellRequest.cmd,
+      shellRequest.args,
+    );
+    const p = new Deno.Command(shellRequest.cmd, {
+      cwd: this.cwd,
+      args: shellRequest.args,
+      stdout: "piped",
+      stderr: "piped",
+    });
+    const output = await p.output();
+    const stdout = new TextDecoder().decode(output.stdout);
+    const stderr = new TextDecoder().decode(output.stderr);
+    if (output.code !== 0) {
+      console.error("Error running shell command", stdout, stderr);
+    }
+
+    return {
+      stderr,
+      stdout,
+      code: output.code,
+    };
+  }
+}

server/spaces/s3_space_primitives.test.ts

@@ -9,6 +9,7 @@ Deno.test("s3_space_primitives", async () => {
     endPoint: "s3.eu-central-1.amazonaws.com",
     region: "eu-central-1",
     bucket: "zef-sb-space",
+    prefix: "test",
   };
 
   const primitives = new S3SpacePrimitives(options);

server/spaces/s3_space_primitives.ts

@@ -7,10 +7,15 @@ import { FileMeta } from "$sb/types.ts";
 
 // TODO: IMPORTANT: This needs a different way to keep meta data (last modified and created dates)
 
+export type S3SpacePrimitivesOptions = ClientOptions & { prefix: string };
+
 export class S3SpacePrimitives implements SpacePrimitives {
   client: S3Client;
-  constructor(options: ClientOptions) {
+  prefix: string;
+  constructor(options: S3SpacePrimitivesOptions) {
     this.client = new S3Client(options);
+    // TODO: Use this
+    this.prefix = options.prefix;
   }
 
   private encodePath(name: string): string {
@@ -18,7 +23,7 @@ export class S3SpacePrimitives implements SpacePrimitives {
   }
 
   private decodePath(encoded: string): string {
-    // AWS only returns ' replace dwith '
+    // AWS only returns ' replace with '
     return encoded.replaceAll("'", "'");
   }
 

server/storage_backend.ts

@@ -0,0 +1,22 @@
+import { DiskSpacePrimitives } from "../common/spaces/disk_space_primitives.ts";
+import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
+import { path } from "./deps.ts";
+import { S3SpacePrimitives } from "./spaces/s3_space_primitives.ts";
+
+export function determineStorageBackend(folder: string): SpacePrimitives {
+  if (folder === "s3://") {
+    console.info("Using S3 as a storage backend");
+    return new S3SpacePrimitives({
+      accessKey: Deno.env.get("AWS_ACCESS_KEY_ID")!,
+      secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!,
+      endPoint: Deno.env.get("AWS_ENDPOINT")!,
+      region: Deno.env.get("AWS_REGION")!,
+      bucket: Deno.env.get("AWS_BUCKET")!,
+      prefix: folder.slice(5),
+    });
+  } else {
+    folder = path.resolve(Deno.cwd(), folder);
+    console.info(`Using local disk as a storage backend: ${folder}`);
+    return new DiskSpacePrimitives(folder);
+  }
+}