Work on #508 (thin client)

server/http_server.ts

@@ -2,12 +2,19 @@ import { Application, Context, Next, oakCors, Router } from "./deps.ts";
 import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
 import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
 import { ensureSettingsAndIndex } from "../common/util.ts";
-import { performLocalFetch } from "../common/proxy_fetch.ts";
 import { BuiltinSettings } from "../web/types.ts";
 import { gitIgnoreCompiler } from "./deps.ts";
 import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts";
 import { Authenticator } from "./auth.ts";
 import { FileMeta } from "$sb/types.ts";
+import {
+  ShellRequest,
+  ShellResponse,
+  SyscallRequest,
+  SyscallResponse,
+} from "./rpc.ts";
+import { SilverBulletHooks } from "../common/manifest.ts";
+import { System } from "../plugos/system.ts";
 
 export type ServerOptions = {
   hostname: string;
@@ -18,11 +25,9 @@ export type ServerOptions = {
   pass?: string;
   certFile?: string;
   keyFile?: string;
-  maxFileSizeMB?: number;
 };
 
 export class HttpServer {
-  app: Application;
   private hostname: string;
   private port: number;
   abortController?: AbortController;
@@ -33,27 +38,19 @@ export class HttpServer {
 
   constructor(
     spacePrimitives: SpacePrimitives,
+    private app: Application,
+    private system: System<SilverBulletHooks> | undefined,
     private options: ServerOptions,
   ) {
     this.hostname = options.hostname;
     this.port = options.port;
-    this.app = new Application();
     this.authenticator = options.authenticator;
     this.clientAssetBundle = options.clientAssetBundle;
 
     let fileFilterFn: (s: string) => boolean = () => true;
     this.spacePrimitives = new FilteredSpacePrimitives(
       spacePrimitives,
-      (meta) => {
-        // Don't list file exceeding the maximum file size
-        if (
-          options.maxFileSizeMB &&
-          meta.size / (1024 * 1024) > options.maxFileSizeMB
-        ) {
-          return false;
-        }
-        return fileFilterFn(meta.name);
-      },
+      (meta) => fileFilterFn(meta.name),
       async () => {
         await this.reloadSettings();
         if (typeof this.settings?.spaceIgnore === "string") {
@@ -71,7 +68,7 @@ export class HttpServer {
       .replaceAll(
         "{{SPACE_PATH}}",
         this.options.pagesPath.replaceAll("\\", "\\\\"),
-      );
+      ).replaceAll("{{THIN_CLIENT_MODE}}", this.system ? "on" : "off");
   }
 
   async start() {
@@ -282,13 +279,6 @@ export class HttpServer {
       const body = await request.body({ type: "json" }).value;
       try {
         switch (body.operation) {
-          // case "fetch": {
-          //   const result = await performLocalFetch(body.url, body.options);
-          //   console.log("Proxying fetch request to", body.url);
-          //   response.headers.set("Content-Type", "application/json");
-          //   response.body = JSON.stringify(result);
-          //   return;
-          // }
           case "shell": {
             // TODO: Have a nicer way to do this
             if (this.options.pagesPath.startsWith("s3://")) {
@@ -300,9 +290,14 @@ export class HttpServer {
               });
               return;
             }
-            console.log("Running shell command:", body.cmd, body.args);
-            const p = new Deno.Command(body.cmd, {
-              args: body.args,
+            const shellCommand: ShellRequest = body;
+            console.log(
+              "Running shell command:",
+              shellCommand.cmd,
+              shellCommand.args,
+            );
+            const p = new Deno.Command(shellCommand.cmd, {
+              args: shellCommand.args,
               cwd: this.options.pagesPath,
               stdout: "piped",
               stderr: "piped",
@@ -316,12 +311,40 @@ export class HttpServer {
               stdout,
               stderr,
               code: output.code,
-            });
+            } as ShellResponse);
             if (output.code !== 0) {
               console.error("Error running shell command", stdout, stderr);
             }
             return;
           }
+          case "syscall": {
+            if (!this.system) {
+              response.headers.set("Content-Type", "text/plain");
+              response.status = 400;
+              response.body = "Unknown operation";
+              return;
+            }
+            const syscallCommand: SyscallRequest = body;
+            try {
+              const result = await this.system.localSyscall(
+                syscallCommand.ctx,
+                syscallCommand.name,
+                syscallCommand.args,
+              );
+              response.headers.set("Content-type", "application/json");
+              response.status = 200;
+              response.body = JSON.stringify({
+                result: result,
+              } as SyscallResponse);
+            } catch (e: any) {
+              response.headers.set("Content-type", "application/json");
+              response.status = 500;
+              response.body = JSON.stringify({
+                error: e.message,
+              } as SyscallResponse);
+            }
+            return;
+          }
           default:
             response.headers.set("Content-Type", "text/plain");
             response.status = 400;
@@ -530,10 +553,3 @@ function utcDateString(mtime: number): string {
 function authCookieName(host: string) {
   return `auth:${host}`;
 }
-
-function copyHeader(fromHeaders: Headers, toHeaders: Headers, header: string) {
-  const value = fromHeaders.get(header);
-  if (value) {
-    toHeaders.set(header, value);
-  }
-}

server/rpc.ts

@@ -0,0 +1,21 @@
+export type ShellRequest = {
+  cmd: string;
+  args: string[];
+};
+
+export type ShellResponse = {
+  stdout: string;
+  stderr: string;
+  code: number;
+};
+
+export type SyscallRequest = {
+  ctx: string; // Plug name requesting
+  name: string;
+  args: any[];
+};
+
+export type SyscallResponse = {
+  result?: any;
+  error?: string;
+};

server/server_system.ts

@@ -0,0 +1,160 @@
+import { PlugNamespaceHook } from "../common/hooks/plug_namespace.ts";
+import { SilverBulletHooks } from "../common/manifest.ts";
+import { loadMarkdownExtensions } from "../common/markdown_parser/markdown_ext.ts";
+import buildMarkdown from "../common/markdown_parser/parser.ts";
+import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives.ts";
+import { FileMetaSpacePrimitives } from "../common/spaces/file_meta_space_primitives.ts";
+import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts";
+import { createSandbox } from "../plugos/environments/webworker_sandbox.ts";
+import { CronHook } from "../plugos/hooks/cron.ts";
+import { EndpointHook } from "../plugos/hooks/endpoint.ts";
+import { EventHook } from "../plugos/hooks/event.ts";
+import { MQHook } from "../plugos/hooks/mq.ts";
+import { DenoKVStore } from "../plugos/lib/kv_store.deno_kv.ts";
+import { DexieMQ } from "../plugos/lib/mq.dexie.ts";
+import assetSyscalls from "../plugos/syscalls/asset.ts";
+import { eventSyscalls } from "../plugos/syscalls/event.ts";
+import { mqSyscalls } from "../plugos/syscalls/mq.dexie.ts";
+import { storeSyscalls } from "../plugos/syscalls/store.ts";
+import { System } from "../plugos/system.ts";
+import { Space } from "../web/space.ts";
+import { debugSyscalls } from "../web/syscalls/debug.ts";
+import { pageIndexSyscalls } from "../cli/syscalls/index.ts";
+import { markdownSyscalls } from "../web/syscalls/markdown.ts";
+import { spaceSyscalls } from "../cli/syscalls/space.ts";
+import { systemSyscalls } from "../web/syscalls/system.ts";
+import { yamlSyscalls } from "../web/syscalls/yaml.ts";
+import { Application, path } from "./deps.ts";
+import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts";
+import { shellSyscalls } from "../plugos/syscalls/shell.deno.ts";
+import { IDBKeyRange, indexedDB } from "https://esm.sh/fake-indexeddb@4.0.2";
+import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
+
+export class ServerSystem {
+  system: System<SilverBulletHooks> = new System("server");
+  spacePrimitives!: SpacePrimitives;
+  requeueInterval?: number;
+  kvStore?: DenoKVStore;
+
+  constructor(
+    private baseSpacePrimitives: SpacePrimitives,
+    private dbPath: string,
+    private app: Application,
+  ) {
+  }
+
+  // Always needs to be invoked right after construction
+  async init() {
+    // Event hook
+    const eventHook = new EventHook();
+    this.system.addHook(eventHook);
+
+    // Cron hook
+    const cronHook = new CronHook(this.system);
+    this.system.addHook(cronHook);
+
+    this.kvStore = new DenoKVStore();
+    await this.kvStore.init(this.dbPath);
+
+    // Endpoint hook
+    this.system.addHook(new EndpointHook(this.app, "/_/"));
+
+    // Use DexieMQ for this, in memory
+    const mq = new DexieMQ("mq", indexedDB, IDBKeyRange);
+
+    this.requeueInterval = setInterval(() => {
+      // Timeout after 5s, retries 3 times, otherwise drops the message (no DLQ)
+      mq.requeueTimeouts(5000, 3, true).catch(console.error);
+    }, 20000); // Look to requeue every 20s
+
+    const pageIndexCalls = pageIndexSyscalls(this.kvStore);
+
+    const plugNamespaceHook = new PlugNamespaceHook();
+    this.system.addHook(plugNamespaceHook);
+
+    this.system.addHook(new MQHook(this.system, mq));
+
+    this.spacePrimitives = new FileMetaSpacePrimitives(
+      new EventedSpacePrimitives(
+        new PlugSpacePrimitives(
+          this.baseSpacePrimitives,
+          plugNamespaceHook,
+        ),
+        eventHook,
+      ),
+      pageIndexCalls,
+    );
+    const space = new Space(this.spacePrimitives, this.kvStore);
+
+    // Add syscalls
+    this.system.registerSyscalls(
+      [],
+      eventSyscalls(eventHook),
+      spaceSyscalls(space),
+      assetSyscalls(this.system),
+      yamlSyscalls(),
+      storeSyscalls(this.kvStore),
+      systemSyscalls(undefined as any, this.system),
+      mqSyscalls(mq),
+      pageIndexCalls,
+      debugSyscalls(),
+      markdownSyscalls(buildMarkdown([])), // Will later be replaced with markdown extensions
+    );
+
+    // Syscalls that require some additional permissions
+    this.system.registerSyscalls(
+      ["fetch"],
+      sandboxFetchSyscalls(),
+    );
+
+    this.system.registerSyscalls(
+      ["shell"],
+      shellSyscalls("."),
+    );
+
+    await this.loadPlugs();
+
+    // for (let plugPath of await space.listPlugs()) {
+    //   plugPath = path.resolve(this.spacePath, plugPath);
+    //   await this.system.load(
+    //     new URL(`file://${plugPath}`),
+    //     createSandbox,
+    //   );
+    // }
+
+    // Load markdown syscalls based on all new syntax (if any)
+    this.system.registerSyscalls(
+      [],
+      markdownSyscalls(buildMarkdown(loadMarkdownExtensions(this.system))),
+    );
+  }
+
+  async loadPlugs() {
+    const tempDir = await Deno.makeTempDir();
+    try {
+      for (const { name } of await this.spacePrimitives.fetchFileList()) {
+        if (
+          name.endsWith(".plug.js") // && !filePath.includes("search.plug.js")
+        ) {
+          const plugPath = path.join(tempDir, name);
+          await Deno.mkdir(path.dirname(plugPath), { recursive: true });
+          await Deno.writeFile(
+            plugPath,
+            (await this.spacePrimitives.readFile(name)).data,
+          );
+          await this.system.load(
+            new URL(`file://${plugPath}`),
+            createSandbox,
+          );
+        }
+      }
+    } finally {
+      await Deno.remove(tempDir, { recursive: true });
+    }
+  }
+
+  async close() {
+    clearInterval(this.requeueInterval);
+    await this.system.unloadAll();
+  }
+}