* Refactored server to use spaces

* Other cleanup

server/api_server.ts

@@ -4,7 +4,7 @@ import { EndpointHook } from "../plugos/hooks/endpoint";
 import { readFile } from "fs/promises";
 import { System } from "../plugos/system";
 import cors from "cors";
-import { DiskStorage, EventedStorage, Storage } from "./disk_storage";
+import { DiskSpacePrimitives } from "../common/spaces/disk_space_primitives";
 import path from "path";
 import bodyParser from "body-parser";
 import { EventHook } from "../plugos/hooks/event";
@@ -15,12 +15,16 @@ import knex, { Knex } from "knex";
 import shellSyscalls from "../plugos/syscalls/shell.node";
 import { NodeCronHook } from "../plugos/hooks/node_cron";
 import { markdownSyscalls } from "../common/syscalls/markdown";
+import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives";
+import { Space } from "../common/spaces/space";
+import { safeRun } from "../webapp/util";
+import { createSandbox } from "../plugos/environments/node_sandbox";
 
 export class ExpressServer {
   app: Express;
   system: System<SilverBulletHooks>;
   private rootPath: string;
-  private storage: Storage;
+  private space: Space;
   private distDir: string;
   private eventHook: EventHook;
   private db: Knex<any, unknown[]>;
@@ -39,9 +43,12 @@ export class ExpressServer {
     // Setup system
     this.eventHook = new EventHook();
     system.addHook(this.eventHook);
-    this.storage = new EventedStorage(
-      new DiskStorage(rootPath),
-      this.eventHook
+    this.space = new Space(
+      new EventedSpacePrimitives(
+        new DiskSpacePrimitives(rootPath),
+        this.eventHook
+      ),
+      true
     );
     this.db = knex({
       client: "better-sqlite3",
@@ -55,10 +62,30 @@ export class ExpressServer {
     system.addHook(new NodeCronHook());
 
     system.registerSyscalls([], pageIndexSyscalls(this.db));
-    system.registerSyscalls([], spaceSyscalls(this.storage));
+    system.registerSyscalls([], spaceSyscalls(this.space));
     system.registerSyscalls([], eventSyscalls(this.eventHook));
     system.registerSyscalls([], markdownSyscalls());
     system.addHook(new EndpointHook(app, "/_/"));
+
+    this.space.on({
+      plugLoaded: (plugName, plug) => {
+        safeRun(async () => {
+          console.log("Plug load", plugName);
+          await system.load(plugName, plug, createSandbox);
+        });
+      },
+      plugUnloaded: (plugName) => {
+        safeRun(async () => {
+          console.log("Plug unload", plugName);
+          await system.unload(plugName);
+        });
+      },
+    });
+
+    setInterval(() => {
+      this.space.updatePageListAsync();
+    }, 5000);
+    this.space.updatePageListAsync();
   }
 
   async init() {
@@ -68,8 +95,9 @@ export class ExpressServer {
 
     // Page list
     fsRouter.route("/").get(async (req, res) => {
-      res.header("Now-Timestamp", "" + Date.now());
-      res.json(await this.storage.listPages());
+      let { nowTimestamp, pages } = await this.space.fetchPageList();
+      res.header("Now-Timestamp", "" + nowTimestamp);
+      res.json([...pages]);
     });
 
     fsRouter.route("/").post(bodyParser.json(), async (req, res) => {});
@@ -80,7 +108,7 @@ export class ExpressServer {
         let pageName = req.params[0];
         // console.log("Getting", pageName);
         try {
-          let pageData = await this.storage.readPage(pageName);
+          let pageData = await this.space.readPage(pageName);
           res.status(200);
           res.header("Last-Modified", "" + pageData.meta.lastModified);
           res.header("Content-Type", "text/markdown");
@@ -97,9 +125,10 @@ export class ExpressServer {
         console.log("Saving", pageName);
 
         try {
-          let meta = await this.storage.writePage(
+          let meta = await this.space.writePage(
             pageName,
             req.body,
+            false,
             req.header("Last-Modified")
               ? +req.header("Last-Modified")!
               : undefined
@@ -116,7 +145,7 @@ export class ExpressServer {
       .options(async (req, res) => {
         let pageName = req.params[0];
         try {
-          const meta = await this.storage.getPageMeta(pageName);
+          const meta = await this.space.getPageMeta(pageName);
           res.status(200);
           res.header("Last-Modified", "" + meta.lastModified);
           res.header("Content-Type", "text/markdown");
@@ -131,7 +160,7 @@ export class ExpressServer {
       .delete(async (req, res) => {
         let pageName = req.params[0];
         try {
-          await this.storage.deletePage(pageName);
+          await this.space.deletePage(pageName);
           res.status(200);
           res.send("OK");
         } catch (e) {

server/disk_storage.ts

@@ -1,176 +0,0 @@
-import { mkdir, readdir, readFile, stat, unlink, utimes, writeFile } from "fs/promises";
-import * as path from "path";
-import { PageMeta } from "../common/types";
-import { EventHook } from "../plugos/hooks/event";
-
-export interface Storage {
-  listPages(): Promise<PageMeta[]>;
-  readPage(pageName: string): Promise<{ text: string; meta: PageMeta }>;
-  writePage(
-    pageName: string,
-    text: string,
-    lastModified?: number
-  ): Promise<PageMeta>;
-  getPageMeta(pageName: string): Promise<PageMeta>;
-  deletePage(pageName: string): Promise<void>;
-}
-
-export class EventedStorage implements Storage {
-  constructor(private wrapped: Storage, private eventHook: EventHook) {}
-
-  listPages(): Promise<PageMeta[]> {
-    return this.wrapped.listPages();
-  }
-
-  readPage(pageName: string): Promise<{ text: string; meta: PageMeta }> {
-    return this.wrapped.readPage(pageName);
-  }
-
-  async writePage(
-    pageName: string,
-    text: string,
-    lastModified?: number
-  ): Promise<PageMeta> {
-    const newPageMeta = this.wrapped.writePage(pageName, text, lastModified);
-    // This can happen async
-    this.eventHook
-      .dispatchEvent("page:saved", pageName)
-      .then(() => {
-        return this.eventHook.dispatchEvent("page:index", {
-          name: pageName,
-          text: text,
-        });
-      })
-      .catch((e) => {
-        console.error("Error dispatching page:saved event", e);
-      });
-    return newPageMeta;
-  }
-
-  getPageMeta(pageName: string): Promise<PageMeta> {
-    return this.wrapped.getPageMeta(pageName);
-  }
-
-  async deletePage(pageName: string): Promise<void> {
-    await this.eventHook.dispatchEvent("page:deleted", pageName);
-    return this.wrapped.deletePage(pageName);
-  }
-}
-
-export class DiskStorage implements Storage {
-  rootPath: string;
-  plugPrefix: string;
-
-  constructor(rootPath: string, plugPrefix: string = "_plug/") {
-    this.rootPath = rootPath;
-    this.plugPrefix = plugPrefix;
-  }
-
-  pageNameToPath(pageName: string) {
-    if (pageName.startsWith(this.plugPrefix)) {
-      return path.join(this.rootPath, pageName + ".plug.json");
-    }
-    return path.join(this.rootPath, pageName + ".md");
-  }
-
-  pathToPageName(fullPath: string): string {
-    let extLength = fullPath.endsWith(".plug.json")
-      ? ".plug.json".length
-      : ".md".length;
-    return fullPath.substring(
-      this.rootPath.length + 1,
-      fullPath.length - extLength
-    );
-  }
-
-  async listPages(): Promise<PageMeta[]> {
-    let fileNames: PageMeta[] = [];
-
-    const walkPath = async (dir: string) => {
-      let files = await readdir(dir);
-      for (let file of files) {
-        const fullPath = path.join(dir, file);
-        let s = await stat(fullPath);
-        // console.log("Encountering", fullPath, s);
-        if (s.isDirectory()) {
-          await walkPath(fullPath);
-        } else {
-          if (file.endsWith(".md") || file.endsWith(".json")) {
-            fileNames.push({
-              name: this.pathToPageName(fullPath),
-              lastModified: s.mtime.getTime(),
-            });
-          }
-        }
-      }
-    };
-    await walkPath(this.rootPath);
-    return fileNames;
-  }
-
-  async readPage(pageName: string): Promise<{ text: string; meta: PageMeta }> {
-    const localPath = this.pageNameToPath(pageName);
-    try {
-      const s = await stat(localPath);
-      return {
-        text: await readFile(localPath, "utf8"),
-        meta: {
-          name: pageName,
-          lastModified: s.mtime.getTime(),
-        },
-      };
-    } catch (e) {
-      // console.error("Error while reading page", pageName, e);
-      throw Error(`Could not read page ${pageName}`);
-    }
-  }
-
-  async writePage(
-    pageName: string,
-    text: string,
-    lastModified?: number
-  ): Promise<PageMeta> {
-    let localPath = this.pageNameToPath(pageName);
-    try {
-      // Ensure parent folder exists
-      await mkdir(path.dirname(localPath), { recursive: true });
-
-      // Actually write the file
-      await writeFile(localPath, text);
-
-      if (lastModified) {
-        let d = new Date(lastModified);
-        console.log("Going to set the modified time", d);
-        await utimes(localPath, d, d);
-      }
-      // Fetch new metadata
-      const s = await stat(localPath);
-      return {
-        name: pageName,
-        lastModified: s.mtime.getTime(),
-      };
-    } catch (e) {
-      console.error("Error while writing page", pageName, e);
-      throw Error(`Could not write ${pageName}`);
-    }
-  }
-
-  async getPageMeta(pageName: string): Promise<PageMeta> {
-    let localPath = this.pageNameToPath(pageName);
-    try {
-      const s = await stat(localPath);
-      return {
-        name: pageName,
-        lastModified: s.mtime.getTime(),
-      };
-    } catch (e) {
-      console.error("Error while getting page meta", pageName, e);
-      throw Error(`Could not get meta for ${pageName}`);
-    }
-  }
-
-  async deletePage(pageName: string): Promise<void> {
-    let localPath = this.pageNameToPath(pageName);
-    await unlink(localPath);
-  }
-}

server/server.ts

@@ -6,7 +6,6 @@ import yargs from "yargs";
 import { hideBin } from "yargs/helpers";
 import { SilverBulletHooks } from "../common/manifest";
 import { ExpressServer } from "./api_server";
-import { DiskPlugLoader } from "../plugos/plug_loader";
 import { System } from "../plugos/system";
 
 let args = yargs(hideBin(process.argv))
@@ -36,12 +35,6 @@ const expressServer = new ExpressServer(app, pagesPath, distDir, system);
 expressServer
   .init()
   .then(async () => {
-    let plugLoader = new DiskPlugLoader(
-      system,
-      `${__dirname}/../../plugs/dist`
-    );
-    await plugLoader.loadPlugs();
-    plugLoader.watcher();
     server.listen(port, () => {
       console.log(`Server listening on port ${port}`);
     });

server/syscalls/space.ts

@@ -1,27 +1,27 @@
 import { PageMeta } from "../../common/types";
 import { SysCallMapping } from "../../plugos/system";
-import { Storage } from "../disk_storage";
+import { Space } from "../../common/spaces/space";
 
-export default (storage: Storage): SysCallMapping => {
+export default (space: Space): SysCallMapping => {
   return {
-    "space.listPages": (ctx): Promise<PageMeta[]> => {
-      return storage.listPages();
+    "space.listPages": async (ctx): Promise<PageMeta[]> => {
+      return [...space.listPages()];
     },
     "space.readPage": async (
       ctx,
       name: string
     ): Promise<{ text: string; meta: PageMeta }> => {
-      return storage.readPage(name);
+      return space.readPage(name);
     },
     "space.writePage": async (
       ctx,
       name: string,
       text: string
     ): Promise<PageMeta> => {
-      return storage.writePage(name, text);
+      return space.writePage(name, text);
     },
     "space.deletePage": async (ctx, name: string) => {
-      return storage.deletePage(name);
+      return space.deletePage(name);
     },
   };
 };