Refactor and renaming

packages/server/api_server.ts

@@ -0,0 +1,296 @@
+import express, { Express } from "express";
+import { SilverBulletHooks } from "@silverbulletmd/common/manifest";
+import { EndpointHook } from "@plugos/plugos/hooks/endpoint";
+import { readFile } from "fs/promises";
+import { System } from "@plugos/plugos/system";
+import cors from "cors";
+import { DiskSpacePrimitives } from "@silverbulletmd/common/spaces/disk_space_primitives";
+import path from "path";
+import bodyParser from "body-parser";
+import { EventHook } from "@plugos/plugos/hooks/event";
+import spaceSyscalls from "./syscalls/space";
+import { eventSyscalls } from "@plugos/plugos/syscalls/event";
+import { ensurePageIndexTable, pageIndexSyscalls } from "./syscalls";
+import knex, { Knex } from "knex";
+import shellSyscalls from "@plugos/plugos/syscalls/shell.node";
+import { NodeCronHook } from "@plugos/plugos/hooks/node_cron";
+import { markdownSyscalls } from "@silverbulletmd/common/syscalls/markdown";
+import { EventedSpacePrimitives } from "@silverbulletmd/common/spaces/evented_space_primitives";
+import { Space } from "@silverbulletmd/common/spaces/space";
+import { safeRun, throttle } from "@silverbulletmd/common/util";
+import { createSandbox } from "@plugos/plugos/environments/node_sandbox";
+import { jwtSyscalls } from "@plugos/plugos/syscalls/jwt";
+import buildMarkdown from "@silverbulletmd/web/parser";
+import { loadMarkdownExtensions } from "@silverbulletmd/web/markdown_ext";
+import http, { Server } from "http";
+
+export class ExpressServer {
+  app: Express;
+  system: System<SilverBulletHooks>;
+  private rootPath: string;
+  private space: Space;
+  private distDir: string;
+  private eventHook: EventHook;
+  private db: Knex<any, unknown[]>;
+  private port: number;
+  private server?: Server;
+
+  constructor(port: number, rootPath: string, distDir: string) {
+    this.port = port;
+    this.app = express();
+    this.rootPath = rootPath;
+    this.distDir = distDir;
+    this.system = new System<SilverBulletHooks>("server");
+
+    // Setup system
+    this.eventHook = new EventHook();
+    this.system.addHook(this.eventHook);
+    this.space = new Space(
+      new EventedSpacePrimitives(
+        new DiskSpacePrimitives(rootPath),
+        this.eventHook
+      ),
+      true
+    );
+    this.db = knex({
+      client: "better-sqlite3",
+      connection: {
+        filename: path.join(rootPath, "data.db"),
+      },
+      useNullAsDefault: true,
+    });
+
+    this.system.registerSyscalls(["shell"], shellSyscalls(rootPath));
+    this.system.addHook(new NodeCronHook());
+
+    this.system.registerSyscalls([], pageIndexSyscalls(this.db));
+    this.system.registerSyscalls([], spaceSyscalls(this.space));
+    this.system.registerSyscalls([], eventSyscalls(this.eventHook));
+    this.system.registerSyscalls([], markdownSyscalls(buildMarkdown([])));
+    this.system.registerSyscalls([], jwtSyscalls());
+    this.system.addHook(new EndpointHook(this.app, "/_/"));
+
+    let throttledRebuildMdExtensions = throttle(() => {
+      this.rebuildMdExtensions();
+    }, 100);
+
+    this.space.on({
+      plugLoaded: (plugName, plug) => {
+        safeRun(async () => {
+          console.log("Plug load", plugName);
+          await this.system.load(plugName, plug, createSandbox);
+        });
+        throttledRebuildMdExtensions();
+      },
+      plugUnloaded: (plugName) => {
+        safeRun(async () => {
+          console.log("Plug unload", plugName);
+          await this.system.unload(plugName);
+        });
+        throttledRebuildMdExtensions();
+      },
+    });
+
+    setInterval(() => {
+      this.space.updatePageListAsync();
+    }, 5000);
+    this.space.updatePageListAsync();
+  }
+
+  rebuildMdExtensions() {
+    this.system.registerSyscalls(
+      [],
+      markdownSyscalls(buildMarkdown(loadMarkdownExtensions(this.system)))
+    );
+  }
+
+  async start() {
+    await ensurePageIndexTable(this.db);
+    console.log("Setting up router");
+
+    this.app.use("/", express.static(this.distDir));
+
+    let fsRouter = express.Router();
+
+    // Page list
+    fsRouter.route("/").get(async (req, res) => {
+      let { nowTimestamp, pages } = await this.space.fetchPageList();
+      res.header("Now-Timestamp", "" + nowTimestamp);
+      res.json([...pages]);
+    });
+
+    fsRouter.route("/").post(bodyParser.json(), async (req, res) => {});
+
+    fsRouter
+      .route(/\/(.+)/)
+      .get(async (req, res) => {
+        let pageName = req.params[0];
+        // console.log("Getting", pageName);
+        try {
+          let pageData = await this.space.readPage(pageName);
+          res.status(200);
+          res.header("Last-Modified", "" + pageData.meta.lastModified);
+          res.header("Content-Type", "text/markdown");
+          res.send(pageData.text);
+        } catch (e) {
+          // CORS
+          res.status(200);
+          res.header("X-Status", "404");
+          res.send("");
+        }
+      })
+      .put(bodyParser.text({ type: "*/*" }), async (req, res) => {
+        let pageName = req.params[0];
+        console.log("Saving", pageName);
+
+        try {
+          let meta = await this.space.writePage(
+            pageName,
+            req.body,
+            false,
+            req.header("Last-Modified")
+              ? +req.header("Last-Modified")!
+              : undefined
+          );
+          res.status(200);
+          res.header("Last-Modified", "" + meta.lastModified);
+          res.send("OK");
+        } catch (err) {
+          res.status(500);
+          res.send("Write failed");
+          console.error("Pipeline failed", err);
+        }
+      })
+      .options(async (req, res) => {
+        let pageName = req.params[0];
+        try {
+          const meta = await this.space.getPageMeta(pageName);
+          res.status(200);
+          res.header("Last-Modified", "" + meta.lastModified);
+          res.header("Content-Type", "text/markdown");
+          res.send("");
+        } catch (e) {
+          // CORS
+          res.status(200);
+          res.header("X-Status", "404");
+          res.send("Not found");
+        }
+      })
+      .delete(async (req, res) => {
+        let pageName = req.params[0];
+        try {
+          await this.space.deletePage(pageName);
+          res.status(200);
+          res.send("OK");
+        } catch (e) {
+          console.error("Error deleting file", e);
+          res.status(500);
+          res.send("OK");
+        }
+      });
+
+    this.app.use(
+      "/fs",
+      cors({
+        methods: "GET,HEAD,PUT,OPTIONS,POST,DELETE",
+        preflightContinue: true,
+      }),
+      fsRouter
+    );
+
+    let plugRouter = express.Router();
+
+    plugRouter.post(
+      "/:plug/syscall/:name",
+      bodyParser.json(),
+      async (req, res) => {
+        const name = req.params.name;
+        const plugName = req.params.plug;
+        const args = req.body as any;
+        const plug = this.system.loadedPlugs.get(plugName);
+        if (!plug) {
+          res.status(404);
+          return res.send(`Plug ${plugName} not found`);
+        }
+        try {
+          const result = await this.system.syscallWithContext(
+            { plug },
+            name,
+            args
+          );
+          res.status(200);
+          res.send(result);
+        } catch (e: any) {
+          res.status(500);
+          return res.send(e.message);
+        }
+      }
+    );
+    plugRouter.post(
+      "/:plug/function/:name",
+      bodyParser.json(),
+      async (req, res) => {
+        const name = req.params.name;
+        const plugName = req.params.plug;
+        const args = req.body as any[];
+        const plug = this.system.loadedPlugs.get(plugName);
+        if (!plug) {
+          res.status(404);
+          return res.send(`Plug ${plugName} not found`);
+        }
+        try {
+          console.log("Invoking", name);
+          const result = await plug.invoke(name, args);
+          res.status(200);
+          res.send(result);
+        } catch (e: any) {
+          res.status(500);
+          console.log("Error invoking function", e);
+          return res.send(e.message);
+        }
+      }
+    );
+
+    this.app.use(
+      "/plug",
+      cors({
+        methods: "GET,HEAD,PUT,OPTIONS,POST,DELETE",
+        preflightContinue: true,
+      }),
+      plugRouter
+    );
+
+    // Fallback, serve index.html
+    let cachedIndex: string | undefined = undefined;
+    this.app.get("/*", async (req, res) => {
+      if (!cachedIndex) {
+        cachedIndex = await readFile(`${this.distDir}/index.html`, "utf8");
+      }
+      res.status(200).header("Content-Type", "text/html").send(cachedIndex);
+    });
+
+    this.server = http.createServer(this.app);
+    this.server.listen(this.port, () => {
+      console.log(`Server listening on port ${this.port}`);
+    });
+  }
+
+  async stop() {
+    if (this.server) {
+      console.log("Stopping");
+      await this.system.unloadAll();
+      console.log("Stopped plugs");
+      return new Promise<void>((resolve, reject) => {
+        this.server!.close((err) => {
+          this.server = undefined;
+          console.log("stopped server");
+          if (err) {
+            reject(err);
+          } else {
+            resolve();
+          }
+        });
+      });
+    }
+  }
+}

packages/server/package.json

@@ -0,0 +1,72 @@
+{
+  "name": "@silverbulletmd/server",
+  "version": "0.0.1",
+  "license": "MIT",
+  "bin": {
+    "silverbullet": "./dist/server.js"
+  },
+  "scripts": {
+    "start": "nodemon -w dist dist/server.js ../../pages"
+  },
+  "targets": {
+    "server": {
+      "source": [
+        "server.ts"
+      ],
+      "outputFormat": "commonjs",
+      "isLibrary": true,
+      "context": "node",
+      "includeNodeModules": [
+        "@plugos/plugos",
+        "@silverbulletmd/common",
+        "@silverbulletmd/web"
+      ]
+    }
+  },
+  "dependencies": {
+    "@codemirror/lang-javascript": "^0.19.7",
+    "@codemirror/lang-markdown": "^0.19.6",
+    "@codemirror/language": "^0.19.0",
+    "@codemirror/legacy-modes": "^0.19.1",
+    "@codemirror/stream-parser": "^0.19.9",
+    "@jest/globals": "^27.5.1",
+    "@lezer/markdown": "^0.15.0",
+    "@silverbulletmd/web": "^0.0.1",
+    "better-sqlite3": "^7.5.0",
+    "body-parser": "^1.19.2",
+    "buffer": "^6.0.3",
+    "cors": "^2.8.5",
+    "dexie": "^3.2.1",
+    "events": "^3.3.0",
+    "express": "^4.17.3",
+    "fake-indexeddb": "^3.1.7",
+    "jest": "^27.5.1",
+    "jsonwebtoken": "^8.5.1",
+    "knex": "^1.0.4",
+    "node-cron": "^3.0.0",
+    "node-fetch": "2",
+    "node-watch": "^0.7.3",
+    "nodemon": "^2.0.15",
+    "vm2": "^3.9.9",
+    "ws": "^8.5.0",
+    "yaml": "^1.10.2",
+    "yargs": "^17.3.1"
+  },
+  "devDependencies": {
+    "@parcel/packager-raw-url": "2.3.2",
+    "@parcel/service-worker": "2.3.2",
+    "@parcel/transformer-inline-string": "2.3.2",
+    "@parcel/transformer-sass": "2.3.2",
+    "@parcel/transformer-webmanifest": "2.3.2",
+    "@parcel/validator-typescript": "2.3.2",
+    "@types/cors": "^2.8.12",
+    "@types/events": "^3.0.0",
+    "@types/express": "^4.17.13",
+    "@types/jest": "^27.4.1",
+    "@types/node": "^17.0.21",
+    "@vscode/sqlite3": "^5.0.7",
+    "assert": "^2.0.0",
+    "parcel": "2.3.2",
+    "typescript": "^4.6.2"
+  }
+}

packages/server/server.ts

@@ -0,0 +1,30 @@
+#!/usr/bin/env node
+
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import { ExpressServer } from "./api_server";
+import { nodeModulesDir } from "@plugos/plugos/environments/node_sandbox";
+
+let args = yargs(hideBin(process.argv))
+  .option("port", {
+    type: "number",
+    default: 3000,
+  })
+  .parse();
+
+if (!args._.length) {
+  console.error("Usage: silverbullet <path-to-pages>");
+  process.exit(1);
+}
+
+const pagesPath = args._[0] as string;
+
+const port = args.port;
+const webappDistDir = `${nodeModulesDir}/node_modules/@silverbulletmd/web/dist`;
+
+// console.log("This is where the static files live", webappDistDir);
+
+const expressServer = new ExpressServer(port, pagesPath, webappDistDir);
+expressServer.start().catch((e) => {
+  console.error(e);
+});

packages/server/syscalls/index.ts

@@ -0,0 +1,116 @@
+import { Knex } from "knex";
+import { SysCallMapping } from "@plugos/plugos/system";
+
+import {
+  ensureTable,
+  storeSyscalls,
+} from "@plugos/plugos/syscalls/store.knex_node";
+
+type IndexItem = {
+  page: string;
+  key: string;
+  value: any;
+};
+
+export type KV = {
+  key: string;
+  value: any;
+};
+
+/*
+ Keyspace design:
+
+ for page lookups:
+ p~page~key
+
+ for global lookups:
+ k~key~page
+
+ */
+
+function pageKey(page: string, key: string) {
+  return `p~${page}~${key}`;
+}
+
+function unpackPageKey(dbKey: string): { page: string; key: string } {
+  const [, page, key] = dbKey.split("~");
+  return { page, key };
+}
+
+function globalKey(page: string, key: string) {
+  return `k~${key}~${page}`;
+}
+
+function unpackGlobalKey(dbKey: string): { page: string; key: string } {
+  const [, key, page] = dbKey.split("~");
+  return { page, key };
+}
+
+export async function ensurePageIndexTable(db: Knex<any, unknown>) {
+  await ensureTable(db, "page_index");
+}
+
+export function pageIndexSyscalls(db: Knex<any, unknown>): SysCallMapping {
+  const storeCalls = storeSyscalls(db, "page_index");
+  const apiObj: SysCallMapping = {
+    "index.set": async (ctx, page: string, key: string, value: any) => {
+      await storeCalls["store.set"](ctx, pageKey(page, key), value);
+      await storeCalls["store.set"](ctx, globalKey(page, key), value);
+    },
+    "index.batchSet": async (ctx, page: string, kvs: KV[]) => {
+      for (let { key, value } of kvs) {
+        await apiObj["index.set"](ctx, page, key, value);
+      }
+    },
+    "index.delete": async (ctx, page: string, key: string) => {
+      await storeCalls["store.delete"](ctx, pageKey(page, key));
+      await storeCalls["store.delete"](ctx, globalKey(page, key));
+    },
+    "index.get": async (ctx, page: string, key: string) => {
+      return storeCalls["store.get"](ctx, pageKey(page, key));
+    },
+    "index.scanPrefixForPage": async (ctx, page: string, prefix: string) => {
+      return (
+        await storeCalls["store.queryPrefix"](ctx, pageKey(page, prefix))
+      ).map(({ key, value }: { key: string; value: any }) => {
+        const { key: pageKey } = unpackPageKey(key);
+        return {
+          page,
+          key: pageKey,
+          value,
+        };
+      });
+    },
+    "index.scanPrefixGlobal": async (ctx, prefix: string) => {
+      return (await storeCalls["store.queryPrefix"](ctx, `k~${prefix}`)).map(
+        ({ key, value }: { key: string; value: any }) => {
+          const { page, key: pageKey } = unpackGlobalKey(key);
+          return {
+            page,
+            key: pageKey,
+            value,
+          };
+        }
+      );
+    },
+    "index.clearPageIndexForPage": async (ctx, page: string) => {
+      await apiObj["index.deletePrefixForPage"](ctx, page, "");
+    },
+    "index.deletePrefixForPage": async (ctx, page: string, prefix: string) => {
+      // Collect all global keys for this page to delete
+      let keysToDelete = (
+        await storeCalls["store.queryPrefix"](ctx, pageKey(page, prefix))
+      ).map(({ key }: { key: string; value: string }) =>
+        globalKey(page, unpackPageKey(key).key)
+      );
+      // Delete all page keys
+      await storeCalls["store.deletePrefix"](ctx, pageKey(page, prefix));
+      // console.log("Deleting keys", keysToDelete);
+      await storeCalls["store.batchDelete"](ctx, keysToDelete);
+    },
+    "index.clearPageIndex": async (ctx) => {
+      await storeCalls["store.deleteAll"](ctx);
+    },
+  };
+  return apiObj;
+}

packages/server/syscalls/space.ts

@@ -0,0 +1,27 @@
+import { PageMeta } from "@silverbulletmd/common/types";
+import { SysCallMapping } from "@plugos/plugos/system";
+import { Space } from "@silverbulletmd/common/spaces/space";
+
+export default (space: Space): SysCallMapping => {
+  return {
+    "space.listPages": async (ctx): Promise<PageMeta[]> => {
+      return [...space.listPages()];
+    },
+    "space.readPage": async (
+      ctx,
+      name: string
+    ): Promise<{ text: string; meta: PageMeta }> => {
+      return space.readPage(name);
+    },
+    "space.writePage": async (
+      ctx,
+      name: string,
+      text: string
+    ): Promise<PageMeta> => {
+      return space.writePage(name, text);
+    },
+    "space.deletePage": async (ctx, name: string) => {
+      return space.deletePage(name);
+    },
+  };
+};

packages/server/util.ts

@@ -0,0 +1,5 @@
+export function safeRun(fn: () => Promise<void>) {
+  fn().catch((e) => {
+    console.error(e);
+  });
+}