Migrate to Deno (#86)

Big bang migration to Deno 🤯

plugos/syscalls/asset.ts

@@ -0,0 +1,14 @@
+import { SysCallMapping, System } from "../system.ts";
+import type { AssetBundle, FileMeta } from "../asset_bundle_reader.ts";
+
+export default function assetSyscalls(system: System<any>): SysCallMapping {
+  return {
+    "asset.readAsset": (
+      ctx,
+      name: string,
+    ): { data: string; meta: FileMeta } => {
+      return (system.loadedPlugs.get(ctx.plug.name)!.manifest!
+        .assets as AssetBundle)[name];
+    },
+  };
+}

plugos/syscalls/esbuild.ts

@@ -0,0 +1,35 @@
+import { sandboxCompile, sandboxCompileModule } from "../compile.ts";
+import { SysCallMapping } from "../system.ts";
+
+// TODO: FIgure out a better way to do this
+const builtinModules = ["yaml", "handlebars"];
+
+export function esbuildSyscalls(): SysCallMapping {
+  return {
+    "esbuild.compile": async (
+      _ctx,
+      filename: string,
+      code: string,
+      functionName?: string,
+      excludeModules: string[] = [],
+    ): Promise<string> => {
+      return await sandboxCompile(
+        filename,
+        code,
+        functionName,
+        {
+          debug: true,
+          excludeModules: [...builtinModules, ...excludeModules],
+        },
+      );
+    },
+    "esbuild.compileModule": async (
+      _ctx,
+      moduleName: string,
+    ): Promise<string> => {
+      return await sandboxCompileModule(moduleName, {
+        excludeModules: builtinModules,
+      });
+    },
+  };
+}

plugos/syscalls/event.ts

@@ -0,0 +1,13 @@
+import { SysCallMapping } from "../system.ts";
+import { EventHook } from "../hooks/event.ts";
+
+export function eventSyscalls(eventHook: EventHook): SysCallMapping {
+  return {
+    "event.dispatch": (_ctx, eventName: string, data: any) => {
+      return eventHook.dispatchEvent(eventName, data);
+    },
+    "event.list": () => {
+      return eventHook.listEvents();
+    },
+  };
+}

plugos/syscalls/fs.deno.ts

@@ -0,0 +1,108 @@
+import type { SysCallMapping } from "../system.ts";
+import { mime, path } from "../../server/deps.ts";
+import { base64Decode, base64Encode } from "../../plugos/base64.ts";
+import type { FileMeta } from "../asset_bundle_reader.ts";
+
+export default function fileSystemSyscalls(root = "/"): SysCallMapping {
+  function resolvedPath(p: string): string {
+    p = path.resolve(root, p);
+    if (!p.startsWith(root)) {
+      throw Error("Path outside root, not allowed");
+    }
+    return p;
+  }
+
+  return {
+    "fs.readFile": async (
+      _ctx,
+      filePath: string,
+      encoding: "utf8" | "dataurl" = "utf8",
+    ): Promise<{ text: string; meta: FileMeta }> => {
+      const p = resolvedPath(filePath);
+      let text = "";
+      if (encoding === "utf8") {
+        text = await Deno.readTextFile(p);
+      } else {
+        text = `data:application/octet-stream,${
+          base64Encode(await Deno.readFile(p))
+        }`;
+      }
+      const s = await Deno.stat(p);
+      return {
+        text,
+        meta: {
+          name: filePath,
+          lastModified: s.mtime!.getTime(),
+          contentType: mime.getType(filePath) || "application/octet-stream",
+          size: s.size,
+          perm: "rw",
+        },
+      };
+    },
+    "fs.getFileMeta": async (_ctx, filePath: string): Promise<FileMeta> => {
+      const p = resolvedPath(filePath);
+      const s = await Deno.stat(p);
+      return {
+        name: filePath,
+        lastModified: s.mtime!.getTime(),
+        contentType: mime.getType(filePath) || "application/octet-stream",
+        size: s.size,
+        perm: "rw",
+      };
+    },
+    "fs.writeFile": async (
+      _ctx,
+      filePath: string,
+      text: string,
+      encoding: "utf8" | "dataurl" = "utf8",
+    ): Promise<FileMeta> => {
+      const p = resolvedPath(filePath);
+      await Deno.mkdir(path.dirname(p), { recursive: true });
+      if (encoding === "utf8") {
+        await Deno.writeTextFile(p, text);
+      } else {
+        await Deno.writeFile(p, base64Decode(text.split(",")[1]));
+      }
+      const s = await Deno.stat(p);
+      return {
+        name: filePath,
+        lastModified: s.mtime!.getTime(),
+        contentType: mime.getType(filePath) || "application/octet-stream",
+        size: s.size,
+        perm: "rw",
+      };
+    },
+    "fs.deleteFile": async (_ctx, filePath: string): Promise<void> => {
+      await Deno.remove(resolvedPath(filePath));
+    },
+    "fs.listFiles": async (
+      _ctx,
+      dirPath: string,
+      recursive: boolean,
+    ): Promise<FileMeta[]> => {
+      dirPath = resolvedPath(dirPath);
+      const allFiles: FileMeta[] = [];
+
+      async function walkPath(dir: string) {
+        const files = await Deno.readDir(dir);
+        for await (const file of files) {
+          const fullPath = path.join(dir, file.name);
+          const s = await Deno.stat(fullPath);
+          if (s.isDirectory && recursive) {
+            await walkPath(fullPath);
+          } else {
+            allFiles.push({
+              name: fullPath.substring(dirPath.length + 1),
+              lastModified: s.mtime!.getTime(),
+              contentType: mime.getType(fullPath) || "application/octet-stream",
+              size: s.size,
+              perm: "rw",
+            });
+          }
+        }
+      }
+      await walkPath(dirPath);
+      return allFiles;
+    },
+  };
+}

plugos/syscalls/fulltext.knex_sqlite.ts

@@ -0,0 +1,57 @@
+import { SQLite } from "../../server/deps.ts";
+import { SysCallMapping } from "../system.ts";
+import { asyncExecute, asyncQuery } from "./store.deno.ts";
+
+type Item = {
+  key: string;
+  value: string;
+};
+
+export function ensureFTSTable(
+  db: SQLite,
+  tableName: string,
+) {
+  const stmt = db.prepare(
+    `SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
+  );
+  const result = stmt.all(tableName);
+  if (result.length === 0) {
+    asyncExecute(
+      db,
+      `CREATE VIRTUAL TABLE ${tableName} USING fts5(key, value);`,
+    );
+
+    console.log(`Created fts5 table ${tableName}`);
+  }
+  return Promise.resolve();
+}
+
+export function fullTextSearchSyscalls(
+  db: SQLite,
+  tableName: string,
+): SysCallMapping {
+  return {
+    "fulltext.index": async (_ctx, key: string, value: string) => {
+      await asyncExecute(db, `DELETE FROM ${tableName} WHERE key = ?`, key);
+      await asyncExecute(
+        db,
+        `INSERT INTO ${tableName} (key, value) VALUES (?, ?)`,
+        key,
+        value,
+      );
+    },
+    "fulltext.delete": async (_ctx, key: string) => {
+      await asyncExecute(db, `DELETE FROM ${tableName} WHERE key = ?`, key);
+    },
+    "fulltext.search": async (_ctx, phrase: string, limit: number) => {
+      return (
+        await asyncQuery<any>(
+          db,
+          `SELECT key, rank FROM ${tableName} WHERE value MATCH ? ORDER BY key, rank LIMIT ?`,
+          phrase,
+          limit,
+        )
+      ).map((item) => ({ name: item.key, rank: item.rank }));
+    },
+  };
+}

plugos/syscalls/sandbox.ts

@@ -0,0 +1,15 @@
+import { LogEntry } from "../sandbox.ts";
+import { SysCallMapping, System } from "../system.ts";
+
+export default function sandboxSyscalls(system: System<any>): SysCallMapping {
+  return {
+    "sandbox.getLogs": (): LogEntry[] => {
+      let allLogs: LogEntry[] = [];
+      for (const plug of system.loadedPlugs.values()) {
+        allLogs = allLogs.concat(plug.sandbox.logBuffer);
+      }
+      allLogs = allLogs.sort((a, b) => a.date - b.date);
+      return allLogs;
+    },
+  };
+}

plugos/syscalls/shell.node.ts

@@ -0,0 +1,23 @@
+import type { SysCallMapping } from "../system.ts";
+
+export default function (cwd: string): SysCallMapping {
+  return {
+    "shell.run": async (
+      _ctx,
+      cmd: string,
+      args: string[],
+    ): Promise<{ stdout: string; stderr: string }> => {
+      const p = Deno.run({
+        cmd: [cmd, ...args],
+        cwd: cwd,
+        stdout: "piped",
+        stderr: "piped",
+      });
+      await p.status();
+      const stdout = new TextDecoder().decode(await p.output());
+      const stderr = new TextDecoder().decode(await p.stderrOutput());
+
+      return { stdout, stderr };
+    },
+  };
+}

plugos/syscalls/store.deno.test.ts

@@ -0,0 +1,106 @@
+import { assertEquals } from "../../test_deps.ts";
+import { SQLite } from "../../server/deps.ts";
+import { createSandbox } from "../environments/deno_sandbox.ts";
+import { System } from "../system.ts";
+import { ensureTable, storeSyscalls } from "./store.deno.ts";
+
+Deno.test("Test store", async () => {
+  const db = new SQLite(":memory:");
+  await ensureTable(db, "test_table");
+  const system = new System("server");
+  const syscalls = storeSyscalls(db, "test_table");
+  system.registerSyscalls([], syscalls);
+  const plug = await system.load(
+    {
+      name: "test",
+      functions: {
+        test1: {
+          code: `(() => {
+          return {
+            default: async () => {
+              await self.syscall("store.set", "name", "Pete");
+              return await self.syscall("store.get", "name");
+            }
+          };
+        })()`,
+        },
+      },
+    },
+    createSandbox,
+  );
+  assertEquals(await plug.invoke("test1", []), "Pete");
+  await system.unloadAll();
+
+  const dummyCtx: any = {};
+
+  await syscalls["store.deleteAll"](dummyCtx);
+  await syscalls["store.batchSet"](dummyCtx, [
+    {
+      key: "pete",
+      value: {
+        age: 20,
+        firstName: "Pete",
+        lastName: "Roberts",
+      },
+    },
+    {
+      key: "petejr",
+      value: {
+        age: 8,
+        firstName: "Pete Jr",
+        lastName: "Roberts",
+      },
+    },
+    {
+      key: "petesr",
+      value: {
+        age: 78,
+        firstName: "Pete Sr",
+        lastName: "Roberts",
+      },
+    },
+  ]);
+
+  let allRoberts = await syscalls["store.query"](dummyCtx, {
+    filter: [{ op: "=", prop: "lastName", value: "Roberts" }],
+    orderBy: "age",
+    orderDesc: true,
+  });
+
+  assertEquals(allRoberts.length, 3);
+  assertEquals(allRoberts[0].key, "petesr");
+
+  allRoberts = await syscalls["store.query"](dummyCtx, {
+    filter: [{ op: "=", prop: "lastName", value: "Roberts" }],
+    orderBy: "age",
+    limit: 1,
+  });
+
+  assertEquals(allRoberts.length, 1);
+  assertEquals(allRoberts[0].key, "petejr");
+
+  allRoberts = await syscalls["store.query"](dummyCtx, {
+    filter: [
+      { op: ">", prop: "age", value: 10 },
+      { op: "<", prop: "age", value: 30 },
+    ],
+    orderBy: "age",
+  });
+
+  assertEquals(allRoberts.length, 1);
+  assertEquals(allRoberts[0].key, "pete");
+
+  // Delete the middle one
+
+  await syscalls["store.deleteQuery"](dummyCtx, {
+    filter: [
+      { op: ">", prop: "age", value: 10 },
+      { op: "<", prop: "age", value: 30 },
+    ],
+  });
+
+  allRoberts = await syscalls["store.query"](dummyCtx, {});
+  assertEquals(allRoberts.length, 2);
+
+  db.close();
+});

plugos/syscalls/store.deno.ts

@@ -0,0 +1,177 @@
+import { SQLite } from "../../server/deps.ts";
+import { SysCallMapping } from "../system.ts";
+
+export type Item = {
+  page: string;
+  key: string;
+  value: any;
+};
+
+export type KV = {
+  key: string;
+  value: any;
+};
+
+export function ensureTable(db: SQLite, tableName: string) {
+  const stmt = db.prepare(
+    `SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
+  );
+  const result = stmt.all(tableName);
+  if (result.length === 0) {
+    db.exec(`CREATE TABLE ${tableName} (key STRING PRIMARY KEY, value TEXT);`);
+    console.log(`Created table ${tableName}`);
+  }
+  return Promise.resolve();
+}
+
+export type Query = {
+  filter?: Filter[];
+  orderBy?: string;
+  orderDesc?: boolean;
+  limit?: number;
+  select?: string[];
+};
+
+export type Filter = {
+  op: string;
+  prop: string;
+  value: any;
+};
+
+export function queryToSql(
+  query: Query,
+): { sql: string; params: any[] } {
+  const whereClauses: string[] = [];
+  const clauses: string[] = [];
+  const params: any[] = [];
+  if (query.filter) {
+    for (const filter of query.filter) {
+      whereClauses.push(
+        `json_extract(value, '$.${filter.prop}') ${filter.op} ?`,
+      );
+      params.push(filter.value);
+    }
+  }
+  if (query.orderBy) {
+    clauses.push(
+      `ORDER BY json_extract(value, '$.${query.orderBy}') ${
+        query.orderDesc ? "desc" : "asc"
+      }`,
+    );
+  }
+  if (query.limit) {
+    clauses.push(`LIMIT ${query.limit}`);
+  }
+  return {
+    sql: whereClauses.length > 0
+      ? `WHERE ${whereClauses.join(" AND ")} ${clauses.join(" ")}`
+      : clauses.join(" "),
+    params,
+  };
+}
+
+export function asyncQuery<T extends Record<string, unknown>>(
+  db: SQLite,
+  query: string,
+  ...params: any[]
+): Promise<T[]> {
+  // console.log("Querying", query, params);
+  return Promise.resolve(db.prepare(query).all<T>(params));
+}
+
+export function asyncExecute(
+  db: SQLite,
+  query: string,
+  ...params: any[]
+): Promise<number> {
+  // console.log("Exdecting", query, params);
+  return Promise.resolve(db.exec(query, params));
+}
+
+export function storeSyscalls(
+  db: SQLite,
+  tableName: string,
+): SysCallMapping {
+  const apiObj: SysCallMapping = {
+    "store.delete": async (_ctx, key: string) => {
+      await asyncExecute(db, `DELETE FROM ${tableName} WHERE key = ?`, key);
+    },
+    "store.deletePrefix": async (_ctx, prefix: string) => {
+      await asyncExecute(
+        db,
+        `DELETE FROM ${tableName} WHERE key LIKE "${prefix}%"`,
+      );
+    },
+    "store.deleteQuery": async (_ctx, query: Query) => {
+      const { sql, params } = queryToSql(query);
+      await asyncExecute(db, `DELETE FROM ${tableName} ${sql}`, ...params);
+    },
+    "store.deleteAll": async () => {
+      await asyncExecute(db, `DELETE FROM ${tableName}`);
+    },
+    "store.set": async (_ctx, key: string, value: any) => {
+      await asyncExecute(
+        db,
+        `UPDATE ${tableName} SET value = ? WHERE key = ?`,
+        JSON.stringify(value),
+        key,
+      );
+      if (db.changes === 0) {
+        await asyncExecute(
+          db,
+          `INSERT INTO ${tableName} (key, value) VALUES (?, ?)`,
+          key,
+          JSON.stringify(value),
+        );
+      }
+    },
+    // TODO: Optimize
+    "store.batchSet": async (ctx, kvs: KV[]) => {
+      for (const { key, value } of kvs) {
+        await apiObj["store.set"](ctx, key, value);
+      }
+    },
+    "store.batchDelete": async (ctx, keys: string[]) => {
+      for (const key of keys) {
+        await apiObj["store.delete"](ctx, key);
+      }
+    },
+    "store.get": async (_ctx, key: string): Promise<any | null> => {
+      const result = await asyncQuery<Item>(
+        db,
+        `SELECT value FROM ${tableName} WHERE key = ?`,
+        key,
+      );
+      if (result.length) {
+        return JSON.parse(result[0].value);
+      } else {
+        return null;
+      }
+    },
+    "store.queryPrefix": async (_ctx, prefix: string) => {
+      return (
+        await asyncQuery<Item>(
+          db,
+          `SELECT key, value FROM ${tableName} WHERE key LIKE "${prefix}%"`,
+        )
+      ).map(({ key, value }) => ({
+        key,
+        value: JSON.parse(value),
+      }));
+    },
+    "store.query": async (_ctx, query: Query) => {
+      const { sql, params } = queryToSql(query);
+      return (
+        await asyncQuery<Item>(
+          db,
+          `SELECT key, value FROM ${tableName} ${sql}`,
+          ...params,
+        )
+      ).map(({ key, value }: { key: string; value: string }) => ({
+        key,
+        value: JSON.parse(value),
+      }));
+    },
+  };
+  return apiObj;
+}

plugos/syscalls/store.dexie_browser.ts

@@ -0,0 +1,66 @@
+import Dexie from "https://esm.sh/dexie@3.2.2";
+import { SysCallMapping } from "../system.ts";
+
+export type KV = {
+  key: string;
+  value: any;
+};
+
+export function storeSyscalls(
+  dbName: string,
+  tableName: string,
+): SysCallMapping {
+  const db = new Dexie(dbName);
+  db.version(1).stores({
+    [tableName]: "key",
+  });
+  const items = db.table(tableName);
+
+  return {
+    "store.delete": async (_ctx, key: string) => {
+      await items.delete(key);
+    },
+
+    "store.deletePrefix": async (_ctx, prefix: string) => {
+      await items.where("key").startsWith(prefix).delete();
+    },
+
+    "store.deleteAll": async () => {
+      await items.clear();
+    },
+
+    "store.set": async (_ctx, key: string, value: any) => {
+      await items.put({
+        key,
+        value,
+      });
+    },
+
+    "store.batchSet": async (_ctx, kvs: KV[]) => {
+      await items.bulkPut(
+        kvs.map(({ key, value }) => ({
+          key,
+          value,
+        })),
+      );
+    },
+
+    "store.get": async (_ctx, key: string): Promise<any | null> => {
+      const result = await items.get({
+        key,
+      });
+      return result ? result.value : null;
+    },
+
+    "store.queryPrefix": async (
+      _ctx,
+      keyPrefix: string,
+    ): Promise<{ key: string; value: any }[]> => {
+      const results = await items.where("key").startsWith(keyPrefix).toArray();
+      return results.map((result) => ({
+        key: result.key,
+        value: result.value,
+      }));
+    },
+  };
+}

plugos/syscalls/transport.ts

@@ -0,0 +1,20 @@
+import { SyscallContext, SysCallMapping } from "../system.ts";
+
+export function proxySyscalls(
+  names: string[],
+  transportCall: (
+    ctx: SyscallContext,
+    name: string,
+    ...args: any[]
+  ) => Promise<any>,
+): SysCallMapping {
+  const syscalls: SysCallMapping = {};
+
+  for (const name of names) {
+    syscalls[name] = (ctx, ...args: any[]) => {
+      return transportCall(ctx, name, ...args);
+    };
+  }
+
+  return syscalls;
+}