Refactor of asset bundles

plugos/asset_bundle.ts

@@ -1,51 +0,0 @@
-import { mime } from "../server/deps.ts";
-import { AssetBundle } from "./asset_bundle_reader.ts";
-import { base64Encode } from "./base64.ts";
-import { globToRegExp, path, walk } from "./deps.ts";
-
-export async function bundleAssets(
-  rootPath: string,
-  patterns: string[],
-): Promise<AssetBundle> {
-  const bundle: AssetBundle = {};
-  for await (
-    const file of walk(rootPath, {
-      match: patterns.map((pat) => globToRegExp(pat)),
-    })
-  ) {
-    await loadIntoBundle(file.path, "", bundle);
-  }
-  return bundle;
-}
-
-export async function bundleFolder(rootPath: string, bundlePath: string) {
-  const bundle: AssetBundle = {};
-  await Deno.mkdir(path.dirname(bundlePath), { recursive: true });
-  for await (
-    const { path: filePath } of walk(rootPath, { includeDirs: false })
-  ) {
-    console.log("Bundling", filePath);
-    await loadIntoBundle(filePath, `${rootPath}/`, bundle);
-  }
-  await Deno.writeTextFile(bundlePath, JSON.stringify(bundle, null, 2));
-}
-
-async function loadIntoBundle(
-  filePath: string,
-  rootPath: string,
-  bundle: AssetBundle,
-) {
-  const b64content = base64Encode(await Deno.readFile(filePath));
-  const s = await Deno.stat(filePath);
-  const cleanPath = filePath.substring(rootPath.length);
-  bundle[cleanPath] = {
-    meta: {
-      name: cleanPath,
-      contentType: mime.getType(cleanPath) || "application/octet-stream",
-      size: s.size,
-      lastModified: s.mtime!.getTime(),
-      perm: "ro",
-    },
-    data: b64content,
-  };
-}

plugos/asset_bundle/base64.test.ts

@@ -1,4 +1,4 @@
-import { assertEquals } from "../test_deps.ts";
+import { assertEquals } from "../../test_deps.ts";
 import { base64Decode } from "./base64.ts";
 import { base64Encode } from "./base64.ts";
 

plugos/asset_bundle/builder.ts

@@ -0,0 +1,34 @@
+import { globToRegExp, path, walk } from "../deps.ts";
+import { AssetBundle } from "./bundle.ts";
+
+export async function bundleAssets(
+  rootPath: string,
+  patterns: string[],
+): Promise<AssetBundle> {
+  const bundle = new AssetBundle();
+  for await (
+    const file of walk(rootPath, {
+      match: patterns.map((pat) => globToRegExp(pat)),
+    })
+  ) {
+    const cleanPath = file.path.substring("".length);
+    await bundle.writeFileSync(cleanPath, await Deno.readFile(file.path));
+  }
+  return bundle;
+}
+
+export async function bundleFolder(rootPath: string, bundlePath: string) {
+  const bundle = new AssetBundle();
+  await Deno.mkdir(path.dirname(bundlePath), { recursive: true });
+  for await (
+    const { path: filePath } of walk(rootPath, { includeDirs: false })
+  ) {
+    console.log("Bundling", filePath);
+    const cleanPath = filePath.substring(`${rootPath}/`.length);
+    await bundle.writeFileSync(cleanPath, await Deno.readFile(filePath));
+  }
+  await Deno.writeTextFile(
+    bundlePath,
+    JSON.stringify(bundle.toJSON(), null, 2),
+  );
+}

plugos/asset_bundle/bundle.test.ts

@@ -0,0 +1,16 @@
+import { AssetBundle } from "./bundle.ts";
+import { assertEquals } from "../../test_deps.ts";
+
+Deno.test("Asset bundle", () => {
+  const assetBundle = new AssetBundle();
+  assetBundle.writeTextFileSync("test.txt", "Sup yo");
+  assertEquals("text/plain", assetBundle.getMimeType("test.txt"));
+  assertEquals("Sup yo", assetBundle.readTextFileSync("test.txt"));
+  const buf = new Uint8Array(3);
+  buf[0] = 1;
+  buf[1] = 2;
+  buf[2] = 3;
+  assetBundle.writeFileSync("test.bin", buf);
+  assertEquals("application/octet-stream", assetBundle.getMimeType("test.bin"));
+  assertEquals(buf, assetBundle.readFileSync("test.bin"));
+});

plugos/asset_bundle/bundle.ts

@@ -0,0 +1,69 @@
+import { base64Decode, base64Encode } from "./base64.ts";
+import { mime } from "../deps.ts";
+
+export type AssetJson = Record<string, string>;
+
+export class AssetBundle {
+  readonly bundle: AssetJson;
+
+  constructor(bundle: AssetJson = {}) {
+    this.bundle = bundle;
+  }
+
+  has(path: string): boolean {
+    return path in this.bundle;
+  }
+
+  listFiles(): string[] {
+    return Object.keys(this.bundle);
+  }
+
+  readFileSync(
+    path: string,
+  ): Uint8Array {
+    const content = this.bundle[path];
+    if (!content) {
+      throw new Error(`No such file ${path}`);
+    }
+    const data = content.split(",", 2)[1];
+    return base64Decode(data);
+  }
+
+  readFileAsDataUrl(path: string): string {
+    const content = this.bundle[path];
+    if (!content) {
+      throw new Error(`No such file ${path}`);
+    }
+    return content;
+  }
+
+  readTextFileSync(
+    path: string,
+  ): string {
+    return new TextDecoder().decode(this.readFileSync(path));
+  }
+
+  getMimeType(
+    path: string,
+  ): string {
+    const content = this.bundle[path];
+    if (!content) {
+      throw new Error(`No such file ${path}`);
+    }
+    return content.split(";")[0].split(":")[1];
+  }
+
+  writeFileSync(path: string, data: Uint8Array) {
+    const encoded = base64Encode(data);
+    const mimeType = mime.getType(path);
+    this.bundle[path] = `data:${mimeType};base64,${encoded}`;
+  }
+
+  writeTextFileSync(path: string, s: string) {
+    this.writeFileSync(path, new TextEncoder().encode(s));
+  }
+
+  toJSON(): AssetJson {
+    return this.bundle;
+  }
+}

plugos/asset_bundle_reader.ts

@@ -1,40 +0,0 @@
-import { base64Decode } from "./base64.ts";
-
-export type FileMeta = {
-  name: string;
-  lastModified: number;
-  contentType: string;
-  size: number;
-  perm: "ro" | "rw";
-};
-
-export type AssetBundle = Record<string, { meta: FileMeta; data: string }>;
-
-export function assetReadFileSync(
-  bundle: AssetBundle,
-  path: string,
-): ArrayBuffer {
-  const content = bundle[path];
-  if (!content) {
-    throw new Error(`No such file ${path}`);
-  }
-  return base64Decode(content.data);
-}
-
-export function assetStatSync(
-  bundle: AssetBundle,
-  path: string,
-): FileMeta {
-  const content = bundle[path];
-  if (!content) {
-    throw new Error(`No such file ${path}`);
-  }
-  return content.meta;
-}
-
-export function assetReadTextFileSync(
-  bundle: AssetBundle,
-  path: string,
-): string {
-  return new TextDecoder().decode(assetReadFileSync(bundle, path));
-}

plugos/bin/plugos-bundle.ts

@@ -11,7 +11,7 @@ import {
 import { path } from "../../server/deps.ts";
 
 import * as flags from "https://deno.land/std@0.158.0/flags/mod.ts";
-import { bundleAssets } from "../../plugos/asset_bundle.ts";
+import { bundleAssets } from "../asset_bundle/builder.ts";
 
 export async function bundle(
   manifestPath: string,
@@ -41,7 +41,7 @@ export async function bundle(
     rootPath,
     manifest.assets as string[] || [],
   );
-  manifest.assets = assetsBundle;
+  manifest.assets = assetsBundle.toJSON();
 
   // Functions
 

plugos/compile.ts

@@ -6,9 +6,9 @@ export const esbuild: typeof esbuildWasm = Deno.run === undefined
   ? esbuildWasm
   : esbuildNative;
 
-import { path } from "../server/deps.ts";
-import { denoPlugin } from "../esbuild_deno_loader/mod.ts";
-import { patchDenoLibJS } from "../common/hack.ts";
+import { path } from "./deps.ts";
+import { denoPlugin } from "./forked/esbuild_deno_loader/mod.ts";
+import { patchDenoLibJS } from "./hack.ts";
 
 export type CompileOptions = {
   debug?: boolean;
@@ -54,20 +54,11 @@ export async function compile(
       treeShaking: true,
       plugins: [
         denoPlugin({
+          // TODO do this differently
           importMapURL: options.importMap ||
             new URL("./../import_map.json", import.meta.url),
         }),
       ],
-      loader: {
-        ".css": "text",
-        ".md": "text",
-        ".txt": "text",
-        ".html": "text",
-        ".hbs": "text",
-        ".png": "dataurl",
-        ".gif": "dataurl",
-        ".jpg": "dataurl",
-      },
       absWorkingDir: path.resolve(path.dirname(inFile)),
     });
 

plugos/deps.ts

@@ -1,3 +1,4 @@
 export { globToRegExp } from "https://deno.land/std@0.158.0/path/glob.ts";
 export { walk } from "https://deno.land/std@0.159.0/fs/mod.ts";
 export * as path from "https://deno.land/std@0.158.0/path/mod.ts";
+export { mime } from "https://deno.land/x/mimetypes@v1.0.0/mod.ts";

plugos/environments/custom_logger.ts

@@ -6,7 +6,7 @@ export class ConsoleLogger {
 
   constructor(
     callback: (level: LogLevel, entry: string) => void,
-    print: boolean = true
+    print: boolean = true,
   ) {
     this.print = print;
     this.callback = callback;

plugos/environments/deno_sandbox.ts

@@ -3,7 +3,7 @@ import { safeRun } from "../util.ts";
 import { Sandbox } from "../sandbox.ts";
 import { WorkerLike } from "./worker.ts";
 import { Plug } from "../plug.ts";
-import { AssetBundle, assetReadTextFileSync } from "../asset_bundle_reader.ts";
+import { AssetBundle } from "../asset_bundle/bundle.ts";
 
 class DenoWorkerWrapper implements WorkerLike {
   private worker: Worker;
@@ -13,7 +13,7 @@ class DenoWorkerWrapper implements WorkerLike {
   constructor(worker: Worker) {
     this.worker = worker;
     this.worker.addEventListener("message", (evt: any) => {
-      let data = evt.data;
+      const data = evt.data;
       if (!data) return;
       safeRun(async () => {
         await this.onMessage!(data);
@@ -30,23 +30,23 @@ class DenoWorkerWrapper implements WorkerLike {
   }
 }
 
-export function sandboxFactory(
-  assetBundle: AssetBundle,
-): (plug: Plug<any>) => Sandbox {
-  return (plug: Plug<any>) => {
-    const workerHref = URL.createObjectURL(
-      new Blob([
-        assetReadTextFileSync(assetBundle, "web/worker.js"),
-      ], {
-        type: "application/javascript",
-      }),
-    );
-    let worker = new Worker(
-      workerHref,
-      {
-        type: "module",
-      },
-    );
-    return new Sandbox(plug, new DenoWorkerWrapper(worker));
-  };
+import workerBundleJson from "./worker_bundle.json" assert { type: "json" };
+
+const workerBundle = new AssetBundle(workerBundleJson);
+
+export function createSandbox(plug: Plug<any>) {
+  const workerHref = URL.createObjectURL(
+    new Blob([
+      workerBundle.readFileSync("worker.js"),
+    ], {
+      type: "application/javascript",
+    }),
+  );
+  const worker = new Worker(
+    workerHref,
+    {
+      type: "module",
+    },
+  );
+  return new Sandbox(plug, new DenoWorkerWrapper(worker));
 }

plugos/environments/worker_bundle.json

@@ -0,0 +1,3 @@
+{
+  "worker.js": "data:application/javascript;base64,KCgpID0+IHsgdmFyIG1vZD0oKCk9PntmdW5jdGlvbiBjKHIpe3IoKS5jYXRjaChlPT57Y29uc29sZS5lcnJvcigiQ2F1Z2h0IGVycm9yIixlLm1lc3NhZ2UpfSl9dmFyIGE9Y2xhc3N7Y29uc3RydWN0b3IoZSxuPSEwKXt0aGlzLnByaW50PW4sdGhpcy5jYWxsYmFjaz1lfWxvZyguLi5lKXt0aGlzLnB1c2goImxvZyIsZSl9d2FybiguLi5lKXt0aGlzLnB1c2goIndhcm4iLGUpfWVycm9yKC4uLmUpe3RoaXMucHVzaCgiZXJyb3IiLGUpfWluZm8oLi4uZSl7dGhpcy5wdXNoKCJpbmZvIixlKX1wdXNoKGUsbil7dGhpcy5jYWxsYmFjayhlLHRoaXMubG9nTWVzc2FnZShuKSksdGhpcy5wcmludCYmY29uc29sZVtlXSguLi5uKX1sb2dNZXNzYWdlKGUpe2xldCBuPVtdO2ZvcihsZXQgdCBvZiBlKXN3aXRjaCh0eXBlb2YgdCl7Y2FzZSJzdHJpbmciOmNhc2UibnVtYmVyIjpuLnB1c2goIiIrdCk7YnJlYWs7Y2FzZSJ1bmRlZmluZWQiOm4ucHVzaCgidW5kZWZpbmVkIik7YnJlYWs7ZGVmYXVsdDp0cnl7bGV0IG89SlNPTi5zdHJpbmdpZnkodCxudWxsLDIpO28ubGVuZ3RoPjUwMCYmKG89by5zdWJzdHJpbmcoMCw1MDApKyIuLi4iKSxuLnB1c2gobyl9Y2F0Y2h7bi5wdXNoKCJbY2lyY3VsYXIgb2JqZWN0XSIpfX1yZXR1cm4gbi5qb2luKCIgIil9fTt0eXBlb2YgRGVubz4idSImJihzZWxmLkRlbm89e2FyZ3M6W10sYnVpbGQ6e2FyY2g6Ing4Nl82NCJ9LGVudjp7Z2V0KCl7fX19KTt2YXIgZD1uZXcgTWFwLGk9bmV3IE1hcDtmdW5jdGlvbiBzKHIpe3R5cGVvZiB3aW5kb3c8InUiJiZ3aW5kb3cucGFyZW50IT09d2luZG93P3dpbmRvdy5wYXJlbnQucG9zdE1lc3NhZ2UociwiKiIpOnNlbGYucG9zdE1lc3NhZ2Uocil9dmFyIGw9MDtzZWxmLnN5c2NhbGw9YXN5bmMociwuLi5lKT0+YXdhaXQgbmV3IFByb21pc2UoKG4sdCk9PntsKyssaS5zZXQobCx7cmVzb2x2ZTpuLHJlamVjdDp0fSkscyh7dHlwZToic3lzY2FsbCIsaWQ6bCxuYW1lOnIsYXJnczplfSl9KTt2YXIgdT1uZXcgTWFwO3NlbGYucmVxdWlyZT1yPT57bGV0IGU9dS5nZXQocik7aWYoIWUpdGhyb3cgbmV3IEVycm9yKGBEeW5hbWljYWxseSBpbXBvcnRpbmcgbm9uLXByZWxvYWRlZCBsaWJyYXJ5ICR7cn1gKTtyZXR1cm4gZX07c2VsZi5jb25zb2xlPW5ldyBhKChyLGUpPT57cyh7dHlwZToibG9nIixsZXZlbDpyLG1lc3NhZ2U6ZX0pfSwhMSk7ZnVuY3Rpb24gZyhyKXtyZXR1cm5gcmV0dXJuICgke3J9KVsiZGVmYXVsdCJdYH1zZWxmLmFkZEV2ZW50TGlzdGVuZXIoIm1lc3NhZ2UiLHI9PntjKGFzeW5jKCk9PntsZXQgZT1yLmRhdGE7c3dpdGNoKGUudHlwZSl7Y2FzZSJsb2FkIjp7bGV0IG49bmV3IEZ1bmN0aW9uKGcoZS5jb2RlKSk7ZC5zZXQoZS5uYW1lLG4oKSkscyh7dHlwZToiaW5pdGVkIixuYW1lOmUubmFtZX0pfWJyZWFrO2Nhc2UibG9hZC1kZXBlbmRlbmN5Ijp7bGV0IHQ9bmV3IEZ1bmN0aW9uKGByZXR1cm4gJHtlLmNvZGV9YCkoKTt1LnNldChlLm5hbWUsdCkscyh7dHlwZToiZGVwZW5kZW5jeS1pbml0ZWQiLG5hbWU6ZS5uYW1lfSl9YnJlYWs7Y2FzZSJpbnZva2UiOntsZXQgbj1kLmdldChlLm5hbWUpO2lmKCFuKXRocm93IG5ldyBFcnJvcihgRnVuY3Rpb24gbm90IGxvYWRlZDogJHtlLm5hbWV9YCk7dHJ5e2xldCB0PWF3YWl0IFByb21pc2UucmVzb2x2ZShuKC4uLmUuYXJnc3x8W10pKTtzKHt0eXBlOiJyZXN1bHQiLGlkOmUuaWQscmVzdWx0OnR9KX1jYXRjaCh0KXtzKHt0eXBlOiJyZXN1bHQiLGlkOmUuaWQsZXJyb3I6dC5tZXNzYWdlLHN0YWNrOnQuc3RhY2t9KX19YnJlYWs7Y2FzZSJzeXNjYWxsLXJlc3BvbnNlIjp7bGV0IG49ZS5pZCx0PWkuZ2V0KG4pO2lmKCF0KXRocm93IGNvbnNvbGUubG9nKCJDdXJyZW50IG91dHN0YW5kaW5nIHJlcXVlc3RzIixpLCJsb29raW5nIHVwIixuKSxFcnJvcigiSW52YWxpZCByZXF1ZXN0IGlkIik7aS5kZWxldGUobiksZS5lcnJvcj90LnJlamVjdChuZXcgRXJyb3IoZS5lcnJvcikpOnQucmVzb2x2ZShlLnJlc3VsdCl9YnJlYWt9fSl9KTt9KSgpOwogcmV0dXJuIG1vZDt9KSgp"
+}

plugos/forked/esbuild_deno_loader/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Luca Casonato
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

plugos/forked/esbuild_deno_loader/README.md

@@ -0,0 +1,21 @@
+# esbuild_deno_loader
+
+Deno module resolution for `esbuild`.
+
+## Example
+
+This example bundles an entrypoint into a single ESM output.
+
+```js
+import * as esbuild from "https://deno.land/x/esbuild@v0.14.51/mod.js";
+import { denoPlugin } from "https://deno.land/x/esbuild_deno_loader@0.5.2/mod.ts";
+
+await esbuild.build({
+  plugins: [denoPlugin()],
+  entryPoints: ["https://deno.land/std@0.150.0/hash/sha1.ts"],
+  outfile: "./dist/sha1.esm.js",
+  bundle: true,
+  format: "esm",
+});
+esbuild.stop();
+```

plugos/forked/esbuild_deno_loader/deps.ts

@@ -0,0 +1,13 @@
+import type * as esbuild from "https://deno.land/x/esbuild@v0.14.54/mod.d.ts";
+export type { esbuild };
+export {
+  fromFileUrl,
+  resolve,
+  toFileUrl,
+} from "https://deno.land/std@0.150.0/path/mod.ts";
+export { basename, extname } from "https://deno.land/std@0.150.0/path/mod.ts";
+export {
+  resolveImportMap,
+  resolveModuleSpecifier,
+} from "https://deno.land/x/importmap@0.2.1/mod.ts";
+export type { ImportMap } from "https://deno.land/x/importmap@0.2.1/mod.ts";

plugos/forked/esbuild_deno_loader/examples/bundle.ts

@@ -0,0 +1,11 @@
+import * as esbuild from "https://deno.land/x/esbuild@v0.14.51/mod.js";
+import { denoPlugin } from "https://deno.land/x/esbuild_deno_loader@0.5.2/mod.ts";
+
+await esbuild.build({
+  plugins: [denoPlugin()],
+  entryPoints: ["https://deno.land/std@0.150.0/hash/sha1.ts"],
+  outfile: "./dist/sha1.esm.js",
+  bundle: true,
+  format: "esm",
+});
+esbuild.stop();

plugos/forked/esbuild_deno_loader/makefile

@@ -0,0 +1,11 @@
+test:
+	deno test -A
+
+lint:
+	deno lint
+
+fmt:
+	deno fmt
+
+fmt/check:
+	deno fmt --check

plugos/forked/esbuild_deno_loader/mod.ts

@@ -0,0 +1,117 @@
+import {
+  esbuild,
+  ImportMap,
+  resolveImportMap,
+  resolveModuleSpecifier,
+  toFileUrl,
+} from "./deps.ts";
+import { load as nativeLoad } from "./src/native_loader.ts";
+import { load as portableLoad } from "./src/portable_loader.ts";
+import { ModuleEntry } from "./src/deno.ts";
+import { resolve } from "https://deno.land/std@0.122.0/path/win32.ts";
+
+export interface DenoPluginOptions {
+  /**
+   * Specify the URL to an import map to use when resolving import specifiers.
+   * The URL must be fetchable with `fetch`.
+   */
+  importMapURL?: URL;
+  /**
+   * Specify which loader to use. By default this will use the `native` loader,
+   * unless `Deno.run` is not available.
+   *
+   * - `native`:     Shells out to the Deno execuatble under the hood to load
+   *                 files. Requires --allow-read and --allow-run.
+   * - `portable`:   Do module downloading and caching with only Web APIs.
+   *                 Requires --allow-net.
+   */
+  loader?: "native" | "portable";
+}
+
+/** The default loader to use. */
+export const DEFAULT_LOADER: "native" | "portable" =
+  typeof Deno.run === "function" ? "native" : "portable";
+
+export function denoPlugin(options: DenoPluginOptions = {}): esbuild.Plugin {
+  const loader = options.loader ?? DEFAULT_LOADER;
+  return {
+    name: "deno",
+    setup(build) {
+      const infoCache = new Map<string, ModuleEntry>();
+      let importMap: ImportMap | null = null;
+
+      build.onStart(async function onStart() {
+        if (options.importMapURL !== undefined) {
+          const resp = await fetch(options.importMapURL.href);
+          const txt = await resp.text();
+          importMap = resolveImportMap(JSON.parse(txt), options.importMapURL);
+        } else {
+          importMap = null;
+        }
+      });
+
+      build.onResolve(
+        { filter: /.*/ },
+        function onResolve(
+          args: esbuild.OnResolveArgs,
+        ): esbuild.OnResolveResult | null | undefined {
+          // console.log("To resolve", args.path);
+          const resolveDir = args.resolveDir
+            ? `${toFileUrl(args.resolveDir).href}/`
+            : "";
+          const referrer = args.importer || resolveDir;
+          let resolved: URL;
+          if (importMap !== null) {
+            const res = resolveModuleSpecifier(
+              args.path,
+              importMap,
+              new URL(referrer) || undefined,
+            );
+            resolved = new URL(res);
+          } else {
+            resolved = new URL(args.path, referrer);
+          }
+          // console.log("Resolved", resolved.href);
+          if (build.initialOptions.external) {
+            for (const external of build.initialOptions.external) {
+              if (resolved.href.startsWith(external)) {
+                // console.log("Got external", args.path, resolved.href);
+                return { path: resolved.href, external: true };
+              }
+            }
+          }
+          const href = resolved.href;
+          // Don't use the deno loader for any of the specific loader file extensions
+          const loaderExts = Object.keys(build.initialOptions.loader || {});
+          for (const ext of loaderExts) {
+            if (href.endsWith(ext)) {
+              console.log("Skipping", href);
+              return {
+                path: resolved.href.substring("file://".length),
+              };
+            }
+          }
+          return { path: resolved.href, namespace: "deno" };
+        },
+      );
+
+      build.onLoad(
+        { filter: /.*/ },
+        function onLoad(
+          args: esbuild.OnLoadArgs,
+        ): Promise<esbuild.OnLoadResult | null> {
+          if (args.path.endsWith(".css")) {
+            return Promise.resolve(null);
+          }
+          const url = new URL(args.path);
+          switch (loader) {
+            case "native":
+              return nativeLoad(infoCache, url, options);
+            case "portable":
+              return portableLoad(url, options);
+          }
+        },
+      );
+    },
+  };
+}

plugos/forked/esbuild_deno_loader/src/deno.ts

@@ -0,0 +1,89 @@
+// Lifted from https://raw.githubusercontent.com/denoland/deno_graph/89affe43c9d3d5c9165c8089687c107d53ed8fe1/lib/media_type.ts
+export type MediaType =
+  | "JavaScript"
+  | "Mjs"
+  | "Cjs"
+  | "JSX"
+  | "TypeScript"
+  | "Mts"
+  | "Cts"
+  | "Dts"
+  | "Dmts"
+  | "Dcts"
+  | "TSX"
+  | "Json"
+  | "Wasm"
+  | "TsBuildInfo"
+  | "SourceMap"
+  | "Unknown";
+
+export interface InfoOutput {
+  roots: string[];
+  modules: ModuleEntry[];
+  redirects: Record<string, string>;
+}
+
+export interface ModuleEntry {
+  specifier: string;
+  size: number;
+  mediaType?: MediaType;
+  local?: string;
+  checksum?: string;
+  emit?: string;
+  map?: string;
+  error?: string;
+}
+
+interface DenoInfoOptions {
+  importMap?: string;
+}
+
+let tempDir: null | string;
+
+export async function info(
+  specifier: URL,
+  options: DenoInfoOptions,
+): Promise<InfoOutput> {
+  const cmd = [
+    Deno.execPath(),
+    "info",
+    "--json",
+  ];
+  if (options.importMap !== undefined) {
+    cmd.push("--import-map", options.importMap);
+  }
+  cmd.push(specifier.href);
+
+  if (!tempDir) {
+    tempDir = Deno.makeTempDirSync();
+  }
+
+  let proc;
+
+  try {
+    proc = Deno.run({
+      cmd,
+      stdout: "piped",
+      cwd: tempDir,
+    });
+    const raw = await proc.output();
+    const status = await proc.status();
+    if (!status.success) {
+      throw new Error(`Failed to call 'deno info' on '${specifier.href}'`);
+    }
+    const txt = new TextDecoder().decode(raw);
+    return JSON.parse(txt);
+  } finally {
+    try {
+      proc?.stdout.close();
+    } catch (err) {
+      if (err instanceof Deno.errors.BadResource) {
+        // ignore the error
+      } else {
+        // deno-lint-ignore no-unsafe-finally
+        throw err;
+      }
+    }
+    proc?.close();
+  }
+}

plugos/forked/esbuild_deno_loader/src/native_loader.ts

@@ -0,0 +1,65 @@
+import { esbuild, fromFileUrl } from "../deps.ts";
+import * as deno from "./deno.ts";
+import { mediaTypeToLoader, transformRawIntoContent } from "./shared.ts";
+
+export interface LoadOptions {
+  importMapURL?: URL;
+}
+
+export async function load(
+  infoCache: Map<string, deno.ModuleEntry>,
+  url: URL,
+  options: LoadOptions,
+): Promise<esbuild.OnLoadResult | null> {
+  switch (url.protocol) {
+    case "http:":
+    case "https:":
+    case "data:":
+      return await loadFromCLI(infoCache, url, options);
+    case "file:": {
+      const res = await loadFromCLI(infoCache, url, options);
+      res.watchFiles = [fromFileUrl(url.href)];
+      return res;
+    }
+  }
+  return null;
+}
+
+async function loadFromCLI(
+  infoCache: Map<string, deno.ModuleEntry>,
+  specifier: URL,
+  options: LoadOptions,
+): Promise<esbuild.OnLoadResult> {
+  const specifierRaw = specifier.href;
+  if (!infoCache.has(specifierRaw)) {
+    const { modules, redirects } = await deno.info(specifier, {
+      importMap: options.importMapURL?.href,
+    });
+    for (const module of modules) {
+      infoCache.set(module.specifier, module);
+    }
+    for (const [specifier, redirect] of Object.entries(redirects)) {
+      const redirected = infoCache.get(redirect);
+      if (!redirected) {
+        throw new TypeError("Unreachable.");
+      }
+      infoCache.set(specifier, redirected);
+    }
+  }
+
+  const module = infoCache.get(specifierRaw);
+  if (!module) {
+    throw new TypeError("Unreachable.");
+  }
+
+  if (module.error) throw new Error(module.error);
+  if (!module.local) throw new Error("Module not downloaded yet.");
+  const mediaType = module.mediaType ?? "Unknown";
+
+  const loader = mediaTypeToLoader(mediaType);
+
+  const raw = await Deno.readFile(module.local);
+  const contents = transformRawIntoContent(raw, mediaType);
+
+  return { contents, loader };
+}

plugos/forked/esbuild_deno_loader/src/portable_loader.ts

@@ -0,0 +1,194 @@
+import { esbuild, extname, fromFileUrl } from "../deps.ts";
+import * as deno from "./deno.ts";
+import { mediaTypeToLoader, transformRawIntoContent } from "./shared.ts";
+
+export interface LoadOptions {
+  importMapURL?: URL;
+}
+
+export async function load(
+  url: URL,
+  _options: LoadOptions,
+): Promise<esbuild.OnLoadResult | null> {
+  switch (url.protocol) {
+    case "http:":
+    case "https:":
+    case "data:":
+      return await loadWithFetch(url);
+    case "file:": {
+      const res = await loadWithReadFile(url);
+      res.watchFiles = [fromFileUrl(url.href)];
+      return res;
+    }
+  }
+  return null;
+}
+
+async function loadWithFetch(
+  specifier: URL,
+): Promise<esbuild.OnLoadResult> {
+  const specifierRaw = specifier.href;
+
+  // TODO(lucacasonato): redirects!
+  const resp = await fetch(specifierRaw);
+  if (!resp.ok) {
+    throw new Error(
+      `Encountered status code ${resp.status} while fetching ${specifierRaw}.`,
+    );
+  }
+
+  const contentType = resp.headers.get("content-type");
+  const mediaType = mapContentType(
+    new URL(resp.url || specifierRaw),
+    contentType,
+  );
+
+  const loader = mediaTypeToLoader(mediaType);
+
+  const raw = new Uint8Array(await resp.arrayBuffer());
+  const contents = transformRawIntoContent(raw, mediaType);
+
+  return { contents, loader };
+}
+
+async function loadWithReadFile(specifier: URL): Promise<esbuild.OnLoadResult> {
+  const path = fromFileUrl(specifier);
+
+  const mediaType = mapContentType(specifier, null);
+  const loader = mediaTypeToLoader(mediaType);
+
+  const raw = await Deno.readFile(path);
+  const contents = transformRawIntoContent(raw, mediaType);
+
+  return { contents, loader };
+}
+
+function mapContentType(
+  specifier: URL,
+  contentType: string | null,
+): deno.MediaType {
+  if (contentType !== null) {
+    const contentTypes = contentType.split(";");
+    const mediaType = contentTypes[0].toLowerCase();
+    switch (mediaType) {
+      case "application/typescript":
+      case "text/typescript":
+      case "video/vnd.dlna.mpeg-tts":
+      case "video/mp2t":
+      case "application/x-typescript":
+        return mapJsLikeExtension(specifier, "TypeScript");
+      case "application/javascript":
+      case "text/javascript":
+      case "application/ecmascript":
+      case "text/ecmascript":
+      case "application/x-javascript":
+      case "application/node":
+        return mapJsLikeExtension(specifier, "JavaScript");
+      case "text/jsx":
+        return "JSX";
+      case "text/tsx":
+        return "TSX";
+      case "application/json":
+      case "text/json":
+        return "Json";
+      case "application/wasm":
+        return "Wasm";
+      case "text/plain":
+      case "application/octet-stream":
+        return mediaTypeFromSpecifier(specifier);
+      default:
+        return "Unknown";
+    }
+  } else {
+    return mediaTypeFromSpecifier(specifier);
+  }
+}
+
+function mapJsLikeExtension(
+  specifier: URL,
+  defaultType: deno.MediaType,
+): deno.MediaType {
+  const path = specifier.pathname;
+  switch (extname(path)) {
+    case ".jsx":
+      return "JSX";
+    case ".mjs":
+      return "Mjs";
+    case ".cjs":
+      return "Cjs";
+    case ".tsx":
+      return "TSX";
+    case ".ts":
+      if (path.endsWith(".d.ts")) {
+        return "Dts";
+      } else {
+        return defaultType;
+      }
+    case ".mts": {
+      if (path.endsWith(".d.mts")) {
+        return "Dmts";
+      } else {
+        return defaultType == "JavaScript" ? "Mjs" : "Mts";
+      }
+    }
+    case ".cts": {
+      if (path.endsWith(".d.cts")) {
+        return "Dcts";
+      } else {
+        return defaultType == "JavaScript" ? "Cjs" : "Cts";
+      }
+    }
+    default:
+      return defaultType;
+  }
+}
+
+function mediaTypeFromSpecifier(specifier: URL): deno.MediaType {
+  const path = specifier.pathname;
+  switch (extname(path)) {
+    case "":
+      if (path.endsWith("/.tsbuildinfo")) {
+        return "TsBuildInfo";
+      } else {
+        return "Unknown";
+      }
+    case ".ts":
+      if (path.endsWith(".d.ts")) {
+        return "Dts";
+      } else {
+        return "TypeScript";
+      }
+    case ".mts":
+      if (path.endsWith(".d.mts")) {
+        return "Dmts";
+      } else {
+        return "Mts";
+      }
+    case ".cts":
+      if (path.endsWith(".d.cts")) {
+        return "Dcts";
+      } else {
+        return "Cts";
+      }
+    case ".tsx":
+      return "TSX";
+    case ".js":
+      return "JavaScript";
+    case ".jsx":
+      return "JSX";
+    case ".mjs":
+      return "Mjs";
+    case ".cjs":
+      return "Cjs";
+    case ".json":
+      return "Json";
+    case ".wasm":
+      return "Wasm";
+    case ".tsbuildinfo":
+      return "TsBuildInfo";
+    case ".map":
+      return "SourceMap";
+    default:
+      return "Unknown";
+  }
+}

plugos/forked/esbuild_deno_loader/src/shared.ts

@@ -0,0 +1,40 @@
+import { esbuild } from "../deps.ts";
+import { MediaType } from "./deno.ts";
+
+export function mediaTypeToLoader(mediaType: MediaType): esbuild.Loader {
+  switch (mediaType) {
+    case "JavaScript":
+    case "Mjs":
+      return "js";
+    case "JSX":
+      return "jsx";
+    case "TypeScript":
+    case "Mts":
+      return "ts";
+    case "TSX":
+      return "tsx";
+    case "Json":
+      return "js";
+    default:
+      throw new Error(`Unhandled media type ${mediaType}.`);
+  }
+}
+
+export function transformRawIntoContent(
+  raw: Uint8Array,
+  mediaType: MediaType,
+): string | Uint8Array {
+  switch (mediaType) {
+    case "Json":
+      return jsonToESM(raw);
+    default:
+      return raw;
+  }
+}
+
+function jsonToESM(source: Uint8Array): string {
+  const sourceString = new TextDecoder().decode(source);
+  let json = JSON.stringify(JSON.parse(sourceString), null, 2);
+  json = json.replaceAll(`"__proto__":`, `["__proto__"]:`);
+  return `export default ${json};`;
+}

plugos/forked/esbuild_deno_loader/test_deps.ts

@@ -0,0 +1,6 @@
+import * as esbuild from "https://deno.land/x/esbuild@v0.14.51/mod.js";
+export { esbuild };
+export {
+  assert,
+  assertEquals,
+} from "https://deno.land/std@0.150.0/testing/asserts.ts";

plugos/gen.ts

@@ -0,0 +1,18 @@
+import { AssetBundle } from "./asset_bundle/bundle.ts";
+import { compile } from "./compile.ts";
+const bundlePath =
+  new URL("./environments/worker_bundle.json", import.meta.url).pathname;
+const workerPath =
+  new URL("./environments/sandbox_worker.ts", import.meta.url).pathname;
+
+const workerCode = await compile(workerPath);
+
+const assetBundle = new AssetBundle();
+assetBundle.writeTextFileSync("worker.js", workerCode);
+Deno.writeTextFile(
+  bundlePath,
+  JSON.stringify(assetBundle.toJSON(), null, 2),
+);
+
+console.log(`Wrote updated bundle to ${bundlePath}`);
+Deno.exit(0);

plugos/hack.ts

@@ -0,0 +1,4 @@
+export function patchDenoLibJS(code: string): string {
+  // The Deno std lib has one occurence of a regex that Webkit JS doesn't (yet parse), we'll strip it because it's likely never invoked anyway, YOLO
+  return code.replaceAll("/(?<=\\n)/", "/()/");
+}

plugos/hooks/endpoint.test.ts

@@ -1,4 +1,4 @@
-import { sandboxFactory } from "../environments/deno_sandbox.ts";
+import { createSandbox } from "../environments/deno_sandbox.ts";
 import { Manifest } from "../types.ts";
 import { EndpointHook, EndpointHookT } from "./endpoint.ts";
 import { System } from "../system.ts";
@@ -6,13 +6,9 @@ import { System } from "../system.ts";
 import { Application } from "../../server/deps.ts";
 import { assertEquals } from "../../test_deps.ts";
 
-import assetBundle from "../../dist/asset_bundle.json" assert { type: "json" };
-import { AssetBundle } from "../asset_bundle_reader.ts";
-
 Deno.test("Run a plugos endpoint server", async () => {
-  const createSandbox = sandboxFactory(assetBundle as AssetBundle);
-  let system = new System<EndpointHookT>("server");
-  let plug = await system.load(
+  const system = new System<EndpointHookT>("server");
+  await system.load(
     {
       name: "test",
       functions: {

plugos/plug.ts

@@ -14,7 +14,7 @@ export class Plug<HookT> {
   constructor(
     system: System<HookT>,
     name: string,
-    sandboxFactory: (plug: Plug<HookT>) => Sandbox
+    sandboxFactory: (plug: Plug<HookT>) => Sandbox,
   ) {
     this.system = system;
     this.name = name;
@@ -55,7 +55,7 @@ export class Plug<HookT> {
       }
       if (!this.canInvoke(name)) {
         throw new Error(
-          `Function ${name} is not available in ${this.runtimeEnv}`
+          `Function ${name} is not available in ${this.runtimeEnv}`,
         );
       }
       await this.sandbox.load(name, funDef.code!);

plugos/runtime.test.ts

@@ -1,4 +1,4 @@
-import { sandboxFactory } from "./environments/deno_sandbox.ts";
+import { createSandbox } from "./environments/deno_sandbox.ts";
 import { System } from "./system.ts";
 
 import {
@@ -6,10 +6,7 @@ import {
   assertEquals,
 } from "https://deno.land/std@0.158.0/testing/asserts.ts";
 
-import assetBundle from "../dist/asset_bundle.json" assert { type: "json" };
 Deno.test("Run a deno sandbox", async () => {
-  const createSandbox = sandboxFactory(assetBundle as AssetBundle);
-
   const system = new System("server");
   system.registerSyscalls([], {
     addNumbers: (_ctx, a, b) => {
@@ -125,12 +122,10 @@ Deno.test("Run a deno sandbox", async () => {
 
 import { bundle as plugOsBundle } from "./bin/plugos-bundle.ts";
 import { esbuild } from "./compile.ts";
-import { AssetBundle } from "./asset_bundle_reader.ts";
+import { AssetBundle } from "./asset_bundle/bundle.ts";
 const __dirname = new URL(".", import.meta.url).pathname;
 
 Deno.test("Preload dependencies", async () => {
-  const createSandbox = sandboxFactory(assetBundle as AssetBundle);
-
   const globalModules = await plugOsBundle(
     `${__dirname}../plugs/global.plug.yaml`,
   );

plugos/syscalls/asset.ts

@@ -1,14 +1,14 @@
 import { SysCallMapping, System } from "../system.ts";
-import type { AssetBundle, FileMeta } from "../asset_bundle_reader.ts";
+import { AssetBundle } from "../asset_bundle/bundle.ts";
 
 export default function assetSyscalls(system: System<any>): SysCallMapping {
   return {
     "asset.readAsset": (
       ctx,
       name: string,
-    ): { data: string; meta: FileMeta } => {
+    ): string => {
       return (system.loadedPlugs.get(ctx.plug.name)!.manifest!
-        .assets as AssetBundle)[name];
+        .assets as AssetBundle).readFileAsDataUrl(name);
     },
   };
 }

plugos/syscalls/fs.deno.ts

@@ -1,7 +1,7 @@
 import type { SysCallMapping } from "../system.ts";
-import { mime, path } from "../../server/deps.ts";
-import { base64Decode, base64Encode } from "../../plugos/base64.ts";
-import type { FileMeta } from "../asset_bundle_reader.ts";
+import { mime, path } from "../deps.ts";
+import { base64Decode, base64Encode } from "../asset_bundle/base64.ts";
+import { FileMeta } from "../../common/types.ts";
 
 export default function fileSystemSyscalls(root = "/"): SysCallMapping {
   function resolvedPath(p: string): string {

plugos/syscalls/store.deno.test.ts

@@ -1,14 +1,10 @@
 import { assertEquals } from "../../test_deps.ts";
 import { SQLite } from "../../server/deps.ts";
-import { sandboxFactory } from "../environments/deno_sandbox.ts";
+import { createSandbox } from "../environments/deno_sandbox.ts";
 import { System } from "../system.ts";
 import { ensureTable, storeSyscalls } from "./store.deno.ts";
 
-import assetBundle from "../../dist/asset_bundle.json" assert { type: "json" };
-import { AssetBundle } from "../asset_bundle_reader.ts";
-
 Deno.test("Test store", async () => {
-  const createSandbox = sandboxFactory(assetBundle as AssetBundle);
   const db = new SQLite(":memory:");
   await ensureTable(db, "test_table");
   const system = new System("server");

plugos/types.ts

@@ -1,10 +1,10 @@
-import { AssetBundle } from "../plugos/asset_bundle_reader.ts";
 import { System } from "./system.ts";
+import { AssetJson } from "./asset_bundle/bundle.ts";
 
 export interface Manifest<HookT> {
   name: string;
   requiredPermissions?: string[];
-  assets?: string[] | AssetBundle;
+  assets?: string[] | AssetJson;
   dependencies?: {
     [key: string]: string;
   };