Added imports

.gitpod.yml

@@ -16,8 +16,8 @@ github:
 tasks:
   - name: Setup
     init: |
-      deno task install
       deno task build
+      deno task install
       gp sync-done setup
   - name: Server watcher
     init: |      

build_plugs.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
 
-deno run -A --unstable plugos/bin/plugos-bundle.ts --dist dist_bundle/_plug $1 --exclude=https://esm.sh/handlebars,https://deno.land/std/encoding/yaml.ts,https://esm.sh/@lezer/lr plugs/*/*.plug.yaml
+deno run -A --unstable plugos/bin/plugos-bundle.ts --dist dist_bundle/_plug $@ --exclude=https://esm.sh/handlebars,https://deno.land/std/encoding/yaml.ts,https://esm.sh/@lezer/lr plugs/*/*.plug.yaml

plugos/asset_bundle/bundle.ts

@@ -1,7 +1,10 @@
 import { base64Decode, base64Encode } from "./base64.ts";
 import { mime } from "../deps.ts";
 
-export type AssetJson = Record<string, string>;
+type DataUrl = string;
+
+// Mapping from path -> `data:mimetype;base64,base64-encoded-data` strings
+export type AssetJson = Record<string, DataUrl>;
 
 export class AssetBundle {
   readonly bundle: AssetJson;

plugos/bin/plugos-bundle.ts

@@ -8,9 +8,8 @@ import {
   esbuild,
   sandboxCompileModule,
 } from "../compile.ts";
-import { path } from "../../server/deps.ts";
+import { cacheDir, flags, path } from "../deps.ts";
 
-import * as flags from "https://deno.land/std@0.158.0/flags/mod.ts";
 import { bundleAssets } from "../asset_bundle/builder.ts";
 
 export async function bundle(
@@ -26,14 +25,9 @@ export async function bundle(
     throw new Error(`Missing 'name' in ${manifestPath}`);
   }
 
-  const allModulesToExclude = options.excludeModules
-    ? options.excludeModules.slice()
-    : [];
-
   // Dependencies
   for (let [name, moduleSpec] of Object.entries(manifest.dependencies || {})) {
     manifest.dependencies![name] = await sandboxCompileModule(moduleSpec);
-    allModulesToExclude.push(name);
   }
 
   // Assets
@@ -43,9 +37,38 @@ export async function bundle(
   );
   manifest.assets = assetsBundle.toJSON();
 
-  // Functions
+  // Imports
+  // Imports currently only "import" dependencies at this point, importing means: assume they're preloaded so we don't need to bundle them
+  const plugCache = path.join(cacheDir()!, "plugos");
+  await Deno.mkdir(plugCache, { recursive: true });
+  const imports: Manifest<any>[] = [];
+  for (const manifestUrl of manifest.imports || []) {
+    // Safe file name
+    const cachedManifestPath = manifestUrl.replaceAll(/[^a-zA-Z0-9]/g, "_");
+    try {
+      if (options.reload) {
+        throw new Error("Forced reload");
+      }
+      // Try to just load from the cache
+      const cachedManifest = JSON.parse(
+        await Deno.readTextFile(path.join(plugCache, cachedManifestPath)),
+      ) as Manifest<any>;
+      imports.push(cachedManifest);
+    } catch {
+      // Otherwise, download and cache
+      console.log("Caching plug", manifestUrl, "to", plugCache);
+      const cachedManifest = await (await fetch(manifestUrl))
+        .json() as Manifest<any>;
+      await Deno.writeTextFile(
+        path.join(plugCache, cachedManifestPath),
+        JSON.stringify(cachedManifest),
+      );
+      imports.push(cachedManifest);
+    }
+  }
 
-  for (let [name, def] of Object.entries(manifest.functions || {})) {
+  // Functions
+  for (const def of Object.values(manifest.functions || {})) {
     let jsFunctionName = "default",
       filePath = path.join(rootPath, def.path!);
     if (filePath.indexOf(":") !== -1) {
@@ -57,7 +80,12 @@ export async function bundle(
       jsFunctionName,
       {
         ...options,
-        excludeModules: allModulesToExclude,
+        imports: [
+          manifest,
+          ...imports,
+          // This is mostly for testing
+          ...options.imports || [],
+        ],
       },
     );
     delete def.path;
@@ -130,15 +158,14 @@ async function bundleRun(
 
 if (import.meta.main) {
   const args = flags.parse(Deno.args, {
-    boolean: ["debug", "watch"],
-    string: ["dist", "exclude", "importmap"],
+    boolean: ["debug", "watch", "reload", "info"],
+    string: ["dist", "importmap"],
     alias: { w: "watch" },
-    // collect: ["exclude"],
   });
 
   if (args._.length === 0) {
     console.log(
-      "Usage: plugos-bundle [--debug] [--dist <path>] [--importmap import_map.json] [--exclude=package1,package2] <manifest.plug.yaml> <manifest2.plug.yaml> ...",
+      "Usage: plugos-bundle [--debug] [--reload] [--dist <path>] [--info] [--importmap import_map.json] [--exclude=package1,package2] <manifest.plug.yaml> <manifest2.plug.yaml> ...",
     );
     Deno.exit(1);
   }
@@ -153,7 +180,8 @@ if (import.meta.main) {
     args.watch,
     {
       debug: args.debug,
-      excludeModules: args.exclude ? args.exclude.split(",") : undefined,
+      reload: args.reload,
+      info: args.info,
       importMap: args.importmap
         ? new URL(args.importmap, `file://${Deno.cwd()}/`)
         : undefined,

plugos/compile.ts

@@ -9,14 +9,33 @@ export const esbuild: typeof esbuildWasm = Deno.run === undefined
 import { path } from "./deps.ts";
 import { denoPlugin } from "./forked/esbuild_deno_loader/mod.ts";
 import { patchDenoLibJS } from "./hack.ts";
+import { Manifest } from "./types.ts";
 
 export type CompileOptions = {
   debug?: boolean;
-  excludeModules?: string[];
-  meta?: boolean;
+  imports?: Manifest<any>[];
   importMap?: URL;
+  // Reload plug import cache
+  reload?: boolean;
+  // Print info on bundle size
+  info?: boolean;
 };
 
+function esBuildExternals(imports?: Manifest<any>[]) {
+  if (!imports) {
+    return [];
+  }
+  const externals: string[] = [];
+  for (const manifest of imports) {
+    for (const dep of Object.keys(manifest.dependencies || {})) {
+      if (!externals.includes(dep)) {
+        externals.push(dep);
+      }
+    }
+  }
+  return externals;
+}
+
 export async function compile(
   filePath: string,
   functionName: string | undefined = undefined,
@@ -49,8 +68,8 @@ export async function compile(
       sourcemap: false, //debug ? "inline" : false,
       minify: !options.debug,
       outfile: outFile,
-      metafile: true,
-      external: options.excludeModules || [],
+      metafile: options.info,
+      external: esBuildExternals(options.imports),
       treeShaking: true,
       plugins: [
         denoPlugin({
@@ -62,8 +81,8 @@ export async function compile(
       absWorkingDir: path.resolve(path.dirname(inFile)),
     });
 
-    if (options.meta) {
-      const text = await esbuild.analyzeMetafile(result.metafile);
+    if (options.info) {
+      const text = await esbuild.analyzeMetafile(result.metafile!);
       console.log("Bundle info for", functionName, text);
     }
 

plugos/deps.ts

@@ -2,3 +2,5 @@ export { globToRegExp } from "https://deno.land/std@0.158.0/path/glob.ts";
 export { walk } from "https://deno.land/std@0.159.0/fs/mod.ts";
 export * as path from "https://deno.land/std@0.158.0/path/mod.ts";
 export { mime } from "https://deno.land/x/mimetypes@v1.0.0/mod.ts";
+export { default as cacheDir } from "https://deno.land/x/cache_dir@0.2.0/mod.ts";
+export * as flags from "https://deno.land/std@0.158.0/flags/mod.ts";

plugos/environments/sandbox_worker.ts

@@ -1,3 +1,4 @@
+// IMPORTANT: After modifiying this file, run `deno task generate` in the SB root to regenerate the asset bundle (`worker_bundle.json`), which will be imported for the runtime.
 import { safeRun } from "../util.ts";
 import { ConsoleLogger } from "./custom_logger.ts";
 import type { ControllerMessage, WorkerMessage } from "./worker.ts";
@@ -18,8 +19,9 @@ if (typeof Deno === "undefined") {
   };
 }
 
-let loadedFunctions = new Map<string, Function>();
-let pendingRequests = new Map<
+// deno-lint-ignore ban-types
+const loadedFunctions = new Map<string, Function>();
+const pendingRequests = new Map<
   number,
   {
     resolve: (result: unknown) => void;
@@ -55,7 +57,7 @@ self.syscall = async (name: string, ...args: any[]) => {
   });
 };
 
-let loadedModules = new Map<string, any>();
+const loadedModules = new Map<string, any>();
 
 // @ts-ignore: global to load dynamic imports
 self.require = (moduleName: string): any => {
@@ -69,7 +71,7 @@ self.require = (moduleName: string): any => {
   return mod;
 };
 
-// @ts-ignore
+// @ts-ignore: global overwrite on purpose
 self.console = new ConsoleLogger((level, message) => {
   workerPostMessage({ type: "log", level, message });
 }, false);
@@ -80,7 +82,7 @@ function wrapScript(code: string) {
 
 self.addEventListener("message", (event: { data: WorkerMessage }) => {
   safeRun(async () => {
-    let data = event.data;
+    const data = event.data;
     switch (data.type) {
       case "load":
         {

plugos/environments/worker_bundle.json

@@ -1,3 +1,3 @@
 {
   "worker.js": "data:application/javascript;base64,KCgpID0+IHsgdmFyIG1vZD0oKCk9PntmdW5jdGlvbiBjKHIpe3IoKS5jYXRjaChlPT57Y29uc29sZS5lcnJvcigiQ2F1Z2h0IGVycm9yIixlLm1lc3NhZ2UpfSl9dmFyIGE9Y2xhc3N7Y29uc3RydWN0b3IoZSxuPSEwKXt0aGlzLnByaW50PW4sdGhpcy5jYWxsYmFjaz1lfWxvZyguLi5lKXt0aGlzLnB1c2goImxvZyIsZSl9d2FybiguLi5lKXt0aGlzLnB1c2goIndhcm4iLGUpfWVycm9yKC4uLmUpe3RoaXMucHVzaCgiZXJyb3IiLGUpfWluZm8oLi4uZSl7dGhpcy5wdXNoKCJpbmZvIixlKX1wdXNoKGUsbil7dGhpcy5jYWxsYmFjayhlLHRoaXMubG9nTWVzc2FnZShuKSksdGhpcy5wcmludCYmY29uc29sZVtlXSguLi5uKX1sb2dNZXNzYWdlKGUpe2xldCBuPVtdO2ZvcihsZXQgdCBvZiBlKXN3aXRjaCh0eXBlb2YgdCl7Y2FzZSJzdHJpbmciOmNhc2UibnVtYmVyIjpuLnB1c2goIiIrdCk7YnJlYWs7Y2FzZSJ1bmRlZmluZWQiOm4ucHVzaCgidW5kZWZpbmVkIik7YnJlYWs7ZGVmYXVsdDp0cnl7bGV0IG89SlNPTi5zdHJpbmdpZnkodCxudWxsLDIpO28ubGVuZ3RoPjUwMCYmKG89by5zdWJzdHJpbmcoMCw1MDApKyIuLi4iKSxuLnB1c2gobyl9Y2F0Y2h7bi5wdXNoKCJbY2lyY3VsYXIgb2JqZWN0XSIpfX1yZXR1cm4gbi5qb2luKCIgIil9fTt0eXBlb2YgRGVubz4idSImJihzZWxmLkRlbm89e2FyZ3M6W10sYnVpbGQ6e2FyY2g6Ing4Nl82NCJ9LGVudjp7Z2V0KCl7fX19KTt2YXIgZD1uZXcgTWFwLGk9bmV3IE1hcDtmdW5jdGlvbiBzKHIpe3R5cGVvZiB3aW5kb3c8InUiJiZ3aW5kb3cucGFyZW50IT09d2luZG93P3dpbmRvdy5wYXJlbnQucG9zdE1lc3NhZ2UociwiKiIpOnNlbGYucG9zdE1lc3NhZ2Uocil9dmFyIGw9MDtzZWxmLnN5c2NhbGw9YXN5bmMociwuLi5lKT0+YXdhaXQgbmV3IFByb21pc2UoKG4sdCk9PntsKyssaS5zZXQobCx7cmVzb2x2ZTpuLHJlamVjdDp0fSkscyh7dHlwZToic3lzY2FsbCIsaWQ6bCxuYW1lOnIsYXJnczplfSl9KTt2YXIgdT1uZXcgTWFwO3NlbGYucmVxdWlyZT1yPT57bGV0IGU9dS5nZXQocik7aWYoIWUpdGhyb3cgbmV3IEVycm9yKGBEeW5hbWljYWxseSBpbXBvcnRpbmcgbm9uLXByZWxvYWRlZCBsaWJyYXJ5ICR7cn1gKTtyZXR1cm4gZX07c2VsZi5jb25zb2xlPW5ldyBhKChyLGUpPT57cyh7dHlwZToibG9nIixsZXZlbDpyLG1lc3NhZ2U6ZX0pfSwhMSk7ZnVuY3Rpb24gZyhyKXtyZXR1cm5gcmV0dXJuICgke3J9KVsiZGVmYXVsdCJdYH1zZWxmLmFkZEV2ZW50TGlzdGVuZXIoIm1lc3NhZ2UiLHI9PntjKGFzeW5jKCk9PntsZXQgZT1yLmRhdGE7c3dpdGNoKGUudHlwZSl7Y2FzZSJsb2FkIjp7bGV0IG49bmV3IEZ1bmN0aW9uKGcoZS5jb2RlKSk7ZC5zZXQoZS5uYW1lLG4oKSkscyh7dHlwZToiaW5pdGVkIixuYW1lOmUubmFtZX0pfWJyZWFrO2Nhc2UibG9hZC1kZXBlbmRlbmN5Ijp7bGV0IHQ9bmV3IEZ1bmN0aW9uKGByZXR1cm4gJHtlLmNvZGV9YCkoKTt1LnNldChlLm5hbWUsdCkscyh7dHlwZToiZGVwZW5kZW5jeS1pbml0ZWQiLG5hbWU6ZS5uYW1lfSl9YnJlYWs7Y2FzZSJpbnZva2UiOntsZXQgbj1kLmdldChlLm5hbWUpO2lmKCFuKXRocm93IG5ldyBFcnJvcihgRnVuY3Rpb24gbm90IGxvYWRlZDogJHtlLm5hbWV9YCk7dHJ5e2xldCB0PWF3YWl0IFByb21pc2UucmVzb2x2ZShuKC4uLmUuYXJnc3x8W10pKTtzKHt0eXBlOiJyZXN1bHQiLGlkOmUuaWQscmVzdWx0OnR9KX1jYXRjaCh0KXtzKHt0eXBlOiJyZXN1bHQiLGlkOmUuaWQsZXJyb3I6dC5tZXNzYWdlLHN0YWNrOnQuc3RhY2t9KX19YnJlYWs7Y2FzZSJzeXNjYWxsLXJlc3BvbnNlIjp7bGV0IG49ZS5pZCx0PWkuZ2V0KG4pO2lmKCF0KXRocm93IGNvbnNvbGUubG9nKCJDdXJyZW50IG91dHN0YW5kaW5nIHJlcXVlc3RzIixpLCJsb29raW5nIHVwIixuKSxFcnJvcigiSW52YWxpZCByZXF1ZXN0IGlkIik7aS5kZWxldGUobiksZS5lcnJvcj90LnJlamVjdChuZXcgRXJyb3IoZS5lcnJvcikpOnQucmVzb2x2ZShlLnJlc3VsdCl9YnJlYWt9fSl9KTt9KSgpOwogcmV0dXJuIG1vZDt9KSgp"
-}
+}

plugos/plug.ts

@@ -1,11 +1,13 @@
 import { Manifest, RuntimeEnvironment } from "./types.ts";
 import { Sandbox } from "./sandbox.ts";
 import { System } from "./system.ts";
+import { AssetBundle, AssetJson } from "./asset_bundle/bundle.ts";
 
 export class Plug<HookT> {
   system: System<HookT>;
   sandbox: Sandbox;
   public manifest?: Manifest<HookT>;
+  public assets?: AssetBundle;
   readonly runtimeEnv: RuntimeEnvironment;
   grantedPermissions: string[] = [];
   name: string;
@@ -25,6 +27,9 @@ export class Plug<HookT> {
 
   async load(manifest: Manifest<HookT>) {
     this.manifest = manifest;
+    this.assets = new AssetBundle(
+      manifest.assets ? manifest.assets as AssetJson : {},
+    );
     // TODO: These need to be explicitly granted, not just taken
     this.grantedPermissions = manifest.requiredPermissions || [];
     for (const [dep, code] of Object.entries(manifest.dependencies || {})) {

plugos/runtime.test.ts

@@ -122,19 +122,18 @@ Deno.test("Run a deno sandbox", async () => {
 
 import { bundle as plugOsBundle } from "./bin/plugos-bundle.ts";
 import { esbuild } from "./compile.ts";
-import { AssetBundle } from "./asset_bundle/bundle.ts";
+
 const __dirname = new URL(".", import.meta.url).pathname;
 
 Deno.test("Preload dependencies", async () => {
   const globalModules = await plugOsBundle(
     `${__dirname}../plugs/global.plug.yaml`,
   );
-  // const globalModules = JSON.parse(
-  //   Deno.readTextFileSync(`${tmpDist}/global.plug.json`),
-  // );
   const testPlugManifest = await plugOsBundle(
     `${__dirname}test.plug.yaml`,
-    { excludeModules: Object.keys(globalModules.dependencies!) },
+    {
+      imports: [globalModules],
+    },
   );
   esbuild.stop();
 

plugos/syscalls/asset.ts

@@ -1,5 +1,4 @@
 import { SysCallMapping, System } from "../system.ts";
-import { AssetBundle } from "../asset_bundle/bundle.ts";
 
 export default function assetSyscalls(system: System<any>): SysCallMapping {
   return {
@@ -7,8 +6,9 @@ export default function assetSyscalls(system: System<any>): SysCallMapping {
       ctx,
       name: string,
     ): string => {
-      return (system.loadedPlugs.get(ctx.plug.name)!.manifest!
-        .assets as AssetBundle).readFileAsDataUrl(name);
+      return system.loadedPlugs.get(ctx.plug.name)!.assets!.readFileAsDataUrl(
+        name,
+      );
     },
   };
 }

plugos/syscalls/esbuild.ts

@@ -1,17 +1,16 @@
 import { sandboxCompile, sandboxCompileModule } from "../compile.ts";
 import { SysCallMapping } from "../system.ts";
+import { Manifest } from "../types.ts";
 
-// TODO: FIgure out a better way to do this
-const builtinModules = ["yaml", "handlebars"];
-
-export function esbuildSyscalls(): SysCallMapping {
+export function esbuildSyscalls(
+  imports: Manifest<any>[],
+): SysCallMapping {
   return {
     "esbuild.compile": async (
       _ctx,
       filename: string,
       code: string,
       functionName?: string,
-      excludeModules: string[] = [],
     ): Promise<string> => {
       return await sandboxCompile(
         filename,
@@ -19,7 +18,7 @@ export function esbuildSyscalls(): SysCallMapping {
         functionName,
         {
           debug: true,
-          excludeModules: [...builtinModules, ...excludeModules],
+          imports,
         },
       );
     },
@@ -28,7 +27,7 @@ export function esbuildSyscalls(): SysCallMapping {
       moduleName: string,
     ): Promise<string> => {
       return await sandboxCompileModule(moduleName, {
-        excludeModules: builtinModules,
+        imports,
       });
     },
   };

plugos/types.ts

@@ -4,6 +4,8 @@ import { AssetJson } from "./asset_bundle/bundle.ts";
 export interface Manifest<HookT> {
   name: string;
   requiredPermissions?: string[];
+  // URLs to plugs whose dependencies are presumed to already be loaded (main use case: global.plug.json)
+  imports?: string[];
   assets?: string[] | AssetJson;
   dependencies?: {
     [key: string]: string;

plugs/core/core.plug.yaml

@@ -1,4 +1,6 @@
 name: core
+imports:
+  - https://get.silverbullet.md/global.plug.json
 syntax:
   Hashtag:
     firstCharacters:

plugs/emoji/emoji.plug.yaml

@@ -1,4 +1,6 @@
 name: emoji
+imports:
+  - https://get.silverbullet.md/global.plug.json
 functions:
   emojiCompleter:
     path: "./emoji.ts:emojiCompleter"

plugs/markdown/markdown.plug.yaml

@@ -1,4 +1,6 @@
 name: markdown
+imports:
+  - https://get.silverbullet.md/global.plug.json
 functions:
   toggle:
     path: "./markdown.ts:togglePreview"

plugs/plugmd/plugmd.plug.yaml

@@ -1,4 +1,6 @@
 name: plugmd
+imports:
+  - https://get.silverbullet.md/global.plug.json
 functions:
   check:
     path: "./plugmd.ts:checkCommand"

plugs/query/query.plug.yaml

@@ -1,4 +1,6 @@
 name: query
+imports:
+  - https://get.silverbullet.md/global.plug.json
 functions:
   updateMaterializedQueriesOnPage:
     path: ./materialized_queries.ts:updateMaterializedQueriesOnPage

plugs/tasks/tasks.plug.yaml

@@ -1,4 +1,6 @@
 name: tasks
+imports:
+  - https://get.silverbullet.md/global.plug.json
 syntax:
   DeadlineDate:
     firstCharacters:

server/http_server.ts

@@ -107,7 +107,7 @@ export class HttpServer {
       spaceSyscalls(this.space),
       eventSyscalls(this.eventHook),
       markdownSyscalls(buildMarkdown([])),
-      esbuildSyscalls(),
+      esbuildSyscalls([this.globalModules]),
       systemSyscalls(this),
       sandboxSyscalls(this.system),
       assetSyscalls(this.system),

web/index.html

@@ -17,6 +17,9 @@
             // return undefined;
           },
         },
+        errors: {
+          AlreadyExists: class extends Error {},
+        }
       };
     </script>
     <style>