Fixes #409

cmd/server.ts

@@ -43,37 +43,33 @@ To allow outside connections, pass -L 0.0.0.0 as a flag, and put a TLS terminato
   }
   let spacePrimitives: SpacePrimitives | undefined;
   if (folder === "s3://") {
-    spacePrimitives = new AssetBundlePlugSpacePrimitives(
+    spacePrimitives = new S3SpacePrimitives({
-      new S3SpacePrimitives({
+      accessKey: Deno.env.get("AWS_ACCESS_KEY_ID")!,
-        accessKey: Deno.env.get("AWS_ACCESS_KEY_ID")!,
+      secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!,
-        secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!,
+      endPoint: Deno.env.get("AWS_ENDPOINT")!,
-        endPoint: Deno.env.get("AWS_ENDPOINT")!,
+      region: Deno.env.get("AWS_REGION")!,
-        region: Deno.env.get("AWS_REGION")!,
+      bucket: Deno.env.get("AWS_BUCKET")!,
-        bucket: Deno.env.get("AWS_BUCKET")!,
+    });
-      }),
-      new AssetBundle(plugAssetBundle as AssetJson),
-    );
     console.log("Running in S3 mode");
   } else {
+    // Regular disk mode
     folder = path.resolve(Deno.cwd(), folder);
-    spacePrimitives = new AssetBundlePlugSpacePrimitives(
+    spacePrimitives = new DiskSpacePrimitives(folder);
-      new DiskSpacePrimitives(folder, {
-        maxFileSizeMB: options.maxFileSizeMB,
-      }),
-      new AssetBundle(plugAssetBundle as AssetJson),
-    );
   }
-  console.log("Serving pages from", folder);
+  spacePrimitives = new AssetBundlePlugSpacePrimitives(
+    spacePrimitives,
+    new AssetBundle(plugAssetBundle as AssetJson),
+  );
 
-  const httpServer = new HttpServer(spacePrimitives, {
+  const httpServer = new HttpServer(spacePrimitives!, {
     hostname,
     port: port,
-    pagesPath: folder,
+    pagesPath: folder!,
     clientAssetBundle: new AssetBundle(clientAssetBundle as AssetJson),
     user: options.user ?? Deno.env.get("SB_USER"),
     keyFile: options.key,
     certFile: options.cert,
     maxFileSizeMB: +maxFileSizeMB,
   });
-  httpServer.start().catch(console.error);
+  return httpServer.start();
 }

common/deps.ts

@@ -124,3 +124,5 @@ export {
 } from "https://esm.sh/@codemirror/lang-javascript@6.1.8?external=@codemirror/language,@codemirror/autocomplete,@codemirror/view,@codemirror/state,@codemirror/lint,@lezer/common,@lezer/lr,@lezer/javascript,@codemirror/commands";
 
 export { mime } from "https://deno.land/x/mimetypes@v1.0.0/mod.ts";
+
+export { compile as gitIgnoreCompiler } from "https://esm.sh/gitignore-parser@0.0.2";

common/spaces/disk_space_primitives.ts

@@ -16,14 +16,10 @@ function normalizeForwardSlashPath(path: string) {
 
 const excludedFiles = ["data.db", "data.db-journal", "sync.json"];
 
-export type DiskSpaceOptions = {
-  maxFileSizeMB?: number;
-};
-
 export class DiskSpacePrimitives implements SpacePrimitives {
   rootPath: string;
 
-  constructor(rootPath: string, private options: DiskSpaceOptions = {}) {
+  constructor(rootPath: string) {
     this.rootPath = Deno.realPathSync(rootPath);
   }
 
@@ -150,13 +146,6 @@ export class DiskSpacePrimitives implements SpacePrimitives {
       const fullPath = file.path;
       try {
         const s = await Deno.stat(fullPath);
-        // Don't list file exceeding the maximum file size
-        if (
-          this.options.maxFileSizeMB &&
-          s.size / (1024 * 1024) > this.options.maxFileSizeMB
-        ) {
-          continue;
-        }
         const name = fullPath.substring(this.rootPath.length + 1);
         if (excludedFiles.includes(name)) {
           continue;

common/spaces/filtered_space_primitives.ts

@@ -0,0 +1,35 @@
+import { FileMeta } from "../types.ts";
+import { SpacePrimitives } from "./space_primitives.ts";
+
+export class FilteredSpacePrimitives implements SpacePrimitives {
+  constructor(
+    private wrapped: SpacePrimitives,
+    private filterFn: (name: FileMeta) => boolean,
+    private onFetchList?: () => Promise<void>,
+  ) {
+  }
+
+  async fetchFileList(): Promise<FileMeta[]> {
+    if (this.onFetchList) {
+      await this.onFetchList();
+    }
+    return (await this.wrapped.fetchFileList()).filter(this.filterFn);
+  }
+  readFile(name: string): Promise<{ data: Uint8Array; meta: FileMeta }> {
+    return this.wrapped.readFile(name);
+  }
+  getFileMeta(name: string): Promise<FileMeta> {
+    return this.wrapped.getFileMeta(name);
+  }
+  writeFile(
+    name: string,
+    data: Uint8Array,
+    selfUpdate?: boolean | undefined,
+    lastModified?: number | undefined,
+  ): Promise<FileMeta> {
+    return this.wrapped.writeFile(name, data, selfUpdate, lastModified);
+  }
+  deleteFile(name: string): Promise<void> {
+    return this.wrapped.deleteFile(name);
+  }
+}

common/util.ts

@@ -36,14 +36,18 @@ export function parseYamlSettings(settingsMarkdown: string): {
 export async function ensureSettingsAndIndex(
   space: SpacePrimitives,
 ): Promise<any> {
+  let settingsText: string | undefined;
   try {
-    await space.getFileMeta("SETTINGS.md");
+    settingsText = new TextDecoder().decode(
+      (await space.readFile("SETTINGS.md")).data,
+    );
   } catch {
     await space.writeFile(
       "SETTINGS.md",
       new TextEncoder().encode(SETTINGS_TEMPLATE),
       true,
     );
+    settingsText = SETTINGS_TEMPLATE;
     // Ok, then let's also write the index page
     try {
       await space.getFileMeta("index.md");
@@ -60,4 +64,6 @@ Loading some onboarding content for you (but doing so does require a working int
       );
     }
   }
+
+  return parseYamlSettings(settingsText);
 }

server/http_server.ts

@@ -4,6 +4,9 @@ import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
 import { base64Decode } from "../plugos/asset_bundle/base64.ts";
 import { ensureSettingsAndIndex } from "../common/util.ts";
 import { performLocalFetch } from "../common/proxy_fetch.ts";
+import { BuiltinSettings } from "../web/types.ts";
+import { gitIgnoreCompiler } from "./deps.ts";
+import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts";
 
 export type ServerOptions = {
   hostname: string;
@@ -22,12 +25,13 @@ export class HttpServer {
   private hostname: string;
   private port: number;
   user?: string;
-  settings: { [key: string]: any } = {};
   abortController?: AbortController;
   clientAssetBundle: AssetBundle;
+  settings?: BuiltinSettings;
+  spacePrimitives: SpacePrimitives;
 
   constructor(
-    private spacePrimitives: SpacePrimitives,
+    spacePrimitives: SpacePrimitives,
     private options: ServerOptions,
   ) {
     this.hostname = options.hostname;
@@ -35,6 +39,29 @@ export class HttpServer {
     this.app = new Application();
     this.user = options.user;
     this.clientAssetBundle = options.clientAssetBundle;
+
+    let fileFilterFn: (s: string) => boolean = () => true;
+    this.spacePrimitives = new FilteredSpacePrimitives(
+      spacePrimitives,
+      (meta) => {
+        // Don't list file exceeding the maximum file size
+        if (
+          options.maxFileSizeMB &&
+          meta.size / (1024 * 1024) > options.maxFileSizeMB
+        ) {
+          return false;
+        }
+        return fileFilterFn(meta.name);
+      },
+      async () => {
+        await this.reloadSettings();
+        if (typeof this.settings?.spaceIgnore === "string") {
+          fileFilterFn = gitIgnoreCompiler(this.settings.spaceIgnore).accepts;
+        } else {
+          fileFilterFn = () => true;
+        }
+      },
+    );
   }
 
   // Replaces some template variables in index.html in a rather ad-hoc manner, but YOLO
@@ -50,8 +77,7 @@ export class HttpServer {
   }
 
   async start() {
-    await ensureSettingsAndIndex(this.spacePrimitives);
+    await this.reloadSettings();
-
     // Serve static files (javascript, css, html)
     this.app.use(async ({ request, response }, next) => {
       if (request.url.pathname === "/") {
@@ -137,6 +163,11 @@ export class HttpServer {
     );
   }
 
+  async reloadSettings() {
+    // TODO: Throttle this?
+    this.settings = await ensureSettingsAndIndex(this.spacePrimitives);
+  }
+
   private addPasswordAuth(app: Application) {
     const excludedPaths = [
       "/manifest.json",

web/editor.tsx

@@ -15,6 +15,7 @@ import {
   EditorSelection,
   EditorState,
   EditorView,
+  gitIgnoreCompiler,
   highlightSpecialChars,
   history,
   historyKeymap,
@@ -134,6 +135,8 @@ import { SyncStatus } from "../common/spaces/sync.ts";
 import { HttpSpacePrimitives } from "../common/spaces/http_space_primitives.ts";
 import { FallbackSpacePrimitives } from "../common/spaces/fallback_space_primitives.ts";
 import { syncSyscalls } from "./syscalls/sync.ts";
+import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts";
+import { globToRegExp } from "https://deno.land/std@0.189.0/path/glob.ts";
 
 const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/;
 
@@ -247,12 +250,24 @@ export class Editor {
       namespaceHook,
     );
 
-    const localSpacePrimitives = new FileMetaSpacePrimitives(
+    let fileFilterFn: (s: string) => boolean = () => true;
-      new EventedSpacePrimitives(
+    const localSpacePrimitives = new FilteredSpacePrimitives(
-        plugSpacePrimitives,
+      new FileMetaSpacePrimitives(
-        this.eventHook,
+        new EventedSpacePrimitives(
+          plugSpacePrimitives,
+          this.eventHook,
+        ),
+        indexSyscalls,
       ),
-      indexSyscalls,
+      (meta) => fileFilterFn(meta.name),
+      async () => {
+        await this.loadSettings();
+        if (typeof this.settings?.spaceIgnore === "string") {
+          fileFilterFn = gitIgnoreCompiler(this.settings.spaceIgnore).accepts;
+        } else {
+          fileFilterFn = () => true;
+        }
+      },
     );
 
     this.space = new Space(localSpacePrimitives);
@@ -458,6 +473,7 @@ export class Editor {
     this.syncService.start();
 
     this.eventHook.addLocalListener("sync:success", async (operations) => {
+      // console.log("Operations", operations);
       if (operations > 0) {
         // Update the page list
         await this.space.updatePageList();

web/types.ts

@@ -33,6 +33,8 @@ export type PanelMode = number;
 
 export type BuiltinSettings = {
   indexPage: string;
+  // Format: compatible with docker ignore
+  spaceIgnore?: string;
 };
 
 export type PanelConfig = {

website/CHANGELOG.md

@@ -1,6 +1,11 @@
 An attempt at documenting the changes/new features introduced in each
 release.
 
+## Next
+
+* Added `spaceIgnore` setting to not sync specific folders or file patterns to the client, see [[SETTINGS]] for documentation
+
+
 ---
 
 ## 0.3.1

website/SETTINGS.md

@@ -20,9 +20,9 @@ weeklyNoteMonday: false
 # Markdown
 previewOnRHS: true
 
-# Sync
+# Defines files to ignore in a format compatible with .gitignore
-sync:
+spaceIgnore: |
-  # Do not sync pages with a specific prefix
+   dist
-  excludePrefixes:
+   largefolder
-  - PLUGS
+   *.mp4
 ```