Robustness and federation sync

plugs/federation/config.ts

@@ -0,0 +1,36 @@
+import { readSetting } from "$sb/lib/settings_page.ts";
+
+type FederationConfig = {
+  uri: string;
+  perm?: "ro" | "rw";
+  // TODO: alias?: string;
+};
+
+let federationConfigs: FederationConfig[] = [];
+let lastFederationUrlFetch = 0;
+
+export async function readFederationConfigs(): Promise<FederationConfig[]> {
+  // Update at most every 5 seconds
+  if (Date.now() > lastFederationUrlFetch + 5000) {
+    federationConfigs = await readSetting("federate", []);
+    if (!Array.isArray(federationConfigs)) {
+      console.error("'federate' setting should be an array of objects");
+      return [];
+    }
+    // Normalize URIs
+    for (const config of federationConfigs) {
+      if (!config.uri) {
+        console.error(
+          "'federate' setting should be an array of objects with at least an 'uri' property",
+          config,
+        );
+        continue;
+      }
+      if (!config.uri.startsWith("!")) {
+        config.uri = `!${config.uri}`;
+      }
+    }
+    lastFederationUrlFetch = Date.now();
+  }
+  return federationConfigs;
+}

plugs/federation/federation.plug.yaml

@@ -2,11 +2,11 @@ name: federation
 requiredPermissions:
   - fetch
 functions:
-  #listFiles:
-  #  path: ./federation.ts:listFiles
-  #  pageNamespace:
-  #    pattern: "!.+"
-  #    operation: listFiles
+  listFiles:
+   path: ./federation.ts:listFiles
+   pageNamespace:
+     pattern: "!.+"
+     operation: listFiles
   readFile:
     path: ./federation.ts:readFile
     pageNamespace:

plugs/federation/federation.ts

@@ -1,90 +1,115 @@
 import "$sb/lib/fetch.ts";
 import type { FileMeta } from "../../common/types.ts";
-import { readSetting } from "$sb/lib/settings_page.ts";
-
-function resolveFederated(pageName: string): string {
-  // URL without the prefix "!""
-  let url = pageName.substring(1);
-  if (!url.startsWith("127.0.0.1") && !url.startsWith("localhost")) {
-    url = `https://${url}`;
-  } else {
-    url = `http://${url}`;
-  }
-  return url;
-}
+import { federatedPathToUrl } from "$sb/lib/resolve.ts";
+import { readFederationConfigs } from "./config.ts";
+import { store } from "$sb/plugos-syscall/mod.ts";
 
 async function responseToFileMeta(
   r: Response,
   name: string,
 ): Promise<FileMeta> {
-  // const perm = r.headers.get("X-Permission") as any || "ro";
-  // const federationConfigs = await readFederationConfigs();
-  // const federationConfig = federationConfigs.find((config) =>
-  //   name.startsWith(config.uri)
-  // );
-  // if (federationConfig?.perm) {
-  //   perm = federationConfig.perm;
-  // }
+  const federationConfigs = await readFederationConfigs();
+
+  // Default permission is "ro" unless explicitly set otherwise
+  let perm: "ro" | "rw" = "ro";
+  const federationConfig = federationConfigs.find((config) =>
+    name.startsWith(config.uri)
+  );
+  if (federationConfig?.perm) {
+    perm = federationConfig.perm;
+  }
   return {
     name: name,
     size: r.headers.get("Content-length")
       ? +r.headers.get("Content-length")!
       : 0,
     contentType: r.headers.get("Content-type")!,
-    perm: "ro",
+    perm,
     lastModified: +(r.headers.get("X-Last-Modified") || "0"),
   };
 }
 
-type FederationConfig = {
-  uri: string;
-  // perm?: "ro" | "rw";
-};
-let federationConfigs: FederationConfig[] = [];
-let lastFederationUrlFetch = 0;
+const fileListingPrefixCacheKey = `federationListCache:`;
+const listingCacheTimeout = 1000 * 30;
 
-async function readFederationConfigs() {
-  // Update at most every 5 seconds
-  if (Date.now() > lastFederationUrlFetch + 5000) {
-    federationConfigs = await readSetting("federate", []);
-    // Normalize URIs
-    for (const config of federationConfigs) {
-      if (!config.uri.startsWith("!")) {
-        config.uri = `!${config.uri}`;
-      }
-    }
-    lastFederationUrlFetch = Date.now();
-  }
-  return federationConfigs;
-}
+type FileListingCacheEntry = {
+  items: FileMeta[];
+  lastUpdated: number;
+};
 
 export async function listFiles(): Promise<FileMeta[]> {
   let fileMetas: FileMeta[] = [];
   // Fetch them all in parallel
-  await Promise.all((await readFederationConfigs()).map(async (config) => {
-    // console.log("Fetching from federated", config);
-    const uriParts = config.uri.split("/");
-    const rootUri = uriParts[0];
-    const prefix = uriParts.slice(1).join("/");
-    const r = await nativeFetch(resolveFederated(rootUri));
-    fileMetas = fileMetas.concat(
-      (await r.json()).filter((meta: FileMeta) => meta.name.startsWith(prefix))
-        .map((meta: FileMeta) => ({
+  try {
+    await Promise.all((await readFederationConfigs()).map(async (config) => {
+      const cachedListing = await store.get(
+        `${fileListingPrefixCacheKey}${config.uri}`,
+      ) as FileListingCacheEntry;
+      if (
+        cachedListing &&
+        cachedListing.lastUpdated > Date.now() - listingCacheTimeout
+      ) {
+        fileMetas = fileMetas.concat(cachedListing.items);
+        return;
+      }
+      console.log("Fetching from federated", config);
+      const uriParts = config.uri.split("/");
+      const rootUri = uriParts[0];
+      const prefix = uriParts.slice(1).join("/");
+      const indexUrl = `${federatedPathToUrl(rootUri)}/index.json`;
+      try {
+        const r = await nativeFetch(indexUrl, {
+          method: "GET",
+          headers: {
+            Accept: "application/json",
+          },
+        });
+        if (r.status !== 200) {
+          console.error(
+            `Failed to fetch ${indexUrl}. Skipping.`,
+            r.status,
+            r.statusText,
+          );
+          if (cachedListing) {
+            console.info("Using cached listing");
+            fileMetas = fileMetas.concat(cachedListing.items);
+          }
+          return;
+        }
+        const jsonResult = await r.json();
+        const items: FileMeta[] = jsonResult.filter((meta: FileMeta) =>
+          meta.name.startsWith(prefix)
+        ).map((meta: FileMeta) => ({
           ...meta,
-          perm: "ro", //config.perm || meta.perm,
+          perm: config.perm || "ro",
           name: `${rootUri}/${meta.name}`,
-        })),
-    );
-  }));
-  // console.log("All of em: ", fileMetas);
-  return fileMetas;
+        }));
+        await store.set(`${fileListingPrefixCacheKey}${config.uri}`, {
+          items,
+          lastUpdated: Date.now(),
+        } as FileListingCacheEntry);
+        fileMetas = fileMetas.concat(items);
+      } catch (e: any) {
+        console.error("Failed to process", indexUrl, e);
+      }
+    }));
+
+    // console.log("All of em: ", fileMetas);
+    return fileMetas;
+  } catch (e: any) {
+    console.error("Error listing federation files", e);
+    return [];
+  }
 }
 
 export async function readFile(
   name: string,
 ): Promise<{ data: Uint8Array; meta: FileMeta } | undefined> {
-  const url = resolveFederated(name);
+  const url = federatedPathToUrl(name);
   const r = await nativeFetch(url);
+  if (r.status === 503) {
+    throw new Error("Offline");
+  }
   const fileMeta = await responseToFileMeta(r, name);
   console.log("Fetching", url);
   if (r.status === 404) {
@@ -151,9 +176,12 @@ export async function deleteFile(
 }
 
 export async function getFileMeta(name: string): Promise<FileMeta> {
-  const url = resolveFederated(name);
+  const url = federatedPathToUrl(name);
   console.log("Fetching federation file meta", url);
   const r = await nativeFetch(url, { method: "HEAD" });
+  if (r.status === 503) {
+    throw new Error("Offline");
+  }
   const fileMeta = await responseToFileMeta(r, name);
   if (!r.ok) {
     throw new Error("Not found");