Refactoring about how proxy fetching happens

plug-api/lib/attribute.ts

@@ -6,11 +6,6 @@ import {
 
 import { YAML } from "$sb/plugos-syscall/mod.ts";
 
-export type Attribute = {
-  name: string;
-  value: string;
-};
-
 /**
  * Extracts attributes from a tree, optionally cleaning them out of the tree.
  * @param tree tree to extract attributes from

plug-api/lib/feed.test.ts

@@ -0,0 +1,49 @@
+import "$sb/lib/syscall_mock.ts";
+import { parse } from "../../common/markdown_parser/parse_tree.ts";
+import buildMarkdown from "../../common/markdown_parser/parser.ts";
+import { assertEquals } from "../../test_deps.ts";
+import { extractFeedItems } from "$sb/lib/feed.ts";
+import { nodeDefToMDExt } from "../../common/markdown_parser/markdown_ext.ts";
+
+const feedSample1 = `---
+test: ignore me
+---
+# My first item
+$myid
+Some text
+
+---
+
+# My second item
+[id: myid2][otherAttribute: 42]
+And some text
+
+---
+
+Completely free form
+`;
+
+Deno.test("Test feed parsing", async () => {
+  // Ad hoc added the NamedAnchor extension from the core plug-in inline here
+  const lang = buildMarkdown([nodeDefToMDExt("NamedAnchor", {
+    firstCharacters: ["$"],
+    regex: "\\$[a-zA-Z\\.\\-\\/]+[\\w\\.\\-\\/]*",
+  })]);
+  const tree = parse(lang, feedSample1);
+  const items = await extractFeedItems(tree);
+  assertEquals(items.length, 3);
+  assertEquals(items[0], {
+    id: "myid",
+    text: "Some text",
+    title: "My first item",
+  });
+  assertEquals(items[1], {
+    id: "myid2",
+    attributes: {
+      otherAttribute: 42,
+    },
+    title: "My second item",
+    text: "And some text",
+  });
+  assertEquals(items[2].text, "Completely free form");
+});

plug-api/lib/feed.ts

@@ -0,0 +1,103 @@
+import {
+  findNodeMatching,
+  findNodeOfType,
+  ParseTree,
+  renderToText,
+} from "$sb/lib/tree.ts";
+
+import { extractAttributes } from "$sb/lib/attribute.ts";
+
+export type FeedItem = {
+  id: string;
+  title?: string;
+  attributes?: Record<string, any>;
+  text: string;
+};
+
+// tree = Document node
+export async function extractFeedItems(tree: ParseTree): Promise<FeedItem[]> {
+  let nodes: ParseTree[] = [];
+  const feedItems: FeedItem[] = [];
+  if (tree.type !== "Document") {
+    throw new Error("Did not get a document");
+  }
+  // Run through the whole document to find the feed items
+  for (const node of tree.children!) {
+    if (node.type === "FrontMatter") {
+      // Not interested
+      console.log("Ignoring", node);
+      continue;
+    }
+    if (node.type === "HorizontalRule") {
+      // Ok we reached the end of a feed item
+      feedItems.push(await nodesToFeedItem(nodes));
+      nodes = [];
+    } else {
+      nodes.push(node);
+    }
+  }
+  if (renderToText({ children: nodes }).trim().length > 0) {
+    feedItems.push(await nodesToFeedItem(nodes));
+  }
+
+  return feedItems;
+}
+
+async function nodesToFeedItem(nodes: ParseTree[]): Promise<FeedItem> {
+  const wrapperNode: ParseTree = {
+    children: nodes,
+  };
+  const attributes = await extractAttributes(wrapperNode, true);
+  let id = attributes.id;
+  delete attributes.id;
+  if (!id) {
+    const anchor = findNodeOfType(wrapperNode, "NamedAnchor");
+    if (anchor) {
+      id = anchor.children![0].text!.substring(1);
+      if (id.startsWith("id/")) {
+        id = id.substring(3);
+      }
+      // Empty it out
+      anchor.children = [];
+    }
+  }
+
+  // Find a title
+  let title: string | undefined;
+  const titleNode = findNodeMatching(
+    wrapperNode,
+    (node) => !!node.type?.startsWith("ATXHeading"),
+  );
+  if (titleNode) {
+    title = titleNode.children![1].text!.trim();
+    titleNode.children = [];
+  }
+
+  const text = renderToText(wrapperNode).trim();
+
+  if (!id) {
+    // If all else fails, generate content based ID
+    id = `gen/${djb2Hash(JSON.stringify({ attributes, text }))}`;
+  }
+  //   console.log("Extracted attributes", attributes);
+  const feedItem: FeedItem = { id, text };
+  if (title) {
+    feedItem.title = title;
+  }
+  if (Object.keys(attributes).length > 0) {
+    feedItem.attributes = attributes;
+  }
+  return feedItem;
+}
+
+function djb2Hash(input: string): string {
+  let hash = 5381; // Initial hash value
+
+  for (let i = 0; i < input.length; i++) {
+    // Update the hash value by shifting and adding the character code
+    hash = (hash * 33) ^ input.charCodeAt(i);
+  }
+
+  // Convert the hash to a hexadecimal string representation
+  return hash.toString(16);
+}

plug-api/plugos-syscall/fetch.ts

@@ -1,15 +1,33 @@
+import { init } from "https://esm.sh/v131/node_events.js";
 import type {
   ProxyFetchRequest,
   ProxyFetchResponse,
 } from "../../common/proxy_fetch.ts";
-import { base64Decode } from "../../plugos/asset_bundle/base64.ts";
+import {
+  base64Decode,
+  base64Encode,
+} from "../../plugos/asset_bundle/base64.ts";
 
-export function sandboxFetch(
-  url: string,
+export async function sandboxFetch(
+  reqInfo: RequestInfo,
   options?: ProxyFetchRequest,
 ): Promise<ProxyFetchResponse> {
+  if (typeof reqInfo !== "string") {
+    // Request as first argument, let's deconstruct it
+    // console.log("fetch", reqInfo);
+    options = {
+      method: reqInfo.method,
+      headers: Object.fromEntries(reqInfo.headers.entries()),
+      base64Body: reqInfo.body
+        ? base64Encode(
+          new Uint8Array(await (new Response(reqInfo.body)).arrayBuffer()),
+        )
+        : undefined,
+    };
+    reqInfo = reqInfo.url;
+  }
   // @ts-ignore: monkey patching fetch
-  return syscall("sandboxFetch.fetch", url, options);
+  return syscall("sandboxFetch.fetch", reqInfo, options);
 }
 
 export function monkeyPatchFetch() {
@@ -17,15 +35,19 @@ export function monkeyPatchFetch() {
   globalThis.nativeFetch = globalThis.fetch;
   // @ts-ignore: monkey patching fetch
   globalThis.fetch = async function (
-    url: string,
+    reqInfo: RequestInfo,
     init?: RequestInit,
   ): Promise<Response> {
     const r = await sandboxFetch(
-      url,
+      reqInfo,
       init && {
         method: init.method,
         headers: init.headers as Record<string, string>,
-        body: init.body as string,
+        base64Body: init.body
+          ? base64Encode(
+            new Uint8Array(await (new Response(init.body)).arrayBuffer()),
+          )
+          : undefined,
       },
     );
     return new Response(r.base64Body ? base64Decode(r.base64Body) : null, {