Fixes #100 implements a custom Markdown renderer

common/markdown_ext.ts

@@ -48,11 +48,11 @@ export function mdExtensionStyleTags({ nodeType, tag }: MDExt): {
 }
 
 export function loadMarkdownExtensions(system: System<any>): MDExt[] {
-  let mdExtensions: MDExt[] = [];
+  const mdExtensions: MDExt[] = [];
-  for (let plug of system.loadedPlugs.values()) {
+  for (const plug of system.loadedPlugs.values()) {
-    let manifest = plug.manifest as Manifest;
+    const manifest = plug.manifest as Manifest;
     if (manifest.syntax) {
-      for (let [nodeType, def] of Object.entries(manifest.syntax)) {
+      for (const [nodeType, def] of Object.entries(manifest.syntax)) {
         mdExtensions.push({
           nodeType,
           tag: Tag.define(),

common/parse_tree.ts

@@ -1,7 +1,5 @@
-import { ParseTree } from "$sb/lib/tree.ts";
+import type { ParseTree } from "$sb/lib/tree.ts";
-
+import type { Language, SyntaxNode } from "./deps.ts";
-import type { SyntaxNode } from "./deps.ts";
-import type { Language } from "./deps.ts";
 
 export function lezerToParseTree(
   text: string,

plug-api/app_event.ts

@@ -1,5 +1,5 @@
-import type { ParseTree } from "./lib/tree.ts";
+import type { ParseTree } from "$sb/lib/tree.ts";
-import { ParsedQuery } from "./lib/query.ts";
+import { ParsedQuery } from "$sb/lib/query.ts";
 
 export type AppEvent =
   | "page:click"

plug-api/silverbullet-syscall/editor.ts

@@ -87,8 +87,8 @@ export function replaceRange(
   return syscall("editor.replaceRange", from, to, text);
 }
 
-export function moveCursor(pos: number): Promise<void> {
+export function moveCursor(pos: number, center = false): Promise<void> {
-  return syscall("editor.moveCursor", pos);
+  return syscall("editor.moveCursor", pos, center);
 }
 
 export function insertAtCursor(text: string): Promise<void> {

plug-api/silverbullet-syscall/markdown.ts

@@ -1,4 +1,4 @@
-import { syscall } from "./syscall.ts";
+import { syscall } from "$sb/silverbullet-syscall/syscall.ts";
 
 import type { ParseTree } from "$sb/lib/tree.ts";
 

plugs/directive/command.ts

@@ -1,13 +1,11 @@
-import { editor, space } from "$sb/silverbullet-syscall/mod.ts";
+import { editor, space, system } from "$sb/silverbullet-syscall/mod.ts";
-import { invokeFunction } from "$sb/silverbullet-syscall/system.ts";
-
 import { renderDirectives } from "./directives.ts";
 
 export async function updateDirectivesOnPageCommand() {
   const currentPage = await editor.getCurrentPage();
   await editor.save();
   if (
-    await invokeFunction(
+    await system.invokeFunction(
       "server",
       "updateDirectivesOnPage",
       currentPage,

plugs/directive/eval_directive.ts

@@ -1,6 +1,6 @@
 // This is some shocking stuff. My profession would kill me for this.
 
-import { YAML } from "../../common/deps.ts";
+import * as YAML from "yaml";
 import { jsonToMDTable, renderTemplate } from "./util.ts";
 
 // Enables plugName.functionName(arg1, arg2) syntax in JS expressions

plugs/markdown/assets/handler.js

@@ -0,0 +1,9 @@
+document.getElementById("root").addEventListener("click", (e) => {
+  // console.log("Got click", e.target)
+  const dataSet = e.target.dataset;
+  if(dataSet["onclick"]) {
+    sendEvent("preview:click", dataSet["onclick"]);
+  } else if(dataSet["pos"]) {
+    sendEvent("preview:click", JSON.stringify(["pos", dataSet["pos"]]));
+  }
+})

plugs/markdown/assets/styles.css

@@ -8,11 +8,24 @@ body {
   padding-right: 20px;
 }
 
+table.front-matter {
+  border: 1px solid #555;
+  font-size: 75%;
+}
+
+table.front-matter .key {
+  font-weight: bold;
+}
+
 table {
   width: 100%;
   border-spacing: 0;
 }
 
+ul li p {
+  margin: 0;
+}
+
 thead tr {
     background-color: #333;
     color: #eee;
@@ -48,3 +61,7 @@ hr:after {
     content: "···";
     letter-spacing: 1em;
 }
+
+span.highlight {
+  background-color: yellow;
+}

plugs/markdown/html_render.test.ts

@@ -0,0 +1,29 @@
+import { assertEquals } from "https://deno.land/std@0.152.0/testing/asserts.ts";
+import { renderHtml } from "./html_render.ts";
+
+Deno.test("HTML Render", () => {
+  assertEquals(
+    renderHtml({
+      name: "b",
+      body: "hello",
+    }),
+    `<b>hello</b>`,
+  );
+  assertEquals(
+    renderHtml({
+      name: "a",
+      attrs: {
+        href: "https://example.com",
+      },
+      body: "hello",
+    }),
+    `<a href="https://example.com">hello</a>`,
+  );
+  assertEquals(
+    renderHtml({
+      name: "span",
+      body: "<>",
+    }),
+    `<span>&lt;&gt;</span>`,
+  );
+});

plugs/markdown/html_render.ts

@@ -0,0 +1,41 @@
+export const Fragment = "FRAGMENT";
+
+export type Tag = {
+  name: string;
+  attrs?: Record<string, string | undefined>;
+  body: Tag[] | string;
+} | string;
+
+function htmlEscape(s: string): string {
+  return s.replace(/&/g, "&amp;")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;");
+}
+
+export function renderHtml(t: Tag | null): string {
+  if (!t) {
+    return "";
+  }
+  if (typeof t === "string") {
+    return htmlEscape(t);
+  }
+  const attrs = t.attrs
+    ? " " + Object.entries(t.attrs)
+      .filter(([, value]) => value !== undefined)
+      .map(([k, v]) => `${k}="${htmlEscape(v!)}"`).join(
+        " ",
+      )
+    : "";
+  const body = typeof t.body === "string"
+    ? htmlEscape(t.body)
+    : t.body.map(renderHtml).join("");
+  if (t.name === Fragment) {
+    return body;
+  }
+  if (t.body) {
+    return `<${t.name}${attrs}>${body}</${t.name}>`;
+  } else {
+    return `<${t.name}${attrs}/>`;
+  }
+}

plugs/markdown/markdown.plug.yaml

@@ -2,7 +2,7 @@ name: markdown
 imports:
   - https://get.silverbullet.md/global.plug.json
 assets:
-  - "*.css"
+  - "assets/*"
 functions:
   toggle:
     path: "./markdown.ts:togglePreview"
@@ -18,3 +18,8 @@ functions:
       - editor:updated
       - editor:pageLoaded
       - editor:pageReloaded
+  previewClickHandler:
+    path: "./preview.ts:previewClickHandler"
+    env: client
+    events:
+      - preview:click

plugs/markdown/markdown_render.test.ts

@@ -0,0 +1,46 @@
+import buildMarkdown from "../../common/parser.ts";
+import { parse } from "../../common/parse_tree.ts";
+import { renderHtml } from "./html_render.ts";
+import { System } from "../../plugos/system.ts";
+
+import corePlug from "../../dist_bundle/_plug/core.plug.json" assert {
+  type: "json",
+};
+import tasksPlug from "../../dist_bundle/_plug/tasks.plug.json" assert {
+  type: "json",
+};
+import { createSandbox } from "../../plugos/environments/deno_sandbox.ts";
+import { loadMarkdownExtensions } from "../../common/markdown_ext.ts";
+import { renderMarkdownToHtml } from "./markdown_render.ts";
+import { assertEquals } from "https://deno.land/std@0.152.0/testing/asserts.ts";
+
+Deno.test("Markdown render", async () => {
+  const system = new System<any>("server");
+  await system.load(corePlug, createSandbox);
+  await system.load(tasksPlug, createSandbox);
+  const lang = buildMarkdown(loadMarkdownExtensions(system));
+  const testFile = Deno.readTextFileSync(
+    new URL("test/example.md", import.meta.url).pathname,
+  );
+  const tree = parse(lang, testFile);
+  renderMarkdownToHtml(tree, {
+    failOnUnknown: true,
+    renderFrontMatter: true,
+  });
+  // console.log("HTML", html);
+});
+
+Deno.test("Smart hard break test", () => {
+  const example = `**Hello**
+*world!*`;
+  const lang = buildMarkdown([]);
+  const tree = parse(lang, example);
+  const html = renderMarkdownToHtml(tree, {
+    failOnUnknown: true,
+    smartHardBreak: true,
+  });
+  assertEquals(
+    html,
+    `<p><strong>Hello</strong><br/><em>world!</em></p>`,
+  );
+});

plugs/markdown/markdown_render.ts

@@ -0,0 +1,350 @@
+import {
+  findNodeOfType,
+  ParseTree,
+  renderToText,
+  traverseTree,
+} from "$sb/lib/tree.ts";
+import * as YAML from "yaml";
+import { Fragment, renderHtml, Tag } from "./html_render.ts";
+
+type MarkdownRenderOptions = {
+  failOnUnknown?: true;
+  smartHardBreak?: true;
+  annotationPositions?: true;
+  renderFrontMatter?: true;
+};
+
+function cleanTags(values: (Tag | null)[]): Tag[] {
+  const result: Tag[] = [];
+  for (const value of values) {
+    if (value) {
+      result.push(value);
+    }
+  }
+  return result;
+}
+
+function preprocess(t: ParseTree, options: MarkdownRenderOptions = {}) {
+  traverseTree(t, (node) => {
+    if (node.type === "Paragraph" && options.smartHardBreak) {
+      for (const child of node.children!) {
+        // If at the paragraph level there's a newline, let's turn it into a hard break
+        if (!child.type && child.text === "\n") {
+          child.type = "HardBreak";
+        }
+      }
+    }
+    return false;
+  });
+}
+
+function posPreservingRender(
+  t: ParseTree,
+  options: MarkdownRenderOptions = {},
+): Tag | null {
+  const tag = render(t, options);
+  if (!options.annotationPositions) {
+    return tag;
+  }
+  if (!tag) {
+    return null;
+  }
+  if (typeof tag === "string") {
+    return tag;
+  }
+  if (t.from) {
+    if (!tag.attrs) {
+      tag.attrs = {};
+    }
+    tag.attrs["data-pos"] = "" + t.from;
+  }
+  return tag;
+}
+
+function render(
+  t: ParseTree,
+  options: MarkdownRenderOptions = {},
+): Tag | null {
+  if (t.type?.endsWith("Mark") || t.type?.endsWith("Delimiter")) {
+    return null;
+  }
+  switch (t.type) {
+    case "Document":
+      return {
+        name: Fragment,
+        body: cleanTags(mapRender(t.children!)),
+      };
+    case "FrontMatter":
+      if (options.renderFrontMatter) {
+        const yamlCode = renderToText(t.children![1]);
+        const parsedYaml = YAML.parse(yamlCode) as Record<string, any>;
+        const rows: Tag[] = [];
+        for (const [k, v] of Object.entries(parsedYaml)) {
+          rows.push({
+            name: "tr",
+            body: [
+              { name: "td", attrs: { class: "key" }, body: k },
+              {
+                name: "td",
+                attrs: { class: "value" },
+                body: YAML.stringify(v),
+              },
+            ],
+          });
+        }
+        return {
+          name: "table",
+          attrs: {
+            class: "front-matter",
+          },
+          body: rows,
+        };
+      } else {
+        return null;
+      }
+    case "CommentBlock":
+      // Remove, for now
+      return null;
+    case "ATXHeading1":
+      return {
+        name: "h1",
+        body: cleanTags(mapRender(t.children!)),
+      };
+    case "ATXHeading2":
+      return {
+        name: "h2",
+        body: cleanTags(mapRender(t.children!)),
+      };
+    case "ATXHeading3":
+      return {
+        name: "h3",
+        body: cleanTags(mapRender(t.children!)),
+      };
+    case "ATXHeading4":
+      return {
+        name: "h4",
+        body: cleanTags(mapRender(t.children!)),
+      };
+    case "ATXHeading5":
+      return {
+        name: "h5",
+        body: cleanTags(mapRender(t.children!)),
+      };
+    case "Paragraph":
+      return {
+        name: "p",
+        body: cleanTags(mapRender(t.children!)),
+      };
+    // Code blocks
+    case "FencedCode":
+    case "CodeBlock": {
+      return {
+        name: "pre",
+        body: cleanTags(mapRender(t.children!)),
+      };
+    }
+    case "CodeText":
+      return t.children![0].text!;
+    case "Blockquote":
+      return {
+        name: "blockquote",
+        body: cleanTags(mapRender(t.children!)),
+      };
+    case "HardBreak":
+      return {
+        name: "br",
+        body: "",
+      };
+    // Basic styling
+    case "Emphasis":
+      return {
+        name: "em",
+        body: cleanTags(mapRender(t.children!)),
+      };
+    case "Highlight":
+      return {
+        name: "span",
+        attrs: {
+          class: "highlight",
+        },
+        body: cleanTags(mapRender(t.children!)),
+      };
+    case "InlineCode":
+      return {
+        name: "tt",
+        body: cleanTags(mapRender(t.children!)),
+      };
+    case "BulletList":
+      return {
+        name: "ul",
+        body: cleanTags(mapRender(t.children!)),
+      };
+    case "OrderedList":
+      return {
+        name: "ol",
+        body: cleanTags(mapRender(t.children!)),
+      };
+    case "ListItem":
+      return {
+        name: "li",
+        body: cleanTags(mapRender(t.children!)),
+      };
+    case "StrongEmphasis":
+      return {
+        name: "strong",
+        body: cleanTags(mapRender(t.children!)),
+      };
+    case "HorizontalRule":
+      return {
+        name: "hr",
+        body: "",
+      };
+    case "Link": {
+      const linkText = t.children![1].text!;
+      const url = findNodeOfType(t, "URL")!.children![0].text!;
+      return {
+        name: "a",
+        attrs: {
+          href: url,
+        },
+        body: linkText,
+      };
+    }
+    case "Image": {
+      const altText = t.children![1].text!;
+      let url = findNodeOfType(t, "URL")!.children![0].text!;
+      if (url.indexOf("://") === -1) {
+        url = `fs/${url}`;
+      }
+      return {
+        name: "img",
+        attrs: {
+          src: url,
+          alt: altText,
+        },
+        body: "",
+      };
+    }
+
+    // Custom stuff
+    case "WikiLink": {
+      // console.log("WikiLink", JSON.stringify(t, null, 2));
+      const ref = findNodeOfType(t, "WikiLinkPage")!.children![0].text!;
+      return {
+        name: "a",
+        attrs: {
+          href: `/${ref}`,
+        },
+        body: ref,
+      };
+    }
+    case "NakedURL": {
+      const url = t.children![0].text!;
+      return {
+        name: "a",
+        attrs: {
+          href: url,
+        },
+        body: url,
+      };
+    }
+    case "Hashtag":
+      return {
+        name: "strong",
+        body: t.children![0].text!,
+      };
+
+    case "Task":
+      return {
+        name: "span",
+        body: cleanTags(mapRender(t.children!)),
+      };
+    case "TaskMarker":
+      return {
+        name: "input",
+        attrs: {
+          type: "checkbox",
+          checked: t.children![0].text !== "[ ]" ? "checked" : undefined,
+          "data-onclick": JSON.stringify(["task", t.to]),
+        },
+        body: "",
+      };
+    case "NamedAnchor":
+      return {
+        name: "a",
+        attrs: {
+          name: t.children![0].text?.substring(1),
+        },
+        body: "",
+      };
+    case "CommandLink": {
+      const commandText = t.children![0].text!.substring(
+        2,
+        t.children![0].text!.length - 2,
+      );
+
+      return {
+        name: "button",
+        attrs: {
+          "data-onclick": JSON.stringify(["command", commandText]),
+        },
+        body: commandText,
+      };
+    }
+
+    case "DeadlineDate":
+      return renderToText(t);
+
+    // Tables
+    case "Table":
+      return {
+        name: "table",
+        body: cleanTags(mapRender(t.children!)),
+      };
+    case "TableHeader":
+      return {
+        name: "thead",
+        body: [
+          {
+            name: "tr",
+            body: cleanTags(mapRender(t.children!)),
+          },
+        ],
+      };
+    case "TableCell":
+      return {
+        name: "td",
+        body: cleanTags(mapRender(t.children!)),
+      };
+    case "TableRow":
+      return {
+        name: "tr",
+        body: cleanTags(mapRender(t.children!)),
+      };
+    // Text
+    case undefined:
+      return t.text!;
+    default:
+      if (options.failOnUnknown) {
+        console.error("Not handling", JSON.stringify(t, null, 2));
+        throw new Error(`Unknown markdown node type ${t.type}`);
+      } else {
+        // Falling back to rendering verbatim
+        console.warn("Not handling", JSON.stringify(t, null, 2));
+        return renderToText(t);
+      }
+  }
+
+  function mapRender(children: ParseTree[]) {
+    return children.map((t) => posPreservingRender(t, options));
+  }
+}
+
+export function renderMarkdownToHtml(
+  t: ParseTree,
+  options: MarkdownRenderOptions = {},
+) {
+  preprocess(t, options);
+  const htmlTree = posPreservingRender(t, options);
+  return renderHtml(htmlTree);
+}

plugs/markdown/preview.ts

@@ -1,28 +1,40 @@
-import MarkdownIt from "https://esm.sh/markdown-it@13.0.1";
+import { clientStore, editor, system } from "$sb/silverbullet-syscall/mod.ts";
-import taskLists from "https://esm.sh/markdown-it-task-lists@2.1.1";
-
-import { clientStore, editor } from "$sb/silverbullet-syscall/mod.ts";
 import { asset } from "$sb/plugos-syscall/mod.ts";
-import { cleanMarkdown } from "./util.ts";
+import { parseMarkdown } from "../../plug-api/silverbullet-syscall/markdown.ts";
-
+import { renderMarkdownToHtml } from "./markdown_render.ts";
-const md = new MarkdownIt({
-  linkify: true,
-  html: false,
-  typographer: true,
-}).use(taskLists);
 
 export async function updateMarkdownPreview() {
   if (!(await clientStore.get("enableMarkdownPreview"))) {
     return;
   }
   const text = await editor.getText();
-  const cleanMd = await cleanMarkdown(text);
+  const mdTree = await parseMarkdown(text);
-  const css = await asset.readAsset("styles.css");
+  // const cleanMd = await cleanMarkdown(text);
+  const css = await asset.readAsset("assets/styles.css");
+  const js = await asset.readAsset("assets/handler.js");
+  const html = renderMarkdownToHtml(mdTree, {
+    smartHardBreak: true,
+    annotationPositions: true,
+    renderFrontMatter: true,
+  });
   await editor.showPanel(
     "rhs",
     2,
-    `<html><head><style>${css}</style></head><body>${
+    `<html><head><style>${css}</style></head><body><div id="root">${html}</div></body></html>`,
-      md.render(cleanMd)
+    js,
-    }</body></html>`,
   );
 }
+
+export async function previewClickHandler(e: any) {
+  const [eventName, arg] = JSON.parse(e);
+  // console.log("Got click", eventName, arg);
+  switch (eventName) {
+    case "pos":
+      // console.log("Moving cursor to", +arg);
+      await editor.moveCursor(+arg, true);
+      break;
+    case "command":
+      await system.invokeCommand(arg);
+      break;
+  }
+}

plugs/markdown/test/example.md

@@ -0,0 +1,82 @@
+---
+name: Sup
+---
+# Hello world
+This is **bold** and _italic_, or *italic*. And a **_mix_**. And ==highlight==!
+
+This is one line
+and this another.
+
+Lists:
+* This
+* Is a 
+* list
+* And here we go nested
+  1. This is a numbered
+  2. Two
+* And different
+  * Bla
+  * More bla
+
+And:
+
+1. Numbered
+2. Two
+
+## Second heading
+
+And some
+
+```
+Code
+bla
+bla
+
+bla
+```
+
+And like this:
+
+    More code
+    Bla
+
+And a blockquote:
+
+> Sup yo
+> Empty line  
+> Second part
+
+<!-- this is a comment -->
+
+And more custom stuff
+[[Page link]]
+
+{[Command button]}
+
+* [ ] #next Task
+* [x] #next Task 2
+* [ ] Task with dealine 📅 2022-05-06 fef
+
+
+
+https://community.mattermost.com
+
+$anchor
+
+[A link](https://silverbullet.md)
+
+## Tables
+
+|type     |actor_login|created_at          |payload_ref           |
+|---------|--------|--------------------|----------------------|
+|PushEvent|avb|2022-10-27T08:27:48Z|refs/heads/master     |
+|PushEvent|avb|2022-10-27T04:31:27Z|refs/heads/jitterSched|
+
+
+Here is something
+
+---
+
+A new thing. 
+
+![alt text](https://image.jpg)

plugs/tasks/task.ts

@@ -88,6 +88,14 @@ export function taskToggle(event: ClickEvent) {
   return taskToggleAtPos(event.pos);
 }
 
+export function previewTaskToggle(eventString: string) {
+  const [eventName, pos] = JSON.parse(eventString);
+  if (eventName === "task") {
+    console.log("Gotta toggle a task at", pos);
+    return taskToggleAtPos(+pos);
+  }
+}
+
 async function toggleTaskMarker(node: ParseTree, moveToPos: number) {
   let changeTo = "[x]";
   if (node.children![0].text === "[x]" || node.children![0].text === "[X]") {
@@ -139,6 +147,7 @@ export async function taskToggleAtPos(pos: number) {
   addParentPointers(mdTree);
 
   const node = nodeAtPos(mdTree, pos);
+  // console.log("Got this node", node?.type);
   if (node && node.type === "TaskMarker") {
     await toggleTaskMarker(node, pos);
   }

plugs/tasks/tasks.plug.yaml

@@ -47,3 +47,8 @@ functions:
       key: Alt-+
       contexts:
         - DeadlineDate
+  previewTaskToggle:
+    env: client
+    path: ./task.ts:previewTaskToggle
+    events:
+      - preview:click

web/components/panel.tsx

@@ -83,8 +83,10 @@ export function Panel({
         editor.dispatchAppEvent(data.name, ...data.args);
       }
     };
+    console.log("Registering event handler");
     globalThis.addEventListener("message", messageListener);
     return () => {
+      console.log("Unregistering event handler");
       globalThis.removeEventListener("message", messageListener);
     };
   }, []);

web/syscalls/editor.ts

@@ -1,5 +1,5 @@
 import { Editor } from "../editor.tsx";
-import { Transaction } from "../deps.ts";
+import { EditorView, Transaction } from "../deps.ts";
 import { SysCallMapping } from "../../plugos/system.ts";
 import { FilterOption } from "../../common/types.ts";
 
@@ -113,12 +113,24 @@ export function editorSyscalls(editor: Editor): SysCallMapping {
         },
       });
     },
-    "editor.moveCursor": (_ctx, pos: number) => {
+    "editor.moveCursor": (_ctx, pos: number, center = false) => {
       editor.editorView!.dispatch({
         selection: {
           anchor: pos,
         },
       });
+      if (center) {
+        editor.editorView!.dispatch({
+          effects: [
+            EditorView.scrollIntoView(
+              pos,
+              {
+                y: "center",
+              },
+            ),
+          ],
+        });
+      }
     },
     "editor.setSelection": (_ctx, from: number, to: number) => {
       const editorView = editor.editorView!;