Work

packages/common/customtags.ts

@@ -0,0 +1,11 @@
+import { Tag } from "@codemirror/highlight";
+
+export const WikiLinkTag = Tag.define();
+export const WikiLinkPageTag = Tag.define();
+export const CodeInfoTag = Tag.define();
+export const TaskTag = Tag.define();
+export const TaskMarkerTag = Tag.define();
+export const CommentTag = Tag.define();
+export const CommentMarkerTag = Tag.define();
+export const BulletList = Tag.define();
+export const OrderedList = Tag.define();

packages/common/markdown/commands.ts

@@ -0,0 +1,366 @@
+import { ChangeSpec, EditorSelection, StateCommand, Text } from "@codemirror/state";
+import { syntaxTree } from "@codemirror/language";
+import { SyntaxNode, Tree } from "@lezer/common";
+import { markdownLanguage } from "./markdown";
+
+function nodeStart(node: SyntaxNode, doc: Text) {
+  return doc.sliceString(node.from, node.from + 50);
+}
+
+class Context {
+  constructor(
+    readonly node: SyntaxNode,
+    readonly from: number,
+    readonly to: number,
+    readonly spaceBefore: string,
+    readonly spaceAfter: string,
+    readonly type: string,
+    readonly item: SyntaxNode | null
+  ) {}
+
+  blank(trailing: boolean = true) {
+    let result = this.spaceBefore;
+    if (this.node.name == "Blockquote") {
+      result += ">";
+    } else if (this.node.name == "Comment") {
+      result += "%%";
+    } else
+      for (
+        let i = this.to - this.from - result.length - this.spaceAfter.length;
+        i > 0;
+        i--
+      )
+        result += " ";
+    return result + (trailing ? this.spaceAfter : "");
+  }
+
+  marker(doc: Text, add: number) {
+    let number =
+      this.node.name == "OrderedList"
+        ? String(+itemNumber(this.item!, doc)[2] + add)
+        : "";
+    return this.spaceBefore + number + this.type + this.spaceAfter;
+  }
+}
+
+function getContext(node: SyntaxNode, line: string, doc: Text) {
+  let nodes = [];
+  for (
+    let cur: SyntaxNode | null = node;
+    cur && cur.name != "Document";
+    cur = cur.parent
+  ) {
+    if (
+      cur.name == "ListItem" ||
+      cur.name == "Blockquote" ||
+      cur.name == "Comment"
+    )
+      nodes.push(cur);
+  }
+  let context = [],
+    pos = 0;
+  for (let i = nodes.length - 1; i >= 0; i--) {
+    let node = nodes[i],
+      match,
+      start = pos;
+    if (
+      node.name == "Blockquote" &&
+      (match = /^[ \t]*>( ?)/.exec(line.slice(pos)))
+    ) {
+      pos += match[0].length;
+      context.push(new Context(node, start, pos, "", match[1], ">", null));
+    } else if (
+      node.name == "Comment" &&
+      (match = /^[ \t]*%%( ?)/.exec(line.slice(pos)))
+    ) {
+      pos += match[0].length;
+      context.push(new Context(node, start, pos, "", match[1], "%%", null));
+    } else if (
+      node.name == "ListItem" &&
+      node.parent!.name == "OrderedList" &&
+      (match = /^([ \t]*)\d+([.)])([ \t]*)/.exec(nodeStart(node, doc)))
+    ) {
+      let after = match[3],
+        len = match[0].length;
+      if (after.length >= 4) {
+        after = after.slice(0, after.length - 4);
+        len -= 4;
+      }
+      pos += len;
+      context.push(
+        new Context(node.parent!, start, pos, match[1], after, match[2], node)
+      );
+    } else if (
+      node.name == "ListItem" &&
+      node.parent!.name == "BulletList" &&
+      (match = /^([ \t]*)([-+*])([ \t]+)/.exec(nodeStart(node, doc)))
+    ) {
+      let after = match[3],
+        len = match[0].length;
+      if (after.length > 4) {
+        after = after.slice(0, after.length - 4);
+        len -= 4;
+      }
+      pos += len;
+      context.push(
+        new Context(node.parent!, start, pos, match[1], after, match[2], node)
+      );
+    }
+  }
+  return context;
+}
+
+function itemNumber(item: SyntaxNode, doc: Text) {
+  return /^(\s*)(\d+)(?=[.)])/.exec(
+    doc.sliceString(item.from, item.from + 10)
+  )!;
+}
+
+function renumberList(
+  after: SyntaxNode,
+  doc: Text,
+  changes: ChangeSpec[],
+  offset = 0
+) {
+  for (let prev = -1, node = after; ; ) {
+    if (node.name == "ListItem") {
+      let m = itemNumber(node, doc);
+      let number = +m[2];
+      if (prev >= 0) {
+        if (number != prev + 1) return;
+        changes.push({
+          from: node.from + m[1].length,
+          to: node.from + m[0].length,
+          insert: String(prev + 2 + offset),
+        });
+      }
+      prev = number;
+    }
+    let next = node.nextSibling;
+    if (!next) break;
+    node = next;
+  }
+}
+
+/// This command, when invoked in Markdown context with cursor
+/// selection(s), will create a new line with the markup for
+/// blockquotes and lists that were active on the old line. If the
+/// cursor was directly after the end of the markup for the old line,
+/// trailing whitespace and list markers are removed from that line.
+///
+/// The command does nothing in non-Markdown context, so it should
+/// not be used as the only binding for Enter (even in a Markdown
+/// document, HTML and code regions might use a different language).
+export const insertNewlineContinueMarkup: StateCommand = ({
+  state,
+  dispatch,
+}) => {
+  let tree = syntaxTree(state),
+    { doc } = state;
+  let dont = null,
+    changes = state.changeByRange((range) => {
+      if (!range.empty || !markdownLanguage.isActiveAt(state, range.from))
+        return (dont = { range });
+      let pos = range.from,
+        line = doc.lineAt(pos);
+      let context = getContext(tree.resolveInner(pos, -1), line.text, doc);
+      while (
+        context.length &&
+        context[context.length - 1].from > pos - line.from
+      )
+        context.pop();
+      if (!context.length) return (dont = { range });
+      let inner = context[context.length - 1];
+      if (inner.to - inner.spaceAfter.length > pos - line.from)
+        return (dont = { range });
+
+      let emptyLine =
+        pos >= inner.to - inner.spaceAfter.length &&
+        !/\S/.test(line.text.slice(inner.to));
+      // Empty line in list
+      if (inner.item && emptyLine) {
+        // First list item or blank line before: delete a level of markup
+        if (
+          inner.node.firstChild!.to >= pos ||
+          (line.from > 0 && !/[^\s>]/.test(doc.lineAt(line.from - 1).text))
+        ) {
+          let next = context.length > 1 ? context[context.length - 2] : null;
+          let delTo,
+            insert = "";
+          if (next && next.item) {
+            // Re-add marker for the list at the next level
+            delTo = line.from + next.from;
+            insert = next.marker(doc, 1);
+          } else {
+            delTo = line.from + (next ? next.to : 0);
+          }
+          let changes: ChangeSpec[] = [{ from: delTo, to: pos, insert }];
+          if (inner.node.name == "OrderedList")
+            renumberList(inner.item!, doc, changes, -2);
+          if (next && next.node.name == "OrderedList")
+            renumberList(next.item!, doc, changes);
+          return {
+            range: EditorSelection.cursor(delTo + insert.length),
+            changes,
+          };
+        } else {
+          // Move this line down
+          let insert = "";
+          for (let i = 0, e = context.length - 2; i <= e; i++)
+            insert += context[i].blank(i < e);
+          insert += state.lineBreak;
+          return {
+            range: EditorSelection.cursor(pos + insert.length),
+            changes: { from: line.from, insert },
+          };
+        }
+      }
+
+      if (inner.node.name == "Blockquote" && emptyLine && line.from) {
+        let prevLine = doc.lineAt(line.from - 1),
+          quoted = />\s*$/.exec(prevLine.text);
+        // Two aligned empty quoted lines in a row
+        if (quoted && quoted.index == inner.from) {
+          let changes = state.changes([
+            { from: prevLine.from + quoted.index, to: prevLine.to },
+            { from: line.from + inner.from, to: line.to },
+          ]);
+          return { range: range.map(changes), changes };
+        }
+      }
+
+      if (inner.node.name == "Comment" && emptyLine && line.from) {
+        let prevLine = doc.lineAt(line.from - 1),
+          commented = /%%\s*$/.exec(prevLine.text);
+        // Two aligned empty quoted lines in a row
+        if (commented && commented.index == inner.from) {
+          let changes = state.changes([
+            { from: prevLine.from + commented.index, to: prevLine.to },
+            { from: line.from + inner.from, to: line.to },
+          ]);
+          return { range: range.map(changes), changes };
+        }
+      }
+
+      let changes: ChangeSpec[] = [];
+      if (inner.node.name == "OrderedList")
+        renumberList(inner.item!, doc, changes);
+      let insert = state.lineBreak;
+      let continued = inner.item && inner.item.from < line.from;
+      // If not dedented
+      if (
+        !continued ||
+        /^[\s\d.)\-+*>]*/.exec(line.text)![0].length >= inner.to
+      ) {
+        for (let i = 0, e = context.length - 1; i <= e; i++)
+          insert +=
+            i == e && !continued
+              ? context[i].marker(doc, 1)
+              : context[i].blank();
+      }
+      let from = pos;
+      while (
+        from > line.from &&
+        /\s/.test(line.text.charAt(from - line.from - 1))
+      )
+        from--;
+      changes.push({ from, to: pos, insert });
+      return { range: EditorSelection.cursor(from + insert.length), changes };
+    });
+  if (dont) return false;
+  dispatch(state.update(changes, { scrollIntoView: true, userEvent: "input" }));
+  return true;
+};
+
+function isMark(node: SyntaxNode) {
+  return node.name == "QuoteMark" || node.name == "ListMark";
+}
+
+function contextNodeForDelete(tree: Tree, pos: number) {
+  let node = tree.resolveInner(pos, -1),
+    scan = pos;
+  if (isMark(node)) {
+    scan = node.from;
+    node = node.parent!;
+  }
+  for (let prev; (prev = node.childBefore(scan)); ) {
+    if (isMark(prev)) {
+      scan = prev.from;
+    } else if (prev.name == "OrderedList" || prev.name == "BulletList") {
+      node = prev.lastChild!;
+      scan = node.to;
+    } else {
+      break;
+    }
+  }
+  return node;
+}
+
+/// This command will, when invoked in a Markdown context with the
+/// cursor directly after list or blockquote markup, delete one level
+/// of markup. When the markup is for a list, it will be replaced by
+/// spaces on the first invocation (a further invocation will delete
+/// the spaces), to make it easy to continue a list.
+///
+/// When not after Markdown block markup, this command will return
+/// false, so it is intended to be bound alongside other deletion
+/// commands, with a higher precedence than the more generic commands.
+export const deleteMarkupBackward: StateCommand = ({ state, dispatch }) => {
+  let tree = syntaxTree(state);
+  let dont = null,
+    changes = state.changeByRange((range) => {
+      let pos = range.from,
+        { doc } = state;
+      if (range.empty && markdownLanguage.isActiveAt(state, range.from)) {
+        let line = doc.lineAt(pos);
+        let context = getContext(
+          contextNodeForDelete(tree, pos),
+          line.text,
+          doc
+        );
+        if (context.length) {
+          let inner = context[context.length - 1];
+          let spaceEnd =
+            inner.to - inner.spaceAfter.length + (inner.spaceAfter ? 1 : 0);
+          // Delete extra trailing space after markup
+          if (
+            pos - line.from > spaceEnd &&
+            !/\S/.test(line.text.slice(spaceEnd, pos - line.from))
+          )
+            return {
+              range: EditorSelection.cursor(line.from + spaceEnd),
+              changes: { from: line.from + spaceEnd, to: pos },
+            };
+          if (pos - line.from == spaceEnd) {
+            let start = line.from + inner.from;
+            // Replace a list item marker with blank space
+            if (
+              inner.item &&
+              inner.node.from < inner.item.from &&
+              /\S/.test(line.text.slice(inner.from, inner.to))
+            )
+              return {
+                range,
+                changes: {
+                  from: start,
+                  to: line.from + inner.to,
+                  insert: inner.blank(),
+                },
+              };
+            // Delete one level of indentation
+            if (start < pos)
+              return {
+                range: EditorSelection.cursor(start),
+                changes: { from: start, to: pos },
+              };
+          }
+        }
+      }
+      return (dont = { range });
+    });
+  if (dont) return false;
+  dispatch(
+    state.update(changes, { scrollIntoView: true, userEvent: "delete" })
+  );
+  return true;
+};

packages/common/markdown/index.ts

@@ -0,0 +1,78 @@
+import { Prec } from "@codemirror/state";
+import { KeyBinding, keymap } from "@codemirror/view";
+import { Language, LanguageDescription, LanguageSupport } from "@codemirror/language";
+import { MarkdownExtension, MarkdownParser, parseCode } from "@lezer/markdown";
+import { html } from "@codemirror/lang-html";
+import { commonmarkLanguage, getCodeParser, markdownLanguage, mkLang } from "./markdown";
+import { deleteMarkupBackward, insertNewlineContinueMarkup } from "./commands";
+
+export {
+  commonmarkLanguage,
+  markdownLanguage,
+  insertNewlineContinueMarkup,
+  deleteMarkupBackward,
+};
+
+/// A small keymap with Markdown-specific bindings. Binds Enter to
+/// [`insertNewlineContinueMarkup`](#lang-markdown.insertNewlineContinueMarkup)
+/// and Backspace to
+/// [`deleteMarkupBackward`](#lang-markdown.deleteMarkupBackward).
+export const markdownKeymap: readonly KeyBinding[] = [
+  { key: "Enter", run: insertNewlineContinueMarkup },
+  { key: "Backspace", run: deleteMarkupBackward },
+];
+
+const htmlNoMatch = html({ matchClosingTags: false });
+
+/// Markdown language support.
+export function markdown(
+  config: {
+    /// When given, this language will be used by default to parse code
+    /// blocks.
+    defaultCodeLanguage?: Language | LanguageSupport;
+    /// A collection of language descriptions to search through for a
+    /// matching language (with
+    /// [`LanguageDescription.matchLanguageName`](#language.LanguageDescription^matchLanguageName))
+    /// when a fenced code block has an info string.
+    codeLanguages?: readonly LanguageDescription[];
+    /// Set this to false to disable installation of the Markdown
+    /// [keymap](#lang-markdown.markdownKeymap).
+    addKeymap?: boolean;
+    /// Markdown parser
+    /// [extensions](https://github.com/lezer-parser/markdown#user-content-markdownextension)
+    /// to add to the parser.
+    extensions?: MarkdownExtension;
+    /// The base language to use. Defaults to
+    /// [`commonmarkLanguage`](#lang-markdown.commonmarkLanguage).
+    base?: Language;
+  } = {}
+) {
+  let {
+    codeLanguages,
+    defaultCodeLanguage,
+    addKeymap = true,
+    base: { parser } = commonmarkLanguage,
+  } = config;
+  if (!(parser instanceof MarkdownParser))
+    throw new RangeError(
+      "Base parser provided to `markdown` should be a Markdown parser"
+    );
+  let extensions = config.extensions ? [config.extensions] : [];
+  let support = [htmlNoMatch.support],
+    defaultCode;
+  if (defaultCodeLanguage instanceof LanguageSupport) {
+    support.push(defaultCodeLanguage.support);
+    defaultCode = defaultCodeLanguage.language;
+  } else if (defaultCodeLanguage) {
+    defaultCode = defaultCodeLanguage;
+  }
+  let codeParser =
+    codeLanguages || defaultCode
+      ? getCodeParser(codeLanguages || [], defaultCode)
+      : undefined;
+  extensions.push(
+    parseCode({ codeParser, htmlParser: htmlNoMatch.language.parser })
+  );
+  if (addKeymap) support.push(Prec.high(keymap.of(markdownKeymap)));
+  return new LanguageSupport(mkLang(parser.configure(extensions)), support);
+}

packages/common/markdown/markdown.ts

@@ -0,0 +1,105 @@
+import {
+  defineLanguageFacet,
+  foldNodeProp,
+  indentNodeProp,
+  Language,
+  languageDataProp,
+  LanguageDescription,
+  ParseContext
+} from "@codemirror/language";
+import { styleTags, tags as t } from "@codemirror/highlight";
+import { Emoji, GFM, MarkdownParser, parser as baseParser, Subscript, Superscript } from "@lezer/markdown";
+
+const data = defineLanguageFacet({ block: { open: "<!--", close: "-->" } });
+
+export const commonmark = baseParser.configure({
+  props: [
+    styleTags({
+      "Blockquote/...": t.quote,
+      HorizontalRule: t.contentSeparator,
+      "ATXHeading1/... SetextHeading1/...": t.heading1,
+      "ATXHeading2/... SetextHeading2/...": t.heading2,
+      "ATXHeading3/...": t.heading3,
+      "ATXHeading4/...": t.heading4,
+      "ATXHeading5/...": t.heading5,
+      "ATXHeading6/...": t.heading6,
+      "Comment CommentBlock": t.comment,
+      Escape: t.escape,
+      Entity: t.character,
+      "Emphasis/...": t.emphasis,
+      "StrongEmphasis/...": t.strong,
+      "Link/... Image/...": t.link,
+      "OrderedList/... BulletList/...": t.list,
+      "InlineCode CodeText": t.monospace,
+      URL: t.url,
+      "HeaderMark HardBreak QuoteMark ListMark LinkMark EmphasisMark CodeMark":
+        t.processingInstruction,
+      "CodeInfo LinkLabel": t.labelName,
+      LinkTitle: t.string,
+      Paragraph: t.content,
+    }),
+    foldNodeProp.add((type) => {
+      if (!type.is("Block") || type.is("Document")) return undefined;
+      return (tree, state) => ({
+        from: state.doc.lineAt(tree.from).to,
+        to: tree.to,
+      });
+    }),
+    indentNodeProp.add({
+      Document: () => null,
+    }),
+    languageDataProp.add({
+      Document: data,
+    }),
+  ],
+});
+
+export function mkLang(parser: MarkdownParser) {
+  return new Language(
+    data,
+    parser,
+    parser.nodeSet.types.find((t) => t.name == "Document")!
+  );
+}
+
+/// Language support for strict CommonMark.
+export const commonmarkLanguage = mkLang(commonmark);
+
+const extended = commonmark.configure([
+  GFM,
+  Subscript,
+  Superscript,
+  Emoji,
+  {
+    props: [
+      styleTags({
+        "TableDelimiter SubscriptMark SuperscriptMark StrikethroughMark":
+          t.processingInstruction,
+        "TableHeader/...": t.heading,
+        "Strikethrough/...": t.strikethrough,
+        TaskMarker: t.atom,
+        Task: t.list,
+        Emoji: t.character,
+        "Subscript Superscript": t.special(t.content),
+        TableCell: t.content,
+      }),
+    ],
+  },
+]);
+
+/// Language support for [GFM](https://github.github.com/gfm/) plus
+/// subscript, superscript, and emoji syntax.
+export const markdownLanguage = mkLang(extended);
+
+export function getCodeParser(
+  languages: readonly LanguageDescription[],
+  defaultLanguage?: Language
+) {
+  return (info: string) => {
+    let found =
+      info && LanguageDescription.matchLanguageName(languages, info, true);
+    if (!found) return defaultLanguage ? defaultLanguage.parser : null;
+    if (found.support) return found.support.language.parser;
+    return ParseContext.getSkippingParser(found.load());
+  };
+}

packages/common/markdown_ext.ts

@@ -0,0 +1,66 @@
+import { Tag } from "@codemirror/highlight";
+import type { MarkdownConfig } from "@lezer/markdown";
+import { System } from "@plugos/plugos/system";
+import { Manifest } from "@silverbulletmd/common/manifest";
+
+export type MDExt = {
+  // unicode char code for efficiency .charCodeAt(0)
+  firstCharCodes: number[];
+  regex: RegExp;
+  nodeType: string;
+  tag: Tag;
+  styles: { [key: string]: string };
+};
+
+export function mdExtensionSyntaxConfig({
+  regex,
+  firstCharCodes,
+  nodeType,
+}: MDExt): MarkdownConfig {
+  return {
+    defineNodes: [nodeType],
+    parseInline: [
+      {
+        name: nodeType,
+        parse(cx, next, pos) {
+          if (!firstCharCodes.includes(next)) {
+            return -1;
+          }
+          let match = regex.exec(cx.slice(pos, cx.end));
+          if (!match) {
+            return -1;
+          }
+          return cx.addElement(cx.elt(nodeType, pos, pos + match[0].length));
+        },
+        // after: "Emphasis",
+      },
+    ],
+  };
+}
+
+export function mdExtensionStyleTags({ nodeType, tag }: MDExt): {
+  [selector: string]: Tag | readonly Tag[];
+} {
+  return {
+    [nodeType]: tag,
+  };
+}
+
+export function loadMarkdownExtensions(system: System<any>): MDExt[] {
+  let mdExtensions: MDExt[] = [];
+  for (let plug of system.loadedPlugs.values()) {
+    let manifest = plug.manifest as Manifest;
+    if (manifest.syntax) {
+      for (let [nodeType, def] of Object.entries(manifest.syntax)) {
+        mdExtensions.push({
+          nodeType,
+          tag: Tag.define(),
+          firstCharCodes: def.firstCharacters.map((ch) => ch.charCodeAt(0)),
+          regex: new RegExp("^" + def.regex),
+          styles: def.styles,
+        });
+      }
+    }
+  }
+  return mdExtensions;
+}

packages/common/package.json

@@ -5,5 +5,13 @@
     "email": "zef@zef.me"
   },
   "version": "0.0.1",
-  "license": "MIT"
+  "license": "MIT",
+  "dependencies": {
+    "@codemirror/highlight": "^0.19.0",
+    "@codemirror/language": "^0.19.0",
+    "@codemirror/stream-parser": "^0.19.9",
+    "@codemirror/lang-javascript": "^0.19.7",
+    "@codemirror/legacy-modes": "^0.19.1",
+    "@lezer/markdown": "^0.15.0"
+  }
 }

packages/common/parser.ts

@@ -0,0 +1,133 @@
+import { styleTags, tags as t } from "@codemirror/highlight";
+import {
+  BlockContext,
+  LeafBlock,
+  LeafBlockParser,
+  MarkdownConfig,
+  parseCode,
+  Table,
+  TaskList,
+} from "@lezer/markdown";
+import { commonmark, getCodeParser, mkLang } from "./markdown/markdown";
+import * as ct from "./customtags";
+import {
+  Language,
+  LanguageDescription,
+  LanguageSupport,
+} from "@codemirror/language";
+import { StreamLanguage } from "@codemirror/stream-parser";
+import { yaml } from "@codemirror/legacy-modes/mode/yaml";
+import {
+  javascriptLanguage,
+  typescriptLanguage,
+} from "@codemirror/lang-javascript";
+import {
+  MDExt,
+  mdExtensionStyleTags,
+  mdExtensionSyntaxConfig,
+} from "./markdown_ext";
+
+export const pageLinkRegex = /^\[\[([^\]]+)\]\]/;
+
+const WikiLink: MarkdownConfig = {
+  defineNodes: ["WikiLink", "WikiLinkPage"],
+  parseInline: [
+    {
+      name: "WikiLink",
+      parse(cx, next, pos) {
+        let match: RegExpMatchArray | null;
+        if (
+          next != 91 /* '[' */ ||
+          !(match = pageLinkRegex.exec(cx.slice(pos, cx.end)))
+        ) {
+          return -1;
+        }
+        return cx.addElement(
+          cx.elt("WikiLink", pos, pos + match[0].length, [
+            cx.elt("WikiLinkPage", pos + 2, pos + match[0].length - 2),
+          ])
+        );
+      },
+      after: "Emphasis",
+    },
+  ],
+};
+
+class CommentParser implements LeafBlockParser {
+  nextLine() {
+    return false;
+  }
+
+  finish(cx: BlockContext, leaf: LeafBlock) {
+    cx.addLeafElement(
+      leaf,
+      cx.elt("Comment", leaf.start, leaf.start + leaf.content.length, [
+        // cx.elt("CommentMarker", leaf.start, leaf.start + 3),
+        ...cx.parser.parseInline(leaf.content.slice(3), leaf.start + 3),
+      ])
+    );
+    return true;
+  }
+}
+export const Comment: MarkdownConfig = {
+  defineNodes: [{ name: "Comment", block: true }],
+  parseBlock: [
+    {
+      name: "Comment",
+      leaf(cx, leaf) {
+        return /^%%\s/.test(leaf.content) ? new CommentParser() : null;
+      },
+      after: "SetextHeading",
+    },
+  ],
+};
+
+export default function buildMarkdown(mdExtensions: MDExt[]): Language {
+  return mkLang(
+    commonmark.configure([
+      WikiLink,
+      TaskList,
+      Comment,
+      Table,
+      ...mdExtensions.map(mdExtensionSyntaxConfig),
+      parseCode({
+        codeParser: getCodeParser([
+          LanguageDescription.of({
+            name: "yaml",
+            alias: ["meta", "data"],
+            support: new LanguageSupport(StreamLanguage.define(yaml)),
+          }),
+          LanguageDescription.of({
+            name: "javascript",
+            alias: ["js"],
+            support: new LanguageSupport(javascriptLanguage),
+          }),
+          LanguageDescription.of({
+            name: "typescript",
+            alias: ["ts"],
+            support: new LanguageSupport(typescriptLanguage),
+          }),
+        ]),
+      }),
+      {
+        props: [
+          styleTags({
+            WikiLink: ct.WikiLinkTag,
+            WikiLinkPage: ct.WikiLinkPageTag,
+            Task: ct.TaskTag,
+            TaskMarker: ct.TaskMarkerTag,
+            Comment: ct.CommentTag,
+            "TableDelimiter SubscriptMark SuperscriptMark StrikethroughMark":
+              t.processingInstruction,
+            "TableHeader/...": t.heading,
+            TableCell: t.content,
+            CodeInfo: ct.CodeInfoTag,
+          }),
+          ...mdExtensions.map((mdExt) =>
+            styleTags(mdExtensionStyleTags(mdExt))
+          ),
+        ],
+      },
+    ])
+  );
+}

packages/common/spaces/disk_space_primitives.ts

@@ -11,21 +11,30 @@ import * as path from "path";
 import { PageMeta } from "../types";
 import { SpacePrimitives } from "./space_primitives";
 import { Plug } from "@plugos/plugos/plug";
+import { realpathSync } from "fs";
 
 export class DiskSpacePrimitives implements SpacePrimitives {
   rootPath: string;
   plugPrefix: string;
 
   constructor(rootPath: string, plugPrefix: string = "_plug/") {
-    this.rootPath = rootPath;
+    this.rootPath = realpathSync(rootPath);
     this.plugPrefix = plugPrefix;
   }
 
+  safePath(p: string): string {
+    let realPath = realpathSync(p);
+    if (!realPath.startsWith(this.rootPath)) {
+      throw Error(`Path ${p} is not in the space`);
+    }
+    return realPath;
+  }
+
   pageNameToPath(pageName: string) {
     if (pageName.startsWith(this.plugPrefix)) {
-      return path.join(this.rootPath, pageName + ".plug.json");
+      return this.safePath(path.join(this.rootPath, pageName + ".plug.json"));
     }
-    return path.join(this.rootPath, pageName + ".md");
+    return this.safePath(path.join(this.rootPath, pageName + ".md"));
   }
 
   pathToPageName(fullPath: string): string {

packages/common/spaces/http_space_primitives.ts

@@ -5,17 +5,34 @@ import { SpacePrimitives } from "./space_primitives";
 export class HttpSpacePrimitives implements SpacePrimitives {
   pageUrl: string;
   private plugUrl: string;
+  token?: string;
 
-  constructor(url: string) {
+  constructor(url: string, token?: string) {
     this.pageUrl = url + "/fs";
     this.plugUrl = url + "/plug";
+    this.token = token;
+  }
+
+  private async authenticatedFetch(
+    url: string,
+    options: any
+  ): Promise<Response> {
+    if (this.token) {
+      options.headers = options.headers || {};
+      options.headers["Authorization"] = `Bearer ${this.token}`;
+    }
+    let result = await fetch(url, options);
+    if (result.status === 401) {
+      throw Error("Unauthorized");
+    }
+    return result;
   }
 
   public async fetchPageList(): Promise<{
     pages: Set<PageMeta>;
     nowTimestamp: number;
   }> {
-    let req = await fetch(this.pageUrl, {
+    let req = await this.authenticatedFetch(this.pageUrl, {
       method: "GET",
     });
 
@@ -35,7 +52,7 @@ export class HttpSpacePrimitives implements SpacePrimitives {
   }
 
   async readPage(name: string): Promise<{ text: string; meta: PageMeta }> {
-    let res = await fetch(`${this.pageUrl}/${name}`, {
+    let res = await this.authenticatedFetch(`${this.pageUrl}/${name}`, {
       method: "GET",
     });
     if (res.headers.get("X-Status") === "404") {
@@ -54,7 +71,7 @@ export class HttpSpacePrimitives implements SpacePrimitives {
     lastModified?: number
   ): Promise<PageMeta> {
     // TODO: lastModified ignored for now
-    let res = await fetch(`${this.pageUrl}/${name}`, {
+    let res = await this.authenticatedFetch(`${this.pageUrl}/${name}`, {
       method: "PUT",
       body: text,
       headers: lastModified
@@ -68,7 +85,7 @@ export class HttpSpacePrimitives implements SpacePrimitives {
   }
 
   async deletePage(name: string): Promise<void> {
-    let req = await fetch(`${this.pageUrl}/${name}`, {
+    let req = await this.authenticatedFetch(`${this.pageUrl}/${name}`, {
       method: "DELETE",
     });
     if (req.status !== 200) {
@@ -77,13 +94,16 @@ export class HttpSpacePrimitives implements SpacePrimitives {
   }
 
   async proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
-    let req = await fetch(`${this.plugUrl}/${plug.name}/syscall/${name}`, {
-      method: "POST",
-      headers: {
-        "Content-type": "application/json",
-      },
-      body: JSON.stringify(args),
-    });
+    let req = await this.authenticatedFetch(
+      `${this.plugUrl}/${plug.name}/syscall/${name}`,
+      {
+        method: "POST",
+        headers: {
+          "Content-type": "application/json",
+        },
+        body: JSON.stringify(args),
+      }
+    );
     if (req.status !== 200) {
       let error = await req.text();
       throw Error(error);
@@ -105,13 +125,16 @@ export class HttpSpacePrimitives implements SpacePrimitives {
       return plug.invoke(name, args);
     }
     // Or dispatch to server
-    let req = await fetch(`${this.plugUrl}/${plug.name}/function/${name}`, {
-      method: "POST",
-      headers: {
-        "Content-type": "application/json",
-      },
-      body: JSON.stringify(args),
-    });
+    let req = await this.authenticatedFetch(
+      `${this.plugUrl}/${plug.name}/function/${name}`,
+      {
+        method: "POST",
+        headers: {
+          "Content-type": "application/json",
+        },
+        body: JSON.stringify(args),
+      }
+    );
     if (req.status !== 200) {
       let error = await req.text();
       throw Error(error);
@@ -127,7 +150,7 @@ export class HttpSpacePrimitives implements SpacePrimitives {
   }
 
   async getPageMeta(name: string): Promise<PageMeta> {
-    let res = await fetch(`${this.pageUrl}/${name}`, {
+    let res = await this.authenticatedFetch(`${this.pageUrl}/${name}`, {
       method: "OPTIONS",
     });
     if (res.headers.get("X-Status") === "404") {

packages/common/tree.test.ts

@@ -7,9 +7,9 @@ import {
   nodeAtPos,
   removeParentPointers,
   renderToText,
-  replaceNodesMatching
+  replaceNodesMatching,
 } from "./tree";
-import wikiMarkdownLang from "@silverbulletmd/web/parser";
+import wikiMarkdownLang from "@silverbulletmd/common/parser";
 
 const mdTest1 = `
 # Heading