Migrated silverbullet-publish into this repo

.gitignore

@@ -6,5 +6,6 @@ dist
 *.js.map
 website_build
 data.db
+publish-data.db
 /index.json
 .idea

cmd/plug_compile.ts

@@ -11,7 +11,6 @@ export async function plugCompileCommand(
   },
   ...manifestPaths: string[]
 ) {
-  console.log("All optiosn", arguments);
   await bundleRun(
     manifestPaths,
     dist,

cmd/publish.ts

@@ -0,0 +1,172 @@
+#!/usr/bin/env node
+import { createSandbox } from "../plugos/environments/deno_sandbox.ts";
+import { EventHook } from "../plugos/hooks/event.ts";
+import { eventSyscalls } from "../plugos/syscalls/event.ts";
+import fileSystemSyscalls from "../plugos/syscalls/fs.deno.ts";
+import {
+  ensureFTSTable,
+  fullTextSearchSyscalls,
+} from "../plugos/syscalls/fulltext.sqlite.ts";
+import sandboxSyscalls from "../plugos/syscalls/sandbox.ts";
+import shellSyscalls from "../plugos/syscalls/shell.deno.ts";
+import {
+  ensureTable as ensureStoreTable,
+  storeSyscalls,
+} from "../plugos/syscalls/store.deno.ts";
+import { System } from "../plugos/system.ts";
+import { Manifest, SilverBulletHooks } from "../common/manifest.ts";
+import { loadMarkdownExtensions } from "../common/markdown_ext.ts";
+import buildMarkdown from "../common/parser.ts";
+import { DiskSpacePrimitives } from "../common/spaces/disk_space_primitives.ts";
+import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives.ts";
+import { Space } from "../common/spaces/space.ts";
+import { markdownSyscalls } from "../common/syscalls/markdown.ts";
+import { PageNamespaceHook } from "../server/hooks/page_namespace.ts";
+import { PlugSpacePrimitives } from "../server/hooks/plug_space_primitives.ts";
+import {
+  ensureTable as ensureIndexTable,
+  pageIndexSyscalls,
+} from "../server/syscalls/index.ts";
+import spaceSyscalls from "../server/syscalls/space.ts";
+
+import { Command } from "https://deno.land/x/cliffy@v0.25.2/command/command.ts";
+
+import assetBundle from "../dist/asset_bundle.json" assert { type: "json" };
+import { AssetBundle, AssetJson } from "../plugos/asset_bundle/bundle.ts";
+import { path } from "../server/deps.ts";
+import { AsyncSQLite } from "../plugos/sqlite/async_sqlite.ts";
+import { AssetBundlePlugSpacePrimitives } from "../common/spaces/asset_bundle_space_primitives.ts";
+import assetSyscalls from "../plugos/syscalls/asset.ts";
+import { faBullseye } from "https://esm.sh/v96/@fortawesome/free-solid-svg-icons@6.2.0/index.d.ts";
+
+export async function publishCommand(options: {
+  index: boolean;
+  watch: boolean;
+  output: string;
+}, pagesPath: string) {
+  const assets = new AssetBundle(assetBundle as AssetJson);
+  // Set up the PlugOS System
+  const system = new System<SilverBulletHooks>("server");
+
+  // Instantiate the event bus hook
+  const eventHook = new EventHook();
+  system.addHook(eventHook);
+
+  // And the page namespace hook
+  const namespaceHook = new PageNamespaceHook();
+  system.addHook(namespaceHook);
+
+  pagesPath = path.resolve(pagesPath);
+
+  // The space
+  const space = new Space(
+    new AssetBundlePlugSpacePrimitives(
+      new EventedSpacePrimitives(
+        new PlugSpacePrimitives(
+          new DiskSpacePrimitives(pagesPath),
+          namespaceHook,
+        ),
+        eventHook,
+      ),
+      assets,
+    ),
+  );
+
+  await space.updatePageList();
+
+  // The database used for persistence (SQLite)
+  const db = new AsyncSQLite(path.join(pagesPath, "publish-data.db"));
+  db.init().catch((e) => {
+    console.error("Error initializing database", e);
+  });
+
+  // Register syscalls available on the server side
+  system.registerSyscalls(
+    [],
+    pageIndexSyscalls(db),
+    storeSyscalls(db, "store"),
+    fullTextSearchSyscalls(db, "fts"),
+    spaceSyscalls(space),
+    eventSyscalls(eventHook),
+    markdownSyscalls(buildMarkdown([])),
+    sandboxSyscalls(system),
+    assetSyscalls(system),
+  );
+  // Danger zone
+  system.registerSyscalls(["shell"], shellSyscalls(pagesPath));
+  system.registerSyscalls(["fs"], fileSystemSyscalls("/"));
+
+  const globalModules = JSON.parse(
+    assets.readTextFileSync(`web/global.plug.json`),
+  );
+
+  system.on({
+    sandboxInitialized: async (sandbox) => {
+      for (
+        const [modName, code] of Object.entries(
+          globalModules.dependencies,
+        )
+      ) {
+        await sandbox.loadDependency(modName, code as string);
+      }
+    },
+  });
+
+  await space.updatePageList();
+
+  const allPlugs = await space.listPlugs();
+
+  console.log("Loading plugs", allPlugs);
+  await Promise.all((await space.listPlugs()).map(async (plugName) => {
+    const { data } = await space.readAttachment(plugName, "string");
+    await system.load(JSON.parse(data as string), createSandbox);
+  }));
+
+  const corePlug = system.loadedPlugs.get("core");
+  if (!corePlug) {
+    console.error("Something went very wrong, 'core' plug not found");
+    return;
+  }
+
+  system.registerSyscalls(
+    [],
+    markdownSyscalls(buildMarkdown(loadMarkdownExtensions(system))),
+  );
+
+  await ensureIndexTable(db);
+  await ensureStoreTable(db, "store");
+  await ensureFTSTable(db, "fts");
+
+  if (options.index) {
+    console.log("Now indexing space");
+    await corePlug.invoke("reindexSpace", []);
+  }
+
+  const outputDir = path.resolve(options.output);
+
+  await Deno.mkdir(outputDir, { recursive: true });
+
+  const publishPlug = system.loadedPlugs.get("publish")!;
+
+  await publishPlug.invoke("publishAll", [outputDir]);
+
+  if (options.watch) {
+    console.log("Watching for changes");
+    let building = false;
+    for await (const _event of Deno.watchFs(pagesPath, { recursive: true })) {
+      console.log("Change detected, republishing");
+      if (building) {
+        continue;
+      }
+      building = true;
+      space.updatePageList().then(async () => {
+        await publishPlug.invoke("publishAll", [outputDir]);
+        building = false;
+      });
+    }
+  } else {
+    console.log("Done!");
+    Deno.exit(0);
+    //   process.exit(0);
+  }
+}

plugs/markdown/markdown_render.ts

@@ -12,6 +12,7 @@ type MarkdownRenderOptions = {
   smartHardBreak?: true;
   annotationPositions?: true;
   renderFrontMatter?: true;
+  attachmentUrlPrefix?: string;
 };
 
 function cleanTags(values: (Tag | null)[]): Tag[] {
@@ -138,11 +139,15 @@ function render(
     // Code blocks
     case "FencedCode":
     case "CodeBlock": {
+      // Clear out top-level indent blocks
+      t.children = t.children!.filter((c) => c.type);
       return {
         name: "pre",
         body: cleanTags(mapRender(t.children!)),
       };
     }
+    case "CodeInfo":
+      return null;
     case "CodeText":
       return t.children![0].text!;
     case "Blockquote":
@@ -201,7 +206,14 @@ function render(
       };
     case "Link": {
       const linkText = t.children![1].text!;
-      const url = findNodeOfType(t, "URL")!.children![0].text!;
+      const urlNode = findNodeOfType(t, "URL");
+      if (!urlNode) {
+        return renderToText(t);
+      }
+      let url = urlNode.children![0].text!;
+      if (url.indexOf("://") === -1) {
+        url = `${options.attachmentUrlPrefix || ""}${url}`;
+      }
       return {
         name: "a",
         attrs: {
@@ -212,9 +224,13 @@ function render(
     }
     case "Image": {
       const altText = t.children![1].text!;
-      let url = findNodeOfType(t, "URL")!.children![0].text!;
+      const urlNode = findNodeOfType(t, "URL");
+      if (!urlNode) {
+        return renderToText(t);
+      }
+      let url = urlNode!.children![0].text!;
       if (url.indexOf("://") === -1) {
-        url = `fs/${url}`;
+        url = `${options.attachmentUrlPrefix || ""}${url}`;
       }
       return {
         name: "img",
@@ -233,7 +249,7 @@ function render(
       return {
         name: "a",
         attrs: {
-          href: `/${ref}`,
+          href: `/${ref.replaceAll(" ", "_").replace("@", "#")}`,
         },
         body: ref,
       };

plugs/markdown/preview.ts

@@ -16,6 +16,7 @@ export async function updateMarkdownPreview() {
     smartHardBreak: true,
     annotationPositions: true,
     renderFrontMatter: true,
+    attachmentUrlPrefix: "fs/",
   });
   await editor.showPanel(
     "rhs",

plugs/markdown/test/example.md

@@ -79,4 +79,8 @@ Here is something
 
 A new thing. 
 
-![alt text](https://image.jpg)
+![alt text](https:/
+
+## Weird stuff
+
+* [{{#if done}}x{{else}} {{/if}}] [[{{page}}@{{pos}}]] {{name}} {{#if deadline}}📅 {{deadline}}{{/if}} {{#each tags}}{{.}} {{/each}}

plugs/publish/README.md

@@ -0,0 +1,45 @@
+# Silver Bullet Publish
+A simple tool to export a subset of your [SilverBullet](https://silverbullet.md) space as a static website.
+
+**Note:** this is highly experimental and not necessarily production ready code, use at your own risk.
+
+silverbullet-publish currentenly publishes a subset of a space in two formats:
+
+* Markdown (.md files)
+* HTML (.html files based on currently hardcoded templates (see `page.hbs` and `style.css`)
+
+The tool can be run in two ways:
+
+1. As a Silver Bullet plug (via the `Silver Bullet Publish: Publish All` command)
+2. As a stand-alone CLI tool (via `npx`)
+
+The latter allows for automatic deployments to e.g. environments like Netlify.
+
+## Configuration
+SilverBullet Publish is configured via the `PUBLISH` page with the following properties:
+
+  ```yaml
+  # Index page to use for public version
+  indexPage: Public
+  # Optional destination folder when used in plug mode
+  destDir: /Users/you/my-website
+  title: Name of the space
+  removeHashtags: true
+  removeUnpublishedLinks: false
+  # Publish all pages with specific tag
+  tags:
+  - "#pub"
+  # Publish all pages with a specifix prefix
+  prefixes:
+  - /public
+  ```
+
+## Running via `npx`
+The easiest way to run SilverBullet Publish is via `npx`, it takes a few optional arguments beyond the path to your SilverBullet space:
+
+* `-o` specifies where to write the output to (defaults to `./web`)
+* `--index` runs a full space index (e.g. to index all hash tags) before publishing, this is primarily useful when run in a CI/CI pipeline (like Netlify) because there no `data.db` in your repo containing this index.
+
+```bash
+npx @silverbulletmd/publish -o web_build --index .
+```

plugs/publish/assets/page.hbs

@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<base href="/" />
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>{{#if pageName}}{{pageName}} — {{config.title}}{{else}}{{config.title}}{{/if}}</title>
+<style>
+{{css}}
+</style>
+</head>
+<body>
+<h1>{{pageName}}</h1>
+{{{body}}}
+</body>
+</html>

plugs/publish/assets/style.css

@@ -0,0 +1,53 @@
+body {
+  font-family: georgia, times, serif;
+  font-size: 14pt;
+  max-width: 800px;
+  margin-left: auto;
+  margin-right: auto;
+  padding-left: 20px;
+  padding-right: 20px;
+}
+
+table {
+  width: 100%;
+  border-spacing: 0;
+}
+
+thead tr {
+  background-color: #333;
+  color: #eee;
+}
+
+th,
+td {
+  padding: 8px;
+}
+
+tbody tr:nth-of-type(even) {
+  background-color: #f3f3f3;
+}
+
+ul li p {
+  margin: 0;
+}
+
+a[href] {
+  text-decoration: none;
+}
+
+blockquote {
+  border-left: 1px solid #333;
+  margin-left: 2px;
+  padding-left: 10px;
+}
+
+.footer {
+  border-top: 1px solid #000;
+  padding-top: 5px;
+  text-align: center;
+  font-size: 70%;
+}
+
+img {
+  max-width: 90%;
+}

plugs/publish/publish.plug.yaml

@@ -0,0 +1,15 @@
+name: publish
+imports:
+  - https://get.silverbullet.md/global.plug.json
+requiredPermissions:
+  - fs
+assets:
+  - "assets/*"
+functions:
+  publishAll:
+    path: "./publish.ts:publishAll"
+    env: server
+  publishAllCommand:
+    path: "./publish.ts:publishAllCommand"
+    command:
+      name: "Silver Bullet Publish: Publish All"

plugs/publish/publish.ts

@@ -0,0 +1,208 @@
+import { asset, fs } from "$sb/plugos-syscall/mod.ts";
+import {
+  editor,
+  index,
+  markdown,
+  space,
+  system,
+} from "$sb/silverbullet-syscall/mod.ts";
+import { readYamlPage } from "$sb/lib/yaml_page.ts";
+import { renderMarkdownToHtml } from "../markdown/markdown_render.ts";
+
+import Handlebars from "handlebars";
+
+import {
+  collectNodesOfType,
+  findNodeOfType,
+  ParseTree,
+  renderToText,
+  replaceNodesMatching,
+} from "$sb/lib/tree.ts";
+
+type PublishConfig = {
+  destDir?: string;
+  title?: string;
+  indexPage?: string;
+  removeHashtags?: boolean;
+  publishAll?: boolean;
+  tags?: string[];
+  prefixes?: string[];
+  footerPage?: string;
+};
+
+async function generatePage(
+  pageName: string,
+  htmlPath: string,
+  mdPath: string,
+  publishedPages: string[],
+  publishConfig: PublishConfig,
+  destDir: string,
+  footerText: string,
+) {
+  const pageTemplate = await asset.readAsset("assets/page.hbs");
+  const pageCSS = await asset.readAsset("assets/style.css");
+  const text = await space.readPage(pageName);
+  const renderPage = Handlebars.compile(pageTemplate);
+  console.log("Writing", pageName);
+  const mdTree = await markdown.parseMarkdown(`${text}\n${footerText}`);
+  const publishMd = cleanMarkdown(
+    mdTree,
+    publishConfig,
+    publishedPages,
+  );
+  const attachments = await collectAttachments(mdTree);
+  for (const attachment of attachments) {
+    try {
+      const result = await space.readAttachment(attachment);
+      console.log("Writing", `${destDir}/${attachment}`);
+      await fs.writeFile(`${destDir}/${attachment}`, result, "dataurl");
+    } catch (e: any) {
+      console.error("Error reading attachment", attachment, e.message);
+    }
+  }
+  // Write .md file
+  await fs.writeFile(mdPath, publishMd);
+  // Write .html file
+  await fs.writeFile(
+    htmlPath,
+    renderPage({
+      pageName,
+      config: publishConfig,
+      css: pageCSS,
+      body: renderMarkdownToHtml(mdTree, {
+        smartHardBreak: true,
+        attachmentUrlPrefix: "/",
+      }),
+    }),
+  );
+}
+
+export async function publishAll(destDir?: string) {
+  const publishConfig: PublishConfig = await readYamlPage("PUBLISH");
+  destDir = destDir || publishConfig.destDir || ".";
+  console.log("Publishing to", destDir);
+  let allPages: any[] = await space.listPages();
+  let allPageMap: Map<string, any> = new Map(
+    allPages.map((pm) => [pm.name, pm]),
+  );
+  for (const { page, value } of await index.queryPrefix("meta:")) {
+    const p = allPageMap.get(page);
+    if (p) {
+      for (const [k, v] of Object.entries(value)) {
+        p[k] = v;
+      }
+    }
+  }
+
+  allPages = [...allPageMap.values()];
+  let publishedPages = new Set<string>();
+  if (publishConfig.publishAll) {
+    publishedPages = new Set(allPages.map((p) => p.name));
+  } else {
+    for (const page of allPages) {
+      if (publishConfig.tags && page.tags) {
+        for (const tag of page.tags) {
+          if (publishConfig.tags.includes(tag)) {
+            publishedPages.add(page.name);
+          }
+        }
+      }
+      // Some sanity checking
+      if (typeof page.name !== "string") {
+        continue;
+      }
+      if (publishConfig.prefixes) {
+        for (const prefix of publishConfig.prefixes) {
+          if (page.name.startsWith(prefix)) {
+            publishedPages.add(page.name);
+          }
+        }
+      }
+    }
+  }
+  console.log("Starting this thing", [...publishedPages]);
+
+  let footer = "";
+
+  if (publishConfig.footerPage) {
+    footer = await space.readPage(publishConfig.footerPage);
+  }
+
+  const publishedPagesArray = [...publishedPages];
+  for (const page of publishedPagesArray) {
+    await generatePage(
+      page,
+      `${destDir}/${page.replaceAll(" ", "_")}/index.html`,
+      `${destDir}/${page}.md`,
+      publishedPagesArray,
+      publishConfig,
+      destDir,
+      footer,
+    );
+  }
+
+  if (publishConfig.indexPage) {
+    console.log("Writing", publishConfig.indexPage);
+    await generatePage(
+      publishConfig.indexPage,
+      `${destDir}/index.html`,
+      `${destDir}/index.md`,
+      publishedPagesArray,
+      publishConfig,
+      destDir,
+      footer,
+    );
+  }
+}
+
+export async function publishAllCommand() {
+  await editor.flashNotification("Publishing...");
+  await await system.invokeFunction("server", "publishAll");
+  await editor.flashNotification("Done!");
+}
+
+export function encodePageUrl(name: string): string {
+  return name.replaceAll(" ", "_");
+}
+
+async function collectAttachments(tree: ParseTree) {
+  const attachments: string[] = [];
+  collectNodesOfType(tree, "URL").forEach((node) => {
+    let url = node.children![0].text!;
+    if (url.indexOf("://") === -1) {
+      attachments.push(url);
+    }
+  });
+  return attachments;
+}
+
+function cleanMarkdown(
+  mdTree: ParseTree,
+  publishConfig: PublishConfig,
+  validPages: string[],
+): string {
+  replaceNodesMatching(mdTree, (n) => {
+    if (n.type === "WikiLink") {
+      let page = n.children![1].children![0].text!;
+      if (page.includes("@")) {
+        page = page.split("@")[0];
+      }
+      if (!validPages.includes(page)) {
+        // Replace with just page text
+        return {
+          text: `_${page}_`,
+        };
+      }
+    }
+    // Simply get rid of these
+    if (n.type === "CommentBlock" || n.type === "Comment") {
+      return null;
+    }
+    if (n.type === "Hashtag") {
+      if (publishConfig.removeHashtags) {
+        return null;
+      }
+    }
+  });
+  return renderToText(mdTree).trim();
+}

scripts/build_website.sh

@@ -1,7 +1,5 @@
 #!/bin/bash
 
-rm -rf website_build
-npx --yes @silverbulletmd/publish -o website_build --index website
 
 echo "Now building Silver Bullet bundle"
 curl -fsSL https://deno.land/install.sh | sh
@@ -11,6 +9,11 @@ echo "Generating version number..."
 echo "export const version = '$(git rev-parse HEAD)';" > version.ts
 echo "Building..."
 deno task build
+deno task install
+
+rm -rf website_build
+silverbullet publish -o website_build --index website
+
 echo "Bundling..."
 deno task bundle
 cp dist/silverbullet.js website_build/

server/http_server.ts

@@ -20,7 +20,7 @@ import {
   fullTextSearchSyscalls,
 } from "../plugos/syscalls/fulltext.sqlite.ts";
 import sandboxSyscalls from "../plugos/syscalls/sandbox.ts";
-import shellSyscalls from "../plugos/syscalls/shell.node.ts";
+import shellSyscalls from "../plugos/syscalls/shell.deno.ts";
 import {
   ensureTable as ensureStoreTable,
   storeSyscalls,

silverbullet.ts

@@ -7,6 +7,7 @@ import { versionCommand } from "./cmd/version.ts";
 import { fixCommand } from "./cmd/fix.ts";
 import { serveCommand } from "./cmd/server.ts";
 import { plugCompileCommand } from "./cmd/plug_compile.ts";
+import { publishCommand } from "./cmd/publish.ts";
 
 await new Command()
   .name("silverbullet")
@@ -42,6 +43,14 @@ await new Command()
   )
   .option("--importmap <path:string>", "Path to import map file to use")
   .action(plugCompileCommand)
+  // publish
+  .command("publish")
+  .description("Publish a SilverBullet site")
+  .arguments("<folder:string>")
+  .option("--index [type:boolean]", "Index space first", { default: false })
+  .option("--watch, -w [type:boolean]", "Watch for changes", { default: false })
+  .option("--output, -o <path:string>", "Output directory", { default: "web" })
+  .action(publishCommand)
   // upgrade
   .command("upgrade", "Upgrade Silver Bullet")
   .action(upgradeCommand)

website/CHANGELOG.md

@@ -11,6 +11,11 @@ release.
 * New PlugOS feature: redirecting function calls. Instead of specifying a `path` for a function, you can now specify `redirect` pointing to another function name, either in the same plug using the `plugName.functionName` syntax.
 * `Cmd-click` or `Ctrl-click` now opens page references in a new window. You can `Alt-click` to put your cursor at the target without navigation.
 * The `Create page` option when navigating pages now always appears as the _second_ option. Let me know how you like it.
+* New `Preview` using a custom markdown renderer offering a lot of extra flexibility (and a much smaller file size). New thing it does:
+  * Render front matter in a table
+  * Makes {[Command buttons]} clickable
+  * Makes todo tasks toggleable
+* Integrated the `silverbullet-publish` plug into core (to be better documented later).
 
 ---
 

website/template/debug.md

@@ -1,4 +1,5 @@
 {{#each .}}
 {{@key}}: {{.}}
 {{/each}}
+
 ---

website/🔌 Directive.md

@@ -26,12 +26,13 @@ Whenever the directives are updated, the body of the directive will be replaced
 $use
 The `#use` directive can be used to use a referenced page as a handbars template. Optionally, a JSON object can be passed as argument to the template:
 
-    <!-- #use [[template/plug]] {"name": "Example plug", "repo": "https://google.com"} -->
+    <!-- #use [[template/plug]] {"name": "🔌 Directive", "repo": "https://google.com", "author": "Pete"} -->
+    
     <!-- /use -->
 
 which renders as follows:
-<!-- #use [[template/plug]] {"name": "Example plug", "repo": "https://google.com"} -->
-* [[Example plug]] by **** ([repo](https://google.com))
+<!-- #use [[template/plug]] {"name": "🔌 Directive", "repo": "https://google.com", "author": "Pete"} -->
+* [[🔌 Directive]] by **Pete** ([repo](https://google.com))
 <!-- /use -->
 
 Note that a string is also a valid JSON value:
@@ -45,7 +46,7 @@ Note that a string is also a valid JSON value:
 which renders as:
 
 <!-- #use [[template/tagged-tasks]] "#test" -->
-* [ ] [[🔌 Directive@1492]] This is a test task  #test
+* [ ] [[🔌 Directive@1537]] This is a test task  #test
 <!-- /use -->
 
 ## Eval
@@ -68,8 +69,8 @@ However, you can also invoke arbitrary plug functions, e.g. the `titleUnfurlOpti
 
 Optionally, you can use a `render` clause to render the result as a template, similar to [[🔌 Directive/Query]]:
 
-<!-- #eval core.titleUnfurlOptions() render [[template/debug]] -->
-id: title-unfurl
-name: Extract title
----
-<!-- /eval -->
+    <!-- #eval core.titleUnfurlOptions() render [[template/debug]] -->
+    id: title-unfurl
+    name: Extract title
+    ---
+    <!-- /eval -->