diff --git a/.gitignore b/.gitignore index 7ce82c4..eafef92 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,6 @@ dist *.js.map website_build data.db +publish-data.db /index.json .idea \ No newline at end of file diff --git a/cmd/plug_compile.ts b/cmd/plug_compile.ts index 724129f..714e2be 100644 --- a/cmd/plug_compile.ts +++ b/cmd/plug_compile.ts @@ -11,7 +11,6 @@ export async function plugCompileCommand( }, ...manifestPaths: string[] ) { - console.log("All optiosn", arguments); await bundleRun( manifestPaths, dist, diff --git a/cmd/publish.ts b/cmd/publish.ts new file mode 100755 index 0000000..685ae50 --- /dev/null +++ b/cmd/publish.ts @@ -0,0 +1,172 @@ +#!/usr/bin/env node +import { createSandbox } from "../plugos/environments/deno_sandbox.ts"; +import { EventHook } from "../plugos/hooks/event.ts"; +import { eventSyscalls } from "../plugos/syscalls/event.ts"; +import fileSystemSyscalls from "../plugos/syscalls/fs.deno.ts"; +import { + ensureFTSTable, + fullTextSearchSyscalls, +} from "../plugos/syscalls/fulltext.sqlite.ts"; +import sandboxSyscalls from "../plugos/syscalls/sandbox.ts"; +import shellSyscalls from "../plugos/syscalls/shell.deno.ts"; +import { + ensureTable as ensureStoreTable, + storeSyscalls, +} from "../plugos/syscalls/store.deno.ts"; +import { System } from "../plugos/system.ts"; +import { Manifest, SilverBulletHooks } from "../common/manifest.ts"; +import { loadMarkdownExtensions } from "../common/markdown_ext.ts"; +import buildMarkdown from "../common/parser.ts"; +import { DiskSpacePrimitives } from "../common/spaces/disk_space_primitives.ts"; +import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives.ts"; +import { Space } from "../common/spaces/space.ts"; +import { markdownSyscalls } from "../common/syscalls/markdown.ts"; +import { PageNamespaceHook } from "../server/hooks/page_namespace.ts"; +import { PlugSpacePrimitives } from "../server/hooks/plug_space_primitives.ts"; +import { + ensureTable as ensureIndexTable, + pageIndexSyscalls, +} from "../server/syscalls/index.ts"; +import spaceSyscalls from "../server/syscalls/space.ts"; + +import { Command } from "https://deno.land/x/cliffy@v0.25.2/command/command.ts"; + +import assetBundle from "../dist/asset_bundle.json" assert { type: "json" }; +import { AssetBundle, AssetJson } from "../plugos/asset_bundle/bundle.ts"; +import { path } from "../server/deps.ts"; +import { AsyncSQLite } from "../plugos/sqlite/async_sqlite.ts"; +import { AssetBundlePlugSpacePrimitives } from "../common/spaces/asset_bundle_space_primitives.ts"; +import assetSyscalls from "../plugos/syscalls/asset.ts"; +import { faBullseye } from "https://esm.sh/v96/@fortawesome/free-solid-svg-icons@6.2.0/index.d.ts"; + +export async function publishCommand(options: { + index: boolean; + watch: boolean; + output: string; +}, pagesPath: string) { + const assets = new AssetBundle(assetBundle as AssetJson); + // Set up the PlugOS System + const system = new System("server"); + + // Instantiate the event bus hook + const eventHook = new EventHook(); + system.addHook(eventHook); + + // And the page namespace hook + const namespaceHook = new PageNamespaceHook(); + system.addHook(namespaceHook); + + pagesPath = path.resolve(pagesPath); + + // The space + const space = new Space( + new AssetBundlePlugSpacePrimitives( + new EventedSpacePrimitives( + new PlugSpacePrimitives( + new DiskSpacePrimitives(pagesPath), + namespaceHook, + ), + eventHook, + ), + assets, + ), + ); + + await space.updatePageList(); + + // The database used for persistence (SQLite) + const db = new AsyncSQLite(path.join(pagesPath, "publish-data.db")); + db.init().catch((e) => { + console.error("Error initializing database", e); + }); + + // Register syscalls available on the server side + system.registerSyscalls( + [], + pageIndexSyscalls(db), + storeSyscalls(db, "store"), + fullTextSearchSyscalls(db, "fts"), + spaceSyscalls(space), + eventSyscalls(eventHook), + markdownSyscalls(buildMarkdown([])), + sandboxSyscalls(system), + assetSyscalls(system), + ); + // Danger zone + system.registerSyscalls(["shell"], shellSyscalls(pagesPath)); + system.registerSyscalls(["fs"], fileSystemSyscalls("/")); + + const globalModules = JSON.parse( + assets.readTextFileSync(`web/global.plug.json`), + ); + + system.on({ + sandboxInitialized: async (sandbox) => { + for ( + const [modName, code] of Object.entries( + globalModules.dependencies, + ) + ) { + await sandbox.loadDependency(modName, code as string); + } + }, + }); + + await space.updatePageList(); + + const allPlugs = await space.listPlugs(); + + console.log("Loading plugs", allPlugs); + await Promise.all((await space.listPlugs()).map(async (plugName) => { + const { data } = await space.readAttachment(plugName, "string"); + await system.load(JSON.parse(data as string), createSandbox); + })); + + const corePlug = system.loadedPlugs.get("core"); + if (!corePlug) { + console.error("Something went very wrong, 'core' plug not found"); + return; + } + + system.registerSyscalls( + [], + markdownSyscalls(buildMarkdown(loadMarkdownExtensions(system))), + ); + + await ensureIndexTable(db); + await ensureStoreTable(db, "store"); + await ensureFTSTable(db, "fts"); + + if (options.index) { + console.log("Now indexing space"); + await corePlug.invoke("reindexSpace", []); + } + + const outputDir = path.resolve(options.output); + + await Deno.mkdir(outputDir, { recursive: true }); + + const publishPlug = system.loadedPlugs.get("publish")!; + + await publishPlug.invoke("publishAll", [outputDir]); + + if (options.watch) { + console.log("Watching for changes"); + let building = false; + for await (const _event of Deno.watchFs(pagesPath, { recursive: true })) { + console.log("Change detected, republishing"); + if (building) { + continue; + } + building = true; + space.updatePageList().then(async () => { + await publishPlug.invoke("publishAll", [outputDir]); + building = false; + }); + } + } else { + console.log("Done!"); + Deno.exit(0); + // process.exit(0); + } +} diff --git a/plugos/syscalls/shell.node.ts b/plugos/syscalls/shell.deno.ts similarity index 100% rename from plugos/syscalls/shell.node.ts rename to plugos/syscalls/shell.deno.ts diff --git a/plugs/markdown/markdown_render.ts b/plugs/markdown/markdown_render.ts index a707268..0dfa118 100644 --- a/plugs/markdown/markdown_render.ts +++ b/plugs/markdown/markdown_render.ts @@ -12,6 +12,7 @@ type MarkdownRenderOptions = { smartHardBreak?: true; annotationPositions?: true; renderFrontMatter?: true; + attachmentUrlPrefix?: string; }; function cleanTags(values: (Tag | null)[]): Tag[] { @@ -138,11 +139,15 @@ function render( // Code blocks case "FencedCode": case "CodeBlock": { + // Clear out top-level indent blocks + t.children = t.children!.filter((c) => c.type); return { name: "pre", body: cleanTags(mapRender(t.children!)), }; } + case "CodeInfo": + return null; case "CodeText": return t.children![0].text!; case "Blockquote": @@ -201,7 +206,14 @@ function render( }; case "Link": { const linkText = t.children![1].text!; - const url = findNodeOfType(t, "URL")!.children![0].text!; + const urlNode = findNodeOfType(t, "URL"); + if (!urlNode) { + return renderToText(t); + } + let url = urlNode.children![0].text!; + if (url.indexOf("://") === -1) { + url = `${options.attachmentUrlPrefix || ""}${url}`; + } return { name: "a", attrs: { @@ -212,9 +224,13 @@ function render( } case "Image": { const altText = t.children![1].text!; - let url = findNodeOfType(t, "URL")!.children![0].text!; + const urlNode = findNodeOfType(t, "URL"); + if (!urlNode) { + return renderToText(t); + } + let url = urlNode!.children![0].text!; if (url.indexOf("://") === -1) { - url = `fs/${url}`; + url = `${options.attachmentUrlPrefix || ""}${url}`; } return { name: "img", @@ -233,7 +249,7 @@ function render( return { name: "a", attrs: { - href: `/${ref}`, + href: `/${ref.replaceAll(" ", "_").replace("@", "#")}`, }, body: ref, }; diff --git a/plugs/markdown/preview.ts b/plugs/markdown/preview.ts index 97b6ea2..e61ee4e 100644 --- a/plugs/markdown/preview.ts +++ b/plugs/markdown/preview.ts @@ -16,6 +16,7 @@ export async function updateMarkdownPreview() { smartHardBreak: true, annotationPositions: true, renderFrontMatter: true, + attachmentUrlPrefix: "fs/", }); await editor.showPanel( "rhs", diff --git a/plugs/markdown/test/example.md b/plugs/markdown/test/example.md index 05e03d1..8b42a2c 100644 --- a/plugs/markdown/test/example.md +++ b/plugs/markdown/test/example.md @@ -79,4 +79,8 @@ Here is something A new thing. -![alt text](https://image.jpg) \ No newline at end of file +![alt text](https:/ + +## Weird stuff + +* [{{#if done}}x{{else}} {{/if}}] [[{{page}}@{{pos}}]] {{name}} {{#if deadline}}📅 {{deadline}}{{/if}} {{#each tags}}{{.}} {{/each}} \ No newline at end of file diff --git a/plugs/publish/README.md b/plugs/publish/README.md new file mode 100644 index 0000000..6474820 --- /dev/null +++ b/plugs/publish/README.md @@ -0,0 +1,45 @@ +# Silver Bullet Publish +A simple tool to export a subset of your [SilverBullet](https://silverbullet.md) space as a static website. + +**Note:** this is highly experimental and not necessarily production ready code, use at your own risk. + +silverbullet-publish currentenly publishes a subset of a space in two formats: + +* Markdown (.md files) +* HTML (.html files based on currently hardcoded templates (see `page.hbs` and `style.css`) + +The tool can be run in two ways: + +1. As a Silver Bullet plug (via the `Silver Bullet Publish: Publish All` command) +2. As a stand-alone CLI tool (via `npx`) + +The latter allows for automatic deployments to e.g. environments like Netlify. + +## Configuration +SilverBullet Publish is configured via the `PUBLISH` page with the following properties: + + ```yaml + # Index page to use for public version + indexPage: Public + # Optional destination folder when used in plug mode + destDir: /Users/you/my-website + title: Name of the space + removeHashtags: true + removeUnpublishedLinks: false + # Publish all pages with specific tag + tags: + - "#pub" + # Publish all pages with a specifix prefix + prefixes: + - /public + ``` + +## Running via `npx` +The easiest way to run SilverBullet Publish is via `npx`, it takes a few optional arguments beyond the path to your SilverBullet space: + +* `-o` specifies where to write the output to (defaults to `./web`) +* `--index` runs a full space index (e.g. to index all hash tags) before publishing, this is primarily useful when run in a CI/CI pipeline (like Netlify) because there no `data.db` in your repo containing this index. + +```bash +npx @silverbulletmd/publish -o web_build --index . +``` \ No newline at end of file diff --git a/plugs/publish/assets/page.hbs b/plugs/publish/assets/page.hbs new file mode 100644 index 0000000..8ac793b --- /dev/null +++ b/plugs/publish/assets/page.hbs @@ -0,0 +1,16 @@ + + + + + + +{{#if pageName}}{{pageName}} — {{config.title}}{{else}}{{config.title}}{{/if}} + + + +

{{pageName}}

+{{{body}}} + + diff --git a/plugs/publish/assets/style.css b/plugs/publish/assets/style.css new file mode 100644 index 0000000..59ef324 --- /dev/null +++ b/plugs/publish/assets/style.css @@ -0,0 +1,53 @@ +body { + font-family: georgia, times, serif; + font-size: 14pt; + max-width: 800px; + margin-left: auto; + margin-right: auto; + padding-left: 20px; + padding-right: 20px; +} + +table { + width: 100%; + border-spacing: 0; +} + +thead tr { + background-color: #333; + color: #eee; +} + +th, +td { + padding: 8px; +} + +tbody tr:nth-of-type(even) { + background-color: #f3f3f3; +} + +ul li p { + margin: 0; +} + +a[href] { + text-decoration: none; +} + +blockquote { + border-left: 1px solid #333; + margin-left: 2px; + padding-left: 10px; +} + +.footer { + border-top: 1px solid #000; + padding-top: 5px; + text-align: center; + font-size: 70%; +} + +img { + max-width: 90%; +} diff --git a/plugs/publish/publish.plug.yaml b/plugs/publish/publish.plug.yaml new file mode 100644 index 0000000..5763049 --- /dev/null +++ b/plugs/publish/publish.plug.yaml @@ -0,0 +1,15 @@ +name: publish +imports: + - https://get.silverbullet.md/global.plug.json +requiredPermissions: + - fs +assets: + - "assets/*" +functions: + publishAll: + path: "./publish.ts:publishAll" + env: server + publishAllCommand: + path: "./publish.ts:publishAllCommand" + command: + name: "Silver Bullet Publish: Publish All" diff --git a/plugs/publish/publish.ts b/plugs/publish/publish.ts new file mode 100644 index 0000000..df1d1d8 --- /dev/null +++ b/plugs/publish/publish.ts @@ -0,0 +1,208 @@ +import { asset, fs } from "$sb/plugos-syscall/mod.ts"; +import { + editor, + index, + markdown, + space, + system, +} from "$sb/silverbullet-syscall/mod.ts"; +import { readYamlPage } from "$sb/lib/yaml_page.ts"; +import { renderMarkdownToHtml } from "../markdown/markdown_render.ts"; + +import Handlebars from "handlebars"; + +import { + collectNodesOfType, + findNodeOfType, + ParseTree, + renderToText, + replaceNodesMatching, +} from "$sb/lib/tree.ts"; + +type PublishConfig = { + destDir?: string; + title?: string; + indexPage?: string; + removeHashtags?: boolean; + publishAll?: boolean; + tags?: string[]; + prefixes?: string[]; + footerPage?: string; +}; + +async function generatePage( + pageName: string, + htmlPath: string, + mdPath: string, + publishedPages: string[], + publishConfig: PublishConfig, + destDir: string, + footerText: string, +) { + const pageTemplate = await asset.readAsset("assets/page.hbs"); + const pageCSS = await asset.readAsset("assets/style.css"); + const text = await space.readPage(pageName); + const renderPage = Handlebars.compile(pageTemplate); + console.log("Writing", pageName); + const mdTree = await markdown.parseMarkdown(`${text}\n${footerText}`); + const publishMd = cleanMarkdown( + mdTree, + publishConfig, + publishedPages, + ); + const attachments = await collectAttachments(mdTree); + for (const attachment of attachments) { + try { + const result = await space.readAttachment(attachment); + console.log("Writing", `${destDir}/${attachment}`); + await fs.writeFile(`${destDir}/${attachment}`, result, "dataurl"); + } catch (e: any) { + console.error("Error reading attachment", attachment, e.message); + } + } + // Write .md file + await fs.writeFile(mdPath, publishMd); + // Write .html file + await fs.writeFile( + htmlPath, + renderPage({ + pageName, + config: publishConfig, + css: pageCSS, + body: renderMarkdownToHtml(mdTree, { + smartHardBreak: true, + attachmentUrlPrefix: "/", + }), + }), + ); +} + +export async function publishAll(destDir?: string) { + const publishConfig: PublishConfig = await readYamlPage("PUBLISH"); + destDir = destDir || publishConfig.destDir || "."; + console.log("Publishing to", destDir); + let allPages: any[] = await space.listPages(); + let allPageMap: Map = new Map( + allPages.map((pm) => [pm.name, pm]), + ); + for (const { page, value } of await index.queryPrefix("meta:")) { + const p = allPageMap.get(page); + if (p) { + for (const [k, v] of Object.entries(value)) { + p[k] = v; + } + } + } + + allPages = [...allPageMap.values()]; + let publishedPages = new Set(); + if (publishConfig.publishAll) { + publishedPages = new Set(allPages.map((p) => p.name)); + } else { + for (const page of allPages) { + if (publishConfig.tags && page.tags) { + for (const tag of page.tags) { + if (publishConfig.tags.includes(tag)) { + publishedPages.add(page.name); + } + } + } + // Some sanity checking + if (typeof page.name !== "string") { + continue; + } + if (publishConfig.prefixes) { + for (const prefix of publishConfig.prefixes) { + if (page.name.startsWith(prefix)) { + publishedPages.add(page.name); + } + } + } + } + } + console.log("Starting this thing", [...publishedPages]); + + let footer = ""; + + if (publishConfig.footerPage) { + footer = await space.readPage(publishConfig.footerPage); + } + + const publishedPagesArray = [...publishedPages]; + for (const page of publishedPagesArray) { + await generatePage( + page, + `${destDir}/${page.replaceAll(" ", "_")}/index.html`, + `${destDir}/${page}.md`, + publishedPagesArray, + publishConfig, + destDir, + footer, + ); + } + + if (publishConfig.indexPage) { + console.log("Writing", publishConfig.indexPage); + await generatePage( + publishConfig.indexPage, + `${destDir}/index.html`, + `${destDir}/index.md`, + publishedPagesArray, + publishConfig, + destDir, + footer, + ); + } +} + +export async function publishAllCommand() { + await editor.flashNotification("Publishing..."); + await await system.invokeFunction("server", "publishAll"); + await editor.flashNotification("Done!"); +} + +export function encodePageUrl(name: string): string { + return name.replaceAll(" ", "_"); +} + +async function collectAttachments(tree: ParseTree) { + const attachments: string[] = []; + collectNodesOfType(tree, "URL").forEach((node) => { + let url = node.children![0].text!; + if (url.indexOf("://") === -1) { + attachments.push(url); + } + }); + return attachments; +} + +function cleanMarkdown( + mdTree: ParseTree, + publishConfig: PublishConfig, + validPages: string[], +): string { + replaceNodesMatching(mdTree, (n) => { + if (n.type === "WikiLink") { + let page = n.children![1].children![0].text!; + if (page.includes("@")) { + page = page.split("@")[0]; + } + if (!validPages.includes(page)) { + // Replace with just page text + return { + text: `_${page}_`, + }; + } + } + // Simply get rid of these + if (n.type === "CommentBlock" || n.type === "Comment") { + return null; + } + if (n.type === "Hashtag") { + if (publishConfig.removeHashtags) { + return null; + } + } + }); + return renderToText(mdTree).trim(); +} diff --git a/scripts/build_website.sh b/scripts/build_website.sh index 336388e..805fa9f 100755 --- a/scripts/build_website.sh +++ b/scripts/build_website.sh @@ -1,7 +1,5 @@ #!/bin/bash -rm -rf website_build -npx --yes @silverbulletmd/publish -o website_build --index website echo "Now building Silver Bullet bundle" curl -fsSL https://deno.land/install.sh | sh @@ -11,6 +9,11 @@ echo "Generating version number..." echo "export const version = '$(git rev-parse HEAD)';" > version.ts echo "Building..." deno task build +deno task install + +rm -rf website_build +silverbullet publish -o website_build --index website + echo "Bundling..." deno task bundle cp dist/silverbullet.js website_build/ diff --git a/server/http_server.ts b/server/http_server.ts index 67066dd..85ea081 100644 --- a/server/http_server.ts +++ b/server/http_server.ts @@ -20,7 +20,7 @@ import { fullTextSearchSyscalls, } from "../plugos/syscalls/fulltext.sqlite.ts"; import sandboxSyscalls from "../plugos/syscalls/sandbox.ts"; -import shellSyscalls from "../plugos/syscalls/shell.node.ts"; +import shellSyscalls from "../plugos/syscalls/shell.deno.ts"; import { ensureTable as ensureStoreTable, storeSyscalls, diff --git a/silverbullet.ts b/silverbullet.ts index 4f3eeef..19b26e8 100755 --- a/silverbullet.ts +++ b/silverbullet.ts @@ -7,6 +7,7 @@ import { versionCommand } from "./cmd/version.ts"; import { fixCommand } from "./cmd/fix.ts"; import { serveCommand } from "./cmd/server.ts"; import { plugCompileCommand } from "./cmd/plug_compile.ts"; +import { publishCommand } from "./cmd/publish.ts"; await new Command() .name("silverbullet") @@ -42,6 +43,14 @@ await new Command() ) .option("--importmap ", "Path to import map file to use") .action(plugCompileCommand) + // publish + .command("publish") + .description("Publish a SilverBullet site") + .arguments("") + .option("--index [type:boolean]", "Index space first", { default: false }) + .option("--watch, -w [type:boolean]", "Watch for changes", { default: false }) + .option("--output, -o ", "Output directory", { default: "web" }) + .action(publishCommand) // upgrade .command("upgrade", "Upgrade Silver Bullet") .action(upgradeCommand) diff --git a/website/CHANGELOG.md b/website/CHANGELOG.md index 6ad8531..5dda708 100644 --- a/website/CHANGELOG.md +++ b/website/CHANGELOG.md @@ -11,6 +11,11 @@ release. * New PlugOS feature: redirecting function calls. Instead of specifying a `path` for a function, you can now specify `redirect` pointing to another function name, either in the same plug using the `plugName.functionName` syntax. * `Cmd-click` or `Ctrl-click` now opens page references in a new window. You can `Alt-click` to put your cursor at the target without navigation. * The `Create page` option when navigating pages now always appears as the _second_ option. Let me know how you like it. +* New `Preview` using a custom markdown renderer offering a lot of extra flexibility (and a much smaller file size). New thing it does: + * Render front matter in a table + * Makes {[Command buttons]} clickable + * Makes todo tasks toggleable +* Integrated the `silverbullet-publish` plug into core (to be better documented later). --- diff --git a/website/template/debug.md b/website/template/debug.md index 45b06d6..939ac19 100644 --- a/website/template/debug.md +++ b/website/template/debug.md @@ -1,4 +1,5 @@ {{#each .}} {{@key}}: {{.}} {{/each}} + --- \ No newline at end of file diff --git a/website/🔌 Directive.md b/website/🔌 Directive.md index 2aed869..fde4080 100644 --- a/website/🔌 Directive.md +++ b/website/🔌 Directive.md @@ -26,12 +26,13 @@ Whenever the directives are updated, the body of the directive will be replaced $use The `#use` directive can be used to use a referenced page as a handbars template. Optionally, a JSON object can be passed as argument to the template: - + + which renders as follows: - -* [[Example plug]] by **** ([repo](https://google.com)) + +* [[🔌 Directive]] by **Pete** ([repo](https://google.com)) Note that a string is also a valid JSON value: @@ -45,7 +46,7 @@ Note that a string is also a valid JSON value: which renders as: -* [ ] [[🔌 Directive@1492]] This is a test task #test +* [ ] [[🔌 Directive@1537]] This is a test task #test ## Eval @@ -68,8 +69,8 @@ However, you can also invoke arbitrary plug functions, e.g. the `titleUnfurlOpti Optionally, you can use a `render` clause to render the result as a template, similar to [[🔌 Directive/Query]]: - -id: title-unfurl -name: Extract title ---- - + + id: title-unfurl + name: Extract title + --- +