1
0

WIP: CLI running of plugs

This commit is contained in:
Zef Hemel 2023-08-04 18:56:55 +02:00
parent de3e385017
commit 3464af0252
35 changed files with 738 additions and 136 deletions

38
cli/plug_run.test.ts Normal file
View File

@ -0,0 +1,38 @@
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
import { compileManifest } from "../plugos/compile.ts";
import { esbuild } from "../plugos/deps.ts";
import { runPlug } from "./plug_run.ts";
import assets from "../dist/plug_asset_bundle.json" assert {
type: "json",
};
import { assertEquals } from "../test_deps.ts";
import { path } from "../common/deps.ts";
Deno.test("Test plug run", async () => {
// const tempDir = await Deno.makeTempDir();
const assetBundle = new AssetBundle(assets);
const testFolder = path.dirname(new URL(import.meta.url).pathname);
const testSpaceFolder = path.join(testFolder, "test_space");
const plugFolder = path.join(testSpaceFolder, "_plug");
await Deno.mkdir(plugFolder, { recursive: true });
await compileManifest(
path.join(testFolder, "test.plug.yaml"),
plugFolder,
);
assertEquals(
await runPlug(
testSpaceFolder,
"test.run",
[],
assetBundle,
),
"Hello",
);
// await Deno.remove(tempDir, { recursive: true });
esbuild.stop();
});

145
cli/plug_run.ts Normal file
View File

@ -0,0 +1,145 @@
import { path } from "../common/deps.ts";
import { PlugNamespaceHook } from "../common/hooks/plug_namespace.ts";
import { SilverBulletHooks } from "../common/manifest.ts";
import { loadMarkdownExtensions } from "../common/markdown_parser/markdown_ext.ts";
import buildMarkdown from "../common/markdown_parser/parser.ts";
import { DiskSpacePrimitives } from "../common/spaces/disk_space_primitives.ts";
import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives.ts";
import { FileMetaSpacePrimitives } from "../common/spaces/file_meta_space_primitives.ts";
import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts";
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
import { createSandbox } from "../plugos/environments/deno_sandbox.ts";
import { CronHook } from "../plugos/hooks/cron.ts";
import { EventHook } from "../plugos/hooks/event.ts";
import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts";
import assetSyscalls from "../plugos/syscalls/asset.ts";
import { eventSyscalls } from "../plugos/syscalls/event.ts";
import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts";
import { shellSyscalls } from "../plugos/syscalls/shell.deno.ts";
import { storeSyscalls } from "../plugos/syscalls/store.ts";
import { System } from "../plugos/system.ts";
import { Space } from "../web/space.ts";
import { debugSyscalls } from "../web/syscalls/debug.ts";
import { markdownSyscalls } from "../web/syscalls/markdown.ts";
import { systemSyscalls } from "../web/syscalls/system.ts";
import { yamlSyscalls } from "../web/syscalls/yaml.ts";
import { pageIndexSyscalls } from "./syscalls/index.ts";
import { spaceSyscalls } from "./syscalls/space.ts";
export async function runPlug(
spacePath: string,
functionName: string,
args: string[] = [],
builtinAssetBundle: AssetBundle,
indexFirst = false,
) {
spacePath = path.resolve(spacePath);
const system = new System<SilverBulletHooks>("cli");
// Event hook
const eventHook = new EventHook();
system.addHook(eventHook);
// Cron hook
const cronHook = new CronHook(system);
system.addHook(cronHook);
const pageIndexCalls = pageIndexSyscalls("run.db");
// TODO: Add endpoint
const plugNamespaceHook = new PlugNamespaceHook();
system.addHook(plugNamespaceHook);
const spacePrimitives = new FileMetaSpacePrimitives(
new EventedSpacePrimitives(
new PlugSpacePrimitives(
new DiskSpacePrimitives(spacePath),
plugNamespaceHook,
),
eventHook,
),
pageIndexCalls,
);
const kvStore = new JSONKVStore();
const space = new Space(spacePrimitives, kvStore);
// Add syscalls
system.registerSyscalls(
[],
eventSyscalls(eventHook),
spaceSyscalls(space),
assetSyscalls(system),
yamlSyscalls(),
storeSyscalls(kvStore),
systemSyscalls(undefined as any, system),
pageIndexCalls,
debugSyscalls(),
markdownSyscalls(buildMarkdown([])), // Will later be replaced with markdown extensions
);
// Syscalls that require some additional permissions
system.registerSyscalls(
["fetch"],
sandboxFetchSyscalls(),
);
system.registerSyscalls(
["shell"],
shellSyscalls("."),
);
await loadPlugsFromAssetBundle(system, builtinAssetBundle);
for (let plugPath of await space.listPlugs()) {
plugPath = path.resolve(spacePath, plugPath);
await system.load(
new URL(`file://${plugPath}`),
createSandbox,
);
}
// Load markdown syscalls based on all new syntax (if any)
system.registerSyscalls(
[],
markdownSyscalls(buildMarkdown(loadMarkdownExtensions(system))),
);
if (indexFirst) {
await system.loadedPlugs.get("core")!.invoke("reindexSpace", []);
}
const [plugName, funcName] = functionName.split(".");
const plug = system.loadedPlugs.get(plugName);
if (!plug) {
throw new Error(`Plug ${plugName} not found`);
}
const result = await plug.invoke(funcName, args);
await system.unloadAll();
await pageIndexCalls["index.close"]({} as any);
return result;
}
async function loadPlugsFromAssetBundle(
system: System<any>,
assetBundle: AssetBundle,
) {
const tempDir = await Deno.makeTempDir();
try {
for (const filePath of assetBundle.listFiles()) {
if (filePath.endsWith(".plug.js")) {
const plugPath = path.join(tempDir, filePath);
await Deno.mkdir(path.dirname(plugPath), { recursive: true });
await Deno.writeFile(plugPath, assetBundle.readFileSync(filePath));
await system.load(
new URL(`file://${plugPath}`),
createSandbox,
);
}
}
} finally {
await Deno.remove(tempDir, { recursive: true });
}
}

7
cli/plug_test.ts Normal file
View File

@ -0,0 +1,7 @@
import { index } from "$sb/silverbullet-syscall/mod.ts";
export async function run() {
console.log("Hello from plug_test.ts");
console.log(await index.queryPrefix(`tag:`));
return "Hello";
}

View File

@ -0,0 +1,34 @@
import { assertEquals } from "../../test_deps.ts";
import { pageIndexSyscalls } from "./index.ts";
Deno.test("Test KV index", async () => {
const ctx: any = {};
const calls = pageIndexSyscalls();
await calls["index.set"](ctx, "page", "test", "value");
assertEquals(await calls["index.get"](ctx, "page", "test"), "value");
await calls["index.delete"](ctx, "page", "test");
assertEquals(await calls["index.get"](ctx, "page", "test"), null);
await calls["index.batchSet"](ctx, "page", [{
key: "attr:test",
value: "value",
}, {
key: "attr:test2",
value: "value2",
}, { key: "random", value: "value3" }]);
await calls["index.batchSet"](ctx, "page2", [{
key: "attr:test",
value: "value",
}, {
key: "attr:test2",
value: "value2",
}, { key: "random", value: "value3" }]);
let results = await calls["index.queryPrefix"](ctx, "attr:");
assertEquals(results.length, 4);
await calls["index.clearPageIndexForPage"](ctx, "page");
results = await calls["index.queryPrefix"](ctx, "attr:");
assertEquals(results.length, 2);
await calls["index.clearPageIndex"](ctx);
results = await calls["index.queryPrefix"](ctx, "");
assertEquals(results.length, 0);
await calls["index.close"](ctx);
});

103
cli/syscalls/index.ts Normal file
View File

@ -0,0 +1,103 @@
/// <reference lib="deno.unstable" />
import type { SysCallMapping } from "../../plugos/system.ts";
type Item = {
page: string;
key: string;
value: any;
};
export type KV = {
key: string;
value: any;
};
// Keyspace:
// ["index", page, key] -> value
// ["indexByKey", key, page] -> value
/**
* Implements the index syscalls using Deno's KV store.
* @param dbFile
* @returns
*/
export function pageIndexSyscalls(dbFile?: string): SysCallMapping {
const kv = Deno.openKv(dbFile);
const apiObj: SysCallMapping = {
"index.set": async (_ctx, page: string, key: string, value: any) => {
const res = await (await kv).atomic()
.set(["index", page, key], value)
.set(["indexByKey", key, page], value)
.commit();
if (!res.ok) {
throw res;
}
},
"index.batchSet": async (_ctx, page: string, kvs: KV[]) => {
// await items.bulkPut(kvs);
for (const { key, value } of kvs) {
await apiObj["index.set"](_ctx, page, key, value);
}
},
"index.delete": async (_ctx, page: string, key: string) => {
const res = await (await kv).atomic()
.delete(["index", page, key])
.delete(["indexByKey", key, page])
.commit();
if (!res.ok) {
throw res;
}
},
"index.get": async (_ctx, page: string, key: string) => {
return (await (await kv).get(["index", page, key])).value;
},
"index.queryPrefix": async (_ctx, prefix: string) => {
const results: { key: string; page: string; value: any }[] = [];
for await (
const result of (await kv).list({
start: ["indexByKey", prefix],
end: [
"indexByKey",
prefix.slice(0, -1) +
// This is a hack to get the next character in the ASCII table (e.g. "a" -> "b")
String.fromCharCode(prefix.charCodeAt(prefix.length - 1) + 1),
],
})
) {
results.push({
key: result.key[1] as string,
page: result.key[2] as string,
value: result.value,
});
}
return results;
},
"index.clearPageIndexForPage": async (ctx, page: string) => {
await apiObj["index.deletePrefixForPage"](ctx, page, "");
},
"index.deletePrefixForPage": async (_ctx, page: string, prefix: string) => {
for await (
const result of (await kv).list({
start: ["index", page, prefix],
end: ["index", page, prefix + "~"],
})
) {
await apiObj["index.delete"](_ctx, page, result.key[2]);
}
},
"index.clearPageIndex": async (ctx) => {
for await (
const result of (await kv).list({
prefix: ["index"],
})
) {
await apiObj["index.delete"](ctx, result.key[1], result.key[2]);
}
},
"index.close": async () => {
(await kv).close();
},
};
return apiObj;
}

61
cli/syscalls/space.ts Normal file
View File

@ -0,0 +1,61 @@
import { SysCallMapping } from "../../plugos/system.ts";
import type { Space } from "../../web/space.ts";
import { AttachmentMeta, PageMeta } from "../../web/types.ts";
/**
* Almost the same as web/syscalls/space.ts except leaving out client-specific stuff
*/
export function spaceSyscalls(space: Space): SysCallMapping {
return {
"space.listPages": (): Promise<PageMeta[]> => {
return space.fetchPageList();
},
"space.readPage": async (
_ctx,
name: string,
): Promise<string> => {
return (await space.readPage(name)).text;
},
"space.getPageMeta": (_ctx, name: string): Promise<PageMeta> => {
return space.getPageMeta(name);
},
"space.writePage": (
_ctx,
name: string,
text: string,
): Promise<PageMeta> => {
return space.writePage(name, text);
},
"space.deletePage": async (_ctx, name: string) => {
await space.deletePage(name);
},
"space.listPlugs": (): Promise<string[]> => {
return space.listPlugs();
},
"space.listAttachments": async (): Promise<AttachmentMeta[]> => {
return await space.fetchAttachmentList();
},
"space.readAttachment": async (
_ctx,
name: string,
): Promise<Uint8Array> => {
return (await space.readAttachment(name)).data;
},
"space.getAttachmentMeta": async (
_ctx,
name: string,
): Promise<AttachmentMeta> => {
return await space.getAttachmentMeta(name);
},
"space.writeAttachment": (
_ctx,
name: string,
data: Uint8Array,
): Promise<AttachmentMeta> => {
return space.writeAttachment(name, data);
},
"space.deleteAttachment": async (_ctx, name: string) => {
await space.deleteAttachment(name);
},
};
}

6
cli/test.plug.yaml Normal file
View File

@ -0,0 +1,6 @@
name: test
requiredPermissions:
- shell
functions:
run:
path: plug_test.ts:run

34
cmd/plug_run.ts Normal file
View File

@ -0,0 +1,34 @@
import { runPlug } from "../cli/plug_run.ts";
import { path } from "../common/deps.ts";
import assets from "../dist/plug_asset_bundle.json" assert {
type: "json",
};
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
export async function plugRunCommand(
{
noIndex,
}: {
noIndex: boolean;
},
spacePath: string,
functionName: string,
...args: string[]
) {
spacePath = path.resolve(spacePath);
console.log("Space path", spacePath);
console.log("Function to run:", functionName, "with arguments", args);
try {
const result = await runPlug(
spacePath,
functionName,
args,
new AssetBundle(assets),
!noIndex,
);
console.log("Output", result);
} catch (e: any) {
console.error(e.message);
Deno.exit(1);
}
}

View File

@ -2,7 +2,7 @@
"tasks": {
"clean": "rm -rf dist dist_client_bundle dist_plug_bundle website_build",
"deep-clean-mac": "rm -f deno.lock && rm -rf $HOME/Library/Caches/deno && deno task clean",
"install": "deno install -f -A --importmap import_map.json silverbullet.ts",
"install": "deno install -f --unstable -A --importmap import_map.json silverbullet.ts",
"check": "find . -name '*.ts*' | xargs deno check",
"test": "deno test -A --unstable",
"build": "deno run -A build_plugs.ts && deno run -A --unstable build_web.ts",

View File

@ -6,8 +6,8 @@ export function createSandbox<HookT>(plug: Plug<HookT>): Sandbox<HookT> {
return new Sandbox(plug, {
deno: {
permissions: {
// Disallow network access
net: false,
// Allow network access
net: true,
// This is required for console logging to work, apparently?
env: true,
// No talking to native code

View File

@ -0,0 +1,29 @@
import { assertEquals } from "../../test_deps.ts";
import { DenoKVStore } from "./kv_store.deno_kv.ts";
Deno.test("Test KV index", async () => {
const kv = new DenoKVStore();
await kv.init("test.db");
await kv.set("name", "Peter");
assertEquals(await kv.get("name"), "Peter");
await kv.del("name");
assertEquals(await kv.has("name"), false);
await kv.batchSet([
{ key: "page:hello", value: "Hello" },
{ key: "page:hello2", value: "Hello 2" },
{ key: "page:hello3", value: "Hello 3" },
{ key: "something", value: "Something" },
]);
const results = await kv.queryPrefix("page:");
assertEquals(results.length, 3);
assertEquals(await kv.batchGet(["page:hello", "page:hello3"]), [
"Hello",
"Hello 3",
]);
await kv.delete();
});

View File

@ -0,0 +1,102 @@
/// <reference lib="deno.unstable" />
import { KV, KVStore } from "./kv_store.ts";
export class DenoKVStore implements KVStore {
kv!: Deno.Kv;
path: string | undefined;
async init(path?: string) {
this.path = path;
this.kv = await Deno.openKv(path);
}
close() {
this.kv.close();
}
async delete() {
this.kv.close();
if (this.path) {
await Deno.remove(this.path);
}
}
async del(key: string): Promise<void> {
const res = await this.kv.atomic()
.delete([key])
.commit();
if (!res.ok) {
throw res;
}
}
async deletePrefix(prefix: string): Promise<void> {
for await (
const result of this.kv.list({
start: [prefix],
end: [endRange(prefix)],
})
) {
await this.del(result.key[0] as string);
}
}
async deleteAll(): Promise<void> {
for await (
const result of this.kv.list({ prefix: [] })
) {
await this.del(result.key[0] as string);
}
}
async set(key: string, value: any): Promise<void> {
const res = await this.kv.atomic()
.set([key], value)
.commit();
if (!res.ok) {
throw res;
}
}
async batchSet(kvs: KV[]): Promise<void> {
for (const { key, value } of kvs) {
await this.set(key, value);
}
}
async batchDelete(keys: string[]): Promise<void> {
for (const key of keys) {
await this.del(key);
}
}
batchGet(keys: string[]): Promise<any[]> {
const results: Promise<any>[] = [];
for (const key of keys) {
results.push(this.get(key));
}
return Promise.all(results);
}
async get(key: string): Promise<any> {
return (await this.kv.get([key])).value;
}
async has(key: string): Promise<boolean> {
return (await this.kv.get([key])).value !== null;
}
async queryPrefix(keyPrefix: string): Promise<{ key: string; value: any }[]> {
const results: { key: string; value: any }[] = [];
for await (
const result of (this.kv).list({
start: [keyPrefix],
end: [endRange(keyPrefix)],
})
) {
results.push({
key: result.key[0] as string,
value: result.value as any,
});
}
return results;
}
}
function endRange(prefix: string) {
const lastChar = prefix[prefix.length - 1];
const nextLastChar = String.fromCharCode(lastChar.charCodeAt(0) + 1);
return prefix.slice(0, -1) + nextLastChar;
}

View File

@ -1,10 +1,10 @@
import { Manifest, RuntimeEnvironment } from "./types.ts";
import { Manifest } from "./types.ts";
import { Sandbox } from "./sandbox.ts";
import { System } from "./system.ts";
import { AssetBundle, AssetJson } from "./asset_bundle/bundle.ts";
export class Plug<HookT> {
readonly runtimeEnv?: RuntimeEnvironment;
readonly runtimeEnv?: string;
public grantedPermissions: string[] = [];
public sandbox: Sandbox<HookT>;

25
plugos/syscalls/fetch.ts Normal file
View File

@ -0,0 +1,25 @@
import type { SysCallMapping } from "../../plugos/system.ts";
import {
ProxyFetchRequest,
ProxyFetchResponse,
} from "../../common/proxy_fetch.ts";
import { base64Encode } from "../asset_bundle/base64.ts";
export function sandboxFetchSyscalls(): SysCallMapping {
return {
"sandboxFetch.fetch": async (
_ctx,
url: string,
options: ProxyFetchRequest,
): Promise<ProxyFetchResponse> => {
// console.log("Got sandbox fetch ", url);
const resp = await fetch(url, options);
return {
status: resp.status,
ok: resp.ok,
headers: Object.fromEntries(resp.headers.entries()),
base64Body: base64Encode(new Uint8Array(await resp.arrayBuffer())),
};
},
};
}

View File

@ -1,21 +1,21 @@
import type { SysCallMapping } from "../system.ts";
export default function (cwd: string): SysCallMapping {
export function shellSyscalls(cwd: string): SysCallMapping {
return {
"shell.run": async (
_ctx,
cmd: string,
args: string[],
): Promise<{ stdout: string; stderr: string }> => {
const p = Deno.run({
cmd: [cmd, ...args],
cwd: cwd,
const p = new Deno.Command(cmd, {
args: args,
cwd,
stdout: "piped",
stderr: "piped",
});
await p.status();
const stdout = new TextDecoder().decode(await p.output());
const stderr = new TextDecoder().decode(await p.stderrOutput());
const output = await p.output();
const stdout = new TextDecoder().decode(output.stdout);
const stderr = new TextDecoder().decode(output.stderr);
return { stdout, stderr };
},

View File

@ -1,9 +1,8 @@
import { SysCallMapping } from "../system.ts";
import { DexieKVStore } from "../lib/kv_store.dexie.ts";
import { KV } from "../lib/kv_store.ts";
import { KV, KVStore } from "../lib/kv_store.ts";
export function storeSyscalls(
db: DexieKVStore,
db: KVStore,
): SysCallMapping {
return {
"store.delete": (_ctx, key: string) => {

View File

@ -1,4 +1,4 @@
import { Hook, RuntimeEnvironment } from "./types.ts";
import { Hook } from "./types.ts";
import { EventEmitter } from "./event.ts";
import type { SandboxFactory } from "./sandbox.ts";
import { Plug } from "./plug.ts";
@ -32,7 +32,7 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
protected registeredSyscalls = new Map<string, Syscall>();
protected enabledHooks = new Set<Hook<HookT>>();
constructor(readonly env?: RuntimeEnvironment) {
constructor(readonly env?: string) {
super();
}

View File

@ -17,11 +17,9 @@ export type FunctionDef<HookT> = {
// Reuse an
// Format: plugName.functionName
redirect?: string;
env?: RuntimeEnvironment;
env?: string;
} & HookT;
export type RuntimeEnvironment = "client" | "server";
export interface Hook<HookT> {
validateManifest(manifest: Manifest<HookT>): string[];
apply(system: System<HookT>): void;

View File

@ -52,6 +52,8 @@ functions:
path: "./page.ts:reindexCommand"
command:
name: "Space: Reindex"
reindexSpace:
path: "./page.ts:reindexSpace"
deletePage:
path: "./page.ts:deletePage"
command:

View File

@ -1,5 +1,6 @@
import { editor, markdown, space, sync } from "$sb/silverbullet-syscall/mod.ts";
import {
ParseTree,
removeParentPointers,
renderToText,
traverseTree,
@ -36,42 +37,7 @@ export async function updateDirectivesOnPageCommand() {
await editor.save();
// Collect all directives and their body replacements
const replacements: { fullMatch: string; textPromise: Promise<string> }[] =
[];
// Convenience array to wait for all promises to resolve
const allPromises: Promise<string>[] = [];
removeParentPointers(tree);
traverseTree(tree, (tree) => {
if (tree.type !== "Directive") {
return false;
}
const fullMatch = text.substring(tree.from!, tree.to!);
try {
const promise = renderDirectives(pageMeta, tree);
replacements.push({
textPromise: promise,
fullMatch,
});
allPromises.push(promise);
} catch (e: any) {
replacements.push({
fullMatch,
textPromise: Promise.resolve(
`${renderToText(tree.children![0])}\n**ERROR:** ${e.message}\n${
renderToText(tree.children![tree.children!.length - 1])
}`,
),
});
}
return true;
});
// Wait for all to have processed
await Promise.all(allPromises);
const replacements = await findReplacements(tree, text, pageMeta);
// Iterate again and replace the bodies. Iterating again (not using previous positions)
// because text may have changed in the mean time (directive processing may take some time)
@ -111,27 +77,27 @@ export async function updateDirectivesOnPageCommand() {
}
}
export async function updateDirectives(
pageMeta: PageMeta,
async function findReplacements(
tree: ParseTree,
text: string,
pageMeta: PageMeta,
) {
const tree = await markdown.parseMarkdown(text);
// Collect all directives and their body replacements
const replacements: { fullMatch: string; textPromise: Promise<string> }[] =
[];
// Convenience array to wait for all promises to resolve
const allPromises: Promise<string>[] = [];
removeParentPointers(tree);
traverseTree(tree, (tree) => {
if (tree.type !== "Directive") {
return false;
}
const fullMatch = text.substring(tree.from!, tree.to!);
try {
const promise = renderDirectives(
pageMeta,
tree,
);
const promise = renderDirectives(pageMeta, tree);
replacements.push({
textPromise: promise,
fullMatch,
@ -153,9 +119,51 @@ export async function updateDirectives(
// Wait for all to have processed
await Promise.all(allPromises);
return replacements;
}
export async function updateDirectivesInSpace() {
const allPages = await space.listPages();
let counter = 0;
for (const page of allPages) {
counter++;
console.log(
`Updating directives in page [${counter}/${allPages.length}]`,
page.name,
);
try {
await updateDirectivesForPage(page.name);
} catch (e: any) {
console.error("Error while updating directives on page", page.name, e);
}
}
}
async function updateDirectivesForPage(
pageName: string,
) {
const pageMeta = await space.getPageMeta(pageName);
const currentText = await space.readPage(pageName);
const newText = await updateDirectives(pageMeta, currentText);
if (newText !== currentText) {
console.info("Content of page changed, saving.");
await space.writePage(pageName, newText);
}
}
export async function updateDirectives(
pageMeta: PageMeta,
text: string,
) {
const tree = await markdown.parseMarkdown(text);
const replacements = await findReplacements(tree, text, pageMeta);
// Iterate again and replace the bodies.
for (const replacement of replacements) {
text = text.replace(replacement.fullMatch, await replacement.textPromise);
text = text.replace(
replacement.fullMatch,
await replacement.textPromise,
);
}
return text;
}

View File

@ -9,6 +9,8 @@ functions:
key: "Alt-q"
events:
- editor:pageLoaded
updateDirectivesInSpace:
path: ./command.ts:updateDirectivesInSpace
indexData:
path: ./data.ts:indexData
events:

View File

@ -1,5 +1,4 @@
import { ParseTree, renderToText } from "$sb/lib/tree.ts";
import { sync } from "../../plug-api/silverbullet-syscall/mod.ts";
import { PageMeta } from "../../web/types.ts";
import { evalDirectiveRenderer } from "./eval_directive.ts";

View File

@ -11,6 +11,7 @@ import { userAdd } from "./cmd/user_add.ts";
import { userPasswd } from "./cmd/user_passwd.ts";
import { userDelete } from "./cmd/user_delete.ts";
import { userChgrp } from "./cmd/user_chgrp.ts";
import { plugRunCommand } from "./cmd/plug_run.ts";
await new Command()
.name("silverbullet")
@ -66,6 +67,13 @@ await new Command()
.option("--importmap <path:string>", "Path to import map file to use")
.option("--runtimeUrl <url:string>", "URL to worker_runtime.ts to use")
.action(plugCompileCommand)
// plug:run
.command("plug:run", "Run a PlugOS function from the CLI")
.arguments("<spacePath> <function> [...args:string]")
.option("--noIndex [type:boolean]", "Do not run a full space index first", {
default: false,
})
.action(plugRunCommand)
.command("user:add", "Add a new user to an authentication file")
.arguments("[username:string]")
.option(

View File

@ -8,7 +8,7 @@ import { createSandbox } from "../plugos/environments/webworker_sandbox.ts";
import assetSyscalls from "../plugos/syscalls/asset.ts";
import { eventSyscalls } from "../plugos/syscalls/event.ts";
import { storeSyscalls } from "../plugos/syscalls/store.dexie_browser.ts";
import { storeSyscalls } from "../plugos/syscalls/store.ts";
import { SysCallMapping, System } from "../plugos/system.ts";
import type { Client } from "./client.ts";
import { CodeWidgetHook } from "./hooks/code_widget.ts";

View File

@ -4,8 +4,8 @@ import { EventEmitter } from "../plugos/event.ts";
import { plugPrefix } from "../common/spaces/constants.ts";
import { safeRun } from "../common/util.ts";
import { AttachmentMeta, PageMeta } from "./types.ts";
import { DexieKVStore } from "../plugos/lib/kv_store.dexie.ts";
import { throttle } from "../common/async_util.ts";
import { KVStore } from "../plugos/lib/kv_store.ts";
export type SpaceEvents = {
pageCreated: (meta: PageMeta) => void;
@ -46,7 +46,7 @@ export class Space extends EventEmitter<SpaceEvents> {
constructor(
readonly spacePrimitives: SpacePrimitives,
private kvStore: DexieKVStore,
private kvStore: KVStore,
) {
super();
this.kvStore.get("imageHeightCache").then((cache) => {

View File

@ -33,7 +33,7 @@ Example query:
<!-- #query page where name = "Attributes" -->
|name |lastModified |contentType |size|perm|pageAttribute|
|----------|-------------|-------------|----|--|-----|
|Attributes|1690384301337|text/markdown|1591|rw|hello|
|Attributes|1691165890257|text/markdown|1609|rw|hello|
<!-- /query -->
This attaches an attribute to an item:
@ -57,5 +57,5 @@ Example query:
<!-- #query task where page = "Attributes" and taskAttribute = "hello" -->
|name|done |taskAttribute|page |pos |
|----|-----|-----|----------|----|
|Task|false|hello|Attributes|1352|
|Task|false|hello|Attributes|1355|
<!-- /query -->

View File

@ -23,18 +23,18 @@ You will notice this whole page section is wrapped in a strange type of block. T
Dont believe me, check this out, heres a list of (max 10) pages in your space ordered by last modified date, it updates (somewhat) dynamically 🤯. Create some new pages and come back here to see that it works:
<!-- #query page select name order by lastModified desc limit 10 -->
|name |
|-------------------------|
|CHANGELOG |
|🔨 Development |
|Server |
|Raspberry Pi Installation|
|STYLES |
|Getting Started |
|Sandbox |
|SETTINGS |
|SilverBullet |
|🔌 Core/Templates |
|name |
|------------------|
|🔌 Directive/Query|
|Attributes |
|Getting Started |
|🔌 Core/Tags |
|🔌 Github |
|🔌 Mattermost |
|🔌 Git |
|🔌 Ghost |
|🔌 Share |
|Install |
<!-- /query -->
That said, the directive used wrapping this page section is `#use` which uses the content of another page as a template and inlines it. Directives recalculate their bodies in two scenarios:

View File

@ -20,13 +20,13 @@ This consists of two steps (unless Deno is already installed — in which case w
With Deno installed, run:
```shell
deno install -f --name silverbullet -A https://get.silverbullet.md
deno install -f --name silverbullet --unstable -A https://get.silverbullet.md
```
This will give you (and when you use `silverbullet upgrade`) the latest stable release. If you prefer to live on the bleeding edge, you can install using the following command instead:
```shell
deno install -f --name silverbullet -A https://silverbullet.md/silverbullet.js
deno install -f --name silverbullet --unstable -A https://silverbullet.md/silverbullet.js
```
This will install `silverbullet` into your `~/.deno/bin` folder (which should already be in your `$PATH` if you followed the Deno install instructions).

View File

@ -18,7 +18,7 @@ and be queried:
<!-- #query item where tags = "core-tag" -->
|name |tags |page |pos|
|-------------------------------|--------|------------|---|
|This is a tagged item #core-tag|core-tag|🔌 Core/Tags|486|
|This is a tagged item #core-tag|core-tag|🔌 Core/Tags|493|
<!-- /query -->
and **tags**:
@ -28,5 +28,5 @@ and **tags**:
And they can be queried this way:
<!-- #query task where tags = "core-tag" render [[template/task]] -->
* [ ] [[🔌 Core/Tags@783]] This is a tagged task #core-tag
* [ ] [[🔌 Core/Tags@804]] This is a tagged task #core-tag
<!-- /query -->

View File

@ -88,8 +88,8 @@ Example:
<!-- #query data where age > 20 and country = "Italy" -->
|name|age|city |country|page |pos |
|----|--|-----|-----|------------------|----|
|John|50|Milan|Italy|🔌 Directive/Query|3293|
|Jane|53|Rome |Italy|🔌 Directive/Query|3294|
|John|50|Milan|Italy|🔌 Directive/Query|2933|
|Jane|53|Rome |Italy|🔌 Directive/Query|2934|
<!-- /query -->
#### 4.2 Plugs data sources
@ -156,11 +156,11 @@ For the sake of simplicity, we will use the `page` data source and limit the res
**Result:** Look at the data. This is more than we need. The query even gives us template pages. Let's try to limit it in the next step.
<!-- #query page limit 3 -->
|name |lastModified |contentType |size|perm|
|--------------|-------------|-------------|----|--|
|API |1688987324351|text/markdown|1405|rw|
|Authelia |1688482500313|text/markdown|866 |rw|
|Authentication|1686682290943|text/markdown|1730|rw|
|name |lastModified |contentType |size |perm|
|--|--|--|--|--|
|Authentication |1686682290943|text/markdown|1730 |rw|
|Guide/Deployment/Cloudflare and Portainer|1690298800145|text/markdown|12899|rw|
|Markdown |1676121406520|text/markdown|1178 |rw|
<!-- /query -->
@ -171,13 +171,13 @@ For the sake of simplicity, we will use the `page` data source and limit the res
**Result:** Okay, this is what we wanted but there is also information such as `perm`, `type` and `lastModified` that we don't need.
<!-- #query page where type = "plug" order by lastModified desc limit 5 -->
|name |lastModified |contentType |size|perm|type|repo |uri |author |
|--|--|--|--|--|--|--|--|--|
|🔌 Directive|1688987324365|text/markdown|2607|rw|plug|https://github.com/silverbulletmd/silverbullet | | |
|🔌 KaTeX |1687099068396|text/markdown|1342|rw|plug|https://github.com/silverbulletmd/silverbullet-katex |github:silverbulletmd/silverbullet-katex/katex.plug.js |Zef Hemel |
|🔌 Core |1687094809367|text/markdown|402 |rw|plug|https://github.com/silverbulletmd/silverbullet | | |
|🔌 Twitter |1685105433212|text/markdown|1266|rw|plug|https://github.com/silverbulletmd/silverbullet-twitter|github:silverbulletmd/silverbullet-twitter/twitter.plug.js|SilverBullet Authors|
|🔌 Mermaid |1685105423879|text/markdown|1096|rw|plug|https://github.com/silverbulletmd/silverbullet-mermaid|github:silverbulletmd/silverbullet-mermaid/mermaid.plug.js|Zef Hemel |
|name |lastModified |contentType |size|perm|type|uri |repo |author |share-support|
|--|--|--|--|--|--|--|--|--|--|
|🔌 Github |1691137925014|text/markdown|2206|rw|plug|github:silverbulletmd/silverbullet-github/github.plug.js |https://github.com/silverbulletmd/silverbullet-github |Zef Hemel|true|
|🔌 Mattermost|1691137924741|text/markdown|3535|rw|plug|github:silverbulletmd/silverbullet-mattermost/mattermost.plug.json|https://github.com/silverbulletmd/silverbullet-mattermost|Zef Hemel|true|
|🔌 Git |1691137924435|text/markdown|1112|rw|plug|github:silverbulletmd/silverbullet-git/git.plug.js |https://github.com/silverbulletmd/silverbullet-git |Zef Hemel| |
|🔌 Ghost |1691137922296|text/markdown|1733|rw|plug|github:silverbulletmd/silverbullet-ghost/ghost.plug.js |https://github.com/silverbulletmd/silverbullet-ghost |Zef Hemel|true|
|🔌 Share |1691137921643|text/markdown|693 |rw|plug| |https://github.com/silverbulletmd/silverbullet | | |
<!-- /query -->
#### 6.3 Query to select only certain fields
@ -189,13 +189,13 @@ and `repo` columns and then sort by last modified time.
from a visual perspective.
<!-- #query page select name, author, repo where type = "plug" order by lastModified desc limit 5 -->
|name |author |repo |
|name |author |repo |
|--|--|--|
|🔌 Directive| |https://github.com/silverbulletmd/silverbullet |
|🔌 KaTeX |Zef Hemel |https://github.com/silverbulletmd/silverbullet-katex |
|🔌 Core | |https://github.com/silverbulletmd/silverbullet |
|🔌 Twitter |SilverBullet Authors|https://github.com/silverbulletmd/silverbullet-twitter|
|🔌 Mermaid |Zef Hemel |https://github.com/silverbulletmd/silverbullet-mermaid|
|🔌 Github |Zef Hemel|https://github.com/silverbulletmd/silverbullet-github |
|🔌 Mattermost|Zef Hemel|https://github.com/silverbulletmd/silverbullet-mattermost|
|🔌 Git |Zef Hemel|https://github.com/silverbulletmd/silverbullet-git |
|🔌 Ghost |Zef Hemel|https://github.com/silverbulletmd/silverbullet-ghost |
|🔌 Share | |https://github.com/silverbulletmd/silverbullet |
<!-- /query -->
#### 6.4 Display the data in a format defined by a template
@ -205,11 +205,11 @@ from a visual perspective.
**Result:** Here you go. This is the result we would like to achieve 🎉. Did you see how I used `render` and `template/plug` in a query? 🚀
<!-- #query page where type = "plug" order by lastModified desc limit 5 render [[template/plug]] -->
* [[🔌 Directive]]
* [[🔌 KaTeX]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-katex))
* [[🔌 Core]]
* [[🔌 Twitter]] by **SilverBullet Authors** ([repo](https://github.com/silverbulletmd/silverbullet-twitter))
* [[🔌 Mermaid]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-mermaid))
* [[🔌 Github]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-github))
* [[🔌 Mattermost]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-mattermost))
* [[🔌 Git]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-git))
* [[🔌 Ghost]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-ghost))
* [[🔌 Share]]
<!-- /query -->
PS: You don't need to select only certain fields to use templates. Templates are

View File

@ -9,8 +9,7 @@ share-support: true
<!-- #include [[https://raw.githubusercontent.com/silverbulletmd/silverbullet-ghost/main/README.md]] -->
# Ghost plug for Silver Bullet
This allows you to publish your pages as [Ghost](https://ghost.org/) pages or
posts. I use it to publish [Zef+](https://zef.plus).
This allows you to publish your pages as [Ghost](https://ghost.org/) pages or posts. I use it to publish [Zef+](https://zef.plus).
## Configuration
@ -22,9 +21,7 @@ In your `SETTINGS` specify the following settings:
url: https://your-ghost-blog.ghost.io
```
Then, create a Custom Integration (in your Ghost control panel under Settings >
Advanced > Integrations > Add Custom Integration). Enter a name (whatever you
want), then copy the full Admin API Key in your `SECRETS` file, mirroring the
Then, create a Custom Integration (in your Ghost control panel under Settings > Advanced > Integrations > Add Custom Integration). Enter a name (whatever you want), then copy the full Admin API Key in your `SECRETS` file, mirroring the
structure of SETTINGS:
```yaml
@ -34,10 +31,8 @@ structure of SETTINGS:
## Usage
The plugin hooks into Silver Bullet's
[Share infrastructure](https://silverbullet.md/%F0%9F%94%8C_Share). Therefore to
share a page as either a Ghost page or post, add a `$share` front matter key.
For posts this should take the shape of:
The plugin hooks into Silver Bullet's [Share infrastructure](https://silverbullet.md/%F0%9F%94%8C_Share). Therefore to
share a page as either a Ghost page or post, add a `$share` front matter key. For posts this should take the shape of:
---
$share:
@ -51,8 +46,7 @@ And for pages:
- ghost:myblog:page:my-page-slug
---
Now, when you {[Share: Publish]} (Cmd-s/Ctrl-s) your post will automatically be
created (as a draft) or updated if it already exists.
Now, when you {[Share: Publish]} (Cmd-s/Ctrl-s) your post will automatically be created (as a draft) or updated if it already exists.
Enjoy!

View File

@ -7,28 +7,37 @@ author: Zef Hemel
<!-- #include [[https://raw.githubusercontent.com/silverbulletmd/silverbullet-git/main/README.md]] -->
# SilverBullet plug for Git
Very basic in functionality, it assumes you have git configured for push and pull in your space. What it does, roughly speaking:
`Git : Sync`:
* Adds all *.md files in your folder to git
* It commits them with a "Snapshot" commit message
* It `git pull`s changes from the remote server
* It `git push`es changes to the remote server
{[Git: Sync]}:
`Git: Snapshot`:
* Asks you for a commit message
* Commits
- Adds all files in your folder to git
- It commits them with a "Snapshot" commit message
- It `git pull`s changes from the remote server
- It `git push`es changes to the remote server
{[Git: Snapshot]}:
- Asks you for a commit message
- Commits
{[Github: Clone]}:
Clones into your space from a Github repository. This will do authentication based on a [personal access token](https://github.com/settings/tokens).
## Installation
Open your `PLUGS` note in SilverBullet and add this plug to the list:
```
- github:silverbulletmd/silverbullet-git/git.plug.json
- github:silverbulletmd/silverbullet-git/git.plug.js
```
Then run the `Plugs: Update` command and off you go!
## To Build
```shell
deno task build
```

View File

@ -17,7 +17,7 @@ Provides various integrations with Github:
Open your `PLUGS` note in SilverBullet and add this plug to the list:
```
- github:silverbulletmd/silverbullet-github/github.plug.json
- github:silverbulletmd/silverbullet-github/github.plug.js
```
Then run the `Plugs: Update` command and off you go!

View File

@ -7,12 +7,12 @@ share-support: true
---
<!-- #include [[https://raw.githubusercontent.com/silverbulletmd/silverbullet-mattermost/main/README.md]] -->
# Mattermost for SilverBullet
# Mattermost for Silver Bullet
This plug provides various integrations with the [Mattermost suite](https://www.mattermost.com) of products. Please follow the installation, configuration sections, and have a look at the example.
Features:
* Integration with [SilverBullet Share](https://silverbullet.md/%F0%9F%94%8C_Share), allowing you to publish and update a page as a post on Mattermost, as well as load existing posts into SB as a page using the {[Share: Mattermost Post: Publish]} (to publish an existing page as a Mattermost post) and {[Share: Mattermost Post: Load]} (to load an existing post into SB) commands.
* Integration with [Silver Bullet Share](https://silverbullet.md/%F0%9F%94%8C_Share), allowing you to publish and update a page as a post on Mattermost, as well as load existing posts into SB as a page using the {[Share: Mattermost Post: Publish]} (to publish an existing page as a Mattermost post) and {[Share: Mattermost Post: Load]} (to load an existing post into SB) commands.
* Access your saved posts via the `mm-saved` query provider
* Unfurl support for posts (after dumping a permalink URL to a post in a page, use the {[Link: Unfurl]} command).
* Boards support is WIP
@ -46,7 +46,7 @@ In `SECRETS` provide a Mattermost personal access token (or hijack one from your
* `mm-saved` fetches (by default 15) saved posts in Mattermost, you need to add a `where server = "community"` (with server name) clause to your query to select the mattermost server to query.
To make the `mm-saved` query results look good, it's recommended you render your query results with a template. Here is one to start with: you can keep it in e.g., `templates/mm-saved`:
To make the `mm-saved` query results look good, it's recommended you render your query results a template. Here is one to start with, you can keep it in e.g. `templates/mm-saved`:
[{{username}}]({{url}}) in {{#if channelName}}**{{channelName}}**{{else}}a DM{{/if}} at _{{updatedAt}}_ {[Unsave]}:

View File

@ -11,7 +11,6 @@ Specific implementations for sharing are implemented in other plugs, specificall
<!-- #query page where share-support = true render [[template/page]] -->
* [[🔌 Ghost]]
* [[🔌 Markdown]]
* [[🔌 Collab]]
* [[🔌 Mattermost]]
* [[🔌 Github]]
<!-- /query -->