WIP: CLI running of plugs
This commit is contained in:
Zef Hemel
@@ -0,0 +1,38 @@
|
||||
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
|
||||
import { compileManifest } from "../plugos/compile.ts";
|
||||
import { esbuild } from "../plugos/deps.ts";
|
||||
import { runPlug } from "./plug_run.ts";
|
||||
import assets from "../dist/plug_asset_bundle.json" assert {
|
||||
type: "json",
|
||||
};
|
||||
import { assertEquals } from "../test_deps.ts";
|
||||
import { path } from "../common/deps.ts";
|
||||
|
||||
Deno.test("Test plug run", async () => {
|
||||
// const tempDir = await Deno.makeTempDir();
|
||||
|
||||
const assetBundle = new AssetBundle(assets);
|
||||
|
||||
const testFolder = path.dirname(new URL(import.meta.url).pathname);
|
||||
const testSpaceFolder = path.join(testFolder, "test_space");
|
||||
|
||||
const plugFolder = path.join(testSpaceFolder, "_plug");
|
||||
await Deno.mkdir(plugFolder, { recursive: true });
|
||||
|
||||
await compileManifest(
|
||||
path.join(testFolder, "test.plug.yaml"),
|
||||
plugFolder,
|
||||
);
|
||||
assertEquals(
|
||||
await runPlug(
|
||||
testSpaceFolder,
|
||||
"test.run",
|
||||
[],
|
||||
assetBundle,
|
||||
),
|
||||
"Hello",
|
||||
);
|
||||
|
||||
// await Deno.remove(tempDir, { recursive: true });
|
||||
esbuild.stop();
|
||||
});
|
||||
+145
@@ -0,0 +1,145 @@
|
||||
import { path } from "../common/deps.ts";
|
||||
import { PlugNamespaceHook } from "../common/hooks/plug_namespace.ts";
|
||||
import { SilverBulletHooks } from "../common/manifest.ts";
|
||||
import { loadMarkdownExtensions } from "../common/markdown_parser/markdown_ext.ts";
|
||||
import buildMarkdown from "../common/markdown_parser/parser.ts";
|
||||
import { DiskSpacePrimitives } from "../common/spaces/disk_space_primitives.ts";
|
||||
import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives.ts";
|
||||
import { FileMetaSpacePrimitives } from "../common/spaces/file_meta_space_primitives.ts";
|
||||
import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts";
|
||||
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
|
||||
import { createSandbox } from "../plugos/environments/deno_sandbox.ts";
|
||||
import { CronHook } from "../plugos/hooks/cron.ts";
|
||||
import { EventHook } from "../plugos/hooks/event.ts";
|
||||
import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts";
|
||||
import assetSyscalls from "../plugos/syscalls/asset.ts";
|
||||
import { eventSyscalls } from "../plugos/syscalls/event.ts";
|
||||
import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts";
|
||||
import { shellSyscalls } from "../plugos/syscalls/shell.deno.ts";
|
||||
import { storeSyscalls } from "../plugos/syscalls/store.ts";
|
||||
import { System } from "../plugos/system.ts";
|
||||
import { Space } from "../web/space.ts";
|
||||
import { debugSyscalls } from "../web/syscalls/debug.ts";
|
||||
import { markdownSyscalls } from "../web/syscalls/markdown.ts";
|
||||
import { systemSyscalls } from "../web/syscalls/system.ts";
|
||||
import { yamlSyscalls } from "../web/syscalls/yaml.ts";
|
||||
import { pageIndexSyscalls } from "./syscalls/index.ts";
|
||||
import { spaceSyscalls } from "./syscalls/space.ts";
|
||||
|
||||
export async function runPlug(
|
||||
spacePath: string,
|
||||
functionName: string,
|
||||
args: string[] = [],
|
||||
builtinAssetBundle: AssetBundle,
|
||||
indexFirst = false,
|
||||
) {
|
||||
spacePath = path.resolve(spacePath);
|
||||
const system = new System<SilverBulletHooks>("cli");
|
||||
|
||||
// Event hook
|
||||
const eventHook = new EventHook();
|
||||
system.addHook(eventHook);
|
||||
|
||||
// Cron hook
|
||||
const cronHook = new CronHook(system);
|
||||
system.addHook(cronHook);
|
||||
|
||||
const pageIndexCalls = pageIndexSyscalls("run.db");
|
||||
|
||||
// TODO: Add endpoint
|
||||
|
||||
const plugNamespaceHook = new PlugNamespaceHook();
|
||||
system.addHook(plugNamespaceHook);
|
||||
|
||||
const spacePrimitives = new FileMetaSpacePrimitives(
|
||||
new EventedSpacePrimitives(
|
||||
new PlugSpacePrimitives(
|
||||
new DiskSpacePrimitives(spacePath),
|
||||
plugNamespaceHook,
|
||||
),
|
||||
eventHook,
|
||||
),
|
||||
pageIndexCalls,
|
||||
);
|
||||
const kvStore = new JSONKVStore();
|
||||
const space = new Space(spacePrimitives, kvStore);
|
||||
|
||||
// Add syscalls
|
||||
system.registerSyscalls(
|
||||
[],
|
||||
eventSyscalls(eventHook),
|
||||
spaceSyscalls(space),
|
||||
assetSyscalls(system),
|
||||
yamlSyscalls(),
|
||||
storeSyscalls(kvStore),
|
||||
systemSyscalls(undefined as any, system),
|
||||
pageIndexCalls,
|
||||
debugSyscalls(),
|
||||
markdownSyscalls(buildMarkdown([])), // Will later be replaced with markdown extensions
|
||||
);
|
||||
|
||||
// Syscalls that require some additional permissions
|
||||
system.registerSyscalls(
|
||||
["fetch"],
|
||||
sandboxFetchSyscalls(),
|
||||
);
|
||||
|
||||
system.registerSyscalls(
|
||||
["shell"],
|
||||
shellSyscalls("."),
|
||||
);
|
||||
|
||||
await loadPlugsFromAssetBundle(system, builtinAssetBundle);
|
||||
|
||||
for (let plugPath of await space.listPlugs()) {
|
||||
plugPath = path.resolve(spacePath, plugPath);
|
||||
await system.load(
|
||||
new URL(`file://${plugPath}`),
|
||||
createSandbox,
|
||||
);
|
||||
}
|
||||
|
||||
// Load markdown syscalls based on all new syntax (if any)
|
||||
system.registerSyscalls(
|
||||
[],
|
||||
markdownSyscalls(buildMarkdown(loadMarkdownExtensions(system))),
|
||||
);
|
||||
|
||||
if (indexFirst) {
|
||||
await system.loadedPlugs.get("core")!.invoke("reindexSpace", []);
|
||||
}
|
||||
|
||||
const [plugName, funcName] = functionName.split(".");
|
||||
|
||||
const plug = system.loadedPlugs.get(plugName);
|
||||
if (!plug) {
|
||||
throw new Error(`Plug ${plugName} not found`);
|
||||
}
|
||||
const result = await plug.invoke(funcName, args);
|
||||
|
||||
await system.unloadAll();
|
||||
await pageIndexCalls["index.close"]({} as any);
|
||||
return result;
|
||||
}
|
||||
|
||||
async function loadPlugsFromAssetBundle(
|
||||
system: System<any>,
|
||||
assetBundle: AssetBundle,
|
||||
) {
|
||||
const tempDir = await Deno.makeTempDir();
|
||||
try {
|
||||
for (const filePath of assetBundle.listFiles()) {
|
||||
if (filePath.endsWith(".plug.js")) {
|
||||
const plugPath = path.join(tempDir, filePath);
|
||||
await Deno.mkdir(path.dirname(plugPath), { recursive: true });
|
||||
await Deno.writeFile(plugPath, assetBundle.readFileSync(filePath));
|
||||
await system.load(
|
||||
new URL(`file://${plugPath}`),
|
||||
createSandbox,
|
||||
);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await Deno.remove(tempDir, { recursive: true });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
import { index } from "$sb/silverbullet-syscall/mod.ts";
|
||||
|
||||
export async function run() {
|
||||
console.log("Hello from plug_test.ts");
|
||||
console.log(await index.queryPrefix(`tag:`));
|
||||
return "Hello";
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import { assertEquals } from "../../test_deps.ts";
|
||||
import { pageIndexSyscalls } from "./index.ts";
|
||||
|
||||
Deno.test("Test KV index", async () => {
|
||||
const ctx: any = {};
|
||||
const calls = pageIndexSyscalls();
|
||||
await calls["index.set"](ctx, "page", "test", "value");
|
||||
assertEquals(await calls["index.get"](ctx, "page", "test"), "value");
|
||||
await calls["index.delete"](ctx, "page", "test");
|
||||
assertEquals(await calls["index.get"](ctx, "page", "test"), null);
|
||||
await calls["index.batchSet"](ctx, "page", [{
|
||||
key: "attr:test",
|
||||
value: "value",
|
||||
}, {
|
||||
key: "attr:test2",
|
||||
value: "value2",
|
||||
}, { key: "random", value: "value3" }]);
|
||||
await calls["index.batchSet"](ctx, "page2", [{
|
||||
key: "attr:test",
|
||||
value: "value",
|
||||
}, {
|
||||
key: "attr:test2",
|
||||
value: "value2",
|
||||
}, { key: "random", value: "value3" }]);
|
||||
let results = await calls["index.queryPrefix"](ctx, "attr:");
|
||||
assertEquals(results.length, 4);
|
||||
await calls["index.clearPageIndexForPage"](ctx, "page");
|
||||
results = await calls["index.queryPrefix"](ctx, "attr:");
|
||||
assertEquals(results.length, 2);
|
||||
await calls["index.clearPageIndex"](ctx);
|
||||
results = await calls["index.queryPrefix"](ctx, "");
|
||||
assertEquals(results.length, 0);
|
||||
await calls["index.close"](ctx);
|
||||
});
|
||||
@@ -0,0 +1,103 @@
|
||||
/// <reference lib="deno.unstable" />
|
||||
|
||||
import type { SysCallMapping } from "../../plugos/system.ts";
|
||||
|
||||
type Item = {
|
||||
page: string;
|
||||
key: string;
|
||||
value: any;
|
||||
};
|
||||
|
||||
export type KV = {
|
||||
key: string;
|
||||
value: any;
|
||||
};
|
||||
|
||||
// Keyspace:
|
||||
// ["index", page, key] -> value
|
||||
// ["indexByKey", key, page] -> value
|
||||
|
||||
/**
|
||||
* Implements the index syscalls using Deno's KV store.
|
||||
* @param dbFile
|
||||
* @returns
|
||||
*/
|
||||
export function pageIndexSyscalls(dbFile?: string): SysCallMapping {
|
||||
const kv = Deno.openKv(dbFile);
|
||||
const apiObj: SysCallMapping = {
|
||||
"index.set": async (_ctx, page: string, key: string, value: any) => {
|
||||
const res = await (await kv).atomic()
|
||||
.set(["index", page, key], value)
|
||||
.set(["indexByKey", key, page], value)
|
||||
.commit();
|
||||
if (!res.ok) {
|
||||
throw res;
|
||||
}
|
||||
},
|
||||
"index.batchSet": async (_ctx, page: string, kvs: KV[]) => {
|
||||
// await items.bulkPut(kvs);
|
||||
for (const { key, value } of kvs) {
|
||||
await apiObj["index.set"](_ctx, page, key, value);
|
||||
}
|
||||
},
|
||||
"index.delete": async (_ctx, page: string, key: string) => {
|
||||
const res = await (await kv).atomic()
|
||||
.delete(["index", page, key])
|
||||
.delete(["indexByKey", key, page])
|
||||
.commit();
|
||||
if (!res.ok) {
|
||||
throw res;
|
||||
}
|
||||
},
|
||||
"index.get": async (_ctx, page: string, key: string) => {
|
||||
return (await (await kv).get(["index", page, key])).value;
|
||||
},
|
||||
"index.queryPrefix": async (_ctx, prefix: string) => {
|
||||
const results: { key: string; page: string; value: any }[] = [];
|
||||
for await (
|
||||
const result of (await kv).list({
|
||||
start: ["indexByKey", prefix],
|
||||
end: [
|
||||
"indexByKey",
|
||||
prefix.slice(0, -1) +
|
||||
// This is a hack to get the next character in the ASCII table (e.g. "a" -> "b")
|
||||
String.fromCharCode(prefix.charCodeAt(prefix.length - 1) + 1),
|
||||
],
|
||||
})
|
||||
) {
|
||||
results.push({
|
||||
key: result.key[1] as string,
|
||||
page: result.key[2] as string,
|
||||
value: result.value,
|
||||
});
|
||||
}
|
||||
return results;
|
||||
},
|
||||
"index.clearPageIndexForPage": async (ctx, page: string) => {
|
||||
await apiObj["index.deletePrefixForPage"](ctx, page, "");
|
||||
},
|
||||
"index.deletePrefixForPage": async (_ctx, page: string, prefix: string) => {
|
||||
for await (
|
||||
const result of (await kv).list({
|
||||
start: ["index", page, prefix],
|
||||
end: ["index", page, prefix + "~"],
|
||||
})
|
||||
) {
|
||||
await apiObj["index.delete"](_ctx, page, result.key[2]);
|
||||
}
|
||||
},
|
||||
"index.clearPageIndex": async (ctx) => {
|
||||
for await (
|
||||
const result of (await kv).list({
|
||||
prefix: ["index"],
|
||||
})
|
||||
) {
|
||||
await apiObj["index.delete"](ctx, result.key[1], result.key[2]);
|
||||
}
|
||||
},
|
||||
"index.close": async () => {
|
||||
(await kv).close();
|
||||
},
|
||||
};
|
||||
return apiObj;
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import { SysCallMapping } from "../../plugos/system.ts";
|
||||
import type { Space } from "../../web/space.ts";
|
||||
import { AttachmentMeta, PageMeta } from "../../web/types.ts";
|
||||
|
||||
/**
|
||||
* Almost the same as web/syscalls/space.ts except leaving out client-specific stuff
|
||||
*/
|
||||
export function spaceSyscalls(space: Space): SysCallMapping {
|
||||
return {
|
||||
"space.listPages": (): Promise<PageMeta[]> => {
|
||||
return space.fetchPageList();
|
||||
},
|
||||
"space.readPage": async (
|
||||
_ctx,
|
||||
name: string,
|
||||
): Promise<string> => {
|
||||
return (await space.readPage(name)).text;
|
||||
},
|
||||
"space.getPageMeta": (_ctx, name: string): Promise<PageMeta> => {
|
||||
return space.getPageMeta(name);
|
||||
},
|
||||
"space.writePage": (
|
||||
_ctx,
|
||||
name: string,
|
||||
text: string,
|
||||
): Promise<PageMeta> => {
|
||||
return space.writePage(name, text);
|
||||
},
|
||||
"space.deletePage": async (_ctx, name: string) => {
|
||||
await space.deletePage(name);
|
||||
},
|
||||
"space.listPlugs": (): Promise<string[]> => {
|
||||
return space.listPlugs();
|
||||
},
|
||||
"space.listAttachments": async (): Promise<AttachmentMeta[]> => {
|
||||
return await space.fetchAttachmentList();
|
||||
},
|
||||
"space.readAttachment": async (
|
||||
_ctx,
|
||||
name: string,
|
||||
): Promise<Uint8Array> => {
|
||||
return (await space.readAttachment(name)).data;
|
||||
},
|
||||
"space.getAttachmentMeta": async (
|
||||
_ctx,
|
||||
name: string,
|
||||
): Promise<AttachmentMeta> => {
|
||||
return await space.getAttachmentMeta(name);
|
||||
},
|
||||
"space.writeAttachment": (
|
||||
_ctx,
|
||||
name: string,
|
||||
data: Uint8Array,
|
||||
): Promise<AttachmentMeta> => {
|
||||
return space.writeAttachment(name, data);
|
||||
},
|
||||
"space.deleteAttachment": async (_ctx, name: string) => {
|
||||
await space.deleteAttachment(name);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
name: test
|
||||
requiredPermissions:
|
||||
- shell
|
||||
functions:
|
||||
run:
|
||||
path: plug_test.ts:run
|
||||
Reference in New Issue
Block a user