1
0
silverbullet/server/server_system.ts

207 lines
7.0 KiB
TypeScript
Raw Normal View History

2023-08-26 06:31:51 +00:00
import { PlugNamespaceHook } from "../common/hooks/plug_namespace.ts";
import { SilverBulletHooks } from "../common/manifest.ts";
import { loadMarkdownExtensions } from "../common/markdown_parser/markdown_ext.ts";
import buildMarkdown from "../common/markdown_parser/parser.ts";
import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives.ts";
import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts";
import { createSandbox } from "../plugos/environments/webworker_sandbox.ts";
import { CronHook } from "../plugos/hooks/cron.ts";
import { EndpointHook } from "../plugos/hooks/endpoint.ts";
import { EventHook } from "../plugos/hooks/event.ts";
import { MQHook } from "../plugos/hooks/mq.ts";
import assetSyscalls from "../plugos/syscalls/asset.ts";
import { eventSyscalls } from "../plugos/syscalls/event.ts";
import { mqSyscalls } from "../plugos/syscalls/mq.ts";
2023-08-26 06:31:51 +00:00
import { System } from "../plugos/system.ts";
import { Space } from "../web/space.ts";
import { debugSyscalls } from "../web/syscalls/debug.ts";
import { markdownSyscalls } from "../common/syscalls/markdown.ts";
2023-08-28 15:12:15 +00:00
import { spaceSyscalls } from "./syscalls/space.ts";
2023-08-26 06:31:51 +00:00
import { systemSyscalls } from "../web/syscalls/system.ts";
import { yamlSyscalls } from "../common/syscalls/yaml.ts";
2023-08-29 19:17:29 +00:00
import { Application } from "./deps.ts";
2023-08-26 06:31:51 +00:00
import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts";
import { shellSyscalls } from "../plugos/syscalls/shell.deno.ts";
import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
2023-08-29 19:17:29 +00:00
import { base64EncodedDataUrl } from "../plugos/asset_bundle/base64.ts";
import { Plug } from "../plugos/plug.ts";
import { DataStore } from "../plugos/lib/datastore.ts";
import { dataStoreSyscalls } from "../plugos/syscalls/datastore.ts";
import { DataStoreMQ } from "../plugos/lib/mq.datastore.ts";
import { languageSyscalls } from "../common/syscalls/language.ts";
import { handlebarsSyscalls } from "../common/syscalls/handlebars.ts";
2023-10-31 09:33:38 +00:00
import { codeWidgetSyscalls } from "../web/syscalls/code_widget.ts";
import { CodeWidgetHook } from "../web/hooks/code_widget.ts";
import { KVPrimitivesManifestCache } from "../plugos/manifest_cache.ts";
import { KvPrimitives } from "../plugos/lib/kv_primitives.ts";
2023-08-26 06:31:51 +00:00
2023-08-27 12:13:18 +00:00
const fileListInterval = 30 * 1000; // 30s
const plugNameExtractRegex = /\/(.+)\.plug\.js$/;
2023-08-26 06:31:51 +00:00
export class ServerSystem {
system!: System<SilverBulletHooks>;
2023-08-26 06:31:51 +00:00
spacePrimitives!: SpacePrimitives;
// denoKv!: Deno.Kv;
2023-08-29 19:17:29 +00:00
listInterval?: number;
ds!: DataStore;
2023-08-26 06:31:51 +00:00
constructor(
private baseSpacePrimitives: SpacePrimitives,
readonly kvPrimitives: KvPrimitives,
2023-08-26 06:31:51 +00:00
) {
}
// Always needs to be invoked right after construction
2023-11-02 11:46:33 +00:00
async init(awaitIndex = false) {
this.ds = new DataStore(this.kvPrimitives);
this.system = new System(
"server",
{
manifestCache: new KVPrimitivesManifestCache(
this.kvPrimitives,
"manifest",
),
plugFlushTimeout: 5 * 60 * 1000, // 5 minutes
},
);
2023-08-26 06:31:51 +00:00
// Event hook
const eventHook = new EventHook();
this.system.addHook(eventHook);
// Cron hook
const cronHook = new CronHook(this.system);
this.system.addHook(cronHook);
const mq = new DataStoreMQ(this.ds);
2023-08-26 06:31:51 +00:00
2023-11-15 08:31:44 +00:00
setInterval(() => {
// Timeout after 5s, retries 3 times, otherwise drops the message (no DLQ)
mq.requeueTimeouts(5000, 3, true).catch(console.error);
}, 20000); // Look to requeue every 20s
2023-08-26 06:31:51 +00:00
const plugNamespaceHook = new PlugNamespaceHook();
this.system.addHook(plugNamespaceHook);
this.system.addHook(new MQHook(this.system, mq));
2023-10-31 09:33:38 +00:00
const codeWidgetHook = new CodeWidgetHook();
this.system.addHook(codeWidgetHook);
this.spacePrimitives = new EventedSpacePrimitives(
new PlugSpacePrimitives(
this.baseSpacePrimitives,
plugNamespaceHook,
2023-08-26 06:31:51 +00:00
),
eventHook,
2023-08-26 06:31:51 +00:00
);
const space = new Space(this.spacePrimitives, this.ds, eventHook);
2023-08-26 06:31:51 +00:00
// Add syscalls
this.system.registerSyscalls(
[],
eventSyscalls(eventHook),
spaceSyscalls(space),
assetSyscalls(this.system),
yamlSyscalls(),
2023-08-29 19:17:29 +00:00
systemSyscalls(this.system),
2023-08-26 06:31:51 +00:00
mqSyscalls(mq),
languageSyscalls(),
handlebarsSyscalls(),
dataStoreSyscalls(this.ds),
2023-08-26 06:31:51 +00:00
debugSyscalls(),
2023-10-31 09:33:38 +00:00
codeWidgetSyscalls(codeWidgetHook),
2023-08-26 06:31:51 +00:00
markdownSyscalls(buildMarkdown([])), // Will later be replaced with markdown extensions
);
// Syscalls that require some additional permissions
this.system.registerSyscalls(
["fetch"],
sandboxFetchSyscalls(),
);
this.system.registerSyscalls(
["shell"],
shellSyscalls("."),
);
await this.loadPlugs();
// Load markdown syscalls based on all new syntax (if any)
this.system.registerSyscalls(
[],
markdownSyscalls(buildMarkdown(loadMarkdownExtensions(this.system))),
);
2023-08-27 12:13:18 +00:00
2023-08-27 16:05:14 +00:00
this.listInterval = setInterval(() => {
2023-08-27 12:13:18 +00:00
space.updatePageList().catch(console.error);
}, fileListInterval);
eventHook.addLocalListener("file:changed", (path, localChange) => {
(async () => {
if (!localChange && path.endsWith(".md")) {
const pageName = path.slice(0, -3);
const data = await this.spacePrimitives.readFile(path);
console.log("Outside page change: reindexing", pageName);
// Change made outside of editor, trigger reindex
await eventHook.dispatchEvent("page:index_text", {
name: pageName,
text: new TextDecoder().decode(data.data),
});
}
2023-08-29 19:17:29 +00:00
if (path.startsWith("_plug/") && path.endsWith(".plug.js")) {
console.log("Plug updated, reloading:", path);
this.system.unload(path);
await this.loadPlugFromSpace(path);
}
2023-08-27 12:13:18 +00:00
})().catch(console.error);
});
2023-08-30 20:36:27 +00:00
// Check if this space was ever indexed before
if (!await this.ds.get(["$initialIndexDone"])) {
2023-08-30 20:36:27 +00:00
console.log("Indexing space for the first time (in the background)");
2023-11-02 11:46:33 +00:00
const indexPromise = this.system.loadedPlugs.get("index")!.invoke(
2023-08-30 20:36:27 +00:00
"reindexSpace",
[],
).then(() => {
2023-11-15 08:31:44 +00:00
console.log("Initial index completed!");
this.ds.set(["$initialIndexDone"], true);
2023-08-30 20:36:27 +00:00
}).catch(console.error);
2023-11-02 11:46:33 +00:00
if (awaitIndex) {
await indexPromise;
}
2023-08-30 20:36:27 +00:00
}
await eventHook.dispatchEvent("system:ready");
2023-08-26 06:31:51 +00:00
}
async loadPlugs() {
2023-08-29 19:17:29 +00:00
for (const { name } of await this.spacePrimitives.fetchFileList()) {
if (name.endsWith(".plug.js")) {
await this.loadPlugFromSpace(name);
2023-08-26 06:31:51 +00:00
}
}
}
2023-08-29 19:17:29 +00:00
async loadPlugFromSpace(path: string): Promise<Plug<SilverBulletHooks>> {
const { meta, data } = await this.spacePrimitives.readFile(path);
const plugName = path.match(plugNameExtractRegex)![1];
2023-08-29 19:17:29 +00:00
return this.system.load(
// Base64 encoding this to support `deno compile` mode
new URL(base64EncodedDataUrl("application/javascript", data)),
plugName,
meta.lastModified,
2023-08-29 19:17:29 +00:00
createSandbox,
);
}
2023-08-26 06:31:51 +00:00
async close() {
2023-08-27 16:05:14 +00:00
clearInterval(this.listInterval);
2023-08-26 06:31:51 +00:00
await this.system.unloadAll();
}
}