SilverBullet pivot to become an offline-first PWA (#403)
This commit is contained in:
Zef Hemel
+2
-2
@@ -1,3 +1,3 @@
|
||||
export * from "../common/deps.ts";
|
||||
export { Application, Router } from "https://deno.land/x/oak@v11.1.0/mod.ts";
|
||||
export * as etag from "https://deno.land/x/oak@v11.1.0/etag.ts";
|
||||
export { Application, Router } from "https://deno.land/x/oak@v12.4.0/mod.ts";
|
||||
export * as etag from "https://deno.land/x/oak@v12.4.0/etag.ts";
|
||||
|
||||
+129
-152
@@ -1,98 +1,71 @@
|
||||
import { Application, path, Router } from "./deps.ts";
|
||||
import { Manifest } from "../common/manifest.ts";
|
||||
import { Application, Router } from "./deps.ts";
|
||||
import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
|
||||
import { EndpointHook } from "../plugos/hooks/endpoint.ts";
|
||||
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
|
||||
import { SpaceSystem } from "./space_system.ts";
|
||||
import { ensureAndLoadSettings } from "../common/util.ts";
|
||||
import { base64Decode } from "../plugos/asset_bundle/base64.ts";
|
||||
import { ensureSettingsAndIndex } from "../common/util.ts";
|
||||
import { performLocalFetch } from "../common/proxy_fetch.ts";
|
||||
|
||||
export type ServerOptions = {
|
||||
hostname: string;
|
||||
port: number;
|
||||
pagesPath: string;
|
||||
dbPath: string;
|
||||
assetBundle: AssetBundle;
|
||||
clientAssetBundle: AssetBundle;
|
||||
user?: string;
|
||||
pass?: string;
|
||||
bareMode?: boolean;
|
||||
certFile?: string;
|
||||
keyFile?: string;
|
||||
maxFileSizeMB?: number;
|
||||
};
|
||||
|
||||
const staticLastModified = new Date().toUTCString();
|
||||
|
||||
export class HttpServer {
|
||||
app: Application;
|
||||
systemBoot: SpaceSystem;
|
||||
private hostname: string;
|
||||
private port: number;
|
||||
user?: string;
|
||||
settings: { [key: string]: any } = {};
|
||||
abortController?: AbortController;
|
||||
bareMode: boolean;
|
||||
clientAssetBundle: AssetBundle;
|
||||
|
||||
constructor(options: ServerOptions) {
|
||||
constructor(
|
||||
private spacePrimitives: SpacePrimitives,
|
||||
private options: ServerOptions,
|
||||
) {
|
||||
this.hostname = options.hostname;
|
||||
this.port = options.port;
|
||||
this.app = new Application(); //{ serverConstructor: FlashServer });
|
||||
this.app = new Application();
|
||||
this.user = options.user ?? Deno.env.get("SB_USER");
|
||||
this.systemBoot = new SpaceSystem(
|
||||
options.assetBundle,
|
||||
options.pagesPath,
|
||||
options.dbPath,
|
||||
);
|
||||
this.bareMode = options.bareMode || false;
|
||||
this.clientAssetBundle = options.clientAssetBundle;
|
||||
}
|
||||
|
||||
// Second, for loading plug JSON files with absolute or relative (from CWD) paths
|
||||
this.systemBoot.eventHook.addLocalListener(
|
||||
"get-plug:file",
|
||||
async (plugPath: string): Promise<Manifest> => {
|
||||
const resolvedPath = path.resolve(plugPath);
|
||||
try {
|
||||
const manifestJson = await Deno.readTextFile(resolvedPath);
|
||||
return JSON.parse(manifestJson);
|
||||
} catch {
|
||||
throw new Error(
|
||||
`No such file: ${resolvedPath} or could not parse as JSON`,
|
||||
);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// Rescan disk every 5s to detect any out-of-process file changes
|
||||
setInterval(() => {
|
||||
this.systemBoot.space.updatePageList().catch(console.error);
|
||||
}, 5000);
|
||||
|
||||
// Register the HTTP endpoint hook (with "/_/<plug-name>"" prefix, hardcoded for now)
|
||||
this.systemBoot.system.addHook(new EndpointHook(this.app, "/_"));
|
||||
// Replaces some template variables in index.html in a rather ad-hoc manner, but YOLO
|
||||
renderIndexHtml() {
|
||||
return this.clientAssetBundle.readTextFileSync(".client/index.html")
|
||||
.replaceAll(
|
||||
"{{SPACE_PATH}}",
|
||||
this.options.pagesPath.replaceAll("\\", "\\\\"),
|
||||
).replaceAll(
|
||||
"{{SYNC_ENDPOINT}}",
|
||||
"/.fs",
|
||||
);
|
||||
}
|
||||
|
||||
async start() {
|
||||
await this.systemBoot.start();
|
||||
await this.systemBoot.ensureSpaceIndex();
|
||||
await ensureAndLoadSettings(this.systemBoot.space, this.bareMode);
|
||||
|
||||
this.addPasswordAuth(this.app);
|
||||
await ensureSettingsAndIndex(this.spacePrimitives);
|
||||
|
||||
// Serve static files (javascript, css, html)
|
||||
this.app.use(async ({ request, response }, next) => {
|
||||
if (request.url.pathname === "/") {
|
||||
if (request.headers.get("If-Modified-Since") === staticLastModified) {
|
||||
response.status = 304;
|
||||
return;
|
||||
}
|
||||
// Note: we're explicitly not setting Last-Modified and If-Modified-Since header here because this page is dynamic
|
||||
response.headers.set("Content-type", "text/html");
|
||||
response.body = this.systemBoot.assetBundle.readTextFileSync(
|
||||
"web/index.html",
|
||||
);
|
||||
response.headers.set("Last-Modified", staticLastModified);
|
||||
response.body = this.renderIndexHtml();
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const assetName = `web${request.url.pathname}`;
|
||||
const assetName = request.url.pathname.slice(1);
|
||||
if (
|
||||
this.systemBoot.assetBundle.has(assetName) &&
|
||||
request.headers.get("If-Modified-Since") === staticLastModified
|
||||
this.clientAssetBundle.has(assetName) &&
|
||||
request.headers.get("If-Modified-Since") ===
|
||||
utcDateString(this.clientAssetBundle.getMtime(assetName))
|
||||
) {
|
||||
response.status = 304;
|
||||
return;
|
||||
@@ -100,14 +73,17 @@ export class HttpServer {
|
||||
response.status = 200;
|
||||
response.headers.set(
|
||||
"Content-type",
|
||||
this.systemBoot.assetBundle.getMimeType(assetName),
|
||||
this.clientAssetBundle.getMimeType(assetName),
|
||||
);
|
||||
const data = this.systemBoot.assetBundle.readFileSync(
|
||||
const data = this.clientAssetBundle.readFileSync(
|
||||
assetName,
|
||||
);
|
||||
response.headers.set("Cache-Control", "no-cache");
|
||||
response.headers.set("Content-length", "" + data.length);
|
||||
response.headers.set("Last-Modified", staticLastModified);
|
||||
response.headers.set(
|
||||
"Last-Modified",
|
||||
utcDateString(this.clientAssetBundle.getMtime(assetName)),
|
||||
);
|
||||
|
||||
if (request.method === "GET") {
|
||||
response.body = data;
|
||||
@@ -117,30 +93,38 @@ export class HttpServer {
|
||||
}
|
||||
});
|
||||
|
||||
// Fallback, serve index.html
|
||||
this.app.use(({ request, response }, next) => {
|
||||
if (
|
||||
!request.url.pathname.startsWith("/.fs") &&
|
||||
request.url.pathname !== "/.auth"
|
||||
) {
|
||||
response.headers.set("Content-type", "text/html");
|
||||
response.body = this.renderIndexHtml();
|
||||
} else {
|
||||
return next();
|
||||
}
|
||||
});
|
||||
|
||||
// Pages API
|
||||
const fsRouter = this.buildFsRouter(this.systemBoot.spacePrimitives);
|
||||
const fsRouter = this.buildFsRouter(this.spacePrimitives);
|
||||
this.addPasswordAuth(this.app);
|
||||
this.app.use(fsRouter.routes());
|
||||
this.app.use(fsRouter.allowedMethods());
|
||||
|
||||
// Plug API
|
||||
const plugRouter = this.buildPlugRouter();
|
||||
this.app.use(plugRouter.routes());
|
||||
this.app.use(plugRouter.allowedMethods());
|
||||
|
||||
// Fallback, serve index.html
|
||||
this.app.use((ctx) => {
|
||||
ctx.response.headers.set("Content-type", "text/html");
|
||||
ctx.response.body = this.systemBoot.assetBundle.readTextFileSync(
|
||||
"web/index.html",
|
||||
);
|
||||
});
|
||||
|
||||
this.abortController = new AbortController();
|
||||
this.app.listen({
|
||||
const listenOptions: any = {
|
||||
hostname: this.hostname,
|
||||
port: this.port,
|
||||
signal: this.abortController.signal,
|
||||
})
|
||||
};
|
||||
if (this.options.keyFile) {
|
||||
listenOptions.key = Deno.readTextFileSync(this.options.keyFile);
|
||||
}
|
||||
if (this.options.certFile) {
|
||||
listenOptions.cert = Deno.readTextFileSync(this.options.certFile);
|
||||
}
|
||||
this.app.listen(listenOptions)
|
||||
.catch((e: any) => {
|
||||
console.log("Server listen error:", e.message);
|
||||
Deno.exit(1);
|
||||
@@ -166,15 +150,16 @@ export class HttpServer {
|
||||
if (!excludedPaths.includes(request.url.pathname)) {
|
||||
const authCookie = await cookies.get("auth");
|
||||
if (!authCookie || authCookie !== b64User) {
|
||||
response.redirect(`/.auth?refer=${request.url.pathname}`);
|
||||
response.status = 401;
|
||||
response.body = "Unauthorized, please authenticate";
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (request.url.pathname === "/.auth") {
|
||||
if (request.method === "GET") {
|
||||
response.headers.set("Content-type", "text/html");
|
||||
response.body = this.systemBoot.assetBundle.readTextFileSync(
|
||||
"web/auth.html",
|
||||
response.body = this.clientAssetBundle.readTextFileSync(
|
||||
".client/auth.html",
|
||||
);
|
||||
return;
|
||||
} else if (request.method === "POST") {
|
||||
@@ -211,18 +196,71 @@ export class HttpServer {
|
||||
// File list
|
||||
fsRouter.get("/", async ({ response }) => {
|
||||
response.headers.set("Content-type", "application/json");
|
||||
response.headers.set("X-Space-Path", this.options.pagesPath);
|
||||
const files = await spacePrimitives.fetchFileList();
|
||||
response.body = JSON.stringify(files);
|
||||
});
|
||||
|
||||
// RPC
|
||||
fsRouter.post("/", async ({ request, response }) => {
|
||||
const body = await request.body({ type: "json" }).value;
|
||||
try {
|
||||
switch (body.operation) {
|
||||
case "fetch": {
|
||||
const result = await performLocalFetch(body.url, body.options);
|
||||
response.headers.set("Content-Type", "application/json");
|
||||
response.body = JSON.stringify(result);
|
||||
return;
|
||||
}
|
||||
case "shell": {
|
||||
// TODO: Have a nicer way to do this
|
||||
if (this.options.pagesPath.startsWith("s3://")) {
|
||||
response.status = 500;
|
||||
response.body = JSON.stringify({
|
||||
stdout: "",
|
||||
stderr: "Cannot run shell commands with S3 backend",
|
||||
code: 500,
|
||||
});
|
||||
return;
|
||||
}
|
||||
const p = new Deno.Command(body.cmd, {
|
||||
args: body.args,
|
||||
cwd: this.options.pagesPath,
|
||||
stdout: "piped",
|
||||
stderr: "piped",
|
||||
});
|
||||
const output = await p.output();
|
||||
const stdout = new TextDecoder().decode(output.stdout);
|
||||
const stderr = new TextDecoder().decode(output.stderr);
|
||||
|
||||
response.headers.set("Content-Type", "application/json");
|
||||
response.body = JSON.stringify({
|
||||
stdout,
|
||||
stderr,
|
||||
code: output.code,
|
||||
});
|
||||
return;
|
||||
}
|
||||
default:
|
||||
response.headers.set("Content-Type", "text/plain");
|
||||
response.status = 400;
|
||||
response.body = "Unknown operation";
|
||||
}
|
||||
} catch (e: any) {
|
||||
console.log("Error", e);
|
||||
response.status = 500;
|
||||
response.body = e.message;
|
||||
return;
|
||||
}
|
||||
});
|
||||
|
||||
fsRouter
|
||||
.get("\/(.+)", async ({ params, response, request }) => {
|
||||
const name = params[0];
|
||||
// console.log("Loading file", name);
|
||||
console.log("Loading file", name);
|
||||
try {
|
||||
const attachmentData = await spacePrimitives.readFile(
|
||||
name,
|
||||
"arraybuffer",
|
||||
);
|
||||
const lastModifiedHeader = new Date(attachmentData.meta.lastModified)
|
||||
.toUTCString();
|
||||
@@ -242,7 +280,7 @@ export class HttpServer {
|
||||
lastModifiedHeader,
|
||||
);
|
||||
response.headers.set("Content-Type", attachmentData.meta.contentType);
|
||||
response.body = attachmentData.data as ArrayBuffer;
|
||||
response.body = attachmentData.data;
|
||||
} catch {
|
||||
// console.error("Error in main router", e);
|
||||
response.status = 404;
|
||||
@@ -266,7 +304,6 @@ export class HttpServer {
|
||||
try {
|
||||
const meta = await spacePrimitives.writeFile(
|
||||
name,
|
||||
"arraybuffer",
|
||||
body,
|
||||
);
|
||||
response.status = 200;
|
||||
@@ -292,7 +329,7 @@ export class HttpServer {
|
||||
response.headers.set("X-Permission", meta.perm);
|
||||
} catch {
|
||||
response.status = 404;
|
||||
response.body = "File not found";
|
||||
response.body = "Not found";
|
||||
// console.error("Options failed", err);
|
||||
}
|
||||
})
|
||||
@@ -308,77 +345,17 @@ export class HttpServer {
|
||||
response.body = e.message;
|
||||
}
|
||||
});
|
||||
return new Router().use("/fs", fsRouter.routes());
|
||||
return new Router().use("/.fs", fsRouter.routes());
|
||||
}
|
||||
|
||||
private buildPlugRouter(): Router {
|
||||
const plugRouter = new Router();
|
||||
const system = this.systemBoot.system;
|
||||
|
||||
plugRouter.post(
|
||||
"/:plug/syscall/:name",
|
||||
async (ctx) => {
|
||||
const name = ctx.params.name;
|
||||
const plugName = ctx.params.plug;
|
||||
const args = await ctx.request.body().value;
|
||||
const plug = system.loadedPlugs.get(plugName);
|
||||
if (!plug) {
|
||||
ctx.response.status = 404;
|
||||
ctx.response.body = `Plug ${plugName} not found`;
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const result = await system.syscallWithContext(
|
||||
{ plug },
|
||||
name,
|
||||
args,
|
||||
);
|
||||
ctx.response.headers.set("Content-Type", "application/json");
|
||||
ctx.response.body = JSON.stringify(result);
|
||||
} catch (e: any) {
|
||||
console.log("Error", e);
|
||||
ctx.response.status = 500;
|
||||
ctx.response.body = e.message;
|
||||
return;
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
plugRouter.post(
|
||||
"/:plug/function/:name",
|
||||
async (ctx) => {
|
||||
const name = ctx.params.name;
|
||||
const plugName = ctx.params.plug;
|
||||
const args = await ctx.request.body().value;
|
||||
const plug = system.loadedPlugs.get(plugName);
|
||||
if (!plug) {
|
||||
ctx.response.status = 404;
|
||||
ctx.response.body = `Plug ${plugName} not found`;
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const result = await plug.invoke(name, args);
|
||||
ctx.response.headers.set("Content-Type", "application/json");
|
||||
ctx.response.body = JSON.stringify(result);
|
||||
} catch (e: any) {
|
||||
ctx.response.status = 500;
|
||||
// console.log("Error invoking function", e);
|
||||
ctx.response.body = e.message;
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
return new Router().use("/plug", plugRouter.routes());
|
||||
}
|
||||
|
||||
async stop() {
|
||||
const system = this.systemBoot.system;
|
||||
stop() {
|
||||
if (this.abortController) {
|
||||
console.log("Stopping");
|
||||
await system.unloadAll();
|
||||
console.log("Stopped plugs");
|
||||
this.abortController.abort();
|
||||
console.log("stopped server");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function utcDateString(mtime: number): string {
|
||||
return new Date(mtime).toUTCString();
|
||||
}
|
||||
|
||||
@@ -1,189 +0,0 @@
|
||||
import { SilverBulletHooks } from "../common/manifest.ts";
|
||||
import { loadMarkdownExtensions } from "../common/markdown_parser/markdown_ext.ts";
|
||||
import buildMarkdown from "../common/markdown_parser/parser.ts";
|
||||
import { DiskSpacePrimitives } from "../common/spaces/disk_space_primitives.ts";
|
||||
import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives.ts";
|
||||
import { Space } from "../common/spaces/space.ts";
|
||||
import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
|
||||
import { markdownSyscalls } from "../common/syscalls/markdown.ts";
|
||||
import { createSandbox } from "../plugos/environments/deno_sandbox.ts";
|
||||
import { EventHook } from "../plugos/hooks/event.ts";
|
||||
import { CronHook } from "../plugos/hooks/cron.ts";
|
||||
import { esbuildSyscalls } from "../plugos/syscalls/esbuild.ts";
|
||||
import { eventSyscalls } from "../plugos/syscalls/event.ts";
|
||||
import fileSystemSyscalls from "../plugos/syscalls/fs.deno.ts";
|
||||
import {
|
||||
ensureFTSTable,
|
||||
fullTextSearchSyscalls,
|
||||
} from "../plugos/syscalls/fulltext.sqlite.ts";
|
||||
import sandboxSyscalls from "../plugos/syscalls/sandbox.ts";
|
||||
import shellSyscalls from "../plugos/syscalls/shell.deno.ts";
|
||||
import {
|
||||
ensureTable as ensureStoreTable,
|
||||
storeSyscalls,
|
||||
} from "../plugos/syscalls/store.sqlite.ts";
|
||||
import { System } from "../plugos/system.ts";
|
||||
import { PageNamespaceHook } from "../common/hooks/page_namespace.ts";
|
||||
import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts";
|
||||
import {
|
||||
ensureTable as ensureIndexTable,
|
||||
pageIndexSyscalls,
|
||||
} from "./syscalls/index.ts";
|
||||
import spaceSyscalls from "../common/syscalls/space.ts";
|
||||
import { systemSyscalls } from "./syscalls/system.ts";
|
||||
import { AssetBundlePlugSpacePrimitives } from "../common/spaces/asset_bundle_space_primitives.ts";
|
||||
import assetSyscalls from "../plugos/syscalls/asset.ts";
|
||||
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
|
||||
import { AsyncSQLite } from "../plugos/sqlite/async_sqlite.ts";
|
||||
import { FileMetaSpacePrimitives } from "../common/spaces/file_meta_space_primitives.ts";
|
||||
import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts";
|
||||
import { syncSyscalls } from "../common/syscalls/sync.ts";
|
||||
export const indexRequiredKey = "$spaceIndexed";
|
||||
|
||||
// A composition of a PlugOS system attached to a Space for server-side use
|
||||
export class SpaceSystem {
|
||||
public system: System<SilverBulletHooks>;
|
||||
public space: Space;
|
||||
public eventHook: EventHook;
|
||||
public spacePrimitives: SpacePrimitives;
|
||||
|
||||
private db: AsyncSQLite;
|
||||
|
||||
constructor(
|
||||
readonly assetBundle: AssetBundle,
|
||||
pagesPath: string,
|
||||
databasePath: string,
|
||||
) {
|
||||
const globalModules = JSON.parse(
|
||||
assetBundle.readTextFileSync(`web/global.plug.json`),
|
||||
);
|
||||
|
||||
// Set up the PlugOS System
|
||||
this.system = new System<SilverBulletHooks>("server");
|
||||
|
||||
// Instantiate the event bus hook
|
||||
this.eventHook = new EventHook();
|
||||
this.system.addHook(this.eventHook);
|
||||
|
||||
// And the page namespace hook
|
||||
const namespaceHook = new PageNamespaceHook();
|
||||
this.system.addHook(namespaceHook);
|
||||
|
||||
// The database used for persistence (SQLite)
|
||||
this.db = new AsyncSQLite(databasePath);
|
||||
this.db.init().catch((e) => {
|
||||
console.error("Error initializing database", e);
|
||||
});
|
||||
|
||||
const indexSyscalls = pageIndexSyscalls(this.db);
|
||||
// The space
|
||||
try {
|
||||
this.spacePrimitives = new FileMetaSpacePrimitives(
|
||||
new AssetBundlePlugSpacePrimitives(
|
||||
new EventedSpacePrimitives(
|
||||
new PlugSpacePrimitives(
|
||||
new DiskSpacePrimitives(pagesPath),
|
||||
namespaceHook,
|
||||
"server",
|
||||
),
|
||||
this.eventHook,
|
||||
),
|
||||
assetBundle,
|
||||
),
|
||||
indexSyscalls,
|
||||
);
|
||||
this.space = new Space(this.spacePrimitives);
|
||||
} catch (e: any) {
|
||||
if (e instanceof Deno.errors.NotFound) {
|
||||
console.error("Pages folder", pagesPath, "not found");
|
||||
} else {
|
||||
console.error(e.message);
|
||||
}
|
||||
Deno.exit(1);
|
||||
}
|
||||
|
||||
// The cron hook
|
||||
this.system.addHook(new CronHook(this.system));
|
||||
|
||||
// Register syscalls available on the server side
|
||||
this.system.registerSyscalls(
|
||||
[],
|
||||
indexSyscalls,
|
||||
storeSyscalls(this.db, "store"),
|
||||
fullTextSearchSyscalls(this.db, "fts"),
|
||||
spaceSyscalls(this.space),
|
||||
syncSyscalls(this.spacePrimitives, this.system),
|
||||
eventSyscalls(this.eventHook),
|
||||
markdownSyscalls(buildMarkdown([])),
|
||||
esbuildSyscalls([globalModules]),
|
||||
systemSyscalls(this.loadPlugsFromSpace.bind(this), this.system),
|
||||
sandboxSyscalls(this.system),
|
||||
assetSyscalls(this.system),
|
||||
sandboxFetchSyscalls(),
|
||||
);
|
||||
|
||||
// Danger zone, these syscalls require requesting specific permissions
|
||||
this.system.registerSyscalls(["shell"], shellSyscalls(pagesPath));
|
||||
this.system.registerSyscalls(["fs"], fileSystemSyscalls("/"));
|
||||
|
||||
this.system.on({
|
||||
sandboxInitialized: async (sandbox) => {
|
||||
for (
|
||||
const [modName, code] of Object.entries(
|
||||
globalModules.dependencies!,
|
||||
)
|
||||
) {
|
||||
await sandbox.loadDependency(modName, code as string);
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Loads all plugs under "_plug/" in the space
|
||||
async loadPlugsFromSpace() {
|
||||
await this.space.updatePageList();
|
||||
|
||||
const allPlugs = await this.space.listPlugs();
|
||||
|
||||
console.log("Going to load", allPlugs.length, "plugs...");
|
||||
await Promise.all(allPlugs.map(async (plugName) => {
|
||||
const { data } = await this.space.readAttachment(plugName, "utf8");
|
||||
await this.system.load(JSON.parse(data as string), createSandbox);
|
||||
}));
|
||||
|
||||
// Re-register the markdown syscall with new markdown extensions
|
||||
this.system.registerSyscalls(
|
||||
[],
|
||||
markdownSyscalls(buildMarkdown(loadMarkdownExtensions(this.system))),
|
||||
);
|
||||
}
|
||||
|
||||
// Checks if the space has been indexed, and if not, does so
|
||||
async ensureSpaceIndex(forceReindex = false) {
|
||||
const corePlug = this.system.loadedPlugs.get("core");
|
||||
if (!corePlug) {
|
||||
console.error("Something went very wrong, 'core' plug not found");
|
||||
return;
|
||||
}
|
||||
|
||||
// Do we need to reindex this space?
|
||||
if (
|
||||
forceReindex ||
|
||||
!(await this.system.localSyscall("core", "store.get", [indexRequiredKey]))
|
||||
) {
|
||||
console.log("Now reindexing space...");
|
||||
await corePlug.invoke("reindexSpace", []);
|
||||
await this.system.localSyscall("core", "store.set", [
|
||||
indexRequiredKey,
|
||||
true,
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
async start() {
|
||||
await ensureIndexTable(this.db);
|
||||
await ensureStoreTable(this.db, "store");
|
||||
await ensureFTSTable(this.db, "fts");
|
||||
await this.loadPlugsFromSpace();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import { S3SpacePrimitives } from "./s3_space_primitives.ts";
|
||||
import { assert, assertEquals } from "../../test_deps.ts";
|
||||
|
||||
Deno.test("s3_space_primitives", async () => {
|
||||
return;
|
||||
const options = {
|
||||
accessKey: Deno.env.get("AWS_ACCESS_KEY_ID")!,
|
||||
secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!,
|
||||
endPoint: "s3.eu-central-1.amazonaws.com",
|
||||
region: "eu-central-1",
|
||||
bucket: "zef-sb-space",
|
||||
};
|
||||
|
||||
const primitives = new S3SpacePrimitives(options);
|
||||
console.log(await primitives.fetchFileList());
|
||||
console.log(
|
||||
await primitives.writeFile("test+'s.txt", stringToBytes("Hello world!")),
|
||||
);
|
||||
assertEquals(
|
||||
stringToBytes("Hello world!"),
|
||||
(await primitives.readFile("test+'s.txt")).data,
|
||||
);
|
||||
await primitives.deleteFile("test+'s.txt");
|
||||
|
||||
try {
|
||||
await primitives.getFileMeta("test+'s.txt");
|
||||
assert(false);
|
||||
} catch (e: any) {
|
||||
assertEquals(e.message, "Not found");
|
||||
}
|
||||
|
||||
// console.log(await primitives.readFile("SETTINGS.md", "utf8"));
|
||||
});
|
||||
|
||||
function stringToBytes(str: string): Uint8Array {
|
||||
return new TextEncoder().encode(str);
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
// We're explicitly using 0.4.0 to be able to hijack the path encoding, which is inconsisently broken in 0.5.0
|
||||
import { S3Client } from "https://deno.land/x/s3_lite_client@0.4.0/mod.ts";
|
||||
import type { ClientOptions } from "https://deno.land/x/s3_lite_client@0.4.0/client.ts";
|
||||
import { SpacePrimitives } from "../../common/spaces/space_primitives.ts";
|
||||
import { FileMeta } from "../../common/types.ts";
|
||||
import { mime } from "../deps.ts";
|
||||
|
||||
export class S3SpacePrimitives implements SpacePrimitives {
|
||||
client: S3Client;
|
||||
constructor(options: ClientOptions) {
|
||||
this.client = new S3Client(options);
|
||||
}
|
||||
|
||||
private encodePath(name: string): string {
|
||||
return uriEscapePath(name);
|
||||
}
|
||||
|
||||
private decodePath(encoded: string): string {
|
||||
// AWS only returns ' replace dwith '
|
||||
return encoded.replaceAll("'", "'");
|
||||
}
|
||||
|
||||
async fetchFileList(): Promise<FileMeta[]> {
|
||||
const allFiles: FileMeta[] = [];
|
||||
|
||||
for await (const obj of this.client.listObjects({ prefix: "" })) {
|
||||
allFiles.push({
|
||||
name: this.decodePath(obj.key),
|
||||
perm: "rw",
|
||||
lastModified: obj.lastModified.getTime(),
|
||||
contentType: mime.getType(obj.key) || "application/octet-stream",
|
||||
size: obj.size,
|
||||
});
|
||||
}
|
||||
return allFiles;
|
||||
}
|
||||
|
||||
async readFile(
|
||||
name: string,
|
||||
): Promise<{ data: Uint8Array; meta: FileMeta }> {
|
||||
try {
|
||||
// console.log("Fetching object", encodeURI(name));
|
||||
const obj = await this.client.getObject(this.encodePath(name));
|
||||
|
||||
const contentType = mime.getType(name) || "application/octet-stream";
|
||||
const meta: FileMeta = {
|
||||
name,
|
||||
perm: "rw",
|
||||
lastModified: new Date(obj.headers.get("Last-Modified")!).getTime(),
|
||||
contentType,
|
||||
size: parseInt(obj.headers.get("Content-Length")!),
|
||||
};
|
||||
|
||||
return {
|
||||
data: new Uint8Array(await obj.arrayBuffer()),
|
||||
meta,
|
||||
};
|
||||
} catch (e: any) {
|
||||
console.log("GOt error", e.message);
|
||||
|
||||
if (e.message.includes("does not exist")) {
|
||||
throw new Error(`Not found`);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
async getFileMeta(name: string): Promise<FileMeta> {
|
||||
try {
|
||||
const stat = await this.client.statObject(this.encodePath(name));
|
||||
return {
|
||||
name,
|
||||
perm: "rw",
|
||||
lastModified: new Date(stat.lastModified).getTime(),
|
||||
size: stat.size,
|
||||
contentType: mime.getType(name) || "application/octet-stream",
|
||||
};
|
||||
} catch (e: any) {
|
||||
if (e.message.includes("404")) {
|
||||
throw new Error(`Not found`);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
async writeFile(
|
||||
name: string,
|
||||
data: Uint8Array,
|
||||
): Promise<FileMeta> {
|
||||
if (data.byteLength === 0) {
|
||||
// S3 doesn't like empty files, so we'll put a space in it. Not ideal, but it works. I hope.
|
||||
data = new TextEncoder().encode(" ");
|
||||
}
|
||||
await this.client.putObject(this.encodePath(name), data);
|
||||
// TODO: Dangerous due to eventual consistency? maybe check with etag or versionid?
|
||||
return this.getFileMeta(name);
|
||||
}
|
||||
async deleteFile(name: string): Promise<void> {
|
||||
await this.client.deleteObject(this.encodePath(name));
|
||||
}
|
||||
}
|
||||
|
||||
// Stolen from https://github.com/aws/aws-sdk-js/blob/master/lib/util.js
|
||||
|
||||
export function uriEscapePath(string: string): string {
|
||||
return string.split("/").map(uriEscape).join("/");
|
||||
}
|
||||
|
||||
function uriEscape(string: string): string {
|
||||
let output = encodeURIComponent(string);
|
||||
output = output.replace(/[^A-Za-z0-9_.~\-%]+/g, escape);
|
||||
|
||||
// AWS percent-encodes some extra non-standard characters in a URI
|
||||
output = output.replace(/[*]/g, function (ch) {
|
||||
return "%" + ch.charCodeAt(0).toString(16).toUpperCase();
|
||||
});
|
||||
|
||||
return output;
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
import { assertEquals } from "https://deno.land/std@0.165.0/testing/asserts.ts";
|
||||
import { AsyncSQLite } from "../../plugos/sqlite/async_sqlite.ts";
|
||||
import { ensureTable, pageIndexSyscalls } from "./index.ts";
|
||||
|
||||
const fakeContext = {} as any;
|
||||
|
||||
Deno.test("Page index", async () => {
|
||||
const db = new AsyncSQLite(":memory:");
|
||||
await db.init();
|
||||
await ensureTable(db);
|
||||
const syscalls = pageIndexSyscalls(db);
|
||||
await syscalls["index.set"](fakeContext, "page1", "key1", "value1");
|
||||
assertEquals(
|
||||
"value1",
|
||||
await syscalls["index.get"](fakeContext, "page1", "key1"),
|
||||
);
|
||||
await syscalls["index.set"](fakeContext, "page1", "key1", "value2");
|
||||
assertEquals(
|
||||
"value2",
|
||||
await syscalls["index.get"](fakeContext, "page1", "key1"),
|
||||
);
|
||||
await syscalls["index.set"](fakeContext, "page1", "key2", "value1");
|
||||
assertEquals(
|
||||
[
|
||||
{ key: "key1", page: "page1", value: "value2" },
|
||||
{ key: "key2", page: "page1", value: "value1" },
|
||||
],
|
||||
await syscalls["index.queryPrefix"](fakeContext, ""),
|
||||
);
|
||||
await syscalls["index.delete"](fakeContext, "page1", "key1");
|
||||
assertEquals(
|
||||
[
|
||||
{ key: "key2", page: "page1", value: "value1" },
|
||||
],
|
||||
await syscalls["index.queryPrefix"](fakeContext, ""),
|
||||
);
|
||||
await syscalls["index.batchSet"](fakeContext, "page1", [
|
||||
{ key: "key1", value: "value1" },
|
||||
{ key: "key2", value: "value2" },
|
||||
{ key: "key3", value: "value3" },
|
||||
]);
|
||||
assertEquals(
|
||||
[
|
||||
{ key: "key1", page: "page1", value: "value1" },
|
||||
{ key: "key2", page: "page1", value: "value2" },
|
||||
{ key: "key3", page: "page1", value: "value3" },
|
||||
],
|
||||
await syscalls["index.queryPrefix"](fakeContext, ""),
|
||||
);
|
||||
db.stop();
|
||||
});
|
||||
@@ -1,123 +0,0 @@
|
||||
// import { Knex } from "knex";
|
||||
import { SysCallMapping } from "../../plugos/system.ts";
|
||||
import { Query, queryToSql } from "../../plugos/syscalls/store.sqlite.ts";
|
||||
import { ISQLite } from "../../plugos/sqlite/sqlite_interface.ts";
|
||||
|
||||
type Item = {
|
||||
page: string;
|
||||
key: string;
|
||||
value: any;
|
||||
};
|
||||
|
||||
export type KV = {
|
||||
key: string;
|
||||
value: any;
|
||||
};
|
||||
|
||||
const tableName = "page_index";
|
||||
|
||||
export async function ensureTable(db: ISQLite): Promise<void> {
|
||||
const result = await db.query(
|
||||
`SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
|
||||
tableName,
|
||||
);
|
||||
if (result.length === 0) {
|
||||
await db.execute(
|
||||
`CREATE TABLE ${tableName} (key STRING, page STRING, value TEXT, PRIMARY KEY (page, key));`,
|
||||
);
|
||||
await db.execute(
|
||||
`CREATE INDEX ${tableName}_idx ON ${tableName}(key);`,
|
||||
);
|
||||
// console.log(`Created table ${tableName}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function pageIndexSyscalls(db: ISQLite): SysCallMapping {
|
||||
const apiObj: SysCallMapping = {
|
||||
"index.set": async (_ctx, page: string, key: string, value: any) => {
|
||||
await db.execute(
|
||||
`INSERT INTO ${tableName}
|
||||
(page, key, value)
|
||||
VALUES (?, ?, ?)
|
||||
ON CONFLICT(page, key)
|
||||
DO UPDATE SET value=excluded.value`,
|
||||
page,
|
||||
key,
|
||||
JSON.stringify(value),
|
||||
);
|
||||
},
|
||||
"index.batchSet": async (_ctx, page: string, kvs: KV[]) => {
|
||||
if (kvs.length === 0) {
|
||||
return;
|
||||
}
|
||||
const values = kvs.flatMap((
|
||||
kv,
|
||||
) => [page, kv.key, JSON.stringify(kv.value)]);
|
||||
await db.execute(
|
||||
`INSERT INTO ${tableName}
|
||||
(page, key, value)
|
||||
VALUES ${kvs.map((_) => "(?, ?, ?)").join(",")}
|
||||
ON CONFLICT(key, page)
|
||||
DO UPDATE SET value=excluded.value`,
|
||||
...values,
|
||||
);
|
||||
},
|
||||
"index.delete": async (_ctx, page: string, key: string) => {
|
||||
await db.execute(
|
||||
`DELETE FROM ${tableName} WHERE key = ? AND page = ?`,
|
||||
key,
|
||||
page,
|
||||
);
|
||||
},
|
||||
"index.get": async (_ctx, page: string, key: string) => {
|
||||
const result = await db.query(
|
||||
`SELECT value FROM ${tableName} WHERE key = ? AND page = ?`,
|
||||
key,
|
||||
page,
|
||||
);
|
||||
if (result.length) {
|
||||
return JSON.parse(result[0].value);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
},
|
||||
"index.queryPrefix": async (_ctx, prefix: string) => {
|
||||
return (
|
||||
await db.query(
|
||||
`SELECT key, page, value FROM ${tableName} WHERE key LIKE ? ORDER BY key, page ASC`,
|
||||
`${prefix}%`,
|
||||
)
|
||||
).map(({ key, value, page }) => ({
|
||||
key,
|
||||
page,
|
||||
value: JSON.parse(value),
|
||||
}));
|
||||
},
|
||||
"index.query": async (_ctx, query: Query) => {
|
||||
const { sql, params } = queryToSql(query);
|
||||
return (
|
||||
await db.query(`SELECT key, value FROM ${tableName} ${sql}`, ...params)
|
||||
).map(({ key, value, page }: any) => ({
|
||||
key,
|
||||
page,
|
||||
value: JSON.parse(value),
|
||||
}));
|
||||
},
|
||||
"index.clearPageIndexForPage": async (ctx, page: string) => {
|
||||
await apiObj["index.deletePrefixForPage"](ctx, page, "");
|
||||
},
|
||||
"index.deletePrefixForPage": async (_ctx, page: string, prefix: string) => {
|
||||
await db.execute(
|
||||
`DELETE FROM ${tableName} WHERE key LIKE ? AND page = ?`,
|
||||
`${prefix}%`,
|
||||
page,
|
||||
);
|
||||
},
|
||||
"index.clearPageIndex": async () => {
|
||||
await db.execute(
|
||||
`DELETE FROM ${tableName}`,
|
||||
);
|
||||
},
|
||||
};
|
||||
return apiObj;
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
import { Plug } from "../../plugos/plug.ts";
|
||||
import { SysCallMapping, System } from "../../plugos/system.ts";
|
||||
|
||||
export function systemSyscalls(
|
||||
plugReloader: () => Promise<void>,
|
||||
system: System<any>,
|
||||
): SysCallMapping {
|
||||
return {
|
||||
"system.invokeFunction": (
|
||||
ctx,
|
||||
// Ignored in this context, always assuming server (this place)
|
||||
_env: string,
|
||||
name: string,
|
||||
...args: any[]
|
||||
) => {
|
||||
if (!ctx.plug) {
|
||||
throw Error("No plug associated with context");
|
||||
}
|
||||
let plug: Plug<any> | undefined = ctx.plug;
|
||||
if (name.indexOf(".") !== -1) {
|
||||
// plug name in the name
|
||||
const [plugName, functionName] = name.split(".");
|
||||
plug = system.loadedPlugs.get(plugName);
|
||||
if (!plug) {
|
||||
throw Error(`Plug ${plugName} not found`);
|
||||
}
|
||||
name = functionName;
|
||||
}
|
||||
return plug.invoke(name, args);
|
||||
},
|
||||
"system.reloadPlugs": () => {
|
||||
return plugReloader();
|
||||
},
|
||||
"system.getEnv": () => {
|
||||
return system.env;
|
||||
},
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user