Monorepo with yarn workspaces requires yarn 3.2
This commit is contained in:
Zef Hemel
@@ -1,272 +0,0 @@
|
||||
import express, { Express } from "express";
|
||||
import { SilverBulletHooks } from "../common/manifest";
|
||||
import { EndpointHook } from "../plugos/hooks/endpoint";
|
||||
import { readFile } from "fs/promises";
|
||||
import { System } from "../plugos/system";
|
||||
import cors from "cors";
|
||||
import { DiskSpacePrimitives } from "../common/spaces/disk_space_primitives";
|
||||
import path from "path";
|
||||
import bodyParser from "body-parser";
|
||||
import { EventHook } from "../plugos/hooks/event";
|
||||
import spaceSyscalls from "./syscalls/space";
|
||||
import { eventSyscalls } from "../plugos/syscalls/event";
|
||||
import { pageIndexSyscalls } from "./syscalls";
|
||||
import knex, { Knex } from "knex";
|
||||
import shellSyscalls from "../plugos/syscalls/shell.node";
|
||||
import { NodeCronHook } from "../plugos/hooks/node_cron";
|
||||
import { markdownSyscalls } from "../common/syscalls/markdown";
|
||||
import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives";
|
||||
import { Space } from "../common/spaces/space";
|
||||
import { safeRun, throttle } from "../webapp/util";
|
||||
import { createSandbox } from "../plugos/environments/node_sandbox";
|
||||
import { jwtSyscalls } from "../plugos/syscalls/jwt";
|
||||
import { fetchSyscalls } from "../plugos/syscalls/fetch.node";
|
||||
import buildMarkdown from "../webapp/parser";
|
||||
import { loadMarkdownExtensions } from "../webapp/markdown_ext";
|
||||
|
||||
export class ExpressServer {
|
||||
app: Express;
|
||||
system: System<SilverBulletHooks>;
|
||||
private rootPath: string;
|
||||
private space: Space;
|
||||
private distDir: string;
|
||||
private eventHook: EventHook;
|
||||
private db: Knex<any, unknown[]>;
|
||||
|
||||
constructor(
|
||||
app: Express,
|
||||
rootPath: string,
|
||||
distDir: string,
|
||||
system: System<SilverBulletHooks>
|
||||
) {
|
||||
this.app = app;
|
||||
this.rootPath = rootPath;
|
||||
this.distDir = distDir;
|
||||
this.system = system;
|
||||
|
||||
// Setup system
|
||||
this.eventHook = new EventHook();
|
||||
system.addHook(this.eventHook);
|
||||
this.space = new Space(
|
||||
new EventedSpacePrimitives(
|
||||
new DiskSpacePrimitives(rootPath),
|
||||
this.eventHook
|
||||
),
|
||||
true
|
||||
);
|
||||
this.db = knex({
|
||||
client: "better-sqlite3",
|
||||
connection: {
|
||||
filename: path.join(rootPath, "data.db"),
|
||||
},
|
||||
useNullAsDefault: true,
|
||||
});
|
||||
|
||||
system.registerSyscalls(["shell"], shellSyscalls(rootPath));
|
||||
system.addHook(new NodeCronHook());
|
||||
|
||||
system.registerSyscalls([], pageIndexSyscalls(this.db));
|
||||
system.registerSyscalls([], spaceSyscalls(this.space));
|
||||
system.registerSyscalls([], eventSyscalls(this.eventHook));
|
||||
system.registerSyscalls([], markdownSyscalls(buildMarkdown([])));
|
||||
system.registerSyscalls([], fetchSyscalls());
|
||||
system.registerSyscalls([], jwtSyscalls());
|
||||
system.addHook(new EndpointHook(app, "/_/"));
|
||||
|
||||
let throttledRebuildMdExtensions = throttle(() => {
|
||||
this.rebuildMdExtensions();
|
||||
}, 100);
|
||||
|
||||
this.space.on({
|
||||
plugLoaded: (plugName, plug) => {
|
||||
safeRun(async () => {
|
||||
console.log("Plug load", plugName);
|
||||
await system.load(plugName, plug, createSandbox);
|
||||
});
|
||||
throttledRebuildMdExtensions();
|
||||
},
|
||||
plugUnloaded: (plugName) => {
|
||||
safeRun(async () => {
|
||||
console.log("Plug unload", plugName);
|
||||
await system.unload(plugName);
|
||||
});
|
||||
throttledRebuildMdExtensions();
|
||||
},
|
||||
});
|
||||
|
||||
setInterval(() => {
|
||||
this.space.updatePageListAsync();
|
||||
}, 5000);
|
||||
this.space.updatePageListAsync();
|
||||
}
|
||||
|
||||
rebuildMdExtensions() {
|
||||
this.system.registerSyscalls(
|
||||
[],
|
||||
markdownSyscalls(buildMarkdown(loadMarkdownExtensions(this.system)))
|
||||
);
|
||||
}
|
||||
|
||||
async init() {
|
||||
console.log("Setting up router");
|
||||
|
||||
let fsRouter = express.Router();
|
||||
|
||||
// Page list
|
||||
fsRouter.route("/").get(async (req, res) => {
|
||||
let { nowTimestamp, pages } = await this.space.fetchPageList();
|
||||
res.header("Now-Timestamp", "" + nowTimestamp);
|
||||
res.json([...pages]);
|
||||
});
|
||||
|
||||
fsRouter.route("/").post(bodyParser.json(), async (req, res) => {});
|
||||
|
||||
fsRouter
|
||||
.route(/\/(.+)/)
|
||||
.get(async (req, res) => {
|
||||
let pageName = req.params[0];
|
||||
// console.log("Getting", pageName);
|
||||
try {
|
||||
let pageData = await this.space.readPage(pageName);
|
||||
res.status(200);
|
||||
res.header("Last-Modified", "" + pageData.meta.lastModified);
|
||||
res.header("Content-Type", "text/markdown");
|
||||
res.send(pageData.text);
|
||||
} catch (e) {
|
||||
// CORS
|
||||
res.status(200);
|
||||
res.header("X-Status", "404");
|
||||
res.send("");
|
||||
}
|
||||
})
|
||||
.put(bodyParser.text({ type: "*/*" }), async (req, res) => {
|
||||
let pageName = req.params[0];
|
||||
console.log("Saving", pageName);
|
||||
|
||||
try {
|
||||
let meta = await this.space.writePage(
|
||||
pageName,
|
||||
req.body,
|
||||
false,
|
||||
req.header("Last-Modified")
|
||||
? +req.header("Last-Modified")!
|
||||
: undefined
|
||||
);
|
||||
res.status(200);
|
||||
res.header("Last-Modified", "" + meta.lastModified);
|
||||
res.send("OK");
|
||||
} catch (err) {
|
||||
res.status(500);
|
||||
res.send("Write failed");
|
||||
console.error("Pipeline failed", err);
|
||||
}
|
||||
})
|
||||
.options(async (req, res) => {
|
||||
let pageName = req.params[0];
|
||||
try {
|
||||
const meta = await this.space.getPageMeta(pageName);
|
||||
res.status(200);
|
||||
res.header("Last-Modified", "" + meta.lastModified);
|
||||
res.header("Content-Type", "text/markdown");
|
||||
res.send("");
|
||||
} catch (e) {
|
||||
// CORS
|
||||
res.status(200);
|
||||
res.header("X-Status", "404");
|
||||
res.send("Not found");
|
||||
}
|
||||
})
|
||||
.delete(async (req, res) => {
|
||||
let pageName = req.params[0];
|
||||
try {
|
||||
await this.space.deletePage(pageName);
|
||||
res.status(200);
|
||||
res.send("OK");
|
||||
} catch (e) {
|
||||
console.error("Error deleting file", e);
|
||||
res.status(500);
|
||||
res.send("OK");
|
||||
}
|
||||
});
|
||||
|
||||
this.app.use(
|
||||
"/fs",
|
||||
cors({
|
||||
methods: "GET,HEAD,PUT,OPTIONS,POST,DELETE",
|
||||
preflightContinue: true,
|
||||
}),
|
||||
fsRouter
|
||||
);
|
||||
|
||||
let plugRouter = express.Router();
|
||||
|
||||
plugRouter.post(
|
||||
"/:plug/syscall/:name",
|
||||
bodyParser.json(),
|
||||
async (req, res) => {
|
||||
const name = req.params.name;
|
||||
const plugName = req.params.plug;
|
||||
const args = req.body as any;
|
||||
const plug = this.system.loadedPlugs.get(plugName);
|
||||
if (!plug) {
|
||||
res.status(404);
|
||||
return res.send(`Plug ${plugName} not found`);
|
||||
}
|
||||
try {
|
||||
const result = await this.system.syscallWithContext(
|
||||
{ plug },
|
||||
name,
|
||||
args
|
||||
);
|
||||
res.status(200);
|
||||
res.send(result);
|
||||
} catch (e: any) {
|
||||
res.status(500);
|
||||
return res.send(e.message);
|
||||
}
|
||||
}
|
||||
);
|
||||
plugRouter.post(
|
||||
"/:plug/function/:name",
|
||||
bodyParser.json(),
|
||||
async (req, res) => {
|
||||
const name = req.params.name;
|
||||
const plugName = req.params.plug;
|
||||
const args = req.body as any[];
|
||||
const plug = this.system.loadedPlugs.get(plugName);
|
||||
if (!plug) {
|
||||
res.status(404);
|
||||
return res.send(`Plug ${plugName} not found`);
|
||||
}
|
||||
try {
|
||||
console.log("Invoking", name);
|
||||
const result = await plug.invoke(name, args);
|
||||
res.status(200);
|
||||
res.send(result);
|
||||
} catch (e: any) {
|
||||
res.status(500);
|
||||
console.log("Error invoking function", e);
|
||||
return res.send(e.message);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
this.app.use(
|
||||
"/plug",
|
||||
cors({
|
||||
methods: "GET,HEAD,PUT,OPTIONS,POST,DELETE",
|
||||
preflightContinue: true,
|
||||
}),
|
||||
plugRouter
|
||||
);
|
||||
|
||||
// Fallback, serve index.html
|
||||
let cachedIndex: string | undefined = undefined;
|
||||
this.app.get("/*", async (req, res) => {
|
||||
if (!cachedIndex) {
|
||||
cachedIndex = await readFile(`${this.distDir}/index.html`, "utf8");
|
||||
}
|
||||
res.status(200).header("Content-Type", "text/html").send(cachedIndex);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import express from "express";
|
||||
import http from "http";
|
||||
import yargs from "yargs";
|
||||
import { hideBin } from "yargs/helpers";
|
||||
import { SilverBulletHooks } from "../common/manifest";
|
||||
import { ExpressServer } from "./api_server";
|
||||
import { System } from "../plugos/system";
|
||||
|
||||
let args = yargs(hideBin(process.argv))
|
||||
.option("port", {
|
||||
type: "number",
|
||||
default: 3000,
|
||||
})
|
||||
.parse();
|
||||
|
||||
if (!args._.length) {
|
||||
console.error("Usage: silverbullet <path-to-pages>");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const pagesPath = args._[0] as string;
|
||||
|
||||
const app = express();
|
||||
const server = http.createServer(app);
|
||||
const system = new System<SilverBulletHooks>("server");
|
||||
|
||||
const port = args.port;
|
||||
const distDir = `${__dirname}/../webapp`;
|
||||
|
||||
app.use("/", express.static(distDir));
|
||||
|
||||
const expressServer = new ExpressServer(app, pagesPath, distDir, system);
|
||||
expressServer
|
||||
.init()
|
||||
.then(async () => {
|
||||
server.listen(port, () => {
|
||||
console.log(`Server listening on port ${port}`);
|
||||
});
|
||||
})
|
||||
.catch((e) => {
|
||||
console.error(e);
|
||||
});
|
||||
@@ -1,113 +0,0 @@
|
||||
import { Knex } from "knex";
|
||||
import { SysCallMapping } from "../../plugos/system";
|
||||
|
||||
import { ensureTable, storeSyscalls } from "../../plugos/syscalls/store.knex_node";
|
||||
|
||||
type IndexItem = {
|
||||
page: string;
|
||||
key: string;
|
||||
value: any;
|
||||
};
|
||||
|
||||
export type KV = {
|
||||
key: string;
|
||||
value: any;
|
||||
};
|
||||
|
||||
/*
|
||||
Keyspace design:
|
||||
|
||||
for page lookups:
|
||||
p~page~key
|
||||
|
||||
for global lookups:
|
||||
k~key~page
|
||||
|
||||
*/
|
||||
|
||||
function pageKey(page: string, key: string) {
|
||||
return `p~${page}~${key}`;
|
||||
}
|
||||
|
||||
function unpackPageKey(dbKey: string): { page: string; key: string } {
|
||||
const [, page, key] = dbKey.split("~");
|
||||
return { page, key };
|
||||
}
|
||||
|
||||
function globalKey(page: string, key: string) {
|
||||
return `k~${key}~${page}`;
|
||||
}
|
||||
|
||||
function unpackGlobalKey(dbKey: string): { page: string; key: string } {
|
||||
const [, key, page] = dbKey.split("~");
|
||||
return { page, key };
|
||||
}
|
||||
|
||||
export async function ensurePageIndexTable(db: Knex<any, unknown>) {
|
||||
await ensureTable(db, "page_index");
|
||||
}
|
||||
|
||||
export function pageIndexSyscalls(db: Knex<any, unknown>): SysCallMapping {
|
||||
const storeCalls = storeSyscalls(db, "page_index");
|
||||
const apiObj: SysCallMapping = {
|
||||
"index.set": async (ctx, page: string, key: string, value: any) => {
|
||||
await storeCalls["store.set"](ctx, pageKey(page, key), value);
|
||||
await storeCalls["store.set"](ctx, globalKey(page, key), value);
|
||||
},
|
||||
"index.batchSet": async (ctx, page: string, kvs: KV[]) => {
|
||||
for (let { key, value } of kvs) {
|
||||
await apiObj["index.set"](ctx, page, key, value);
|
||||
}
|
||||
},
|
||||
"index.delete": async (ctx, page: string, key: string) => {
|
||||
await storeCalls["store.delete"](ctx, pageKey(page, key));
|
||||
await storeCalls["store.delete"](ctx, globalKey(page, key));
|
||||
},
|
||||
"index.get": async (ctx, page: string, key: string) => {
|
||||
return storeCalls["store.get"](ctx, pageKey(page, key));
|
||||
},
|
||||
"index.scanPrefixForPage": async (ctx, page: string, prefix: string) => {
|
||||
return (
|
||||
await storeCalls["store.queryPrefix"](ctx, pageKey(page, prefix))
|
||||
).map(({ key, value }: { key: string; value: any }) => {
|
||||
const { key: pageKey } = unpackPageKey(key);
|
||||
return {
|
||||
page,
|
||||
key: pageKey,
|
||||
value,
|
||||
};
|
||||
});
|
||||
},
|
||||
"index.scanPrefixGlobal": async (ctx, prefix: string) => {
|
||||
return (await storeCalls["store.queryPrefix"](ctx, `k~${prefix}`)).map(
|
||||
({ key, value }: { key: string; value: any }) => {
|
||||
const { page, key: pageKey } = unpackGlobalKey(key);
|
||||
return {
|
||||
page,
|
||||
key: pageKey,
|
||||
value,
|
||||
};
|
||||
}
|
||||
);
|
||||
},
|
||||
"index.clearPageIndexForPage": async (ctx, page: string) => {
|
||||
await apiObj["index.deletePrefixForPage"](ctx, page, "");
|
||||
},
|
||||
"index.deletePrefixForPage": async (ctx, page: string, prefix: string) => {
|
||||
// Collect all global keys for this page to delete
|
||||
let keysToDelete = (
|
||||
await storeCalls["store.queryPrefix"](ctx, pageKey(page, prefix))
|
||||
).map(({ key }: { key: string; value: string }) =>
|
||||
globalKey(page, unpackPageKey(key).key)
|
||||
);
|
||||
// Delete all page keys
|
||||
await storeCalls["store.deletePrefix"](ctx, pageKey(page, prefix));
|
||||
// console.log("Deleting keys", keysToDelete);
|
||||
await storeCalls["store.batchDelete"](ctx, keysToDelete);
|
||||
},
|
||||
"index.clearPageIndex": async (ctx) => {
|
||||
await storeCalls["store.deleteAll"](ctx);
|
||||
},
|
||||
};
|
||||
return apiObj;
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
import { PageMeta } from "../../common/types";
|
||||
import { SysCallMapping } from "../../plugos/system";
|
||||
import { Space } from "../../common/spaces/space";
|
||||
|
||||
export default (space: Space): SysCallMapping => {
|
||||
return {
|
||||
"space.listPages": async (ctx): Promise<PageMeta[]> => {
|
||||
return [...space.listPages()];
|
||||
},
|
||||
"space.readPage": async (
|
||||
ctx,
|
||||
name: string
|
||||
): Promise<{ text: string; meta: PageMeta }> => {
|
||||
return space.readPage(name);
|
||||
},
|
||||
"space.writePage": async (
|
||||
ctx,
|
||||
name: string,
|
||||
text: string
|
||||
): Promise<PageMeta> => {
|
||||
return space.writePage(name, text);
|
||||
},
|
||||
"space.deletePage": async (ctx, name: string) => {
|
||||
return space.deletePage(name);
|
||||
},
|
||||
};
|
||||
};
|
||||
@@ -1,5 +0,0 @@
|
||||
export function safeRun(fn: () => Promise<void>) {
|
||||
fn().catch((e) => {
|
||||
console.error(e);
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user