1
0

* Refactored server to use spaces

* Other cleanup
This commit is contained in:
Zef Hemel 2022-04-08 17:46:09 +02:00
parent e10f41031c
commit 6ebf8e7f15
24 changed files with 256 additions and 160 deletions

View File

@ -1,63 +1,10 @@
import { mkdir, readdir, readFile, stat, unlink, utimes, writeFile } from "fs/promises";
import * as path from "path";
import { PageMeta } from "../common/types";
import { EventHook } from "../plugos/hooks/event";
import { PageMeta } from "../types";
import { SpacePrimitives } from "./space_primitives";
import { Plug } from "../../plugos/plug";
export interface Storage {
listPages(): Promise<PageMeta[]>;
readPage(pageName: string): Promise<{ text: string; meta: PageMeta }>;
writePage(
pageName: string,
text: string,
lastModified?: number
): Promise<PageMeta>;
getPageMeta(pageName: string): Promise<PageMeta>;
deletePage(pageName: string): Promise<void>;
}
export class EventedStorage implements Storage {
constructor(private wrapped: Storage, private eventHook: EventHook) {}
listPages(): Promise<PageMeta[]> {
return this.wrapped.listPages();
}
readPage(pageName: string): Promise<{ text: string; meta: PageMeta }> {
return this.wrapped.readPage(pageName);
}
async writePage(
pageName: string,
text: string,
lastModified?: number
): Promise<PageMeta> {
const newPageMeta = this.wrapped.writePage(pageName, text, lastModified);
// This can happen async
this.eventHook
.dispatchEvent("page:saved", pageName)
.then(() => {
return this.eventHook.dispatchEvent("page:index", {
name: pageName,
text: text,
});
})
.catch((e) => {
console.error("Error dispatching page:saved event", e);
});
return newPageMeta;
}
getPageMeta(pageName: string): Promise<PageMeta> {
return this.wrapped.getPageMeta(pageName);
}
async deletePage(pageName: string): Promise<void> {
await this.eventHook.dispatchEvent("page:deleted", pageName);
return this.wrapped.deletePage(pageName);
}
}
export class DiskStorage implements Storage {
export class DiskSpacePrimitives implements SpacePrimitives {
rootPath: string;
plugPrefix: string;
@ -83,31 +30,6 @@ export class DiskStorage implements Storage {
);
}
async listPages(): Promise<PageMeta[]> {
let fileNames: PageMeta[] = [];
const walkPath = async (dir: string) => {
let files = await readdir(dir);
for (let file of files) {
const fullPath = path.join(dir, file);
let s = await stat(fullPath);
// console.log("Encountering", fullPath, s);
if (s.isDirectory()) {
await walkPath(fullPath);
} else {
if (file.endsWith(".md") || file.endsWith(".json")) {
fileNames.push({
name: this.pathToPageName(fullPath),
lastModified: s.mtime.getTime(),
});
}
}
}
};
await walkPath(this.rootPath);
return fileNames;
}
async readPage(pageName: string): Promise<{ text: string; meta: PageMeta }> {
const localPath = this.pageNameToPath(pageName);
try {
@ -128,6 +50,7 @@ export class DiskStorage implements Storage {
async writePage(
pageName: string,
text: string,
selfUpdate: boolean,
lastModified?: number
): Promise<PageMeta> {
let localPath = this.pageNameToPath(pageName);
@ -173,4 +96,47 @@ export class DiskStorage implements Storage {
let localPath = this.pageNameToPath(pageName);
await unlink(localPath);
}
async fetchPageList(): Promise<{
pages: Set<PageMeta>;
nowTimestamp: number;
}> {
let pages = new Set<PageMeta>();
const walkPath = async (dir: string) => {
let files = await readdir(dir);
for (let file of files) {
const fullPath = path.join(dir, file);
let s = await stat(fullPath);
if (s.isDirectory()) {
await walkPath(fullPath);
} else {
if (file.endsWith(".md") || file.endsWith(".json")) {
pages.add({
name: this.pathToPageName(fullPath),
lastModified: s.mtime.getTime(),
});
}
}
}
};
await walkPath(this.rootPath);
return {
pages: pages,
nowTimestamp: Date.now(),
};
}
invokeFunction(
plug: Plug<any>,
env: string,
name: string,
args: any[]
): Promise<any> {
return plug.invoke(name, args);
}
proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
return plug.syscall(name, args);
}
}

View File

@ -0,0 +1,65 @@
import { SpacePrimitives } from "./space_primitives";
import { EventHook } from "../../plugos/hooks/event";
import { PageMeta } from "../types";
import { Plug } from "../../plugos/plug";
export class EventedSpacePrimitives implements SpacePrimitives {
constructor(private wrapped: SpacePrimitives, private eventHook: EventHook) {}
fetchPageList(): Promise<{ pages: Set<PageMeta>; nowTimestamp: number }> {
return this.wrapped.fetchPageList();
}
proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
return this.wrapped.proxySyscall(plug, name, args);
}
invokeFunction(
plug: Plug<any>,
env: string,
name: string,
args: any[]
): Promise<any> {
return this.wrapped.invokeFunction(plug, env, name, args);
}
readPage(pageName: string): Promise<{ text: string; meta: PageMeta }> {
return this.wrapped.readPage(pageName);
}
async writePage(
pageName: string,
text: string,
selfUpdate: boolean,
lastModified?: number
): Promise<PageMeta> {
const newPageMeta = await this.wrapped.writePage(
pageName,
text,
selfUpdate,
lastModified
);
// This can happen async
this.eventHook
.dispatchEvent("page:saved", pageName)
.then(() => {
return this.eventHook.dispatchEvent("page:index", {
name: pageName,
text: text,
});
})
.catch((e) => {
console.error("Error dispatching page:saved event", e);
});
return newPageMeta;
}
getPageMeta(pageName: string): Promise<PageMeta> {
return this.wrapped.getPageMeta(pageName);
}
async deletePage(pageName: string): Promise<void> {
await this.eventHook.dispatchEvent("page:deleted", pageName);
return this.wrapped.deletePage(pageName);
}
}

View File

@ -1,4 +1,4 @@
import { PageMeta } from "../../common/types";
import { PageMeta } from "../types";
import { Plug } from "../../plugos/plug";
import { SpacePrimitives } from "./space_primitives";

View File

@ -1,5 +1,5 @@
import { SpacePrimitives } from "./space_primitives";
import { PageMeta } from "../../common/types";
import { PageMeta } from "../types";
import Dexie, { Table } from "dexie";
import { Plug } from "../../plugos/plug";

View File

@ -1,9 +1,9 @@
import { SpacePrimitives } from "./space_primitives";
import { safeRun } from "../util";
import { PageMeta } from "../../common/types";
import { EventEmitter } from "../../common/event";
import { safeRun } from "../../webapp/util";
import { PageMeta } from "../types";
import { EventEmitter } from "../event";
import { Plug } from "../../plugos/plug";
import { Manifest } from "../../common/manifest";
import { Manifest } from "../manifest";
const pageWatchInterval = 2000;
const trashPrefix = "_trash/";
@ -46,7 +46,6 @@ export class Space extends EventEmitter<SpaceEvents> {
pageMeta.name.substring(plugPrefix.length),
JSON.parse(pageData.text)
);
this.watchPage(pageMeta.name);
}
},
});
@ -104,10 +103,8 @@ export class Space extends EventEmitter<SpaceEvents> {
this.watchedPages.delete(pageName);
continue;
}
const newMeta = await this.space.getPageMeta(pageName);
if (oldMeta.lastModified !== newMeta.lastModified) {
this.emit("pageChanged", newMeta);
}
// This seems weird, but simply fetching it will compare to local cache and trigger an event if necessary
await this.getPageMeta(pageName);
}
});
}, pageWatchInterval);
@ -134,7 +131,15 @@ export class Space extends EventEmitter<SpaceEvents> {
}
async getPageMeta(name: string): Promise<PageMeta> {
return this.metaCacher(name, await this.space.getPageMeta(name));
let oldMeta = this.pageMetaCache.get(name);
let newMeta = await this.space.getPageMeta(name);
if (oldMeta) {
if (oldMeta.lastModified !== newMeta.lastModified) {
// Changed on disk, trigger event
this.emit("pageChanged", newMeta);
}
}
return this.metaCacher(name, newMeta);
}
invokeFunction(
@ -185,6 +190,13 @@ export class Space extends EventEmitter<SpaceEvents> {
async readPage(name: string): Promise<{ text: string; meta: PageMeta }> {
let pageData = await this.space.readPage(name);
let previousMeta = this.pageMetaCache.get(name);
if (previousMeta) {
if (previousMeta.lastModified !== pageData.meta.lastModified) {
// Page changed since last cached metadata, trigger event
this.emit("pageChanged", pageData.meta);
}
}
this.pageMetaCache.set(name, pageData.meta);
return pageData;
}

View File

@ -1,5 +1,5 @@
import { Plug } from "../../plugos/plug";
import { PageMeta } from "../../common/types";
import { PageMeta } from "../types";
export interface SpacePrimitives {
// Pages

View File

@ -1,7 +1,7 @@
import { expect, test } from "@jest/globals";
import { IndexedDBSpacePrimitives } from "./indexeddb_space_primitives";
import { SpaceSync } from "./sync";
import { PageMeta } from "../../common/types";
import { PageMeta } from "../types";
import { Space } from "./space";
// For testing in node.js

View File

@ -1,5 +1,5 @@
import { Space } from "./space";
import { PageMeta } from "../../common/types";
import { PageMeta } from "../types";
import { SpacePrimitives } from "./space_primitives";
export class SpaceSync {

View File

@ -35,7 +35,7 @@
"context": "node"
},
"test": {
"source": ["plugs/lib/tree.test.ts", "webapp/spaces/sync.test.ts"],
"source": ["plugs/lib/tree.test.ts", "common/spaces/sync.test.ts"],
"outputFormat": "commonjs",
"isLibrary": true,
"context": "node"

View File

@ -88,7 +88,6 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
sandboxFactory: SandboxFactory<HookT>
): Promise<Plug<HookT>> {
if (this.plugs.has(name)) {
console.log("Unloading", name);
await this.unload(name);
}
// Validate

View File

@ -43,6 +43,7 @@ export async function updateMaterializedQueriesCommand() {
// Called from client, running on server
export async function updateMaterializedQueriesOnPage(pageName: string) {
let { text } = await readPage(pageName);
text = await replaceAsync(text, queryRegex, async (match, ...args) => {
let { table, filter, groupBy, limit, orderBy, orderDesc } =
args[args.length - 1];

View File

@ -1,34 +1,48 @@
import { IndexEvent } from "../../webapp/app_event";
import { pageLinkRegex } from "../../webapp/constant";
import {
batchSet,
clearPageIndex as clearPageIndexSyscall,
clearPageIndexForPage,
scanPrefixGlobal
} from "plugos-silverbullet-syscall/index";
import { flashNotification, getCurrentPage, getText, matchBefore, navigate } from "plugos-silverbullet-syscall/editor";
import {
flashNotification,
getCurrentPage,
getText,
matchBefore,
navigate,
prompt
} from "plugos-silverbullet-syscall/editor";
import { dispatch } from "plugos-syscall/event";
import { deletePage as deletePageSyscall, listPages, readPage, writePage } from "plugos-silverbullet-syscall/space";
import { invokeFunction } from "plugos-silverbullet-syscall/system";
const wikilinkRegex = new RegExp(pageLinkRegex, "g");
import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
import {
addParentPointers,
collectNodesMatching,
MarkdownTree,
renderMarkdown,
replaceNodesMatching
} from "../lib/tree";
export async function indexLinks({ name, text }: IndexEvent) {
let backLinks: { key: string; value: string }[] = [];
// [[Style Links]]
console.log("Now indexing", name);
for (let match of text.matchAll(wikilinkRegex)) {
let toPage = match[1];
if (toPage.includes("@")) {
toPage = toPage.split("@")[0];
let mdTree = await parseMarkdown(text);
collectNodesMatching(mdTree, (n) => n.type === "WikiLinkPage").forEach(
(n) => {
let toPage = n.children![0].text!;
if (toPage.includes("@")) {
toPage = toPage.split("@")[0];
}
backLinks.push({
key: `pl:${toPage}:${n.from}`,
value: name,
});
}
let pos = match.index!;
backLinks.push({
key: `pl:${toPage}:${pos}`,
value: name,
});
}
);
console.log("Found", backLinks.length, "wiki link(s)");
await batchSet(name, backLinks);
}
@ -69,12 +83,31 @@ export async function renamePage() {
for (let pageToUpdate of pageToUpdateSet) {
console.log("Now going to update links in", pageToUpdate);
let { text } = await readPage(pageToUpdate);
console.log("Received text", text);
// console.log("Received text", text);
if (!text) {
// Page likely does not exist, but at least we can skip it
continue;
}
let newText = text.replaceAll(`[[${oldName}]]`, `[[${newName}]]`);
let mdTree = await parseMarkdown(text);
addParentPointers(mdTree);
replaceNodesMatching(mdTree, (n): MarkdownTree | undefined | null => {
if (n.type === "WikiLinkPage") {
let pageName = n.children![0].text!;
if (pageName === oldName) {
n.children![0].text = newName;
return n;
}
// page name with @pos position
if (pageName.startsWith(`${oldName}@`)) {
let [, pos] = pageName.split("@");
n.children![0].text = `${newName}@${pos}`;
return n;
}
}
return;
});
// let newText = text.replaceAll(`[[${oldName}]]`, `[[${newName}]]`);
let newText = renderMarkdown(mdTree);
if (text !== newText) {
console.log("Changes made, saving...");
await writePage(pageToUpdate, newText);

View File

@ -57,6 +57,7 @@ export function collectNodesMatching(
return results;
}
// return value: returning undefined = not matched, continue, null = delete, new node = replace
export function replaceNodesMatching(
mdTree: MarkdownTree,
substituteFn: (mdTree: MarkdownTree) => MarkdownTree | null | undefined

View File

@ -48,7 +48,7 @@ export async function updateMarkdownPreview() {
}
});
let html = md.render(renderMarkdown(mdTree));
await showRhs(`<html><body>${html}</body></html>`, 1);
await showRhs(`<html><body>${html}</body></html>`, 2);
}
async function hideMarkdownPreview() {

View File

@ -4,7 +4,7 @@ import { EndpointHook } from "../plugos/hooks/endpoint";
import { readFile } from "fs/promises";
import { System } from "../plugos/system";
import cors from "cors";
import { DiskStorage, EventedStorage, Storage } from "./disk_storage";
import { DiskSpacePrimitives } from "../common/spaces/disk_space_primitives";
import path from "path";
import bodyParser from "body-parser";
import { EventHook } from "../plugos/hooks/event";
@ -15,12 +15,16 @@ import knex, { Knex } from "knex";
import shellSyscalls from "../plugos/syscalls/shell.node";
import { NodeCronHook } from "../plugos/hooks/node_cron";
import { markdownSyscalls } from "../common/syscalls/markdown";
import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives";
import { Space } from "../common/spaces/space";
import { safeRun } from "../webapp/util";
import { createSandbox } from "../plugos/environments/node_sandbox";
export class ExpressServer {
app: Express;
system: System<SilverBulletHooks>;
private rootPath: string;
private storage: Storage;
private space: Space;
private distDir: string;
private eventHook: EventHook;
private db: Knex<any, unknown[]>;
@ -39,9 +43,12 @@ export class ExpressServer {
// Setup system
this.eventHook = new EventHook();
system.addHook(this.eventHook);
this.storage = new EventedStorage(
new DiskStorage(rootPath),
this.eventHook
this.space = new Space(
new EventedSpacePrimitives(
new DiskSpacePrimitives(rootPath),
this.eventHook
),
true
);
this.db = knex({
client: "better-sqlite3",
@ -55,10 +62,30 @@ export class ExpressServer {
system.addHook(new NodeCronHook());
system.registerSyscalls([], pageIndexSyscalls(this.db));
system.registerSyscalls([], spaceSyscalls(this.storage));
system.registerSyscalls([], spaceSyscalls(this.space));
system.registerSyscalls([], eventSyscalls(this.eventHook));
system.registerSyscalls([], markdownSyscalls());
system.addHook(new EndpointHook(app, "/_/"));
this.space.on({
plugLoaded: (plugName, plug) => {
safeRun(async () => {
console.log("Plug load", plugName);
await system.load(plugName, plug, createSandbox);
});
},
plugUnloaded: (plugName) => {
safeRun(async () => {
console.log("Plug unload", plugName);
await system.unload(plugName);
});
},
});
setInterval(() => {
this.space.updatePageListAsync();
}, 5000);
this.space.updatePageListAsync();
}
async init() {
@ -68,8 +95,9 @@ export class ExpressServer {
// Page list
fsRouter.route("/").get(async (req, res) => {
res.header("Now-Timestamp", "" + Date.now());
res.json(await this.storage.listPages());
let { nowTimestamp, pages } = await this.space.fetchPageList();
res.header("Now-Timestamp", "" + nowTimestamp);
res.json([...pages]);
});
fsRouter.route("/").post(bodyParser.json(), async (req, res) => {});
@ -80,7 +108,7 @@ export class ExpressServer {
let pageName = req.params[0];
// console.log("Getting", pageName);
try {
let pageData = await this.storage.readPage(pageName);
let pageData = await this.space.readPage(pageName);
res.status(200);
res.header("Last-Modified", "" + pageData.meta.lastModified);
res.header("Content-Type", "text/markdown");
@ -97,9 +125,10 @@ export class ExpressServer {
console.log("Saving", pageName);
try {
let meta = await this.storage.writePage(
let meta = await this.space.writePage(
pageName,
req.body,
false,
req.header("Last-Modified")
? +req.header("Last-Modified")!
: undefined
@ -116,7 +145,7 @@ export class ExpressServer {
.options(async (req, res) => {
let pageName = req.params[0];
try {
const meta = await this.storage.getPageMeta(pageName);
const meta = await this.space.getPageMeta(pageName);
res.status(200);
res.header("Last-Modified", "" + meta.lastModified);
res.header("Content-Type", "text/markdown");
@ -131,7 +160,7 @@ export class ExpressServer {
.delete(async (req, res) => {
let pageName = req.params[0];
try {
await this.storage.deletePage(pageName);
await this.space.deletePage(pageName);
res.status(200);
res.send("OK");
} catch (e) {

View File

@ -6,7 +6,6 @@ import yargs from "yargs";
import { hideBin } from "yargs/helpers";
import { SilverBulletHooks } from "../common/manifest";
import { ExpressServer } from "./api_server";
import { DiskPlugLoader } from "../plugos/plug_loader";
import { System } from "../plugos/system";
let args = yargs(hideBin(process.argv))
@ -36,12 +35,6 @@ const expressServer = new ExpressServer(app, pagesPath, distDir, system);
expressServer
.init()
.then(async () => {
let plugLoader = new DiskPlugLoader(
system,
`${__dirname}/../../plugs/dist`
);
await plugLoader.loadPlugs();
plugLoader.watcher();
server.listen(port, () => {
console.log(`Server listening on port ${port}`);
});

View File

@ -1,27 +1,27 @@
import { PageMeta } from "../../common/types";
import { SysCallMapping } from "../../plugos/system";
import { Storage } from "../disk_storage";
import { Space } from "../../common/spaces/space";
export default (storage: Storage): SysCallMapping => {
export default (space: Space): SysCallMapping => {
return {
"space.listPages": (ctx): Promise<PageMeta[]> => {
return storage.listPages();
"space.listPages": async (ctx): Promise<PageMeta[]> => {
return [...space.listPages()];
},
"space.readPage": async (
ctx,
name: string
): Promise<{ text: string; meta: PageMeta }> => {
return storage.readPage(name);
return space.readPage(name);
},
"space.writePage": async (
ctx,
name: string,
text: string
): Promise<PageMeta> => {
return storage.writePage(name, text);
return space.writePage(name, text);
},
"space.deletePage": async (ctx, name: string) => {
return storage.deletePage(name);
return space.deletePage(name);
},
};
};

View File

@ -1,9 +1,9 @@
import { Editor } from "./editor";
import { safeRun } from "./util";
import { Space } from "./spaces/space";
import { HttpSpacePrimitives } from "./spaces/http_space_primitives";
import { IndexedDBSpacePrimitives } from "./spaces/indexeddb_space_primitives";
import { SpaceSync } from "./spaces/sync";
import { Space } from "../common/spaces/space";
import { HttpSpacePrimitives } from "../common/spaces/http_space_primitives";
import { IndexedDBSpacePrimitives } from "../common/spaces/indexeddb_space_primitives";
import { SpaceSync } from "../common/spaces/sync";
let localSpace = new Space(new IndexedDBSpacePrimitives("pages"), true);
localSpace.watch();

View File

@ -1 +0,0 @@
export const pageLinkRegex = /\[\[([\w\s\/:,\.@\-]+)\]\]/;

View File

@ -29,7 +29,7 @@ import { PathPageNavigator } from "./navigator";
import customMarkDown from "./parser";
import reducer from "./reducer";
import { smartQuoteKeymap } from "./smart_quotes";
import { Space } from "./spaces/space";
import { Space } from "../common/spaces/space";
import customMarkdownStyle from "./style";
import { editorSyscalls } from "./syscalls/editor";
import { indexerSyscalls } from "./syscalls";
@ -429,9 +429,7 @@ export class Editor implements AppEventDispatcher {
let pageState = this.openPages.get(this.currentPage);
if (pageState) {
pageState.selection = this.editorView!.state.selection;
pageState.scrollTop =
this.editorView!.scrollDOM.parentElement!.parentElement!.scrollTop;
// pageState.scrollTop = this.editorView!.scrollDOM.scrollTop;
pageState.scrollTop = this.editorView!.scrollDOM.scrollTop;
// console.log("Saved pageState", this.currentPage, pageState);
}
this.space.unwatchPage(this.currentPage);
@ -466,8 +464,7 @@ export class Editor implements AppEventDispatcher {
editorView.dispatch({
selection: pageState.selection,
});
editorView.scrollDOM.parentElement!.parentElement!.scrollTop =
pageState!.scrollTop;
editorView.scrollDOM.scrollTop = pageState!.scrollTop;
}
this.space.watchPage(pageName);

View File

@ -2,11 +2,12 @@ import { styleTags, tags as t } from "@codemirror/highlight";
import { BlockContext, LeafBlock, LeafBlockParser, MarkdownConfig, TaskList } from "@lezer/markdown";
import { commonmark, mkLang } from "./markdown/markdown";
import * as ct from "./customtags";
import { pageLinkRegex } from "./constant";
const pageLinkRegexPrefix = new RegExp(
"^" + pageLinkRegex.toString().slice(1, -1)
);
export const pageLinkRegexPrefix = /^\[\[([\w\s\/:,\.@\-]+)\]\]/;
// const pageLinkRegexPrefix = new RegExp(
// "^" + pageLinkRegex.toString().slice(1, -1)
// );
const WikiLink: MarkdownConfig = {
defineNodes: ["WikiLink", "WikiLinkPage"],

View File

@ -6,7 +6,7 @@
.cm-editor {
font-size: var(--ident);
overflow-y: hidden;
//overflow-y: hidden;
.cm-content {
font-family: var(--editor-font);

View File

@ -1,6 +1,6 @@
import { SysCallMapping } from "../../plugos/system";
import { proxySyscalls } from "../../plugos/syscalls/transport";
import { Space } from "../spaces/space";
import { Space } from "../../common/spaces/space";
export function indexerSyscalls(space: Space): SysCallMapping {
return proxySyscalls(

View File

@ -1,5 +1,5 @@
import { SysCallMapping } from "../../plugos/system";
import { Space } from "../spaces/space";
import { Space } from "../../common/spaces/space";
export function systemSyscalls(space: Space): SysCallMapping {
return {