neil/silverbullet
1
0
Fork 0
Code Activity

Sync engine (#298)

Browse Source 
Fixes #261
This commit is contained in:
Zef Hemel
2023-01-13 15:41:29 +01:00
committed by GitHub
parent de6f531e91
commit a56e14bff1
51 changed files with 1033 additions and 1418 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/hooks/page_namespace.ts
-97
View File
@@ -1,97 +0,0 @@
import { Plug } from "../../plugos/plug.ts";
import { System } from "../../plugos/system.ts";
import { Hook, Manifest } from "../../plugos/types.ts";
export type NamespaceOperation =
| "readFile"
| "writeFile"
| "listFiles"
| "getFileMeta"
| "deleteFile";
export type PageNamespaceDef = {
pattern: string;
operation: NamespaceOperation;
};
export type PageNamespaceHookT = {
pageNamespace?: PageNamespaceDef;
};
type SpaceFunction = {
operation: NamespaceOperation;
pattern: RegExp;
plug: Plug<PageNamespaceHookT>;
name: string;
env?: string;
};
export class PageNamespaceHook implements Hook<PageNamespaceHookT> {
spaceFunctions: SpaceFunction[] = [];
constructor() {}
apply(system: System<PageNamespaceHookT>): void {
system.on({
plugLoaded: () => {
this.updateCache(system);
},
plugUnloaded: () => {
this.updateCache(system);
},
});
}
updateCache(system: System<PageNamespaceHookT>) {
this.spaceFunctions = [];
for (const plug of system.loadedPlugs.values()) {
if (plug.manifest?.functions) {
for (
const [funcName, funcDef] of Object.entries(
plug.manifest.functions,
)
) {
if (funcDef.pageNamespace) {
this.spaceFunctions.push({
operation: funcDef.pageNamespace.operation,
pattern: new RegExp(funcDef.pageNamespace.pattern),
plug,
name: funcName,
env: funcDef.env,
});
}
}
}
}
}
validateManifest(manifest: Manifest<PageNamespaceHookT>): string[] {
const errors: string[] = [];
if (!manifest.functions) {
return [];
}
for (let [funcName, funcDef] of Object.entries(manifest.functions)) {
if (funcDef.pageNamespace) {
if (!funcDef.pageNamespace.pattern) {
errors.push(`Function ${funcName} has a namespace but no pattern`);
}
if (!funcDef.pageNamespace.operation) {
errors.push(`Function ${funcName} has a namespace but no operation`);
}
if (
![
"readFile",
"writeFile",
"getFileMeta",
"listFiles",
"deleteFile",
].includes(funcDef.pageNamespace.operation)
) {
errors.push(
`Function ${funcName} has an invalid operation ${funcDef.pageNamespace.operation}`,
);
}
}
}
return errors;
}
}
server/hooks/plug_space_primitives.ts
-128
View File
@@ -1,128 +0,0 @@
import { Plug } from "../../plugos/plug.ts";
import {
FileData,
FileEncoding,
SpacePrimitives,
} from "../../common/spaces/space_primitives.ts";
import { FileMeta } from "../../common/types.ts";
import { NamespaceOperation, PageNamespaceHook } from "./page_namespace.ts";
import { base64DecodeDataUrl } from "../../plugos/asset_bundle/base64.ts";
export class PlugSpacePrimitives implements SpacePrimitives {
constructor(
private wrapped: SpacePrimitives,
private hook: PageNamespaceHook,
private env?: string,
) {}
performOperation(
type: NamespaceOperation,
pageName: string,
...args: any[]
): Promise<any> | false {
for (
const { operation, pattern, plug, name, env } of this.hook.spaceFunctions
) {
if (
operation === type && pageName.match(pattern) &&
(!this.env || (env && env === this.env))
) {
return plug.invoke(name, [pageName, ...args]);
}
}
return false;
}
async fetchFileList(): Promise<FileMeta[]> {
const allFiles: FileMeta[] = [];
for (const { plug, name, operation } of this.hook.spaceFunctions) {
if (operation === "listFiles") {
try {
for (const pm of await plug.invoke(name, [])) {
allFiles.push(pm);
}
} catch (e) {
console.error("Error listing files", e);
}
}
}
const result = await this.wrapped.fetchFileList();
for (const pm of result) {
allFiles.push(pm);
}
return allFiles;
}
async readFile(
name: string,
encoding: FileEncoding,
): Promise<{ data: FileData; meta: FileMeta }> {
const wantArrayBuffer = encoding === "arraybuffer";
const result: { data: FileData; meta: FileMeta } | false = await this
.performOperation(
"readFile",
name,
wantArrayBuffer ? "dataurl" : encoding,
);
if (result) {
if (wantArrayBuffer) {
return {
data: base64DecodeDataUrl(result.data as string),
meta: result.meta,
};
} else {
return result;
}
}
return this.wrapped.readFile(name, encoding);
}
getFileMeta(name: string): Promise<FileMeta> {
const result = this.performOperation("getFileMeta", name);
if (result) {
return result;
}
return this.wrapped.getFileMeta(name);
}
writeFile(
name: string,
encoding: FileEncoding,
data: FileData,
selfUpdate?: boolean,
): Promise<FileMeta> {
const result = this.performOperation(
"writeFile",
name,
encoding,
data,
selfUpdate,
);
if (result) {
return result;
}
return this.wrapped.writeFile(name, encoding, data, selfUpdate);
}
deleteFile(name: string): Promise<void> {
const result = this.performOperation("deleteFile", name);
if (result) {
return result;
}
return this.wrapped.deleteFile(name);
}
proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
return this.wrapped.proxySyscall(plug, name, args);
}
invokeFunction(
plug: Plug<any>,
env: string,
name: string,
args: any[],
): Promise<any> {
return this.wrapped.invokeFunction(plug, env, name, args);
}
}
server/http_server.ts
+18 -5
View File
@@ -5,6 +5,7 @@ import { EndpointHook } from "../plugos/hooks/endpoint.ts";
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
import { SpaceSystem } from "./space_system.ts";
import { ensureAndLoadSettings } from "../common/util.ts";
import { base64Decode } from "../plugos/asset_bundle/base64.ts";
export type ServerOptions = {
hostname: string;
@@ -14,6 +15,7 @@ export type ServerOptions = {
assetBundle: AssetBundle;
user?: string;
pass?: string;
bareMode?: boolean;
};
const staticLastModified = new Date().toUTCString();
@@ -26,6 +28,7 @@ export class HttpServer {
user?: string;
settings: { [key: string]: any } = {};
abortController?: AbortController;
bareMode: boolean;
constructor(options: ServerOptions) {
this.hostname = options.hostname;
@@ -37,6 +40,7 @@ export class HttpServer {
options.pagesPath,
options.dbPath,
);
this.bareMode = options.bareMode || false;
// Second, for loading plug JSON files with absolute or relative (from CWD) paths
this.systemBoot.eventHook.addLocalListener(
@@ -66,7 +70,7 @@ export class HttpServer {
async start() {
await this.systemBoot.start();
await this.systemBoot.ensureSpaceIndex();
await ensureAndLoadSettings(this.systemBoot.space);
await ensureAndLoadSettings(this.systemBoot.space, this.bareMode);
this.addPasswordAuth(this.app);
@@ -207,7 +211,8 @@ export class HttpServer {
// File list
fsRouter.get("/", async ({ response }) => {
response.headers.set("Content-type", "application/json");
response.body = JSON.stringify(await spacePrimitives.fetchFileList());
const files = await spacePrimitives.fetchFileList();
response.body = JSON.stringify(files);
});
fsRouter
@@ -248,12 +253,21 @@ export class HttpServer {
const name = params[0];
console.log("Saving file", name);
let body: Uint8Array;
if (
request.headers.get("X-Content-Base64")
) {
const content = await request.body({ type: "text" }).value;
body = base64Decode(content);
} else {
body = await request.body({ type: "bytes" }).value;
}
try {
const meta = await spacePrimitives.writeFile(
name,
"arraybuffer",
await request.body().value,
false,
body,
);
response.status = 200;
response.headers.set("Content-Type", meta.contentType);
@@ -299,7 +313,6 @@ export class HttpServer {
private buildPlugRouter(): Router {
const plugRouter = new Router();
// this.addPasswordAuth(plugRouter);
const system = this.systemBoot.system;
plugRouter.post(
server/space_system.ts
+6 -4
View File
@@ -23,13 +23,13 @@ import {
storeSyscalls,
} from "../plugos/syscalls/store.sqlite.ts";
import { System } from "../plugos/system.ts";
import { PageNamespaceHook } from "./hooks/page_namespace.ts";
import { PlugSpacePrimitives } from "./hooks/plug_space_primitives.ts";
import { PageNamespaceHook } from "../common/hooks/page_namespace.ts";
import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts";
import {
ensureTable as ensureIndexTable,
pageIndexSyscalls,
} from "./syscalls/index.ts";
import spaceSyscalls from "./syscalls/space.ts";
import spaceSyscalls from "../common/syscalls/space.ts";
import { systemSyscalls } from "./syscalls/system.ts";
import { AssetBundlePlugSpacePrimitives } from "../common/spaces/asset_bundle_space_primitives.ts";
import assetSyscalls from "../plugos/syscalls/asset.ts";
@@ -37,6 +37,7 @@ import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
import { AsyncSQLite } from "../plugos/sqlite/async_sqlite.ts";
import { FileMetaSpacePrimitives } from "../common/spaces/file_meta_space_primitives.ts";
import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts";
import { syncSyscalls } from "../common/syscalls/sync.ts";
export const indexRequiredKey = "$spaceIndexed";
// A composition of a PlugOS system attached to a Space for server-side use
@@ -111,6 +112,7 @@ export class SpaceSystem {
storeSyscalls(this.db, "store"),
fullTextSearchSyscalls(this.db, "fts"),
spaceSyscalls(this.space),
syncSyscalls(this.spacePrimitives),
eventSyscalls(this.eventHook),
markdownSyscalls(buildMarkdown([])),
esbuildSyscalls([globalModules]),
@@ -145,7 +147,7 @@ export class SpaceSystem {
console.log("Going to load", allPlugs.length, "plugs...");
await Promise.all(allPlugs.map(async (plugName) => {
const { data } = await this.space.readAttachment(plugName, "string");
const { data } = await this.space.readAttachment(plugName, "utf8");
await this.system.load(JSON.parse(data as string), createSandbox);
}));
server/syscalls/space.ts
-63
View File
@@ -1,63 +0,0 @@
import { AttachmentMeta, PageMeta } from "../../common/types.ts";
import { SysCallMapping } from "../../plugos/system.ts";
import { Space } from "../../common/spaces/space.ts";
import {
FileData,
FileEncoding,
} from "../../common/spaces/space_primitives.ts";
export default (space: Space): SysCallMapping => {
return {
"space.listPages": (): PageMeta[] => {
return [...space.listPages()];
},
"space.readPage": async (
_ctx,
name: string,
): Promise<string> => {
return (await space.readPage(name)).text;
},
"space.getPageMeta": (_ctx, name: string): Promise<PageMeta> => {
return space.getPageMeta(name);
},
"space.writePage": (
_ctx,
name: string,
text: string,
): Promise<PageMeta> => {
return space.writePage(name, text);
},
"space.deletePage": (_ctx, name: string) => {
return space.deletePage(name);
},
"space.listPlugs": (): Promise<string[]> => {
return space.listPlugs();
},
"space.listAttachments": async (): Promise<AttachmentMeta[]> => {
return await space.fetchAttachmentList();
},
"space.readAttachment": async (
_ctx,
name: string,
): Promise<FileData> => {
return (await space.readAttachment(name, "dataurl")).data;
},
"space.getAttachmentMeta": async (
_ctx,
name: string,
): Promise<AttachmentMeta> => {
return await space.getAttachmentMeta(name);
},
"space.writeAttachment": async (
_ctx,
name: string,
encoding: FileEncoding,
data: string,
): Promise<AttachmentMeta> => {
return await space.writeAttachment(name, encoding, data);
},
"space.deleteAttachment": async (_ctx, name: string) => {
await space.deleteAttachment(name);
},
};
};