1
0
This commit is contained in:
Zef Hemel 2023-05-29 09:53:49 +02:00
parent 50651dc185
commit c6a45be4bb
10 changed files with 129 additions and 47 deletions

View File

@ -43,37 +43,33 @@ To allow outside connections, pass -L 0.0.0.0 as a flag, and put a TLS terminato
} }
let spacePrimitives: SpacePrimitives | undefined; let spacePrimitives: SpacePrimitives | undefined;
if (folder === "s3://") { if (folder === "s3://") {
spacePrimitives = new AssetBundlePlugSpacePrimitives( spacePrimitives = new S3SpacePrimitives({
new S3SpacePrimitives({
accessKey: Deno.env.get("AWS_ACCESS_KEY_ID")!, accessKey: Deno.env.get("AWS_ACCESS_KEY_ID")!,
secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!, secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!,
endPoint: Deno.env.get("AWS_ENDPOINT")!, endPoint: Deno.env.get("AWS_ENDPOINT")!,
region: Deno.env.get("AWS_REGION")!, region: Deno.env.get("AWS_REGION")!,
bucket: Deno.env.get("AWS_BUCKET")!, bucket: Deno.env.get("AWS_BUCKET")!,
}), });
new AssetBundle(plugAssetBundle as AssetJson),
);
console.log("Running in S3 mode"); console.log("Running in S3 mode");
} else { } else {
// Regular disk mode
folder = path.resolve(Deno.cwd(), folder); folder = path.resolve(Deno.cwd(), folder);
spacePrimitives = new DiskSpacePrimitives(folder);
}
spacePrimitives = new AssetBundlePlugSpacePrimitives( spacePrimitives = new AssetBundlePlugSpacePrimitives(
new DiskSpacePrimitives(folder, { spacePrimitives,
maxFileSizeMB: options.maxFileSizeMB,
}),
new AssetBundle(plugAssetBundle as AssetJson), new AssetBundle(plugAssetBundle as AssetJson),
); );
}
console.log("Serving pages from", folder);
const httpServer = new HttpServer(spacePrimitives, { const httpServer = new HttpServer(spacePrimitives!, {
hostname, hostname,
port: port, port: port,
pagesPath: folder, pagesPath: folder!,
clientAssetBundle: new AssetBundle(clientAssetBundle as AssetJson), clientAssetBundle: new AssetBundle(clientAssetBundle as AssetJson),
user: options.user ?? Deno.env.get("SB_USER"), user: options.user ?? Deno.env.get("SB_USER"),
keyFile: options.key, keyFile: options.key,
certFile: options.cert, certFile: options.cert,
maxFileSizeMB: +maxFileSizeMB, maxFileSizeMB: +maxFileSizeMB,
}); });
httpServer.start().catch(console.error); return httpServer.start();
} }

View File

@ -124,3 +124,5 @@ export {
} from "https://esm.sh/@codemirror/lang-javascript@6.1.8?external=@codemirror/language,@codemirror/autocomplete,@codemirror/view,@codemirror/state,@codemirror/lint,@lezer/common,@lezer/lr,@lezer/javascript,@codemirror/commands"; } from "https://esm.sh/@codemirror/lang-javascript@6.1.8?external=@codemirror/language,@codemirror/autocomplete,@codemirror/view,@codemirror/state,@codemirror/lint,@lezer/common,@lezer/lr,@lezer/javascript,@codemirror/commands";
export { mime } from "https://deno.land/x/mimetypes@v1.0.0/mod.ts"; export { mime } from "https://deno.land/x/mimetypes@v1.0.0/mod.ts";
export { compile as gitIgnoreCompiler } from "https://esm.sh/gitignore-parser@0.0.2";

View File

@ -16,14 +16,10 @@ function normalizeForwardSlashPath(path: string) {
const excludedFiles = ["data.db", "data.db-journal", "sync.json"]; const excludedFiles = ["data.db", "data.db-journal", "sync.json"];
export type DiskSpaceOptions = {
maxFileSizeMB?: number;
};
export class DiskSpacePrimitives implements SpacePrimitives { export class DiskSpacePrimitives implements SpacePrimitives {
rootPath: string; rootPath: string;
constructor(rootPath: string, private options: DiskSpaceOptions = {}) { constructor(rootPath: string) {
this.rootPath = Deno.realPathSync(rootPath); this.rootPath = Deno.realPathSync(rootPath);
} }
@ -150,13 +146,6 @@ export class DiskSpacePrimitives implements SpacePrimitives {
const fullPath = file.path; const fullPath = file.path;
try { try {
const s = await Deno.stat(fullPath); const s = await Deno.stat(fullPath);
// Don't list file exceeding the maximum file size
if (
this.options.maxFileSizeMB &&
s.size / (1024 * 1024) > this.options.maxFileSizeMB
) {
continue;
}
const name = fullPath.substring(this.rootPath.length + 1); const name = fullPath.substring(this.rootPath.length + 1);
if (excludedFiles.includes(name)) { if (excludedFiles.includes(name)) {
continue; continue;

View File

@ -0,0 +1,35 @@
import { FileMeta } from "../types.ts";
import { SpacePrimitives } from "./space_primitives.ts";
export class FilteredSpacePrimitives implements SpacePrimitives {
constructor(
private wrapped: SpacePrimitives,
private filterFn: (name: FileMeta) => boolean,
private onFetchList?: () => Promise<void>,
) {
}
async fetchFileList(): Promise<FileMeta[]> {
if (this.onFetchList) {
await this.onFetchList();
}
return (await this.wrapped.fetchFileList()).filter(this.filterFn);
}
readFile(name: string): Promise<{ data: Uint8Array; meta: FileMeta }> {
return this.wrapped.readFile(name);
}
getFileMeta(name: string): Promise<FileMeta> {
return this.wrapped.getFileMeta(name);
}
writeFile(
name: string,
data: Uint8Array,
selfUpdate?: boolean | undefined,
lastModified?: number | undefined,
): Promise<FileMeta> {
return this.wrapped.writeFile(name, data, selfUpdate, lastModified);
}
deleteFile(name: string): Promise<void> {
return this.wrapped.deleteFile(name);
}
}

View File

@ -36,14 +36,18 @@ export function parseYamlSettings(settingsMarkdown: string): {
export async function ensureSettingsAndIndex( export async function ensureSettingsAndIndex(
space: SpacePrimitives, space: SpacePrimitives,
): Promise<any> { ): Promise<any> {
let settingsText: string | undefined;
try { try {
await space.getFileMeta("SETTINGS.md"); settingsText = new TextDecoder().decode(
(await space.readFile("SETTINGS.md")).data,
);
} catch { } catch {
await space.writeFile( await space.writeFile(
"SETTINGS.md", "SETTINGS.md",
new TextEncoder().encode(SETTINGS_TEMPLATE), new TextEncoder().encode(SETTINGS_TEMPLATE),
true, true,
); );
settingsText = SETTINGS_TEMPLATE;
// Ok, then let's also write the index page // Ok, then let's also write the index page
try { try {
await space.getFileMeta("index.md"); await space.getFileMeta("index.md");
@ -60,4 +64,6 @@ Loading some onboarding content for you (but doing so does require a working int
); );
} }
} }
return parseYamlSettings(settingsText);
} }

View File

@ -4,6 +4,9 @@ import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
import { base64Decode } from "../plugos/asset_bundle/base64.ts"; import { base64Decode } from "../plugos/asset_bundle/base64.ts";
import { ensureSettingsAndIndex } from "../common/util.ts"; import { ensureSettingsAndIndex } from "../common/util.ts";
import { performLocalFetch } from "../common/proxy_fetch.ts"; import { performLocalFetch } from "../common/proxy_fetch.ts";
import { BuiltinSettings } from "../web/types.ts";
import { gitIgnoreCompiler } from "./deps.ts";
import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts";
export type ServerOptions = { export type ServerOptions = {
hostname: string; hostname: string;
@ -22,12 +25,13 @@ export class HttpServer {
private hostname: string; private hostname: string;
private port: number; private port: number;
user?: string; user?: string;
settings: { [key: string]: any } = {};
abortController?: AbortController; abortController?: AbortController;
clientAssetBundle: AssetBundle; clientAssetBundle: AssetBundle;
settings?: BuiltinSettings;
spacePrimitives: SpacePrimitives;
constructor( constructor(
private spacePrimitives: SpacePrimitives, spacePrimitives: SpacePrimitives,
private options: ServerOptions, private options: ServerOptions,
) { ) {
this.hostname = options.hostname; this.hostname = options.hostname;
@ -35,6 +39,29 @@ export class HttpServer {
this.app = new Application(); this.app = new Application();
this.user = options.user; this.user = options.user;
this.clientAssetBundle = options.clientAssetBundle; this.clientAssetBundle = options.clientAssetBundle;
let fileFilterFn: (s: string) => boolean = () => true;
this.spacePrimitives = new FilteredSpacePrimitives(
spacePrimitives,
(meta) => {
// Don't list file exceeding the maximum file size
if (
options.maxFileSizeMB &&
meta.size / (1024 * 1024) > options.maxFileSizeMB
) {
return false;
}
return fileFilterFn(meta.name);
},
async () => {
await this.reloadSettings();
if (typeof this.settings?.spaceIgnore === "string") {
fileFilterFn = gitIgnoreCompiler(this.settings.spaceIgnore).accepts;
} else {
fileFilterFn = () => true;
}
},
);
} }
// Replaces some template variables in index.html in a rather ad-hoc manner, but YOLO // Replaces some template variables in index.html in a rather ad-hoc manner, but YOLO
@ -50,8 +77,7 @@ export class HttpServer {
} }
async start() { async start() {
await ensureSettingsAndIndex(this.spacePrimitives); await this.reloadSettings();
// Serve static files (javascript, css, html) // Serve static files (javascript, css, html)
this.app.use(async ({ request, response }, next) => { this.app.use(async ({ request, response }, next) => {
if (request.url.pathname === "/") { if (request.url.pathname === "/") {
@ -137,6 +163,11 @@ export class HttpServer {
); );
} }
async reloadSettings() {
// TODO: Throttle this?
this.settings = await ensureSettingsAndIndex(this.spacePrimitives);
}
private addPasswordAuth(app: Application) { private addPasswordAuth(app: Application) {
const excludedPaths = [ const excludedPaths = [
"/manifest.json", "/manifest.json",

View File

@ -15,6 +15,7 @@ import {
EditorSelection, EditorSelection,
EditorState, EditorState,
EditorView, EditorView,
gitIgnoreCompiler,
highlightSpecialChars, highlightSpecialChars,
history, history,
historyKeymap, historyKeymap,
@ -134,6 +135,8 @@ import { SyncStatus } from "../common/spaces/sync.ts";
import { HttpSpacePrimitives } from "../common/spaces/http_space_primitives.ts"; import { HttpSpacePrimitives } from "../common/spaces/http_space_primitives.ts";
import { FallbackSpacePrimitives } from "../common/spaces/fallback_space_primitives.ts"; import { FallbackSpacePrimitives } from "../common/spaces/fallback_space_primitives.ts";
import { syncSyscalls } from "./syscalls/sync.ts"; import { syncSyscalls } from "./syscalls/sync.ts";
import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts";
import { globToRegExp } from "https://deno.land/std@0.189.0/path/glob.ts";
const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/; const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/;
@ -247,12 +250,24 @@ export class Editor {
namespaceHook, namespaceHook,
); );
const localSpacePrimitives = new FileMetaSpacePrimitives( let fileFilterFn: (s: string) => boolean = () => true;
const localSpacePrimitives = new FilteredSpacePrimitives(
new FileMetaSpacePrimitives(
new EventedSpacePrimitives( new EventedSpacePrimitives(
plugSpacePrimitives, plugSpacePrimitives,
this.eventHook, this.eventHook,
), ),
indexSyscalls, indexSyscalls,
),
(meta) => fileFilterFn(meta.name),
async () => {
await this.loadSettings();
if (typeof this.settings?.spaceIgnore === "string") {
fileFilterFn = gitIgnoreCompiler(this.settings.spaceIgnore).accepts;
} else {
fileFilterFn = () => true;
}
},
); );
this.space = new Space(localSpacePrimitives); this.space = new Space(localSpacePrimitives);
@ -458,6 +473,7 @@ export class Editor {
this.syncService.start(); this.syncService.start();
this.eventHook.addLocalListener("sync:success", async (operations) => { this.eventHook.addLocalListener("sync:success", async (operations) => {
// console.log("Operations", operations);
if (operations > 0) { if (operations > 0) {
// Update the page list // Update the page list
await this.space.updatePageList(); await this.space.updatePageList();

View File

@ -33,6 +33,8 @@ export type PanelMode = number;
export type BuiltinSettings = { export type BuiltinSettings = {
indexPage: string; indexPage: string;
// Format: compatible with docker ignore
spaceIgnore?: string;
}; };
export type PanelConfig = { export type PanelConfig = {

View File

@ -1,6 +1,11 @@
An attempt at documenting the changes/new features introduced in each An attempt at documenting the changes/new features introduced in each
release. release.
## Next
* Added `spaceIgnore` setting to not sync specific folders or file patterns to the client, see [[SETTINGS]] for documentation
--- ---
## 0.3.1 ## 0.3.1

View File

@ -20,9 +20,9 @@ weeklyNoteMonday: false
# Markdown # Markdown
previewOnRHS: true previewOnRHS: true
# Sync # Defines files to ignore in a format compatible with .gitignore
sync: spaceIgnore: |
# Do not sync pages with a specific prefix dist
excludePrefixes: largefolder
- PLUGS *.mp4
``` ```