diff --git a/.github/workflows/docker-s3.yml b/.github/workflows/docker-s3.yml index f9c8426..f0340a2 100644 --- a/.github/workflows/docker-s3.yml +++ b/.github/workflows/docker-s3.yml @@ -7,7 +7,7 @@ on: tags: - "*" env: - DENO_VERSION: v1.33 + DENO_VERSION: v1.34 # Docker & Registries ARCHITECTURES: linux/amd64,linux/arm64 IMAGE_NAME: silverbullet-s3 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 60c56ff..b7d2bd2 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -7,7 +7,7 @@ on: tags: - "*" env: - DENO_VERSION: v1.33 + DENO_VERSION: v1.34 # Docker & Registries ARCHITECTURES: linux/amd64,linux/arm64 IMAGE_NAME: silverbullet diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 39196c0..81c8204 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,7 @@ jobs: - name: Setup Deno uses: denoland/setup-deno@v1 with: - deno-version: v1.33 + deno-version: v1.34 - name: Run build run: deno task build - name: Bundle diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4503568..b7c2a1d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,7 +20,7 @@ jobs: - name: Setup Deno uses: denoland/setup-deno@v1 with: - deno-version: v1.33 + deno-version: v1.34 - name: Run build run: deno task build diff --git a/.gitignore b/.gitignore index f5c8858..e7d7cd7 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,7 @@ website_build .idea deno.lock fly.toml -env.sh \ No newline at end of file +env.sh +node_modules +*.db +test_space diff --git a/build_bundle.ts b/build_bundle.ts index a1c4b55..886de3e 100644 --- a/build_bundle.ts +++ b/build_bundle.ts @@ -13,12 +13,23 @@ await esbuild.build({ sourcemap: false, minify: false, plugins: [ + // ESBuild plugin to make npm modules external + { + name: "npm-external", + setup(build: any) { + build.onResolve({ filter: /^npm:/ }, (args: any) => { + return { + path: args.path, + external: true, + }; + }); + }, + }, { name: "json", setup: (build) => build.onLoad({ filter: /\.json$/ }, () => ({ loader: "json" })), }, - ...denoPlugins({ importMapURL: new URL("./import_map.json", import.meta.url) .toString(), diff --git a/build_web.ts b/build_web.ts index f374c02..0858d67 100644 --- a/build_web.ts +++ b/build_web.ts @@ -1,4 +1,3 @@ -import { denoPlugins } from "https://deno.land/x/esbuild_deno_loader@0.7.0/mod.ts"; import { copy } from "https://deno.land/std@0.165.0/fs/copy.ts"; import sass from "https://deno.land/x/denosass@1.0.4/mod.ts"; @@ -6,7 +5,7 @@ import { bundleFolder } from "./plugos/asset_bundle/builder.ts"; import * as flags from "https://deno.land/std@0.165.0/flags/mod.ts"; import { patchDenoLibJS } from "./plugos/compile.ts"; -import { esbuild } from "./plugos/deps.ts"; +import { denoPlugins, esbuild } from "./plugos/deps.ts"; export async function bundleAll( watch: boolean, @@ -43,7 +42,7 @@ export async function copyAssets(dist: string) { await copy("web/auth.html", `${dist}/auth.html`, { overwrite: true, }); - await copy("web/reset.html", `${dist}/reset.html`, { + await copy("web/logout.html", `${dist}/logout.html`, { overwrite: true, }); await copy("web/images/favicon.png", `${dist}/favicon.png`, { diff --git a/cmd/server.ts b/cmd/server.ts index 9ebd4cc..c9f560c 100644 --- a/cmd/server.ts +++ b/cmd/server.ts @@ -11,8 +11,10 @@ import { AssetBundlePlugSpacePrimitives } from "../common/spaces/asset_bundle_sp import { DiskSpacePrimitives } from "../common/spaces/disk_space_primitives.ts"; import { SpacePrimitives } from "../common/spaces/space_primitives.ts"; import { S3SpacePrimitives } from "../server/spaces/s3_space_primitives.ts"; +import { Authenticator } from "../server/auth.ts"; +import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts"; -export function serveCommand( +export async function serveCommand( options: any, folder?: string, ) { @@ -61,12 +63,42 @@ To allow outside connections, pass -L 0.0.0.0 as a flag, and put a TLS terminato new AssetBundle(plugAssetBundle as AssetJson), ); + const authStore = new JSONKVStore(); + const authenticator = new Authenticator(authStore); + + const flagUser = options.user ?? Deno.env.get("SB_USER"); + if (flagUser) { + // If explicitly added via env/parameter, add on the fly + const [username, password] = flagUser.split(":"); + await authenticator.register(username, password, ["admin"], ""); + } + + if (options.auth) { + // Load auth file + const authFile: string = options.auth; + console.log("Loading authentication credentials from", authFile); + await authStore.load(authFile); + (async () => { + // Asynchronously kick off file watcher + for await (const _event of Deno.watchFs(options.auth)) { + console.log("Authentication file changed, reloading..."); + await authStore.load(authFile); + } + })().catch(console.error); + } + + const envAuth = Deno.env.get("SB_AUTH"); + if (envAuth) { + console.log("Loading authentication from SB_AUTH"); + authStore.loadString(envAuth); + } + const httpServer = new HttpServer(spacePrimitives!, { hostname, port: port, pagesPath: folder!, clientAssetBundle: new AssetBundle(clientAssetBundle as AssetJson), - user: options.user ?? Deno.env.get("SB_USER"), + authenticator, keyFile: options.key, certFile: options.cert, maxFileSizeMB: +maxFileSizeMB, diff --git a/cmd/user_add.ts b/cmd/user_add.ts new file mode 100644 index 0000000..53e2f38 --- /dev/null +++ b/cmd/user_add.ts @@ -0,0 +1,35 @@ +import getpass from "https://deno.land/x/getpass@0.3.1/mod.ts"; +import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts"; +import { Authenticator } from "../server/auth.ts"; + +export async function userAdd( + options: any, + username?: string, +) { + const authFile = options.auth || ".auth.json"; + console.log("Using auth file", authFile); + if (!username) { + username = prompt("Username:")!; + } + if (!username) { + return; + } + const pw = getpass("Password: "); + if (!pw) { + return; + } + + console.log("Adding user to groups", options.group); + + const store = new JSONKVStore(); + try { + await store.load(authFile); + } catch (e: any) { + if (e instanceof Deno.errors.NotFound) { + console.log("Creating new auth database because it didn't exist."); + } + } + const auth = new Authenticator(store); + await auth.register(username!, pw!, options.group); + await store.save(authFile); +} diff --git a/cmd/user_chgrp.ts b/cmd/user_chgrp.ts new file mode 100644 index 0000000..34f1b54 --- /dev/null +++ b/cmd/user_chgrp.ts @@ -0,0 +1,30 @@ +import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts"; +import { Authenticator } from "../server/auth.ts"; + +export async function userChgrp( + options: any, + username?: string, +) { + const authFile = options.auth || ".auth.json"; + console.log("Using auth file", authFile); + if (!username) { + username = prompt("Username:")!; + } + if (!username) { + return; + } + + console.log("Setting groups for user:", options.group); + + const store = new JSONKVStore(); + try { + await store.load(authFile); + } catch (e: any) { + if (e instanceof Deno.errors.NotFound) { + console.log("Creating new auth database because it didn't exist."); + } + } + const auth = new Authenticator(store); + await auth.setGroups(username!, options.group); + await store.save(authFile); +} diff --git a/cmd/user_delete.ts b/cmd/user_delete.ts new file mode 100644 index 0000000..64a0472 --- /dev/null +++ b/cmd/user_delete.ts @@ -0,0 +1,37 @@ +import getpass from "https://deno.land/x/getpass@0.3.1/mod.ts"; +import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts"; +import { Authenticator } from "../server/auth.ts"; + +export async function userDelete( + options: any, + username?: string, +) { + const authFile = options.auth || ".auth.json"; + console.log("Using auth file", authFile); + if (!username) { + username = prompt("Username:")!; + } + if (!username) { + return; + } + + const store = new JSONKVStore(); + try { + await store.load(authFile); + } catch (e: any) { + if (e instanceof Deno.errors.NotFound) { + console.log("Creating new auth database because it didn't exist."); + } + } + const auth = new Authenticator(store); + + const user = await auth.getUser(username); + + if (!user) { + console.error("User", username, "not found."); + Deno.exit(1); + } + + await auth.deleteUser(username!); + await store.save(authFile); +} diff --git a/cmd/user_passwd.ts b/cmd/user_passwd.ts new file mode 100644 index 0000000..567e090 --- /dev/null +++ b/cmd/user_passwd.ts @@ -0,0 +1,42 @@ +import getpass from "https://deno.land/x/getpass@0.3.1/mod.ts"; +import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts"; +import { Authenticator } from "../server/auth.ts"; + +export async function userPasswd( + options: any, + username?: string, +) { + const authFile = options.auth || ".auth.json"; + console.log("Using auth file", authFile); + if (!username) { + username = prompt("Username:")!; + } + if (!username) { + return; + } + + const store = new JSONKVStore(); + try { + await store.load(authFile); + } catch (e: any) { + if (e instanceof Deno.errors.NotFound) { + console.log("Creating new auth database because it didn't exist."); + } + } + const auth = new Authenticator(store); + + const user = await auth.getUser(username); + + if (!user) { + console.error("User", username, "not found."); + Deno.exit(1); + } + + const pw = getpass("New password: "); + if (!pw) { + return; + } + + await auth.setPassword(username!, pw!); + await store.save(authFile); +} diff --git a/collab-data/m9DT-f0Z71eMhGUMB22TX b/collab-data/m9DT-f0Z71eMhGUMB22TX new file mode 100644 index 0000000..fea61ab Binary files /dev/null and b/collab-data/m9DT-f0Z71eMhGUMB22TX differ diff --git a/common/deps.ts b/common/deps.ts index 508154a..2a2d9bb 100644 --- a/common/deps.ts +++ b/common/deps.ts @@ -61,7 +61,7 @@ export { } from "@codemirror/view"; export type { DecorationSet, KeyBinding } from "@codemirror/view"; -export { markdown } from "https://esm.sh/@codemirror/lang-markdown@6.1.1?external=@codemirror/state,@lezer/common,@codemirror/language,@lezer/markdown,@codemirror/view,@lezer/highlight,@@codemirror/lang-html"; +export { markdown } from "https://esm.sh/@codemirror/lang-markdown@6.1.1?external=@codemirror/state,@lezer/common,@codemirror/language,@lezer/markdown,@codemirror/view,@lezer/highlight,@codemirror/lang-html"; export { EditorSelection, diff --git a/common/spaces/asset_bundle_space_primitives.ts b/common/spaces/asset_bundle_space_primitives.ts index 5fb6444..695529f 100644 --- a/common/spaces/asset_bundle_space_primitives.ts +++ b/common/spaces/asset_bundle_space_primitives.ts @@ -1,9 +1,7 @@ import { FileMeta } from "../types.ts"; import { SpacePrimitives } from "./space_primitives.ts"; import { AssetBundle } from "../../plugos/asset_bundle/bundle.ts"; -import { mime } from "../deps.ts"; -const bootTime = Date.now(); export class AssetBundlePlugSpacePrimitives implements SpacePrimitives { constructor( private wrapped: SpacePrimitives, @@ -16,8 +14,8 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives { return this.assetBundle.listFiles() .map((p) => ({ name: p, - contentType: mime.getType(p) || "application/octet-stream", - lastModified: bootTime, + contentType: this.assetBundle.getMimeType(p), + lastModified: this.assetBundle.getMtime(p), perm: "ro", size: -1, } as FileMeta)).concat(files); @@ -32,10 +30,10 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives { return Promise.resolve({ data, meta: { - lastModified: bootTime, + contentType: this.assetBundle.getMimeType(name), + lastModified: this.assetBundle.getMtime(name), size: data.byteLength, perm: "ro", - contentType: this.assetBundle.getMimeType(name), } as FileMeta, }); } @@ -46,10 +44,10 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives { if (this.assetBundle.has(name)) { const data = this.assetBundle.readFileSync(name); return Promise.resolve({ - lastModified: bootTime, + contentType: this.assetBundle.getMimeType(name), + lastModified: this.assetBundle.getMtime(name), size: data.byteLength, perm: "ro", - contentType: this.assetBundle.getMimeType(name), } as FileMeta); } return this.wrapped.getFileMeta(name); diff --git a/common/spaces/disk_space_primitives.ts b/common/spaces/disk_space_primitives.ts index 614a359..1634bc6 100644 --- a/common/spaces/disk_space_primitives.ts +++ b/common/spaces/disk_space_primitives.ts @@ -63,7 +63,7 @@ export class DiskSpacePrimitives implements SpacePrimitives { }; } catch { // console.error("Error while reading file", name, e); - throw Error(`Could not read file ${name}`); + throw Error("Not found"); } } diff --git a/common/spaces/evented_space_primitives.ts b/common/spaces/evented_space_primitives.ts index b878aa2..f49b612 100644 --- a/common/spaces/evented_space_primitives.ts +++ b/common/spaces/evented_space_primitives.ts @@ -36,7 +36,7 @@ export class EventedSpacePrimitives implements SpacePrimitives { text = decoder.decode(data); this.eventHook - .dispatchEvent("page:saved", pageName) + .dispatchEvent("page:saved", pageName, newMeta) .then(() => { return this.eventHook.dispatchEvent("page:index_text", { name: pageName, diff --git a/common/spaces/sync.ts b/common/spaces/sync.ts index fabc38d..8857670 100644 --- a/common/spaces/sync.ts +++ b/common/spaces/sync.ts @@ -37,7 +37,10 @@ export class SpaceSync { ) { } - async syncFiles(snapshot: Map): Promise { + async syncFiles( + snapshot: Map, + isSyncCandidate = this.options.isSyncCandidate, + ): Promise { let operations = 0; console.log("[sync]", "Fetching snapshot from primary"); const primaryAllPages = this.syncCandidates( @@ -73,6 +76,9 @@ export class SpaceSync { // console.log("[sync]", "Iterating over all files"); let filesProcessed = 0; for (const name of sortedFilenames) { + if (isSyncCandidate && !isSyncCandidate(name)) { + continue; + } try { operations += await this.syncFile( snapshot, diff --git a/common/util.ts b/common/util.ts index 4ce85fb..39fe16d 100644 --- a/common/util.ts +++ b/common/util.ts @@ -41,14 +41,19 @@ export async function ensureSettingsAndIndex( settingsText = new TextDecoder().decode( (await space.readFile("SETTINGS.md")).data, ); - } catch { - await space.writeFile( - "SETTINGS.md", - new TextEncoder().encode(SETTINGS_TEMPLATE), - true, - ); + } catch (e: any) { + if (e.message === "Not found") { + await space.writeFile( + "SETTINGS.md", + new TextEncoder().encode(SETTINGS_TEMPLATE), + true, + ); + } else { + console.error("Error reading settings", e.message); + console.error("Falling back to default settings"); + } settingsText = SETTINGS_TEMPLATE; - // Ok, then let's also write the index page + // Ok, then let's also check the index page try { await space.getFileMeta("index.md"); } catch { diff --git a/import_map.json b/import_map.json index 8ebad8a..b3601a2 100644 --- a/import_map.json +++ b/import_map.json @@ -1,10 +1,11 @@ { "imports": { "@lezer/common": "https://esm.sh/@lezer/common@1.0.2", - "@lezer/lr": "https://esm.sh/@lezer/lr@1.3.5?external=@lezer/common", - "@lezer/markdown": "https://esm.sh/@lezer/markdown@1.0.2?external=@lezer/common,@codemirror/language,@lezer/highlight", - "@lezer/javascript": "https://esm.sh/@lezer/javascript@1.4.3?external=@lezer/common,@codemirror/language,@lezer/highlight", - "@lezer/highlight": "https://esm.sh/@lezer/highlight@1.1.6?external=@lezer/common", + "@lezer/lr": "https://esm.sh/@lezer/lr@1.3.5?external=@lezer/common&target=deno", + "@lezer/markdown": "https://esm.sh/@lezer/markdown@1.0.2?external=@lezer/common,@codemirror/language,@lezer/highlight,@lezer/lr", + "@lezer/javascript": "https://esm.sh/@lezer/javascript@1.4.3?external=@lezer/common,@codemirror/language,@lezer/highlight,@lezer/lr", + "@lezer/highlight": "https://esm.sh/@lezer/highlight@1.1.6?external=@lezer/common,@lezer/lr", + "@lezer/html": "https://esm.sh/@lezer/html@1.3.4?external=@lezer/common,@lezer/lr", "@codemirror/state": "https://esm.sh/@codemirror/state@6.2.1", "@codemirror/language": "https://esm.sh/@codemirror/language@6.7.0?external=@codemirror/state,@lezer/common,@lezer/lr,@codemirror/view,@lezer/highlight", @@ -12,11 +13,12 @@ "@codemirror/view": "https://esm.sh/@codemirror/view@6.9.0?external=@codemirror/state,@lezer/common", "@codemirror/autocomplete": "https://esm.sh/@codemirror/autocomplete@6.7.1?external=@codemirror/state,@codemirror/commands,@lezer/common,@codemirror/view", "@codemirror/lint": "https://esm.sh/@codemirror/lint@6.2.1?external=@codemirror/state,@lezer/common", - "@codemirror/lang-html": "https://esm.sh/@codemirror/lang-html@6.4.3?external=@codemirror/language,@codemirror/autocomplete,@codemirror/lang-css,@codemirror/state", + "@codemirror/lang-css": "https://esm.sh/@codemirror/lang-css@6.2.0?external=@codemirror/language,@codemirror/autocomplete,@codemirror/state,@lezer/lr,@lezer/html", + "@codemirror/lang-html": "https://esm.sh/@codemirror/lang-html@6.4.3?external=@codemirror/language,@codemirror/autocomplete,@codemirror/lang-css,@codemirror/state,@lezer/lr,@lezer/html", "@codemirror/search": "https://esm.sh/@codemirror/search@6.4.0?external=@codemirror/state,@codemirror/view", "preact": "https://esm.sh/preact@10.11.1", - "yjs": "https://esm.sh/yjs@13.5.42", + "yjs": "https://esm.sh/yjs@13.5.42?target=es2022", "$sb/": "./plug-api/", "handlebars": "https://esm.sh/handlebars@4.7.7?target=es2022", "dexie": "https://esm.sh/dexie@3.2.2" diff --git a/plug-api/app_event.ts b/plug-api/app_event.ts index 253f806..1008f71 100644 --- a/plug-api/app_event.ts +++ b/plug-api/app_event.ts @@ -9,6 +9,7 @@ export type AppEvent = | "editor:init" | "editor:pageLoaded" | "editor:pageReloaded" + | "editor:pageSaved" | "editor:modeswitch" | "plugs:loaded"; diff --git a/plug-api/silverbullet-syscall/editor.ts b/plug-api/silverbullet-syscall/editor.ts index 89b13c7..fe0613a 100644 --- a/plug-api/silverbullet-syscall/editor.ts +++ b/plug-api/silverbullet-syscall/editor.ts @@ -42,8 +42,8 @@ export function reloadPage(): Promise { return syscall("editor.reloadPage"); } -export function openUrl(url: string): Promise { - return syscall("editor.openUrl", url); +export function openUrl(url: string, existingWindow = false): Promise { + return syscall("editor.openUrl", url, existingWindow); } // Force the client to download the file in dataUrl with filename as file name diff --git a/plugos/deps.ts b/plugos/deps.ts index fe1098e..7d4f941 100644 --- a/plugos/deps.ts +++ b/plugos/deps.ts @@ -6,5 +6,5 @@ export { mime } from "https://deno.land/x/mimetypes@v1.0.0/mod.ts"; export { default as cacheDir } from "https://deno.land/x/cache_dir@0.2.0/mod.ts"; export * as flags from "https://deno.land/std@0.165.0/flags/mod.ts"; export * as esbuild from "https://deno.land/x/esbuild@v0.17.18/mod.js"; -export { denoPlugins } from "https://deno.land/x/esbuild_deno_loader@0.7.0/mod.ts"; +export { denoPlugins } from "https://deno.land/x/esbuild_deno_loader@0.8.1/mod.ts"; export * as YAML from "https://deno.land/std@0.184.0/yaml/mod.ts"; diff --git a/plugos/hooks/event.ts b/plugos/hooks/event.ts index 1df0ad8..85b87d2 100644 --- a/plugos/hooks/event.ts +++ b/plugos/hooks/event.ts @@ -10,9 +10,9 @@ export type EventHookT = { export class EventHook implements Hook { private system?: System; - public localListeners: Map any)[]> = new Map(); + public localListeners: Map any)[]> = new Map(); - addLocalListener(eventName: string, callback: (data: any) => any) { + addLocalListener(eventName: string, callback: (...args: any[]) => any) { if (!this.localListeners.has(eventName)) { this.localListeners.set(eventName, []); } @@ -41,13 +41,13 @@ export class EventHook implements Hook { return [...eventNames]; } - async dispatchEvent(eventName: string, data?: any): Promise { + async dispatchEvent(eventName: string, ...args: any[]): Promise { if (!this.system) { throw new Error("Event hook is not initialized"); } const responses: any[] = []; for (const plug of this.system.loadedPlugs.values()) { - const manifest = await plug.manifest; + const manifest = plug.manifest; for ( const [name, functionDef] of Object.entries( manifest!.functions, @@ -56,7 +56,7 @@ export class EventHook implements Hook { if (functionDef.events && functionDef.events.includes(eventName)) { // Only dispatch functions that can run in this environment if (await plug.canInvoke(name)) { - const result = await plug.invoke(name, [data]); + const result = await plug.invoke(name, args); if (result !== undefined) { responses.push(result); } @@ -67,7 +67,7 @@ export class EventHook implements Hook { const localListeners = this.localListeners.get(eventName); if (localListeners) { for (const localListener of localListeners) { - const result = await Promise.resolve(localListener(data)); + const result = await Promise.resolve(localListener(...args)); if (result) { responses.push(result); } diff --git a/plugos/lib/kv_store.json_file.ts b/plugos/lib/kv_store.json_file.ts new file mode 100644 index 0000000..782d703 --- /dev/null +++ b/plugos/lib/kv_store.json_file.ts @@ -0,0 +1,71 @@ +import { KV, KVStore } from "./kv_store.ts"; + +export class JSONKVStore implements KVStore { + private data: { [key: string]: any } = {}; + + async load(path: string) { + this.loadString(await Deno.readTextFile(path)); + } + + loadString(jsonString: string) { + this.data = JSON.parse(jsonString); + } + + async save(path: string) { + await Deno.writeTextFile(path, JSON.stringify(this.data)); + } + + del(key: string): Promise { + delete this.data[key]; + return Promise.resolve(); + } + + deletePrefix(prefix: string): Promise { + for (const key in this.data) { + if (key.startsWith(prefix)) { + delete this.data[key]; + } + } + return Promise.resolve(); + } + + deleteAll(): Promise { + this.data = {}; + return Promise.resolve(); + } + + set(key: string, value: any): Promise { + this.data[key] = value; + return Promise.resolve(); + } + batchSet(kvs: KV[]): Promise { + for (const kv of kvs) { + this.data[kv.key] = kv.value; + } + return Promise.resolve(); + } + batchDelete(keys: string[]): Promise { + for (const key of keys) { + delete this.data[key]; + } + return Promise.resolve(); + } + batchGet(keys: string[]): Promise { + return Promise.resolve(keys.map((key) => this.data[key])); + } + get(key: string): Promise { + return Promise.resolve(this.data[key]); + } + has(key: string): Promise { + return Promise.resolve(key in this.data); + } + queryPrefix(keyPrefix: string): Promise<{ key: string; value: any }[]> { + const results: { key: string; value: any }[] = []; + for (const key in this.data) { + if (key.startsWith(keyPrefix)) { + results.push({ key, value: this.data[key] }); + } + } + return Promise.resolve(results); + } +} diff --git a/plugs/collab/constants.ts b/plugs/collab/constants.ts new file mode 100644 index 0000000..53ec7f9 --- /dev/null +++ b/plugs/collab/constants.ts @@ -0,0 +1 @@ +export const collabPingInterval = 2500; diff --git a/plugs/core/account.ts b/plugs/core/account.ts new file mode 100644 index 0000000..bef45eb --- /dev/null +++ b/plugs/core/account.ts @@ -0,0 +1,5 @@ +import { editor } from "$sb/silverbullet-syscall/mod.ts"; + +export async function accountLogoutCommand() { + await editor.openUrl("/.client/logout.html", true); +} diff --git a/plugs/core/core.plug.yaml b/plugs/core/core.plug.yaml index 99ba90f..102930d 100644 --- a/plugs/core/core.plug.yaml +++ b/plugs/core/core.plug.yaml @@ -326,22 +326,21 @@ functions: path: ./debug.ts:parsePageCommand command: name: "Debug: Parse Document" - resetClientCommand: - path: ./debug.ts:resetClientCommand - command: - name: "Debug: Reset Client" - versionCommand: path: ./help.ts:versionCommand command: name: "Help: Version" - gettingStartedCommand: path: ./help.ts:gettingStartedCommand command: name: "Help: Getting Started" + accountLogoutCommand: + path: ./account.ts:accountLogoutCommand + command: + name: "Account: Logout" + # Link unfurl infrastructure unfurlLink: path: ./link.ts:unfurlCommand diff --git a/plugs/core/debug.ts b/plugs/core/debug.ts index 811a41c..3b77cb4 100644 --- a/plugs/core/debug.ts +++ b/plugs/core/debug.ts @@ -10,7 +10,3 @@ export async function parsePageCommand() { ), ); } - -export async function resetClientCommand() { - editor.openUrl("/.client/reset.html"); -} diff --git a/plugs/core/page.ts b/plugs/core/page.ts index ab70b7e..0bfe8a1 100644 --- a/plugs/core/page.ts +++ b/plugs/core/page.ts @@ -9,7 +9,6 @@ import { index, markdown, space, - system, } from "$sb/silverbullet-syscall/mod.ts"; import { events } from "$sb/plugos-syscall/mod.ts"; diff --git a/plugs/core/plugmanager.ts b/plugs/core/plugmanager.ts index d8fa451..4a845da 100644 --- a/plugs/core/plugmanager.ts +++ b/plugs/core/plugmanager.ts @@ -1,5 +1,4 @@ import { events } from "$sb/plugos-syscall/mod.ts"; -import type { Manifest } from "../../common/manifest.ts"; import { editor, space, system } from "$sb/silverbullet-syscall/mod.ts"; import { readYamlPage } from "$sb/lib/yaml_page.ts"; import { builtinPlugNames } from "../builtin_plugs.ts"; @@ -21,7 +20,7 @@ export async function updatePlugsCommand() { ); } } catch (e: any) { - if (e.message.includes("Could not read file")) { + if (e.message.includes("Not found")) { console.warn("No PLUGS page found, not loading anything"); return; } diff --git a/server/auth.ts b/server/auth.ts new file mode 100644 index 0000000..ef98ca7 --- /dev/null +++ b/server/auth.ts @@ -0,0 +1,120 @@ +import { KVStore } from "../plugos/lib/kv_store.ts"; + +export type User = { + username: string; + passwordHash: string; // hashed password + salt: string; + groups: string[]; // special "admin" +}; + +async function createUser( + username: string, + password: string, + groups: string[], + salt = generateSalt(16), +): Promise { + return { + username, + passwordHash: await hashSHA256(`${salt}${password}`), + salt, + groups, + }; +} + +const userPrefix = `u:`; + +export class Authenticator { + constructor(private store: KVStore) { + } + + async register( + username: string, + password: string, + groups: string[], + salt?: string, + ): Promise { + await this.store.set( + `${userPrefix}${username}`, + await createUser(username, password, groups, salt), + ); + } + + async authenticateHashed( + username: string, + hashedPassword: string, + ): Promise { + const user = await this.store.get(`${userPrefix}${username}`) as User; + if (!user) { + return false; + } + return user.passwordHash === hashedPassword; + } + + async authenticate( + username: string, + password: string, + ): Promise { + const user = await this.store.get(`${userPrefix}${username}`) as User; + if (!user) { + return undefined; + } + const hashedPassword = await hashSHA256(`${user.salt}${password}`); + return user.passwordHash === hashedPassword ? hashedPassword : undefined; + } + + async getAllUsers(): Promise { + return (await this.store.queryPrefix(userPrefix)).map((item) => item.value); + } + + getUser(username: string): Promise { + return this.store.get(`${userPrefix}${username}`); + } + + async setPassword(username: string, password: string): Promise { + const user = await this.getUser(username); + if (!user) { + throw new Error(`User does not exist`); + } + user.passwordHash = await hashSHA256(`${user.salt}${password}`); + await this.store.set(`${userPrefix}${username}`, user); + } + + async deleteUser(username: string): Promise { + const user = await this.getUser(username); + if (!user) { + throw new Error(`User does not exist`); + } + await this.store.del(`${userPrefix}${username}`); + } + + async setGroups(username: string, groups: string[]): Promise { + const user = await this.getUser(username); + if (!user) { + throw new Error(`User does not exist`); + } + user.groups = groups; + await this.store.set(`${userPrefix}${username}`, user); + } +} + +async function hashSHA256(message: string): Promise { + // Transform the string into an ArrayBuffer + const encoder = new TextEncoder(); + const data = encoder.encode(message); + + // Generate the hash + const hashBuffer = await window.crypto.subtle.digest("SHA-256", data); + + // Transform the hash into a hex string + return Array.from(new Uint8Array(hashBuffer)).map((b) => + b.toString(16).padStart(2, "0") + ).join(""); +} + +function generateSalt(length: number): string { + const array = new Uint8Array(length / 2); // because two characters represent one byte in hex + crypto.getRandomValues(array); + return Array.from(array, (byte) => ("00" + byte.toString(16)).slice(-2)).join( + "", + ); +} diff --git a/server/collab.test.ts b/server/collab.test.ts new file mode 100644 index 0000000..6f5008d --- /dev/null +++ b/server/collab.test.ts @@ -0,0 +1,46 @@ +import { assert, assertEquals } from "../test_deps.ts"; +import { CollabServer } from "./collab.ts"; + +Deno.test("Collab server", async () => { + const collabServer = new CollabServer(null as any); + console.log("Client 1 joins page 1"); + assertEquals(collabServer.updatePresence("client1", "page1"), {}); + assertEquals(collabServer.pages.size, 1); + console.log("CLient 1 leaves page 1"); + assertEquals(collabServer.updatePresence("client1", undefined, "page1"), {}); + assertEquals(collabServer.pages.size, 0); + assertEquals(collabServer.updatePresence("client1", "page1"), {}); + console.log("Client 1 joins page 2"); + assertEquals(collabServer.updatePresence("client1", "page2", "page1"), {}); + assertEquals(collabServer.pages.size, 1); + console.log("Client 2 joins to page 2, collab id created, but not exposed"); + assertEquals( + collabServer.updatePresence("client2", "page2").collabId, + undefined, + ); + assert( + collabServer.updatePresence("client1", "page2").collabId !== undefined, + ); + console.log("Client 2 moves to page 1, collab id destroyed"); + assertEquals(collabServer.updatePresence("client2", "page1", "page2"), {}); + assertEquals(collabServer.updatePresence("client1", "page2", "page2"), {}); + assertEquals(collabServer.pages.get("page2")!.collabId, undefined); + assertEquals(collabServer.pages.get("page1")!.collabId, undefined); + console.log("Going to cleanup, which should have no effect"); + collabServer.cleanup(50); + assertEquals(collabServer.pages.size, 2); + collabServer.updatePresence("client2", "page2", "page1"); + console.log("Going to sleep 20ms"); + await sleep(20); + console.log("Then client 1 pings, but client 2 does not"); + collabServer.updatePresence("client1", "page2", "page2"); + await sleep(20); + console.log("Going to cleanup, which should clean client 2"); + collabServer.cleanup(35); + assertEquals(collabServer.pages.get("page2")!.collabId, undefined); + console.log(collabServer); +}); + +function sleep(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/server/collab.ts b/server/collab.ts new file mode 100644 index 0000000..a0cd6d6 --- /dev/null +++ b/server/collab.ts @@ -0,0 +1,231 @@ +import { getAvailablePortSync } from "https://deno.land/x/port@1.0.0/mod.ts"; +import { nanoid } from "https://esm.sh/nanoid@4.0.0"; +import { race, timeout } from "../common/async_util.ts"; +import { Application } from "./deps.ts"; +import { SpacePrimitives } from "../common/spaces/space_primitives.ts"; +import { collabPingInterval } from "../plugs/collab/constants.ts"; +import { Hocuspocus } from "./deps.ts"; + +type CollabPage = { + clients: Map; // clientId -> lastPing + collabId?: string; + // The currently elected provider of the initial document + masterClientId: string; +}; + +export class CollabServer { + // clients: Map = new Map(); + pages: Map = new Map(); + yCollabServer?: Hocuspocus; + + constructor(private spacePrimitives: SpacePrimitives) { + } + + start() { + setInterval(() => { + this.cleanup(3 * collabPingInterval); + }, collabPingInterval); + } + + updatePresence( + clientId: string, + currentPage?: string, + previousPage?: string, + ): { collabId?: string } { + if (previousPage && currentPage !== previousPage) { + // Client switched pages + // Update last page record + const lastCollabPage = this.pages.get(previousPage); + if (lastCollabPage) { + lastCollabPage.clients.delete(clientId); + if (lastCollabPage.clients.size === 1) { + delete lastCollabPage.collabId; + } + + if (lastCollabPage.clients.size === 0) { + this.pages.delete(previousPage); + } else { + // Elect a new master client + if (lastCollabPage.masterClientId === clientId) { + // Any is fine, really + lastCollabPage.masterClientId = + [...lastCollabPage.clients.keys()][0]; + } + } + } + } + + if (currentPage) { + // Update new page + let nextCollabPage = this.pages.get(currentPage); + if (!nextCollabPage) { + // Newly opened page (no other clients on this page right now) + nextCollabPage = { + clients: new Map(), + masterClientId: clientId, + }; + this.pages.set(currentPage, nextCollabPage); + } + // Register last ping from us + nextCollabPage.clients.set(clientId, Date.now()); + + if (nextCollabPage.clients.size > 1 && !nextCollabPage.collabId) { + // Create a new collabId + nextCollabPage.collabId = nanoid(); + } + // console.log("State", this.pages); + if (nextCollabPage.collabId) { + // We will now expose this collabId, except when we're just starting this session + // in which case we'll wait for the original client to publish the document + const existingyCollabSession = this.yCollabServer?.documents.get( + buildCollabId(nextCollabPage.collabId, `${currentPage}.md`), + ); + if (existingyCollabSession) { + // console.log("Found an existing collab session already, let's join!"); + return { collabId: nextCollabPage.collabId }; + } else if (clientId === nextCollabPage.masterClientId) { + // console.log("We're the master, so we should connect"); + return { collabId: nextCollabPage.collabId }; + } else { + // We're not the first client, so we need to wait for the first client to connect + // console.log("We're not the master, so we should wait"); + return {}; + } + } else { + return {}; + } + } else { + return {}; + } + } + + cleanup(timeout: number) { + // Clean up pages and their clients that haven't pinged for some time + for (const [pageName, page] of this.pages) { + for (const [clientId, lastPing] of page.clients) { + if (Date.now() - lastPing > timeout) { + // Eject client + page.clients.delete(clientId); + // Elect a new master client + if (page.masterClientId === clientId && page.clients.size > 0) { + page.masterClientId = [...page.clients.keys()][0]; + } + } + } + if (page.clients.size === 1) { + // If there's only one client left, we don't need to keep the collabId around anymore + delete page.collabId; + } + if (page.clients.size === 0) { + // And if we have no clients left, well... + this.pages.delete(pageName); + } + } + } + + route(app: Application) { + // The way this works is that we spin up a separate WS server locally and then proxy requests to it + // This is the only way I could get Hocuspocus to work with Deno + const internalPort = getAvailablePortSync(); + this.yCollabServer = new Hocuspocus({ + port: internalPort, + address: "127.0.0.1", + quiet: true, + onStoreDocument: async (data) => { + const [_, path] = splitCollabId(data.documentName); + const text = data.document.getText("codemirror").toString(); + console.log( + "[Hocuspocus]", + "Persisting", + path, + "to space on server", + ); + const meta = await this.spacePrimitives.writeFile( + path, + new TextEncoder().encode(text), + ); + // Broadcast new persisted lastModified date + data.document.broadcastStateless( + JSON.stringify({ + type: "persisted", + path, + lastModified: meta.lastModified, + }), + ); + return; + }, + onDisconnect: (client) => { + console.log("[Hocuspocus]", "Client disconnected", client.clientsCount); + if (client.clientsCount === 0) { + console.log( + "[Hocuspocus]", + "Last client disconnected from", + client.documentName, + "purging from memory", + ); + this.yCollabServer!.documents.delete(client.documentName); + } + return Promise.resolve(); + }, + }); + + this.yCollabServer.listen(); + + app.use((ctx) => { + // if (ctx.request.url.pathname === "/.ws") { + // const sock = ctx.upgrade(); + // sock.onmessage = (e) => { + // console.log("WS: Got message", e.data); + // }; + // } + // Websocket proxy to hocuspocus + if (ctx.request.url.pathname === "/.ws-collab") { + const sock = ctx.upgrade(); + + const ws = new WebSocket(`ws://localhost:${internalPort}`); + const wsReady = race([ + new Promise((resolve) => { + ws.onopen = () => { + resolve(); + }; + }), + timeout(1000), + ]).catch(() => { + console.error("Timeout waiting for collab to open websocket"); + sock.close(); + }); + sock.onmessage = (e) => { + // console.log("Got message", e); + wsReady.then(() => ws.send(e.data)).catch(console.error); + }; + sock.onclose = () => { + if (ws.OPEN) { + ws.close(); + } + }; + ws.onmessage = (e) => { + if (sock.OPEN) { + sock.send(e.data); + } else { + console.error("Got message from websocket but socket is not open"); + } + }; + ws.onclose = () => { + if (sock.OPEN) { + sock.close(); + } + }; + } + }); + } +} + +function splitCollabId(documentName: string): [string, string] { + const [collabId, ...pathPieces] = documentName.split("/"); + const path = pathPieces.join("/"); + return [collabId, path]; +} + +function buildCollabId(collabId: string, path: string): string { + return `${collabId}/${path}`; +} diff --git a/server/deps.ts b/server/deps.ts index ca2011b..15932b1 100644 --- a/server/deps.ts +++ b/server/deps.ts @@ -1,3 +1,5 @@ export * from "../common/deps.ts"; export { Application, Router } from "https://deno.land/x/oak@v12.4.0/mod.ts"; export * as etag from "https://deno.land/x/oak@v12.4.0/etag.ts"; + +export { Hocuspocus } from "npm:@hocuspocus/server@2.1.0"; diff --git a/server/http_server.ts b/server/http_server.ts index 620b0b9..6362d45 100644 --- a/server/http_server.ts +++ b/server/http_server.ts @@ -7,13 +7,15 @@ import { performLocalFetch } from "../common/proxy_fetch.ts"; import { BuiltinSettings } from "../web/types.ts"; import { gitIgnoreCompiler } from "./deps.ts"; import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts"; +import { CollabServer } from "./collab.ts"; +import { Authenticator } from "./auth.ts"; export type ServerOptions = { hostname: string; port: number; pagesPath: string; clientAssetBundle: AssetBundle; - user?: string; + authenticator: Authenticator; pass?: string; certFile?: string; keyFile?: string; @@ -24,11 +26,12 @@ export class HttpServer { app: Application; private hostname: string; private port: number; - user?: string; abortController?: AbortController; clientAssetBundle: AssetBundle; settings?: BuiltinSettings; spacePrimitives: SpacePrimitives; + collab: CollabServer; + authenticator: Authenticator; constructor( spacePrimitives: SpacePrimitives, @@ -37,7 +40,7 @@ export class HttpServer { this.hostname = options.hostname; this.port = options.port; this.app = new Application(); - this.user = options.user; + this.authenticator = options.authenticator; this.clientAssetBundle = options.clientAssetBundle; let fileFilterFn: (s: string) => boolean = () => true; @@ -62,6 +65,8 @@ export class HttpServer { } }, ); + this.collab = new CollabServer(this.spacePrimitives); + this.collab.start(); } // Replaces some template variables in index.html in a rather ad-hoc manner, but YOLO @@ -123,7 +128,8 @@ export class HttpServer { this.app.use(({ request, response }, next) => { if ( !request.url.pathname.startsWith("/.fs") && - request.url.pathname !== "/.auth" + request.url.pathname !== "/.auth" && + !request.url.pathname.startsWith("/.ws") ) { response.headers.set("Content-type", "text/html"); response.body = this.renderIndexHtml(); @@ -134,10 +140,12 @@ export class HttpServer { // Pages API const fsRouter = this.buildFsRouter(this.spacePrimitives); - this.addPasswordAuth(this.app); + await this.addPasswordAuth(this.app); this.app.use(fsRouter.routes()); this.app.use(fsRouter.allowedMethods()); + this.collab.route(this.app); + this.abortController = new AbortController(); const listenOptions: any = { hostname: this.hostname, @@ -168,25 +176,40 @@ export class HttpServer { this.settings = await ensureSettingsAndIndex(this.spacePrimitives); } - private addPasswordAuth(app: Application) { + private async addPasswordAuth(app: Application) { const excludedPaths = [ "/manifest.json", "/favicon.png", "/logo.png", "/.auth", ]; - if (this.user) { - const b64User = btoa(this.user); + if ((await this.authenticator.getAllUsers()).length > 0) { app.use(async ({ request, response, cookies }, next) => { if (!excludedPaths.includes(request.url.pathname)) { const authCookie = await cookies.get("auth"); - if (!authCookie || authCookie !== b64User) { + if (!authCookie) { response.status = 401; response.body = "Unauthorized, please authenticate"; return; } + const [username, hashedPassword] = authCookie.split(":"); + if ( + !await this.authenticator.authenticateHashed( + username, + hashedPassword, + ) + ) { + response.status = 401; + response.body = "Invalid username/password, please reauthenticate"; + return; + } } + if (request.url.pathname === "/.auth") { + if (request.url.search === "?logout") { + await cookies.delete("auth"); + // Implicit fallthrough to login page + } if (request.method === "GET") { response.headers.set("Content-type", "text/html"); response.body = this.clientAssetBundle.readTextFileSync( @@ -195,11 +218,15 @@ export class HttpServer { return; } else if (request.method === "POST") { const values = await request.body({ type: "form" }).value; - const username = values.get("username"), - password = values.get("password"), + const username = values.get("username")!, + password = values.get("password")!, refer = values.get("refer"); - if (this.user === `${username}:${password}`) { - await cookies.set("auth", b64User, { + const hashedPassword = await this.authenticator.authenticate( + username, + password, + ); + if (hashedPassword) { + await cookies.set("auth", `${username}:${hashedPassword}`, { expires: new Date(Date.now() + 1000 * 60 * 60 * 24 * 7), // in a week sameSite: "strict", }); @@ -273,6 +300,19 @@ export class HttpServer { }); return; } + case "presence": { + // RPC to check (for collab purposes) which client has what page open + response.headers.set("Content-Type", "application/json"); + console.log("Got presence update", body); + response.body = JSON.stringify( + this.collab.updatePresence( + body.clientId, + body.currentPage, + body.previousPage, + ), + ); + return; + } default: response.headers.set("Content-Type", "text/plain"); response.status = 400; @@ -290,6 +330,11 @@ export class HttpServer { .get("\/(.+)", async ({ params, response, request }) => { const name = params[0]; console.log("Loading file", name); + if (name.startsWith(".")) { + // Don't expose hidden files + response.status = 404; + return; + } try { const attachmentData = await spacePrimitives.readFile( name, @@ -322,6 +367,11 @@ export class HttpServer { .put("\/(.+)", async ({ request, response, params }) => { const name = params[0]; console.log("Saving file", name); + if (name.startsWith(".")) { + // Don't expose hidden files + response.status = 403; + return; + } let body: Uint8Array; if ( diff --git a/silverbullet.ts b/silverbullet.ts index 3a785bf..0267e94 100755 --- a/silverbullet.ts +++ b/silverbullet.ts @@ -7,6 +7,10 @@ import { upgradeCommand } from "./cmd/upgrade.ts"; import { versionCommand } from "./cmd/version.ts"; import { serveCommand } from "./cmd/server.ts"; import { plugCompileCommand } from "./cmd/plug_compile.ts"; +import { userAdd } from "./cmd/user_add.ts"; +import { userPasswd } from "./cmd/user_passwd.ts"; +import { userDelete } from "./cmd/user_delete.ts"; +import { userChgrp } from "./cmd/user_chgrp.ts"; await new Command() .name("silverbullet") @@ -27,6 +31,10 @@ await new Command() "--user ", "'username:password' combo for BasicAuth authentication", ) + .option( + "--auth ", + "User authentication file to use for authentication", + ) .option( "--cert ", "Path to TLS certificate", @@ -58,6 +66,42 @@ await new Command() .option("--importmap ", "Path to import map file to use") .option("--runtimeUrl ", "URL to worker_runtime.ts to use") .action(plugCompileCommand) + .command("user:add", "Add a new user to an authentication file") + .arguments("[username:string]") + .option( + "--auth ", + "User authentication file to use", + ) + .option("-G, --group ", "Add user to group", { + collect: true, + default: [] as string[], + }) + .action(userAdd) + .command("user:delete", "Delete an existing user") + .arguments("[username:string]") + .option( + "--auth ", + "User authentication file to use", + ) + .action(userDelete) + .command("user:chgrp", "Update user groups") + .arguments("[username:string]") + .option( + "--auth ", + "User authentication file to use", + ) + .option("-G, --group ", "Groups to put user into", { + collect: true, + default: [] as string[], + }) + .action(userChgrp) + .command("user:passwd", "Set the password for an existing user") + .arguments("[username:string]") + .option( + "--auth ", + "User authentication file to use", + ) + .action(userPasswd) // upgrade .command("upgrade", "Upgrade SilverBullet") .action(upgradeCommand) diff --git a/web/auth.html b/web/auth.html index 5ba25da..085d227 100644 --- a/web/auth.html +++ b/web/auth.html @@ -56,8 +56,14 @@

Login to SilverBullet

+
-
+
diff --git a/web/cm_plugins/collab.ts b/web/cm_plugins/collab.ts index c7c6559..d2352ed 100644 --- a/web/cm_plugins/collab.ts +++ b/web/cm_plugins/collab.ts @@ -1,4 +1,6 @@ -import { Extension, WebsocketProvider, Y, yCollab } from "../deps.ts"; +import { safeRun } from "../../common/util.ts"; +import { Extension, HocuspocusProvider, Y, yCollab } from "../deps.ts"; +import { SyncService } from "../sync_service.ts"; const userColors = [ { color: "#30bced", light: "#30bced33" }, @@ -12,27 +14,45 @@ const userColors = [ ]; export class CollabState { - ydoc: Y.Doc; - collabProvider: WebsocketProvider; - ytext: Y.Text; - yundoManager: Y.UndoManager; + public ytext: Y.Text; + collabProvider: HocuspocusProvider; + private yundoManager: Y.UndoManager; + interval?: number; - constructor(serverUrl: string, token: string, username: string) { - this.ydoc = new Y.Doc(); - this.collabProvider = new WebsocketProvider( - serverUrl, - token, - this.ydoc, - ); + constructor( + serverUrl: string, + readonly path: string, + readonly token: string, + username: string, + private syncService: SyncService, + public isLocalCollab: boolean, + ) { + this.collabProvider = new HocuspocusProvider({ + url: serverUrl, + name: token, + + // Receive broadcasted messages from the server (right now only "page has been persisted" notifications) + onStateless: ( + { payload }, + ) => { + const message = JSON.parse(payload); + switch (message.type) { + case "persisted": { + // Received remote persist notification, updating snapshot + syncService.updateRemoteLastModified( + message.path, + message.lastModified, + ).catch(console.error); + } + } + }, + }); this.collabProvider.on("status", (e: any) => { console.log("Collab status change", e); }); - this.collabProvider.on("sync", (e: any) => { - console.log("Sync status", e); - }); - this.ytext = this.ydoc.getText("codemirror"); + this.ytext = this.collabProvider.document.getText("codemirror"); this.yundoManager = new Y.UndoManager(this.ytext); const randomColor = @@ -43,10 +63,32 @@ export class CollabState { color: randomColor.color, colorLight: randomColor.light, }); + if (isLocalCollab) { + syncService.excludeFromSync(path).catch(console.error); + + this.interval = setInterval(() => { + // Ping the store to make sure the file remains in exclusion + syncService.excludeFromSync(path).catch(console.error); + }, 1000); + } } stop() { + console.log("[COLLAB] Destroying collab provider"); + if (this.interval) { + clearInterval(this.interval); + } this.collabProvider.destroy(); + // For whatever reason, destroy() doesn't properly clean up everything so we need to help a bit + this.collabProvider.configuration.websocketProvider.webSocket = null; + this.collabProvider.configuration.websocketProvider.destroy(); + + // When stopping collaboration, we're going back to sync mode. Make sure we got the latest and greatest remote timestamp to avoid + // conflicts + safeRun(async () => { + await this.syncService.unExcludeFromSync(this.path); + await this.syncService.fetchAndPersistRemoteLastModified(this.path); + }); } collabExtension(): Extension { diff --git a/web/collab_manager.ts b/web/collab_manager.ts new file mode 100644 index 0000000..ec7f049 --- /dev/null +++ b/web/collab_manager.ts @@ -0,0 +1,88 @@ +import { nanoid } from "https://esm.sh/nanoid@4.0.0"; +import type { Editor } from "./editor.tsx"; + +const collabPingInterval = 2500; + +export class CollabManager { + clientId = nanoid(); + localCollabServer: string; + + constructor(private editor: Editor) { + this.localCollabServer = location.protocol === "http:" + ? `ws://${location.host}/.ws-collab` + : `wss://${location.host}/.ws-collab`; + editor.eventHook.addLocalListener( + "editor:pageLoaded", + (pageName, previousPage) => { + console.log("Page loaded", pageName, previousPage); + this.updatePresence(pageName, previousPage).catch(console.error); + }, + ); + } + + start() { + setInterval(() => { + this.updatePresence(this.editor.currentPage!).catch(console.error); + }, collabPingInterval); + } + + async updatePresence(currentPage?: string, previousPage?: string) { + try { + const resp = await this.editor.remoteSpacePrimitives.authenticatedFetch( + this.editor.remoteSpacePrimitives.url, + { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + operation: "presence", + clientId: this.clientId, + previousPage, + currentPage, + }), + keepalive: true, // important for beforeunload event + }, + ); + const { collabId } = await resp.json(); + + if (this.editor.collabState && !this.editor.collabState.isLocalCollab) { + // We're in a remote collab mode, don't do anything + return; + } + + // console.log("Collab ID", collabId); + const previousCollabId = this.editor.collabState?.token.split("/")[0]; + if (!collabId && this.editor.collabState) { + // Stop collab + console.log("Stopping collab"); + if (this.editor.collabState.path === `${currentPage}.md`) { + this.editor.flashNotification( + "Other users have left this page, switched back to single-user mode.", + ); + } + this.editor.stopCollab(); + } else if (collabId && collabId !== previousCollabId) { + // Start collab + console.log("Starting collab"); + this.editor.flashNotification( + "Opening page in multi-user mode.", + ); + this.editor.startCollab( + this.localCollabServer, + `${collabId}/${currentPage}.md`, + this.editor.getUsername(), + true, + ); + } + } catch (e: any) { + // console.error("Ping error", e); + if ( + e.message.toLowerCase().includes("failed") && this.editor.collabState + ) { + console.log("Offline, stopping collab"); + this.editor.stopCollab(); + } + } + } +} diff --git a/web/components/top_bar.tsx b/web/components/top_bar.tsx index 0028d47..f38a9e3 100644 --- a/web/components/top_bar.tsx +++ b/web/components/top_bar.tsx @@ -13,6 +13,7 @@ export type ActionButton = { icon: FunctionalComponent; description: string; callback: () => void; + href?: string; }; export function TopBar({ @@ -118,17 +119,21 @@ export function TopBar({
)}
- {actionButtons.map((actionButton) => ( - - ))} + {actionButtons.map((actionButton) => { + const button = + + + return actionButton.href !== undefined ? ({button}) : button; + })}
diff --git a/web/deps.ts b/web/deps.ts index c0fecd2..9e1e121 100644 --- a/web/deps.ts +++ b/web/deps.ts @@ -21,7 +21,7 @@ export { yCollab, yUndoManagerKeymap, } from "https://esm.sh/y-codemirror.next@0.3.2?external=yjs,@codemirror/state,@codemirror/commands,@codemirror/history,@codemirror/view"; -export { WebsocketProvider } from "https://esm.sh/y-websocket@1.4.5?external=yjs"; +export { HocuspocusProvider } from "https://esm.sh/@hocuspocus/provider@2.1.0?external=yjs,ws&target=es2022"; // Vim mode export { diff --git a/web/editor.tsx b/web/editor.tsx index 9acbd9a..93236b9 100644 --- a/web/editor.tsx +++ b/web/editor.tsx @@ -136,7 +136,7 @@ import { HttpSpacePrimitives } from "../common/spaces/http_space_primitives.ts"; import { FallbackSpacePrimitives } from "../common/spaces/fallback_space_primitives.ts"; import { syncSyscalls } from "./syscalls/sync.ts"; import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts"; -import { globToRegExp } from "https://deno.land/std@0.189.0/path/glob.ts"; +import { CollabManager } from "./collab_manager.ts"; const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/; @@ -193,6 +193,7 @@ export class Editor { syncService: SyncService; settings?: BuiltinSettings; kvStore: DexieKVStore; + collabManager: CollabManager; constructor( parent: Element, @@ -214,6 +215,8 @@ export class Editor { this.eventHook = new EventHook(); system.addHook(this.eventHook); + this.collabManager = new CollabManager(this); + // Cron hook const cronHook = new CronHook(system); system.addHook(cronHook); @@ -368,6 +371,11 @@ export class Editor { } }); + globalThis.addEventListener("beforeunload", (e) => { + console.log("Pinging with with undefined page name"); + this.collabManager.updatePresence(undefined, this.currentPage); + }); + this.eventHook.addLocalListener("plug:changed", async (fileName) => { console.log("Plug updated, reloading:", fileName); system.unload(fileName); @@ -389,7 +397,8 @@ export class Editor { this.space.on({ pageChanged: (meta) => { - if (this.currentPage === meta.name) { + // Only reload when watching the current page (to avoid reloading when switching pages and in collab mode) + if (this.space.watchInterval && this.currentPage === meta.name) { console.log("Page changed elsewhere, reloading"); this.flashNotification("Page changed elsewhere, reloading"); this.reloadPage(); @@ -471,6 +480,7 @@ export class Editor { // Kick off background sync this.syncService.start(); + this.collabManager.start(); this.eventHook.addLocalListener("sync:success", async (operations) => { // console.log("Operations", operations); @@ -557,8 +567,13 @@ export class Editor { this.editorView!.state.sliceDoc(0), true, ) - .then(() => { + .then(async (meta) => { this.viewDispatch({ type: "page-saved" }); + await this.dispatchAppEvent( + "editor:pageSaved", + this.currentPage, + meta, + ); resolve(); }) .catch((e) => { @@ -656,8 +671,8 @@ export class Editor { }); } - dispatchAppEvent(name: AppEvent, data?: any): Promise { - return this.eventHook.dispatchEvent(name, data); + dispatchAppEvent(name: AppEvent, ...args: any[]): Promise { + return this.eventHook.dispatchEvent(name, ...args); } createEditorState( @@ -950,38 +965,42 @@ export class Editor { touchCount = 0; }, mousedown: (event: MouseEvent, view: EditorView) => { - // Make sure tags are clicked without moving the cursor there - if (!event.altKey && event.target instanceof Element) { - const parentA = event.target.closest("a"); - if (parentA) { - event.stopPropagation(); - event.preventDefault(); - const clickEvent: ClickEvent = { - page: pageName, - ctrlKey: event.ctrlKey, - metaKey: event.metaKey, - altKey: event.altKey, - pos: view.posAtCoords({ - x: event.x, - y: event.y, - })!, - }; - this.dispatchAppEvent("page:click", clickEvent).catch( - console.error, - ); - } - } - }, - click: (event: MouseEvent, view: EditorView) => { safeRun(async () => { - const clickEvent: ClickEvent = { + const pos = view.posAtCoords(event); + if (!pos) { + return; + } + const potentialClickEvent: ClickEvent = { page: pageName, ctrlKey: event.ctrlKey, metaKey: event.metaKey, altKey: event.altKey, - pos: view.posAtCoords(event)!, + pos: view.posAtCoords({ + x: event.x, + y: event.y, + })!, }; - await this.dispatchAppEvent("page:click", clickEvent); + // Make sure tags are clicked without moving the cursor there + if (!event.altKey && event.target instanceof Element) { + const parentA = event.target.closest("a"); + if (parentA) { + event.stopPropagation(); + event.preventDefault(); + await this.dispatchAppEvent( + "page:click", + potentialClickEvent, + ); + return; + } + } + + const distanceX = event.x - view.coordsAtPos(pos)!.left; + // What we're trying to determine here is if the click occured anywhere near the looked up position + // this may not be the case with locations that expand signifcantly based on live preview (such as links), we don't want any accidental clicks + // Fixes #357 + if (distanceX <= view.defaultCharacterWidth) { + await this.dispatchAppEvent("page:click", potentialClickEvent); + } }); }, }), @@ -1107,6 +1126,10 @@ export class Editor { this.editorView!.focus(); } + getUsername(): string { + return localStorage.getItem("username") || "you"; + } + async navigate( name: string, pos?: number | string, @@ -1144,8 +1167,7 @@ export class Editor { await this.save(true); // And stop the collab session if (this.collabState) { - this.collabState.stop(); - this.collabState = undefined; + this.stopCollab(); } } } @@ -1187,9 +1209,10 @@ export class Editor { // Note: these events are dispatched asynchronously deliberately (not waiting for results) if (loadingDifferentPage) { - this.eventHook.dispatchEvent("editor:pageLoaded", pageName).catch( - console.error, - ); + this.eventHook.dispatchEvent("editor:pageLoaded", pageName, previousPage) + .catch( + console.error, + ); } else { this.eventHook.dispatchEvent("editor:pageReloaded", pageName).catch( console.error, @@ -1226,10 +1249,18 @@ export class Editor { if (pageState) { // Restore state editorView.scrollDOM.scrollTop = pageState!.scrollTop; - editorView.dispatch({ - selection: pageState.selection, - scrollIntoView: true, - }); + try { + editorView.dispatch({ + selection: pageState.selection, + scrollIntoView: true, + }); + } catch { + // This is fine, just go to the top + editorView.dispatch({ + selection: { anchor: 0 }, + scrollIntoView: true, + }); + } } else { editorView.scrollDOM.scrollTop = 0; editorView.dispatch({ @@ -1405,6 +1436,7 @@ export class Editor { callback: () => { editor.navigate(""); }, + href: "", }, { icon: BookIcon, @@ -1501,19 +1533,49 @@ export class Editor { return; } - startCollab(serverUrl: string, token: string, username: string) { + startCollab( + serverUrl: string, + token: string, + username: string, + isLocalCollab = false, + ) { if (this.collabState) { // Clean up old collab state this.collabState.stop(); } const initialText = this.editorView!.state.sliceDoc(); - this.collabState = new CollabState(serverUrl, token, username); - this.collabState.collabProvider.once("sync", (synced: boolean) => { - if (this.collabState?.ytext.toString() === "") { - console.log("Synced value is empty, putting back original text"); - this.collabState?.ytext.insert(0, initialText); + this.collabState = new CollabState( + serverUrl, + `${this.currentPage!}.md`, + token, + username, + this.syncService, + isLocalCollab, + ); + + this.collabState.collabProvider.on("synced", () => { + if (this.collabState!.ytext.toString() === "") { + console.log( + "[Collab]", + "Synced value is empty (new collab session), inserting local copy", + ); + this.collabState!.ytext.insert(0, initialText); } }); + this.rebuildEditorState(); + + // Don't watch for local changes in this mode + this.space.unwatch(); + } + + stopCollab() { + if (this.collabState) { + this.collabState.stop(); + this.collabState = undefined; + this.rebuildEditorState(); + } + // Start file watching again + this.space.watch(); } } diff --git a/web/reset.html b/web/logout.html similarity index 84% rename from web/reset.html rename to web/logout.html index 3194fb6..107c83b 100644 --- a/web/reset.html +++ b/web/logout.html @@ -55,13 +55,16 @@
-

Reset page

+

Logout

- - + +