Refactoring

.gitignore

@@ -13,5 +13,3 @@ node_modules
 *.db
 test_space
 silverbullet
-# Local Netlify folder
-.netlify

cli/syscalls/space.ts

@@ -1,3 +1,4 @@
+import { FileMeta } from "$sb/types.ts";
 import { SysCallMapping } from "../../plugos/system.ts";
 import type { Space } from "../../web/space.ts";
 import { AttachmentMeta, PageMeta } from "../../web/types.ts";
@@ -57,5 +58,26 @@ export function spaceSyscalls(space: Space): SysCallMapping {
     "space.deleteAttachment": async (_ctx, name: string) => {
       await space.deleteAttachment(name);
     },
+
+    // FS
+    "space.listFiles": (): Promise<FileMeta[]> => {
+      return space.spacePrimitives.fetchFileList();
+    },
+    "space.getFileMeta": (_ctx, name: string): Promise<FileMeta> => {
+      return space.spacePrimitives.getFileMeta(name);
+    },
+    "space.readFile": async (_ctx, name: string): Promise<Uint8Array> => {
+      return (await space.spacePrimitives.readFile(name)).data;
+    },
+    "space.writeFile": (
+      _ctx,
+      name: string,
+      data: Uint8Array,
+    ): Promise<FileMeta> => {
+      return space.spacePrimitives.writeFile(name, data);
+    },
+    "space.deleteFile": (_ctx, name: string) => {
+      return space.spacePrimitives.deleteFile(name);
+    },
   };
 }

common/deps.ts

@@ -18,7 +18,7 @@ export { styleTags, Tag, tagHighlighter, tags } from "@lezer/highlight";
 export * as YAML from "https://deno.land/std@0.189.0/yaml/mod.ts";
 export * as path from "https://deno.land/std@0.189.0/path/mod.ts";
 
-export { readAll } from "https://deno.land/std@0.165.0/streams/conversion.ts";
+// export { readAll } from "https://deno.land/std@0.165.0/streams/conversion.ts";
 
 export type {
   BlockContext,

common/spaces/asset_bundle_space_primitives.ts

@@ -1,6 +1,6 @@
-import { FileMeta } from "../types.ts";
 import { SpacePrimitives } from "./space_primitives.ts";
 import { AssetBundle } from "../../plugos/asset_bundle/bundle.ts";
+import { FileMeta } from "$sb/types.ts";
 
 export class AssetBundlePlugSpacePrimitives implements SpacePrimitives {
   constructor(

common/spaces/deno_kv_space_primitives.test.ts

@@ -0,0 +1,31 @@
+import { assertEquals } from "../../test_deps.ts";
+import { DenoKVSpacePrimitives } from "./deno_kv_space_primitives.ts";
+
+Deno.test("deno_kv_space_primitives", async () => {
+  const tempFile = await Deno.makeTempFile({ suffix: ".db" });
+  const spacePrimitives = new DenoKVSpacePrimitives();
+  await spacePrimitives.init(tempFile);
+  await spacePrimitives.writeFile("test.txt", new TextEncoder().encode("test"));
+  let result = await spacePrimitives.readFile("test.txt");
+  assertEquals(result.data, new TextEncoder().encode("test"));
+  let listing = await spacePrimitives.fetchFileList();
+  assertEquals(listing.length, 1);
+  await spacePrimitives.writeFile(
+    "test.txt",
+    new TextEncoder().encode("test2"),
+  );
+  result = await spacePrimitives.readFile("test.txt");
+  assertEquals(result.data, new TextEncoder().encode("test2"));
+  await spacePrimitives.deleteFile("test.txt");
+  listing = await spacePrimitives.fetchFileList();
+  try {
+    await spacePrimitives.readFile("test.txt");
+    throw new Error("Should not be here");
+  } catch (e: any) {
+    assertEquals(e.message, "Not found");
+  }
+  assertEquals(listing.length, 0);
+
+  spacePrimitives.close();
+  await Deno.remove(tempFile);
+});

common/spaces/deno_kv_space_primitives.ts

@@ -0,0 +1,83 @@
+/// <reference lib="deno.unstable" />
+
+import { FileMeta } from "$sb/types.ts";
+import type { SpacePrimitives } from "./space_primitives.ts";
+import { mime } from "https://deno.land/x/mimetypes@v1.0.0/mod.ts";
+
+export class DenoKVSpacePrimitives implements SpacePrimitives {
+  private kv!: Deno.Kv;
+  private dataAttribute = "file";
+  private metaAttribute = "meta";
+
+  async init(path?: string) {
+    this.kv = await Deno.openKv(path);
+  }
+
+  close() {
+    this.kv.close();
+  }
+
+  async fetchFileList(): Promise<FileMeta[]> {
+    const results: FileMeta[] = [];
+    for await (
+      const result of this.kv.list({
+        prefix: [this.metaAttribute],
+      })
+    ) {
+      results.push(result.value as FileMeta);
+    }
+    return results;
+  }
+  async readFile(name: string): Promise<{ data: Uint8Array; meta: FileMeta }> {
+    const [meta, data] = await this.kv.getMany([[this.metaAttribute, name], [
+      this.dataAttribute,
+      name,
+    ]]);
+    if (!meta.value) {
+      throw new Error("Not found");
+    }
+    return {
+      data: data.value as Uint8Array,
+      meta: meta.value as FileMeta,
+    };
+  }
+  async getFileMeta(name: string): Promise<FileMeta> {
+    const result = await this.kv.get([this.metaAttribute, name]);
+    if (result.value) {
+      return result.value as FileMeta;
+    } else {
+      throw new Error("Not found");
+    }
+  }
+  async writeFile(
+    name: string,
+    data: Uint8Array,
+    _selfUpdate?: boolean | undefined,
+    suggestedMeta?: FileMeta | undefined,
+  ): Promise<FileMeta> {
+    const meta: FileMeta = {
+      name,
+      lastModified: suggestedMeta?.lastModified || Date.now(),
+      contentType: mime.getType(name) || "application/octet-stream",
+      size: data.byteLength,
+      perm: suggestedMeta?.perm || "rw",
+    };
+    const res = await this.kv.atomic()
+      .set([this.dataAttribute, name], data)
+      .set([this.metaAttribute, name], meta)
+      .commit();
+    if (!res.ok) {
+      throw res;
+    }
+    return meta;
+  }
+  async deleteFile(name: string): Promise<void> {
+    const res = await this.kv.atomic()
+      .delete([this.dataAttribute, name])
+      .delete([this.metaAttribute, name])
+      .commit();
+    if (!res.ok) {
+      throw res;
+    }
+  }
+}

common/spaces/disk_space_primitives.ts

@@ -1,10 +1,9 @@
-// import { mkdir, readdir, readFile, stat, unlink, writeFile } from "fs/promises";
+import * as path from "https://deno.land/std@0.189.0/path/mod.ts";
-import { path } from "../deps.ts";
+import { readAll } from "https://deno.land/std@0.165.0/streams/conversion.ts";
-import { readAll } from "../deps.ts";
-import { FileMeta } from "../types.ts";
 import { SpacePrimitives } from "./space_primitives.ts";
 import { mime } from "https://deno.land/x/mimetypes@v1.0.0/mod.ts";
 import { walk } from "https://deno.land/std@0.198.0/fs/walk.ts";
+import { FileMeta } from "$sb/types.ts";
 
 function lookupContentType(path: string): string {
   return mime.getType(path) || "application/octet-stream";

common/spaces/evented_space_primitives.ts

@@ -1,6 +1,6 @@
+import { FileMeta } from "$sb/types.ts";
 import { EventHook } from "../../plugos/hooks/event.ts";
 
-import { FileMeta } from "../types.ts";
 import type { SpacePrimitives } from "./space_primitives.ts";
 
 export class EventedSpacePrimitives implements SpacePrimitives {

common/spaces/fallback_space_primitives.ts

@@ -1,4 +1,4 @@
-import { FileMeta } from "../types.ts";
+import { FileMeta } from "$sb/types.ts";
 import type { SpacePrimitives } from "./space_primitives.ts";
 
 /**

common/spaces/file_meta_space_primitives.ts

@@ -1,6 +1,6 @@
-import { FileMeta } from "../types.ts";
 import { SpacePrimitives } from "./space_primitives.ts";
 import type { SysCallMapping } from "../../plugos/system.ts";
+import { FileMeta } from "$sb/types.ts";
 
 // Enriches the file list listing with custom metadata from the page index
 export class FileMetaSpacePrimitives implements SpacePrimitives {

common/spaces/filtered_space_primitives.ts

@@ -1,4 +1,4 @@
-import { FileMeta } from "../types.ts";
+import { FileMeta } from "$sb/types.ts";
 import { SpacePrimitives } from "./space_primitives.ts";
 
 export class FilteredSpacePrimitives implements SpacePrimitives {

common/spaces/http_space_primitives.ts

@@ -1,6 +1,6 @@
-import { FileMeta } from "../types.ts";
 import { SpacePrimitives } from "./space_primitives.ts";
 import { flushCachesAndUnregisterServiceWorker } from "../sw_util.ts";
+import { FileMeta } from "$sb/types.ts";
 
 export class HttpSpacePrimitives implements SpacePrimitives {
   constructor(

common/spaces/indexeddb_space_primitives.ts

@@ -1,7 +1,7 @@
-import type { FileMeta } from "../types.ts";
 import type { SpacePrimitives } from "./space_primitives.ts";
 import Dexie, { Table } from "dexie";
 import { mime } from "../deps.ts";
+import { FileMeta } from "$sb/types.ts";
 
 export type FileContent = {
   name: string;

common/spaces/plug_space_primitives.ts

@@ -1,5 +1,5 @@
 import { SpacePrimitives } from "../../common/spaces/space_primitives.ts";
-import { FileMeta } from "../../common/types.ts";
+import { FileMeta } from "$sb/types.ts";
 import {
   NamespaceOperation,
   PlugNamespaceHook,

common/spaces/space_primitives.ts

@@ -1,8 +1,8 @@
-import type { FileMeta } from "../types.ts";
-
 // export type FileEncoding = "utf8" | "arraybuffer" | "dataurl";
 // export type FileData = ArrayBuffer | string;
 
+import { FileMeta } from "$sb/types.ts";
+
 export interface SpacePrimitives {
   // Returns a list of file meta data as well as the timestamp of this snapshot
   fetchFileList(): Promise<FileMeta[]>;

common/spaces/sync.ts

@@ -1,9 +1,9 @@
 import { renderToText, replaceNodesMatching } from "../../plug-api/lib/tree.ts";
 import buildMarkdown from "../markdown_parser/parser.ts";
 import { parse } from "../markdown_parser/parse_tree.ts";
-import type { FileMeta } from "../types.ts";
 import { SpacePrimitives } from "./space_primitives.ts";
 import { EventEmitter } from "../../plugos/event.ts";
+import { FileMeta } from "$sb/types.ts";
 
 type SyncHash = number;
 

common/types.ts

@@ -1,10 +0,0 @@
-export const maximumAttachmentSize = 20 * 1024 * 1024; // 10 MB
-
-export type FileMeta = {
-  name: string;
-  lastModified: number;
-  contentType: string;
-  size: number;
-  perm: "ro" | "rw";
-  noSync?: boolean;
-} & Record<string, any>;

plug-api/lib/secrets_page.ts

@@ -5,10 +5,10 @@ import { readYamlPage } from "./yaml_page.ts";
 // of not decising this SECRETS page to other places
 export async function readSecrets(keys: string[]): Promise<any[]> {
   try {
-    let allSecrets = await readYamlPage("SECRETS", ["yaml", "secrets"]);
+    const allSecrets = await readYamlPage("SECRETS", ["yaml", "secrets"]);
-    let collectedSecrets: any[] = [];
+    const collectedSecrets: any[] = [];
-    for (let key of keys) {
+    for (const key of keys) {
-      let secret = allSecrets[key];
+      const secret = allSecrets[key];
       if (secret) {
         collectedSecrets.push(secret);
       } else {

plug-api/silverbullet-syscall/space.ts

@@ -1,5 +1,6 @@
 import { syscall } from "./syscall.ts";
 import type { AttachmentMeta, PageMeta } from "../../web/types.ts";
+import { FileMeta } from "$sb/types.ts";
 
 export function listPages(unfiltered = false): Promise<PageMeta[]> {
   return syscall("space.listPages", unfiltered);
@@ -27,7 +28,7 @@ export function listPlugs(): Promise<string[]> {
   return syscall("space.listPlugs");
 }
 
-export function listAttachments(): Promise<PageMeta[]> {
+export function listAttachments(): Promise<AttachmentMeta[]> {
   return syscall("space.listAttachments");
 }
 
@@ -66,3 +67,27 @@ export function writeAttachment(
 export function deleteAttachment(name: string): Promise<void> {
   return syscall("space.deleteAttachment", name);
 }
+
+// FS
+export function listFiles(): Promise<FileMeta[]> {
+  return syscall("space.listFiles");
+}
+
+export function readFile(name: string): Promise<Uint8Array> {
+  return syscall("space.readFile", name);
+}
+
+export function getFileMeta(name: string): Promise<FileMeta> {
+  return syscall("space.getFileMeta", name);
+}
+
+export function writeFile(
+  name: string,
+  data: Uint8Array,
+): Promise<FileMeta> {
+  return syscall("space.writeFile", name, data);
+}
+
+export function deleteFile(name: string): Promise<void> {
+  return syscall("space.deleteFile", name);
+}

plug-api/types.ts

@@ -10,3 +10,12 @@ export type QueueStats = {
   processing: number;
   dlq: number;
 };
+
+export type FileMeta = {
+  name: string;
+  lastModified: number;
+  contentType: string;
+  size: number;
+  perm: "ro" | "rw";
+  noSync?: boolean;
+} & Record<string, any>;

plugos/syscalls/fs.deno.test.ts

@@ -1,5 +1,5 @@
+import { FileMeta } from "$sb/types.ts";
 import { assert } from "../../test_deps.ts";
-import { FileMeta } from "../../common/types.ts";
 import { path } from "../deps.ts";
 import fileSystemSyscalls from "./fs.deno.ts";
 

plugos/syscalls/fs.deno.ts

@@ -1,7 +1,7 @@
 import type { SysCallMapping } from "../system.ts";
 import { mime, path, walk } from "../deps.ts";
 import { base64DecodeDataUrl, base64Encode } from "../asset_bundle/base64.ts";
-import { FileMeta } from "../../common/types.ts";
+import { FileMeta } from "$sb/types.ts";
 
 export default function fileSystemSyscalls(root = "/"): SysCallMapping {
   function resolvedPath(p: string): string {

plugs/core/cloud.ts

@@ -1,6 +1,6 @@
 import { renderToText, replaceNodesMatching } from "$sb/lib/tree.ts";
-import type { FileMeta } from "../../common/types.ts";
 import { parseMarkdown } from "$sb/silverbullet-syscall/markdown.ts";
+import { FileMeta } from "$sb/types.ts";
 
 export const cloudPrefix = "💭 ";
 

plugs/federation/federation.ts

@@ -1,8 +1,8 @@
 import "$sb/lib/fetch.ts";
-import type { FileMeta } from "../../common/types.ts";
 import { federatedPathToUrl } from "$sb/lib/resolve.ts";
 import { readFederationConfigs } from "./config.ts";
 import { store } from "$sb/plugos-syscall/mod.ts";
+import { FileMeta } from "$sb/types.ts";
 
 async function responseToFileMeta(
   r: Response,

plugs/search/search.ts

@@ -3,9 +3,8 @@ import { renderToText } from "$sb/lib/tree.ts";
 import { store } from "$sb/plugos-syscall/mod.ts";
 import { applyQuery } from "$sb/lib/query.ts";
 import { editor, index } from "$sb/silverbullet-syscall/mod.ts";
-import { base64EncodedDataUrl } from "../../plugos/asset_bundle/base64.ts";
 import { BatchKVStore, SimpleSearchEngine } from "./engine.ts";
-import { FileMeta } from "../../common/types.ts";
+import { FileMeta } from "$sb/types.ts";
 
 const searchPrefix = "🔍 ";
 

scripts/generate_fs_list.ts

@@ -1,8 +1,7 @@
-import type { FileMeta } from "../common/types.ts";
-
 import { walk } from "https://deno.land/std@0.165.0/fs/mod.ts";
 import { resolve } from "https://deno.land/std@0.165.0/path/mod.ts";
 import { mime } from "https://deno.land/x/mimetypes@v1.0.0/mod.ts";
+import { FileMeta } from "$sb/types.ts";
 
 const rootDir = resolve("website_build");
 

server/http_server.ts

@@ -7,7 +7,7 @@ import { BuiltinSettings } from "../web/types.ts";
 import { gitIgnoreCompiler } from "./deps.ts";
 import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts";
 import { Authenticator } from "./auth.ts";
-import { FileMeta } from "../common/types.ts";
+import { FileMeta } from "$sb/types.ts";
 
 export type ServerOptions = {
   hostname: string;

server/spaces/s3_space_primitives.ts

@@ -2,8 +2,8 @@
 import { S3Client } from "https://deno.land/x/s3_lite_client@0.4.0/mod.ts";
 import type { ClientOptions } from "https://deno.land/x/s3_lite_client@0.4.0/client.ts";
 import { SpacePrimitives } from "../../common/spaces/space_primitives.ts";
-import { FileMeta } from "../../common/types.ts";
 import { mime } from "../deps.ts";
+import { FileMeta } from "$sb/types.ts";
 
 export class S3SpacePrimitives implements SpacePrimitives {
   client: S3Client;

silverbullet.ts

@@ -1,4 +1,5 @@
 import.meta.main = false;
+// import { Command } from "https://deno.land/x/cliffy@v1.0.0-rc.3/command/command.ts";
 import { Command } from "https://deno.land/x/cliffy@v0.25.2/command/command.ts";
 
 import { version } from "./version.ts";
@@ -30,7 +31,7 @@ await new Command()
   .option("-p, --port <port:number>", "Port to listen on")
   .option(
     "--user <user:string>",
-    "'username:password' combo for BasicAuth authentication",
+    "'username:password' combo for authentication",
   )
   .option(
     "--auth <auth.json:string>",

web/cm_plugins/editor_paste.ts

@@ -1,5 +1,4 @@
 import { EditorView, syntaxTree, ViewPlugin, ViewUpdate } from "../deps.ts";
-import { maximumAttachmentSize } from "../../common/types.ts";
 import { Client } from "../client.ts";
 
 // We use turndown to convert HTML to Markdown
@@ -18,6 +17,7 @@ import {
   nodeAtPos,
 } from "../../plug-api/lib/tree.ts";
 import { folderName, resolve } from "../../common/path.ts";
+import { maximumAttachmentSize } from "../constants.ts";
 
 const turndownService = new TurndownService({
   hr: "---",

web/constants.ts

@@ -0,0 +1 @@
+export const maximumAttachmentSize = 1024 * 1024 * 10; // 10MB

web/service_worker.ts

@@ -2,7 +2,7 @@ import Dexie from "https://esm.sh/v120/dexie@3.2.2/dist/dexie.js";
 
 import type { FileContent } from "../common/spaces/indexeddb_space_primitives.ts";
 import { simpleHash } from "../common/crypto.ts";
-import type { FileMeta } from "../common/types.ts";
+import { FileMeta } from "$sb/types.ts";
 
 const CACHE_NAME = "{{CACHE_NAME}}";
 

web/space.ts

@@ -1,11 +1,11 @@
 import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
-import { FileMeta } from "../common/types.ts";
 import { EventEmitter } from "../plugos/event.ts";
 import { plugPrefix } from "../common/spaces/constants.ts";
 import { safeRun } from "../common/util.ts";
 import { AttachmentMeta, PageMeta } from "./types.ts";
 import { throttle } from "../common/async_util.ts";
 import { KVStore } from "../plugos/lib/kv_store.ts";
+import { FileMeta } from "$sb/types.ts";
 
 export type SpaceEvents = {
   pageCreated: (meta: PageMeta) => void;

web/syscalls/space.ts

@@ -1,6 +1,7 @@
 import { Client } from "../client.ts";
 import { SysCallMapping } from "../../plugos/system.ts";
 import { AttachmentMeta, PageMeta } from "../types.ts";
+import { FileMeta } from "$sb/types.ts";
 
 export function spaceSyscalls(editor: Client): SysCallMapping {
   return {
@@ -61,5 +62,26 @@ export function spaceSyscalls(editor: Client): SysCallMapping {
     "space.deleteAttachment": async (_ctx, name: string) => {
       await editor.space.deleteAttachment(name);
     },
+
+    // FS
+    "space.listFiles": (): Promise<FileMeta[]> => {
+      return editor.space.spacePrimitives.fetchFileList();
+    },
+    "space.getFileMeta": (_ctx, name: string): Promise<FileMeta> => {
+      return editor.space.spacePrimitives.getFileMeta(name);
+    },
+    "space.readFile": async (_ctx, name: string): Promise<Uint8Array> => {
+      return (await editor.space.spacePrimitives.readFile(name)).data;
+    },
+    "space.writeFile": (
+      _ctx,
+      name: string,
+      data: Uint8Array,
+    ): Promise<FileMeta> => {
+      return editor.space.spacePrimitives.writeFile(name, data);
+    },
+    "space.deleteFile": (_ctx, name: string) => {
+      return editor.space.spacePrimitives.deleteFile(name);
+    },
   };
 }