From 70501bc3e456ab615d4b0a0d9d3a891e8b935c49 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Wed, 19 Oct 2022 09:52:29 +0200 Subject: [PATCH] Fixes #90: Re-enables full text search --- common/spaces/disk_space_primitives.ts | 10 +++---- common/spaces/http_space_primitives.ts | 37 ++++++++----------------- plug-api/plugos-syscall/asset.ts | 4 +-- plugos/asset_bundle/base64.test.ts | 11 +++++++- plugos/asset_bundle/base64.ts | 12 ++++++++ plugos/asset_bundle/bundle.ts | 7 ++--- plugos/syscalls/fs.deno.ts | 4 +-- plugos/syscalls/fulltext.knex_sqlite.ts | 28 ++++++++----------- plugs/core/core.plug.yaml | 22 +++++++-------- plugs/core/search.ts | 36 +++++++++++++++++++----- server/deps.ts | 3 +- server/hooks/plug_space_primitives.ts | 20 +++++++++++-- server/http_server.ts | 14 ++++++++-- 13 files changed, 128 insertions(+), 80 deletions(-) diff --git a/common/spaces/disk_space_primitives.ts b/common/spaces/disk_space_primitives.ts index b16d966..15140ec 100644 --- a/common/spaces/disk_space_primitives.ts +++ b/common/spaces/disk_space_primitives.ts @@ -6,8 +6,8 @@ import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives.ts"; import { Plug } from "../../plugos/plug.ts"; import { mime } from "https://deno.land/x/mimetypes@v1.0.0/mod.ts"; import { - base64Decode, - base64Encode, + base64DecodeDataUrl, + base64EncodedDataUrl, } from "../../plugos/asset_bundle/base64.ts"; function lookupContentType(path: string): string { @@ -53,10 +53,10 @@ export class DiskSpacePrimitives implements SpacePrimitives { case "dataurl": { const f = await Deno.open(localPath, { read: true }); - const buf = base64Encode(await readAll(f)); + const buf = await readAll(f); Deno.close(f.rid); - data = `data:${contentType};base64,${buf}`; + data = base64EncodedDataUrl(contentType, buf); } break; case "arraybuffer": @@ -103,7 +103,7 @@ export class DiskSpacePrimitives implements SpacePrimitives { case "dataurl": await Deno.writeFile( localPath, - base64Decode((data as string).split(",")[1]), + base64DecodeDataUrl(data as string), ); break; case "arraybuffer": diff --git a/common/spaces/http_space_primitives.ts b/common/spaces/http_space_primitives.ts index d001807..a68966b 100644 --- a/common/spaces/http_space_primitives.ts +++ b/common/spaces/http_space_primitives.ts @@ -1,6 +1,11 @@ import { FileMeta } from "../types.ts"; import { Plug } from "../../plugos/plug.ts"; import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives.ts"; +import { + base64DecodeDataUrl, + base64EncodedDataUrl, +} from "../../plugos/asset_bundle/base64.ts"; +import { mime } from "../../plugos/deps.ts"; export class HttpSpacePrimitives implements SpacePrimitives { fsUrl: string; @@ -50,14 +55,16 @@ export class HttpSpacePrimitives implements SpacePrimitives { switch (encoding) { case "arraybuffer": { - const abBlob = await res.blob(); - data = await abBlob.arrayBuffer(); + data = await res.arrayBuffer(); + // data = await abBlob.arrayBuffer(); } break; case "dataurl": { - const dUBlob = await res.blob(); - data = arrayBufferToDataUrl(await dUBlob.arrayBuffer()); + data = base64EncodedDataUrl( + mime.getType(name) || "application/octet-stream", + new Uint8Array(await res.arrayBuffer()), + ); } break; case "string": @@ -83,7 +90,7 @@ export class HttpSpacePrimitives implements SpacePrimitives { body = data; break; case "dataurl": - data = dataUrlToArrayBuffer(data as string); + data = base64DecodeDataUrl(data as string); break; } const res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, { @@ -184,23 +191,3 @@ export class HttpSpacePrimitives implements SpacePrimitives { } } } - -function dataUrlToArrayBuffer(dataUrl: string): ArrayBuffer { - const binary_string = atob(dataUrl.split(",")[1]); - const len = binary_string.length; - const bytes = new Uint8Array(len); - for (let i = 0; i < len; i++) { - bytes[i] = binary_string.charCodeAt(i); - } - return bytes.buffer; -} - -function arrayBufferToDataUrl(buffer: ArrayBuffer): string { - let binary = ""; - const bytes = new Uint8Array(buffer); - const len = bytes.byteLength; - for (let i = 0; i < len; i++) { - binary += String.fromCharCode(bytes[i]); - } - return `data:application/octet-stream,${btoa(binary)}`; -} diff --git a/plug-api/plugos-syscall/asset.ts b/plug-api/plugos-syscall/asset.ts index f4b799e..fe15b56 100644 --- a/plug-api/plugos-syscall/asset.ts +++ b/plug-api/plugos-syscall/asset.ts @@ -1,4 +1,4 @@ -import { base64Decode } from "../../plugos/asset_bundle/base64.ts"; +import { base64DecodeDataUrl } from "../../plugos/asset_bundle/base64.ts"; import { syscall } from "./syscall.ts"; export async function readAsset( @@ -8,7 +8,7 @@ export async function readAsset( const dataUrl = await syscall("asset.readAsset", name) as string; switch (encoding) { case "utf8": - return new TextDecoder().decode(base64Decode(dataUrl.split(",")[1])); + return new TextDecoder().decode(base64DecodeDataUrl(dataUrl)); case "dataurl": return dataUrl; } diff --git a/plugos/asset_bundle/base64.test.ts b/plugos/asset_bundle/base64.test.ts index ce7e39d..9d82afc 100644 --- a/plugos/asset_bundle/base64.test.ts +++ b/plugos/asset_bundle/base64.test.ts @@ -1,5 +1,9 @@ import { assertEquals } from "../../test_deps.ts"; -import { base64Decode } from "./base64.ts"; +import { + base64Decode, + base64DecodeDataUrl, + base64EncodedDataUrl, +} from "./base64.ts"; import { base64Encode } from "./base64.ts"; Deno.test("Base 64 encoding", () => { @@ -9,4 +13,9 @@ Deno.test("Base 64 encoding", () => { buf[2] = 3; assertEquals(buf, base64Decode(base64Encode(buf))); + + assertEquals( + buf, + base64DecodeDataUrl(base64EncodedDataUrl("application/octet-stream", buf)), + ); }); diff --git a/plugos/asset_bundle/base64.ts b/plugos/asset_bundle/base64.ts index cbfb0e8..1401ede 100644 --- a/plugos/asset_bundle/base64.ts +++ b/plugos/asset_bundle/base64.ts @@ -16,3 +16,15 @@ export function base64Encode(buffer: Uint8Array): string { } return btoa(binary); } + +export function base64EncodedDataUrl( + mimeType: string, + buffer: Uint8Array, +): string { + return `data:${mimeType};base64,${base64Encode(buffer)}`; +} + +export function base64DecodeDataUrl(dataUrl: string): Uint8Array { + const b64Encoded = dataUrl.split(",", 2)[1]; + return base64Decode(b64Encoded); +} diff --git a/plugos/asset_bundle/bundle.ts b/plugos/asset_bundle/bundle.ts index a032c2f..fb615f6 100644 --- a/plugos/asset_bundle/bundle.ts +++ b/plugos/asset_bundle/bundle.ts @@ -1,4 +1,4 @@ -import { base64Decode, base64Encode } from "./base64.ts"; +import { base64Decode, base64EncodedDataUrl } from "./base64.ts"; import { mime } from "../deps.ts"; type DataUrl = string; @@ -57,9 +57,8 @@ export class AssetBundle { } writeFileSync(path: string, data: Uint8Array) { - const encoded = base64Encode(data); - const mimeType = mime.getType(path); - this.bundle[path] = `data:${mimeType};base64,${encoded}`; + const mimeType = mime.getType(path) || "application/octet-stream"; + this.bundle[path] = base64EncodedDataUrl(mimeType, data); } writeTextFileSync(path: string, s: string) { diff --git a/plugos/syscalls/fs.deno.ts b/plugos/syscalls/fs.deno.ts index eb2ad49..de197e0 100644 --- a/plugos/syscalls/fs.deno.ts +++ b/plugos/syscalls/fs.deno.ts @@ -1,6 +1,6 @@ import type { SysCallMapping } from "../system.ts"; import { mime, path } from "../deps.ts"; -import { base64Decode, base64Encode } from "../asset_bundle/base64.ts"; +import { base64DecodeDataUrl, base64Encode } from "../asset_bundle/base64.ts"; import { FileMeta } from "../../common/types.ts"; export default function fileSystemSyscalls(root = "/"): SysCallMapping { @@ -51,7 +51,7 @@ export default function fileSystemSyscalls(root = "/"): SysCallMapping { if (encoding === "utf8") { await Deno.writeTextFile(p, text); } else { - await Deno.writeFile(p, base64Decode(text.split(",")[1])); + await Deno.writeFile(p, base64DecodeDataUrl(text)); } const s = await Deno.stat(p); return { diff --git a/plugos/syscalls/fulltext.knex_sqlite.ts b/plugos/syscalls/fulltext.knex_sqlite.ts index b2e7f94..155af36 100644 --- a/plugos/syscalls/fulltext.knex_sqlite.ts +++ b/plugos/syscalls/fulltext.knex_sqlite.ts @@ -2,27 +2,22 @@ import { SQLite } from "../../server/deps.ts"; import { SysCallMapping } from "../system.ts"; import { asyncExecute, asyncQuery } from "./store.deno.ts"; -type Item = { - key: string; - value: string; -}; - export function ensureFTSTable( db: SQLite, tableName: string, ) { - // const stmt = db.prepare( - // `SELECT name FROM sqlite_master WHERE type='table' AND name=?`, - // ); - // const result = stmt.all(tableName); - // if (result.length === 0) { - // asyncExecute( - // db, - // `CREATE VIRTUAL TABLE ${tableName} USING fts5(key, value);`, - // ); + const result = db.query( + `SELECT name FROM sqlite_master WHERE type='table' AND name=?`, + [tableName], + ); + if (result.length === 0) { + asyncExecute( + db, + `CREATE VIRTUAL TABLE ${tableName} USING fts5(key, value);`, + ); - // console.log(`Created fts5 table ${tableName}`); - // } + console.log(`Created fts5 table ${tableName}`); + } return Promise.resolve(); } @@ -44,6 +39,7 @@ export function fullTextSearchSyscalls( await asyncExecute(db, `DELETE FROM ${tableName} WHERE key = ?`, key); }, "fulltext.search": async (_ctx, phrase: string, limit: number) => { + console.log("Got search query", phrase); return ( await asyncQuery( db, diff --git a/plugs/core/core.plug.yaml b/plugs/core/core.plug.yaml index c929a41..657dab8 100644 --- a/plugs/core/core.plug.yaml +++ b/plugs/core/core.plug.yaml @@ -138,15 +138,15 @@ functions: - page:complete # Full text search - # searchIndex: - # path: ./search.ts:pageIndex - # events: - # - page:index - # searchUnindex: - # path: "./search.ts:pageUnindex" - # env: server - # events: - # - page:deleted + searchIndex: + path: ./search.ts:pageIndex + events: + - page:index + searchUnindex: + path: "./search.ts:pageUnindex" + env: server + events: + - page:deleted searchQueryProvider: path: ./search.ts:queryProvider events: @@ -158,12 +158,12 @@ functions: key: Ctrl-Shift-f mac: Cmd-Shift-f readPageSearch: - path: ./search.ts:readPageSearch + path: ./search.ts:readFileSearch pageNamespace: pattern: "🔍 .+" operation: readFile getPageMetaSearch: - path: ./search.ts:getPageMetaSearch + path: ./search.ts:getFileMetaSearch pageNamespace: pattern: "🔍 .+" operation: getFileMeta diff --git a/plugs/core/search.ts b/plugs/core/search.ts index 14f1c7b..827709f 100644 --- a/plugs/core/search.ts +++ b/plugs/core/search.ts @@ -1,9 +1,18 @@ import { fulltext } from "$sb/plugos-syscall/mod.ts"; import { renderToText } from "$sb/lib/tree.ts"; -import type { PageMeta } from "../../common/types.ts"; +import type { FileMeta, PageMeta } from "../../common/types.ts"; import { editor, index } from "$sb/silverbullet-syscall/mod.ts"; import { IndexTreeEvent, QueryProviderEvent } from "$sb/app_event.ts"; import { applyQuery, removeQueries } from "$sb/lib/query.ts"; +import { + FileData, + FileEncoding, +} from "../../common/spaces/space_primitives.ts"; +import { + base64DecodeDataUrl, + base64Encode, + base64EncodedDataUrl, +} from "../../plugos/asset_bundle/base64.ts"; const searchPrefix = "🔍 "; @@ -46,16 +55,20 @@ export async function queryProvider({ } export async function searchCommand() { - const phrase = await prompt("Search for: "); + const phrase = await editor.prompt("Search for: "); if (phrase) { await editor.navigate(`${searchPrefix}${phrase}`); } } -export async function readPageSearch( +export async function readFileSearch( name: string, -): Promise<{ text: string; meta: PageMeta }> { - const phrase = name.substring(searchPrefix.length); + encoding: FileEncoding, +): Promise<{ data: FileData; meta: FileMeta }> { + const phrase = name.substring( + searchPrefix.length, + name.length - ".md".length, + ); const results = await fulltext.fullTextSearch(phrase, 100); const text = `# Search results for "${phrase}"\n${ results @@ -63,19 +76,28 @@ export async function readPageSearch( .join("\n") } `; + return { - text: text, + // encoding === "arraybuffer" is not an option, so either it's "string" or "dataurl" + data: encoding === "string" ? text : base64EncodedDataUrl( + "text/markdown", + new TextEncoder().encode(text), + ), meta: { name, + contentType: "text/markdown", + size: text.length, lastModified: 0, perm: "ro", }, }; } -export function getPageMetaSearch(name: string): PageMeta { +export function getFileMetaSearch(name: string): FileMeta { return { name, + contentType: "text/markdown", + size: -1, lastModified: 0, perm: "ro", }; diff --git a/server/deps.ts b/server/deps.ts index ef6143a..1228c7f 100644 --- a/server/deps.ts +++ b/server/deps.ts @@ -1,3 +1,4 @@ export * from "../common/deps.ts"; -export { DB as SQLite } from "https://deno.land/x/sqlite@v3.5.0/mod.ts"; +export { DB as SQLite } from "../plugos/forked/deno-sqlite/mod.ts"; export { Application, Router } from "https://deno.land/x/oak@v11.1.0/mod.ts"; +export * as etag from "https://deno.land/x/oak@v11.1.0/etag.ts"; diff --git a/server/hooks/plug_space_primitives.ts b/server/hooks/plug_space_primitives.ts index d5254f0..3ad3455 100644 --- a/server/hooks/plug_space_primitives.ts +++ b/server/hooks/plug_space_primitives.ts @@ -6,6 +6,7 @@ import { } from "../../common/spaces/space_primitives.ts"; import { FileMeta } from "../../common/types.ts"; import { NamespaceOperation, PageNamespaceHook } from "./page_namespace.ts"; +import { base64DecodeDataUrl } from "../../plugos/asset_bundle/base64.ts"; export class PlugSpacePrimitives implements SpacePrimitives { constructor( @@ -46,13 +47,26 @@ export class PlugSpacePrimitives implements SpacePrimitives { return allFiles; } - readFile( + async readFile( name: string, encoding: FileEncoding, ): Promise<{ data: FileData; meta: FileMeta }> { - const result = this.performOperation("readFile", name); + const wantArrayBuffer = encoding === "arraybuffer"; + const result: { data: FileData; meta: FileMeta } | false = await this + .performOperation( + "readFile", + name, + wantArrayBuffer ? "dataurl" : encoding, + ); if (result) { - return result; + if (wantArrayBuffer) { + return { + data: base64DecodeDataUrl(result.data as string), + meta: result.meta, + }; + } else { + return result; + } } return this.wrapped.readFile(name, encoding); } diff --git a/server/http_server.ts b/server/http_server.ts index 5a8fae7..e67f3a8 100644 --- a/server/http_server.ts +++ b/server/http_server.ts @@ -1,4 +1,4 @@ -import { Application, path, Router, SQLite } from "./deps.ts"; +import { Application, etag, path, Router, SQLite } from "./deps.ts"; import { Manifest, SilverBulletHooks } from "../common/manifest.ts"; import { loadMarkdownExtensions } from "../common/markdown_ext.ts"; import buildMarkdown from "../common/parser.ts"; @@ -15,7 +15,10 @@ import { DenoCronHook } from "../plugos/hooks/cron.deno.ts"; import { esbuildSyscalls } from "../plugos/syscalls/esbuild.ts"; import { eventSyscalls } from "../plugos/syscalls/event.ts"; import fileSystemSyscalls from "../plugos/syscalls/fs.deno.ts"; -import { fullTextSearchSyscalls } from "../plugos/syscalls/fulltext.knex_sqlite.ts"; +import { + ensureFTSTable, + fullTextSearchSyscalls, +} from "../plugos/syscalls/fulltext.knex_sqlite.ts"; import sandboxSyscalls from "../plugos/syscalls/sandbox.ts"; import shellSyscalls from "../plugos/syscalls/shell.node.ts"; import { @@ -43,6 +46,7 @@ export type ServerOptions = { }; const indexRequiredKey = "$spaceIndexed"; +const staticLastModified = new Date().toUTCString(); export class HttpServer { app: Application; @@ -198,7 +202,7 @@ export class HttpServer { async start() { await ensureIndexTable(this.db); await ensureStoreTable(this.db, "store"); - // await ensureFTSTable(this.db, "fts"); + await ensureFTSTable(this.db, "fts"); await this.ensureAndLoadSettings(); // Load plugs @@ -211,6 +215,8 @@ export class HttpServer { response.body = this.assetBundle.readTextFileSync( "web/index.html", ); + response.headers.set("Last-Modified", staticLastModified); + response.headers.set("ETag", await etag.calculate(response.body)); return; } try { @@ -224,6 +230,8 @@ export class HttpServer { assetName, ); response.headers.set("Content-length", "" + data.length); + response.headers.set("Last-Modified", staticLastModified); + response.headers.set("ETag", await etag.calculate(data)); if (request.method === "GET") { response.body = data;