Initial search implementation

package-lock.json

@@ -4692,6 +4692,11 @@
       "version": "1.1.1",
       "license": "MIT"
     },
+    "node_modules/fuzzysort": {
+      "version": "1.9.0",
+      "resolved": "https://registry.npmjs.org/fuzzysort/-/fuzzysort-1.9.0.tgz",
+      "integrity": "sha512-MOxCT0qLTwLqmEwc7UtU045RKef7mc8Qz8eR4r2bLNEq9dy/c3ZKMEFp6IEst69otkQdFZ4FfgH2dmZD+ddX1g=="
+    },
     "node_modules/gauge": {
       "version": "2.7.4",
       "license": "ISC",
@@ -9315,7 +9320,6 @@
     },
     "node_modules/typescript": {
       "version": "4.6.3",
-      "devOptional": true,
       "license": "Apache-2.0",
       "bin": {
         "tsc": "bin/tsc",
@@ -9891,6 +9895,7 @@
         "node-fetch": "2",
         "node-watch": "^0.7.3",
         "supertest": "^6.2.2",
+        "typescript": "^4.6.2",
         "vm2": "^3.9.9",
         "ws": "^8.5.0",
         "yaml": "^1.10.2",
@@ -9920,8 +9925,7 @@
         "assert": "^2.0.0",
         "events": "^3.3.0",
         "parcel": "2.3.2",
-        "prettier": "^2.5.1",
+        "prettier": "^2.5.1"
-        "typescript": "^4.6.2"
       }
     },
     "packages/plugos-silverbullet-syscall": {
@@ -10042,6 +10046,7 @@
         "@jest/globals": "^27.5.1",
         "@lezer/markdown": "^0.15.0",
         "fake-indexeddb": "^3.1.7",
+        "fuzzysort": "^1.9.0",
         "jest": "^27.5.1",
         "knex": "^1.0.4",
         "react": "^17.0.2",
@@ -11645,6 +11650,7 @@
         "@types/react-dom": "^17.0.11",
         "assert": "^2.0.0",
         "fake-indexeddb": "^3.1.7",
+        "fuzzysort": "^1.9.0",
         "jest": "^27.5.1",
         "knex": "^1.0.4",
         "parcel": "2.3.2",
@@ -13189,6 +13195,11 @@
     "function-bind": {
       "version": "1.1.1"
     },
+    "fuzzysort": {
+      "version": "1.9.0",
+      "resolved": "https://registry.npmjs.org/fuzzysort/-/fuzzysort-1.9.0.tgz",
+      "integrity": "sha512-MOxCT0qLTwLqmEwc7UtU045RKef7mc8Qz8eR4r2bLNEq9dy/c3ZKMEFp6IEst69otkQdFZ4FfgH2dmZD+ddX1g=="
+    },
     "gauge": {
       "version": "2.7.4",
       "requires": {
@@ -15955,8 +15966,7 @@
       }
     },
     "typescript": {
-      "version": "4.6.3",
+      "version": "4.6.3"
-      "devOptional": true
     },
     "typeson": {
       "version": "6.1.0"

packages/plugos-syscall/fulltext.ts

@@ -0,0 +1,13 @@
+import { syscall } from "./syscall";
+
+export async function fullTextIndex(key: string, value: string) {
+  return syscall("fulltext.index", key, value);
+}
+
+export async function fullTextDelete(key: string) {
+  return syscall("fulltext.index", key);
+}
+
+export async function fullTextSearch(phrase: string, limit: number = 100) {
+  return syscall("fulltext.search", phrase, limit);
+}

packages/plugos/syscalls/fulltext.knex_sqlite.ts

@@ -0,0 +1,42 @@
+import { Knex } from "knex";
+import { SysCallMapping } from "../system";
+
+type Item = {
+  key: string;
+  value: string;
+};
+
+export async function ensureFTSTable(
+  db: Knex<any, unknown>,
+  tableName: string
+) {
+  if (!(await db.schema.hasTable(tableName))) {
+    await db.raw(`CREATE VIRTUAL TABLE ${tableName} USING fts5(key, value);`);
+
+    console.log(`Created fts5 table ${tableName}`);
+  }
+}
+
+export function fullTextSearchSyscalls(
+  db: Knex<any, unknown>,
+  tableName: string
+): SysCallMapping {
+  return {
+    "fulltext.index": async (ctx, key: string, value: string) => {
+      await db<Item>(tableName).where({ key }).del();
+      await db<Item>(tableName).insert({ key, value });
+    },
+    "fulltext.delete": async (ctx, key: string) => {
+      await db<Item>(tableName).where({ key }).del();
+    },
+    "fulltext.search": async (ctx, phrase: string, limit: number) => {
+      return (
+        await db<any>(tableName)
+          .whereRaw(`value MATCH ?`, [phrase])
+          .select(["key", "rank"])
+          .orderBy("rank")
+          .limit(limit)
+      ).map((item) => ({ name: item.key, rank: item.rank }));
+    },
+  };
+}

packages/plugos/syscalls/store.knex_node.ts

@@ -19,6 +19,7 @@ export async function ensureTable(db: Knex<any, unknown>, tableName: string) {
       table.text("value");
       table.primary(["key"]);
     });
+
     console.log(`Created table ${tableName}`);
   }
 }

packages/plugs/core/template.ts

@@ -72,7 +72,6 @@ export async function quickNoteCommand() {
   let [date, time] = isoDate.split("T");
   time = time.split(".")[0];
   let pageName = `📥 ${date} ${time}`;
-  await writePage(pageName, "");
   await navigate(pageName);
 }
 

packages/plugs/search/search.plug.yaml

@@ -0,0 +1,10 @@
+name: search
+functions:
+  index:
+    path: ./search.ts:index
+    events:
+      - page:index
+  queryProvider:
+    path: ./search.ts:queryProvider
+    events:
+      - query:full-text

packages/plugs/search/search.ts

@@ -0,0 +1,38 @@
+import { fullTextIndex, fullTextSearch } from "@plugos/plugos-syscall/fulltext";
+import { renderToText } from "@silverbulletmd/common/tree";
+import { scanPrefixGlobal } from "@silverbulletmd/plugos-silverbullet-syscall";
+import { IndexTreeEvent } from "@silverbulletmd/web/app_event";
+import { applyQuery, QueryProviderEvent } from "../query/engine";
+import { removeQueries } from "../query/util";
+
+export async function index(data: IndexTreeEvent) {
+  removeQueries(data.tree);
+  let cleanText = renderToText(data.tree);
+  await fullTextIndex(data.name, cleanText);
+}
+
+export async function queryProvider({
+  query,
+}: QueryProviderEvent): Promise<any[]> {
+  let phraseFilter = query.filter.find((f) => f.prop === "phrase");
+  if (!phraseFilter) {
+    throw Error("No 'phrase' filter specified, this is mandatory");
+  }
+  let results = await fullTextSearch(phraseFilter.value, 100);
+
+  let allPageMap: Map<string, any> = new Map(results.map((r) => [r.name, r]));
+  for (let { page, value } of await scanPrefixGlobal("meta:")) {
+    let p = allPageMap.get(page);
+    if (p) {
+      for (let [k, v] of Object.entries(value)) {
+        p[k] = v;
+      }
+    }
+  }
+
+  // Remove the "phrase" filter
+  query.filter.splice(query.filter.indexOf(phraseFilter), 1);
+
+  results = applyQuery(query, results);
+  return results;
+}

packages/server/_in.ts

@@ -1 +0,0 @@
-import {pullDataCommand} from ".//var/folders/s2/4nqrw2192hngtxg672qzc0nr0000gn/T/plugos-0.8739407042390945/file.js";export default pullDataCommand;

packages/server/express_server.ts

@@ -32,6 +32,10 @@ import sandboxSyscalls from "@plugos/plugos/syscalls/sandbox";
 import globalModules from "../common/dist/global.plug.json";
 
 import { safeRun } from "./util";
+import {
+  ensureFTSTable,
+  fullTextSearchSyscalls,
+} from "@plugos/plugos/syscalls/fulltext.knex_sqlite";
 
 const safeFilename = /^[a-zA-Z0-9_\-\.]+$/;
 
@@ -86,6 +90,7 @@ export class ExpressServer {
     this.system.registerSyscalls(
       [],
       pageIndexSyscalls(this.db),
+      fullTextSearchSyscalls(this.db, "fts"),
       spaceSyscalls(this.space),
       eventSyscalls(this.eventHook),
       markdownSyscalls(buildMarkdown([])),
@@ -196,6 +201,7 @@ export class ExpressServer {
         };
 
     await ensurePageIndexTable(this.db);
+    await ensureFTSTable(this.db, "fts");
     console.log("Setting up router");
 
     let auth = new Authenticator(this.db);

packages/web/components/command_palette.tsx

@@ -6,9 +6,11 @@ import { FilterOption } from "@silverbulletmd/common/types";
 
 export function CommandPalette({
   commands,
+  recentCommands,
   onTrigger,
 }: {
   commands: Map<string, AppCommand>;
+  recentCommands: Map<string, Date>;
   onTrigger: (command: AppCommand | undefined) => void;
 }) {
   let options: FilterOption[] = [];
@@ -17,6 +19,9 @@ export function CommandPalette({
     options.push({
       name: name,
       hint: isMac && def.command.mac ? def.command.mac : def.command.key,
+      orderId: recentCommands.has(name)
+        ? -recentCommands.get(name)!.getTime()
+        : 0,
     });
   }
   return (

packages/web/components/filter.tsx

@@ -2,49 +2,24 @@ import React, { useEffect, useRef, useState } from "react";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import { IconDefinition } from "@fortawesome/free-solid-svg-icons";
 import { FilterOption } from "@silverbulletmd/common/types";
+import fuzzysort from "fuzzysort";
 
 function magicSorter(a: FilterOption, b: FilterOption): number {
   if (a.orderId && b.orderId) {
     return a.orderId < b.orderId ? -1 : 1;
   }
-  return a.name.toLowerCase() < b.name.toLowerCase() ? -1 : 1;
+  if (a.orderId) {
+    return -1;
+  }
+  if (b.orderId) {
+    return 1;
+  }
+  return 0;
 }
 
-function escapeRegExp(str: string): string {
+type FilterResult = FilterOption & {
-  return str.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&");
+  result?: any;
-}
+};
-
-function fuzzyFilter(pattern: string, options: FilterOption[]): FilterOption[] {
-  let closeMatchRegex = escapeRegExp(pattern);
-  closeMatchRegex = closeMatchRegex.split(/\s+/).join(".*?");
-  closeMatchRegex = closeMatchRegex.replace(/\\\//g, ".*?\\/.*?");
-  const distantMatchRegex = escapeRegExp(pattern).split("").join(".*?");
-  const r1 = new RegExp(closeMatchRegex, "i");
-  const r2 = new RegExp(distantMatchRegex, "i");
-  let matches = [];
-  if (!pattern) {
-    return options;
-  }
-  for (let option of options) {
-    let m = r1.exec(option.name);
-    if (m) {
-      matches.push({
-        ...option,
-        orderId: 100000 - (options.length - m[0].length - m.index),
-      });
-    } else {
-      // Let's try the distant matcher
-      var m2 = r2.exec(option.name);
-      if (m2) {
-        matches.push({
-          ...option,
-          orderId: 10000 - (options.length - m2[0].length - m2.index),
-        });
-      }
-    }
-  }
-  return matches;
-}
 
 function simpleFilter(
   pattern: string,
@@ -56,6 +31,25 @@ function simpleFilter(
   });
 }
 
+function escapeHtml(unsafe: string): string {
+  return unsafe
+    .replaceAll("&", "&amp;")
+    .replaceAll("<", "&lt;")
+    .replaceAll(">", "&gt;")
+    .replaceAll('"', "&quot;")
+    .replaceAll("'", "&#039;");
+}
+
+function fuzzySorter(pattern: string, options: FilterOption[]): FilterResult[] {
+  return fuzzysort
+    .go(pattern, options, {
+      all: true,
+      key: "name",
+    })
+    .map((result) => ({ ...result.obj, result: result }))
+    .sort(magicSorter);
+}
+
 export function FilterList({
   placeholder,
   options,
@@ -82,7 +76,7 @@ export function FilterList({
   const searchBoxRef = useRef<HTMLInputElement>(null);
   const [text, setText] = useState("");
   const [matchingOptions, setMatchingOptions] = useState(
-    options.sort(magicSorter)
+    fuzzySorter("", options)
   );
   const [selectedOption, setSelectionOption] = useState(0);
 
@@ -93,12 +87,8 @@ export function FilterList({
   }
 
   function updateFilter(originalPhrase: string) {
-    const searchPhrase = originalPhrase.toLowerCase();
-
-    if (searchPhrase) {
     let foundExactMatch = false;
-      let results = simpleFilter(searchPhrase, options);
+    let results = fuzzySorter(originalPhrase, options);
-      results = results.sort(magicSorter);
     if (allowNew && !foundExactMatch) {
       results.push({
         name: originalPhrase,
@@ -106,10 +96,6 @@ export function FilterList({
       });
     }
     setMatchingOptions(results);
-    } else {
-      let results = options.sort(magicSorter);
-      setMatchingOptions(results);
-    }
 
     setText(originalPhrase);
     setSelectionOption(0);
@@ -201,7 +187,14 @@ export function FilterList({
                 <span className="icon">
                   {icon && <FontAwesomeIcon icon={icon} />}
                 </span>
-                <span className="name">{option.name}</span>
+                <span
+                  className="name"
+                  dangerouslySetInnerHTML={{
+                    __html: option?.result?.indexes
+                      ? fuzzysort.highlight(option.result, "<b>", "</b>")!
+                      : escapeHtml(option.name),
+                  }}
+                ></span>
                 {option.hint && <span className="hint">{option.hint}</span>}
               </div>
             ))

packages/web/editor.tsx

@@ -602,12 +602,14 @@ export class Editor {
               dispatch({ type: "hide-palette" });
               editor!.focus();
               if (cmd) {
+                dispatch({ type: "command-run", command: cmd.command.name });
                 cmd.run().catch((e) => {
                   console.error("Error running command", e.message);
                 });
               }
             }}
             commands={viewState.commands}
+            recentCommands={viewState.recentCommands}
           />
         )}
         {viewState.showFilterBox && (

packages/web/editor_paste.ts

@@ -1,8 +1,10 @@
 import { ViewPlugin, ViewUpdate } from "@codemirror/view";
+import { createImportSpecifier } from "typescript";
 
 const urlRegexp =
   /^https?:\/\/[-a-zA-Z0-9@:%._\+~#=]{1,256}([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/;
 
+// Known iOS Safari paste issue (unrelated to this implementation): https://voxpelli.com/2015/03/ios-safari-url-copy-paste-bug/
 export const pasteLinkExtension = ViewPlugin.fromClass(
   class {
     update(update: ViewUpdate): void {
@@ -19,6 +21,7 @@ export const pasteLinkExtension = ViewPlugin.fromClass(
           let pastedString = pastedText.join("");
           if (pastedString.match(urlRegexp)) {
             let selection = update.startState.selection.main;
+            console.log("It's a URL and selection empty?", selection.empty);
             if (!selection.empty) {
               setTimeout(() => {
                 update.view.dispatch({

packages/web/package.json

@@ -48,6 +48,7 @@
     "@jest/globals": "^27.5.1",
     "@lezer/markdown": "^0.15.0",
     "fake-indexeddb": "^3.1.7",
+    "fuzzysort": "^1.9.0",
     "jest": "^27.5.1",
     "knex": "^1.0.4",
     "react": "^17.0.2",

packages/web/reducer.ts

@@ -66,6 +66,11 @@ export default function reducer(
         ...state,
         showCommandPalette: false,
       };
+    case "command-run":
+      return {
+        ...state,
+        recentCommands: state.recentCommands.set(action.command, new Date()),
+      };
     case "update-commands":
       return {
         ...state,

packages/web/types.ts

@@ -1,4 +1,4 @@
-import { AppCommand } from "./hooks/command";
+import { AppCommand, CommandDef } from "./hooks/command";
 import { FilterOption, PageMeta } from "@silverbulletmd/common/types";
 
 export const slashCommandRegexp = /\/[\w\-]*/;
@@ -34,6 +34,7 @@ export type AppViewState = {
   commands: Map<string, AppCommand>;
   notifications: Notification[];
   actionButtons: ActionButton[];
+  recentCommands: Map<string, Date>;
 
   showFilterBox: boolean;
   filterBoxLabel: string;
@@ -55,6 +56,7 @@ export const initialViewState: AppViewState = {
   bhsHTML: "",
   allPages: new Set(),
   commands: new Map(),
+  recentCommands: new Map(),
   notifications: [],
   actionButtons: [],
   showFilterBox: false,
@@ -87,6 +89,7 @@ export type Action =
   | { type: "hide-lhs" }
   | { type: "show-bhs"; html: string; flex: number; script?: string }
   | { type: "hide-bhs" }
+  | { type: "command-run"; command: string }
   | {
       type: "show-filterbox";
       options: FilterOption[];