perf(search): use fuzzysort for both pages and blocks searching

package.json

@@ -44,17 +44,15 @@
     "dependencies": {
         "codemirror": "^5.58.1",
         "diff": "^4.0.2",
-        "dropbox": "^5.2.0",
+        "fuzzysort": "^1.1.4",
         "ignore": "^5.1.8",
         "jszip": "^3.5.0",
-        "localforage": "^1.7.3",
+        "localforage": "^1.9.0",
         "mousetrap": "^1.6.5",
-        "parinfer-codemirror": "^1.4.2",
         "react": "^16.12.0",
         "react-dom": "^16.12.0",
         "react-textarea-autosize": "^8.0.1",
         "react-transition-group": "^4.3.0",
-        "webdav": "^3.3.0",
         "yargs-parser": "^20.2.4"
     }
 }

src/main/frontend/components/search.cljs

@@ -85,7 +85,8 @@
                                           :path-params {:path data}})
 
                         :block
-                        (let [page (:page/name (:block/page data))
+                        (let [block-uuid (uuid (:block/uuid data))
+                              page (:page/name (:block/page (db/entity [:block/uuid block-uuid])))
                               path (str "/page/" (util/encode-str page) "#ls-block-" (:block/uuid data))]
                           (route/redirect-with-fragment! path))
                         nil))
@@ -100,7 +101,8 @@
                                  {:page page}))
 
                               :block
-                              (let [block (db/entity [:block/uuid (:block/uuid data)])]
+                              (let [block-uuid (uuid (:block/uuid data))
+                                    block (db/entity [:block/uuid block-uuid])]
                                 (state/sidebar-add-block!
                                  (state/get-current-repo)
                                  (:db/id block)

src/main/frontend/db.cljs

@@ -1,6 +1,7 @@
 (ns frontend.db
   (:require [datascript.core :as d]
             [frontend.date :as date]
+            [frontend.search.db :as search-db]
             [medley.core :as medley]
             [datascript.transit :as dt]
             [frontend.format :as format]
@@ -21,7 +22,8 @@
             [frontend.db-schema :as db-schema]
             [clojure.core.async :as async]
             [lambdaisland.glogi :as log]
-            [frontend.idb :as idb]))
+            [frontend.idb :as idb]
+            [cljs-bean.core :as bean]))
 
 ;; Query atom of map of Key ([repo q inputs]) -> atom
 ;; TODO: replace with LRUCache, only keep the latest 20 or 50 items?
@@ -1930,14 +1932,27 @@
     (swap! persistent-jobs assoc repo job)))
 
 ;; only save when user's idle
+
+;; TODO: pass as a parameter
+(defonce *sync-search-indice-f (atom nil))
 (defn- repo-listen-to-tx!
   [repo conn files-db?]
   (d/listen! conn :persistence
              (fn [tx-report]
                (let [tx-id (get-tx-id tx-report)]
                  (state/set-last-transact-time! repo (util/time-ms))
                  ;; (state/persist-transaction! repo files-db? tx-id (:tx-data tx-report))
-                 (persist-if-idle! repo)))))
+                 (persist-if-idle! repo))
+
+               ;; rebuild search indices
+               (when-not files-db?
+                 (let [data (:tx-data tx-report)
+                       datoms (filter
+                               (fn [datom]
+                                 (contains? #{:page/name :block/content} (:a datom)))
+                               data)]
+                   (when-let [f @*sync-search-indice-f]
+                     (f datoms)))))))
 
 (defn- listen-and-persist!
   [repo]
@@ -2415,6 +2430,18 @@
        (reset! blocks-count-cache n)
        n))))
 
+;; block/uuid and block/content
+(defn get-all-block-contents
+  []
+  (->> (d/datoms (get-conn) :avet :block/uuid)
+       (map :v)
+       (map (fn [id]
+              (let [e (entity [:block/uuid id])]
+                {:db/id (:db/id e)
+                 :block/uuid id
+                 :block/content (:block/content e)
+                 :block/format (:block/format e)})))))
+
 (defn get-all-templates
   []
   (let [pred (fn [db properties]

src/main/frontend/extensions/code.cljs

@@ -36,9 +36,7 @@
             ["codemirror/mode/smalltalk/smalltalk"]
             ["codemirror/mode/sql/sql"]
             ["codemirror/mode/swift/swift"]
-            ["codemirror/mode/xml/xml"]
-            ;; ["parinfer-codemirror" :as par-cm]
-))
+            ["codemirror/mode/xml/xml"]))
 
 ;; codemirror
 

src/main/frontend/handler.cljs

@@ -9,6 +9,8 @@
             [promesa.core :as p]
             [cljs-bean.core :as bean]
             [frontend.date :as date]
+            [frontend.search :as search]
+            [frontend.search.db :as search-db]
             [frontend.handler.notification :as notification]
             [frontend.handler.page :as page-handler]
             [frontend.handler.repo :as repo-handler]
@@ -25,9 +27,16 @@
 
 (defn- watch-for-date!
   []
-  (js/setInterval (fn []
-                    (when-not (state/nfs-refreshing?)
-                      (repo-handler/create-today-journal!))) 1000))
+  (let [f (fn []
+            (when-not (state/nfs-refreshing?)
+              (repo-handler/create-today-journal!))
+            (when-let [repo (state/get-current-repo)]
+              (when (and (search-db/empty? repo)
+                         (not (state/file-in-writing!))
+                         (state/input-idle? repo))
+                (search/rebuild-indices!))))]
+    (f)
+    (js/setInterval f 5000)))
 
 (defn store-schema!
   []
@@ -157,6 +166,7 @@
                       :example? true}])]
         (state/set-repos! repos)
         (restore-and-setup! me repos logged?)))
+    (reset! db/*sync-search-indice-f search/sync-search-indice!)
     (db/run-batch-txs!)
     (file-handler/run-writes-chan!)
     (editor-handler/periodically-save!)))

src/main/frontend/handler/editor.cljs

@@ -1547,17 +1547,11 @@
         editing-page (and block
                           (when-let [page-id (:db/id (:block/page block))]
                             (:page/name (db/entity page-id))))]
-    (let [pages (db/get-pages (state/get-current-repo))
-          pages (if editing-page
-                  ;; To prevent self references
-                  (remove (fn [p] (= (string/lower-case p) editing-page)) pages)
-                  pages)]
-      (filter
-       (fn [page]
-         (string/index-of
-          (string/lower-case page)
-          (string/lower-case q)))
-       pages))))
+    (let [pages (search/page-search q 20)]
+      (if editing-page
+        ;; To prevent self references
+        (remove (fn [p] (= (string/lower-case p) editing-page)) pages)
+        pages))))
 
 (defn get-matched-blocks
   [q]
@@ -1567,7 +1561,7 @@
      (fn [h]
        (= (:block/uuid current-block)
           (:block/uuid h)))
-     (search/search q 21))))
+     (search/search q 10))))
 
 (defn get-matched-templates
   [q]

src/main/frontend/handler/search.cljs

@@ -2,14 +2,15 @@
   (:require [goog.object :as gobj]
             [frontend.state :as state]
             [goog.dom :as gdom]
-            [frontend.search :as search]))
+            [frontend.search :as search]
+            [frontend.handler.notification :as notification-handler]))
 
 (defn search
   [q]
   (swap! state/state assoc :search/result
          {:pages (search/page-search q)
           :files (search/file-search q)
-          :blocks (search/search q)}))
+          :blocks (search/search q 10)}))
 
 (defn clear-search!
   []
@@ -18,3 +19,11 @@
          :search/q "")
   (when-let [input (gdom/getElement "search_field")]
     (gobj/set input "value" "")))
+
+(defn rebuild-indices!
+  []
+  (println "Starting to rebuild search indices!")
+  (search/rebuild-indices!)
+  (notification-handler/show!
+   "Search indices rebuilt successfully!"
+   :success))

src/main/frontend/keyboards.cljs

@@ -3,7 +3,9 @@
             [frontend.handler.history :as history-handler]
             [frontend.handler.ui :as ui-handler]
             [frontend.handler.route :as route-handler]
+            [frontend.handler.search :as search-handler]
             [frontend.state :as state]
+            [frontend.search :as search]
             [frontend.util :as util]
             [medley.core :as medley]
             ["mousetrap" :as mousetrap]
@@ -70,7 +72,8 @@
     "ctrl+h" editor-handler/highlight-format!
     "ctrl+shift+a" editor-handler/select-all-blocks!
     "alt+shift+up" (fn [state e] (editor-handler/move-up-down e true))
-    "alt+shift+down" (fn [state e] (editor-handler/move-up-down e false))}
+    "alt+shift+down" (fn [state e] (editor-handler/move-up-down e false))
+    "ctrl+c ctrl+s" (fn [state e] (search-handler/rebuild-indices!))}
    (medley/map-keys util/->system-modifier)))
 
 (defonce chords
@@ -79,7 +82,8 @@
    "t t" state/toggle-theme!
    "t r" ui-handler/toggle-right-sidebar!
    "t e" state/toggle-new-block-shortcut!
-   "s" route-handler/toggle-between-page-and-file!})
+   "s" route-handler/toggle-between-page-and-file!
+   })
 
 (defonce bind! (gobj/get mousetrap "bind"))