Skip to content

Commit

Permalink
feat(full-text-search): use flexsearch instead of fuzzysort
Browse files Browse the repository at this point in the history 
For full-text search.
  • Loading branch information
@tiensonqin
tiensonqin committed Feb 14, 2021
1 parent e1ae3bf commit 42f053d
Show file tree
Hide file tree
Showing 7 changed files with 104 additions and 47 deletions.
2 changes: 2 additions & 0 deletions externs.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ dummy.getRangeAt = function() {};
dummy.getElementsByClassName = function() {};
dummy.containsNode = function() {};
dummy.select = function() {};
dummy.search = function() {};
dummy.add = function() {};
dummy.closest = function () {};
dummy.setAttribute = function() {};
dummy.getAttribute = function() {};
Expand Down
1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
"diff": "5.0.0",
"diff-match-patch": "^1.0.5",
"electron": "^11.2.0",
"flexsearch": "^0.6.32",
"fs": "^0.0.1-security",
"fuzzysort": "git+https://github.com/getstation/fuzzysort#a66f5813825d2415b606cc69129070c4eb612ae2",
"gulp-cached": "^1.1.1",
Expand Down
37 changes: 36 additions & 1 deletion src/main/frontend/components/search.cljs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,41 @@
(let [switch (reductions not= true (map pred? coll (rest coll)))]
(map (partial map first) (partition-by second (map list coll switch)))))

(defn highlight-exact-query
[content q]
(let [q-words (string/split q #" ")
lc-content (string/lower-case content)
lc-q (string/lower-case q)]
(if (or (string/includes? lc-content lc-q)
(not (re-find #" " q)))
(let [i (string/index-of lc-content lc-q)
[before after] [(subs content 0 i) (subs content (+ i (count q)))]]
[:p
(when-not (string/blank? before)
[:span before])
[:mark (subs content i (+ i (count q)))]
(when-not (string/blank? after)
[:span after])])
(let [elements (loop [words q-words
content content
result []]
(if (and (seq words) content)
(let [word (first words)
lc-word (string/lower-case word)
lc-content (string/lower-case content)]
(if-let [i (string/index-of lc-content lc-word)]
(recur (rest words)
(subs content (+ i (count word)))
(vec
(concat result
[[:span (subs content 0 i)]
[:mark (subs content i (+ i (count word)))]])))
(recur nil
content
result)))
(conj result [:span content])))]
[:p elements]))))

(rum/defc highlight-fuzzy
[content indexes]
(let [n (count content)
Expand Down Expand Up @@ -182,7 +217,7 @@
(:page/name page))]
[:div.flex-1
[:div.text-sm.font-medium (str "-> " page)]
(highlight-fuzzy content indexes)])
(highlight-exact-query content search-q)])

nil))})])))

Expand Down
2 changes: 1 addition & 1 deletion src/main/frontend/handler/editor.cljs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1767,7 +1767,7 @@
(remove
(fn [h]
(contains? current-and-parents (:block/uuid h)))
(search/search q 10))))
(search/block-search q 10))))

(defn get-matched-templates
[q]
Expand Down
13 changes: 8 additions & 5 deletions src/main/frontend/handler/search.cljs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,17 @@
[frontend.state :as state]
[goog.dom :as gdom]
[frontend.search :as search]
[frontend.handler.notification :as notification-handler]))
[frontend.handler.notification :as notification-handler]
[promesa.core :as p]))

(defn search
[q]
(swap! state/state assoc :search/result
{:pages (search/page-search q)
:files (search/file-search q)
:blocks (search/search q 10)}))
;; TODO: separate rendering for blocks
(p/let [blocks-result (search/block-search q 10)]
(swap! state/state assoc :search/result
{:pages (search/page-search q)
:files (search/file-search q)
:blocks blocks-result})))

(defn clear-search!
[]
Expand Down
91 changes: 51 additions & 40 deletions src/main/frontend/search.cljs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,35 +12,48 @@
[cljs-bean.core :as bean]
[goog.object :as gobj]
["fuzzysort" :as fuzzy]
[medley.core :as medley]))
["flexsearch" :as flexsearch]
[medley.core :as medley]
[promesa.core :as p]))

(def fuzzy-go (gobj/get fuzzy "go"))
(defonce prepare (gobj/get fuzzy "prepare"))
(defonce highlight (gobj/get fuzzy "highlight"))

(defn go
[q indice opts]
(fuzzy-go q indice opts))
[q indice-type indice opts]
(case indice-type
:page
(fuzzy-go q indice opts)

:block
(.search indice q opts)))

(defn block->index
[{:block/keys [uuid content format] :as block}]
(when (<= (count content) 1000) ; performance
(when-let [result (->> (text/remove-level-spaces content format)
(text/remove-properties!)
(prepare))]
(gobj/set result "id" (:db/id block))
(gobj/set result "uuid" (str uuid))
result)))
(when-let [result (->> (text/remove-level-spaces content format)
(text/remove-properties!))]
{:id (:db/id block)
:uuid (str uuid)
:content result}))

(defn make-blocks-indice!
[]
(when-let [repo (state/get-current-repo)]
(let [blocks (->> (db/get-all-block-contents)
(map block->index)
(remove nil?)
(bean/->js))]
(swap! indices assoc-in [repo :blocks] blocks)
blocks)))
(bean/->js))
indice (flexsearch.
(clj->js
{:encode "icase"
:tokenize "full"
:doc {:id "id"
:field ["uuid" "content"]}
:async true}))]
(p/let [result (.add indice blocks)]
(swap! indices assoc-in [repo :blocks] indice))
indice)))

(defn make-pages-indice!
[]
Expand Down Expand Up @@ -136,32 +149,27 @@
:score (score query (.toLowerCase s))})))))
(map :data))))

(defn search
"Block search"
(defn block-search
([q]
(search q 10))
(block-search q 10))
([q limit]
(when-let [repo (state/get-current-repo)]
(when-not (string/blank? q)
(let [q (string/lower-case q)
q (escape-str q)]
(when-not (string/blank? q)
(let [indice (or (get-in @indices [repo :blocks])
(make-blocks-indice!))
result (->
(go q indice (clj->js {:limit limit
:allowTypo false
:threshold -10000}))
(bean/->clj))]
(->>
(map
(fn [{:keys [target uuid indexes]}]
{:block/uuid uuid
:block/content target
:block/page (:block/page (db/entity [:block/uuid (medley/uuid (str uuid))]))
:block/indexes indexes}) ; For result highlight
result)
(remove nil?)))))))))
(make-blocks-indice!))]
(p/let [result (go q :block indice (clj->js {:limit limit}))
result (bean/->clj result)]
(->>
(map
(fn [{:keys [content uuid] :as block}]
{:block/uuid uuid
:block/content content
:block/page (:block/page (db/entity [:block/uuid (medley/uuid (str uuid))]))})
result)
(remove nil?))))))))))

(defn page-search
([q]
Expand All @@ -173,11 +181,12 @@
(when-not (string/blank? q)
(let [indice (or (get-in @indices [repo :pages])
(make-pages-indice!))
result (->> (go q indice (clj->js {:limit limit
:key "name"
:allowTypo false
:threshold -10000}))
result (->> (go q :page indice (clj->js {:limit limit
:key "name"
:allowTypo false
:threshold -10000}))
(bean/->clj))]
;; TODO: add indexes for highlights
(->> (map
(fn [{:keys [obj]}]
(:name obj))
Expand Down Expand Up @@ -250,8 +259,10 @@
(map :e)
(set))]
(swap! search-db/indices update-in [repo :blocks]
(fn [blocks]
(let [blocks (or blocks (array))
blocks (.filter blocks (fn [block]
(not (contains? blocks-to-remove-set (gobj/get block "id")))))]
(.concat blocks (bean/->js blocks-to-add)))))))))))
(fn [indice]
(when indice
(doseq [block-id blocks-to-remove-set]
(.remove indice #js {:id block-id}))
(when (seq blocks-to-add)
(.add indice (bean/->js blocks-to-add))))
indice))))))))
5 changes: 5 additions & 0 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2312,6 +2312,11 @@ flatted@^3.1.0:
resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.1.0.tgz#a5d06b4a8b01e3a63771daa5cb7a1903e2e57067"
integrity sha512-tW+UkmtNg/jv9CSofAKvgVcO7c2URjhTdW1ZTkcAritblu8tajiYy7YisnIflEwtKssCtOxpnBRoCB7iap0/TA==

flexsearch@^0.6.32:
version "0.6.32"
resolved "https://registry.yarnpkg.com/flexsearch/-/flexsearch-0.6.32.tgz#1e20684d317af65baa445cdd9864a5f5b320f510"
integrity sha512-EF1BWkhwoeLtbIlDbY/vDSLBen/E5l/f1Vg7iX5CDymQCamcx1vhlc3tIZxIDplPjgi0jhG37c67idFbjg+v+Q==

flush-write-stream@^1.0.2:
version "1.1.1"
resolved "https://registry.yarnpkg.com/flush-write-stream/-/flush-write-stream-1.1.1.tgz#8dd7d873a1babc207d94ead0c2e0e44276ebf2e8"
Expand Down

0 comments on commit 42f053d

Please sign in to comment.