perf: reduce start time from minutes to seconds for large graphs

lugenx · Dec 5, 2021 · d7c4392 · d7c4392
1 parent 117cc9f
commit d7c4392
Show file tree

Hide file tree

Showing 8 changed files with 25 additions and 26 deletions.
diff --git a/package.json b/package.json
@@ -81,7 +81,7 @@
         "d3-force": "3.0.0",
         "diff": "5.0.0",
         "diff-match-patch": "1.0.5",
-        "electron": "15.1.2",
+        "electron": "16.0.4",
         "fs": "0.0.1-security",
         "fs-extra": "9.1.0",
         "fuse.js": "6.4.6",

diff --git a/resources/package.json b/resources/package.json
@@ -11,14 +11,14 @@
     "electron:make": "electron-forge make",
     "electron:make-macos-arm64": "electron-forge make --platform=darwin --arch=arm64",
     "electron:publish:github": "electron-forge publish",
-    "rebuild:better-sqlite3": "electron-rebuild -v 15.1.2 -f -w better-sqlite3",
+    "rebuild:better-sqlite3": "electron-rebuild -v 16.0.4 -f -w better-sqlite3",
     "postinstall": "install-app-deps"
   },
   "config": {
     "forge": "./forge.config.js"
   },
   "dependencies": {
-    "better-sqlite3": "7.4.4",
+    "better-sqlite3": "7.4.5",
     "chokidar": "^3.5.1",
     "dugite": "1.103.0",
     "electron-log": "4.3.1",
@@ -38,10 +38,9 @@
     "@electron-forge/maker-rpm": "^6.0.0-beta.57",
     "@electron-forge/maker-squirrel": "^6.0.0-beta.57",
     "@electron-forge/maker-zip": "^6.0.0-beta.57",
-    "electron": "15.1.2",
+    "electron": "16.0.4",
     "electron-builder": "^22.11.7",
     "electron-forge-maker-appimage": "trusktr/electron-forge-maker-appimage#patch-1",
     "electron-rebuild": "3.2.5"
-  },
-  "resolutions": {  "**/node-abi": "^3.2.0" }
+  }
 }
diff --git a/src/electron/electron/fs_watcher.cljs b/src/electron/electron/fs_watcher.cljs
@@ -26,7 +26,8 @@
                           (clj->js
                            {:ignored (fn [path]
                                        (utils/ignored-path? dir path))
-                            :ignoreInitial false
+                            ;; :ignoreInitial false
+                            :ignoreInitial true
                             :ignorePermissionErrors true
                             :interval polling-interval
                             :binaryInterval polling-interval

diff --git a/src/electron/electron/handler.cljs b/src/electron/electron/handler.cljs
@@ -163,7 +163,8 @@
 (defmethod handle :rebuild-blocks-indice [window [_ repo data]]
   (search/truncate-blocks-table! repo)
   ;; unneeded serialization
-  (search/upsert-blocks! repo (bean/->js data)))
+  (search/upsert-blocks! repo (bean/->js data))
+  [])
 
 (defmethod handle :transact-blocks [window [_ repo data]]
   (let [{:keys [blocks-to-remove-set blocks-to-add]} data]

diff --git a/src/main/frontend/debug.cljs b/src/main/frontend/debug.cljs
@@ -1,4 +1,5 @@
 (ns frontend.debug
+  (:refer-clojure :exclude [print])
   (:require [cljs.pprint :as pprint]
             [frontend.state :as state]
             [frontend.util :as util]
@@ -10,6 +11,11 @@
     (doseq [x xs]
       (pprint/pprint x))))
 
+(defn print
+  [& xs]
+  (println "Time: " (str (js/Date.)))
+  (apply println xs))
+
 (defonce ack-wait-timeouts (atom {}))
 
 (defonce default-write-ack-timeout 10000)

diff --git a/src/main/frontend/handler.cljs b/src/main/frontend/handler.cljs
@@ -62,12 +62,7 @@
                             (* 60 1000)))
                 (let [total (srs/get-srs-cards-total)]
                   (state/set-state! :srs/cards-due-count total)
-                  (reset! cards-last-check-time (util/time-ms))))
-
-              (when (and repo
-                         (search-db/empty? repo)
-                         (state/input-idle? repo))
-                (search/rebuild-indices!))))]
+                  (reset! cards-last-check-time (util/time-ms))))))]
     (f)
     (js/setInterval f 5000)))
 
@@ -133,7 +128,7 @@
                                  (js/console.error "Failed to request GitHub app tokens."))))
 
                             (watch-for-date!)
-                            (file-handler/watch-for-local-dirs!)
+                            (file-handler/watch-for-current-graph-dir!)
                             ;; (when-not (state/logged?)
                             ;;   (state/pub-event! [:after-db-restore repos]))
                             ))

diff --git a/src/main/frontend/handler/file.cljs b/src/main/frontend/handler/file.cljs
@@ -330,14 +330,11 @@
       (recur))
     chan))
 
-(defn watch-for-local-dirs!
+(defn watch-for-current-graph-dir!
   []
   (when (util/electron?)
-    (let [repos (->> (state/get-repos)
-                     (filter (fn [repo]
-                               (config/local-db? (:url repo)))))
-          directories (map (fn [repo] (config/get-repo-dir (:url repo))) repos)]
-      (doseq [dir directories]
+    (when-let [repo (state/get-current-repo)]
+      (when-let [dir (config/get-repo-dir repo)]
         (fs/watch-dir! dir)))))
 
 (defn create-metadata-file

diff --git a/src/main/frontend/search.cljs b/src/main/frontend/search.cljs
@@ -214,11 +214,11 @@
    (when repo
      (when-let [engine (get-engine repo)]
        (let [pages (search-db/make-pages-indice!)]
-        (p/let [blocks (protocol/rebuild-blocks-indice! engine)]
-          (let [result {:pages pages
-                        :blocks blocks}]
-            (swap! indices assoc repo result)
-            indices)))))))
+         (p/let [blocks (protocol/rebuild-blocks-indice! engine)]
+           (let [result {:pages pages
+                         :blocks blocks}]
+             (swap! indices assoc repo result)
+             indices)))))))
 
 (defn reset-indice!
   [repo]