Index analysis by file and bucket (clojure-lsp#1096)

Index analysis by file and bucket, instead of just file as we did before. This is helpful because the majority of queries need to look in only select buckets.

CHANGELOG.md

@@ -7,6 +7,7 @@
   - Improve call hierarchy performance by parsing less frequently. #1092
   - Improve system wide performance by keeping a graph of dependencies between namespaces. #990 #1053
     Enable setting `:experimental {:dep-graph-queries true}` to beta test this feature.
+  - Improve performance by adding second level of analysis indexing.
 
 ## 2022.06.29-19.32.13
 

lib/src/clojure_lsp/db.clj

@@ -21,7 +21,7 @@
 (defonce created-watched-files-chan (async/chan 1))
 (defonce edits-chan (async/chan 1))
 
-(def version 3)
+(def version 4)
 
 (defn ^:private sqlite-db-file [project-root]
   (io/file (str project-root) ".lsp" ".cache" "sqlite.db"))

lib/src/clojure_lsp/dep_graph.clj

@@ -224,19 +224,44 @@
 (defn ^:private remove-usages [db usages] (update-usages db ms-disj usages))
 (defn ^:private add-usages    [db usages] (update-usages db ms-conj usages))
 
-(defn ^:private update-definition [db f {:keys [name filename]}]
-  (let [uri (filename-to-uri db filename)
-        in-doc (get-in db [:documents uri])]
-    (-> db
-        (update-in [:dep-graph name :uris] f uri)
-        (update-in [:dep-graph name :internal?] #(or % (:internal? in-doc)))
-        (update-in [:documents uri :namespaces] f name))))
+(def ^:private user-ns-def
+  ;; add :filename
+  '{:bucket :namespace-definitions
+    :name user})
+
+(def ^:private clojure-core-ns-usage
+  ;; add :filename and :from ns
+  '{:bucket :namespace-usages
+    :name clojure.core})
 
-(defn ^:private update-definitions [db f namespace-definitions]
-  (reduce #(update-definition %1 f %2) db namespace-definitions))
+(def ^:private cljs-core-ns-usage
+  ;; add :filename and :from ns
+  '{:bucket :namespace-usages
+    :name cljs.core})
 
-(defn ^:private remove-definitions [db namespace-definitions] (update-definitions db s-disj namespace-definitions))
-(defn ^:private add-definitions    [db namespace-definitions] (update-definitions db s-conj namespace-definitions))
+(defn ^:private update-definition [db s-f ms-f {:keys [name filename]}]
+  (let [uri (filename-to-uri db filename)
+        in-doc (get-in db [:documents uri])
+        doc-langs (:langs in-doc)
+        db (-> db
+               (update-in [:dep-graph name :uris] s-f uri)
+               (update-in [:dep-graph name :internal?] #(or % (:internal? in-doc)))
+               (update-in [:documents uri :namespaces] s-f name))]
+    (cond-> db
+      (contains? doc-langs :clj)
+      (update-usage ms-f (assoc clojure-core-ns-usage
+                                :from name
+                                :filename filename))
+      (contains? doc-langs :cljs)
+      (update-usage ms-f (assoc cljs-core-ns-usage
+                                :from name
+                                :filename filename)))))
+
+(defn ^:private update-definitions [db s-f ms-f namespace-definitions]
+  (reduce #(update-definition %1 s-f ms-f %2) db namespace-definitions))
+
+(defn ^:private remove-definitions [db namespace-definitions] (update-definitions db s-disj ms-disj namespace-definitions))
+(defn ^:private add-definitions    [db namespace-definitions] (update-definitions db s-conj ms-conj namespace-definitions))
 
 (defn ^:private ensure-file [db filename internal?]
   (let [uri (shared/filename->uri filename db)]
@@ -253,59 +278,19 @@
 (defn ^:private ensure-files [db filenames internal?]
   (reduce #(ensure-file %1 %2 internal?) db filenames))
 
-(def ^:private user-ns-def
-  ;; add :filename
-  '{:bucket :namespace-definitions
-    :name user})
-
-(def ^:private clojure-core-ns-usage
-  ;; add :filename and :from ns
-  '{:bucket :namespace-usages
-    :name clojure.core})
-
-(def ^:private cljs-core-ns-usage
-  ;; add :filename and :from ns
-  '{:bucket :namespace-usages
-    :name cljs.core})
-
 (defn ^:private ns-definitions-and-usages [analysis]
-  (letfn [(result-with-element [result element]
-            (let [result (if (= 'user (:ns element))
-                           (assoc result :in-user true)
-                           result)]
-              (case (:bucket element)
-                :namespace-definitions (update result :defs conj element)
-                :namespace-usages      (update result :usages conj element)
-                :var-usages            (case (:to element)
-                                         clojure.core (update result :to-clj conj (:from element))
-                                         cljs.core    (update result :to-cljs conj (:from element))
-                                         result)
-                result)))]
-    (let [{:keys [defs usages]}
-          (reduce-kv (fn [result filename elements]
-                       (let [result (reduce result-with-element
-                                            (assoc result
-                                                   :in-user false
-                                                   :to-clj #{}
-                                                   :to-cljs #{})
-                                            elements)
-                             implicit-elements (concat
-                                                 (when (:in-user result)
-                                                   [(assoc user-ns-def :filename filename)])
-                                                 (map (fn [from-ns]
-                                                        (assoc clojure-core-ns-usage
-                                                               :from from-ns
-                                                               :filename filename))
-                                                      (:to-clj result))
-                                                 (map (fn [from-ns]
-                                                        (assoc cljs-core-ns-usage
-                                                               :from from-ns
-                                                               :filename filename))
-                                                      (:to-cljs result)))]
-                         (reduce result-with-element result implicit-elements)))
-                     {:defs [] :usages []}
-                     analysis)]
-      [defs usages])))
+  (let [{:keys [defs usages]}
+        (reduce-kv (fn [result filename {:keys [var-definitions namespace-definitions namespace-usages]}]
+                     (let [defs (cond-> namespace-definitions
+                                  ;; implicitly in user ns
+                                  (some #(= 'user (:ns %)) var-definitions)
+                                  (conj (assoc user-ns-def :filename filename)))]
+                       (-> result
+                           (update :defs into defs)
+                           (update :usages into namespace-usages))))
+                   {:defs [] :usages []}
+                   analysis)]
+    [defs usages]))
 
 (defn refresh-analysis [db old-analysis new-analysis internal?]
   ;; NOTE: When called during startup this takes a little time (500ms in medium

lib/src/clojure_lsp/feature/clean_ns.clj

@@ -210,10 +210,9 @@
                          (z/find-next-value ':as)
                          z/right
                          z/sexpr)]
-    (let [local-elements (get-in db [:analysis filename])
-          used-alias? (some #(and (= :var-usages (:bucket %))
-                                  (= alias (:alias %)))
-                            local-elements)]
+    (let [local-var-usages (get-in db [:analysis filename :var-usages])
+          used-alias? (some #(= alias (:alias %))
+                            local-var-usages)]
       (if used-alias?
         node
         (z/remove node)))