fix(core): improve file gathering performance (nrwl#20377)

Cargo.toml

@@ -1,8 +1,8 @@
 
 [workspace]
-resolver = "2"
+resolver = '2'
 members = [
-	'packages/nx'
+	'packages/nx',
 ]
 
 [profile.release]

packages/nx/Cargo.toml

@@ -7,7 +7,7 @@ edition = '2021'
 anyhow = "1.0.71"
 colored = "2"
 crossbeam-channel = '0.5'
-dashmap = { version = "5.5.3", features= ["rayon"] }
+dashmap = { version = "5.5.3", features = ["rayon"] }
 fs_extra = "1.3.0"
 globset = "0.4.10"
 hashbrown = { version = "0.14.0", features = ["rayon"] }
@@ -25,8 +25,6 @@ napi-derive = '2.9.3'
 nom = '7.1.3'
 regex = "1.9.1"
 rayon = "1.7.0"
-serde = "1"
-serde_json = "1"
 thiserror = "1.0.40"
 tokio = { version = "1.28.2", features = ["fs"] }
 tracing = "0.1.37"

packages/nx/src/native/hasher.rs

@@ -1,6 +1,7 @@
-use crate::native::utils::Normalize;
-use crate::native::walker::nx_walker;
-use std::collections::HashMap;
+use std::fs::File;
+use std::io::{BufRead, BufReader};
+use std::path::Path;
+use tracing::trace;
 use xxhash_rust::xxh3;
 
 pub fn hash(content: &[u8]) -> String {
@@ -16,25 +17,24 @@ pub fn hash_array(input: Vec<String>) -> String {
 
 #[napi]
 pub fn hash_file(file: String) -> Option<String> {
-    let Ok(content) = std::fs::read(file) else {
+    hash_file_path(file)
+}
+
+#[inline]
+pub fn hash_file_path<P: AsRef<Path>>(path: P) -> Option<String> {
+    let path = path.as_ref();
+    let Ok(file) = File::open(path) else {
+        trace!("could not open file: {path:?}");
         return None;
     };
 
-    Some(hash(&content))
-}
+    let mut buffer = BufReader::new(file);
+    let Ok(content) = buffer.fill_buf() else {
+        trace!("could not read file: {path:?}");
+        return None;
+    };
 
-#[napi]
-pub fn hash_files(workspace_root: String) -> HashMap<String, String> {
-    nx_walker(workspace_root, |rec| {
-        let mut collection: HashMap<String, String> = HashMap::new();
-        for (path, content) in rec {
-            collection.insert(
-                path.to_normalized_string(),
-                xxh3::xxh3_64(&content).to_string(),
-            );
-        }
-        collection
-    })
+    Some(hash(content))
 }
 
 #[cfg(test)]

packages/nx/src/native/index.d.ts

@@ -23,7 +23,6 @@ export function remove(src: string): void
 export function copy(src: string, dest: string): void
 export function hashArray(input: Array<string>): string
 export function hashFile(file: string): string | null
-export function hashFiles(workspaceRoot: string): Record<string, string>
 export function findImports(projectFileMap: Record<string, Array<string>>): Array<ImportResult>
 /**
  * Transfer the project graph from the JS world to the Rust world, so that we can pass the project graph via memory quicker

packages/nx/src/native/index.js

@@ -246,15 +246,14 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { expandOutputs, getFilesForOutputs, remove, copy, hashArray, hashFile, hashFiles, ImportResult, findImports, transferProjectGraph, HashPlanner, TaskHasher, EventType, Watcher, WorkspaceContext, WorkspaceErrors } = nativeBinding
+const { expandOutputs, getFilesForOutputs, remove, copy, hashArray, hashFile, ImportResult, findImports, transferProjectGraph, HashPlanner, TaskHasher, EventType, Watcher, WorkspaceContext, WorkspaceErrors } = nativeBinding
 
 module.exports.expandOutputs = expandOutputs
 module.exports.getFilesForOutputs = getFilesForOutputs
 module.exports.remove = remove
 module.exports.copy = copy
 module.exports.hashArray = hashArray
 module.exports.hashFile = hashFile
-module.exports.hashFiles = hashFiles
 module.exports.ImportResult = ImportResult
 module.exports.findImports = findImports
 module.exports.transferProjectGraph = transferProjectGraph

packages/nx/src/native/plugins/js/ts_import_locators.rs

@@ -1361,16 +1361,11 @@ import('./dynamic-import.vue')
         ancestors.next();
         let root = PathBuf::from(ancestors.next().unwrap());
 
-        let files = nx_walker(root.clone(), move |receiver| {
-            let mut files = vec![];
-            let glob = build_glob_set(&["**/*.[jt]s"]).unwrap();
-            for (path, _) in receiver {
-                if glob.is_match(&path) {
-                    files.push(root.join(path).to_normalized_string());
-                }
-            }
-            files
-        });
+        let glob = build_glob_set(&["**/*.[jt]s"]).unwrap();
+        let files = nx_walker(root.clone())
+            .filter(|(full_path, _)| glob.is_match(full_path))
+            .map(|(full_path, _)| full_path.to_normalized_string())
+            .collect::<Vec<_>>();
 
         let results: HashMap<_, _> =
             find_imports(HashMap::from([(String::from("nx"), files.clone())]))

packages/nx/src/native/walker.rs

@@ -2,8 +2,9 @@ use std::path::{Path, PathBuf};
 use std::thread;
 use std::thread::available_parallelism;
 
-use crossbeam_channel::{unbounded, Receiver};
+use crossbeam_channel::unbounded;
 use ignore::WalkBuilder;
+use tracing::trace;
 
 use crate::native::glob::build_glob_set;
 
@@ -35,11 +36,9 @@ where
 }
 
 /// Walk the directory and ignore files from .gitignore and .nxignore
-pub fn nx_walker<P, Fn, Re>(directory: P, f: Fn) -> Re
+pub fn nx_walker<P>(directory: P) -> impl Iterator<Item = (PathBuf, PathBuf)>
 where
     P: AsRef<Path>,
-    Fn: FnOnce(Receiver<(PathBuf, Vec<u8>)>) -> Re + Send + 'static,
-    Re: Send + 'static,
 {
     let directory = directory.as_ref();
     let nx_ignore = directory.join(".nxignore");
@@ -59,10 +58,11 @@ where
 
     let cpus = available_parallelism().map_or(2, |n| n.get()) - 1;
 
-    let (sender, receiver) = unbounded::<(PathBuf, Vec<u8>)>();
+    let (sender, receiver) = unbounded();
 
-    let receiver_thread = thread::spawn(|| f(receiver));
+    trace!(?directory, "walking");
 
+    let now = std::time::Instant::now();
     walker.threads(cpus).build_parallel().run(|| {
         let tx = sender.clone();
         Box::new(move |entry| {
@@ -72,27 +72,29 @@ where
                 return Continue;
             };
 
-            let Ok(content) = std::fs::read(dir_entry.path()) else {
+            if dir_entry.file_type().is_some_and(|d| d.is_dir()) {
                 return Continue;
-            };
+            }
 
             let Ok(file_path) = dir_entry.path().strip_prefix(directory) else {
                 return Continue;
             };
 
-            tx.send((file_path.into(), content)).ok();
+            tx.send((dir_entry.path().to_owned(), file_path.to_owned()))
+                .ok();
 
             Continue
         })
     });
+    trace!("walked in {:?}", now.elapsed());
 
+    let receiver_thread = thread::spawn(move || receiver.into_iter());
     drop(sender);
     receiver_thread.join().unwrap()
 }
 
 #[cfg(test)]
 mod test {
-    use std::collections::HashMap;
     use std::{assert_eq, vec};
 
     use assert_fs::prelude::*;
@@ -124,32 +126,21 @@ mod test {
     #[test]
     fn it_walks_a_directory() {
         // handle empty workspaces
-        let content = nx_walker("/does/not/exist", |rec| {
-            let mut paths = vec![];
-            for (path, _) in rec {
-                paths.push(path);
-            }
-            paths
-        });
+        let content = nx_walker("/does/not/exist").collect::<Vec<_>>();
         assert!(content.is_empty());
 
         let temp_dir = setup_fs();
 
-        let content = nx_walker(temp_dir, |rec| {
-            let mut paths = HashMap::new();
-            for (path, content) in rec {
-                paths.insert(path, content);
-            }
-            paths
-        });
+        let mut content = nx_walker(&temp_dir).collect::<Vec<_>>();
+        content.sort();
         assert_eq!(
             content,
-            HashMap::from([
-                (PathBuf::from("baz/qux.txt"), "content@qux".into()),
-                (PathBuf::from("foo.txt"), "content1".into()),
-                (PathBuf::from("test.txt"), "content".into()),
-                (PathBuf::from("bar.txt"), "content2".into()),
-            ])
+            vec![
+                (temp_dir.join("bar.txt"), PathBuf::from("bar.txt")),
+                (temp_dir.join("baz/qux.txt"), PathBuf::from("baz/qux.txt")),
+                (temp_dir.join("foo.txt"), PathBuf::from("foo.txt")),
+                (temp_dir.join("test.txt"), PathBuf::from("test.txt")),
+            ]
         );
     }
 
@@ -180,13 +171,10 @@ nested/child-two/
             )
             .unwrap();
 
-        let mut file_names = nx_walker(temp_dir, |rec| {
-            let mut file_names = vec![];
-            for (path, _) in rec {
-                file_names.push(path.to_normalized_string());
-            }
-            file_names
-        });
+        let mut file_names = nx_walker(temp_dir)
+            .into_iter()
+            .map(|(_, p)| p.to_normalized_string())
+            .collect::<Vec<_>>();
 
         file_names.sort();