Skip to content

Commit

Permalink
replace std thread with rayon
Browse files Browse the repository at this point in the history
  • Loading branch information
lukaszKielar committed Oct 16, 2020
1 parent 0cb55df commit 413a942
Show file tree
Hide file tree
Showing 3 changed files with 139 additions and 28 deletions.
124 changes: 120 additions & 4 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ env_logger = "0.7.1"
glob = "0.3.0"
lazy_static = "1.4.0"
log = "0.4"
regex = "1.3.9"
rayon = "1.4.1"
regex = "1.4.1"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
structopt = "0.3"
40 changes: 17 additions & 23 deletions src/utils.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::thread;

use lazy_static::lazy_static;
use rayon::prelude::*;
use regex::Regex;

use crate::metadata::Metadata;
Expand Down Expand Up @@ -68,29 +67,24 @@ pub(crate) fn get_conda_meta_path() -> PathBuf {
pub(crate) fn get_conda_metadata() -> HashMap<String, Metadata> {
let conda_meta = get_conda_meta_path();

let mut thread_handles = vec![];

let conda_metadata: Arc<Mutex<HashMap<String, Metadata>>> =
Arc::new(Mutex::new(HashMap::new()));

for entry in conda_meta.read_dir().unwrap() {
let c_conda_metadata = conda_metadata.clone();
thread_handles.push(thread::spawn(move || {
let path = entry.unwrap().path();
if path.is_file() & path.to_str().unwrap().ends_with(".json") {
let metadata = Metadata::from_json(&path).unwrap();
let mut conda_metadata = c_conda_metadata.lock().unwrap();
conda_metadata.insert(metadata.name.clone(), metadata);
}
}))
}
// read conda meta directory and get all of the json metadata files
let json_metadata_files: Vec<_> = conda_meta
.read_dir()
.unwrap()
.map(|direntry| direntry.unwrap().path())
.filter(|path| path.is_file() & path.to_str().unwrap().ends_with(".json"))
.collect();

for handle in thread_handles {
handle.join().unwrap()
}
// iterate over json files and create hashmap of all packages installed
let conda_metadata: HashMap<String, Metadata> = json_metadata_files
.par_iter()
.map(|path| {
let metadata = Metadata::from_json(&path).unwrap();
(metadata.name.clone(), metadata)
})
.collect();

let conda_metadata = &*conda_metadata.lock().unwrap();
conda_metadata.clone()
conda_metadata
}

// TODO this function should take a conda_metadata as an argument
Expand Down

0 comments on commit 413a942

Please sign in to comment.