Update to v1.0.5

- Fix full/partial scan handling, by checking for presence of master
  branch
- Update progress indicator at end of task, not beginning of task
- Allow platforms not listed in PLATFORMS_UNIX/WINDOWS

Author: k3d3

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "panamax"
-version = "1.0.4"
+version = "1.0.5"
 authors = ["k3d3 <[email protected]>"]
 description = "Mirror rustup and crates.io repositories, for offline Rust and Cargo usage."
 license = "MIT/Apache-2.0"
@@ -21,7 +21,7 @@ env_logger = "0.9.0"
 sha2 = "0.9.6"
 url = "2.2.2"
 glob = "0.3.0"
-git2 = "0.13.21"
+git2 = "0.14"
 serde_json = "1.0.67"
 thiserror = "1.0.29"
 tokio = { version = "1.11.0", features = ["full"] }

Cargo.toml

@@ -105,18 +105,11 @@ pub async fn sync_crates_files(
     let origin_master = repo.find_reference("refs/remotes/origin/master")?;
     let origin_master_tree = origin_master.peel_to_tree()?;
 
-    let master = repo.find_reference("refs/heads/master")?;
-    let master_tree = master.peel_to_tree()?;
-
-    // Perform a full scan if master and origin/master match
-    let do_full_scan = origin_master.peel_to_commit()?.id() == master.peel_to_commit()?.id();
+    let master = repo.find_reference("refs/heads/master").ok();
+    let master_tree = master.as_ref().and_then(|m| m.peel_to_tree().ok());
 
     // Diff between master and origin/master (i.e. everything since the last fetch)
-    let diff = if do_full_scan {
-        repo.diff_tree_to_tree(None, Some(&origin_master_tree), None)?
-    } else {
-        repo.diff_tree_to_tree(Some(&master_tree), Some(&origin_master_tree), None)?
-    };
+    let diff = repo.diff_tree_to_tree(master_tree.as_ref(), Some(&origin_master_tree), None)?;
 
     let mut changed_crates = Vec::new();
     let mut removed_crates = Vec::new();
@@ -214,16 +207,18 @@ pub async fn sync_crates_files(
             let pb = pb.clone();
 
             tokio::spawn(async move {
-                pb.inc(1);
-
-                sync_one_crate_entry(
+                let out = sync_one_crate_entry(
                     &path,
                     crates_source.as_deref(),
                     mirror_retries,
                     &c,
                     &user_agent,
                 )
-                .await
+                .await;
+
+                pb.inc(1);
+
+                out
             })
         })
         .buffer_unordered(crates.download_threads)

src/crates.rs

@@ -77,7 +77,10 @@ pub fn sync_crates_repo(mirror_path: &Path, crates: &ConfigCrates) -> Result<(),
     fetch_opts.proxy_options(proxy_opts);
 
     if !repo_path.join(".git").exists() {
-        clone_repository(fetch_opts, &crates.source_index, &repo_path)?
+        clone_repository(fetch_opts, &crates.source_index, &repo_path)?;
+        // Remove master in order to ensure full scan is performed
+        let repo = Repository::open(&repo_path)?;
+        repo.find_reference("refs/heads/master")?.delete()?;
     } else {
         // Get (fetch) the branch's latest remote "master" commit
         let repo = Repository::open(&repo_path)?;

src/crates_index.rs

@@ -181,35 +181,11 @@ impl Platforms {
 
 pub fn get_platforms(rustup: &ConfigRustup) -> Result<Platforms, MirrorError> {
     let unix = match &rustup.platforms_unix {
-        Some(p) => {
-            let bad_platforms: Vec<&String> = p
-                .iter()
-                .filter(|x| !PLATFORMS_UNIX.contains(&x.as_str()))
-                .collect();
-            if !bad_platforms.is_empty() {
-                eprintln!("Bad values in unix platforms: {:?}", bad_platforms);
-                return Err(MirrorError::Config(
-                    "bad value for 'platforms_unix'".to_string(),
-                ));
-            }
-            p.clone()
-        }
+        Some(p) => p.clone(),
         None => PLATFORMS_UNIX.iter().map(|x| x.to_string()).collect(),
     };
     let windows = match &rustup.platforms_windows {
-        Some(p) => {
-            let bad_platforms: Vec<&String> = p
-                .iter()
-                .filter(|x| !PLATFORMS_WINDOWS.contains(&x.as_str()))
-                .collect();
-            if !bad_platforms.is_empty() {
-                eprintln!("Bad values in windows platforms: {:?}", bad_platforms);
-                return Err(MirrorError::Config(
-                    "bad value for 'platforms_windows'".to_string(),
-                ));
-            }
-            p.clone()
-        }
+        Some(p) => p.clone(),
         None => PLATFORMS_WINDOWS.iter().map(|x| x.to_string()).collect(),
     };
     Ok(Platforms { unix, windows })
@@ -290,9 +266,7 @@ async fn create_sync_tasks(
             let pb = pb.clone();
 
             tokio::spawn(async move {
-                pb.inc(1);
-
-                sync_one_init(
+                let out = sync_one_init(
                     &path,
                     &source,
                     platform.as_str(),
@@ -301,7 +275,11 @@ async fn create_sync_tasks(
                     retries,
                     &user_agent,
                 )
-                .await
+                .await;
+
+                pb.inc(1);
+
+                out
             })
         })
         .buffer_unordered(threads)
@@ -676,9 +654,12 @@ pub async fn sync_rustup_channel(
             let pb = pb.clone();
 
             tokio::spawn(async move {
+                let out =
+                    sync_one_rustup_target(&path, &source, &url, &hash, retries, &user_agent).await;
+
                 pb.inc(1);
 
-                sync_one_rustup_target(&path, &source, &url, &hash, retries, &user_agent).await
+                out
             })
         })
         .buffer_unordered(threads)