Stop on fail consensus recovery (qdrant#1347)

godhelpjun · Feb 6, 2023 · 6669f95 · 6669f95
1 parent f5924e0
commit 6669f95
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 2 deletions.
diff --git a/src/consensus.rs b/src/consensus.rs
@@ -337,7 +337,7 @@ impl Consensus {
                     RECOVERY_RETRY_TIMEOUT * (RECOVERY_MAX_RETRY_COUNT - tries) as u32;
                 sleep(exp_timeout).await;
             }
-            log::error!("Failed to recover from any peer");
+            return Err(anyhow::anyhow!("Failed to recover from any known peers"));
         }
 
         Ok(())

diff --git a/tests/consensus_tests/utils.py b/tests/consensus_tests/utils.py
@@ -156,7 +156,11 @@ def get_cluster_info(peer_api_uri: str) -> dict:
 
 def print_clusters_info(peer_api_uris: [str]):
     for uri in peer_api_uris:
-        print(json.dumps(get_cluster_info(uri), indent=4))
+        try:
+            # do not crash if the peer is not online
+            print(json.dumps(get_cluster_info(uri), indent=4))
+        except requests.exceptions.ConnectionError:
+            print(f"Can't retrieve cluster info for offline peer {uri}")
 
 
 def get_collection_cluster_info(peer_api_uri: str, collection_name: str) -> dict: