Fix simtest failure due to pruned checkpoints after long sync timeout (…

…MystenLabs#16205)

Also adds improved debug logging in a few places.

crates/sui-config/src/p2p.rs

@@ -96,7 +96,7 @@ pub struct StateSyncConfig {
     /// - in case of a fork, to prevent the node from syncing to the wrong chain.
     /// - in case of a network stall, to force the node to proceed with a manually-injected
     ///   checkpoint.
-    #[serde(skip_serializing_if = "Vec::is_empty")]
+    #[serde(skip_serializing_if = "Vec::is_empty", default)]
     pub pinned_checkpoints: Vec<(CheckpointSequenceNumber, CheckpointDigest)>,
 
     /// Query peers for their latest checkpoint every interval period.

crates/sui-e2e-tests/tests/protocol_version_tests.rs

@@ -178,8 +178,10 @@ mod sim_only_tests {
             .with_supported_protocol_version_callback(Arc::new(|idx, name| {
                 if name.is_some() && idx == 0 {
                     // first validator only does not support version FINISH.
+                    info!("node {name:?} supports protocol versions up to {START}");
                     SupportedProtocolVersions::new_for_testing(START, START)
                 } else {
+                    info!("node {name:?} supports protocol versions up to {FINISH}");
                     SupportedProtocolVersions::new_for_testing(START, FINISH)
                 }
             }))

crates/sui-network/src/state_sync/mod.rs

@@ -1316,6 +1316,7 @@ where
         tokio::time::sleep(duration).await;
         return Err(checkpoint);
     };
+    debug!("completed checkpoint contents sync");
     Ok(checkpoint)
 }
 
@@ -1346,6 +1347,11 @@ where
     // Iterate through our selected peers trying each one in turn until we're able to
     // successfully get the target checkpoint
     for mut peer in peers {
+        debug!(
+            ?timeout,
+            "requesting checkpoint contents from {}",
+            peer.inner().peer_id(),
+        );
         let request = Request::new(digest).with_timeout(timeout);
         if let Some(contents) = peer
             .get_checkpoint_contents(request)

crates/sui-swarm-config/src/node_config_builder.rs

@@ -17,7 +17,7 @@ use sui_config::node::{
     DEFAULT_GRPC_CONCURRENCY_LIMIT,
 };
 use sui_config::node::{default_zklogin_oauth_providers, ConsensusProtocol, RunWithRange};
-use sui_config::p2p::{P2pConfig, SeedPeer};
+use sui_config::p2p::{P2pConfig, SeedPeer, StateSyncConfig};
 use sui_config::{
     local_ip_utils, ConsensusConfig, NodeConfig, AUTHORITIES_DB_NAME, CONSENSUS_DB_NAME,
     FULL_NODE_DB_PATH,
@@ -128,6 +128,12 @@ impl ValidatorConfigBuilder {
                     .unwrap()
             }),
             external_address: Some(validator.p2p_address),
+            // Set a shorter timeout for checkpoint content download in tests, since
+            // checkpoint pruning also happens much faster, and network is local.
+            state_sync: Some(StateSyncConfig {
+                checkpoint_content_timeout_ms: Some(10_000),
+                ..Default::default()
+            }),
             ..Default::default()
         };
 
@@ -364,6 +370,12 @@ impl FullnodeConfigBuilder {
                     .p2p_external_address
                     .or(Some(validator_config.p2p_address.clone())),
                 seed_peers,
+                // Set a shorter timeout for checkpoint content download in tests, since
+                // checkpoint pruning also happens much faster, and network is local.
+                state_sync: Some(StateSyncConfig {
+                    checkpoint_content_timeout_ms: Some(10_000),
+                    ..Default::default()
+                }),
                 ..Default::default()
             }
         };

crates/sui-swarm-config/tests/snapshots/snapshot_tests__network_config_snapshot_matches.snap

@@ -50,6 +50,8 @@ validator_configs:
     grpc-concurrency-limit: 20000000000
     p2p-config:
       listen-address: "0.0.0.0:1"
+      state-sync:
+        checkpoint-content-timeout-ms: 10000
     genesis:
       genesis: "[fake genesis]"
     authority-store-pruning-config:
@@ -171,6 +173,8 @@ validator_configs:
     grpc-concurrency-limit: 20000000000
     p2p-config:
       listen-address: "0.0.0.0:1"
+      state-sync:
+        checkpoint-content-timeout-ms: 10000
     genesis:
       genesis: "[fake genesis]"
     authority-store-pruning-config:
@@ -292,6 +296,8 @@ validator_configs:
     grpc-concurrency-limit: 20000000000
     p2p-config:
       listen-address: "0.0.0.0:1"
+      state-sync:
+        checkpoint-content-timeout-ms: 10000
     genesis:
       genesis: "[fake genesis]"
     authority-store-pruning-config:
@@ -413,6 +419,8 @@ validator_configs:
     grpc-concurrency-limit: 20000000000
     p2p-config:
       listen-address: "0.0.0.0:1"
+      state-sync:
+        checkpoint-content-timeout-ms: 10000
     genesis:
       genesis: "[fake genesis]"
     authority-store-pruning-config:
@@ -534,6 +542,8 @@ validator_configs:
     grpc-concurrency-limit: 20000000000
     p2p-config:
       listen-address: "0.0.0.0:1"
+      state-sync:
+        checkpoint-content-timeout-ms: 10000
     genesis:
       genesis: "[fake genesis]"
     authority-store-pruning-config:
@@ -655,6 +665,8 @@ validator_configs:
     grpc-concurrency-limit: 20000000000
     p2p-config:
       listen-address: "0.0.0.0:1"
+      state-sync:
+        checkpoint-content-timeout-ms: 10000
     genesis:
       genesis: "[fake genesis]"
     authority-store-pruning-config:
@@ -776,6 +788,8 @@ validator_configs:
     grpc-concurrency-limit: 20000000000
     p2p-config:
       listen-address: "0.0.0.0:1"
+      state-sync:
+        checkpoint-content-timeout-ms: 10000
     genesis:
       genesis: "[fake genesis]"
     authority-store-pruning-config:

crates/sui-swarm/src/memory/swarm.rs

@@ -13,7 +13,6 @@ use std::{
     ops,
     path::{Path, PathBuf},
 };
-
 use sui_config::node::{DBCheckpointConfig, OverloadThresholdConfig, RunWithRange};
 use sui_config::NodeConfig;
 use sui_macros::nondeterministic;
@@ -28,6 +27,7 @@ use sui_swarm_config::node_config_builder::FullnodeConfigBuilder;
 use sui_types::base_types::AuthorityName;
 use sui_types::object::Object;
 use tempfile::TempDir;
+use tracing::info;
 
 pub struct SwarmBuilder<R = OsRng> {
     rng: R,
@@ -295,7 +295,13 @@ impl<R: rand::RngCore + rand::CryptoRng> SwarmBuilder<R> {
         let mut nodes: HashMap<_, _> = network_config
             .validator_configs()
             .iter()
-            .map(|config| (config.protocol_public_key(), Node::new(config.to_owned())))
+            .map(|config| {
+                info!(
+                    "SwarmBuilder configuring validator with name {}",
+                    config.protocol_public_key()
+                );
+                (config.protocol_public_key(), Node::new(config.to_owned()))
+            })
             .collect();
 
         let mut fullnode_config_builder = FullnodeConfigBuilder::new()
@@ -326,6 +332,10 @@ impl<R: rand::RngCore + rand::CryptoRng> SwarmBuilder<R> {
                     }
                 }
                 let config = builder.build(&mut OsRng, &network_config);
+                info!(
+                    "SwarmBuilder configuring full node with name {}",
+                    config.protocol_public_key()
+                );
                 nodes.insert(config.protocol_public_key(), Node::new(config));
             });
         }

crates/test-cluster/src/lib.rs

@@ -283,7 +283,9 @@ impl TestCluster {
             unreachable!("Broken reconfig channel");
         })
         .await
-        .expect("Timed out waiting for cluster to target epoch")
+        .unwrap_or_else(|_| {
+            panic!("Timed out waiting for cluster to target epoch {target_epoch:?}")
+        })
     }
 
     pub async fn wait_for_run_with_range_shutdown_signal(&self) -> Option<RunWithRange> {