From e8c29edc79d9ac9552b704f1f5458ae6b645013f Mon Sep 17 00:00:00 2001 From: Fedor Korotkov Date: Thu, 20 Aug 2015 02:58:30 -0400 Subject: [PATCH] Retries for resolve.go Testing Done: https://travis-ci.org/fkorotkov/pants/builds/76133778 Reviewed at https://rbcommons.com/s/twitter/r/2652/ --- .../src/python/pants/contrib/go/subsystems/fetchers.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/contrib/go/src/python/pants/contrib/go/subsystems/fetchers.py b/contrib/go/src/python/pants/contrib/go/subsystems/fetchers.py index 7da654d03c9..a75eba1aa9f 100644 --- a/contrib/go/src/python/pants/contrib/go/subsystems/fetchers.py +++ b/contrib/go/src/python/pants/contrib/go/subsystems/fetchers.py @@ -297,6 +297,8 @@ def register_options(cls, register): default=10 * 1024, # 10KB in case jumbo frames are in play. help='The number of bytes of archive content to buffer in memory before flushing to ' 'disk when downloading an archive.') + register('--retries', default=1, advanced=True, + help='How many times to retry to fetch a remote library.') @memoized_property def _matchers(self): @@ -358,7 +360,12 @@ def _fetch(self, url): def _download(self, url): # TODO(jsirois): Wrap with workunits, progress meters, checksums. self.logger.info('Downloading {}...'.format(url)) - with closing(requests.get(url, stream=True)) as res: + session = requests.session() + # Override default http adapters with a retriable one. + retriable_http_adapter = requests.adapters.HTTPAdapter(max_retries=self.get_options().retries) + session.mount("http://", retriable_http_adapter) + session.mount("https://", retriable_http_adapter) + with closing(session.get(url, stream=True)) as res: if not res.status_code == requests.codes.ok: raise self.FetchError('Failed to download {} ({} error)'.format(url, res.status_code)) with temporary_file() as archive_fp: