Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

explicit retry_decorator dependency #153

Merged
merged 1 commit into from
Sep 4, 2013
Merged

explicit retry_decorator dependency #153

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitmodules
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,6 @@
[submodule "third_party/socksipy-branch"]
path = third_party/socksipy-branch
url = git://github.com/gsutil-mirrors/socksipy-branch.git
[submodule "third_party/retry_decorator"]
path = third_party/retry_decorator
url = git://github.com/pnpnpn/retry-decorator.git
2 changes: 1 addition & 1 deletion gslib/commands/acl.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -354,7 +354,7 @@ def _ApplyExceptionHandler(self, exception):
self.logger.error('Encountered a problem: {0}'.format(exception))
self.everything_set_okay = False

@Retry(GSResponseError, tries=3, delay=1, backoff=2)
@Retry(GSResponseError, tries=3, timeout_secs=1)
def ApplyAclChanges(self, uri_or_expansion_result):
"""Applies the changes in self.changes to the provided URI."""
if isinstance(uri_or_expansion_result, name_expansion.NameExpansionResult):
Expand Down
2 changes: 1 addition & 1 deletion gslib/commands/defacl.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ def _ChDefAcl(self):
for uri in bucket_uris:
self.ApplyAclChanges(uri)

@Retry(GSResponseError, tries=3, delay=1, backoff=2)
@Retry(GSResponseError, tries=3, timeout_secs=1)
def ApplyAclChanges(self, uri):
"""Applies the changes in self.changes to the provided URI."""
try:
Expand Down
2 changes: 1 addition & 1 deletion gslib/commands/setmeta.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ def _SetMetadataExceptionHandler(e):
self.logger.error(str(e))
self.everything_set_okay = False

@Retry(GSResponseError, tries=3, delay=1, backoff=2)
@Retry(GSResponseError, tries=3, timeout_secs=1)
def _SetMetadataFunc(name_expansion_result):
exp_src_uri = self.suri_builder.StorageUri(
name_expansion_result.GetExpandedUriStr())
Expand Down
4 changes: 2 additions & 2 deletions gslib/tests/test_acl.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ def test_acl_set_version_specific_uri(self):

# Find out the two object version IDs.
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _GetVersions():
stdout = self.RunGsUtil(['ls', '-a', uri.uri], return_stdout=True)
lines = stdout.split('\n')
Expand Down Expand Up @@ -398,7 +398,7 @@ def testMultiVersionSupport(self):
bucket_uri=bucket, object_name=object_name, contents='Another thing')

# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1, logger=self.logger)
@Retry(AssertionError, tries=3, timeout_secs=1, fptr=self.logger)
def _getObjects():
stdout = self.RunGsUtil(['ls', '-a', suri(obj)], return_stdout=True)
lines = stdout.strip().split('\n')
Expand Down
32 changes: 16 additions & 16 deletions gslib/tests/test_cp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,15 +88,15 @@ def test_detect_content_type(self):

self.RunGsUtil(['cp', self._get_test_file('test.mp3'), dsturi])
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check1():
stdout = self.RunGsUtil(['ls', '-L', dsturi], return_stdout=True)
self.assertRegexpMatches(stdout, 'Content-Type:\s+audio/mpeg')
_Check1()

self.RunGsUtil(['cp', self._get_test_file('test.gif'), dsturi])
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check2():
stdout = self.RunGsUtil(['ls', '-L', dsturi], return_stdout=True)
self.assertRegexpMatches(stdout, 'Content-Type:\s+image/gif')
Expand All @@ -109,7 +109,7 @@ def test_content_type_override(self):
self.RunGsUtil(['-h', 'Content-Type:', 'cp',
self._get_test_file('test.mp3'), dsturi])
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check1():
stdout = self.RunGsUtil(['ls', '-L', dsturi], return_stdout=True)
self.assertRegexpMatches(stdout, 'Content-Type:\s+binary/octet-stream')
Expand All @@ -118,7 +118,7 @@ def _Check1():
self.RunGsUtil(['-h', 'Content-Type:', 'cp',
self._get_test_file('test.gif'), dsturi])
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check2():
stdout = self.RunGsUtil(['ls', '-L', dsturi], return_stdout=True)
self.assertRegexpMatches(stdout, 'Content-Type:\s+binary/octet-stream')
Expand All @@ -131,7 +131,7 @@ def test_foo_noct(self):
fpath = self.CreateTempFile(contents='foo/bar\n')
self.RunGsUtil(['cp', fpath, dsturi])
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check1():
stdout = self.RunGsUtil(['ls', '-L', dsturi], return_stdout=True)
USE_MAGICFILE = boto.config.getbool('GSUtil', 'use_magicfile', False)
Expand All @@ -149,7 +149,7 @@ def test_content_type_mismatches(self):
self.RunGsUtil(['-h', 'Content-Type:image/gif', 'cp',
self._get_test_file('test.mp3'), dsturi])
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check1():
stdout = self.RunGsUtil(['ls', '-L', dsturi], return_stdout=True)
self.assertRegexpMatches(stdout, 'Content-Type:\s+image/gif')
Expand All @@ -158,15 +158,15 @@ def _Check1():
self.RunGsUtil(['-h', 'Content-Type:image/gif', 'cp',
self._get_test_file('test.gif'), dsturi])
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check2():
stdout = self.RunGsUtil(['ls', '-L', dsturi], return_stdout=True)
self.assertRegexpMatches(stdout, 'Content-Type:\s+image/gif')
_Check2()

self.RunGsUtil(['-h', 'Content-Type:image/gif', 'cp', fpath, dsturi])
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check3():
stdout = self.RunGsUtil(['ls', '-L', dsturi], return_stdout=True)
self.assertRegexpMatches(stdout, 'Content-Type:\s+image/gif')
Expand All @@ -181,7 +181,7 @@ def test_content_type_header_case_insensitive(self):
self.RunGsUtil(['-h', 'content-Type:text/plain', 'cp',
fpath, dsturi])
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check1():
stdout = self.RunGsUtil(['ls', '-L', dsturi], return_stdout=True)
self.assertRegexpMatches(stdout, 'Content-Type:\s+text/plain')
Expand All @@ -192,7 +192,7 @@ def _Check1():
'-h', 'content-type:image/gif',
'cp', fpath, dsturi])
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check2():
stdout = self.RunGsUtil(['ls', '-L', dsturi], return_stdout=True)
self.assertRegexpMatches(stdout, 'Content-Type:\s+image/gif')
Expand Down Expand Up @@ -254,7 +254,7 @@ def test_recursive_copying_versioned_bucket(self):
# Recursively copy to second versioned bucket.
self.RunGsUtil(['cp', '-R', suri(bucket1_uri, '*'), suri(bucket2_uri)])
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check1():
listing1 = self.RunGsUtil(['ls', '-la', suri(bucket1_uri)],
return_stdout=True).split('\n')
Expand Down Expand Up @@ -325,7 +325,7 @@ def _run_cp_minus_v_test(self, opt, src_str, dst_str):
self.assertIsNotNone(match)
created_uri = match.group(1)
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check1():
stdout = self.RunGsUtil(['ls', '-a', dst_str], return_stdout=True)
lines = stdout.split('\n')
Expand All @@ -343,7 +343,7 @@ def test_stdin_args(self):
self.RunGsUtil(['cp', '-I', suri(bucket_uri)],
stdin='\n'.join((fpath1, fpath2)))
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check1():
stdout = self.RunGsUtil(['ls', suri(bucket_uri)], return_stdout=True)
self.assertIn(os.path.basename(fpath1), stdout)
Expand Down Expand Up @@ -377,7 +377,7 @@ def test_daisy_chain_cp(self):
stderr = self.RunGsUtil(['cp', '-Dpn', suri(key_uri), suri(bucket2_uri)],
return_stderr=True)
self.assertNotIn('Copy-in-the-cloud disallowed', stderr)
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check():
uri = suri(bucket2_uri, key_uri.object_name)
stdout = self.RunGsUtil(['ls', '-L', uri], return_stdout=True)
Expand Down Expand Up @@ -412,7 +412,7 @@ def test_copy_bucket_to_bucket(self):
self.CreateObject(bucket_uri=src_bucket_uri, object_name='obj1',
contents='def')
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _CopyAndCheck():
self.RunGsUtil(['cp', '-R', suri(src_bucket_uri),
suri(dst_bucket_uri)])
Expand All @@ -435,7 +435,7 @@ def test_copy_bucket_to_dir(self):
self.CreateObject(bucket_uri=src_bucket_uri, object_name='obj1',
contents='def')
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _CopyAndCheck():
self.RunGsUtil(['cp', '-R', suri(src_bucket_uri), dst_dir])
dir_list = []
Expand Down
24 changes: 12 additions & 12 deletions gslib/tests/test_du.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ def _create_nested_subdir(self):
def test_object(self):
obj_uri = self.CreateObject(contents='foo')
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check():
stdout = self.RunGsUtil(['du', suri(obj_uri)], return_stdout=True)
self.assertEqual(stdout, '%-10s %s\n' % (3, suri(obj_uri)))
Expand All @@ -46,7 +46,7 @@ def test_bucket(self):
bucket_uri = self.CreateBucket()
obj_uri = self.CreateObject(bucket_uri=bucket_uri, contents='foo')
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check():
stdout = self.RunGsUtil(['du', suri(bucket_uri)], return_stdout=True)
self.assertEqual(stdout, '%-10s %s\n' % (3, suri(obj_uri)))
Expand All @@ -56,7 +56,7 @@ def test_subdirs(self):
bucket_uri, obj_uris = self._create_nested_subdir()

# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check():
stdout = self.RunGsUtil(['du', suri(bucket_uri)], return_stdout=True)
self.assertSetEqual(set(stdout.splitlines()), set([
Expand All @@ -74,7 +74,7 @@ def test_multi_args(self):
obj_uri1 = self.CreateObject(bucket_uri=bucket_uri, contents='foo')
obj_uri2 = self.CreateObject(bucket_uri=bucket_uri, contents='foo2')
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check():
stdout = self.RunGsUtil(['du', suri(obj_uri1), suri(obj_uri2)],
return_stdout=True)
Expand All @@ -89,7 +89,7 @@ def test_total(self):
obj_uri1 = self.CreateObject(bucket_uri=bucket_uri, contents='foo')
obj_uri2 = self.CreateObject(bucket_uri=bucket_uri, contents='zebra')
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check():
stdout = self.RunGsUtil(['du', '-c', suri(bucket_uri)],
return_stdout=True)
Expand All @@ -103,7 +103,7 @@ def _Check():
def test_human_readable(self):
obj_uri = self.CreateObject(contents='x' * 2048)
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check():
stdout = self.RunGsUtil(['du', '-h', suri(obj_uri)], return_stdout=True)
self.assertEqual(stdout, '%-10s %s\n' % ('2 KB', suri(obj_uri)))
Expand All @@ -114,7 +114,7 @@ def test_summary(self):
bucket_uri2, _ = self._create_nested_subdir()

# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check():
stdout = self.RunGsUtil([
'du', '-s', suri(bucket_uri1), suri(bucket_uri2)], return_stdout=True)
Expand All @@ -132,14 +132,14 @@ def test_versioned(self):
bucket_uri=bucket_uri, object_name='foo', contents='foo2')

# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check1():
stdout = self.RunGsUtil(['du', suri(bucket_uri)], return_stdout=True)
self.assertEqual(stdout, '%-10s %s\n' % (4, suri(object_uri2)))
_Check1()

# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check2():
stdout = self.RunGsUtil(['du', '-a', suri(bucket_uri)],
return_stdout=True)
Expand All @@ -156,7 +156,7 @@ def test_null_endings(self):
obj_uri1 = self.CreateObject(bucket_uri=bucket_uri, contents='foo')
obj_uri2 = self.CreateObject(bucket_uri=bucket_uri, contents='zebra')
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check():
stdout = self.RunGsUtil(['du', '-0c', suri(bucket_uri)],
return_stdout=True)
Expand All @@ -172,7 +172,7 @@ def test_excludes(self):
bucket_uri, obj_uris = self._create_nested_subdir()

# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check():
stdout = self.RunGsUtil([
'du', '-e', '*sub2/five*', '-e', '*sub1/four',
Expand All @@ -190,7 +190,7 @@ def test_excludes_file(self):
fpath = self.CreateTempFile(contents='*sub2/five*\n*sub1/four')

# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check():
stdout = self.RunGsUtil([
'du', '-X', fpath, suri(bucket_uri)], return_stdout=True)
Expand Down
Loading