Skip to content

Commit

Permalink
Add support for golang.org/x remote libs.
Browse files Browse the repository at this point in the history
Although golang.org/x supports the `go get` <meta> protocol, pants does
not have a `Fetcher` that supports that protocol yet, so just use the
standard `ArchiveFetcher` with a regex mapping that redirects directly
to github (this appears to be the consistent pattern of th golang.org/x
<meta> re-directs).

Add a test to cover this new default fetcher configuration.

Testing Done:
Manually tested this in a custom Medium pants.ini setup:
```ini
[fetchers]
mapping: {
    # Builtin fetchers:
    r'^bitbucket.org/.*$': 'ArchiveFetcher',
    r'^github.com/.*$':'ArchiveFetcher',
    r'^gopkg.in/.*$':'GopkgInFetcher',

    # Our workaround custom fetcher until
    # pantsbuild#2378 or
    # pantsbuild#2379 is resolved.
    r'^golang.org/x/.*$':'ArchiveFetcher',
  }

[archive-fetcher]
matchers: {
    # Builtin matchers:
    r'bitbucket.org/(?P<user>[^/]+)/(?P<repo>[^/]+)':
      ('https://bitbucket.org/\g<user>/\g<repo>/get/{rev}.tar.gz', 'tip', 1),
    r'github.com/(?P<user>[^/]+)/(?P<repo>[^/]+)':
      ('https://github.com/\g<user>/\g<repo>/archive/{rev}.tar.gz', 'master', 1),

    # Our workaround custom fetcher until
    # pantsbuild#2378 or
    # pantsbuild#2379 is resolved.
    r'golang.org/x/(?P<user>[^/]+)':
      ('https://github.com/golang/\g<user>/archive/{rev}.tar.gz', 'master', 1),
  }
```

CI went green here:
  https://travis-ci.org/pantsbuild/pants/builds/85667224

Bugs closed: 2378, 2379, 2380

Reviewed at https://rbcommons.com/s/twitter/r/2976/
  • Loading branch information
jsirois committed Oct 16, 2015
1 parent e4e0d62 commit 18e00f7
Show file tree
Hide file tree
Showing 2 changed files with 50 additions and 3 deletions.
9 changes: 7 additions & 2 deletions contrib/go/src/python/pants/contrib/go/subsystems/fetchers.py
Original file line number Diff line number Diff line change
Expand Up @@ -280,14 +280,18 @@ def rev(self, rev):
options_scope = 'archive-fetcher'

_DEFAULT_MATCHERS = {
r'bitbucket.org/(?P<user>[^/]+)/(?P<repo>[^/]+)':
r'bitbucket\.org/(?P<user>[^/]+)/(?P<repo>[^/]+)':
UrlInfo(url_format='https://bitbucket.org/\g<user>/\g<repo>/get/{rev}.tar.gz',
default_rev='tip',
strip_level=1),
r'github.com/(?P<user>[^/]+)/(?P<repo>[^/]+)':
r'github\.com/(?P<user>[^/]+)/(?P<repo>[^/]+)':
UrlInfo(url_format='https://github.com/\g<user>/\g<repo>/archive/{rev}.tar.gz',
default_rev='master',
strip_level=1),
r'golang\.org/x/(?P<repo>[^/]+)':
UrlInfo(url_format='https://github.com/golang/\g<repo>/archive/{rev}.tar.gz',
default_rev='master',
strip_level=1),
}

@classmethod
Expand Down Expand Up @@ -569,3 +573,4 @@ def _select_highest_compatible(self, candidates, raw_rev):
Fetchers.advertise(ArchiveFetcher, namespace='')
Fetchers._register_default(r'^bitbucket\.org/.*$', ArchiveFetcher)
Fetchers._register_default(r'^github\.com/.*$', ArchiveFetcher)
Fetchers._register_default(r'^golang\.org/x/.*$', ArchiveFetcher)
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from pants.util.contextutil import temporary_dir
from pants_test.subsystem.subsystem_util import subsystem_instance

from pants.contrib.go.subsystems.fetchers import Fetchers, GopkgInFetcher
from pants.contrib.go.subsystems.fetchers import ArchiveFetcher, Fetchers, GopkgInFetcher


class FetchersTest(unittest.TestCase):
Expand Down Expand Up @@ -51,10 +51,52 @@ def test_default_github(self):
self.check_default('github.com/docker/docker/daemon/events',
expected_root='github.com/docker/docker')

def test_default_golang(self):
self.check_default('golang.org/x/oauth2',
expected_root='golang.org/x/oauth2')
self.check_default('golang.org/x/net/context',
expected_root='golang.org/x/net')

def test_default_gopkg(self):
self.check_default('gopkg.in/check.v1', expected_root='gopkg.in/check.v1')


class GolangOrgFetcherTest(unittest.TestCase):

def do_fetch(self, import_path, expected_url, rev=None):
with subsystem_instance(ArchiveFetcher) as fetcher:
fetcher._fetch = mock.MagicMock(spec=fetcher._fetch)
with temporary_dir() as dest:
fetcher.fetch(import_path, dest, rev=rev)

# For some reason using `assert_called_once_with` like so:
# fetcher._fetch.assert_called_once_with(expected_url)
#
# Yields this error:
# E AssertionError:
# Expected call: mock(u'https://github.com/golang/oauth2/archive/master.tar.gz')
# E Actual call: mock(u'https://github.com/golang/oauth2/archive/master.tar.gz')
# E too many positional arguments
#
# So we manually check called once with in 2 steps.
self.assertEqual(1, fetcher._fetch.call_count)
self.assertEqual(mock.call(expected_url), fetcher._fetch.call_args)

def test_fetch(self):
self.do_fetch(import_path='golang.org/x/oauth2',
expected_url='https://github.com/golang/oauth2/archive/master.tar.gz')
self.do_fetch(import_path='golang.org/x/net/context',
expected_url='https://github.com/golang/net/archive/master.tar.gz')

def test_fetch_rev(self):
self.do_fetch(import_path='golang.org/x/oauth2',
rev='abc123',
expected_url='https://github.com/golang/oauth2/archive/abc123.tar.gz')
self.do_fetch(import_path='golang.org/x/net/context',
rev='def456',
expected_url='https://github.com/golang/net/archive/def456.tar.gz')


class GopkgInFetcherTest(unittest.TestCase):

def do_fetch(self, import_path, version_override=None, github_api_responses=None,
Expand Down

0 comments on commit 18e00f7

Please sign in to comment.