Merge branch 'valentindavid/netrc' into 'master'

Add support for .netrc in remote/tar/zip plugins

Closes apache#723

See merge request BuildStream/buildstream!908

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-image: buildstream/testsuite-debian:9-master-119-552f5fc6
+image: buildstream/testsuite-debian:9-master-123-7ce6581b
 
 cache:
   key: "$CI_JOB_NAME-"
@@ -140,7 +140,7 @@ tests-unix:
 
 tests-fedora-missing-deps:
   # Ensure that tests behave nicely while missing bwrap and ostree
-  image: buildstream/testsuite-fedora:28-master-119-552f5fc6
+  image: buildstream/testsuite-fedora:28-master-123-7ce6581b
   <<: *tests
 
   script:

buildstream/plugins/sources/_downloadablefilesource.py

@@ -5,16 +5,77 @@
 import urllib.error
 import contextlib
 import shutil
+import netrc
 
 from buildstream import Source, SourceError, Consistency
 from buildstream import utils
 
 
+class _NetrcFTPOpener(urllib.request.FTPHandler):
+
+    def __init__(self, netrc_config):
+        self.netrc = netrc_config
+
+    def _split(self, netloc):
+        userpass, hostport = urllib.parse.splituser(netloc)
+        host, port = urllib.parse.splitport(hostport)
+        if userpass:
+            user, passwd = urllib.parse.splitpasswd(userpass)
+        else:
+            user = None
+            passwd = None
+        return host, port, user, passwd
+
+    def _unsplit(self, host, port, user, passwd):
+        if port:
+            host = '{}:{}'.format(host, port)
+        if user:
+            if passwd:
+                user = '{}:{}'.format(user, passwd)
+            host = '{}@{}'.format(user, host)
+
+        return host
+
+    def ftp_open(self, req):
+        host, port, user, passwd = self._split(req.host)
+
+        if user is None and self.netrc:
+            entry = self.netrc.authenticators(host)
+            if entry:
+                user, _, passwd = entry
+
+        req.host = self._unsplit(host, port, user, passwd)
+
+        return super().ftp_open(req)
+
+
+class _NetrcPasswordManager:
+
+    def __init__(self, netrc_config):
+        self.netrc = netrc_config
+
+    def add_password(self, realm, uri, user, passwd):
+        pass
+
+    def find_user_password(self, realm, authuri):
+        if not self.netrc:
+            return None, None
+        parts = urllib.parse.urlsplit(authuri)
+        entry = self.netrc.authenticators(parts.hostname)
+        if not entry:
+            return None, None
+        else:
+            login, _, password = entry
+            return login, password
+
+
 class DownloadableFileSource(Source):
     # pylint: disable=attribute-defined-outside-init
 
     COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ['url', 'ref', 'etag']
 
+    __urlopener = None
+
     def configure(self, node):
         self.original_url = self.node_get_member(node, str, 'url')
         self.ref = self.node_get_member(node, str, 'ref', None)
@@ -118,7 +179,8 @@ def _ensure_mirror(self):
                     if etag and self.get_consistency() == Consistency.CACHED:
                         request.add_header('If-None-Match', etag)
 
-                with contextlib.closing(urllib.request.urlopen(request)) as response:
+                opener = self.__get_urlopener()
+                with contextlib.closing(opener.open(request)) as response:
                     info = response.info()
 
                     etag = info['ETag'] if 'ETag' in info else None
@@ -164,3 +226,19 @@ def _get_mirror_dir(self):
 
     def _get_mirror_file(self, sha=None):
         return os.path.join(self._get_mirror_dir(), sha or self.ref)
+
+    def __get_urlopener(self):
+        if not DownloadableFileSource.__urlopener:
+            try:
+                netrc_config = netrc.netrc()
+            except FileNotFoundError:
+                DownloadableFileSource.__urlopener = urllib.request.build_opener()
+            except netrc.NetrcParseError as e:
+                self.warn('{}: While reading .netrc: {}'.format(self, e))
+                return urllib.request.build_opener()
+            else:
+                netrc_pw_mgr = _NetrcPasswordManager(netrc_config)
+                http_auth = urllib.request.HTTPBasicAuthHandler(netrc_pw_mgr)
+                ftp_handler = _NetrcFTPOpener(netrc_config)
+                DownloadableFileSource.__urlopener = urllib.request.build_opener(http_auth, ftp_handler)
+        return DownloadableFileSource.__urlopener

dev-requirements.txt

@@ -9,3 +9,4 @@ pytest-pep8
 pytest-pylint
 pytest-xdist
 pytest-timeout
+pyftpdlib

tests/sources/remote.py

@@ -5,6 +5,7 @@
 from buildstream._exceptions import ErrorDomain
 from buildstream import _yaml
 from tests.testutils import cli
+from tests.testutils.file_server import create_file_server
 
 DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
@@ -22,6 +23,16 @@ def generate_project(project_dir, tmpdir):
     }, project_file)
 
 
+def generate_project_file_server(server, project_dir):
+    project_file = os.path.join(project_dir, "project.conf")
+    _yaml.dump({
+        'name': 'foo',
+        'aliases': {
+            'tmpdir': server.base_url()
+        }
+    }, project_file)
+
+
 # Test that without ref, consistency is set appropriately.
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref'))
 def test_no_ref(cli, tmpdir, datafiles):
@@ -164,3 +175,35 @@ def test_executable(cli, tmpdir, datafiles):
     assert (mode & stat.S_IEXEC)
     # Assert executable by anyone
     assert(mode & (stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH))
+
+
+@pytest.mark.parametrize('server_type', ('FTP', 'HTTP'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'single-file'))
+def test_use_netrc(cli, datafiles, server_type, tmpdir):
+    fake_home = os.path.join(str(tmpdir), 'fake_home')
+    os.makedirs(fake_home, exist_ok=True)
+    project = str(datafiles)
+    checkoutdir = os.path.join(str(tmpdir), 'checkout')
+
+    os.environ['HOME'] = fake_home
+    with open(os.path.join(fake_home, '.netrc'), 'wb') as f:
+        os.fchmod(f.fileno(), 0o700)
+        f.write(b'machine 127.0.0.1\n')
+        f.write(b'login testuser\n')
+        f.write(b'password 12345\n')
+
+    with create_file_server(server_type) as server:
+        server.add_user('testuser', '12345', project)
+        generate_project_file_server(server, project)
+
+        server.start()
+
+        result = cli.run(project=project, args=['fetch', 'target.bst'])
+        result.assert_success()
+        result = cli.run(project=project, args=['build', 'target.bst'])
+        result.assert_success()
+        result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir])
+        result.assert_success()
+
+        checkout_file = os.path.join(checkoutdir, 'file')
+        assert(os.path.exists(checkout_file))

tests/sources/tar.py

@@ -3,11 +3,13 @@
 import tarfile
 import tempfile
 import subprocess
+import urllib.parse
 from shutil import copyfile, rmtree
 
 from buildstream._exceptions import ErrorDomain
 from buildstream import _yaml
 from tests.testutils import cli
+from tests.testutils.file_server import create_file_server
 from tests.testutils.site import HAVE_LZIP
 from . import list_dir_contents
 
@@ -49,6 +51,16 @@ def generate_project(project_dir, tmpdir):
     }, project_file)
 
 
+def generate_project_file_server(base_url, project_dir):
+    project_file = os.path.join(project_dir, "project.conf")
+    _yaml.dump({
+        'name': 'foo',
+        'aliases': {
+            'tmpdir': base_url
+        }
+    }, project_file)
+
+
 # Test that without ref, consistency is set appropriately.
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref'))
 def test_no_ref(cli, tmpdir, datafiles):
@@ -302,3 +314,77 @@ def make_dir_writable(fn, path, excinfo):
             else:
                 os.remove(path)
         rmtree(str(tmpdir), onerror=make_dir_writable)
+
+
+@pytest.mark.parametrize('server_type', ('FTP', 'HTTP'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+def test_use_netrc(cli, datafiles, server_type, tmpdir):
+    file_server_files = os.path.join(str(tmpdir), 'file_server')
+    fake_home = os.path.join(str(tmpdir), 'fake_home')
+    os.makedirs(file_server_files, exist_ok=True)
+    os.makedirs(fake_home, exist_ok=True)
+    project = str(datafiles)
+    checkoutdir = os.path.join(str(tmpdir), 'checkout')
+
+    os.environ['HOME'] = fake_home
+    with open(os.path.join(fake_home, '.netrc'), 'wb') as f:
+        os.fchmod(f.fileno(), 0o700)
+        f.write(b'machine 127.0.0.1\n')
+        f.write(b'login testuser\n')
+        f.write(b'password 12345\n')
+
+    with create_file_server(server_type) as server:
+        server.add_user('testuser', '12345', file_server_files)
+        generate_project_file_server(server.base_url(), project)
+
+        src_tar = os.path.join(file_server_files, 'a.tar.gz')
+        _assemble_tar(os.path.join(str(datafiles), 'content'), 'a', src_tar)
+
+        server.start()
+
+        result = cli.run(project=project, args=['track', 'target.bst'])
+        result.assert_success()
+        result = cli.run(project=project, args=['fetch', 'target.bst'])
+        result.assert_success()
+        result = cli.run(project=project, args=['build', 'target.bst'])
+        result.assert_success()
+        result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir])
+        result.assert_success()
+
+        original_dir = os.path.join(str(datafiles), 'content', 'a')
+        original_contents = list_dir_contents(original_dir)
+        checkout_contents = list_dir_contents(checkoutdir)
+        assert(checkout_contents == original_contents)
+
+
+@pytest.mark.parametrize('server_type', ('FTP', 'HTTP'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+def test_netrc_already_specified_user(cli, datafiles, server_type, tmpdir):
+    file_server_files = os.path.join(str(tmpdir), 'file_server')
+    fake_home = os.path.join(str(tmpdir), 'fake_home')
+    os.makedirs(file_server_files, exist_ok=True)
+    os.makedirs(fake_home, exist_ok=True)
+    project = str(datafiles)
+    checkoutdir = os.path.join(str(tmpdir), 'checkout')
+
+    os.environ['HOME'] = fake_home
+    with open(os.path.join(fake_home, '.netrc'), 'wb') as f:
+        os.fchmod(f.fileno(), 0o700)
+        f.write(b'machine 127.0.0.1\n')
+        f.write(b'login testuser\n')
+        f.write(b'password 12345\n')
+
+    with create_file_server(server_type) as server:
+        server.add_user('otheruser', '12345', file_server_files)
+        parts = urllib.parse.urlsplit(server.base_url())
+        base_url = urllib.parse.urlunsplit([parts[0]] + ['otheruser@{}'.format(parts[1])] + list(parts[2:]))
+        generate_project_file_server(base_url, project)
+
+        src_tar = os.path.join(file_server_files, 'a.tar.gz')
+        _assemble_tar(os.path.join(str(datafiles), 'content'), 'a', src_tar)
+
+        server.start()
+
+        result = cli.run(project=project, args=['track', 'target.bst'])
+        result.assert_main_error(ErrorDomain.STREAM, None)
+        result.assert_task_error(ErrorDomain.SOURCE, None)

tests/sources/zip.py

@@ -5,6 +5,7 @@
 from buildstream._exceptions import ErrorDomain
 from buildstream import _yaml
 from tests.testutils import cli
+from tests.testutils.file_server import create_file_server
 from . import list_dir_contents
 
 DATA_DIR = os.path.join(
@@ -35,6 +36,16 @@ def generate_project(project_dir, tmpdir):
     }, project_file)
 
 
+def generate_project_file_server(server, project_dir):
+    project_file = os.path.join(project_dir, "project.conf")
+    _yaml.dump({
+        'name': 'foo',
+        'aliases': {
+            'tmpdir': server.base_url()
+        }
+    }, project_file)
+
+
 # Test that without ref, consistency is set appropriately.
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref'))
 def test_no_ref(cli, tmpdir, datafiles):
@@ -176,3 +187,44 @@ def test_stage_explicit_basedir(cli, tmpdir, datafiles):
     original_contents = list_dir_contents(original_dir)
     checkout_contents = list_dir_contents(checkoutdir)
     assert(checkout_contents == original_contents)
+
+
+@pytest.mark.parametrize('server_type', ('FTP', 'HTTP'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+def test_use_netrc(cli, datafiles, server_type, tmpdir):
+    file_server_files = os.path.join(str(tmpdir), 'file_server')
+    fake_home = os.path.join(str(tmpdir), 'fake_home')
+    os.makedirs(file_server_files, exist_ok=True)
+    os.makedirs(fake_home, exist_ok=True)
+    project = str(datafiles)
+    checkoutdir = os.path.join(str(tmpdir), 'checkout')
+
+    os.environ['HOME'] = fake_home
+    with open(os.path.join(fake_home, '.netrc'), 'wb') as f:
+        os.fchmod(f.fileno(), 0o700)
+        f.write(b'machine 127.0.0.1\n')
+        f.write(b'login testuser\n')
+        f.write(b'password 12345\n')
+
+    with create_file_server(server_type) as server:
+        server.add_user('testuser', '12345', file_server_files)
+        generate_project_file_server(server, project)
+
+        src_zip = os.path.join(file_server_files, 'a.zip')
+        _assemble_zip(os.path.join(str(datafiles), 'content'), src_zip)
+
+        server.start()
+
+        result = cli.run(project=project, args=['track', 'target.bst'])
+        result.assert_success()
+        result = cli.run(project=project, args=['fetch', 'target.bst'])
+        result.assert_success()
+        result = cli.run(project=project, args=['build', 'target.bst'])
+        result.assert_success()
+        result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir])
+        result.assert_success()
+
+        original_dir = os.path.join(str(datafiles), 'content', 'a')
+        original_contents = list_dir_contents(original_dir)
+        checkout_contents = list_dir_contents(checkoutdir)
+        assert(checkout_contents == original_contents)

tests/testutils/file_server.py

@@ -0,0 +1,19 @@
+from contextlib import contextmanager
+
+from .ftp_server import SimpleFtpServer
+from .http_server import SimpleHttpServer
+
+
+@contextmanager
+def create_file_server(file_server_type):
+    if file_server_type == 'FTP':
+        server = SimpleFtpServer()
+    elif file_server_type == 'HTTP':
+        server = SimpleHttpServer()
+    else:
+        assert False
+
+    try:
+        yield server
+    finally:
+        server.stop()