Merge pull request docker#2223 from docker/3.7.0-release

3.7.0 release

Jenkinsfile

@@ -45,10 +45,13 @@ def getDockerVersions = { ->
 }
 
 def getAPIVersion = { engineVersion ->
-  def versionMap = ['17.06': '1.30', '17.12': '1.35', '18.02': '1.36', '18.03': '1.37']
+  def versionMap = [
+    '17.06': '1.30', '17.12': '1.35', '18.02': '1.36', '18.03': '1.37',
+    '18.06': '1.38', '18.09': '1.39'
+  ]
   def result = versionMap[engineVersion.substring(0, 5)]
   if (!result) {
-    return '1.37'
+    return '1.39'
   }
   return result
 }
@@ -88,7 +91,7 @@ def runTests = { Map settings ->
             --network ${testNetwork} \\
             --volumes-from ${dindContainerName} \\
             ${testImage} \\
-            py.test -v -rxs tests/integration
+            py.test -v -rxs --cov=docker tests/
           """
         } finally {
           sh """

docker/api/build.py

@@ -19,7 +19,8 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None,
               forcerm=False, dockerfile=None, container_limits=None,
               decode=False, buildargs=None, gzip=False, shmsize=None,
               labels=None, cache_from=None, target=None, network_mode=None,
-              squash=None, extra_hosts=None, platform=None, isolation=None):
+              squash=None, extra_hosts=None, platform=None, isolation=None,
+              use_config_proxy=False):
         """
         Similar to the ``docker build`` command. Either ``path`` or ``fileobj``
         needs to be set. ``path`` can be a local path (to a directory
@@ -103,6 +104,10 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None,
             platform (str): Platform in the format ``os[/arch[/variant]]``
             isolation (str): Isolation technology used during build.
                 Default: `None`.
+            use_config_proxy (bool): If ``True``, and if the docker client
+                configuration file (``~/.docker/config.json`` by default)
+                contains a proxy configuration, the corresponding environment
+                variables will be set in the container being built.
 
         Returns:
             A generator for the build output.
@@ -168,6 +173,10 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None,
         }
         params.update(container_limits)
 
+        if use_config_proxy:
+            proxy_args = self._proxy_configs.get_environment()
+            for k, v in proxy_args.items():
+                buildargs.setdefault(k, v)
         if buildargs:
             params.update({'buildargs': json.dumps(buildargs)})
 
@@ -286,48 +295,31 @@ def _set_auth_headers(self, headers):
 
         # If we don't have any auth data so far, try reloading the config
         # file one more time in case anything showed up in there.
-        if not self._auth_configs:
+        if not self._auth_configs or self._auth_configs.is_empty:
             log.debug("No auth config in memory - loading from filesystem")
-            self._auth_configs = auth.load_config()
+            self._auth_configs = auth.load_config(
+                credstore_env=self.credstore_env
+            )
 
         # Send the full auth configuration (if any exists), since the build
         # could use any (or all) of the registries.
         if self._auth_configs:
-            auth_cfgs = self._auth_configs
-            auth_data = {}
-            if auth_cfgs.get('credsStore'):
-                # Using a credentials store, we need to retrieve the
-                # credentials for each registry listed in the config.json file
-                # Matches CLI behavior: https://github.com/docker/docker/blob/
-                # 67b85f9d26f1b0b2b240f2d794748fac0f45243c/cliconfig/
-                # credentials/native_store.go#L68-L83
-                for registry in auth_cfgs.get('auths', {}).keys():
-                    auth_data[registry] = auth.resolve_authconfig(
-                        auth_cfgs, registry,
-                        credstore_env=self.credstore_env,
-                    )
-            else:
-                for registry in auth_cfgs.get('credHelpers', {}).keys():
-                    auth_data[registry] = auth.resolve_authconfig(
-                        auth_cfgs, registry,
-                        credstore_env=self.credstore_env
-                    )
-                for registry, creds in auth_cfgs.get('auths', {}).items():
-                    if registry not in auth_data:
-                        auth_data[registry] = creds
-                # See https://github.com/docker/docker-py/issues/1683
-                if auth.INDEX_NAME in auth_data:
-                    auth_data[auth.INDEX_URL] = auth_data[auth.INDEX_NAME]
+            auth_data = self._auth_configs.get_all_credentials()
+
+            # See https://github.com/docker/docker-py/issues/1683
+            if auth.INDEX_URL not in auth_data and auth.INDEX_URL in auth_data:
+                auth_data[auth.INDEX_URL] = auth_data.get(auth.INDEX_NAME, {})
 
             log.debug(
                 'Sending auth config ({0})'.format(
                     ', '.join(repr(k) for k in auth_data.keys())
                 )
             )
 
-            headers['X-Registry-Config'] = auth.encode_header(
-                auth_data
-            )
+            if auth_data:
+                headers['X-Registry-Config'] = auth.encode_header(
+                    auth_data
+                )
         else:
             log.debug('No auth config found')
 

docker/api/client.py

@@ -32,8 +32,9 @@
 from ..tls import TLSConfig
 from ..transport import SSLAdapter, UnixAdapter
 from ..utils import utils, check_resource, update_headers, config
-from ..utils.socket import frames_iter, socket_raw_iter
+from ..utils.socket import frames_iter, consume_socket_output, demux_adaptor
 from ..utils.json_stream import json_stream
+from ..utils.proxy import ProxyConfig
 try:
     from ..transport import NpipeAdapter
 except ImportError:
@@ -114,8 +115,17 @@ def __init__(self, base_url=None, version=None,
         self.headers['User-Agent'] = user_agent
 
         self._general_configs = config.load_general_config()
+
+        proxy_config = self._general_configs.get('proxies', {})
+        try:
+            proxies = proxy_config[base_url]
+        except KeyError:
+            proxies = proxy_config.get('default', {})
+
+        self._proxy_configs = ProxyConfig.from_dict(proxies)
+
         self._auth_configs = auth.load_config(
-            config_dict=self._general_configs
+            config_dict=self._general_configs, credstore_env=credstore_env,
         )
         self.credstore_env = credstore_env
 
@@ -381,19 +391,23 @@ def _stream_raw_result(self, response, chunk_size=1, decode=True):
         for out in response.iter_content(chunk_size, decode):
             yield out
 
-    def _read_from_socket(self, response, stream, tty=False):
+    def _read_from_socket(self, response, stream, tty=True, demux=False):
         socket = self._get_raw_response_socket(response)
 
-        gen = None
-        if tty is False:
-            gen = frames_iter(socket)
+        gen = frames_iter(socket, tty)
+
+        if demux:
+            # The generator will output tuples (stdout, stderr)
+            gen = (demux_adaptor(*frame) for frame in gen)
         else:
-            gen = socket_raw_iter(socket)
+            # The generator will output strings
+            gen = (data for (_, data) in gen)
 
         if stream:
             return gen
         else:
-            return six.binary_type().join(gen)
+            # Wait for all the frames, concatenate them, and return the result
+            return consume_socket_output(gen, demux=demux)
 
     def _disable_socket_timeout(self, socket):
         """ Depending on the combination of python version and whether we're
@@ -476,4 +490,6 @@ def reload_config(self, dockercfg_path=None):
         Returns:
             None
         """
-        self._auth_configs = auth.load_config(dockercfg_path)
+        self._auth_configs = auth.load_config(
+            dockercfg_path, credstore_env=self.credstore_env
+        )

docker/api/config.py

@@ -42,7 +42,7 @@ def inspect_config(self, id):
             Retrieve config metadata
 
             Args:
-                id (string): Full ID of the config to remove
+                id (string): Full ID of the config to inspect
 
             Returns (dict): A dictionary of metadata
 

docker/api/container.py

@@ -13,7 +13,7 @@
 class ContainerApiMixin(object):
     @utils.check_resource('container')
     def attach(self, container, stdout=True, stderr=True,
-               stream=False, logs=False):
+               stream=False, logs=False, demux=False):
         """
         Attach to a container.
 
@@ -28,11 +28,15 @@ def attach(self, container, stdout=True, stderr=True,
             stream (bool): Return container output progressively as an iterator
                 of strings, rather than a single string.
             logs (bool): Include the container's previous output.
+            demux (bool): Keep stdout and stderr separate.
 
         Returns:
-            By default, the container's output as a single string.
+            By default, the container's output as a single string (two if
+            ``demux=True``: one for stdout and one for stderr).
 
-            If ``stream=True``, an iterator of output strings.
+            If ``stream=True``, an iterator of output strings. If
+            ``demux=True``, two iterators are returned: one for stdout and one
+            for stderr.
 
         Raises:
             :py:class:`docker.errors.APIError`
@@ -54,8 +58,7 @@ def attach(self, container, stdout=True, stderr=True,
         response = self._post(u, headers=headers, params=params, stream=True)
 
         output = self._read_from_socket(
-            response, stream, self._check_is_tty(container)
-        )
+            response, stream, self._check_is_tty(container), demux=demux)
 
         if stream:
             return CancellableStream(output, response)
@@ -218,7 +221,8 @@ def create_container(self, image, command=None, hostname=None, user=None,
                          working_dir=None, domainname=None, host_config=None,
                          mac_address=None, labels=None, stop_signal=None,
                          networking_config=None, healthcheck=None,
-                         stop_timeout=None, runtime=None):
+                         stop_timeout=None, runtime=None,
+                         use_config_proxy=False):
         """
         Creates a container. Parameters are similar to those for the ``docker
         run`` command except it doesn't support the attach options (``-a``).
@@ -387,6 +391,10 @@ def create_container(self, image, command=None, hostname=None, user=None,
             runtime (str): Runtime to use with this container.
             healthcheck (dict): Specify a test to perform to check that the
                 container is healthy.
+            use_config_proxy (bool): If ``True``, and if the docker client
+                configuration file (``~/.docker/config.json`` by default)
+                contains a proxy configuration, the corresponding environment
+                variables will be set in the container being created.
 
         Returns:
             A dictionary with an image 'Id' key and a 'Warnings' key.
@@ -400,6 +408,14 @@ def create_container(self, image, command=None, hostname=None, user=None,
         if isinstance(volumes, six.string_types):
             volumes = [volumes, ]
 
+        if isinstance(environment, dict):
+            environment = utils.utils.format_environment(environment)
+
+        if use_config_proxy:
+            environment = self._proxy_configs.inject_proxy_environment(
+                environment
+            )
+
         config = self.create_container_config(
             image, command, hostname, user, detach, stdin_open, tty,
             ports, environment, volumes,

docker/api/daemon.py

@@ -42,8 +42,8 @@ def events(self, since=None, until=None, filters=None, decode=None):
 
         Example:
 
-            >>> for event in client.events()
-            ...   print event
+            >>> for event in client.events(decode=True)
+            ...   print(event)
             {u'from': u'image/with:tag',
              u'id': u'container-id',
              u'status': u'start',
@@ -54,7 +54,7 @@ def events(self, since=None, until=None, filters=None, decode=None):
 
             >>> events = client.events()
             >>> for event in events:
-            ...   print event
+            ...   print(event)
             >>> # and cancel from another thread
             >>> events.close()
         """
@@ -124,13 +124,15 @@ def login(self, username, password=None, email=None, registry=None,
         # If dockercfg_path is passed check to see if the config file exists,
         # if so load that config.
         if dockercfg_path and os.path.exists(dockercfg_path):
-            self._auth_configs = auth.load_config(dockercfg_path)
-        elif not self._auth_configs:
-            self._auth_configs = auth.load_config()
-
-        authcfg = auth.resolve_authconfig(
-            self._auth_configs, registry, credstore_env=self.credstore_env,
-        )
+            self._auth_configs = auth.load_config(
+                dockercfg_path, credstore_env=self.credstore_env
+            )
+        elif not self._auth_configs or self._auth_configs.is_empty:
+            self._auth_configs = auth.load_config(
+                credstore_env=self.credstore_env
+            )
+
+        authcfg = self._auth_configs.resolve_authconfig(registry)
         # If we found an existing auth config for this registry and username
         # combination, we can return it immediately unless reauth is requested.
         if authcfg and authcfg.get('username', None) == username \
@@ -146,9 +148,7 @@ def login(self, username, password=None, email=None, registry=None,
 
         response = self._post_json(self._url('/auth'), data=req_data)
         if response.status_code == 200:
-            if 'auths' not in self._auth_configs:
-                self._auth_configs['auths'] = {}
-            self._auth_configs['auths'][registry or auth.INDEX_NAME] = req_data
+            self._auth_configs.add_auth(registry or auth.INDEX_NAME, req_data)
         return self._result(response, json=True)
 
     def ping(self):

docker/api/exec_api.py

@@ -118,7 +118,7 @@ def exec_resize(self, exec_id, height=None, width=None):
 
     @utils.check_resource('exec_id')
     def exec_start(self, exec_id, detach=False, tty=False, stream=False,
-                   socket=False):
+                   socket=False, demux=False):
         """
         Start a previously set up exec instance.
 
@@ -130,11 +130,14 @@ def exec_start(self, exec_id, detach=False, tty=False, stream=False,
             stream (bool): Stream response data. Default: False
             socket (bool): Return the connection socket to allow custom
                 read/write operations.
+            demux (bool): Return stdout and stderr separately
 
         Returns:
-            (generator or str): If ``stream=True``, a generator yielding
-            response chunks. If ``socket=True``, a socket object for the
-            connection. A string containing response data otherwise.
+
+            (generator or str or tuple): If ``stream=True``, a generator
+            yielding response chunks. If ``socket=True``, a socket object for
+            the connection. A string containing response data otherwise. If
+            ``demux=True``, stdout and stderr are separated.
 
         Raises:
             :py:class:`docker.errors.APIError`
@@ -162,4 +165,4 @@ def exec_start(self, exec_id, detach=False, tty=False, stream=False,
             return self._result(res)
         if socket:
             return self._get_raw_response_socket(res)
-        return self._read_from_socket(res, stream, tty)
+        return self._read_from_socket(res, stream, tty=tty, demux=demux)

docker/api/image.py

@@ -353,8 +353,8 @@ def pull(self, repository, tag=None, stream=False, auth_config=None,
 
         Example:
 
-            >>> for line in cli.pull('busybox', stream=True):
-            ...     print(json.dumps(json.loads(line), indent=4))
+            >>> for line in cli.pull('busybox', stream=True, decode=True):
+            ...     print(json.dumps(line, indent=4))
             {
                 "status": "Pulling image (latest) from busybox",
                 "progressDetail": {},
@@ -429,12 +429,12 @@ def push(self, repository, tag=None, stream=False, auth_config=None,
                 If the server returns an error.
 
         Example:
-            >>> for line in cli.push('yourname/app', stream=True):
-            ...   print line
-            {"status":"Pushing repository yourname/app (1 tags)"}
-            {"status":"Pushing","progressDetail":{},"id":"511136ea3c5a"}
-            {"status":"Image already pushed, skipping","progressDetail":{},
-             "id":"511136ea3c5a"}
+            >>> for line in cli.push('yourname/app', stream=True, decode=True):
+            ...   print(line)
+            {'status': 'Pushing repository yourname/app (1 tags)'}
+            {'status': 'Pushing','progressDetail': {}, 'id': '511136ea3c5a'}
+            {'status': 'Image already pushed, skipping', 'progressDetail':{},
+             'id': '511136ea3c5a'}
             ...
 
         """