Add tests for S3 channels using minio locally (conda#11644)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Jannis Leidel <[email protected]>

.github/workflows/ci.yml

@@ -164,17 +164,11 @@ jobs:
       - name: Setup environment
         shell: bash -l {0}
         run: |
-          # restoring the default for changeps1 to have parity with dev
-          conda config --set changeps1 true
-          # make sure the caching works correctly
-          conda config --set use_only_tar_bz2 true
-          # install all test requirements
-          conda install --name conda-test-env --yes --file tests/requirements.txt
-          conda update openssl ca-certificates certifi
+          ./dev/macos/setup.sh
 
       - name: Python ${{ matrix.python-version }} ${{ matrix.test-type }} tests, group ${{ matrix.test-group }}
         shell: bash -l {0}
-        run : |
+        run: |
           ./dev/macos/${{ matrix.test-type }}.sh
 
       - uses: codecov/codecov-action@v2

conda/testing/gateways/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2012 Anaconda, Inc
+# SPDX-License-Identifier: BSD-3-Clause

conda/testing/gateways/fixtures.py

@@ -0,0 +1,126 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2012 Anaconda, Inc
+# SPDX-License-Identifier: BSD-3-Clause
+
+import json
+import os
+import socket
+from pathlib import Path
+
+import pytest
+import boto3
+from botocore.client import Config
+from xprocess import ProcessStarter
+
+from ...cli.find_commands import find_executable
+
+MINIO_EXE = find_executable("minio")
+
+
+def minio_s3_server(xprocess, tmp_path):
+    """
+    Mock a local S3 server using `minio`
+
+    This requires:
+    - pytest-xprocess: runs the background process
+    - minio: the executable must be in PATH
+
+    Note, the given S3 server will be EMPTY! The test function needs
+    to populate it. You can use
+    `conda.testing.helpers.populate_s3_server` for that.
+    """
+
+    class Minio:
+        # The 'name' below will be the name of the S3 bucket containing
+        # keys like `noarch/repodata.json`
+        name = "minio_s3_server"
+        port = 9000
+
+        def __init__(self):
+            (Path(tmp_path) / self.name).mkdir()
+
+        @property
+        def server_url(self):
+            return f"http://localhost:{self.port}/{self.name}"
+
+        def populate_bucket(self, endpoint, bucket_name, channel_dir):
+            "prepare the s3 connection for our minio instance"
+
+            # Make the minio bucket public first
+            # https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-example-bucket-policies.html#set-a-bucket-policy
+            session = boto3.session.Session()
+            client = session.client(
+                "s3",
+                endpoint_url=endpoint,
+                aws_access_key_id="minioadmin",
+                aws_secret_access_key="minioadmin",
+                config=Config(signature_version="s3v4"),
+                region_name="us-east-1",
+            )
+            bucket_policy = json.dumps(
+                {
+                    "Version": "2012-10-17",
+                    "Statement": [
+                        {
+                            "Sid": "AddPerm",
+                            "Effect": "Allow",
+                            "Principal": "*",
+                            "Action": ["s3:GetObject"],
+                            "Resource": f"arn:aws:s3:::{bucket_name}/*",
+                        }
+                    ],
+                }
+            )
+            client.put_bucket_policy(Bucket=bucket_name, Policy=bucket_policy)
+
+            # Minio has to start with an empty directory; once available,
+            # we can import all channel files by "uploading" them
+            for current, _, files in os.walk(channel_dir):
+                for f in files:
+                    path = Path(current, f)
+                    key = path.relative_to(channel_dir)
+                    client.upload_file(
+                        str(path),
+                        bucket_name,
+                        str(key),
+                        ExtraArgs={"ACL": "public-read"},
+                    )
+
+    print("Starting mock_s3_server")
+    minio = Minio()
+
+    class Starter(ProcessStarter):
+
+        pattern = "https://docs.min.io"
+        terminate_on_interrupt = True
+        timeout = 10
+        args = [
+            MINIO_EXE,
+            "server",
+            f"--address=:{minio.port}",
+            tmp_path,
+        ]
+
+        def startup_check(self, port=minio.port):
+            s = socket.socket()
+            address = "localhost"
+            error = False
+            try:
+                s.connect((address, port))
+            except Exception as e:
+                print("something's wrong with %s:%d. Exception is %s" % (address, port, e))
+                error = True
+            finally:
+                s.close()
+
+            return not error
+
+    # ensure process is running and return its logfile
+    pid, logfile = xprocess.ensure(minio.name, Starter)
+    print(f"Server (PID: {pid}) log file can be found here: {logfile}")
+    yield minio
+    xprocess.getinfo(minio.name).terminate()
+
+
+if MINIO_EXE is not None:
+    minio_s3_server = pytest.fixture()(minio_s3_server)

dev/linux/setup.sh

@@ -10,6 +10,11 @@ apt-get install -y --no-install-recommends \
 apt-get clean
 rm -rf /var/lib/apt/lists/*
 
+# Download the Minio server, needed for S3 tests
+wget https://dl.minio.io/server/minio/release/linux-amd64/minio
+chmod +x minio
+sudo mv minio /usr/local/bin/minio
+
 useradd -m -s /bin/bash test_user
 usermod -u 1001 test_user
 groupmod -g 1001 test_user

dev/macos/setup.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -ex
+
+# Download the Minio server, needed for S3 tests
+curl -LO https://dl.minio.io/server/minio/release/darwin-amd64/minio
+chmod +x minio
+sudo mv minio /usr/local/bin/minio
+
+# restoring the default for changeps1 to have parity with dev
+conda config --set changeps1 true
+# make sure the caching works correctly
+conda config --set use_only_tar_bz2 true
+# install all test requirements
+conda install --name conda-test-env --yes --file tests/requirements.txt
+conda update openssl ca-certificates certifi

dev/windows/setup.bat

@@ -19,6 +19,11 @@ CALL conda create -n conda-test-env -y python=%PYTHON% pywin32 --file=tests\requ
 CALL conda activate conda-test-env || goto :error
 CALL conda update openssl ca-certificates certifi || goto :error
 python -m conda init cmd.exe --dev || goto :error
+
+:: Download minio server needed for S3 tests and place it in our conda environment so is in PATH
+:: certutil somehow is able to download arbitrary files; don't aske me why: https://superuser.com/a/1545689
+certutil -urlcache -split -f "https://dl.minio.io/server/minio/release/windows-amd64/minio.exe" "%CONDA_PREFIX%\minio.exe" || goto :error
+
 goto :EOF
 
 :error

tests/conftest.py

@@ -8,6 +8,7 @@
 
 pytest_plugins = (
     # Add testing fixtures and internal pytest plugins here
+    "conda.testing.gateways.fixtures",
     "conda.testing.notices.fixtures",
     "conda.testing.fixtures",
 )

tests/gateways/test_connection.py

@@ -5,24 +5,27 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 from logging import getLogger
-from conda.auxlib.compat import Utf8NamedTemporaryFile
+from pathlib import Path
 from unittest import TestCase
-import warnings
+from unittest.mock import patch
 
 import pytest
 from requests import HTTPError
 
+from conda.auxlib.compat import Utf8NamedTemporaryFile
 from conda.common.compat import ensure_binary
 from conda.common.url import path_to_url
+from conda.exceptions import CondaExitZero
 from conda.gateways.anaconda_client import remove_binstar_token, set_binstar_token
 from conda.gateways.connection.session import CondaHttpAuth, CondaSession
 from conda.gateways.disk.delete import rm_rf
+from conda.testing.gateways.fixtures import MINIO_EXE
+from conda.testing.integration import make_temp_env, env_var
 
 log = getLogger(__name__)
 
 
 class CondaHttpAuthTests(TestCase):
-
     def test_add_binstar_token(self):
         try:
             # # token already exists in url, don't add anything
@@ -43,15 +46,14 @@ def test_add_binstar_token(self):
 
 
 class CondaSessionTests(TestCase):
-
     def test_local_file_adapter_404(self):
         session = CondaSession()
-        test_path = 'file:///some/location/doesnt/exist'
+        test_path = "file:///some/location/doesnt/exist"
         r = session.get(test_path)
         with pytest.raises(HTTPError) as exc:
             r.raise_for_status()
         assert r.status_code == 404
-        assert r.json()['path'] == test_path[len('file://'):]
+        assert r.json()["path"] == test_path[len("file://") :]
 
     def test_local_file_adapter_200(self):
         test_path = None
@@ -65,7 +67,48 @@ def test_local_file_adapter_200(self):
             r = session.get(test_url)
             r.raise_for_status()
             assert r.status_code == 200
-            assert r.json()['content'] == "file content"
+            assert r.json()["content"] == "file content"
         finally:
             if test_path is not None:
                 rm_rf(test_path)
+
+
+@pytest.mark.skipif(MINIO_EXE is None, reason=f"Minio server not available")
+@pytest.mark.integration
+def test_s3_server(minio_s3_server):
+    import boto3
+    from botocore.client import Config
+
+    endpoint, bucket_name = minio_s3_server.server_url.rsplit("/", 1)
+    channel_dir = Path(__file__).parent.parent / "data" / "conda_format_repo"
+    minio_s3_server.populate_bucket(endpoint, bucket_name, channel_dir)
+
+    # We patch the default kwargs values in boto3.session.Session.resource(...)
+    # which is used in conda.gateways.connection.s3.S3Adapter to initialize the S3
+    # connection; otherwise it would default to a real AWS instance
+    patched_defaults = (
+        "us-east-1",  # region_name
+        None,  # api_version
+        True,  # use_ssl
+        None,  # verify
+        endpoint,  # endpoint_url
+        "minioadmin",  # aws_access_key_id
+        "minioadmin",  # aws_secret_access_key
+        None,  # aws_session_token
+        Config(signature_version="s3v4"),  # config
+    )
+    with pytest.raises(CondaExitZero):
+        with patch.object(boto3.session.Session.resource, "__defaults__", patched_defaults):
+            # the .conda files in this repo are somehow corrupted
+            with env_var("CONDA_USE_ONLY_TAR_BZ2", "True"):
+                with make_temp_env(
+                    "--override-channels",
+                    f"--channel=s3://{bucket_name}",
+                    "--download-only",
+                    "--no-deps",  # this fake repo only includes the zlib tarball
+                    "zlib",
+                    use_exception_handler=False,
+                    no_capture=True,
+                ):
+                    # we just want to run make_temp_env and cleanup after
+                    pass

tests/requirements.txt

@@ -2,11 +2,13 @@
 # run as 'conda install --file tests/requirements.txt -c defaults'
 anaconda-client
 beautifulsoup4
+boto3 # NOTE: we also need the `minio` server, installed manually in CI
 chardet
 conda
 conda-build
 conda-forge::pre-commit
 conda-forge::pytest-split
+conda-forge::pytest-xprocess
 conda-forge::xdoctest
 conda-package-handling
 conda-verify