Fixes to submgr download

Download sub-mgr from UBI 8:
 The subscription-manager-rhsm-certificates on CentOS 8 has a bug
(https://bugs.centos.org/view.php?id=17907) so we're downloading
the package from Red Hat UBI 8 instead.

Use CentOS 6 Vault:
 The CentOS 6 is not available through standard ways anymore. It's
necessary to use Vault (archive).

Fix setting the virtual pty window size on CentOS 7:

Create temporary directory for RHSM related files:
 The dir did not exist so creating any file in it was failing.

With new tests, increase the minimum test coverage to 84%.

convert2rhel/main.py

@@ -160,7 +160,7 @@ def pre_ponr_conversion():
 
     if not toolopts.tool_opts.disable_submgr:
         loggerinst.task("Convert: Subscription Manager - Download packages")
-        subscription.download_subscription_manager_packages()
+        subscription.download_rhsm_pkgs()
         loggerinst.task("Convert: Subscription Manager - Replace")
         subscription.replace_subscription_manager()
         loggerinst.task("Convert: Install RHEL certificates for RHSM")

convert2rhel/pkghandler.py

@@ -102,7 +102,7 @@ def call_yum_cmd(command, args="", print_output=True, enable_repos=None, disable
         cmd += " --releasever=%s" % system_info.releasever
 
     # Without the release package installed, dnf can't determine the modularity platform ID.
-    if int(system_info.version) == 8:
+    if system_info.version.major == 8:
         cmd += " --setopt=module_platform_id=platform:el8"
 
     repos_to_enable = []
@@ -111,8 +111,7 @@ def call_yum_cmd(command, args="", print_output=True, enable_repos=None, disable
     else:
         # When using subscription-manager for the conversion, use those repos for the yum call that have been enabled
         # through subscription-manager
-       repos_to_enable = system_info.submgr_enabled_repos if (
-               not tool_opts.disable_submgr and system_info.submgr_enabled_repos) else tool_opts.enablerepo
+        repos_to_enable = system_info.submgr_enabled_repos if not tool_opts.disable_submgr else tool_opts.enablerepo
 
     for repo in repos_to_enable:
         cmd += " --enablerepo=%s" % repo
@@ -599,9 +598,7 @@ def get_kernel(kernels_raw):
 
 
 def replace_non_rhel_installed_kernel(version):
-    """Replace the installed non-RHEL kernel with RHEL kernel with same
-    version.
-    """
+    """Replace the installed non-RHEL kernel with RHEL kernel with same version."""
     loggerinst = logging.getLogger(__name__)
     loggerinst.warning("The convert2rhel is going to force-replace the only"
                        " kernel installed, which has the same NEVRA as the"
@@ -616,8 +613,8 @@ def replace_non_rhel_installed_kernel(version):
     pkg = "kernel-%s" % version
 
     path = utils.download_pkg(
-        pkg=pkg, dest=utils.TMP_DIR, disablerepo=tool_opts.disablerepo,
-        enablerepo=tool_opts.enablerepo)
+        pkg=pkg, dest=utils.TMP_DIR, disable_repos=tool_opts.disablerepo,
+        enable_repos=tool_opts.enablerepo)
     if not path:
         loggerinst.critical("Unable to download the RHEL kernel package.")
 
@@ -626,7 +623,7 @@ def replace_non_rhel_installed_kernel(version):
         'rpm -i --force --replacepkgs %s*' % os.path.join(utils.TMP_DIR, pkg),
         print_output=False)
     if ret_code != 0:
-        loggerinst.critical("Unable to replace kernel package: %s" % output)
+        loggerinst.critical("Unable to replace the kernel package: %s" % output)
 
     loggerinst.info("\nRHEL %s installed.\n" % pkg)
 
@@ -673,7 +670,7 @@ def fix_invalid_grub2_entries():
     The solution handled by this function is to remove the non-functioning boot entries upon the removal of the original
     OS kernels, and setting the RHEL kernel as default.
     """
-    if int(system_info.version) < 8 or system_info.arch == "s390x":
+    if system_info.version.major < 8 or system_info.arch == "s390x":
         # Applicable only on systems derived from RHEL 8 and later, and systems using GRUB2 (s390x uses zipl)
         return
 

convert2rhel/redhatrelease.py

@@ -30,9 +30,9 @@ def get_release_pkg_name():
 
     For RHEL 8, the name is redhat-release.
     """
-    if system_info.version in ["6", "7"]:
+    if system_info.version.major in [6, 7]:
         return "redhat-release-server"
-    elif system_info.version == "8":
+    elif system_info.version.major == 8:
         return "redhat-release"
 
 

convert2rhel/subscription.py

@@ -14,6 +14,7 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
 from collections import namedtuple
 import os
 import re
@@ -29,6 +30,30 @@
 from convert2rhel.systeminfo import system_info
 
 SUBMGR_RPMS_DIR = os.path.join(utils.DATA_DIR, "subscription-manager")
+_RHSM_TMP_DIR = os.path.join(utils.TMP_DIR, "rhsm")
+_CENTOS_6_REPO_CONTENT = \
+        '[centos-6-contrib-convert2rhel]\n' \
+        'name=CentOS 6 - Contrib added by Convert2RHEL\n' \
+        'baseurl=https://vault.centos.org/centos/6/contrib/$basearch/\n' \
+        'gpgcheck=0\n' \
+        'enabled=1\n'
+_CENTOS_6_REPO_PATH = os.path.join(_RHSM_TMP_DIR, "centos_6.repo")
+_CENTOS_7_REPO_CONTENT = \
+        '[centos-7-convert2rhel]\n' \
+        'name=CentOS 7 added by Convert2RHEL\n' \
+        'mirrorlist=http://mirrorlist.centos.org/?release=7&arch=$basearch&repo=os\n' \
+        'gpgcheck=0\n' \
+        'enabled=1\n'
+_CENTOS_7_REPO_PATH = os.path.join(_RHSM_TMP_DIR, "centos_7.repo")
+# We are using UBI 8 instead of CentOS 8 because there's a bug in subscription-manager-rhsm-certificates on CentOS 8
+# https://bugs.centos.org/view.php?id=17907
+_UBI_8_REPO_CONTENT = \
+        '[ubi-8-baseos-convert2rhel]\n' \
+        'name=Red Hat Universal Base Image 8 - BaseOS added by Convert2RHEL\n' \
+        'baseurl=https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi8/8/$basearch/baseos/os/\n' \
+        'gpgcheck=0\n' \
+        'enabled=1\n'
+_UBI_8_REPO_PATH = os.path.join(_RHSM_TMP_DIR, "ubi_8.repo")
 
 
 def subscribe_system():
@@ -412,45 +437,69 @@ def check_needed_repos_availability(repo_ids_needed):
         loggerinst.info("Needed RHEL repos are available.")
 
 
-def download_subscription_manager_packages():
+def download_rhsm_pkgs():
+    """Download all the packages necessary for a successful registration to the Red Hat Subscription Management.
+
+    The packages are available in non-standard repositories, so additional repofiles need to be used. The downloaded
+    RPMs are to be installed in a later stage of the conversion.
+    """
+    utils.mkdir_p(_RHSM_TMP_DIR)
+    pkgs_to_download = ["subscription-manager",
+                        "subscription-manager-rhsm-certificates"]
+
+    if system_info.version.major == 6:
+        pkgs_to_download.append("subscription-manager-rhsm")
+        _download_rhsm_pkgs(pkgs_to_download, _CENTOS_6_REPO_PATH, _CENTOS_6_REPO_CONTENT)
+
+    elif system_info.version.major == 7:
+        pkgs_to_download += ["subscription-manager-rhsm", "python-syspurpose"]
+        _download_rhsm_pkgs(pkgs_to_download, _CENTOS_7_REPO_PATH, _CENTOS_7_REPO_CONTENT)
+        _get_rhsm_cert_on_centos_7()
+
+    elif system_info.version.major == 8:
+        pkgs_to_download += ["python3-subscription-manager-rhsm", "dnf-plugin-subscription-manager",
+                             "python3-syspurpose"]
+        _download_rhsm_pkgs(pkgs_to_download, _UBI_8_REPO_PATH, _UBI_8_REPO_CONTENT)
+
+
+def _download_rhsm_pkgs(pkgs_to_download, repo_path, repo_content):
+    downloaddir = os.path.join(utils.DATA_DIR, "subscription-manager")
+    utils.store_content_to_file(filename=repo_path, content=repo_content)
+    paths = utils.download_pkgs(pkgs_to_download, dest=downloaddir, reposdir=_RHSM_TMP_DIR)
+    exit_on_failed_download(paths)
+
+
+def exit_on_failed_download(paths):
     loggerinst = logging.getLogger(__name__)
+    if None in paths:
+        loggerinst.critical("Unable to download the subscription-manager package or its dependencies. See details of"
+                            " the failed yumdownloader call above. These packages are necessary for the conversion"
+                            " unless you use the --disable-submgr option.")
+
+
+def _get_rhsm_cert_on_centos_7():
+    """There's a RHSM-related bug on CentOS 7: https://bugs.centos.org/view.php?id=14785
+    - The subscription-manager-rhsm-certificates is missing the necessary /etc/rhsm/ca/redhat-uep.pem.
+    - This cert is still available in the python-rhsm-certificates package which is not possible to install
+      (because it is obsoleted by the subscription-manager-rhsm-certificates).
+    The workaround is to download the python-rhsm-certificates and extract the certificate from it.
+    """
+    loggerinst = logging.getLogger(__name__)
+    cert_pkg_path = utils.download_pkg(pkg="python-rhsm-certificates", dest=_RHSM_TMP_DIR, reposdir=_RHSM_TMP_DIR)
+    exit_on_failed_download([cert_pkg_path])
 
-    _CENTOS_CONTRIB_REPO = '[contrib]\n' \
-                           'name=CentOS-$releasever - Contrib\n' \
-                           'mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=contrib\n' \
-                           'gpgcheck=0\n' \
-                           'enabled=1\n' \
-                           'protect=0\n'
-
-    _TEMP_REPOFILE_DIR = '/tmp/convert2rhel/centos_contrib.repo'
-
-    version = utils.parse_version_number(system_info.system_release_file_content)
-
-    if version['major'] <= 6 and (version['minor'] is None or ('minor' in version and version['minor'] <= 6)):
-        loggerinst.error(msg='Unable to download subscription-manager on CentOS/Oracle Linux 6.6 and older. '
-                             'Please use the --disable-submgr option or update to CentOS/Oracle Linux 6.7 or later')
-        sys.exit(1)
-
-    if version['major'] == 6:
-        utils.store_content_to_file(filename=_TEMP_REPOFILE_DIR, content=_CENTOS_CONTRIB_REPO)
-        pkghandler.call_yum_cmd(command="install",
-                                args="subscription-manager "
-                                     "subscription-manager-rhsm "
-                                     "subscription-manager-rhsm-certificates"
-                                     " --setopt=reposdir=/tmp/convert2rhel/ --downloadonly"
-                                     " --downloaddir=/usr/share/convert2rhel/subscription-manager")
-
-    elif version['major'] == 7:
-        pkghandler.call_yum_cmd(command="install", args="subscription-manager subscription-manager-rhsm "
-                                                        "subscription-manager-rhsm-certificates "
-                                                        "--downloadonly "
-                                                        "--downloaddir=/usr/share/convert2rhel/subscription-manager")
-
-        utils.download_pkg(pkg='python-rhsm-certificates', dest='/tmp/')
-
-        utils.mkdir_p("/etc/rhsm/ca/")
-        rpm2cpio = subprocess.Popen("rpm2cpio /tmp/python-rhsm-certificates*.rpm", stdout=subprocess.PIPE, shell=True)
-        cpio = subprocess.Popen("cpio -iv --to-stdout ./etc/rhsm/ca/redhat-uep.pem > /etc/rhsm/ca/redhat-uep.pem",
-                                stdin=rpm2cpio.stdout, stdout=None, shell=True)
-        rpm2cpio.wait()
-        cpio.wait()
+    output, ret_code = utils.run_subprocess("rpm2cpio %s" % cert_pkg_path, print_output=False)
+    if ret_code != 0:
+        loggerinst.critical("Failed to extract cpio archive from the %s package." % cert_pkg_path)
+
+    cpio_filepath = cert_pkg_path + ".cpio"
+    utils.store_content_to_file(filename=cpio_filepath, content=output)
+
+    cert_path = "/etc/rhsm/ca/redhat-uep.pem"
+    utils.mkdir_p("/etc/rhsm/ca/")
+    output, ret_code = utils.run_subprocess("cpio --quiet -F %s -iv --to-stdout .%s" % (cpio_filepath, cert_path),
+                                            print_output=False)
+    # cpio return code 0 even if the requested file is not in the archive - but then the output is 0 chars
+    if ret_code != 0 or not output:
+        loggerinst.critical("Failed to extract the %s certificate from the %s archive." % (cert_path, cpio_filepath))
+    utils.store_content_to_file(cert_path, output)

convert2rhel/systeminfo.py

@@ -15,6 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+from collections import namedtuple
 try:
     import ConfigParser as configparser
 except ImportError:
@@ -41,7 +42,7 @@ def __init__(self):
         self.name = None
         # Single-word lowercase identificator of the system (e.g. oracle)
         self.id = None  # pylint: disable=C0103
-        # Major version of the operating system (e.g. 6)
+        # Major and minor version of the operating system (e.g. version.major == 6, version.minor == 10)
         self.version = None
         # Platform architecture
         self.arch = None
@@ -99,10 +100,22 @@ def _get_system_name(self):
         return name
 
     def _get_system_version(self):
-        version = re.search(r".+?(\d+)\.?",
-                            self.system_release_file_content).group(1)
+        """Return a namedtuple with major and minor elements, both of an int type.
+        
+        Examples:
+        Oracle Linux Server release 6.10
+        Oracle Linux Server release 7.8
+        CentOS release 6.10 (Final)
+        CentOS Linux release 7.6.1810 (Core)
+        CentOS Linux release 8.1.1911 (Core)
+        """
+        match = re.search(r".+?(\d+)\.(\d+)\D?", self.system_release_file_content)
+        if not match:
+            from convert2rhel import redhatrelease
+            self.logger.critical("Couldn't get system version from %s" % redhatrelease.get_system_release_filepath())
+        version = namedtuple("Version", ["major", "minor"])(int(match.group(1)), int(match.group(2)))
 
-        self.logger.info("%-20s %s" % ("OS major version:", version))
+        self.logger.info("%-20s %d.%d" % ("OS version:", version.major, version.minor))
         return version
 
     def _get_architecture(self):
@@ -112,8 +125,8 @@ def _get_architecture(self):
         return arch
 
     def _get_cfg_filename(self):
-        cfg_filename = "%s-%s-%s.cfg" % (self.id,
-                                         self.version,
+        cfg_filename = "%s-%d-%s.cfg" % (self.id,
+                                         self.version.major,
                                          self.arch)
         self.logger.info("%-20s %s" % ("Config filename:", cfg_filename))
         return cfg_filename

convert2rhel/unit_tests/main_test.py

@@ -157,15 +157,15 @@ def test_rollback_changes(self):
     @mock_calls(subscription, "check_needed_repos_availability", CallOrderMocked)
     @mock_calls(subscription, "disable_repos", CallOrderMocked)
     @mock_calls(subscription, "enable_repos", CallOrderMocked)
-    @mock_calls(subscription, "download_subscription_manager_packages", CallOrderMocked)
+    @mock_calls(subscription, "download_rhsm_pkgs", CallOrderMocked)
     def test_pre_ponr_conversion_order_with_rhsm(self):
         self.CallOrderMocked.reset()
         main.pre_ponr_conversion()
 
         intended_call_order = OrderedDict()
         intended_call_order["list_third_party_pkgs"] = 1
         intended_call_order["remove_excluded_pkgs"] = 1
-        intended_call_order["download_subscription_manager_packages"] = 1
+        intended_call_order["download_rhsm_pkgs"] = 1
         intended_call_order["replace_subscription_manager"] = 1
         intended_call_order["install"] = 1
         intended_call_order["subscribe_system"] = 1
@@ -196,7 +196,7 @@ def test_pre_ponr_conversion_order_with_rhsm(self):
     @mock_calls(subscription, "check_needed_repos_availability", CallOrderMocked)
     @mock_calls(subscription, "disable_repos", CallOrderMocked)
     @mock_calls(subscription, "enable_repos", CallOrderMocked)
-    @mock_calls(subscription, "download_subscription_manager_packages", CallOrderMocked)
+    @mock_calls(subscription, "download_rhsm_pkgs", CallOrderMocked)
     def test_pre_ponr_conversion_order_without_rhsm(self):
         self.CallOrderMocked.reset()
         main.pre_ponr_conversion()
@@ -207,7 +207,7 @@ def test_pre_ponr_conversion_order_without_rhsm(self):
         intended_call_order["remove_excluded_pkgs"] = 1
 
         # Do not expect this one to be called - related to RHSM
-        intended_call_order["download_subscription_manager_packages"] = 1
+        intended_call_order["download_rhsm_pkgs"] = 0
         intended_call_order["replace_subscription_manager"] = 0
         intended_call_order["install"] = 0
         intended_call_order["subscribe_system"] = 0