Make certbot constraint file independent from certbot-auto + update c…

…ryptography (certbot#8649)

* Refactor to not depend on certbot-auto dependencies pinning anymore

* Update constraints

* Replaces references

* Upgrade AWS dependencies pinning

* Fix script

* Fix Windows installer builds

* Fixing sdists letstest script

* Pin cryptography on 3.1.1 specifically for RHEL/CentOS 7 to avoid build failures during test_sdists test.

* Finish fix

* Fix VERSION_ID in RHEL 7

snap/snapcraft.yaml

@@ -73,10 +73,10 @@ parts:
     build-packages: [gcc, libffi-dev, libssl-dev, git, libaugeas-dev, python3-dev]
     build-environment:
       - SNAPCRAFT_PYTHON_VENV_ARGS: --upgrade
-      # Constraints are passed through the environment variable PIP_CONSTRAINTS instead of using the 
+      # Constraints are passed through the environment variable PIP_CONSTRAINTS instead of using the
       # parts.[part_name].constraints option available in snapcraft.yaml when the Python plugin is
       # used. This is done to let these constraints be applied not only on the certbot package
-      # build, but also on any isolated build that pip could trigger when building wheels for 
+      # build, but also on any isolated build that pip could trigger when building wheels for
       # dependencies. See https://github.com/certbot/certbot/pull/8443 for more info.
       - PIP_CONSTRAINT: $SNAPCRAFT_PART_SRC/snap-constraints.txt
     override-build: |
@@ -85,7 +85,7 @@ parts:
       snapcraftctl build
     override-pull: |
       snapcraftctl pull
-      python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/letsencrypt-auto-source/pieces/dependency-requirements.txt" | grep -v python-augeas >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt"
+      python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/tools/certbot_constraints.txt" | grep -v python-augeas >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt"
       python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/tools/pipstrap_constraints.txt" >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt"
       echo "$(python3 "${SNAPCRAFT_PART_SRC}/tools/merge_requirements.py" "${SNAPCRAFT_PART_SRC}/snap-constraints.txt")" > "${SNAPCRAFT_PART_SRC}/snap-constraints.txt"
       snapcraftctl set-version `grep -oP "__version__ = '\K.*(?=')" "${SNAPCRAFT_PART_SRC}/certbot/certbot/__init__.py"`

tests/letstest/scripts/test_sdists.sh

@@ -12,13 +12,21 @@ sudo $BOOTSTRAP_SCRIPT
 # We strip the hashes because the venv creation script includes unhashed
 # constraints in the commands given to pip and the mix of hashed and unhashed
 # packages makes pip error out.
-python3 tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt > requirements.txt
-# We also strip out the requirement for enum34 because it cannot be installed
-# in newer versions of Python 3, tools/strip_hashes.py removes the environment
-# marker that'd normally prevent it from being installed, and this package is
-# not needed for any OS tested here.
-sed -i '/enum34/d' requirements.txt
-CERTBOT_PIP_NO_BINARY=:all: tools/venv.py --requirement requirements.txt
+python3 tools/strip_hashes.py tools/pipstrap_constraints.txt > constraints.txt
+python3 tools/strip_hashes.py tools/certbot_constraints.txt > requirements.txt
+
+# We pin cryptography to 3.1.1 and pyOpenSSL to 19.1.0 specifically for CentOS 7 / RHEL 7
+# because these systems ship only with OpenSSL 1.0.2, and this OpenSSL version support has been
+# dropped on cryptography>=3.2 and pyOpenSSL>=20.0.0.
+# Using this old version of OpenSSL would break the cryptography and pyOpenSSL wheels builds.
+if [ -f /etc/redhat-release ] && [ "$(. /etc/os-release 2> /dev/null && echo "$VERSION_ID" | cut -d '.' -f1)" -eq 7 ]; then
+  sed -i 's|cryptography==.*|cryptography==3.1.1|g' requirements.txt
+  sed -i 's|pyOpenSSL==.*|pyOpenSSL==19.1.0|g' requirements.txt
+fi
+
+python3 -m venv $VENV_PATH
+$VENV_PATH/bin/python3 tools/pipstrap.py
+PIP_CONSTRAINT=constraints.txt PIP_NO_BINARY=:all: $VENV_PATH/bin/python3 -m pip install --requirement requirements.txt
 . "$VENV_PATH/bin/activate"
 # pytest is needed to run tests on some of our packages so we install a pinned version here.
 tools/pip_install.py pytest

tools/dev_constraints.txt

@@ -1,7 +1,7 @@
 # Specifies Python package versions for development and building Docker images.
 # It includes in particular packages not specified in letsencrypt-auto's requirements file.
 # Some dev package versions specified here may be overridden by higher level constraints
-# files during tests (eg. letsencrypt-auto-source/pieces/dependency-requirements.txt).
+# files during tests (eg. tools/certbot_constraints.txt).
 alabaster==0.7.10
 apacheconfig==0.3.2
 apipkg==1.4
@@ -16,8 +16,8 @@ backports.functools-lru-cache==1.5
 backports.shutil-get-terminal-size==1.0.0
 backports.ssl-match-hostname==3.7.0.1
 bcrypt==3.1.6
-boto3==1.11.7
-botocore==1.14.7
+boto3==1.17.4
+botocore==1.20.4
 cached-property==1.5.1
 cloudflare==2.3.1
 configparser==3.7.4

tools/docker/core/Dockerfile

@@ -14,10 +14,6 @@ WORKDIR /opt/certbot
 
 # Copy certbot code
 COPY CHANGELOG.md README.rst src/
-# We keep the relative path to the requirements file the same because, as of
-# writing this, tools/pip_install.py is used in the Dockerfile for Certbot
-# plugins and this script expects to find the requirements file there.
-COPY letsencrypt-auto-source/pieces/dependency-requirements.txt letsencrypt-auto-source/pieces/
 COPY tools tools
 COPY acme src/acme
 COPY certbot src/certbot

tools/pip_install.py

@@ -57,7 +57,7 @@ def certbot_oldest_processing(tools_path, args, test_constraints):
 def certbot_normal_processing(tools_path, test_constraints):
     repo_path = os.path.dirname(tools_path)
     certbot_requirements = os.path.normpath(os.path.join(
-        repo_path, 'letsencrypt-auto-source/pieces/dependency-requirements.txt'))
+        repo_path, 'tools/certbot_constraints.txt'))
     with open(certbot_requirements, 'r') as fd:
         certbot_reqs = fd.readlines()
     with open(os.path.join(tools_path, 'pipstrap_constraints.txt'), 'r') as fd:
@@ -76,8 +76,7 @@ def merge_requirements(tools_path, requirements, test_constraints, all_constrain
     # Here is the order by increasing priority:
     # 1) The general development constraints (tools/dev_constraints.txt)
     # 2) The general tests constraints (oldest_requirements.txt or
-    #    certbot-auto's dependency-requirements.txt + pipstrap's constraints
-    #    for the normal processing)
+    #    certbot_constraints.txt + pipstrap's constraints for the normal processing)
     # 3) The local requirement file, typically local-oldest-requirement in oldest tests
     files = [os.path.join(tools_path, 'dev_constraints.txt'), test_constraints]
     if requirements:
@@ -134,6 +133,7 @@ def main(args):
                 pip_install_with_print('--force-reinstall --no-deps --requirement "{0}"'
                                        .format(requirements))
 
+            print(' '.join(args))
             pip_install_with_print(' '.join(args), env=env)
 
 

letsencrypt-auto-source/rebuild_dependencies.py → tools/rebuild_certbot_constraints.py

@@ -4,12 +4,12 @@
 on various Linux distributions. It generates a requirements file contained the pinned and hashed
 versions, ready to be used by pip to install the certbot dependencies.
 
-This script is typically used to update the certbot-requirements.txt file of certbot-auto.
+This script is typically used to update the certbot_constraints.txt file.
 
 To achieve its purpose, this script will start a certbot installation with unpinned dependencies,
 then gather them, on various distributions started as Docker containers.
 
-Usage: letsencrypt-auto-source/rebuild_dependencies new_requirements.txt
+Usage: tools/rebuild_certbot_constraints.py new_requirements.txt
 
 NB1: Docker must be installed on the machine running this script.
 NB2: Python library 'hashin' must be installed on the machine running this script.
@@ -26,52 +26,41 @@
 
 # The list of docker distributions to test dependencies against with.
 DISTRIBUTION_LIST = [
-    'ubuntu:18.04', 'ubuntu:16.04',
-    'debian:stretch',
-    'centos:7', 'centos:6',
-    'opensuse/leap:15',
-    'fedora:29',
+    'ubuntu:20.04', 'ubuntu:18.04', 'debian:buster',
+    'centos:8', 'centos:7', 'fedora:29',
 ]
 
 # These constraints will be added while gathering dependencies on each distribution.
 # It can be used because a particular version for a package is required for any reason,
 # or to solve a version conflict between two distributions requirements.
 AUTHORITATIVE_CONSTRAINTS = {
-    # Using an older version of mock here prevents regressions of #5276.
-    'mock': '1.3.0',
     # Too touchy to move to a new version. And will be removed soon
     # in favor of pure python parser for Apache.
     'python-augeas': '0.5.0',
-    # Package enum34 needs to be explicitly limited to Python2.x, in order to avoid
-    # certbot-auto failures on Python 3.6+ which enum34 doesn't support. See #5456.
-    'enum34': '1.1.10; python_version < \'3.4\'',
-    # Cryptography 2.9+ drops support for OpenSSL 1.0.1, but we still want to support it
-    # for officially supported non-x86_64 ancient distributions like RHEL 6.
-    'cryptography': '2.8',
-    # Parsedatetime 2.6 is broken on Python 2.7, see https://github.com/bear/parsedatetime/issues/246
-    'parsedatetime': '2.5',
+    # We avoid cryptography 3.4+ since it requires Rust to compile the wheels, and
+    # this needs some work on the snap builds.
+    'cryptography': '3.3.2',
 }
 
-# ./certbot/letsencrypt-auto-source/rebuild_dependencies.py (2 levels from certbot root path)
+# ./certbot/tools/rebuild_certbot_constraints.py (2 levels from certbot root path)
 CERTBOT_REPO_PATH = dirname(dirname(abspath(__file__)))
 
 # The script will be used to gather dependencies for a given distribution.
-#   - certbot-auto is used to install relevant OS packages, and set up an initial venv
+#   - bootstrap_os_packages.sh is used to install relevant OS packages, and set up an initial venv
 #   - then this venv is used to consistently construct an empty new venv
-#   - once pipstraped, this new venv pip-installs certbot runtime (including apache/nginx),
+#   - once pipstrap.py, this new venv pip-installs certbot runtime (including apache/nginx),
 #     without pinned dependencies, and respecting input authoritative requirements
 #   - `certbot plugins` is called to check we have a healthy environment
 #   - finally current set of dependencies is extracted out of the docker using pip freeze
 SCRIPT = r"""#!/bin/sh
-set -e
+set -ex
 
 cd /tmp/certbot
-letsencrypt-auto-source/letsencrypt-auto --install-only -n
-PYVER=`/opt/eff.org/certbot/venv/bin/python --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'`
+tests/letstest/scripts/bootstrap_os_packages.sh
 
-/opt/eff.org/certbot/venv/bin/python letsencrypt-auto-source/pieces/create_venv.py /tmp/venv "$PYVER" 1
+python3 -m venv /tmp/venv
 
-/tmp/venv/bin/python letsencrypt-auto-source/pieces/pipstrap.py
+/tmp/venv/bin/python tools/pipstrap.py
 /tmp/venv/bin/pip install -e acme -e certbot -e certbot-apache -e certbot-nginx -c /tmp/constraints.txt
 /tmp/venv/bin/certbot plugins
 /tmp/venv/bin/pip freeze >> /tmp/workspace/requirements.txt
@@ -109,6 +98,7 @@ def _requirements_from_one_distribution(distribution, verbose):
             '{0}=={1}'.format(package, version) for package, version in AUTHORITATIVE_CONSTRAINTS.items()))
 
         command = ['docker', 'run', '--rm', '--cidfile', cid_file,
+                   '--network=host',
                    '-v', '{0}:/tmp/certbot'.format(CERTBOT_REPO_PATH),
                    '-v', '{0}:/tmp/workspace'.format(workspace),
                    '-v', '{0}:/tmp/constraints.txt'.format(authoritative_constraints),
@@ -158,7 +148,7 @@ def _parse_and_merge_requirements(dependencies_map, requirements_file_lines, dis
     """
     for line in requirements_file_lines:
         match = re.match(r'([^=]+)==([^=]+)', line.strip())
-        if not line.startswith('-e') and match:
+        if not line.startswith('-e') and not line.startswith('#') and match:
             package, version = match.groups()
             if package not in ['acme', 'certbot', 'certbot-apache', 'certbot-nginx', 'pkg-resources']:
                 dependencies_map.setdefault(package, []).append((version, distribution))
@@ -215,11 +205,11 @@ def _write_requirements(dest_file, requirements, conflicts):
     print('===> Calculating hashes for the requirement file.')
 
     _write_to(dest_file, '''\
-# This is the flattened list of packages certbot-auto installs.
+# This is the flattened list of pinned packages to build certbot deployable artifacts.
 # To generate this, do (with docker and package hashin installed):
 # ```
-# letsencrypt-auto-source/rebuild_dependencies.py \\
-#   letsencrypt-auto-source/pieces/dependency-requirements.txt
+# tools/rebuild_certbot_contraints.py \\
+#   tools/certbot_constraints.txt
 # ```
 # If you want to update a single dependency, run commands similar to these:
 # ```
@@ -264,8 +254,8 @@ def _gather_dependencies(dest_file, verbose):
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(
-        description=('Build a sanitized, pinned and hashed requirements file for certbot-auto, '
-                     'validated against several OS distributions using Docker.'))
+        description=('Build a sanitized, pinned and hashed requirements file for certbot deployable'
+                     ' artifacts, validated against several OS distributions using Docker.'))
     parser.add_argument('requirements_path',
                         help='path for the generated requirements file')
     parser.add_argument('--verbose', '-v', action='store_true',

tools/snap/generate_dnsplugins_all.sh

@@ -10,7 +10,7 @@ for PLUGIN_PATH in "${CERTBOT_DIR}"/certbot-dns-*; do
   bash "${CERTBOT_DIR}"/tools/snap/generate_dnsplugins_postrefreshhook.sh $PLUGIN_PATH
   # Create constraints file
   "${CERTBOT_DIR}"/tools/merge_requirements.py tools/dev_constraints.txt \
-    <("${CERTBOT_DIR}"/tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt) \
+    <("${CERTBOT_DIR}"/tools/strip_hashes.py tools/certbot_constraints.txt) \
     <("${CERTBOT_DIR}"/tools/strip_hashes.py tools/pipstrap_constraints.txt) \
     > "${PLUGIN_PATH}"/snap-constraints.txt
 done

windows-installer/construct.py

@@ -2,6 +2,7 @@
 import contextlib
 import ctypes
 import os
+import re
 import shutil
 import struct
 import subprocess
@@ -52,6 +53,21 @@ def _compile_wheels(repo_path, build_path, venv_python):
         command.extend(wheels_project)
         subprocess.check_call(command, env=env)
 
+    # Cryptography uses now a unique wheel name "cryptography-VERSION-cpXX-abi3-win32.whl where
+    # cpXX is the lowest supported version of Python (eg. cp36 says that the wheel is compatible
+    # with Python 3.6+). While technically valid to describe a wheel compliant with the Stable
+    # Application Binary Interface, this naming convention makes pynsist falsely think that the
+    # wheel is compatible with Python 3.6 only.
+    # Let's trick pynsist by renaming the wheel until this is fixed upstream.
+    for file in os.listdir(wheels_path):
+        # Given that our Python version is 3.8, this rename files like
+        # cryptography-VERSION-cpXX-abi3-win32.whl into cryptography-VERSION-cp38-abi3-win32.whl
+        renamed = re.sub(r'^(.*)-cp\d+-abi3-(\w+)\.whl$', r'\1-cp{0}{1}-abi3-\2.whl'
+                         .format(PYTHON_VERSION[0], PYTHON_VERSION[1]), file)
+        print(renamed)
+        if renamed != file:
+            os.replace(os.path.join(wheels_path, file), os.path.join(wheels_path, renamed))
+
 
 def _prepare_build_tools(venv_path, venv_python, repo_path):
     print('Prepare build tools')
@@ -63,7 +79,7 @@ def _prepare_build_tools(venv_path, venv_python, repo_path):
 
 @contextlib.contextmanager
 def _prepare_constraints(repo_path):
-    reqs_certbot = os.path.join(repo_path, 'letsencrypt-auto-source', 'pieces', 'dependency-requirements.txt')
+    reqs_certbot = os.path.join(repo_path, 'tools', 'certbot_constraints.txt')
     reqs_pipstrap = os.path.join(repo_path, 'tools', 'pipstrap_constraints.txt')
     constraints_certbot = subprocess.check_output(
         [sys.executable, os.path.join(repo_path, 'tools', 'strip_hashes.py'), reqs_certbot],