Build zlib from the source on Linux with the latest version (opencv#715)

* Added zlib build from the source on Linux, handling auditwheel issues * Removed unused variable * Removed bzip2 from dependencies for OpenCV Python packages
cudawarped · Aug 26, 2022 · 379d281 · 379d281
1 parent 7bb3e77
commit 379d281
Show file tree

Hide file tree

Showing 7 changed files with 75 additions and 59 deletions.
diff --git a/.github/workflows/build_wheels_linux.yml b/.github/workflows/build_wheels_linux.yml
@@ -35,14 +35,13 @@ jobs:
     env:
       ACTIONS_ALLOW_UNSECURE_COMMANDS: true
       REPO_DIR: .
-      BUILD_COMMIT: ${{ github.ref }}
       PROJECT_SPEC: opencv-python
       MB_PYTHON_VERSION: ${{ matrix.python-version }}
       TRAVIS_PYTHON_VERSION: ${{ matrix.python-version }}
       MB_ML_VER: 2014
       TRAVIS_BUILD_DIR: ${{ github.workspace }}
       CONFIG_PATH: travis_config.sh
-      DOCKER_IMAGE: quay.io/opencv-ci/opencv-python-manylinux2014-x86-64:20220628
+      DOCKER_IMAGE: quay.io/opencv-ci/opencv-python-manylinux2014-x86-64:20220824
       USE_CCACHE: 0
       UNICODE_WIDTH: 32
       PLAT: x86_64
@@ -55,9 +54,6 @@ jobs:
       working-directory: ${{ github.workspace }}
     - name: Setup environment
       run: |
-        if [[ "${{ github.event.pull_request }}" == "true" ]]; then
-          echo "BUILD_COMMIT=${{ github.base_ref }}" >> $GITHUB_ENV
-        fi
         if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then
           echo "ENABLE_ROLLING=1" >> $GITHUB_ENV
         fi
@@ -135,7 +131,6 @@ jobs:
     env:
       ACTIONS_ALLOW_UNSECURE_COMMANDS: true
       REPO_DIR: .
-      BUILD_COMMIT: ${{ github.ref }}
       PROJECT_SPEC: opencv-python
       PLAT: x86_64
       MB_PYTHON_VERSION: ${{ matrix.python-version }}
@@ -156,9 +151,6 @@ jobs:
       working-directory: ${{ github.workspace }}
     - name: Setup environment
       run: |
-        if [[ "${{ github.event.pull_request }}" == "true" ]]; then
-          echo "BUILD_COMMIT=${{ github.base_ref }}" >> $GITHUB_ENV
-        fi
         if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then
           echo "ENABLE_ROLLING=1" >> $GITHUB_ENV
         fi

diff --git a/.github/workflows/build_wheels_linux_arm.yml b/.github/workflows/build_wheels_linux_arm.yml
@@ -35,15 +35,14 @@ jobs:
     env:
       ACTIONS_ALLOW_UNSECURE_COMMANDS: true
       REPO_DIR: .
-      BUILD_COMMIT: ${{ github.ref }}
       PROJECT_SPEC: opencv-python
       MB_PYTHON_VERSION: ${{ matrix.python-version }}
       TRAVIS_PYTHON_VERSION: ${{ matrix.python-version }}
       PLAT: aarch64
       MB_ML_VER: 2014
       TRAVIS_BUILD_DIR: ${{ github.workspace }}
       CONFIG_PATH: travis_config.sh
-      DOCKER_IMAGE: quay.io/opencv-ci/opencv-python-manylinux2014-aarch64:20220628
+      DOCKER_IMAGE: quay.io/opencv-ci/opencv-python-manylinux2014-aarch64:20220824
       USE_CCACHE: 0
       UNICODE_WIDTH: 32
       SDIST: ${{ matrix.build_sdist || 0 }}
@@ -55,9 +54,6 @@ jobs:
       working-directory: ${{ github.workspace }}
     - name: Setup environment
       run: |
-        if [[ "${{ github.event.pull_request }}" == "true" ]]; then
-          echo "BUILD_COMMIT=${{ github.base_ref }}" >> $GITHUB_ENV
-        fi
         if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then
           echo "ENABLE_ROLLING=1" >> $GITHUB_ENV
         fi
@@ -97,7 +93,6 @@ jobs:
       ACTIONS_ALLOW_UNSECURE_COMMANDS: true
       MB_PYTHON_VERSION: ${{ matrix.python-version }}
       PLAT: aarch64
-      BUILD_COMMIT: ${{ github.ref }}
       NP_TEST_DEP: numpy==1.19.4
       NP_TEST_DEP_LATEST: numpy==1.21.4
       CONFIG_PATH: travis_config.sh
@@ -107,11 +102,6 @@ jobs:
     - name: Cleanup
       run: find . -mindepth 1 -delete
       working-directory: ${{ github.workspace }}
-    - name: Setup environment
-      run: |
-        if [[ "${{ github.event.pull_request }}" == "true" ]]; then
-          echo "BUILD_COMMIT=${{ github.base_ref }}" >> $GITHUB_ENV
-        fi
     - name: Checkout
       uses: actions/checkout@v3
       with:

diff --git a/.github/workflows/build_wheels_macos.yml b/.github/workflows/build_wheels_macos.yml
@@ -35,7 +35,6 @@ jobs:
     env:
       ACTIONS_ALLOW_UNSECURE_COMMANDS: true
       REPO_DIR: .
-      BUILD_COMMIT: ${{ github.ref }}
       PROJECT_SPEC: opencv-python
       MB_PYTHON_VERSION: ${{ matrix.python-version }}
       TRAVIS_PYTHON_VERSION: ${{ matrix.python-version }}
@@ -57,9 +56,6 @@ jobs:
       working-directory: ${{ github.workspace }}
     - name: Setup environment
       run: |
-        if [[ "${{ github.event.pull_request }}" == "true" ]]; then
-          echo "BUILD_COMMIT=${{ github.base_ref }}" >> $GITHUB_ENV
-        fi
         if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then
           echo "ENABLE_ROLLING=1" >> $GITHUB_ENV
         fi

diff --git a/.github/workflows/build_wheels_macos_m1.yml b/.github/workflows/build_wheels_macos_m1.yml
@@ -30,7 +30,6 @@ jobs:
         without_gui: [0, 1]
         build_sdist: [0]
     env:
-      BUILD_COMMIT: ${{ github.ref }}
       CI_BUILD: 1
       SDIST: ${{ matrix.build_sdist || 0 }}
       ENABLE_HEADLESS: ${{ matrix.without_gui }}
@@ -41,9 +40,6 @@ jobs:
       working-directory: ${{ github.workspace }}
     - name: Setup environment
       run: |
-        if [[ "${{ github.event.pull_request }}" == "true" ]]; then
-          echo "BUILD_COMMIT=${{ github.base_ref }}" >> $GITHUB_ENV
-        fi
         if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then
           echo "ENABLE_ROLLING=1" >> $GITHUB_ENV
         fi
@@ -84,18 +80,12 @@ jobs:
         without_gui: [0, 1]
         build_sdist: [0]
     env:
-      BUILD_COMMIT: ${{ github.ref }}
       OPENCV_TEST_DATA_PATH: ${{ github.workspace }}/opencv_extra/testdata
       PYLINT_TEST_FILE: ${{ github.workspace }}/opencv/samples/python/squares.py
     steps:
     - name: Cleanup
       run: find . -mindepth 1 -delete
       working-directory: ${{ github.workspace }}
-    - name: Setup environment
-      run: |
-        if [[ "${{ github.event.pull_request }}" == "true" ]]; then
-          echo "BUILD_COMMIT=${{ github.base_ref }}" >> $GITHUB_ENV
-        fi
     - name: Checkout
       uses: actions/checkout@v3
       with:

diff --git a/docker/manylinux2014/Dockerfile_aarch64 b/docker/manylinux2014/Dockerfile_aarch64
@@ -1,4 +1,4 @@
-# Version: 20220628
+# Version: 20220824
 # Image name: quay.io/opencv-ci/opencv-python-manylinux2014-aarch64
 
 FROM quay.io/pypa/manylinux2014_aarch64:latest
@@ -11,16 +11,28 @@ ARG NASM_VERSION=2.15.04
 ARG OPENSSL_VERSION=1_1_1o
 ARG QT_VERSION=5.15.0
 ARG YASM_VERSION=1.3.0
+ARG ZLIB_VERSION=1.2.12
 
 ENV LD_LIBRARY_PATH /usr/local/lib:$LD_LIBRARY_PATH
 
 # epel-release need for aarch64 to get openblas packages
-RUN yum install bzip2-devel curl-devel zlib-devel xcb-util-renderutil-devel xcb-util-devel xcb-util-image-devel xcb-util-keysyms-devel xcb-util-wm-devel mesa-libGL-devel libxkbcommon-devel libxkbcommon-x11-devel libXi-devel lapack-devel epel-release -y && \
+RUN yum install bzip2-devel curl-devel xcb-util-renderutil-devel xcb-util-devel xcb-util-image-devel xcb-util-keysyms-devel xcb-util-wm-devel mesa-libGL-devel libxkbcommon-devel libxkbcommon-x11-devel libXi-devel lapack-devel epel-release -y && \
     yum install openblas-devel -y && \
     cp /usr/include/lapacke/lapacke*.h /usr/include/ && \
     curl https://raw.githubusercontent.com/xianyi/OpenBLAS/v0.3.3/cblas.h -o /usr/include/cblas.h && \
     # libpng will be built from source
-    yum remove libpng -y
+    yum remove libpng zlib-devel bzip2-devel -y
+
+RUN mkdir ~/zlib_sources && \
+    cd ~/zlib_sources && \
+    curl -O -L https://www.zlib.net/zlib-${ZLIB_VERSION}.tar.gz && \
+    tar -xf zlib-${ZLIB_VERSION}.tar.gz && \
+    cd zlib-${ZLIB_VERSION} && \
+    ./configure --prefix=/usr/local && \
+    make && \
+    make install && \
+    cd .. && \
+    rm -rf ~/zlib_sources
 
 RUN mkdir ~/libpng_sources && \
     cd ~/libpng_sources && \
@@ -71,8 +83,8 @@ RUN mkdir ~/openssl_sources && \
 
 RUN mkdir ~/nasm_sources && \
     cd ~/nasm_sources && \
-    curl -O -L http://www.nasm.us/pub/nasm/releasebuilds/${NASM_VERSION}/nasm-${NASM_VERSION}.tar.bz2 && \
-    tar -xf nasm-${NASM_VERSION}.tar.bz2 && cd nasm-${NASM_VERSION} && ./autogen.sh && \
+    curl -O -L http://www.nasm.us/pub/nasm/releasebuilds/${NASM_VERSION}/nasm-${NASM_VERSION}.tar.gz && \
+    tar -xf nasm-${NASM_VERSION}.tar.gz && cd nasm-${NASM_VERSION} && ./autogen.sh && \
     ./configure --prefix="/ffmpeg_build" --bindir="$HOME/bin" && \
     make -j$(getconf _NPROCESSORS_ONLN) && \
     make install && \
@@ -102,17 +114,16 @@ RUN mkdir ~/libvpx_sources && \
 
 RUN mkdir ~/ffmpeg_sources && \
     cd ~/ffmpeg_sources && \
-    curl -O -L https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \
-    tar -xf ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \
+    curl -O -L https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.gz && \
+    tar -xf ffmpeg-${FFMPEG_VERSION}.tar.gz && \
     cd ffmpeg-${FFMPEG_VERSION} && \
     PATH=~/bin:$PATH && \
     PKG_CONFIG_PATH="/ffmpeg_build/lib/pkgconfig" ./configure --prefix="/ffmpeg_build" --extra-cflags="-I/ffmpeg_build/include" --extra-ldflags="-L/ffmpeg_build/lib" --enable-openssl --enable-libvpx --enable-shared --enable-pic --bindir="$HOME/bin" && \
     make -j$(getconf _NPROCESSORS_ONLN) && \
     make install && \
     echo "/ffmpeg_build/lib/" >> /etc/ld.so.conf && \
     ldconfig && \
-    rm -rf ~/ffmpeg_sources && \
-    yum remove bzip2-devel -y
+    rm -rf ~/ffmpeg_sources
 
 RUN curl -O -L https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/ccache-${CCACHE_VERSION}.tar.gz && \
     tar -xf ccache-${CCACHE_VERSION}.tar.gz && \

diff --git a/docker/manylinux2014/Dockerfile_x86_64 b/docker/manylinux2014/Dockerfile_x86_64
@@ -1,4 +1,4 @@
-# Version: 20220628
+# Version: 20220824
 # Image name: quay.io/opencv-ci/opencv-python-manylinux2014-x86-64
 
 FROM quay.io/pypa/manylinux2014_x86_64:latest
@@ -11,16 +11,28 @@ ARG NASM_VERSION=2.15.04
 ARG OPENSSL_VERSION=1_1_1o
 ARG QT_VERSION=5.15.0
 ARG YASM_VERSION=1.3.0
+ARG ZLIB_VERSION=1.2.12
 
 ENV LD_LIBRARY_PATH /usr/local/lib:$LD_LIBRARY_PATH
 
 # epel-release need for aarch64 to get openblas packages
-RUN yum install bzip2-devel curl-devel zlib-devel xcb-util-renderutil-devel xcb-util-devel xcb-util-image-devel xcb-util-keysyms-devel xcb-util-wm-devel mesa-libGL-devel libxkbcommon-devel libxkbcommon-x11-devel libXi-devel lapack-devel epel-release -y && \
+RUN yum install bzip2-devel curl-devel xcb-util-renderutil-devel xcb-util-devel xcb-util-image-devel xcb-util-keysyms-devel xcb-util-wm-devel mesa-libGL-devel libxkbcommon-devel libxkbcommon-x11-devel libXi-devel lapack-devel epel-release -y && \
     yum install openblas-devel -y && \
     cp /usr/include/lapacke/lapacke*.h /usr/include/ && \
     curl https://raw.githubusercontent.com/xianyi/OpenBLAS/v0.3.3/cblas.h -o /usr/include/cblas.h && \
     # libpng will be built from source
-    yum remove libpng -y
+    yum remove libpng zlib-devel bzip2-devel -y
+
+RUN mkdir ~/zlib_sources && \
+    cd ~/zlib_sources && \
+    curl -O -L https://www.zlib.net/zlib-${ZLIB_VERSION}.tar.gz && \
+    tar -xf zlib-${ZLIB_VERSION}.tar.gz && \
+    cd zlib-${ZLIB_VERSION} && \
+    ./configure --prefix=/usr/local && \
+    make && \
+    make install && \
+    cd .. && \
+    rm -rf ~/zlib_sources
 
 RUN mkdir ~/libpng_sources && \
     cd ~/libpng_sources && \
@@ -71,8 +83,8 @@ RUN mkdir ~/openssl_sources && \
 
 RUN mkdir ~/nasm_sources && \
     cd ~/nasm_sources && \
-    curl -O -L http://www.nasm.us/pub/nasm/releasebuilds/${NASM_VERSION}/nasm-${NASM_VERSION}.tar.bz2 && \
-    tar -xf nasm-${NASM_VERSION}.tar.bz2 && cd nasm-${NASM_VERSION} && ./autogen.sh && \
+    curl -O -L http://www.nasm.us/pub/nasm/releasebuilds/${NASM_VERSION}/nasm-${NASM_VERSION}.tar.gz && \
+    tar -xf nasm-${NASM_VERSION}.tar.gz && cd nasm-${NASM_VERSION} && ./autogen.sh && \
     ./configure --prefix="/ffmpeg_build" --bindir="$HOME/bin" && \
     make -j$(getconf _NPROCESSORS_ONLN) && \
     make install && \
@@ -102,17 +114,16 @@ RUN mkdir ~/libvpx_sources && \
 
 RUN mkdir ~/ffmpeg_sources && \
     cd ~/ffmpeg_sources && \
-    curl -O -L https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \
-    tar -xf ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \
+    curl -O -L https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.gz && \
+    tar -xf ffmpeg-${FFMPEG_VERSION}.tar.gz && \
     cd ffmpeg-${FFMPEG_VERSION} && \
     PATH=~/bin:$PATH && \
     PKG_CONFIG_PATH="/ffmpeg_build/lib/pkgconfig" ./configure --prefix="/ffmpeg_build" --extra-cflags="-I/ffmpeg_build/include" --extra-ldflags="-L/ffmpeg_build/lib" --enable-openssl --enable-libvpx --enable-shared --enable-pic --bindir="$HOME/bin" && \
     make -j$(getconf _NPROCESSORS_ONLN) && \
     make install && \
     echo "/ffmpeg_build/lib/" >> /etc/ld.so.conf && \
     ldconfig && \
-    rm -rf ~/ffmpeg_sources && \
-    yum remove bzip2-devel -y
+    rm -rf ~/ffmpeg_sources
 
 RUN curl -O -L https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/ccache-${CCACHE_VERSION}.tar.gz && \
     tar -xf ccache-${CCACHE_VERSION}.tar.gz && \

diff --git a/patch_auditwheel_whitelist.py b/patch_auditwheel_whitelist.py
@@ -3,13 +3,39 @@
 
 from auditwheel import policy
 
-policies = None
 
-with open(join(dirname(abspath(policy.__file__)), "manylinux-policy.json")) as f:
-    policies = json.load(f)
+def add_zlib_versions():
+    with open(join(dirname(abspath(policy.__file__)), "manylinux-policy.json"), 'r') as manylinux_policy:
+        manylinux_policy_lines = manylinux_policy.readlines()
 
-for p in policies:
-    p["lib_whitelist"].append("libxcb.so.1")
+    manylinux_policy_data = ""
+    for line in manylinux_policy_lines:
+        if "ZLIB" in line:
+            if len(line) > 22:
+                updated_line = line[:-2] + ', "1.2.9", "1.2.12"]'
+            else:
+                updated_line = line[:-2] + '"1.2.9", "1.2.12"]'
+            manylinux_policy_replacement = line.replace(line, updated_line)
+        else:
+            manylinux_policy_replacement = line
+        manylinux_policy_data = manylinux_policy_data + manylinux_policy_replacement
 
-with open(join(dirname(abspath(policy.__file__)), "manylinux-policy.json"), "w") as f:
-    f.write(json.dumps(policies))
+    with open(join(dirname(abspath(policy.__file__)), "manylinux-policy.json"), 'w') as manylinux_final_policy:
+        manylinux_final_policy.write(manylinux_policy_data)
+
+def add_whitelisted_libs():
+    policies = None
+
+    with open(join(dirname(abspath(policy.__file__)), "manylinux-policy.json")) as f:
+        policies = json.load(f)
+
+    for p in policies:
+        p["lib_whitelist"].append("libxcb.so.1")
+
+    with open(join(dirname(abspath(policy.__file__)), "manylinux-policy.json"), "w") as f:
+        f.write(json.dumps(policies))
+
+
+if __name__ == '__main__':
+    add_zlib_versions()
+    add_whitelisted_libs()