[Python] Added build for Arm64 on Linux and Python 3.10 (apache#15004)

* [Python] Added build for Arm64 on Linux and Python 3.10

* Fixed docker-build-python3.9.sh

* Added arch in docker/pulsar/pom.xml

* Fixed spacing in docker/pulsar/pom.xml

* Fixed build-wheels.sh when taking args

* Removed debug statement

* Fixed variable expansion

* Pass the arch to the script inside docker

docker/pulsar/pom.xml

@@ -75,7 +75,7 @@
                   <executable>${project.basedir}/../../pulsar-client-cpp/docker/build-wheels.sh</executable>
                   <arguments>
                     <!-- build python 3.8 -->
-                    <argument>3.8 cp38-cp38</argument>
+                    <argument>3.8 cp38-cp38 x86_64</argument>
                   </arguments>
                 </configuration>
               </execution>

pulsar-client-cpp/docker-build-python3.9.sh

@@ -29,16 +29,18 @@ cd $ROOT_DIR/pulsar-client-cpp
 # Build manylinux2014 build image
 PYTHON_VERSION="3.9"
 PYTHON_SPEC="cp39-cp39"
-IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC
+ARCH="x86_64"
+IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC-$ARCH
 
 docker build -t $IMAGE_NAME ./docker/manylinux2014 \
         --build-arg PYTHON_VERSION=$PYTHON_VERSION \
-        --build-arg PYTHON_SPEC=$PYTHON_SPEC
+        --build-arg PYTHON_SPEC=$PYTHON_SPEC \
+        --build-arg ARCH=$ARCH
 
 
 # Build wheel file
 BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-pulsar-build}"
-IMAGE=$BUILD_IMAGE_NAME:manylinux-$PYTHON_SPEC
+IMAGE=$BUILD_IMAGE_NAME:manylinux-$PYTHON_SPEC-$ARCH
 
 VOLUME_OPTION=${VOLUME_OPTION:-"-v $ROOT_DIR:/pulsar"}
 COMMAND="/pulsar/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh"

pulsar-client-cpp/docker/build-wheel-file-within-docker.sh

@@ -32,7 +32,7 @@ cmake . -DPYTHON_INCLUDE_DIR=/opt/python/$PYTHON_SPEC/include/python$PYTHON_VERS
         -DBUILD_WIRESHARK=OFF
 
 make clean
-make _pulsar -j3 VERBOSE=1
+make _pulsar -j3
 
 cd python
 python setup.py bdist_wheel
@@ -43,4 +43,4 @@ python setup.py bdist_wheel
 # Audit wheel will make sure no external dependencies are needed for
 # the shared library and that only symbols supported by most linux
 # distributions are used.
-auditwheel repair dist/pulsar_client*-$PYTHON_SPEC-linux_x86_64.whl
+auditwheel repair dist/pulsar_client*-$PYTHON_SPEC-linux_${ARCH}.whl

pulsar-client-cpp/docker/build-wheels.sh

@@ -28,32 +28,41 @@ ROOT_DIR=`cd $(dirname $0)/../..; pwd`
 cd $ROOT_DIR
 
 PYTHON_VERSIONS=(
-   '2.7 cp27-cp27mu'
-   '2.7 cp27-cp27m'
-   '3.5 cp35-cp35m'
-   '3.6 cp36-cp36m'
-   '3.7 cp37-cp37m'
-   '3.8 cp38-cp38'
-   '3.9 cp39-cp39'
+   '2.7  cp27-cp27mu  x86_64'
+   '2.7  cp27-cp27m   x86_64'
+   '3.5  cp35-cp35m   x86_64'
+   '3.6  cp36-cp36m   x86_64'
+   '3.7  cp37-cp37m   x86_64'
+   '3.8  cp38-cp38    x86_64'
+   '3.9  cp39-cp39    x86_64'
+   '3.10 cp310-cp310  x86_64'
+   '3.7  cp37-cp37m   aarch64'
+   '3.8  cp38-cp38    aarch64'
+   '3.9  cp39-cp39    aarch64'
+   '3.10 cp310-cp310  aarch64'
 )
 
-function contains() {
-    local n=$#
-    local value=${!n}
-    for ((i=1;i < $#;i++)) {
-        if [ "${!i}" == "${value}" ]; then
-            echo "y"
-            return 0
+function contains_build_version {
+    for line in "${PYTHON_VERSIONS[@]}"; do
+        read -r -a v <<< "$line"
+        value="${v[0]} ${v[1]} ${v[2]}"
+
+        if [ "${build_version}" == "${value}" ]; then
+            # found
+            res=1
+            return
         fi
-    }
-    echo "n"
-    return 1
+    done
+
+    # not found
+    res=0
 }
 
 
 if [ $# -ge 1 ]; then
     build_version=$@
-    if [ $(contains "${PYTHON_VERSIONS[@]}" "${build_version}") == "y" ]; then
+    contains_build_version
+    if [ $res == 1 ]; then
         PYTHON_VERSIONS=(
             "${build_version}"
         )
@@ -69,16 +78,17 @@ for line in "${PYTHON_VERSIONS[@]}"; do
     read -r -a PY <<< "$line"
     PYTHON_VERSION=${PY[0]}
     PYTHON_SPEC=${PY[1]}
-    echo "--------- Build Python wheel for $PYTHON_VERSION -- $PYTHON_SPEC"
+    ARCH=${PY[2]}
+    echo "--------- Build Python wheel for $PYTHON_VERSION -- $PYTHON_SPEC -- $ARCH"
 
-    IMAGE=$BUILD_IMAGE_NAME:manylinux-$PYTHON_SPEC
+    IMAGE=$BUILD_IMAGE_NAME:manylinux-$PYTHON_SPEC-$ARCH
 
     echo "Using image: $IMAGE"
 
     VOLUME_OPTION=${VOLUME_OPTION:-"-v $ROOT_DIR:/pulsar"}
     COMMAND="/pulsar/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh"
     DOCKER_CMD="docker run -i ${VOLUME_OPTION} -e USE_FULL_POM_NAME -e NAME_POSTFIX ${IMAGE}"
 
-    $DOCKER_CMD bash -c "${COMMAND}"
+    $DOCKER_CMD bash -c "ARCH=$ARCH ${COMMAND}"
 
 done

pulsar-client-cpp/docker/create-images.sh

@@ -24,27 +24,34 @@
 set -e
 
 PYTHON_VERSIONS=(
-   '2.7 cp27-cp27mu manylinux1'
-   '2.7 cp27-cp27m manylinux1'
-   '3.5 cp35-cp35m manylinux2014'
-   '3.6 cp36-cp36m manylinux2014'
-   '3.7 cp37-cp37m manylinux2014'
-   '3.8 cp38-cp38 manylinux2014'
-   '3.9 cp39-cp39 manylinux2014'
+   '2.7  cp27-cp27mu manylinux1    x86_64'
+   '2.7  cp27-cp27m  manylinux1    x86_64'
+   '3.5  cp35-cp35m  manylinux2014 x86_64'
+   '3.6  cp36-cp36m  manylinux2014 x86_64'
+   '3.7  cp37-cp37m  manylinux2014 x86_64'
+   '3.8  cp38-cp38   manylinux2014 x86_64'
+   '3.9  cp39-cp39   manylinux2014 x86_64'
+   '3.10 cp310-cp310 manylinux2014 x86_64'
+   '3.7  cp37-cp37m  manylinux2014 aarch64'
+   '3.8  cp38-cp38   manylinux2014 aarch64'
+   '3.9  cp39-cp39   manylinux2014 aarch64'
+   '3.10 cp310-cp310 manylinux2014 aarch64'
 )
 
 for line in "${PYTHON_VERSIONS[@]}"; do
     read -r -a PY <<< "$line"
     PYTHON_VERSION=${PY[0]}
     PYTHON_SPEC=${PY[1]}
     BASE_IMAGE=${PY[2]}
-    echo "--------- Build Docker image for $PYTHON_VERSION -- $PYTHON_SPEC"
+    ARCH=${PY[3]}
+    echo "--------- Build Docker image for $PYTHON_VERSION -- $PYTHON_SPEC -- $ARCH"
 
-    IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC
+    IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC-$ARCH
 
     docker build -t $IMAGE_NAME $BASE_IMAGE \
             --build-arg PYTHON_VERSION=$PYTHON_VERSION \
-            --build-arg PYTHON_SPEC=$PYTHON_SPEC
+            --build-arg PYTHON_SPEC=$PYTHON_SPEC \
+            --build-arg ARCH=$ARCH
 
     echo "==== Successfully built image $IMAGE_NAME"
 done

pulsar-client-cpp/docker/manylinux2014/Dockerfile

@@ -17,17 +17,19 @@
 # under the License.
 #
 
-
-FROM quay.io/pypa/manylinux2014_x86_64
-
-RUN yum install -y gtest-devel
+ARG ARCH
+FROM quay.io/pypa/manylinux2014_${ARCH}
 
 ARG PYTHON_VERSION
 ARG PYTHON_SPEC
 
 ENV PYTHON_VERSION=${PYTHON_VERSION}
 ENV PYTHON_SPEC=${PYTHON_SPEC}
 
+ARG ARCH
+ENV ARCH=${ARCH}
+
+
 ENV PATH="/opt/python/${PYTHON_SPEC}/bin:${PATH}"
 
 RUN ln -s /opt/python/${PYTHON_SPEC}/include/python${PYTHON_VERSION}m /opt/python/${PYTHON_SPEC}/include/python${PYTHON_VERSION}
@@ -50,80 +52,47 @@ RUN curl -O -L https://zlib.net/zlib-1.2.12.tar.gz && \
     tar xvfz zlib-1.2.12.tar.gz && \
     cd zlib-1.2.12 && \
     CFLAGS="-fPIC -O3" ./configure && \
-    make && make install && \
+    make -j8 && make install && \
     rm -rf /zlib-1.2.12.tar.gz /zlib-1.2.12
 
 # Compile OpenSSL
-RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_0j.tar.gz && \
-    tar xvfz OpenSSL_1_1_0j.tar.gz && \
-    cd openssl-OpenSSL_1_1_0j/ && \
-    ./Configure -fPIC --prefix=/usr/local/ssl/ no-shared linux-x86_64 && \
-    make && make install && \
-    rm -rf /OpenSSL_1_1_0j.tar.gz /openssl-OpenSSL_1_1_0j
+RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_1n.tar.gz && \
+    tar xvfz OpenSSL_1_1_1n.tar.gz && \
+    cd openssl-OpenSSL_1_1_1n/ && \
+    ./Configure -fPIC --prefix=/usr/local/ssl/ no-shared linux-${ARCH} && \
+    make -j8 && make install && \
+    rm -rf /OpenSSL_1_1_1n.tar.gz /openssl-OpenSSL_1_1_1n
 
 # Download and compile boost
-RUN curl -O -L https://boostorg.jfrog.io/artifactory/main/release/1.68.0/source/boost_1_68_0.tar.gz && \
-    tar xvfz boost_1_68_0.tar.gz && \
-    cd /boost_1_68_0 && \
+RUN curl -O -L https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz && \
+    tar xvfz boost_1_78_0.tar.gz && \
+    cd /boost_1_78_0 && \
     ./bootstrap.sh --with-libraries=program_options,filesystem,regex,thread,system,python && \
-    ./b2 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install && \
-    rm -rf /boost_1_68_0.tar.gz /boost_1_68_0
+    ./b2 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install -j8 && \
+    rm -rf /boost_1_78_0.tar.gz /boost_1_78_0
 
 # Download and copile protoubf
 RUN curl -O -L  https://github.com/google/protobuf/releases/download/v3.3.0/protobuf-cpp-3.3.0.tar.gz && \
     tar xvfz protobuf-cpp-3.3.0.tar.gz && \
     cd protobuf-3.3.0/ && \
     CXXFLAGS=-fPIC ./configure && \
-    make && make install && ldconfig && \
+    make -j8 && make install && ldconfig && \
     rm -rf /protobuf-cpp-3.3.0.tar.gz /protobuf-3.3.0
 
-# Compile APR
-RUN curl -O -L  http://archive.apache.org/dist/apr/apr-1.5.2.tar.gz && \
-    tar xvfz apr-1.5.2.tar.gz && \
-    cd apr-1.5.2 && \
-    CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure && \
-    make && make install && \
-    rm -rf /apr-1.5.2.tar.gz /apr-1.5.2
-
-# Compile APR-Util
-RUN curl -O -L  http://archive.apache.org/dist/apr/apr-util-1.5.4.tar.gz && \
-    tar xvfz apr-util-1.5.4.tar.gz && \
-    cd apr-util-1.5.4 && \
-    CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure -with-apr=/usr/local/apr && \
-    make && make install && \
-    rm -rf /apr-util-1.5.4.tar.gz /apr-util-1.5.4
-
-# Libtool
-RUN curl -L -O https://ftp.gnu.org/gnu/libtool/libtool-2.4.6.tar.gz && \
-    tar xvfz libtool-2.4.6.tar.gz && \
-    cd libtool-2.4.6 && \
-    ./configure && \
-    make && make install && \
-    rm -rf /libtool-2.4.6.tar.gz /libtool-2.4.6
-
-# Compile log4cxx
-RUN curl -O -L https://github.com/apache/logging-log4cxx/archive/v0.11.0.tar.gz && \
-    tar xvfz v0.11.0.tar.gz && \
-    cd logging-log4cxx-0.11.0 && \
-    ./autogen.sh && \
-    CXXFLAGS=-fPIC ./configure && \
-    make && make install && \
-    rm -rf /v0.11.0.tar.gz /logging-log4cxx-0.11.0
-
 # Compile expat
 RUN curl -O -L  https://github.com/libexpat/libexpat/archive/R_2_2_0.tar.gz && \
     tar xfvz R_2_2_0.tar.gz && \
     cd libexpat-R_2_2_0/expat && \
     ./buildconf.sh && \
     CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure  && \
-    make && make installlib && \
+    make -j8 && make installlib && \
     rm -rf /R_2_2_0.tar.gz /libexpat-R_2_2_0
 
 RUN curl -O -L https://github.com/Kitware/CMake/archive/v3.12.1.tar.gz && \
     tar xvfz v3.12.1.tar.gz && \
     cd CMake-3.12.1 && \
     ./configure && \
-    make && make install && \
+    make -j8 && make install && \
     rm -rf /v3.12.1.tar.gz /CMake-3.12.1
 
 # Zstandard
@@ -139,15 +108,15 @@ RUN curl -O -L https://github.com/google/snappy/releases/download/1.1.3/snappy-1
     tar xvfz snappy-1.1.3.tar.gz && \
     cd snappy-1.1.3 && \
     CXXFLAGS="-fPIC -O3" ./configure && \
-    make && make install && \
+    make -j8 && make install && \
     rm -rf /snappy-1.1.3 /snappy-1.1.3.tar.gz
 
 # LibCurl
 RUN curl -O -L  https://github.com/curl/curl/releases/download/curl-7_61_0/curl-7.61.0.tar.gz && \
     tar xvfz curl-7.61.0.tar.gz && \
     cd curl-7.61.0 && \
     CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ && \
-    make && make install && \
+    make -j8 && make install && \
     rm -rf /curl-7.61.0.tar.gz /curl-7.61.0
 
 RUN pip install twine

pulsar-client-cpp/python/CMakeLists.txt

@@ -60,6 +60,10 @@ if (NOT DEFINED ${Boost_PYTHON39-MT_LIBRARY})
   set(Boost_PYTHON39-MT_LIBRARY ${Boost_PYTHON39_LIBRARY})
 endif()
 
+if (NOT DEFINED ${Boost_PYTHON310-MT_LIBRARY})
+  set(Boost_PYTHON310-MT_LIBRARY ${Boost_PYTHON310_LIBRARY})
+endif()
+
 # Try all possible boost-python variable namings
 set(PYTHON_WRAPPER_LIBS ${Boost_PYTHON_LIBRARY}
                         ${Boost_PYTHON3_LIBRARY}
@@ -69,7 +73,9 @@ set(PYTHON_WRAPPER_LIBS ${Boost_PYTHON_LIBRARY}
                         ${Boost_PYTHON35_LIBRARY}
                         ${Boost_PYTHON36_LIBRARY}
                         ${Boost_PYTHON38_LIBRARY}
-                        ${Boost_PYTHON39_LIBRARY})
+                        ${Boost_PYTHON39_LIBRARY}
+                        ${Boost_PYTHON310_LIBRARY}
+    )
 
 if (APPLE)
     if (Boost_PYTHON27-MT_LIBRARY_RELEASE)
@@ -84,6 +90,9 @@ if (APPLE)
     if (Boost_PYTHON39-MT_LIBRARY_RELEASE)
         set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} ${Boost_PYTHON39-MT_LIBRARY_RELEASE})
     endif ()
+    if (Boost_PYTHON310-MT_LIBRARY_RELEASE)
+        set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} ${Boost_PYTHON310-MT_LIBRARY_RELEASE})
+    endif ()
 endif()
 
 message(STATUS "Using Boost Python libs: ${PYTHON_WRAPPER_LIBS}")