Bump protobuf from 3.16.0 to 3.18.3 (onnx#4544)

* Bump protobuf from 3.16.0 to 3.18.3 Signed-off-by: Chun-Wei Chen <[email protected]> * nit Signed-off-by: Chun-Wei Chen <[email protected]> * update 3.18.3 in workflow scripts as well Signed-off-by: Chun-Wei Chen <[email protected]> * use 3.19.5 instead Signed-off-by: Chun-Wei Chen <[email protected]> * 3.13.0 Signed-off-by: Chun-Wei Chen <[email protected]> * 3.20.2 Signed-off-by: Chun-Wei Chen <[email protected]> * mini 3.16.0 Signed-off-by: Chun-Wei Chen <[email protected]> * mini 3.19.5 Signed-off-by: Chun-Wei Chen <[email protected]> * debug and 3.20.0 Signed-off-by: Chun-Wei Chen <[email protected]> * try 3.19.5 Signed-off-by: Chun-Wei Chen <[email protected]> * use 3.18.3 Signed-off-by: Chun-Wei Chen <[email protected]> * sync version in used conda in Windows CI Signed-off-by: Chun-Wei Chen <[email protected]> * mini 3.12.2; 3.19.5 for mac Signed-off-by: Chun-Wei Chen <[email protected]> * -m pip install -q -r requirements-release.txt Signed-off-by: Chun-Wei Chen <[email protected]> Signed-off-by: Chun-Wei Chen <[email protected]>
aipul · Oct 20, 2022 · 604af9c · 604af9c
1 parent 1010de3
commit 604af9c
Show file tree

Hide file tree

Showing 9 changed files with 39 additions and 25 deletions.
diff --git a/.azure-pipelines/MacOS-CI.yml b/.azure-pipelines/MacOS-CI.yml
@@ -40,20 +40,25 @@ jobs:
       versionSpec: '$(python.version)'
 
   - script: |
-      # Install protobuf 3.16.0 from source
+      # Install Protobuf from source
       export NUM_CORES=`sysctl -n hw.logicalcpu`
-      source workflow_scripts/protobuf/build_protobuf_unix.sh $NUM_CORES $(pwd)/protobuf/protobuf_install
+      if [ '$(onnx_debug)' == '1' ]; then
+        export DEBUG=1
+        source workflow_scripts/protobuf/build_protobuf_unix.sh $NUM_CORES $(pwd)/protobuf/protobuf_install Debug
+      else
+        source workflow_scripts/protobuf/build_protobuf_unix.sh $NUM_CORES $(pwd)/protobuf/protobuf_install
+      fi
 
       git submodule update --init --recursive
       python -m pip install -q --upgrade pip
       python -m pip install -q -r requirements-release.txt
-      export ONNX_BUILD_TESTS=1
-      if [ '$(onnx_debug)' == '1' ]; then
-        export DEBUG=1
-      fi
+      # Mac cannot work with Python Protobuf 3.18.3 https://github.com/protocolbuffers/protobuf/issues/10691
+      python -m pip install protobuf==3.19.5
+
       if [ '$(onnx_ml)' == '1' ]; then
         export ONNX_ML=1
       fi
+      export ONNX_BUILD_TESTS=1
       export CMAKE_ARGS="-DONNX_WERROR=ON"
       if [ '$(onnx_lite)' == '1' ]; then
         export CMAKE_ARGS="${CMAKE_ARGS} -DONNX_USE_LITE_PROTO=ON"

diff --git a/.azure-pipelines/Windows-CI.yml b/.azure-pipelines/Windows-CI.yml
@@ -41,7 +41,7 @@ jobs:
 
   - script: |
       conda create --yes --quiet --name py$(python.version) python=$(python.version)
-      conda install -n py$(python.version) -y -c conda-forge numpy libprotobuf=3.16.0
+      conda install -n py$(python.version) -y -c conda-forge numpy libprotobuf=3.18
     displayName: Create Anaconda environment
 
   - script: |

diff --git a/.github/workflows/release_linux_aarch64.yml b/.github/workflows/release_linux_aarch64.yml
@@ -47,7 +47,7 @@ jobs:
           ${{ env.img }} \
           bash -exc '${{ env.py }} -m pip install -q virtualenv && ${{ env.py }} -m venv .env && \
           source .env/bin/activate && \
-          ${{ env.py }} -m pip install -U numpy protobuf==3.16.0 && \
+          ${{ env.py }} -m pip install -q -r requirements-release.txt && \
           yum install -y protobuf-compiler protobuf-devel
           deactivate'
 

diff --git a/.github/workflows/release_mac.yml b/.github/workflows/release_mac.yml
@@ -41,14 +41,16 @@ jobs:
       run: |
         python -m pip install -q --upgrade pip
         python -m pip install -q -r requirements-release.txt
+        # Mac cannot work with Python Protobuf 3.18.3 https://github.com/protocolbuffers/protobuf/issues/10691
+        python -m pip install protobuf==3.19.5
 
     - name: Build wheel and install
       env:
         CC: "clang"
         CXX: "clang++"
         ONNX_ML: 1
       run: |
-        # Install protobuf from source
+        # Install Protobuf from source
         export NUM_CORES=`sysctl -n hw.logicalcpu`
         source workflow_scripts/protobuf/build_protobuf_unix.sh $NUM_CORES $(pwd)/protobuf/protobuf_install
         export CMAKE_ARGS="-DONNX_USE_LITE_PROTO=ON"
@@ -102,15 +104,15 @@ jobs:
         python onnx/test/test_with_ort.py
 
     - name: Build and upload source distribution to TestPyPI weekly
-      if: github.event_name == 'schedule' && matrix.python-version == '3.9' # Only triggered by weekly event on Python 3.9
+      if: github.event_name == 'schedule' && matrix.python-version == '3.10' # Only triggered by weekly event on Python 3.10
       run: |
         # Build and upload source distribution to TestPyPI
         git clean -xdf
         python setup.py sdist --weekly_build
         twine upload dist/* --repository-url https://test.pypi.org/legacy/ -u ${{ secrets.TESTPYPI_USERNAME }} -p ${{ secrets.TESTPYPI_PASSWORD }}
 
     - name: Test source distribution from TestPyPI
-      if: github.event_name == 'schedule' && matrix.python-version == '3.9'
+      if: github.event_name == 'schedule' && matrix.python-version == '3.10'
       run: |
         python -m pip uninstall -y onnx
         python -m pip install setuptools

diff --git a/.github/workflows/weekly_mac_ci.yml b/.github/workflows/weekly_mac_ci.yml
@@ -34,6 +34,8 @@ jobs:
         set -e
         python -m pip install -q --upgrade pip
         python -m pip install -q -r requirements-release.txt
+        # Mac cannot work with Python Protobuf 3.18.3 https://github.com/protocolbuffers/protobuf/issues/10691
+        python -m pip install protobuf==3.19.5
 
     - name: Build, install and test ONNX
       shell: bash

diff --git a/README.md b/README.md
@@ -94,15 +94,15 @@ The ON/OFF depends on what kind of protobuf library you have. Shared libraries a
 
 
 ### Windows
-If you are building ONNX from source, it is recommended that you also build Protobuf locally as a static library. The version distributed with conda-forge is a DLL, but ONNX expects it to be a static library. Building protobuf locally also lets you control the version of protobuf. The tested and recommended version is 3.16.0.
+If you are building ONNX from source, it is recommended that you also build Protobuf locally as a static library. The version distributed with conda-forge is a DLL, but ONNX expects it to be a static library. Building protobuf locally also lets you control the version of protobuf. The tested and recommended version is 3.18.3.
 
 The instructions in this README assume you are using Visual Studio.  It is recommended that you run all the commands from a shell started from "x64 Native Tools Command Prompt for VS 2019" and keep the build system generator for cmake (e.g., cmake -G "Visual Studio 16 2019") consistent while building protobuf as well as ONNX.
 
 You can get protobuf by running the following commands:
 ```bat
 git clone https://github.com/protocolbuffers/protobuf.git
 cd protobuf
-git checkout v3.16.0
+git checkout v3.18.3
 cd cmake
 cmake -G "Visual Studio 16 2019" -A x64 -DCMAKE_INSTALL_PREFIX=<protobuf_install_dir> -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_BUILD_EXAMPLES=OFF .
 msbuild protobuf.sln /m /p:Configuration=Release
@@ -150,7 +150,7 @@ A more general way is to build and install it from source. See the instructions
   ```bash
     git clone https://github.com/protocolbuffers/protobuf.git
     cd protobuf
-    git checkout v3.16.0
+    git checkout v3.18.3
     git submodule update --init --recursive
     mkdir build_source && cd build_source
     cmake ../cmake -Dprotobuf_BUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release
@@ -162,7 +162,7 @@ A more general way is to build and install it from source. See the instructions
   ```bash
     git clone https://github.com/protocolbuffers/protobuf.git
     cd protobuf
-    git checkout v3.16.0
+    git checkout v3.18.3
     git submodule update --init --recursive
     mkdir build_source && cd build_source
     cmake ../cmake  -DCMAKE_INSTALL_LIBDIR=lib64 -Dprotobuf_BUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release
@@ -193,9 +193,9 @@ pip install -e .
 export NUM_CORES=`sysctl -n hw.ncpu`
 brew update
 brew install autoconf && brew install automake
-wget https://github.com/protocolbuffers/protobuf/releases/download/v3.16.0/protobuf-cpp-3.16.0.tar.gz
-tar -xvf protobuf-cpp-3.16.0.tar.gz
-cd protobuf-3.16.0
+wget https://github.com/protocolbuffers/protobuf/releases/download/v3.18.3/protobuf-cpp-3.18.3.tar.gz
+tar -xvf protobuf-cpp-3.18.3.tar.gz
+cd protobuf-3.18.3
 mkdir build_source && cd build_source
 cmake ../cmake -Dprotobuf_BUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release
 make -j${NUM_CORES}

diff --git a/requirements-release.txt b/requirements-release.txt
@@ -1,5 +1,5 @@
 numpy == 1.21.5
-protobuf == 3.16.0
+protobuf == 3.18.3
 pytest
 nbval
 ipython

diff --git a/workflow_scripts/protobuf/build_protobuf_unix.sh b/workflow_scripts/protobuf/build_protobuf_unix.sh
@@ -4,6 +4,7 @@
 
 export CORE_NUMBER=$1
 export INSTALL_PROTOBUF_PATH=$2
+export BUILD_TYPE=$3
 
 if [[ -z "$CORE_NUMBER" ]]; then
    export CORE_NUMBER=1
@@ -13,14 +14,18 @@ if [[ -z "$INSTALL_PROTOBUF_PATH" ]]; then
    export INSTALL_PROTOBUF_PATH=/usr
 fi
 
+if [[ -z "$BUILD_TYPE" ]]; then
+   export BUILD_TYPE=Release
+fi
+
 # Build protobuf from source with -fPIC on Unix-like system
 ORIGINAL_PATH=$(pwd)
 cd ..
-wget https://github.com/protocolbuffers/protobuf/releases/download/v3.16.0/protobuf-cpp-3.16.0.tar.gz
-tar -xvf protobuf-cpp-3.16.0.tar.gz
-cd protobuf-3.16.0
+wget https://github.com/protocolbuffers/protobuf/releases/download/v3.18.3/protobuf-cpp-3.18.3.tar.gz
+tar -xvf protobuf-cpp-3.18.3.tar.gz
+cd protobuf-3.18.3
 mkdir build_source && cd build_source
-cmake ../cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX=$INSTALL_PROTOBUF_PATH -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release
+cmake ../cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX=$INSTALL_PROTOBUF_PATH -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=$BUILD_TYPE
 make -j$CORE_NUMBER
 if [ "$INSTALL_PROTOBUF_PATH" == "/usr" ]; then
     # install protobuf on default system path so it needs sudo permission

diff --git a/workflow_scripts/protobuf/build_protobuf_win.ps1 b/workflow_scripts/protobuf/build_protobuf_win.ps1
@@ -6,9 +6,9 @@ param(
 )
 
 echo "Build protobuf from source on Windows."
-Invoke-WebRequest -Uri https://github.com/protocolbuffers/protobuf/releases/download/v3.16.0/protobuf-cpp-3.16.0.tar.gz -OutFile protobuf.tar.gz -Verbose
+Invoke-WebRequest -Uri https://github.com/protocolbuffers/protobuf/releases/download/v3.18.3/protobuf-cpp-3.18.3.tar.gz -OutFile protobuf.tar.gz -Verbose
 tar -xvf protobuf.tar.gz
-cd protobuf-3.16.0
+cd protobuf-3.18.3
 $protobuf_root_dir = Get-Location
 mkdir protobuf_install
 cd cmake