Add ONNX Runtime tests in CI pipelines with installed ONNX Runtime (o…

…nnx#4966) * Add ONNX Runtime tests in CI pipelines with installed ONNX Runtime Signed-off-by: Chun-Wei Chen <[email protected]> * unittest test_with_ort Signed-off-by: Chun-Wei Chen <[email protected]> * skip unsupported ONNX ir by ORT Signed-off-by: Chun-Wei Chen <[email protected]> * skip tests correctly Signed-off-by: Chun-Wei Chen <[email protected]> * install onnx from source Signed-off-by: Chun-Wei Chen <[email protected]> * introduce max supported opset version Signed-off-by: Chun-Wei Chen <[email protected]> * fix lint Signed-off-by: Chun-Wei Chen <[email protected]> * refactor test_with_ort Signed-off-by: Chun-Wei Chen <[email protected]> * black Signed-off-by: Chun-Wei Chen <[email protected]> * use make_model_gen_version to cover more test cases when IR bumps Signed-off-by: Chun-Wei Chen <[email protected]> * fix lint Signed-off-by: Chun-Wei Chen <[email protected]> * use env variables instead of helper.py for max supported version from ORT Signed-off-by: Chun-Wei Chen <[email protected]> * add missing Signed-off-by: Chun-Wei Chen <[email protected]> * string to int Signed-off-by: Chun-Wei Chen <[email protected]> * set env variables in CIs Signed-off-by: Chun-Wei Chen <[email protected]> --------- Signed-off-by: Chun-Wei Chen <[email protected]>
praasz · Mar 21, 2023 · d9f7739 · d9f7739
1 parent 9f8ee3b
commit d9f7739
Show file tree

Hide file tree

Showing 8 changed files with 193 additions and 88 deletions.
diff --git a/.github/workflows/release_linux_aarch64.yml b/.github/workflows/release_linux_aarch64.yml
@@ -123,6 +123,11 @@ jobs:
           ${{ env.img }} \
           bash -exc '\
           source .env/bin/activate && \
+          python -m pip uninstall -y onnx protobuf numpy && python -m pip install -q -r requirements-release.txt && \
+          python -m pip install dist/*manylinux2014_aarch64.whl && \
           python -m pip install -q onnxruntime && \
-          python onnx/test/test_with_ort.py && \
+          export ORT_MAX_IR_SUPPORTED_VERSION=8 \
+          export ORT_MAX_ML_OPSET_SUPPORTED_VERSION=3 \
+          export ORT_MAX_ONNX_OPSET_SUPPORTED_VERSION=18 \
+          pytest && \
           deactivate'
diff --git a/.github/workflows/release_linux_x86_64.yml b/.github/workflows/release_linux_x86_64.yml
@@ -94,5 +94,10 @@ jobs:
     - name: Verify ONNX with ONNX Runtime PyPI package
       if: matrix.python-version != '3.11'
       run: |
+        python -m pip uninstall -y onnx protobuf numpy && python -m pip install -q -r requirements-release.txt
+        python -m pip install dist/*manylinux2014_x86_64.whl
         python -m pip install -q onnxruntime
-        python onnx/test/test_with_ort.py
+        export ORT_MAX_IR_SUPPORTED_VERSION=8
+        export ORT_MAX_ML_OPSET_SUPPORTED_VERSION=3
+        export ORT_MAX_ONNX_OPSET_SUPPORTED_VERSION=18
+        pytest
diff --git a/.github/workflows/release_mac.yml b/.github/workflows/release_mac.yml
@@ -108,12 +108,6 @@ jobs:
         for file in dist/*.whl; do python -m pip install --upgrade $file; done
         pytest
 
-    - name: Verify ONNX with ONNX Runtime PyPI package
-      if: matrix.python-version != '3.11'
-      run: |
-        python -m pip install -q onnxruntime
-        python onnx/test/test_with_ort.py
-
     # Only triggered by weekly event on certain CI
     - name: Build and upload source distribution to PyPI weekly
       if: github.event_name == 'schedule' && matrix.python-version == '3.10' && matrix.target-architecture == 'x86_64'
@@ -128,3 +122,14 @@ jobs:
         python -m pip install setuptools
         python -m pip install --index-url https://upload.pypi.org/legacy/ --use-deprecated=legacy-resolver --no-use-pep517 --no-binary onnx-weekly onnx-weekly
         pytest
+
+    - name: Verify ONNX with ONNX Runtime PyPI package
+      if: matrix.python-version != '3.11'
+      run: |
+        python -m pip uninstall -y onnx protobuf numpy && python -m pip install -q -r requirements-release.txt
+        for file in dist/*.whl; do python -m pip install --upgrade $file; done
+        python -m pip install -q onnxruntime
+        export ORT_MAX_IR_SUPPORTED_VERSION=8
+        export ORT_MAX_ML_OPSET_SUPPORTED_VERSION=3
+        export ORT_MAX_ONNX_OPSET_SUPPORTED_VERSION=18
+        pytest
diff --git a/.github/workflows/release_win.yml b/.github/workflows/release_win.yml
@@ -131,5 +131,11 @@ jobs:
     - name: Verify ONNX with ONNX Runtime PyPI package
       if: matrix.python-version != '3.11'
       run: |
+        cd onnx
+        python -m pip uninstall -y onnx protobuf numpy && python -m pip install -q -r requirements-release.txt
+        Get-ChildItem -Path dist/*.whl | foreach {python -m pip install --upgrade $_.fullname}
         python -m pip install -q onnxruntime
-        python onnx\onnx\test\test_with_ort.py
+        $Env:ORT_MAX_IR_SUPPORTED_VERSION=8
+        $Env:ORT_MAX_ML_OPSET_SUPPORTED_VERSION=3
+        $Env:ORT_MAX_ONNX_OPSET_SUPPORTED_VERSION=18
+        pytest
diff --git a/onnx/test/reference_evaluator_backend_test.py b/onnx/test/reference_evaluator_backend_test.py
@@ -29,6 +29,8 @@
         StrictVersion as version,
     )
 
+from os import getenv
+
 import numpy as np
 from numpy import __version__ as npver
 from numpy import object_ as dtype_object
@@ -41,6 +43,13 @@
 from onnx.reference import ReferenceEvaluator
 from onnx.reference.ops.op_cast import cast_to
 
+# TODO (https://github.com/microsoft/onnxruntime/issues/14932): Get max supported version from onnxruntime directly
+# For now, bump the version in CIs whenever there is a new onnxruntime release
+ORT_MAX_IR_SUPPORTED_VERSION = int(getenv("ORT_MAX_IR_SUPPORTED_VERSION", "8"))
+ORT_MAX_ONNX_OPSET_SUPPORTED_VERSION = int(
+    getenv("ORT_MAX_ONNX_OPSET_SUPPORTED_VERSION", "18")
+)
+
 # Number of tests expected to pass without raising an exception.
 MIN_PASSING_TESTS = 1235
 
@@ -617,6 +626,22 @@ def common_test_onnx_test_run(
                 print("CHECK RUNTIME onnxruntime")
                 from onnxruntime import InferenceSession
 
+                onnx_domain_opset = ORT_MAX_ONNX_OPSET_SUPPORTED_VERSION
+                for opset in te.onnx_model.opset_import:
+                    if opset.domain in ("", "ai.onnx"):
+                        onnx_domain_opset = opset.version
+                        break
+
+                # The new IR or opset version is not supported by onnxruntime yet
+                if (
+                    te.onnx_model.ir_version > ORT_MAX_IR_SUPPORTED_VERSION
+                    or onnx_domain_opset > ORT_MAX_ONNX_OPSET_SUPPORTED_VERSION
+                ):
+                    print(
+                        "Skip test because of IR or opset version is not supported by onnxruntime yet"
+                    )
+                    return
+
                 te.run(
                     lambda obj: InferenceSession(obj.SerializeToString()),
                     lambda *a, **b: TestOnnxBackEndWithReferenceEvaluator.run_fct(