Allow setting optimization level for numerical and backend tests (onn…

…x#1407) * - Add cmake variable ONNX_MLIR_TEST_OPTLEVEL to allow setting optimization level for numerical tests - Add -O|--Optlevel option to allow setting optimization level for backend tests - Prefix env var in test/backend/variables.py with TEST_ to avoid collision - Run tests in dev image with -O0 to cover this case Signed-off-by: Gong Su <[email protected]> * Minor updates Signed-off-by: Gong Su <[email protected]>
python3kgae · May 6, 2022 · ef48b4a · ef48b4a
1 parent 088a691
commit ef48b4a
Show file tree

Hide file tree

Showing 8 changed files with 51 additions and 24 deletions.
diff --git a/docker/Dockerfile.onnx-mlir-dev b/docker/Dockerfile.onnx-mlir-dev
@@ -36,6 +36,7 @@ RUN LLVM_PROJECT_ROOT=${WORK_DIR}/llvm-project \
                                   [ "$(uname -m)" = "ppc64le" ] && echo || echo)} \
     && cmake -DMLIR_DIR=${LLVM_PROJECT_ROOT}/build/lib/cmake/mlir \
              -DCMAKE_BUILD_TYPE=Debug \
+             -DONNX_MLIR_TEST_OPTLEVEL=0 \
              -DONNX_MLIR_ACCELERATORS=${BUILD_NNPA} .. \
     && make -j${NPROC} \
     && make -j${NPROC} LIT_OPTS=-v check-onnx-lit \
@@ -44,10 +45,12 @@ RUN LLVM_PROJECT_ROOT=${WORK_DIR}/llvm-project \
                    [ "$(uname -m)" = "x86_64" ] &&  echo || \
                    [ "$(uname -m)" = "ppc64le" ] && echo || echo) \
     && TEST_ARGS="-mcpu=${TEST_MCPU}" \
+    && TEST_OPTLEVEL=0 \
     && make NPROC=${NPROC} \
             CTEST_PARALLEL_LEVEL=${NPROC} \
             TEST_MCPU=${TEST_MCPU} \
             TEST_ARGS="${TEST_ARGS}" \
+            TEST_OPTLEVEL=${TEST_OPTLEVEL} \
             -j${NPROC} -f CMakeFiles/Makefile2 \
             check-onnx-backend \
             check-onnx-backend-dynamic \

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
@@ -1,5 +1,18 @@
 # SPDX-License-Identifier: Apache-2.0
 
+# Valid optimization levels are 0, 1, 2, and 3
+# Default to 3 if ONNX_MLIR_TEST_OPTLEVEL not set or cached
+set(OPTLEVELS 0 1 2 3)
+if (NOT DEFINED ONNX_MLIR_TEST_OPTLEVEL)
+  set(ONNX_MLIR_TEST_OPTLEVEL 3 CACHE STRING "" FORCE)
+endif()
+# Fail if specified/cached ONNX_MLIR_TEST_OPTLEVEL is not valid
+if (NOT ${ONNX_MLIR_TEST_OPTLEVEL} IN_LIST OPTLEVELS)
+  unset(ONNX_MLIR_TEST_OPTLEVEL CACHE)
+  message(FATAL_ERROR "ONNX_MLIR_TEST_OPTLEVEL must be one of ${OPTLEVELS}")
+endif()
+message(STATUS "Tests optimization level : ${ONNX_MLIR_TEST_OPTLEVEL}")
+
 # The backend tests require ONNX package installation.
 add_subdirectory(backend)
 

diff --git a/test/backend/common.py b/test/backend/common.py
@@ -1,4 +1,4 @@
-#!/usr/local/bin/python3
+#!/usr/bin/env python3
 
 ##################### common.py ################################################
 #
@@ -60,7 +60,7 @@ def execute_commands(cmds, dynamic_inputs_dims):
                     first_dim = False
                 else:
                     env_string += "," + str(dim_index)
-        my_env["IMPORTER_FORCE_DYNAMIC"] = env_string
+        my_env["TEST_IMPORTER_FORCE_DYNAMIC"] = env_string
     subprocess.run(cmds, env=my_env)
 
 
@@ -103,6 +103,8 @@ def compile_model(model, emit):
 
     # Command
     command_list = [TEST_DRIVER]
+    if args.Optlevel:
+        command_list.append("-O" + args.Optlevel)
     if args.mcpu:
         command_list.append("--mcpu=" + args.mcpu)
     if args.march:
@@ -113,7 +115,6 @@ def compile_model(model, emit):
         command_list.append("--invokeOnnxVersionConverter=true")
     command_list.append(target[emit])
     command_list.append(model_name)
-    command_list.append('-O3')
     command_list.append("-o=" + exec_base)
 
     # Call frontend to process model_name.onnx, bit code will be generated.

diff --git a/test/backend/inference_backend.py b/test/backend/inference_backend.py
@@ -1,4 +1,4 @@
-#!/usr/local/bin/python3
+#!/usr/bin/env python3
 
 ##################### inference_backend.py #####################################
 #

diff --git a/test/backend/signature_backend.py b/test/backend/signature_backend.py
@@ -1,4 +1,4 @@
-#!/usr/local/bin/python3
+#!/usr/bin/env python3
 
 ##################### signature_backend.py #####################################
 #

diff --git a/test/backend/test.py b/test/backend/test.py
@@ -1,4 +1,4 @@
-#!/usr/local/bin/python3
+#!/usr/bin/env python3
 
 ####################### test.py ################################################
 #

diff --git a/test/backend/variables.py b/test/backend/variables.py
@@ -1,4 +1,4 @@
-#!/usr/local/bin/python3
+#!/usr/bin/env python3
 
 ############################ variables.py #####################################
 #
@@ -26,15 +26,15 @@
 
 
 def get_args_from_env():
-    # Casting with "bool" does not work well. When you specify VERBOSE=xxx,
+    # Casting with "bool" does not work well. When you specify TEST_VERBOSE=xxx,
     # regardless of the value of xxx (e.g., true, false, y, n, etc.) the
     # casted bool value will be true. Only if xxx is empty, the casted bool
     # value will be false. This is a bit counter intuitive. So we use strtobool
     # to do the conversion. But note that strtobool can't take an emtpy string.
 
-    VERBOSE = os.getenv("VERBOSE")
-    INVOKECONVERTER = os.getenv("INVOKECONVERTER")
-    IMPORTER_FORCE_DYNAMIC = os.getenv("IMPORTER_FORCE_DYNAMIC")
+    TEST_VERBOSE = os.getenv("TEST_VERBOSE")
+    TEST_INVOKECONVERTER = os.getenv("TEST_INVOKECONVERTER")
+    TEST_IMPORTER_FORCE_DYNAMIC = os.getenv("TEST_IMPORTER_FORCE_DYNAMIC")
     # Force input tensors to constants. Set this to a list of input indices.
     # E.g.
     #   - "0, 2" for the first and third input tensors.
@@ -88,6 +88,12 @@ def get_args_from_env():
         default=os.getenv("TEST_DIM", -1),
         help="dimensions to be changed to unknown (default: all dimensions if TEST_DIM env var not set)",
     )
+    parser.add_argument(
+        "--converter",
+        action="store_true",
+        default=(strtobool(TEST_INVOKECONVERTER) if TEST_INVOKECONVERTER else False),
+        help="invoke version converter (default: false if TEST_INVOKECONVERTER env var not set)",
+    )
     parser.add_argument(
         "-e",
         "--emit",
@@ -96,13 +102,6 @@ def get_args_from_env():
         default=os.getenv("TEST_EMIT", "lib"),
         help="emit lib or jni for testing (default: lib)",
     )
-    parser.add_argument(
-        "-v",
-        "--verbose",
-        action="store_true",
-        default=(strtobool(VERBOSE) if VERBOSE else False),
-        help="verbose output (default: false if VERBOSE env var not set)",
-    )
     parser.add_argument(
         "--mtriple",
         type=str,
@@ -122,10 +121,19 @@ def get_args_from_env():
         help="target a specific architecture, passed to the compiler",
     )
     parser.add_argument(
-        "--converter",
+        "-O",
+        "--Optlevel",
+        type=str,
+        choices=["0", "1", "2", "3"],
+        default=os.getenv("TEST_OPTLEVEL", "3"),
+        help="set compiler optimization level (default: 3 if TEST_OPTLEVEL env var not set)",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
         action="store_true",
-        default=(strtobool(INVOKECONVERTER) if INVOKECONVERTER else False),
-        help="invoke version converter (default: false if INVOKECONVERTER env var not set)",
+        default=(strtobool(TEST_VERBOSE) if TEST_VERBOSE else False),
+        help="verbose output (default: false if TEST_VERBOSE env var not set)",
     )
     parser.add_argument("unittest_args", nargs="*")
     args = parser.parse_args()

diff --git a/test/numerical/CMakeLists.txt b/test/numerical/CMakeLists.txt
@@ -29,9 +29,11 @@ function(add_numerical_unittest test_name)
     set_property(TARGET ${test_name} PROPERTY FOLDER "${test_suite_folder}")
   endif ()
 
-  # Force -O3 when invoking the numerical tests, so that the models in the test
-  # are compiled with -O3. TODO: make that a runtime env variable.
-  add_test(NAME ${test_name} COMMAND ${test_name} -O3 WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+  # Optimization level set by ONNX_MLIR_TEST_OPTLEVEL, defaults to 3
+  add_test(NAME ${test_name}
+    COMMAND ${test_name} -O${ONNX_MLIR_TEST_OPTLEVEL}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    )
   set_tests_properties(${test_name} PROPERTIES LABELS numerical)
 endfunction()