ci: build and test only modified components related test cases

TrackliDev · May 26, 2023 · 22e2738 · 22e2738
1 parent 2cbcafc
commit 22e2738
Show file tree

Hide file tree

Showing 12 changed files with 274 additions and 100 deletions.
diff --git a/.gitignore b/.gitignore
@@ -97,3 +97,5 @@ managed_components
 
 # pytest log
 pytest_embedded_log/
+list_job_*.txt
+size_info.txt
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -20,9 +20,11 @@ workflow:
     - if: $CI_OPEN_MERGE_REQUESTS != null
       variables:
         PIPELINE_COMMIT_SHA: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
+        IS_MR_PIPELINE: 1
     - if: $CI_OPEN_MERGE_REQUESTS == null
       variables:
         PIPELINE_COMMIT_SHA: $CI_COMMIT_SHA
+        IS_MR_PIPELINE: 0
     - when: always
 
 variables:

diff --git a/.gitlab/ci/README.md b/.gitlab/ci/README.md
@@ -21,11 +21,7 @@
       - [Shell Script Related](#shell-script-related)
   - [Manifest File to Control the Build/Test apps](#manifest-file-to-control-the-buildtest-apps)
     - [Grammar](#grammar)
-      - [Operands](#operands)
-      - [Operators](#operators)
-      - [Limitation:](#limitation)
-    - [How does it work?](#how-does-it-work)
-    - [Example](#example)
+    - [Special Rules](#special-rules)
 
 ## General Workflow
 
@@ -242,3 +238,10 @@ We're using the latest version of [idf-build-apps][idf-build-apps]. Please refer
 
 [idf-build-apps]: https://github.com/espressif/idf-build-apps
 [manifest-doc]: https://docs.espressif.com/projects/idf-build-apps/en/latest/manifest.html
+
+### Special Rules
+
+In ESP-IDF CI, there's a few more special rules are additionally supported to disable the check app dependencies feature:
+
+- Add MR labels `BUILD_AND_TEST_ALL_APPS`
+- Run in protected branches
diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml
@@ -21,6 +21,8 @@
   needs:
     - job: fast_template_app
       artifacts: false
+    - job: mr_variables
+      optional: true  # only MR pipelines would have this
   artifacts:
     paths:
       - "**/build*/size.json"
@@ -37,7 +39,7 @@
       - "**/build*/sdkconfig"
       - "**/build*/bootloader/*.bin"
       - "**/build*/partition_table/*.bin"
-      - list_job_*.json
+      - list_job_*.txt
       - size_info.txt
       # unit test specific
       - components/idf_test/unit_test/*.yml
@@ -66,18 +68,18 @@
     # would be clean up after 4 days
     - mc share download shiny-s3/idf-artifacts/${CI_PIPELINE_ID}/${CI_JOB_ID}.zip --expire=96h
   script:
-    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    # CI specific options start from "--parallel-count xxx". could ignore when running locally
     - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
       -t $IDF_TARGET
       --copy-sdkconfig
-      --collect-size-info size_info.txt
-      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
       --extra-preserve-dirs
         examples/bluetooth/esp_ble_mesh/ble_mesh_console
         examples/bluetooth/hci/controller_hci_uart_esp32
         examples/wifi/iperf
+      --modified-components ${MR_MODIFIED_COMPONENTS}
+      --modified-files ${MR_MODIFIED_FILES}
 
 .build_cmake_clang_template:
   extends:
@@ -87,14 +89,14 @@
     TEST_BUILD_OPTS_EXTRA: ""
     TEST_DIR: tools/test_apps/system/cxx_pthread_bluetooth
   script:
-    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    # CI specific options start from "--parallel-count xxx". could ignore when running locally
     - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
       -t $IDF_TARGET
       --copy-sdkconfig
-      --collect-size-info size_info.txt
-      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
+      --modified-components ${MR_MODIFIED_COMPONENTS}
+      --modified-files ${MR_MODIFIED_FILES}
       $TEST_BUILD_OPTS_EXTRA
 
 .build_pytest_template:
@@ -114,30 +116,32 @@
       - "**/build*/config/sdkconfig.json"
       - "**/build*/bootloader/*.bin"
       - "**/build*/partition_table/*.bin"
-      - list_job_*.json
+      - list_job_*.txt
       - size_info.txt
     when: always
     expire_in: 4 days
   script:
-    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    # CI specific options start from "--parallel-count xxx". could ignore when running locally
     - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
       -t $IDF_TARGET
       --pytest-apps
-      --collect-size-info size_info.txt
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
+      --modified-components ${MR_MODIFIED_COMPONENTS}
+      --modified-files ${MR_MODIFIED_FILES}
 
 .build_pytest_no_jtag_template:
   extends: .build_pytest_template
   script:
-    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    # CI specific options start from "--parallel-count xxx". could ignore when running locally
     - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
       -t $IDF_TARGET
       -m \"not host_test and not jtag\"
       --pytest-apps
-      --collect-size-info size_info.txt
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
+      --modified-components ${MR_MODIFIED_COMPONENTS}
+      --modified-files ${MR_MODIFIED_FILES}
 
 .build_pytest_jtag_template:
   extends:
@@ -156,19 +160,20 @@
       - "**/build*/config/sdkconfig.json"
       - "**/build*/bootloader/*.bin"
       - "**/build*/partition_table/*.bin"
-      - list_job_*.json
+      - list_job_*.txt
       - size_info.txt
     when: always
     expire_in: 4 days
   script:
-    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    # CI specific options start from "--parallel-count xxx". could ignore when running locally
     - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
       -t $IDF_TARGET
       -m \"not host_test and jtag\"
       --pytest-apps
-      --collect-size-info size_info.txt
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
+      --modified-components ${MR_MODIFIED_COMPONENTS}
+      --modified-files ${MR_MODIFIED_FILES}
 
 build_pytest_examples_esp32:
   extends:
@@ -310,13 +315,13 @@ build_only_components_apps:
   parallel: 5
   script:
     - set_component_ut_vars
-    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    # CI specific options start from "--parallel-count xxx". could ignore when running locally
     - run_cmd python tools/ci/ci_build_apps.py $COMPONENT_UT_DIRS -v
       -t all
-      --collect-size-info size_info.txt
-      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
+      --modified-components ${MR_MODIFIED_COMPONENTS}
+      --modified-files ${MR_MODIFIED_FILES}
 
 .build_pytest_test_apps_template:
   extends: .build_pytest_template
@@ -336,7 +341,7 @@ build_only_components_apps:
       - "**/build*/bootloader/*.bin"
       - "**/build*/partition_table/*.bin"
       - "**/build*/project_description.json"
-      - list_job_*.json
+      - list_job_*.txt
       - size_info.txt
     when: always
     expire_in: 4 days
@@ -404,13 +409,13 @@ build_only_tools_test_apps:
     - .rules:build:custom_test
   parallel: 9
   script:
-    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    # CI specific options start from "--parallel-count xxx". could ignore when running locally
     - run_cmd python tools/ci/ci_build_apps.py tools/test_apps -v
       -t all
-      --collect-size-info size_info.txt
-      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
+      --modified-components ${MR_MODIFIED_COMPONENTS}
+      --modified-files ${MR_MODIFIED_FILES}
 
 .build_template_app_template:
   extends:
@@ -529,20 +534,18 @@ build_ssc_esp32h2:
       - "**/build*/sdkconfig"
       - "**/build*/bootloader/*.bin"
       - "**/build*/partition_table/*.bin"
-      - list_job_*.json
+      - list_job_*.txt
       - size_info.txt
       - components/idf_test/unit_test/*.yml
     when: always
     expire_in: 4 days
   script:
-    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    # CI specific options start from "--parallel-count xxx". could ignore when running locally
     - run_cmd python tools/ci/ci_build_apps.py tools/unit-test-app -v
       -t $IDF_TARGET
       --config "configs/*="
       --copy-sdkconfig
       --preserve-all
-      --collect-size-info size_info.txt
-      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
     - run_cmd python tools/unit-test-app/tools/UnitTestParser.py tools/unit-test-app ${CI_NODE_INDEX:-1}

diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml
@@ -186,3 +186,22 @@ check_configure_ci_environment_parsing:
   script:
     - cd tools/ci
     - python -m unittest ci_build_apps.py
+
+mr_variables:
+  extends:
+    - .pre_check_template
+    - .rules:mr
+    - .before_script_minimal
+  tags:
+    - build
+  script:
+    - echo "MR_MODIFIED_FILES=$(python tools/ci/ci_get_mr_info.py files ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} | xargs)" >> mr.env
+    - echo "MR_MODIFIED_COMPONENTS=$(python tools/ci/ci_get_mr_info.py components ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} | xargs)" >> mr.env
+    - >
+      if echo "$CI_MERGE_REQUEST_LABELS" | egrep "^([^,\n\r]+,)*BUILD_AND_TEST_ALL_APPS(,[^,\n\r]+)*$"; then
+        echo "BUILD_AND_TEST_ALL_APPS=1" >> mr.env
+      fi
+  artifacts:
+    reports:
+      dotenv: mr.env
+    expire_in: 4 days
diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml
@@ -374,6 +374,7 @@
 #########
 # Rules #
 #########
+### Branches ###
 .rules:protected:
   rules:
     - <<: *if-protected
@@ -382,6 +383,30 @@
   rules:
     - <<: *if-protected-no_label
 
+.rules:dev:
+  rules:
+    - <<: *if-trigger
+    - <<: *if-dev-push
+
+.rules:mr:
+  rules:
+    - <<: *if-dev-push
+
+.rules:tag:release:
+  rules:
+    - <<: *if-tag-release
+
+.rules:ref:master-schedule:
+  rules:
+    - <<: *if-ref-master
+    - <<: *if-schedule
+
+.rules:ref:master-always:
+  rules:
+    - <<: *if-ref-master
+      when: always
+
+### Patterns ###
 .rules:patterns:python-cache:
   rules:
     - *if-schedule
@@ -404,25 +429,6 @@
     - <<: *if-dev-push
       changes: *patterns-danger-npm
 
-.rules:dev:
-  rules:
-    - <<: *if-trigger
-    - <<: *if-dev-push
-
-.rules:tag:release:
-  rules:
-    - <<: *if-tag-release
-
-.rules:ref:master-schedule:
-  rules:
-    - <<: *if-ref-master
-    - <<: *if-schedule
-
-.rules:ref:master-always:
-  rules:
-    - <<: *if-ref-master
-      when: always
-
 .rules:patterns:clang_tidy:
   rules:
     - <<: *if-protected

diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml
@@ -39,6 +39,7 @@
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
       ${PYTEST_EXTRA_FLAGS}
+      --app-info-filepattern \"list_job_*.txt\"
 
 .pytest_examples_dir_template:
   extends: .pytest_template

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -143,7 +143,7 @@ repos:
         require_serial: true
         additional_dependencies:
           - PyYAML == 5.3.1
-          - idf_build_apps
+          - idf_build_apps~=1.0
       - id: sort-build-test-rules-ymls
         name: sort .build-test-rules.yml files
         entry: tools/ci/check_build_test_rules.py sort-yaml