From 22e2738f7831d00bbfdaa1ad7641221df9775b94 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Wed, 24 May 2023 10:53:57 +0800 Subject: [PATCH] ci: build and test only modified components related test cases --- .gitignore | 2 + .gitlab-ci.yml | 2 + .gitlab/ci/README.md | 13 +- .gitlab/ci/build.yml | 55 +++---- .gitlab/ci/pre_check.yml | 19 +++ .gitlab/ci/rules.yml | 44 +++--- .gitlab/ci/target-test.yml | 1 + .pre-commit-config.yaml | 2 +- conftest.py | 89 +++++++++-- pytest.ini | 2 +- tools/ci/ci_build_apps.py | 143 ++++++++++++++---- .../python_packages/ttfw_idf/IDFAssignTest.py | 2 +- 12 files changed, 274 insertions(+), 100 deletions(-) diff --git a/.gitignore b/.gitignore index 6e1a94fe6885..20428dc70e20 100644 --- a/.gitignore +++ b/.gitignore @@ -97,3 +97,5 @@ managed_components # pytest log pytest_embedded_log/ +list_job_*.txt +size_info.txt diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e0d1e6edd827..d44f2fee4b0f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,9 +20,11 @@ workflow: - if: $CI_OPEN_MERGE_REQUESTS != null variables: PIPELINE_COMMIT_SHA: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA + IS_MR_PIPELINE: 1 - if: $CI_OPEN_MERGE_REQUESTS == null variables: PIPELINE_COMMIT_SHA: $CI_COMMIT_SHA + IS_MR_PIPELINE: 0 - when: always variables: diff --git a/.gitlab/ci/README.md b/.gitlab/ci/README.md index 831a913cd40a..e37dbf3384c3 100644 --- a/.gitlab/ci/README.md +++ b/.gitlab/ci/README.md @@ -21,11 +21,7 @@ - [Shell Script Related](#shell-script-related) - [Manifest File to Control the Build/Test apps](#manifest-file-to-control-the-buildtest-apps) - [Grammar](#grammar) - - [Operands](#operands) - - [Operators](#operators) - - [Limitation:](#limitation) - - [How does it work?](#how-does-it-work) - - [Example](#example) + - [Special Rules](#special-rules) ## General Workflow @@ -242,3 +238,10 @@ We're using the latest version of [idf-build-apps][idf-build-apps]. Please refer [idf-build-apps]: https://github.com/espressif/idf-build-apps [manifest-doc]: https://docs.espressif.com/projects/idf-build-apps/en/latest/manifest.html + +### Special Rules + +In ESP-IDF CI, there's a few more special rules are additionally supported to disable the check app dependencies feature: + +- Add MR labels `BUILD_AND_TEST_ALL_APPS` +- Run in protected branches diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index b3883ede7e5e..3e0e7e9ca2de 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -21,6 +21,8 @@ needs: - job: fast_template_app artifacts: false + - job: mr_variables + optional: true # only MR pipelines would have this artifacts: paths: - "**/build*/size.json" @@ -37,7 +39,7 @@ - "**/build*/sdkconfig" - "**/build*/bootloader/*.bin" - "**/build*/partition_table/*.bin" - - list_job_*.json + - list_job_*.txt - size_info.txt # unit test specific - components/idf_test/unit_test/*.yml @@ -66,18 +68,18 @@ # would be clean up after 4 days - mc share download shiny-s3/idf-artifacts/${CI_PIPELINE_ID}/${CI_JOB_ID}.zip --expire=96h script: - # CI specific options start from "--collect-size-info xxx". could ignore when running locally + # CI specific options start from "--parallel-count xxx". could ignore when running locally - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v -t $IDF_TARGET --copy-sdkconfig - --collect-size-info size_info.txt - --collect-app-info list_job_${CI_NODE_INDEX:-1}.json --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} --extra-preserve-dirs examples/bluetooth/esp_ble_mesh/ble_mesh_console examples/bluetooth/hci/controller_hci_uart_esp32 examples/wifi/iperf + --modified-components ${MR_MODIFIED_COMPONENTS} + --modified-files ${MR_MODIFIED_FILES} .build_cmake_clang_template: extends: @@ -87,14 +89,14 @@ TEST_BUILD_OPTS_EXTRA: "" TEST_DIR: tools/test_apps/system/cxx_pthread_bluetooth script: - # CI specific options start from "--collect-size-info xxx". could ignore when running locally + # CI specific options start from "--parallel-count xxx". could ignore when running locally - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v -t $IDF_TARGET --copy-sdkconfig - --collect-size-info size_info.txt - --collect-app-info list_job_${CI_NODE_INDEX:-1}.json --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} + --modified-components ${MR_MODIFIED_COMPONENTS} + --modified-files ${MR_MODIFIED_FILES} $TEST_BUILD_OPTS_EXTRA .build_pytest_template: @@ -114,30 +116,32 @@ - "**/build*/config/sdkconfig.json" - "**/build*/bootloader/*.bin" - "**/build*/partition_table/*.bin" - - list_job_*.json + - list_job_*.txt - size_info.txt when: always expire_in: 4 days script: - # CI specific options start from "--collect-size-info xxx". could ignore when running locally + # CI specific options start from "--parallel-count xxx". could ignore when running locally - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v -t $IDF_TARGET --pytest-apps - --collect-size-info size_info.txt --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} + --modified-components ${MR_MODIFIED_COMPONENTS} + --modified-files ${MR_MODIFIED_FILES} .build_pytest_no_jtag_template: extends: .build_pytest_template script: - # CI specific options start from "--collect-size-info xxx". could ignore when running locally + # CI specific options start from "--parallel-count xxx". could ignore when running locally - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v -t $IDF_TARGET -m \"not host_test and not jtag\" --pytest-apps - --collect-size-info size_info.txt --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} + --modified-components ${MR_MODIFIED_COMPONENTS} + --modified-files ${MR_MODIFIED_FILES} .build_pytest_jtag_template: extends: @@ -156,19 +160,20 @@ - "**/build*/config/sdkconfig.json" - "**/build*/bootloader/*.bin" - "**/build*/partition_table/*.bin" - - list_job_*.json + - list_job_*.txt - size_info.txt when: always expire_in: 4 days script: - # CI specific options start from "--collect-size-info xxx". could ignore when running locally + # CI specific options start from "--parallel-count xxx". could ignore when running locally - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v -t $IDF_TARGET -m \"not host_test and jtag\" --pytest-apps - --collect-size-info size_info.txt --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} + --modified-components ${MR_MODIFIED_COMPONENTS} + --modified-files ${MR_MODIFIED_FILES} build_pytest_examples_esp32: extends: @@ -310,13 +315,13 @@ build_only_components_apps: parallel: 5 script: - set_component_ut_vars - # CI specific options start from "--collect-size-info xxx". could ignore when running locally + # CI specific options start from "--parallel-count xxx". could ignore when running locally - run_cmd python tools/ci/ci_build_apps.py $COMPONENT_UT_DIRS -v -t all - --collect-size-info size_info.txt - --collect-app-info list_job_${CI_NODE_INDEX:-1}.json --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} + --modified-components ${MR_MODIFIED_COMPONENTS} + --modified-files ${MR_MODIFIED_FILES} .build_pytest_test_apps_template: extends: .build_pytest_template @@ -336,7 +341,7 @@ build_only_components_apps: - "**/build*/bootloader/*.bin" - "**/build*/partition_table/*.bin" - "**/build*/project_description.json" - - list_job_*.json + - list_job_*.txt - size_info.txt when: always expire_in: 4 days @@ -404,13 +409,13 @@ build_only_tools_test_apps: - .rules:build:custom_test parallel: 9 script: - # CI specific options start from "--collect-size-info xxx". could ignore when running locally + # CI specific options start from "--parallel-count xxx". could ignore when running locally - run_cmd python tools/ci/ci_build_apps.py tools/test_apps -v -t all - --collect-size-info size_info.txt - --collect-app-info list_job_${CI_NODE_INDEX:-1}.json --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} + --modified-components ${MR_MODIFIED_COMPONENTS} + --modified-files ${MR_MODIFIED_FILES} .build_template_app_template: extends: @@ -529,20 +534,18 @@ build_ssc_esp32h2: - "**/build*/sdkconfig" - "**/build*/bootloader/*.bin" - "**/build*/partition_table/*.bin" - - list_job_*.json + - list_job_*.txt - size_info.txt - components/idf_test/unit_test/*.yml when: always expire_in: 4 days script: - # CI specific options start from "--collect-size-info xxx". could ignore when running locally + # CI specific options start from "--parallel-count xxx". could ignore when running locally - run_cmd python tools/ci/ci_build_apps.py tools/unit-test-app -v -t $IDF_TARGET --config "configs/*=" --copy-sdkconfig --preserve-all - --collect-size-info size_info.txt - --collect-app-info list_job_${CI_NODE_INDEX:-1}.json --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} - run_cmd python tools/unit-test-app/tools/UnitTestParser.py tools/unit-test-app ${CI_NODE_INDEX:-1} diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index 0e86577b7251..a70f2fd2e05a 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -186,3 +186,22 @@ check_configure_ci_environment_parsing: script: - cd tools/ci - python -m unittest ci_build_apps.py + +mr_variables: + extends: + - .pre_check_template + - .rules:mr + - .before_script_minimal + tags: + - build + script: + - echo "MR_MODIFIED_FILES=$(python tools/ci/ci_get_mr_info.py files ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} | xargs)" >> mr.env + - echo "MR_MODIFIED_COMPONENTS=$(python tools/ci/ci_get_mr_info.py components ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} | xargs)" >> mr.env + - > + if echo "$CI_MERGE_REQUEST_LABELS" | egrep "^([^,\n\r]+,)*BUILD_AND_TEST_ALL_APPS(,[^,\n\r]+)*$"; then + echo "BUILD_AND_TEST_ALL_APPS=1" >> mr.env + fi + artifacts: + reports: + dotenv: mr.env + expire_in: 4 days diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 108c16ef6b6a..cb1ebb12bc5d 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -374,6 +374,7 @@ ######### # Rules # ######### +### Branches ### .rules:protected: rules: - <<: *if-protected @@ -382,6 +383,30 @@ rules: - <<: *if-protected-no_label +.rules:dev: + rules: + - <<: *if-trigger + - <<: *if-dev-push + +.rules:mr: + rules: + - <<: *if-dev-push + +.rules:tag:release: + rules: + - <<: *if-tag-release + +.rules:ref:master-schedule: + rules: + - <<: *if-ref-master + - <<: *if-schedule + +.rules:ref:master-always: + rules: + - <<: *if-ref-master + when: always + +### Patterns ### .rules:patterns:python-cache: rules: - *if-schedule @@ -404,25 +429,6 @@ - <<: *if-dev-push changes: *patterns-danger-npm -.rules:dev: - rules: - - <<: *if-trigger - - <<: *if-dev-push - -.rules:tag:release: - rules: - - <<: *if-tag-release - -.rules:ref:master-schedule: - rules: - - <<: *if-ref-master - - <<: *if-schedule - -.rules:ref:master-always: - rules: - - <<: *if-ref-master - when: always - .rules:patterns:clang_tidy: rules: - <<: *if-protected diff --git a/.gitlab/ci/target-test.yml b/.gitlab/ci/target-test.yml index c7530dcf349d..82365927979c 100644 --- a/.gitlab/ci/target-test.yml +++ b/.gitlab/ci/target-test.yml @@ -39,6 +39,7 @@ --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} ${PYTEST_EXTRA_FLAGS} + --app-info-filepattern \"list_job_*.txt\" .pytest_examples_dir_template: extends: .pytest_template diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 412b2562bea6..14d38277993e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -143,7 +143,7 @@ repos: require_serial: true additional_dependencies: - PyYAML == 5.3.1 - - idf_build_apps + - idf_build_apps~=1.0 - id: sort-build-test-rules-ymls name: sort .build-test-rules.yml files entry: tools/ci/check_build_test_rules.py sort-yaml diff --git a/conftest.py b/conftest.py index 385fe9fb26c5..881d01d18241 100644 --- a/conftest.py +++ b/conftest.py @@ -13,6 +13,8 @@ # This is an experimental feature, and if you found any bug or have any question, please report to # https://github.com/espressif/pytest-embedded/issues +import glob +import json import logging import os import re @@ -36,11 +38,11 @@ from pytest_embedded_idf.dut import IdfDut try: - from idf_ci_utils import to_list + from idf_ci_utils import IDF_PATH, to_list from idf_unity_tester import CaseTester except ImportError: sys.path.append(os.path.join(os.path.dirname(__file__), 'tools', 'ci')) - from idf_ci_utils import to_list + from idf_ci_utils import IDF_PATH, to_list from idf_unity_tester import CaseTester try: @@ -252,7 +254,7 @@ def test_case_name(request: FixtureRequest, target: str, config: str) -> str: @pytest.fixture @multi_dut_fixture -def build_dir(app_path: str, target: Optional[str], config: Optional[str]) -> str: +def build_dir(request: FixtureRequest, app_path: str, target: Optional[str], config: Optional[str]) -> str: """ Check local build dir with the following priority: @@ -261,11 +263,6 @@ def build_dir(app_path: str, target: Optional[str], config: Optional[str]) -> st 3. build_ 4. build - Args: - app_path: app path - target: target - config: config - Returns: valid build directory """ @@ -278,6 +275,25 @@ def build_dir(app_path: str, target: Optional[str], config: Optional[str]) -> st check_dirs.append(f'build_{config}') check_dirs.append('build') + idf_pytest_embedded = request.config.stash[_idf_pytest_embedded_key] + + build_dir = None + if idf_pytest_embedded.apps_list is not None: + for check_dir in check_dirs: + binary_path = os.path.join(app_path, check_dir) + if binary_path in idf_pytest_embedded.apps_list: + build_dir = check_dir + break + + if build_dir is None: + pytest.skip( + f'app path {app_path} with target {target} and config {config} is not listed in app info list files' + ) + return '' # not reachable, to fool mypy + + if build_dir: + check_dirs = [build_dir] + for check_dir in check_dirs: binary_path = os.path.join(app_path, check_dir) if os.path.isdir(binary_path): @@ -286,9 +302,8 @@ def build_dir(app_path: str, target: Optional[str], config: Optional[str]) -> st logging.warning('checking binary path: %s... missing... try another place', binary_path) - recommend_place = check_dirs[0] raise ValueError( - f'no build dir valid. Please build the binary via "idf.py -B {recommend_place} build" and run pytest again' + f'no build dir valid. Please build the binary via "idf.py -B {check_dirs[0]} build" and run pytest again' ) @@ -412,20 +427,32 @@ def dev_user(request: FixtureRequest) -> str: # Hook functions # ################## def pytest_addoption(parser: pytest.Parser) -> None: - base_group = parser.getgroup('idf') - base_group.addoption( + idf_group = parser.getgroup('idf') + idf_group.addoption( '--sdkconfig', help='sdkconfig postfix, like sdkconfig.ci.. (Default: None, which would build all found apps)', ) - base_group.addoption('--known-failure-cases-file', help='known failure cases file path') - base_group.addoption( + idf_group.addoption('--known-failure-cases-file', help='known failure cases file path') + idf_group.addoption( '--dev-user', help='user name associated with some specific device/service used during the test execution', ) - base_group.addoption( + idf_group.addoption( '--dev-passwd', help='password associated with some specific device/service used during the test execution', ) + idf_group.addoption( + '--app-info-basedir', + default=IDF_PATH, + help='app info base directory. specify this value when you\'re building under a ' + 'different IDF_PATH. (Default: $IDF_PATH)', + ) + idf_group.addoption( + '--app-info-filepattern', + help='glob pattern to specify the files that include built app info generated by ' + '`idf-build-apps --collect-app-info ...`. will not raise ValueError when binary ' + 'paths not exist in local file system if not listed recorded in the app info.', + ) _idf_pytest_embedded_key = pytest.StashKey['IdfPytestEmbedded']() @@ -446,10 +473,34 @@ def pytest_configure(config: Config) -> None: if not target: # also could specify through markexpr via "-m" target = get_target_marker_from_expr(config.getoption('markexpr') or '') + apps_list = None + app_info_basedir = config.getoption('app_info_basedir') + app_info_filepattern = config.getoption('app_info_filepattern') + if app_info_filepattern: + apps_list = [] + for file in glob.glob(os.path.join(IDF_PATH, app_info_filepattern)): + with open(file) as fr: + for line in fr.readlines(): + if not line.strip(): + continue + + # each line is a valid json + app_info = json.loads(line.strip()) + if app_info_basedir and app_info['app_dir'].startswith(app_info_basedir): + relative_app_dir = os.path.relpath(app_info['app_dir'], app_info_basedir) + apps_list.append(os.path.join(IDF_PATH, os.path.join(relative_app_dir, app_info['build_dir']))) + print('Detected app: ', apps_list[-1]) + else: + print( + f'WARNING: app_info base dir {app_info_basedir} not recognizable in {app_info["app_dir"]}, skipping...' + ) + continue + config.stash[_idf_pytest_embedded_key] = IdfPytestEmbedded( target=target, sdkconfig=config.getoption('sdkconfig'), known_failure_cases_file=config.getoption('known_failure_cases_file'), + apps_list=apps_list, ) config.pluginmanager.register(config.stash[_idf_pytest_embedded_key]) @@ -470,11 +521,13 @@ def __init__( target: str, sdkconfig: Optional[str] = None, known_failure_cases_file: Optional[str] = None, + apps_list: Optional[List[str]] = None, ): # CLI options to filter the test cases self.target = target.lower() self.sdkconfig = sdkconfig self.known_failure_patterns = self._parse_known_failure_cases_file(known_failure_cases_file) + self.apps_list = apps_list self._failed_cases: List[Tuple[str, bool, bool]] = [] # (test_case_name, is_known_failure_cases, is_xfail) @@ -599,7 +652,11 @@ def pytest_runtest_makereport(self, item: Function, call: CallInfo[None]) -> Opt test_case_name = item.funcargs.get('test_case_name', '') if test_case_name: self._failed_cases.append( - (test_case_name, self._is_known_failure(test_case_name), report.keywords.get('xfail', False)) + ( + test_case_name, + self._is_known_failure(test_case_name), + report.keywords.get('xfail', False), + ) ) return report diff --git a/pytest.ini b/pytest.ini index ffc14c7019fc..c9c0c4356040 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,7 +5,7 @@ python_files = pytest_*.py # ignore PytestExperimentalApiWarning for record_xml_attribute # set traceback to "short" to prevent the overwhelming tracebacks addopts = - -s + -s -vv --embedded-services esp,idf --tb short --strict-markers diff --git a/tools/ci/ci_build_apps.py b/tools/ci/ci_build_apps.py index 90961eb21130..b7443395a0af 100644 --- a/tools/ci/ci_build_apps.py +++ b/tools/ci/ci_build_apps.py @@ -8,10 +8,10 @@ import argparse import os import sys +import typing as t import unittest from collections import defaultdict from pathlib import Path -from typing import List, Optional, Set import yaml from idf_build_apps import LOGGER, App, build_apps, find_apps, setup_logging @@ -20,25 +20,28 @@ CI_ENV_VARS = { 'EXTRA_CFLAGS': '-Werror -Werror=deprecated-declarations -Werror=unused-variable ' - '-Werror=unused-but-set-variable -Werror=unused-function -Wstrict-prototypes', + '-Werror=unused-but-set-variable -Werror=unused-function -Wstrict-prototypes', 'EXTRA_CXXFLAGS': '-Werror -Werror=deprecated-declarations -Werror=unused-variable ' - '-Werror=unused-but-set-variable -Werror=unused-function', + '-Werror=unused-but-set-variable -Werror=unused-function', 'LDGEN_CHECK_MAPPING': '1', } def get_pytest_apps( - paths: List[str], + paths: t.List[str], target: str, - config_rules_str: List[str], + config_rules_str: t.List[str], marker_expr: str, filter_expr: str, preserve_all: bool = False, - extra_default_build_targets: Optional[List[str]] = None, -) -> List[App]: + extra_default_build_targets: t.Optional[t.List[str]] = None, + modified_components: t.Optional[t.List[str]] = None, + modified_files: t.Optional[t.List[str]] = None, + ignore_app_dependencies_filepatterns: t.Optional[t.List[str]] = None, +) -> t.List[App]: pytest_cases = get_pytest_cases(paths, target, marker_expr, filter_expr) - _paths: Set[str] = set() + _paths: t.Set[str] = set() test_related_app_configs = defaultdict(set) for case in pytest_cases: for app in case.apps: @@ -53,6 +56,9 @@ def get_pytest_apps( if not case.nightly_run: test_related_app_configs[app.path].add(app.config) + if not extra_default_build_targets: + extra_default_build_targets = [] + app_dirs = list(_paths) if not app_dirs: raise RuntimeError('No apps found') @@ -68,9 +74,12 @@ def get_pytest_apps( build_log_path='build_log.txt', size_json_path='size.json', check_warnings=True, + manifest_rootpath=IDF_PATH, manifest_files=[str(p) for p in Path(IDF_PATH).glob('**/.build-test-rules.yml')], default_build_targets=SUPPORTED_TARGETS + extra_default_build_targets, - manifest_rootpath=IDF_PATH, + modified_components=modified_components, + modified_files=modified_files, + ignore_app_dependencies_filepatterns=ignore_app_dependencies_filepatterns, ) for app in apps: @@ -85,12 +94,15 @@ def get_pytest_apps( def get_cmake_apps( - paths: List[str], + paths: t.List[str], target: str, - config_rules_str: List[str], + config_rules_str: t.List[str], preserve_all: bool = False, - extra_default_build_targets: Optional[List[str]] = None, -) -> List[App]: + extra_default_build_targets: t.Optional[t.List[str]] = None, + modified_components: t.Optional[t.List[str]] = None, + modified_files: t.Optional[t.List[str]] = None, + ignore_app_dependencies_filepatterns: t.Optional[t.List[str]] = None, +) -> t.List[App]: ttfw_app_dirs = get_ttfw_app_paths(paths, target) apps = find_apps( @@ -103,9 +115,12 @@ def get_cmake_apps( size_json_path='size.json', check_warnings=True, preserve=False, + manifest_rootpath=IDF_PATH, manifest_files=[str(p) for p in Path(IDF_PATH).glob('**/.build-test-rules.yml')], default_build_targets=SUPPORTED_TARGETS + extra_default_build_targets, - manifest_rootpath=IDF_PATH, + modified_components=modified_components, + modified_files=modified_files, + ignore_app_dependencies_filepatterns=ignore_app_dependencies_filepatterns, ) apps_for_build = [] @@ -130,7 +145,7 @@ def get_cmake_apps( def main(args: argparse.Namespace) -> None: - extra_default_build_targets: List[str] = [] + extra_default_build_targets: t.List[str] = [] if args.default_build_test_rules: with open(args.default_build_test_rules) as fr: configs = yaml.safe_load(fr) @@ -148,6 +163,9 @@ def main(args: argparse.Namespace) -> None: args.filter_expr, args.preserve_all, extra_default_build_targets, + args.modified_components, + args.modified_files, + args.ignore_app_dependencies_filepatterns, ) else: LOGGER.info('build apps. will skip pytest apps with pytest scripts') @@ -157,6 +175,9 @@ def main(args: argparse.Namespace) -> None: args.config, args.preserve_all, extra_default_build_targets, + args.modified_components, + args.modified_files, + args.ignore_app_dependencies_filepatterns, ) LOGGER.info('Found %d apps after filtering', len(apps)) @@ -175,22 +196,28 @@ def main(args: argparse.Namespace) -> None: if abs_extra_preserve_dir == abs_app_dir or abs_extra_preserve_dir in abs_app_dir.parents: app.preserve = True - sys.exit( - build_apps( - apps, - parallel_count=args.parallel_count, - parallel_index=args.parallel_index, - dry_run=False, - build_verbose=args.build_verbose, - keep_going=True, - collect_size_info=args.collect_size_info, - collect_app_info=args.collect_app_info, - ignore_warning_strs=args.ignore_warning_str, - ignore_warning_file=args.ignore_warning_file, - copy_sdkconfig=args.copy_sdkconfig, - ) + res = build_apps( + apps, + parallel_count=args.parallel_count, + parallel_index=args.parallel_index, + dry_run=False, + build_verbose=args.build_verbose, + keep_going=True, + collect_size_info='size_info.txt', + collect_app_info='list_job_@p.txt', + ignore_warning_strs=args.ignore_warning_str, + ignore_warning_file=args.ignore_warning_file, + copy_sdkconfig=args.copy_sdkconfig, + modified_components=args.modified_components, + modified_files=args.modified_files, + ignore_app_dependencies_filepatterns=args.ignore_app_dependencies_filepatterns, ) + if isinstance(res, tuple): + sys.exit(res[0]) + else: + sys.exit(res) + if __name__ == '__main__': parser = argparse.ArgumentParser( @@ -249,8 +276,7 @@ def main(args: argparse.Namespace) -> None: parser.add_argument( '--ignore-warning-str', nargs='+', - help='Ignore the warning string that match the specified regex in the build output. ' - 'Can be specified multiple times.', + help='Ignore the warning string that match the specified regex in the build output. space-separated list', ) parser.add_argument( '--ignore-warning-file', @@ -298,6 +324,30 @@ def main(args: argparse.Namespace) -> None: help='by default this script would set the build flags exactly the same as the CI ones. ' 'Set this flag to use your local build flags.', ) + parser.add_argument( + '--modified-components', + nargs='*', + default=None, + help='space-separated list which specifies the modified components. app with `depends_components` set in the ' + 'corresponding manifest files would only be built if depends on any of the specified components.', + ) + parser.add_argument( + '--modified-files', + nargs='*', + default=None, + help='space-separated list which specifies the modified files. app with `depends_filepatterns` set in the ' + 'corresponding manifest files would only be built if any of the specified file pattern matches any of the ' + 'specified modified files.', + ) + parser.add_argument( + '-if', + '--ignore-app-dependencies-filepatterns', + nargs='*', + default=None, + help='space-separated list which specifies the file patterns used for ignoring checking the app dependencies. ' + 'The `depends_components` and `depends_filepatterns` set in the manifest files will be ignored when any of the ' + 'specified file patterns matches any of the modified files. Must be used together with --modified-files', + ) arguments = parser.parse_args() @@ -309,6 +359,37 @@ def main(args: argparse.Namespace) -> None: os.environ[_k] = _v LOGGER.info(f'env var {_k} set to "{_v}"') + if os.getenv('IS_MR_PIPELINE') == '0' or os.getenv('BUILD_AND_TEST_ALL_APPS') == '1': + # if it's not MR pipeline or env var BUILD_AND_TEST_ALL_APPS=1, + # remove component dependency related arguments + if 'modified_components' in arguments: + arguments.modified_components = None + if 'modified_files' in arguments: + arguments.modified_files = None + + # file patterns to tigger full build + if 'modified_components' in arguments and not arguments.ignore_app_dependencies_filepatterns: + arguments.ignore_app_dependencies_filepatterns = [ + # tools + 'tools/cmake/**/*', + 'tools/tools.json', + # components + 'components/cxx/**/*', + 'components/esp_common/**/*', + 'components/esp_hw_support/**/*', + 'components/esp_rom/**/*', + 'components/esp_system/**/*', + 'components/esp_timer/**/*', + 'components/freertos/**/*', + 'components/hal/**/*', + 'components/heap/**/*', + 'components/log/**/*', + 'components/newlib/**/*', + 'components/riscv/**/*', + 'components/soc/**/*', + 'components/xtensa/**/*', + ] + main(arguments) diff --git a/tools/ci/python_packages/ttfw_idf/IDFAssignTest.py b/tools/ci/python_packages/ttfw_idf/IDFAssignTest.py index 3b8262f8ba79..68a151e32362 100644 --- a/tools/ci/python_packages/ttfw_idf/IDFAssignTest.py +++ b/tools/ci/python_packages/ttfw_idf/IDFAssignTest.py @@ -54,7 +54,7 @@ def __init__(self, test_case_path, ci_config_file, case_group=IDFCaseGroup): super(IDFAssignTest, self).__init__(test_case_path, ci_config_file, case_group) def format_build_log_path(self, parallel_num): - return 'list_job_{}.json'.format(parallel_num) + return 'list_job_{}.txt'.format(parallel_num) def create_artifact_index_file(self, project_id=None, pipeline_id=None): if project_id is None: