Use component manager if available as python package

.gitignore

@@ -71,8 +71,12 @@ test_multi_heap_host
 .idea/
 cmake-build-*/
 
-# Results for the checking of the Python coding style
+# Results for the checking of the Python coding style and static analysis
+.mypy_cache
 flake8_output.txt
 
 # ESP-IDF default build directory name
 build
+
+# lock files for examples and components
+dependencies.lock

.gitmodules

@@ -78,3 +78,4 @@
 [submodule "components/esp_wifi/lib"]
 	path = components/esp_wifi/lib
 	url = ../../espressif/esp32-wifi-lib.git
+

requirements.txt

@@ -11,6 +11,7 @@ future>=0.15.2
 cryptography>=2.1.4
 pyparsing>=2.0.3,<2.4.0
 pyelftools>=0.22
+
 # windows-curses are required in Windows command line but cannot be installed in MSYS2. A requirement like
 # "windows-curses; sys_platform == 'win32'" would want to install the package on both of them. There is no environment
 # marker for detecting MSYS2. So instead, a dummy custom package is used with "windows-curses" dependency for Windows

tools/ci/test_build_system_cmake.sh

@@ -556,8 +556,8 @@ endmenu\n" >> ${IDF_PATH}/Kconfig
     idf.py -DSDKCONFIG_DEFAULTS="sdkconfig.defaults1;sdkconfig.defaults2" reconfigure > /dev/null
     grep "CONFIG_PARTITION_TABLE_OFFSET=0x10000" sdkconfig || failure "The define from sdkconfig.defaults1 should be in sdkconfig"
     grep "CONFIG_PARTITION_TABLE_TWO_OTA=y" sdkconfig || failure "The define from sdkconfig.defaults2 should be in sdkconfig"
-    rm sdkconfig.defaults1
-    rm sdkconfig.defaults2
+    rm sdkconfig.defaults1 sdkconfig.defaults2 sdkconfig
+    git checkout sdkconfig.defaults
 
     print_status "Supports git worktree"
     clean_build_dir
@@ -577,6 +577,17 @@ endmenu\n" >> ${IDF_PATH}/Kconfig
     git checkout CMakeLists.txt
     rm -f log.txt
 
+    print_status "Compiles with dependencies delivered by component manager"
+    clean_build_dir
+    printf "\n#include \"test_component.h\"\n" >> main/main.c
+    printf "dependencies:\n  test_component:\n    path: test_component\n    git: ${COMPONENT_MANAGER_TEST_REPO}\n" >> idf_project.yml
+    ! idf.py build || failure "Build should fail if dependencies are not installed"
+    pip install ${COMPONENT_MANAGER_REPO}
+    idf.py reconfigure build || failure "Build succeeds once requirements are installed"
+    pip uninstall -y idf_component_manager
+    rm idf_project.yml
+    git checkout main/main.c
+
     print_status "All tests completed"
     if [ -n "${FAILURES}" ]; then
         echo "Some failures were detected:"

tools/cmake/component.cmake

@@ -215,13 +215,32 @@ function(__component_get_requirements)
         -D "COMPONENT_REQUIRES_FILE=${component_requires_file}"
         -P "${idf_path}/tools/cmake/scripts/component_get_requirements.cmake"
         RESULT_VARIABLE result
-        ERROR_VARIABLE error
-        )
+        ERROR_VARIABLE error)
 
     if(NOT result EQUAL 0)
         message(FATAL_ERROR "${error}")
     endif()
 
+    idf_build_get_property(idf_component_manager IDF_COMPONENT_MANAGER)
+    if(idf_component_manager AND idf_component_manager EQUAL "1")
+        # Call for component manager once again to inject dependencies
+        idf_build_get_property(python PYTHON)
+        execute_process(COMMAND ${python}
+            "-m"
+            "idf_component_manager.prepare_components"
+            "--project_dir=${project_dir}"
+            "inject_requrements"
+            "--idf_path=${idf_path}"
+            "--build_dir=${build_dir}"
+            "--component_requires_file=${component_requires_file}"
+            RESULT_VARIABLE result
+            ERROR_VARIABLE error)
+
+        if(NOT result EQUAL 0)
+            message(FATAL_ERROR "${error}")
+        endif()
+    endif()
+
     include(${component_requires_file})
 
     file(REMOVE ${build_properties_file})
@@ -409,6 +428,10 @@ function(idf_component_register)
     __component_check_target()
     __component_add_sources(sources)
 
+    # Add component manifest and lock files to list of dependencies
+    set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${COMPONENT_DIR}/idf_component.yml")
+    set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${COMPONENT_DIR}/dependencies.lock")
+
     # Create the final target for the component. This target is the target that is
     # visible outside the build system.
     __component_get_target(component_target ${COMPONENT_ALIAS})

tools/cmake/project.cmake

@@ -169,6 +169,8 @@ function(__project_init components_var test_components_var)
         endif()
     endfunction()
 
+    idf_build_set_property(IDF_COMPONENT_MANAGER "$ENV{IDF_COMPONENT_MANAGER}")
+
     # Add component directories to the build, given the component filters, exclusions
     # extra directories, etc. passed from the root CMakeLists.txt.
     if(COMPONENT_DIRS)
@@ -179,6 +181,43 @@ function(__project_init components_var test_components_var)
             __project_component_dir(${component_dir})
         endforeach()
     else()
+        # Add project manifest and lock file to the list of dependencies
+        set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${CMAKE_CURRENT_LIST_DIR}/idf_project.yml")
+        set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${CMAKE_CURRENT_LIST_DIR}/dependencies.lock")
+
+        idf_build_get_property(idf_component_manager IDF_COMPONENT_MANAGER)
+        if(idf_component_manager)
+            if(idf_component_manager EQUAL "0")
+                message(VERBOSE "IDF Component manager was explicitly disabled by setting IDF_COMPONENT_MANAGER=0")
+            elseif(idf_component_manager EQUAL "1")
+                set(managed_components_list_file ${CMAKE_BINARY_DIR}/managed_components_list.temp.cmake)
+
+                # Call for package manager to prepare remote dependencies
+                execute_process(COMMAND ${PYTHON}
+                    "-m"
+                    "idf_component_manager.prepare_components"
+                    "--project_dir=${CMAKE_CURRENT_LIST_DIR}"
+                    "prepare_dependencies"
+                    "--managed_components_list_file=${managed_components_list_file}"
+                    RESULT_VARIABLE result
+                    ERROR_VARIABLE error)
+
+                if(NOT result EQUAL 0)
+                    message(FATAL_ERROR "${error}")
+                endif()
+
+                # Include managed components
+                include(${managed_components_list_file})
+                file(REMOVE ${managed_components_list_file})
+            else()
+                message(WARNING "IDF_COMPONENT_MANAGER environment variable is set to unknown value "
+                        "\"${idf_component_manager}\". If you want to use component manager set it to 1.")
+            endif()
+        elseif(EXISTS "${CMAKE_CURRENT_LIST_DIR}/idf_project.yml")
+            message(WARNING "\"idf_project.yml\" file is found in project directory, "
+                    "but component manager is not enabled. Please set IDF_COMPONENT_MANAGER environment variable.")
+        endif()
+
         # Look for components in the usual places: CMAKE_CURRENT_LIST_DIR/main,
         # CMAKE_CURRENT_LIST_DIR/components, and the extra component dirs
         if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/main")

tools/idf.py

@@ -649,6 +649,17 @@ def parse_project_dir(project_dir):
             if name.endswith('_ext'):
                 extensions[name] = import_module(name)
 
+    # Load component manager if available and not explicitly disabled
+    if os.getenv('IDF_COMPONENT_MANAGER', None) != '0':
+        try:
+            from idf_component_manager import idf_extensions
+
+            extensions['component_manager_ext'] = idf_extensions
+            os.environ['IDF_COMPONENT_MANAGER'] = '1'
+
+        except ImportError:
+            pass
+
     for name, extension in extensions.items():
         try:
             all_actions = merge_action_lists(all_actions, extension.action_extensions(all_actions, project_dir))

tools/idf_py_actions/errors.py

@@ -1,6 +1,6 @@
 class FatalError(RuntimeError):
     """
-    Wrapper class for runtime errors that aren't caused by bugs in idf.py or the build proces.s
+    Wrapper class for runtime errors that aren't caused by bugs in idf.py or the build process.
     """
 
     pass