Terraform: support updating local path modules

Add support for fetching and updating local path modules:

```
module "project" {
  source = "./project"
}
```

Terraform will resolve all local path modules when updating the
lockfile, so we need to make sure all requirements for the same
dependency have been updated, and written to the temporary directory
before we run `terraform providers lock _provider_`.

This also updates the file fetcher to not make an api request when
checking folder contents.

common/lib/dependabot/file_fetchers/base.rb

@@ -157,11 +157,12 @@ def repo_contents(dir: ".", ignore_base_directory: false,
         path = Pathname.new(File.join(dir)).cleanpath.to_path.gsub(%r{^/*}, "")
 
         @repo_contents ||= {}
-        @repo_contents[dir] ||= _fetch_repo_contents(
-          path,
-          raise_errors: raise_errors,
-          fetch_submodules: fetch_submodules
-        )
+        @repo_contents[dir] ||= if repo_contents_path
+                                  _cloned_repo_contents(path)
+                                else
+                                  _fetch_repo_contents(path, raise_errors: raise_errors,
+                                                             fetch_submodules: fetch_submodules)
+                                end
       end
 
       #################################################
@@ -225,6 +226,22 @@ def _github_repo_contents(repo, path, commit)
         github_response.map { |f| _build_github_file_struct(f) }
       end
 
+      def _cloned_repo_contents(relative_path)
+        repo_path = File.join(clone_repo_contents, relative_path)
+        return [] unless Dir.exist?(repo_path)
+
+        Dir.entries(repo_path).map do |name|
+          next if [".", ".."].include?(name)
+
+          OpenStruct.new(
+            name: name,
+            path: Pathname.new(File.join(relative_path, name)).cleanpath.to_path,
+            type: Dir.exist?(File.join(repo_path, name)) ? "dir" : "file",
+            size: 0 # NOTE: added for parity with github contents API
+          )
+        end.compact
+      end
+
       def update_linked_paths(repo, path, commit, github_response)
         case github_response.type
         when "submodule"

terraform/lib/dependabot/terraform/file_fetcher.rb

@@ -9,6 +9,9 @@ module Terraform
     class FileFetcher < Dependabot::FileFetchers::Base
       include FileSelector
 
+      # https://www.terraform.io/docs/language/modules/sources.html#local-paths
+      LOCAL_PATH_SOURCE = %r{source\s*=\s*['"](?<path>..?\/[^'"]+)}.freeze
+
       def self.required_files_in?(filenames)
         filenames.any? { |f| f.end_with?(".tf", ".hcl") }
       end
@@ -23,6 +26,7 @@ def fetch_files
         fetched_files = []
         fetched_files += terraform_files
         fetched_files += terragrunt_files
+        fetched_files += local_path_module_files(terraform_files)
         fetched_files += [lock_file] if lock_file
 
         return fetched_files if fetched_files.any?
@@ -47,6 +51,35 @@ def terragrunt_files
           map { |f| fetch_file_from_host(f.name) }
       end
 
+      def local_path_module_files(files, dir: ".")
+        terraform_files = []
+
+        files.each do |file|
+          terraform_file_local_module_details(file).each do |path|
+            base_path = Pathname.new(File.join(dir, path)).cleanpath.to_path
+            nested_terraform_files =
+              repo_contents(dir: base_path).
+              select { |f| f.type == "file" && f.name.end_with?(".tf") }.
+              map { |f| fetch_file_from_host(File.join(base_path, f.name)) }
+            terraform_files += nested_terraform_files
+            terraform_files += local_path_module_files(nested_terraform_files, dir: path)
+          end
+        end
+
+        # NOTE: The `support_file` attribute is not used but we set this to
+        # match what we do in other ecosystems
+        terraform_files.tap { |fs| fs.each { |f| f.support_file = true } }
+      end
+
+      def terraform_file_local_module_details(file)
+        return [] unless file.name.end_with?(".tf")
+        return [] unless file.content.match?(LOCAL_PATH_SOURCE)
+
+        file.content.scan(LOCAL_PATH_SOURCE).flatten.map do |path|
+          Pathname.new(path).cleanpath.to_path
+        end
+      end
+
       def lock_file
         @lock_file ||= fetch_file_if_present(".terraform.lock.hcl")
       end

terraform/lib/dependabot/terraform/file_updater.rb

@@ -29,7 +29,7 @@ def updated_dependency_files
 
           updated_files << updated_file(file: file, content: updated_content)
         end
-        updated_lockfile_content = update_lockfile_declaration
+        updated_lockfile_content = update_lockfile_declaration(updated_files)
 
         if updated_lockfile_content && lock_file.content != updated_lockfile_content
           updated_files << updated_file(file: lock_file, content: updated_lockfile_content)
@@ -92,7 +92,7 @@ def update_registry_declaration(new_req, old_req, updated_content)
         end
       end
 
-      def update_lockfile_declaration # rubocop:disable Metrics/AbcSize
+      def update_lockfile_declaration(updated_manifest_files) # rubocop:disable Metrics/AbcSize
         return if lock_file.nil?
 
         new_req = dependency.requirements.first
@@ -106,6 +106,9 @@ def update_lockfile_declaration # rubocop:disable Metrics/AbcSize
 
         base_dir = dependency_files.first.directory
         SharedHelpers.in_a_temporary_repo_directory(base_dir, repo_contents_path) do
+          # Update the provider requirements in case the previous requirement doesn't allow the new version
+          updated_manifest_files.each { |f| File.write(f.name, f.content) }
+
           File.write(".terraform.lock.hcl", lockfile_dependency_removed)
           SharedHelpers.run_shell_command("terraform providers lock #{provider_source}")
 

terraform/spec/dependabot/terraform/file_fetcher_spec.rb

@@ -14,68 +14,30 @@
       directory: directory
     )
   end
+
   let(:file_fetcher_instance) do
-    described_class.new(source: source, credentials: credentials)
-  end
-  let(:directory) { "/" }
-  let(:github_url) { "https://api.github.com/" }
-  let(:url) { github_url + "repos/gocardless/bump/contents/" }
-  let(:credentials) do
-    [{
-      "type" => "git_source",
-      "host" => "github.com",
-      "username" => "x-access-token",
-      "password" => "token"
-    }]
+    described_class.new(source: source, credentials: [], repo_contents_path: repo_contents_path)
   end
 
-  before { allow(file_fetcher_instance).to receive(:commit).and_return("sha") }
+  let(:project_name) { "provider" }
+  let(:directory) { "/" }
+  let(:repo_contents_path) { build_tmp_repo(project_name) }
 
-  context "with a Terraform file" do
-    before do
-      stub_request(:get, url + "?ref=sha").
-        with(headers: { "Authorization" => "token token" }).
-        to_return(
-          status: 200,
-          body: fixture("github", "contents_terraform_repo.json"),
-          headers: { "content-type" => "application/json" }
-        )
+  after do
+    FileUtils.rm_rf(repo_contents_path)
+  end
 
-      %w(main.tf outputs.tf variables.tf).each do |nm|
-        stub_request(:get, File.join(url, "#{nm}?ref=sha")).
-          with(headers: { "Authorization" => "token token" }).
-          to_return(
-            status: 200,
-            body: fixture("github", "contents_terraform_file.json"),
-            headers: { "content-type" => "application/json" }
-          )
-      end
-    end
+  context "with Terraform files" do
+    let(:project_name) { "versions_file" }
 
     it "fetches the Terraform files" do
       expect(file_fetcher_instance.files.map(&:name)).
-        to match_array(%w(main.tf outputs.tf variables.tf))
+        to match_array(%w(main.tf versions.tf))
     end
   end
 
   context "with a HCL based terragrunt file" do
-    before do
-      stub_request(:get, url + "?ref=sha").
-        with(headers: { "Authorization" => "token token" }).
-        to_return(
-          status: 200,
-          body: fixture("github", "contents_terragrunt_hcl_repo.json"),
-          headers: { "content-type" => "application/json" }
-        )
-
-      stub_request(:get, File.join(url, "terragrunt.hcl?ref=sha")).
-        with(headers: { "Authorization" => "token token" }).
-        to_return(
-          status: 200,
-          body: fixture("github", "contents_terraform_file.json"),
-          headers: { "content-type" => "application/json" }
-        )
-    end
+    let(:project_name) { "terragrunt_hcl" }
 
     it "fetches the Terragrunt file" do
       expect(file_fetcher_instance.files.map(&:name)).
@@ -84,23 +46,7 @@
   end
 
   context "with a lockfile" do
-    before do
-      stub_request(:get, url + "?ref=sha").
-        with(headers: { "Authorization" => "token token" }).
-        to_return(
-          status: 200,
-          body: fixture("github", "contents_lockfile_repo.json"),
-          headers: { "content-type" => "application/json" }
-        )
-
-      stub_request(:get, File.join(url, ".terraform.lock.hcl?ref=sha")).
-        with(headers: { "Authorization" => "token token" }).
-        to_return(
-          status: 200,
-          body: fixture("github", "contents_terraform_file.json"),
-          headers: { "content-type" => "application/json" }
-        )
-    end
+    let(:project_name) { "terraform_lock_only" }
 
     it "fetches the lockfile" do
       expect(file_fetcher_instance.files.map(&:name)).
@@ -111,19 +57,22 @@
   context "with a directory that doesn't exist" do
     let(:directory) { "/nonexistent" }
 
-    before do
-      stub_request(:get, url + "nonexistent?ref=sha").
-        with(headers: { "Authorization" => "token token" }).
-        to_return(
-          status: 404,
-          body: fixture("github", "not_found.json"),
-          headers: { "content-type" => "application/json" }
-        )
-    end
-
     it "raises a helpful error" do
       expect { file_fetcher_instance.files }.
         to raise_error(Dependabot::DependencyFileNotFound)
     end
   end
+
+  context "when fetching nested local path modules" do
+    let(:project_name) { "provider_with_multiple_local_path_modules" }
+
+    it "fetches nested terraform files" do
+      expect(file_fetcher_instance.files.map(&:name)).
+        to match_array(
+          %w(.terraform.lock.hcl loader.tf providers.tf
+             loader/providers.tf loader/projects.tf
+             loader/project/providers.tf)
+        )
+    end
+  end
 end

terraform/spec/dependabot/terraform/file_updater_spec.rb

@@ -698,7 +698,7 @@ module "consul" {
           <<~DEP
             provider "registry.terraform.io/hashicorp/aws" {
               version     = "3.45.0"
-              constraints = ">= 3.37.0, < 3.46.0"
+              constraints = ">= 3.42.0, < 3.46.0"
           DEP
         )
       end
@@ -1055,7 +1055,7 @@ module "caf" {
       end
     end
 
-    describe "when updating a provider with local path modules" do
+    describe "when updating a provider with mixed case path" do
       let(:project_name) { "provider_with_mixed_case" }
       let(:dependencies) do
         [
@@ -1099,5 +1099,92 @@ module "caf" {
         )
       end
     end
+
+    describe "when updating a provider with multiple local path modules" do
+      let(:project_name) { "provider_with_multiple_local_path_modules" }
+      let(:dependencies) do
+        [
+          Dependabot::Dependency.new(
+            name: "Mongey/confluentcloud",
+            version: "0.0.10",
+            previous_version: "0.0.6",
+            requirements: [{
+              requirement: "0.0.10",
+              groups: [],
+              file: "providers.tf",
+              source: {
+                type: "provider",
+                registry_hostname: "registry.terraform.io",
+                module_identifier: "Mongey/confluentcloud"
+              }
+            }, {
+              requirement: "0.0.10",
+              groups: [],
+              file: "loader/providers.tf",
+              source: {
+                type: "provider",
+                registry_hostname: "registry.terraform.io",
+                module_identifier: "Mongey/confluentcloud"
+              }
+            }, {
+              requirement: "0.0.10",
+              groups: [],
+              file: "loader/project/providers.tf",
+              source: {
+                type: "provider",
+                registry_hostname: "registry.terraform.io",
+                module_identifier: "Mongey/confluentcloud"
+              }
+            }],
+            previous_requirements: [{
+              requirement: "0.0.6",
+              groups: [],
+              file: "providers.tf",
+              source: {
+                type: "provider",
+                registry_hostname: "registry.terraform.io",
+                module_identifier: "Mongey/confluentcloud"
+              }
+            }, {
+              requirement: "0.0.6",
+              groups: [],
+              file: "loader/providers.tf",
+              source: {
+                type: "provider",
+                registry_hostname: "registry.terraform.io",
+                module_identifier: "Mongey/confluentcloud"
+              }
+            }, {
+              requirement: "0.0.6",
+              groups: [],
+              file: "loader/project/providers.tf",
+              source: {
+                type: "provider",
+                registry_hostname: "registry.terraform.io",
+                module_identifier: "Mongey/confluentcloud"
+              }
+            }],
+            package_manager: "terraform"
+          )
+        ]
+      end
+
+      it "updates the module version across all nested providers" do
+        updated_files = subject
+        lockfile = updated_files.find { |file| file.name == ".terraform.lock.hcl" }
+        provider_files = updated_files.select { |file| file.name.end_with?(".tf") }
+
+        expect(provider_files.count).to eq(3)
+        provider_files.each do |file|
+          expect(file.content).to include("version = \"0.0.10\"")
+        end
+        expect(lockfile.content).to include(
+          <<~DEP
+            provider "registry.terraform.io/mongey/confluentcloud" {
+              version     = "0.0.10"
+          DEP
+        )
+      end
+    end
   end
 end

terraform/spec/fixtures/projects/provider_with_multiple_local_path_modules/loader.tf

@@ -0,0 +1,3 @@
+module "loader" {
+  source = "./loader"
+}