add git submodule support (argoproj#2495)

common/common.go

@@ -135,6 +135,8 @@ const (
 	EnvVarTLSDataPath = "ARGOCD_TLS_DATA_PATH"
 	// Specifies number of git remote operations attempts count
 	EnvGitAttemptsCount = "ARGOCD_GIT_ATTEMPTS_COUNT"
+	// Overrides git submodule support, true by default
+	EnvGitSubmoduleEnabled = "ARGOCD_GIT_MODULES_ENABLED"
 )
 
 const (

docs/user-guide/private-repositories.md

@@ -344,6 +344,10 @@ argocd repo add [email protected]:argoproj/argocd-example-apps.git --ssh-private-ke
 !!! warning "This does not work for Kustomize remote bases or custom plugins"
     For Kustomize support, see [#827](https://github.com/argoproj/argo-cd/issues/827).
 
+## Git Submodules
+
+Submodules are supported and will be picked up automatically. If the submodule repository requires authentication then the credentials will need to match the credentials of the parent repository. Set ARGOCD_GIT_MODULES_ENABLED=false to disable submodule support
+
 ## Declarative Configuration
 
 See [declarative setup](../../operator-manual/declarative-setup#Repositories)

test/e2e/fixture/app/actions.go

@@ -91,6 +91,10 @@ func (a *Actions) CreateFromFile(handler func(app *Application)) *Actions {
 		logrus.Fatal("Application parameters or json tlas are not supported")
 	}
 
+	if a.context.directoryRecurse {
+		app.Spec.Source.Directory = &ApplicationSourceDirectory{Recurse: true}
+	}
+
 	handler(app)
 	data := json.MustMarshal(app)
 	tmpFile, err := ioutil.TempFile("", "")

test/e2e/fixture/app/context.go

@@ -36,6 +36,7 @@ type Context struct {
 	project                string
 	revision               string
 	force                  bool
+	directoryRecurse       bool
 }
 
 func Given(t *testing.T) *Context {
@@ -54,12 +55,12 @@ func (c *Context) CustomSSHKnownHostsAdded() *Context {
 }
 
 func (c *Context) HTTPSRepoURLAdded(withCreds bool) *Context {
-	repos.AddHTTPSRepo(false, withCreds)
+	repos.AddHTTPSRepo(false, withCreds, fixture.RepoURLTypeHTTPS)
 	return c
 }
 
 func (c *Context) HTTPSInsecureRepoURLAdded(withCreds bool) *Context {
-	repos.AddHTTPSRepo(true, withCreds)
+	repos.AddHTTPSRepo(true, withCreds, fixture.RepoURLTypeHTTPS)
 	return c
 }
 
@@ -73,13 +74,25 @@ func (c *Context) HTTPSRepoURLWithClientCertAdded() *Context {
 	return c
 }
 
+func (c *Context) SubmoduleHTTPSRepoURLAdded(withCreds bool) *Context {
+	fixture.CreateSubmoduleRepos("https")
+	repos.AddHTTPSRepo(false, withCreds, fixture.RepoURLTypeHTTPSSubmoduleParent)
+	return c
+}
+
 func (c *Context) SSHRepoURLAdded(withCreds bool) *Context {
-	repos.AddSSHRepo(false, withCreds)
+	repos.AddSSHRepo(false, withCreds, fixture.RepoURLTypeSSH)
 	return c
 }
 
 func (c *Context) SSHInsecureRepoURLAdded(withCreds bool) *Context {
-	repos.AddSSHRepo(true, withCreds)
+	repos.AddSSHRepo(true, withCreds, fixture.RepoURLTypeSSH)
+	return c
+}
+
+func (c *Context) SubmoduleSSHRepoURLAdded(withCreds bool) *Context {
+	fixture.CreateSubmoduleRepos("ssh")
+	repos.AddSSHRepo(false, withCreds, fixture.RepoURLTypeSSHSubmoduleParent)
 	return c
 }
 
@@ -123,6 +136,11 @@ func (c *Context) Path(path string) *Context {
 	return c
 }
 
+func (c *Context) Recurse() *Context {
+	c.directoryRecurse = true
+	return c
+}
+
 func (c *Context) Chart(chart string) *Context {
 	c.chart = chart
 	return c

test/e2e/fixture/fixture.go

@@ -40,8 +40,10 @@ const (
 	ArgoCDNamespace  = "argocd-e2e"
 
 	// ensure all repos are in one directory tree, so we can easily clean them up
-	TmpDir  = "/tmp/argo-e2e"
-	repoDir = "testdata.git"
+	TmpDir             = "/tmp/argo-e2e"
+	repoDir            = "testdata.git"
+	submoduleDir       = "submodule.git"
+	submoduleParentDir = "submoduleParent.git"
 
 	GuestbookPath = "guestbook"
 )
@@ -62,13 +64,17 @@ var (
 type RepoURLType string
 
 const (
-	RepoURLTypeFile            = "file"
-	RepoURLTypeHTTPS           = "https"
-	RepoURLTypeHTTPSClientCert = "https-cc"
-	RepoURLTypeSSH             = "ssh"
-	RepoURLTypeHelm            = "helm"
-	GitUsername                = "admin"
-	GitPassword                = "password"
+	RepoURLTypeFile                 = "file"
+	RepoURLTypeHTTPS                = "https"
+	RepoURLTypeHTTPSClientCert      = "https-cc"
+	RepoURLTypeHTTPSSubmodule       = "https-sub"
+	RepoURLTypeHTTPSSubmoduleParent = "https-par"
+	RepoURLTypeSSH                  = "ssh"
+	RepoURLTypeSSHSubmodule         = "ssh-sub"
+	RepoURLTypeSSHSubmoduleParent   = "ssh-par"
+	RepoURLTypeHelm                 = "helm"
+	GitUsername                     = "admin"
+	GitPassword                     = "password"
 )
 
 // getKubeConfig creates new kubernetes client config using specified config path and config overrides variables
@@ -132,17 +138,36 @@ func repoDirectory() string {
 	return path.Join(TmpDir, repoDir)
 }
 
+func submoduleDirectory() string {
+	return path.Join(TmpDir, submoduleDir)
+}
+
+func submoduleParentDirectory() string {
+	return path.Join(TmpDir, submoduleParentDir)
+}
+
 func RepoURL(urlType RepoURLType) string {
 	switch urlType {
 	// Git server via SSH
 	case RepoURLTypeSSH:
 		return "ssh://root@localhost:2222/tmp/argo-e2e/testdata.git"
+	// Git submodule repo
+	case RepoURLTypeSSHSubmodule:
+		return "ssh://root@localhost:2222/tmp/argo-e2e/submodule.git"
+		// Git submodule parent repo
+	case RepoURLTypeSSHSubmoduleParent:
+		return "ssh://root@localhost:2222/tmp/argo-e2e/submoduleParent.git"
 	// Git server via HTTPS
 	case RepoURLTypeHTTPS:
 		return "https://localhost:9443/argo-e2e/testdata.git"
 	// Git server via HTTPS - Client Cert protected
 	case RepoURLTypeHTTPSClientCert:
 		return "https://localhost:9444/argo-e2e/testdata.git"
+	case RepoURLTypeHTTPSSubmodule:
+		return "https://localhost:9443/argo-e2e/submodule.git"
+		// Git submodule parent repo
+	case RepoURLTypeHTTPSSubmoduleParent:
+		return "https://localhost:9443/argo-e2e/submoduleParent.git"
 	// Default - file based Git repository
 	case RepoURLTypeHelm:
 		return "https://localhost:9444/argo-e2e/testdata.git/helm-repo"
@@ -471,3 +496,27 @@ func Declarative(filename string, values interface{}) (string, error) {
 
 	return Run("", "kubectl", "-n", ArgoCDNamespace, "apply", "-f", tmpFile.Name())
 }
+
+func CreateSubmoduleRepos(repoType string) {
+
+	// set-up submodule repo
+	FailOnErr(Run("", "cp", "-Rf", "testdata/git-submodule/", submoduleDirectory()))
+	FailOnErr(Run(submoduleDirectory(), "chmod", "777", "."))
+	FailOnErr(Run(submoduleDirectory(), "git", "init"))
+	FailOnErr(Run(submoduleDirectory(), "git", "add", "."))
+	FailOnErr(Run(submoduleDirectory(), "git", "commit", "-q", "-m", "initial commit"))
+
+	// set-up submodule parent repo
+	FailOnErr(Run("", "mkdir", submoduleParentDirectory()))
+	FailOnErr(Run(submoduleParentDirectory(), "chmod", "777", "."))
+	FailOnErr(Run(submoduleParentDirectory(), "git", "init"))
+	FailOnErr(Run(submoduleParentDirectory(), "git", "add", "."))
+	FailOnErr(Run(submoduleParentDirectory(), "git", "submodule", "add", "-b", "master", "../submodule.git", "submodule/test"))
+	if repoType == "ssh" {
+		FailOnErr(Run(submoduleParentDirectory(), "git", "config", "--file=.gitmodules", "submodule.submodule/test.url", RepoURL(RepoURLTypeSSHSubmodule)))
+	} else if repoType == "https" {
+		FailOnErr(Run(submoduleParentDirectory(), "git", "config", "--file=.gitmodules", "submodule.submodule/test.url", RepoURL(RepoURLTypeHTTPSSubmodule)))
+	}
+	FailOnErr(Run(submoduleParentDirectory(), "git", "add", "--all"))
+	FailOnErr(Run(submoduleParentDirectory(), "git", "commit", "-q", "-m", "commit with submodule"))
+}

test/e2e/fixture/repos/repos.go

@@ -19,10 +19,10 @@ func mustToAbsPath(relativePath string) string {
 }
 
 // sets the current repo as the default SSH test repo
-func AddSSHRepo(insecure bool, credentials bool) {
+func AddSSHRepo(insecure bool, credentials bool, repoURLType fixture.RepoURLType) {
 	keyPath, err := filepath.Abs("../fixture/testrepos/id_rsa")
 	errors.CheckError(err)
-	args := []string{"repo", "add", fixture.RepoURL(fixture.RepoURLTypeSSH)}
+	args := []string{"repo", "add", fixture.RepoURL(repoURLType)}
 	if credentials {
 		args = append(args, "--ssh-private-key-path", keyPath)
 	}
@@ -33,9 +33,8 @@ func AddSSHRepo(insecure bool, credentials bool) {
 }
 
 // sets the current repo as the default HTTPS test repo
-func AddHTTPSRepo(insecure bool, credentials bool) {
+func AddHTTPSRepo(insecure bool, credentials bool, repoURLType fixture.RepoURLType) {
 	// This construct is somewhat necessary to satisfy the compiler
-	var repoURLType fixture.RepoURLType = fixture.RepoURLTypeHTTPS
 	args := []string{"repo", "add", fixture.RepoURL(repoURLType)}
 	if credentials {
 		args = append(args, "--username", fixture.GitUsername, "--password", fixture.GitPassword)

test/e2e/git_submodule_test.go

@@ -0,0 +1,42 @@
+package e2e
+
+import (
+	"testing"
+
+	v1 "k8s.io/api/core/v1"
+
+	"github.com/argoproj/argo-cd/test/e2e/fixture"
+
+	. "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
+	. "github.com/argoproj/argo-cd/test/e2e/fixture/app"
+)
+
+func TestGitSubmoduleSSHSupport(t *testing.T) {
+	Given(t).
+		RepoURLType(fixture.RepoURLTypeSSHSubmoduleParent).
+		Path("submodule").
+		Recurse().
+		CustomSSHKnownHostsAdded().
+		SubmoduleSSHRepoURLAdded(true).
+		When().
+		CreateFromFile(func(app *Application) {}).
+		Sync().
+		Then().
+		Expect(SyncStatusIs(SyncStatusCodeSynced)).
+		Expect(Pod(func(p v1.Pod) bool { return p.Name == "pod-in-submodule" }))
+}
+
+func TestGitSubmoduleHTTPSSupport(t *testing.T) {
+	Given(t).
+		RepoURLType(fixture.RepoURLTypeHTTPSSubmoduleParent).
+		Path("submodule").
+		Recurse().
+		CustomCACertAdded().
+		SubmoduleHTTPSRepoURLAdded(true).
+		When().
+		CreateFromFile(func(app *Application) {}).
+		Sync().
+		Then().
+		Expect(SyncStatusIs(SyncStatusCodeSynced)).
+		Expect(Pod(func(p v1.Pod) bool { return p.Name == "pod-in-submodule" }))
+}

test/e2e/testdata/git-submodule/submodule-pod.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-in-submodule
+spec:
+  containers:
+    - name: main
+      image: alpine:3.10.2
+      imagePullPolicy: IfNotPresent
+      command:
+        - "true"
+  restartPolicy: Never

util/git/client.go

@@ -303,6 +303,13 @@ func (m *nativeGitClient) Checkout(revision string) error {
 			return err
 		}
 	}
+	if _, err := os.Stat(m.root + "/.gitmodules"); !os.IsNotExist(err) {
+		if submoduleEnabled := os.Getenv(common.EnvGitSubmoduleEnabled); submoduleEnabled != "false" {
+			if err := m.runCredentialedCmd("git", "submodule", "update", "--init", "--recursive"); err != nil {
+				return err
+			}
+		}
+	}
 	if _, err := m.runCmd("clean", "-fdx"); err != nil {
 		return err
 	}