cmd/go: don't fetch from insecure repositories without -insecure

Fixes golang#9637
Fixes golang#10120

Change-Id: I3728239089efb94d04cd4115c9f840afd7badeaf
Reviewed-on: https://go-review.googlesource.com/9715
Reviewed-by: Brad Fitzpatrick <[email protected]>
Reviewed-by: Russ Cox <[email protected]>

src/cmd/go/alldocs.go

@@ -424,7 +424,7 @@ Download and install packages and dependencies
 
 Usage:
 
-	go get [-d] [-f] [-fix] [-t] [-u] [build flags] [packages]
+	go get [-d] [-f] [-fix] [-insecure] [-t] [-u] [build flags] [packages]
 
 Get downloads and installs the packages named by the import paths,
 along with their dependencies.
@@ -440,6 +440,9 @@ of the original.
 The -fix flag instructs get to run the fix tool on the downloaded packages
 before resolving dependencies or building the code.
 
+The -insecure flag permits fetching from repositories and resolving
+custom domains using insecure schemes such as HTTP. Use with caution.
+
 The -t flag instructs get to also download the packages required to build
 the tests for the specified packages.
 

src/cmd/go/bootstrap.go

@@ -29,7 +29,7 @@ func httpGET(url string) ([]byte, error) {
 	return nil, errHTTP
 }
 
-func httpsOrHTTP(importPath string) (string, io.ReadCloser, error) {
+func httpsOrHTTP(importPath string, security securityMode) (string, io.ReadCloser, error) {
 	return "", nil, errHTTP
 }
 

src/cmd/go/get.go

@@ -16,7 +16,7 @@ import (
 )
 
 var cmdGet = &Command{
-	UsageLine: "get [-d] [-f] [-fix] [-t] [-u] [build flags] [packages]",
+	UsageLine: "get [-d] [-f] [-fix] [-insecure] [-t] [-u] [build flags] [packages]",
 	Short:     "download and install packages and dependencies",
 	Long: `
 Get downloads and installs the packages named by the import paths,
@@ -33,6 +33,9 @@ of the original.
 The -fix flag instructs get to run the fix tool on the downloaded packages
 before resolving dependencies or building the code.
 
+The -insecure flag permits fetching from repositories and resolving
+custom domains using insecure schemes such as HTTP. Use with caution.
+
 The -t flag instructs get to also download the packages required to build
 the tests for the specified packages.
 
@@ -62,6 +65,7 @@ var getF = cmdGet.Flag.Bool("f", false, "")
 var getT = cmdGet.Flag.Bool("t", false, "")
 var getU = cmdGet.Flag.Bool("u", false, "")
 var getFix = cmdGet.Flag.Bool("fix", false, "")
+var getInsecure = cmdGet.Flag.Bool("insecure", false, "")
 
 func init() {
 	addBuildFlags(cmdGet)
@@ -279,6 +283,12 @@ func downloadPackage(p *Package) error {
 		repo, rootPath string
 		err            error
 	)
+
+	security := secure
+	if *getInsecure {
+		security = insecure
+	}
+
 	if p.build.SrcRoot != "" {
 		// Directory exists.  Look for checkout along path to src.
 		vcs, rootPath, err = vcsForDir(p)
@@ -288,32 +298,39 @@ func downloadPackage(p *Package) error {
 		repo = "<local>" // should be unused; make distinctive
 
 		// Double-check where it came from.
-		if *getU && vcs.remoteRepo != nil && !*getF {
+		if *getU && vcs.remoteRepo != nil {
 			dir := filepath.Join(p.build.SrcRoot, rootPath)
 			if remote, err := vcs.remoteRepo(vcs, dir); err == nil {
-				if rr, err := repoRootForImportPath(p.ImportPath); err == nil {
-					repo := rr.repo
-					if rr.vcs.resolveRepo != nil {
-						resolved, err := rr.vcs.resolveRepo(rr.vcs, dir, repo)
-						if err == nil {
-							repo = resolved
+				repo = remote
+
+				if !*getF {
+					if rr, err := repoRootForImportPath(p.ImportPath, security); err == nil {
+						repo := rr.repo
+						if rr.vcs.resolveRepo != nil {
+							resolved, err := rr.vcs.resolveRepo(rr.vcs, dir, repo)
+							if err == nil {
+								repo = resolved
+							}
+						}
+						if remote != repo {
+							return fmt.Errorf("%s is a custom import path for %s, but %s is checked out from %s", rr.root, repo, dir, remote)
 						}
-					}
-					if remote != repo {
-						return fmt.Errorf("%s is a custom import path for %s, but %s is checked out from %s", rr.root, repo, dir, remote)
 					}
 				}
 			}
 		}
 	} else {
 		// Analyze the import path to determine the version control system,
 		// repository, and the import path for the root of the repository.
-		rr, err := repoRootForImportPath(p.ImportPath)
+		rr, err := repoRootForImportPath(p.ImportPath, security)
 		if err != nil {
 			return err
 		}
 		vcs, repo, rootPath = rr.vcs, rr.repo, rr.root
 	}
+	if !vcs.isSecure(repo) && !*getInsecure {
+		return fmt.Errorf("cannot download, %v uses insecure protocol", repo)
+	}
 
 	if p.build.SrcRoot == "" {
 		// Package not found.  Put in first directory of $GOPATH.

src/cmd/go/go_test.go

@@ -530,6 +530,17 @@ func (tg *testgoData) cleanup() {
 	}
 }
 
+// failSSH puts an ssh executable in the PATH that always fails.
+// This is to stub out uses of ssh by go get.
+func (tg *testgoData) failSSH() {
+	wd, err := os.Getwd()
+	if err != nil {
+		tg.t.Fatal(err)
+	}
+	fail := filepath.Join(wd, "testdata/failssh")
+	tg.setenv("PATH", fmt.Sprintf("%v%c%v", fail, filepath.ListSeparator, os.Getenv("PATH")))
+}
+
 func TestFileLineInErrorMessages(t *testing.T) {
 	tg := testgo(t)
 	defer tg.cleanup()
@@ -1848,3 +1859,61 @@ func TestIssue4210(t *testing.T) {
 	tg.runFail("build", "y")
 	tg.grepBoth("is a program", `did not find expected error message ("is a program")`)
 }
+
+func TestGoGetInsecure(t *testing.T) {
+	testenv.MustHaveExternalNetwork(t)
+
+	tg := testgo(t)
+	defer tg.cleanup()
+	tg.makeTempdir()
+	tg.setenv("GOPATH", tg.path("."))
+	tg.failSSH()
+
+	const repo = "wh3rd.net/git.git"
+
+	// Try go get -d of HTTP-only repo (should fail).
+	tg.runFail("get", "-d", repo)
+
+	// Try again with -insecure (should succeed).
+	tg.run("get", "-d", "-insecure", repo)
+
+	// Try updating without -insecure (should fail).
+	tg.runFail("get", "-d", "-u", "-f", repo)
+}
+
+func TestGoGetUpdateInsecure(t *testing.T) {
+	testenv.MustHaveExternalNetwork(t)
+
+	tg := testgo(t)
+	defer tg.cleanup()
+	tg.makeTempdir()
+	tg.setenv("GOPATH", tg.path("."))
+
+	const repo = "github.com/golang/example"
+
+	// Clone the repo via HTTP manually.
+	cmd := exec.Command("git", "clone", "-q", "http://"+repo, tg.path("src/"+repo))
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatalf("cloning %v repo: %v\n%s", repo, err, out)
+	}
+
+	// Update without -insecure should fail.
+	// Update with -insecure should succeed.
+	// We need -f to ignore import comments.
+	const pkg = repo + "/hello"
+	tg.runFail("get", "-d", "-u", "-f", pkg)
+	tg.run("get", "-d", "-u", "-f", "-insecure", pkg)
+}
+
+func TestGoGetInsecureCustomDomain(t *testing.T) {
+	testenv.MustHaveExternalNetwork(t)
+
+	tg := testgo(t)
+	defer tg.cleanup()
+	tg.makeTempdir()
+	tg.setenv("GOPATH", tg.path("."))
+
+	const repo = "wh3rd.net/repo"
+	tg.runFail("get", "-d", repo)
+	tg.run("get", "-d", "-insecure", repo)
+}

src/cmd/go/http.go

@@ -18,11 +18,15 @@ import (
 	"log"
 	"net/http"
 	"net/url"
+	"time"
 )
 
 // httpClient is the default HTTP client, but a variable so it can be
 // changed by tests, without modifying http.DefaultClient.
 var httpClient = http.DefaultClient
+var impatientHTTPClient = &http.Client{
+	Timeout: time.Duration(5 * time.Second),
+}
 
 type httpError struct {
 	status     string
@@ -55,7 +59,7 @@ func httpGET(url string) ([]byte, error) {
 
 // httpsOrHTTP returns the body of either the importPath's
 // https resource or, if unavailable, the http resource.
-func httpsOrHTTP(importPath string) (urlStr string, body io.ReadCloser, err error) {
+func httpsOrHTTP(importPath string, security securityMode) (urlStr string, body io.ReadCloser, err error) {
 	fetch := func(scheme string) (urlStr string, res *http.Response, err error) {
 		u, err := url.Parse(scheme + "://" + importPath)
 		if err != nil {
@@ -66,7 +70,11 @@ func httpsOrHTTP(importPath string) (urlStr string, body io.ReadCloser, err erro
 		if buildV {
 			log.Printf("Fetching %s", urlStr)
 		}
-		res, err = httpClient.Get(urlStr)
+		if security == insecure && scheme == "https" { // fail earlier
+			res, err = impatientHTTPClient.Get(urlStr)
+		} else {
+			res, err = httpClient.Get(urlStr)
+		}
 		return
 	}
 	closeBody := func(res *http.Response) {
@@ -84,7 +92,9 @@ func httpsOrHTTP(importPath string) (urlStr string, body io.ReadCloser, err erro
 			}
 		}
 		closeBody(res)
-		urlStr, res, err = fetch("http")
+		if security == insecure {
+			urlStr, res, err = fetch("http")
+		}
 	}
 	if err != nil {
 		closeBody(res)

src/cmd/go/testdata/failssh/ssh

@@ -0,0 +1,2 @@
+#!/bin/sh
+exit 1