cmd/go: merge module support from x/vgo repo

This CL corresponds to CL 123361, the final manual CL in that repo,
making this the final manual sync.

All future commits will happen in this repo (the main Go repo),
and we'll update x/vgo automatically with a fixed patch+script.

Change-Id: I572243309c1809727604fd704705a23c30e85d1a
Reviewed-on: https://go-review.googlesource.com/123576
Run-TryBot: Russ Cox <[email protected]>
Reviewed-by: Bryan C. Mills <[email protected]>

src/cmd/go/go_test.go

@@ -8,6 +8,7 @@ import (
 	"bytes"
 	"debug/elf"
 	"debug/macho"
+	"flag"
 	"fmt"
 	"go/format"
 	"internal/race"
@@ -112,6 +113,12 @@ func TestMain(m *testing.M) {
 	}
 	os.Unsetenv("GOROOT_FINAL")
 
+	flag.Parse()
+	if *proxyAddr != "" {
+		StartProxy()
+		select {}
+	}
+
 	if canRun {
 		args := []string{"build", "-tags", "testgo", "-o", "testgo" + exeSuffix}
 		if race.Enabled {
@@ -417,7 +424,8 @@ func (tg *testgoData) doRun(args []string) error {
 func (tg *testgoData) run(args ...string) {
 	tg.t.Helper()
 	if status := tg.doRun(args); status != nil {
-		tg.t.Logf("go %v failed unexpectedly: %v", args, status)
+		wd, _ := os.Getwd()
+		tg.t.Logf("go %v failed unexpectedly in %s: %v", args, wd, status)
 		tg.t.FailNow()
 	}
 }
@@ -760,24 +768,51 @@ func (tg *testgoData) wantNotStale(pkg, reason, msg string) {
 	}
 }
 
+// If -testwork is specified, the test prints the name of the temp directory
+// and does not remove it when done, so that a programmer can
+// poke at the test file tree afterward.
+var testWork = flag.Bool("testwork", false, "")
+
 // cleanup cleans up a test that runs testgo.
 func (tg *testgoData) cleanup() {
 	tg.t.Helper()
 	if tg.wd != "" {
+		wd, _ := os.Getwd()
+		tg.t.Logf("ended in %s", wd)
+
 		if err := os.Chdir(tg.wd); err != nil {
 			// We are unlikely to be able to continue.
 			fmt.Fprintln(os.Stderr, "could not restore working directory, crashing:", err)
 			os.Exit(2)
 		}
 	}
+	if *testWork {
+		tg.t.Logf("TESTWORK=%s\n", tg.path("."))
+		return
+	}
 	for _, path := range tg.temps {
-		tg.check(os.RemoveAll(path))
+		tg.check(removeAll(path))
 	}
 	if tg.tempdir != "" {
-		tg.check(os.RemoveAll(tg.tempdir))
+		tg.check(removeAll(tg.tempdir))
 	}
 }
 
+func removeAll(dir string) error {
+	// module cache has 0444 directories;
+	// make them writable in order to remove content.
+	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return nil // ignore errors walking in file system
+		}
+		if info.IsDir() {
+			os.Chmod(path, 0777)
+		}
+		return nil
+	})
+	return os.RemoveAll(dir)
+}
+
 // failSSH puts an ssh executable in the PATH that always fails.
 // This is to stub out uses of ssh by go get.
 func (tg *testgoData) failSSH() {
@@ -1745,8 +1780,8 @@ func TestGoGetTestOnlyPkg(t *testing.T) {
 	defer tg.cleanup()
 	tg.tempDir("gopath")
 	tg.setenv("GOPATH", tg.path("gopath"))
-	tg.run("get", "golang.org/x/tour/content")
-	tg.run("get", "-t", "golang.org/x/tour/content")
+	tg.run("get", "golang.org/x/tour/content...")
+	tg.run("get", "-t", "golang.org/x/tour/content...")
 }
 
 func TestInstalls(t *testing.T) {
@@ -3165,11 +3200,25 @@ func TestBuildDashIInstallsDependencies(t *testing.T) {
 	checkbar("cmd")
 }
 
-func TestGoBuildInTestOnlyDirectoryFailsWithAGoodError(t *testing.T) {
+func TestGoBuildTestOnly(t *testing.T) {
 	tg := testgo(t)
 	defer tg.cleanup()
-	tg.runFail("build", "./testdata/testonly")
-	tg.grepStderr("no non-test Go files in", "go build ./testdata/testonly produced unexpected error")
+	tg.makeTempdir()
+	tg.setenv("GOPATH", tg.path("."))
+	tg.tempFile("src/testonly/t_test.go", `package testonly`)
+	tg.tempFile("src/testonly2/t.go", `package testonly2`)
+	tg.cd(tg.path("src"))
+
+	// Named explicitly, test-only packages should be reported as unbuildable/uninstallable,
+	// even if there is a wildcard also matching.
+	tg.runFail("build", "testonly", "testonly...")
+	tg.grepStderr("no non-test Go files in", "go build ./xtestonly produced unexpected error")
+	tg.runFail("install", "./testonly")
+	tg.grepStderr("no non-test Go files in", "go install ./testonly produced unexpected error")
+
+	// Named through a wildcards, the test-only packages should be silently ignored.
+	tg.run("build", "testonly...")
+	tg.run("install", "./testonly...")
 }
 
 func TestGoTestDetectsTestOnlyImportCycles(t *testing.T) {
@@ -6115,12 +6164,12 @@ func TestBadCommandLines(t *testing.T) {
 	tg.tempFile("src/@x/x.go", "package x\n")
 	tg.setenv("GOPATH", tg.path("."))
 	tg.runFail("build", "@x")
-	tg.grepStderr("invalid input directory name \"@x\"", "did not reject @x directory")
+	tg.grepStderr("invalid input directory name \"@x\"|cannot use path@version syntax", "did not reject @x directory")
 
 	tg.tempFile("src/@x/y/y.go", "package y\n")
 	tg.setenv("GOPATH", tg.path("."))
 	tg.runFail("build", "@x/y")
-	tg.grepStderr("invalid import path \"@x/y\"", "did not reject @x/y import path")
+	tg.grepStderr("invalid import path \"@x/y\"|cannot use path@version syntax", "did not reject @x/y import path")
 
 	tg.tempFile("src/-x/x.go", "package x\n")
 	tg.setenv("GOPATH", tg.path("."))

src/cmd/go/internal/cfg/cfg.go

@@ -21,6 +21,7 @@ var (
 	BuildA                 bool   // -a flag
 	BuildBuildmode         string // -buildmode flag
 	BuildContext           = build.Default
+	BuildGetmode           string             // -getmode flag
 	BuildI                 bool               // -i flag
 	BuildLinkshared        bool               // -linkshared flag
 	BuildMSan              bool               // -msan flag
@@ -67,6 +68,11 @@ var (
 	Goos      = BuildContext.GOOS
 	ExeSuffix string
 	Gopath    = filepath.SplitList(BuildContext.GOPATH)
+
+	// ModulesEnabled specifies whether the go command is running
+	// in module-aware mode (as opposed to GOPATH mode).
+	// It is equal to modload.Enabled, but not all packages can import modload.
+	ModulesEnabled bool
 )
 
 func init() {

src/cmd/go/internal/envcmd/env.go

@@ -9,14 +9,15 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"path/filepath"
 	"runtime"
 	"strings"
 
 	"cmd/go/internal/base"
 	"cmd/go/internal/cache"
 	"cmd/go/internal/cfg"
 	"cmd/go/internal/load"
-	"cmd/go/internal/vgo"
+	"cmd/go/internal/modload"
 	"cmd/go/internal/work"
 )
 
@@ -112,6 +113,18 @@ func findEnv(env []cfg.EnvVar, name string) string {
 
 // ExtraEnvVars returns environment variables that should not leak into child processes.
 func ExtraEnvVars() []cfg.EnvVar {
+	gomod := ""
+	if modload.Init(); modload.ModRoot != "" {
+		gomod = filepath.Join(modload.ModRoot, "go.mod")
+	}
+	return []cfg.EnvVar{
+		{Name: "GOMOD", Value: gomod},
+	}
+}
+
+// ExtraEnvVarsCostly returns environment variables that should not leak into child processes
+// but are costly to evaluate.
+func ExtraEnvVarsCostly() []cfg.EnvVar {
 	var b work.Builder
 	b.Init()
 	cppflags, cflags, cxxflags, fflags, ldflags, err := b.CFlags(&load.Package{})
@@ -121,6 +134,7 @@ func ExtraEnvVars() []cfg.EnvVar {
 		return nil
 	}
 	cmd := b.GccCmd(".", "")
+
 	return []cfg.EnvVar{
 		// Note: Update the switch in runEnv below when adding to this list.
 		{Name: "CGO_CFLAGS", Value: strings.Join(cflags, " ")},
@@ -130,19 +144,19 @@ func ExtraEnvVars() []cfg.EnvVar {
 		{Name: "CGO_LDFLAGS", Value: strings.Join(ldflags, " ")},
 		{Name: "PKG_CONFIG", Value: b.PkgconfigCmd()},
 		{Name: "GOGCCFLAGS", Value: strings.Join(cmd[3:], " ")},
-		{Name: "VGOMODROOT", Value: vgo.ModRoot},
 	}
 }
 
 func runEnv(cmd *base.Command, args []string) {
 	env := cfg.CmdEnv
+	env = append(env, ExtraEnvVars()...)
 
-	// Do we need to call ExtraEnvVars, which is a bit expensive?
+	// Do we need to call ExtraEnvVarsCostly, which is a bit expensive?
 	// Only if we're listing all environment variables ("go env")
 	// or the variables being requested are in the extra list.
-	needExtra := true
+	needCostly := true
 	if len(args) > 0 {
-		needExtra = false
+		needCostly = false
 		for _, arg := range args {
 			switch arg {
 			case "CGO_CFLAGS",
@@ -152,12 +166,12 @@ func runEnv(cmd *base.Command, args []string) {
 				"CGO_LDFLAGS",
 				"PKG_CONFIG",
 				"GOGCCFLAGS":
-				needExtra = true
+				needCostly = true
 			}
 		}
 	}
-	if needExtra {
-		env = append(env, ExtraEnvVars()...)
+	if needCostly {
+		env = append(env, ExtraEnvVarsCostly()...)
 	}
 
 	if len(args) > 0 {

src/cmd/go/internal/fix/fix.go

@@ -9,8 +9,8 @@ import (
 	"cmd/go/internal/base"
 	"cmd/go/internal/cfg"
 	"cmd/go/internal/load"
+	"cmd/go/internal/modload"
 	"cmd/go/internal/str"
-	"cmd/go/internal/vgo"
 	"fmt"
 	"os"
 )
@@ -34,9 +34,9 @@ See also: go fmt, go vet.
 func runFix(cmd *base.Command, args []string) {
 	printed := false
 	for _, pkg := range load.Packages(args) {
-		if vgo.Enabled() && !pkg.Module.Top {
+		if modload.Enabled() && !pkg.Module.Main {
 			if !printed {
-				fmt.Fprintf(os.Stderr, "vgo: not fixing packages in dependency modules\n")
+				fmt.Fprintf(os.Stderr, "go: not fixing packages in dependency modules\n")
 				printed = true
 			}
 			continue

src/cmd/go/internal/fmtcmd/fmt.go

@@ -16,8 +16,8 @@ import (
 	"cmd/go/internal/base"
 	"cmd/go/internal/cfg"
 	"cmd/go/internal/load"
+	"cmd/go/internal/modload"
 	"cmd/go/internal/str"
-	"cmd/go/internal/vgo"
 )
 
 func init() {
@@ -60,9 +60,9 @@ func runFmt(cmd *base.Command, args []string) {
 		}()
 	}
 	for _, pkg := range load.PackagesAndErrors(args) {
-		if vgo.Enabled() && !pkg.Module.Top {
+		if modload.Enabled() && !pkg.Module.Main {
 			if !printed {
-				fmt.Fprintf(os.Stderr, "vgo: not formatting packages in dependency modules\n")
+				fmt.Fprintf(os.Stderr, "go: not formatting packages in dependency modules\n")
 				printed = true
 			}
 			continue

src/cmd/go/internal/generate/generate.go

@@ -21,7 +21,7 @@ import (
 	"cmd/go/internal/base"
 	"cmd/go/internal/cfg"
 	"cmd/go/internal/load"
-	"cmd/go/internal/vgo"
+	"cmd/go/internal/modload"
 	"cmd/go/internal/work"
 )
 
@@ -161,9 +161,9 @@ func runGenerate(cmd *base.Command, args []string) {
 	// Even if the arguments are .go files, this loop suffices.
 	printed := false
 	for _, pkg := range load.Packages(args) {
-		if vgo.Enabled() && !pkg.Module.Top {
+		if modload.Enabled() && !pkg.Module.Main {
 			if !printed {
-				fmt.Fprintf(os.Stderr, "vgo: not generating in packages in dependency modules\n")
+				fmt.Fprintf(os.Stderr, "go: not generating in packages in dependency modules\n")
 				printed = true
 			}
 			continue

src/cmd/go/internal/get/discovery.go

@@ -28,7 +28,7 @@ func charsetReader(charset string, input io.Reader) (io.Reader, error) {
 
 // parseMetaGoImports returns meta imports from the HTML in r.
 // Parsing ends at the end of the <head> section or the beginning of the <body>.
-func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) {
+func parseMetaGoImports(r io.Reader, mod ModuleMode) (imports []metaImport, err error) {
 	d := xml.NewDecoder(r)
 	d.CharsetReader = charsetReader
 	d.Strict = false
@@ -39,13 +39,13 @@ func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) {
 			if err == io.EOF || len(imports) > 0 {
 				err = nil
 			}
-			return
+			break
 		}
 		if e, ok := t.(xml.StartElement); ok && strings.EqualFold(e.Name.Local, "body") {
-			return
+			break
 		}
 		if e, ok := t.(xml.EndElement); ok && strings.EqualFold(e.Name.Local, "head") {
-			return
+			break
 		}
 		e, ok := t.(xml.StartElement)
 		if !ok || !strings.EqualFold(e.Name.Local, "meta") {
@@ -55,20 +55,34 @@ func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) {
 			continue
 		}
 		if f := strings.Fields(attrValue(e.Attr, "content")); len(f) == 3 {
-			// Ignore VCS type "mod", which is new Go modules.
-			// This code is for old go get and must ignore the new mod lines.
-			// Otherwise matchGoImport will complain about two
-			// different metaImport lines for the same Prefix.
-			if f[1] == "mod" {
-				continue
-			}
 			imports = append(imports, metaImport{
 				Prefix:   f[0],
 				VCS:      f[1],
 				RepoRoot: f[2],
 			})
 		}
 	}
+
+	// Extract mod entries if we are paying attention to them.
+	var list []metaImport
+	var have map[string]bool
+	if mod == PreferMod {
+		have = make(map[string]bool)
+		for _, m := range imports {
+			if m.VCS == "mod" {
+				have[m.Prefix] = true
+				list = append(list, m)
+			}
+		}
+	}
+
+	// Append non-mod entries, ignoring those superseded by a mod entry.
+	for _, m := range imports {
+		if m.VCS != "mod" && !have[m.Prefix] {
+			list = append(list, m)
+		}
+	}
+	return list, nil
 }
 
 // attrValue returns the attribute value for the case-insensitive key