travis, build, internal: use own Go bundle for PPA builds (ethereum#2…

…0240)

* build: bump PPAs to Go 1.13 (via longsleep), keep Trusty on 1.11

* travis, build, vendor: use own Go bundle for PPA builds

* travis, build, internal, vendor: smarter Go bundler, own untar

* build: updated ci-notes with new Go bundling, only make, don't test

.travis.yml

@@ -75,9 +75,12 @@ jobs:
             - fakeroot
             - python-bzrlib
             - python-paramiko
+      cache:
+        directories:
+          - $HOME/.gobundle
       script:
         - echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
-        - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <[email protected]>"
+        - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <[email protected]>" -goversion 1.13.4 -gohash 95dbeab442ee2746b9acf0934c8e2fc26414a0565c008631b04addb8c02e7624 -gobundle $HOME/.gobundle/go.tar.gz
 
     # This builder does the Linux Azure uploads
     - stage: build

build/ci-notes.md

@@ -22,30 +22,29 @@ variables `PPA_SIGNING_KEY` and `PPA_SSH_KEY` on Travis.
 
 We want to build go-ethereum with the most recent version of Go, irrespective of the Go
 version that is available in the main Ubuntu repository. In order to make this possible,
-our PPA depends on the ~gophers/ubuntu/archive PPA. Our source package build-depends on
-golang-1.11, which is co-installable alongside the regular golang package. PPA dependencies
-can be edited at https://launchpad.net/%7Eethereum/+archive/ubuntu/ethereum/+edit-dependencies
+we bundle the entire Go sources into our own source archive and start the built job by
+compiling Go and then using that to build go-ethereum. On Trusty we have a special case
+requiring the `~gophers/ubuntu/archive` PPA since Trusty can't even build Go itself. PPA
+deps are set at https://launchpad.net/%7Eethereum/+archive/ubuntu/ethereum/+edit-dependencies
 
 ## Building Packages Locally (for testing)
 
 You need to run Ubuntu to do test packaging.
 
-Add the gophers PPA and install Go 1.11 and Debian packaging tools:
+Install any version of Go and Debian packaging tools:
 
-    $ sudo apt-add-repository ppa:gophers/ubuntu/archive
-    $ sudo apt-get update
-    $ sudo apt-get install build-essential golang-1.11 devscripts debhelper python-bzrlib python-paramiko
+    $ sudo apt-get install build-essential golang-go devscripts debhelper python-bzrlib python-paramiko
 
 Create the source packages:
 
     $ go run build/ci.go debsrc -workdir dist
 
 Then go into the source package directory for your running distribution and build the package:
 
-    $ cd dist/ethereum-unstable-1.6.0+xenial
+    $ cd dist/ethereum-unstable-1.9.6+bionic
     $ dpkg-buildpackage
 
 Built packages are placed in the dist/ directory.
 
     $ cd ..
-    $ dpkg-deb -c geth-unstable_1.6.0+xenial_amd64.deb
+    $ dpkg-deb -c geth-unstable_1.9.6+bionic_amd64.deb

build/ci.go

@@ -58,6 +58,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/internal/build"
 	"github.com/ethereum/go-ethereum/params"
 )
@@ -138,7 +139,18 @@ var (
 	// Note: zesty is unsupported because it was officially deprecated on Launchpad.
 	// Note: artful is unsupported because it was officially deprecated on Launchpad.
 	// Note: cosmic is unsupported because it was officially deprecated on Launchpad.
-	debDistros = []string{"trusty", "xenial", "bionic", "disco", "eoan"}
+	debDistroGoBoots = map[string]string{
+		"trusty": "golang-1.11",
+		"xenial": "golang-go",
+		"bionic": "golang-go",
+		"disco":  "golang-go",
+		"eoan":   "golang-go",
+	}
+
+	debGoBootPaths = map[string]string{
+		"golang-1.11": "/usr/lib/go-1.11",
+		"golang-go":   "/usr/lib/go",
+	}
 )
 
 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
@@ -459,11 +471,14 @@ func maybeSkipArchive(env build.Environment) {
 // Debian Packaging
 func doDebianSource(cmdline []string) {
 	var (
-		signer  = flag.String("signer", "", `Signing key name, also used as package author`)
-		upload  = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`)
-		sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`)
-		workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
-		now     = time.Now()
+		goversion = flag.String("goversion", "", `Go version to build with (will be included in the source package)`)
+		gobundle  = flag.String("gobundle", "/tmp/go.tar.gz", `Filesystem path to cache the downloaded Go bundles at`)
+		gohash    = flag.String("gohash", "", `SHA256 checksum of the Go sources requested to build with`)
+		signer    = flag.String("signer", "", `Signing key name, also used as package author`)
+		upload    = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`)
+		sshUser   = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`)
+		workdir   = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
+		now       = time.Now()
 	)
 	flag.CommandLine.Parse(cmdline)
 	*workdir = makeWorkdir(*workdir)
@@ -476,12 +491,25 @@ func doDebianSource(cmdline []string) {
 		gpg.Stdin = bytes.NewReader(key)
 		build.MustRun(gpg)
 	}
-
+	// Download and verify the Go source package
+	if err := build.EnsureGoSources(*goversion, hexutil.MustDecode("0x"+*gohash), *gobundle); err != nil {
+		log.Fatalf("Failed to ensure Go source package: %v", err)
+	}
 	// Create Debian packages and upload them
 	for _, pkg := range debPackages {
-		for _, distro := range debDistros {
-			meta := newDebMetadata(distro, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables)
+		for distro, goboot := range debDistroGoBoots {
+			// Prepare the debian package with the go-ethereum sources
+			meta := newDebMetadata(distro, goboot, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables)
 			pkgdir := stageDebianSource(*workdir, meta)
+
+			// Ship the Go sources along so we have a proper thing to build with
+			if err := build.ExtractTarballArchive(*gobundle, pkgdir); err != nil {
+				log.Fatalf("Failed to extract Go sources: %v", err)
+			}
+			if err := os.Rename(filepath.Join(pkgdir, "go"), filepath.Join(pkgdir, ".go")); err != nil {
+				log.Fatalf("Failed to rename Go source folder: %v", err)
+			}
+			// Run the packaging and upload to the PPA
 			debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz")
 			debuild.Dir = pkgdir
 			build.MustRun(debuild)
@@ -561,7 +589,9 @@ type debPackage struct {
 }
 
 type debMetadata struct {
-	Env build.Environment
+	Env           build.Environment
+	GoBootPackage string
+	GoBootPath    string
 
 	PackageName string
 
@@ -590,19 +620,21 @@ func (d debExecutable) Package() string {
 	return d.BinaryName
 }
 
-func newDebMetadata(distro, author string, env build.Environment, t time.Time, name string, version string, exes []debExecutable) debMetadata {
+func newDebMetadata(distro, goboot, author string, env build.Environment, t time.Time, name string, version string, exes []debExecutable) debMetadata {
 	if author == "" {
 		// No signing key, use default author.
 		author = "Ethereum Builds <[email protected]>"
 	}
 	return debMetadata{
-		PackageName: name,
-		Env:         env,
-		Author:      author,
-		Distro:      distro,
-		Version:     version,
-		Time:        t.Format(time.RFC1123Z),
-		Executables: exes,
+		GoBootPackage: goboot,
+		GoBootPath:    debGoBootPaths[goboot],
+		PackageName:   name,
+		Env:           env,
+		Author:        author,
+		Distro:        distro,
+		Version:       version,
+		Time:          t.Format(time.RFC1123Z),
+		Executables:   exes,
 	}
 }
 
@@ -667,7 +699,6 @@ func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) {
 	if err := os.Mkdir(pkgdir, 0755); err != nil {
 		log.Fatal(err)
 	}
-
 	// Copy the source code.
 	build.MustRunCommand("git", "checkout-index", "-a", "--prefix", pkgdir+string(filepath.Separator))
 
@@ -685,7 +716,6 @@ func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) {
 		build.Render("build/deb/"+meta.PackageName+"/deb.install", install, 0644, exe)
 		build.Render("build/deb/"+meta.PackageName+"/deb.docs", docs, 0644, exe)
 	}
-
 	return pkgdir
 }
 

build/deb/ethereum/deb.control

@@ -2,7 +2,7 @@ Source: {{.Name}}
 Section: science
 Priority: extra
 Maintainer: {{.Author}}
-Build-Depends: debhelper (>= 8.0.0), golang-1.11
+Build-Depends: debhelper (>= 8.0.0), {{.GoBootPackage}}
 Standards-Version: 3.9.5
 Homepage: https://ethereum.org
 Vcs-Git: git://github.com/ethereum/go-ethereum.git

build/deb/ethereum/deb.rules

@@ -6,9 +6,11 @@
 
 # Launchpad rejects Go's access to $HOME/.cache, use custom folder
 export GOCACHE=/tmp/go-build
+export GOROOT_BOOTSTRAP={{.GoBootPath}}
 
 override_dh_auto_build:
-	build/env.sh /usr/lib/go-1.11/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
+	(cd .go/src && ./make.bash)
+	build/env.sh .go/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
 
 override_dh_auto_test:
 

internal/build/archive.go

@@ -183,3 +183,49 @@ func (a *TarballArchive) Close() error {
 	}
 	return a.file.Close()
 }
+
+func ExtractTarballArchive(archive string, dest string) error {
+	// We're only interested in gzipped archives, wrap the reader now
+	ar, err := os.Open(archive)
+	if err != nil {
+		return err
+	}
+	defer ar.Close()
+
+	gzr, err := gzip.NewReader(ar)
+	if err != nil {
+		return err
+	}
+	defer gzr.Close()
+
+	// Iterate over all the files in the tarball
+	tr := tar.NewReader(gzr)
+	for {
+		// Fetch the next tarball header and abort if needed
+		header, err := tr.Next()
+		if err != nil {
+			if err == io.EOF {
+				return nil
+			}
+			return err
+		}
+		// Figure out the target and create it
+		target := filepath.Join(dest, header.Name)
+
+		switch header.Typeflag {
+		case tar.TypeDir:
+			if err := os.MkdirAll(target, 0755); err != nil {
+				return err
+			}
+		case tar.TypeReg:
+			file, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
+			if err != nil {
+				return err
+			}
+			if _, err := io.Copy(file, tr); err != nil {
+				return err
+			}
+			file.Close()
+		}
+	}
+}

internal/build/gosrc.go

@@ -0,0 +1,81 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package build
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// EnsureGoSources ensures that path contains a file with the given SHA256 hash,
+// and if not, it downloads a fresh Go source package from upstream and replaces
+// path with it (if the hash matches).
+func EnsureGoSources(version string, hash []byte, path string) error {
+	// Sanity check the destination path to ensure we don't do weird things
+	if !strings.HasSuffix(path, ".tar.gz") {
+		return fmt.Errorf("destination path (%s) must end with .tar.gz", path)
+	}
+	// If the file exists, validate it's hash
+	if archive, err := ioutil.ReadFile(path); err == nil { // Go sources are ~20MB, it's fine to read all
+		hasher := sha256.New()
+		hasher.Write(archive)
+		have := hasher.Sum(nil)
+
+		if bytes.Equal(have, hash) {
+			fmt.Printf("Go %s [%x] available at %s\n", version, hash, path)
+			return nil
+		}
+		fmt.Printf("Go %s hash mismatch (have %x, want %x) at %s, deleting old archive\n", version, have, hash, path)
+		if err := os.Remove(path); err != nil {
+			return err
+		}
+	}
+	// Archive missing or bad hash, download a new one
+	fmt.Printf("Downloading Go %s [want %x] into %s\n", version, hash, path)
+
+	res, err := http.Get(fmt.Sprintf("https://dl.google.com/go/go%s.src.tar.gz", version))
+	if err != nil || res.StatusCode != http.StatusOK {
+		return fmt.Errorf("failed to access Go sources: code %d, err %v", res.StatusCode, err)
+	}
+	defer res.Body.Close()
+
+	archive, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return err
+	}
+	// Sanity check the downloaded archive, save if checks out
+	hasher := sha256.New()
+	hasher.Write(archive)
+
+	if have := hasher.Sum(nil); !bytes.Equal(have, hash) {
+		return fmt.Errorf("downloaded Go %s hash mismatch (have %x, want %x)", version, have, hash)
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+		return err
+	}
+	if err := ioutil.WriteFile(path, archive, 0644); err != nil {
+		return err
+	}
+	fmt.Printf("Downloaded Go %s [%x] into %s\n", version, hash, path)
+	return nil
+}