Parse .netrc for HTTP Basic authentication credentials when BAZELISK_…

…BASE_URL points to protected resource (bazelbuild#292)

* Use ~/.netrc for Basic HTTP auth

* Update README

* Make tryFindNetrcFileCreds private

* Fix build

* Fix review issues

* Fix build and add more logs

.gitignore

@@ -7,3 +7,4 @@
 
 # Binary created by 'go build'
 /bazelisk
+/bazelisk.exe

README.md

@@ -73,7 +73,7 @@ As mentioned in the previous section, the `<FORK>/<VERSION>` version format allo
 If you want to create a fork with your own releases, you have to follow the naming conventions that we use in `bazelbuild/bazel` for the binary file names.
 The URL format looks like `https://github.com/<FORK>/bazel/releases/download/<VERSION>/<FILENAME>`.
 
-You can also override the URL by setting the environment variable `$BAZELISK_BASE_URL`. Bazelisk will then append `/<VERSION>/<FILENAME>` to the base URL instead of using the official release server.
+You can also override the URL by setting the environment variable `$BAZELISK_BASE_URL`. Bazelisk will then append `/<VERSION>/<FILENAME>` to the base URL instead of using the official release server. Bazelisk will read file [`~/.netrc`](https://everything.curl.dev/usingcurl/netrc) for credentials for Basic authentication.
 
 ## Ensuring that your developers use Bazelisk rather than Bazel
 

WORKSPACE

@@ -42,6 +42,12 @@ go_repository(
     version = "v1.1.0",
 )
 
+go_repository(
+    name = "com_github_jdxcode_netrc",
+    importpath = "github.com/jdxcode/netrc",
+    commit = "926c7f70242abe00179235c2b06bb647c0c53a12",
+)
+
 go_rules_dependencies()
 
 go_register_toolchains(version = "1.16.4")
@@ -59,8 +65,6 @@ load("@build_bazel_rules_nodejs//:index.bzl", "node_repositories")
 
 node_repositories()
 
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
-
 http_archive(
     name = "platforms",
     sha256 = "079945598e4b6cc075846f7fd6a9d0857c33a7afc0de868c2ccb96405225135d",

go.mod

@@ -5,5 +5,6 @@ go 1.15
 require (
 	github.com/bazelbuild/rules_go v0.29.0
 	github.com/hashicorp/go-version v1.3.0
+	github.com/jdxcode/netrc v0.0.0-20210204082910-926c7f70242a // indirect
 	github.com/mitchellh/go-homedir v1.1.0
 )

go.sum

@@ -2,5 +2,11 @@ github.com/bazelbuild/rules_go v0.29.0 h1:SfxjyO/V68rVnzOHop92fB0gv/Aa75KNLAN0PM
 github.com/bazelbuild/rules_go v0.29.0/go.mod h1:MC23Dc/wkXEyk3Wpq6lCqz0ZAYOZDw2DR5y3N1q2i7M=
 github.com/hashicorp/go-version v1.3.0 h1:McDWVJIU/y+u1BRV06dPaLfLCaT7fUTJLp5r04x7iNw=
 github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/jdxcode/netrc v0.0.0-20210204082910-926c7f70242a h1:d4+I1YEKVmWZrgkt6jpXBnLgV2ZjO0YxEtLDdfIZfH4=
+github.com/jdxcode/netrc v0.0.0-20210204082910-926c7f70242a/go.mod h1:Zi/ZFkEqFHTm7qkjyNJjaWH4LQA9LQhGJyF0lTYGpxw=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

httputil/BUILD

@@ -10,6 +10,10 @@ go_library(
         "fake.go",
         "httputil.go",
     ],
+    deps = [
+        "@com_github_mitchellh_go_homedir//:go_default_library",
+        "@com_github_jdxcode_netrc//:go_default_library"
+    ],
     importpath = "github.com/bazelbuild/bazelisk/httputil",
     visibility = ["//visibility:public"],
 )

httputil/httputil.go

@@ -2,24 +2,29 @@
 package httputil
 
 import (
+	b64 "encoding/base64"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
 	"math/rand"
 	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
 	"regexp"
 	"strconv"
 	"time"
+
+	netrc "github.com/jdxcode/netrc"
+	homedir "github.com/mitchellh/go-homedir"
 )
 
 var (
 	// DefaultTransport specifies the http.RoundTripper that is used for any network traffic, and may be replaced with a dummy implementation for unit testing.
 	DefaultTransport = http.DefaultTransport
 	// UserAgent is passed to every HTTP request as part of the 'User-Agent' header.
-	UserAgent = "Bazelisk"
+	UserAgent   = "Bazelisk"
 	linkPattern = regexp.MustCompile(`<(.*?)>; rel="(\w+)"`)
 
 	// RetryClock is used for waiting between HTTP request retries.
@@ -28,7 +33,7 @@ var (
 	MaxRetries = 4
 	// MaxRequestDuration defines the maximum amount of time that a request and its retries may take in total
 	MaxRequestDuration = time.Second * 30
-	retryHeaders = []string{"Retry-After", "X-RateLimit-Reset", "Rate-Limit-Reset"}
+	retryHeaders       = []string{"Retry-After", "X-RateLimit-Reset", "Rate-Limit-Reset"}
 )
 
 // Clock keeps track of time. It can return the current time, as well as move forward by sleeping for a certain period.
@@ -37,7 +42,7 @@ type Clock interface {
 	Now() time.Time
 }
 
-type realClock struct {}
+type realClock struct{}
 
 func (*realClock) Sleep(d time.Duration) {
 	time.Sleep(d)
@@ -47,12 +52,12 @@ func (*realClock) Now() time.Time {
 	return time.Now()
 }
 
-// ReadRemoteFile returns the contents of the given file, using the supplied Authorization token, if set. It also returns the HTTP headers.
+// ReadRemoteFile returns the contents of the given file, using the supplied Authorization header value, if set. It also returns the HTTP headers.
 // If the request fails with a transient error it will retry the request for at most MaxRetries times.
 // It obeys HTTP headers such as "Retry-After" when calculating the start time of the next attempt.
 // If no such header is present, it uses an exponential backoff strategy.
-func ReadRemoteFile(url string, token string) ([]byte, http.Header, error) {
-	res, err := get(url, token)
+func ReadRemoteFile(url string, auth string) ([]byte, http.Header, error) {
+	res, err := get(url, auth)
 	if err != nil {
 		return nil, nil, fmt.Errorf("could not fetch %s: %v", url, err)
 	}
@@ -69,15 +74,15 @@ func ReadRemoteFile(url string, token string) ([]byte, http.Header, error) {
 	return body, res.Header, nil
 }
 
-func get(url, token string) (*http.Response, error) {
+func get(url, auth string) (*http.Response, error) {
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
 		return nil, fmt.Errorf("could not create request: %v", err)
 	}
 
 	req.Header.Set("User-Agent", UserAgent)
-	if token != "" {
-		req.Header.Set("Authorization", "token "+token)
+	if auth != "" {
+		req.Header.Set("Authorization", auth)
 	}
 	client := &http.Client{Transport: DefaultTransport}
 	deadline := RetryClock.Now().Add(MaxRequestDuration)
@@ -118,7 +123,7 @@ func getWaitPeriod(res *http.Response, attempt int) (time.Duration, error) {
 		}
 	}
 	// Let's just use exponential backoff: 1s + d1, 2s + d2, 4s + d3, 8s + d4 with dx being a random value in [0ms, 500ms]
-	return time.Duration(1 << uint(attempt)) * time.Second + time.Duration(rand.Intn(500)) * time.Millisecond, nil
+	return time.Duration(1<<uint(attempt))*time.Second + time.Duration(rand.Intn(500))*time.Millisecond, nil
 }
 
 func parseRetryHeader(value string) (time.Duration, error) {
@@ -133,6 +138,36 @@ func parseRetryHeader(value string) (time.Duration, error) {
 	return time.Until(t), nil
 }
 
+// tryFindNetrcFileCreds returns base64-encoded login:password found in ~/.netrc file for a given `host`
+func tryFindNetrcFileCreds(host string) (string, error) {
+	dir, err := homedir.Dir()
+	if err != nil {
+		return "", err
+	}
+
+	var file = filepath.Join(dir, ".netrc")
+	n, err := netrc.Parse(file)
+	if err != nil {
+		// netrc does not exist or we can't read it
+		return "", err
+	}
+
+	m := n.Machine(host)
+	if m == nil {
+		// if host is not found, we should proceed without providing any Authorization header,
+		// because remote host may not have auth at all.
+		log.Printf("Skipping basic authentication for %s because no credentials found in %s", host, file)
+		return "", fmt.Errorf("could not find creds for %s in netrc %s", host, file)
+	}
+
+	log.Printf("Using basic authentication credentials for host %s from %s", host, file)
+
+	login := m.Get("login")
+	pwd := m.Get("password")
+	token := b64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", login, pwd)))
+	return fmt.Sprintf("Basic %s", token), nil
+}
+
 // DownloadBinary downloads a file from the given URL into the specified location, marks it executable and returns its full path.
 func DownloadBinary(originURL, destDir, destFile string) (string, error) {
 	err := os.MkdirAll(destDir, 0755)
@@ -153,8 +188,22 @@ func DownloadBinary(originURL, destDir, destFile string) (string, error) {
 			}
 		}()
 
+		u, err := url.Parse(originURL)
+		if err != nil {
+			// originURL supposed to be valid
+			return "", err
+		}
+
 		log.Printf("Downloading %s...", originURL)
-		resp, err := get(originURL, "")
+
+		var auth string = ""
+		t, err := tryFindNetrcFileCreds(u.Host)
+		if err == nil {
+			// successfully parsed netrc for given host
+			auth = t
+		}
+
+		resp, err := get(originURL, auth)
 		if err != nil {
 			return "", fmt.Errorf("HTTP GET %s failed: %v", originURL, err)
 		}
@@ -190,7 +239,8 @@ type ContentMerger func([][]byte) ([]byte, error)
 // MaybeDownload downloads a file from the given url and caches the result under bazeliskHome.
 // It skips the download if the file already exists and is not outdated.
 // Parameter ´description´ is only used to provide better error messages.
-func MaybeDownload(bazeliskHome, url, filename, description, token string, merger ContentMerger) ([]byte, error) {
+// Parameter `auth` is a value of "Authorization" HTTP header.
+func MaybeDownload(bazeliskHome, url, filename, description, auth string, merger ContentMerger) ([]byte, error) {
 	cachePath := filepath.Join(bazeliskHome, filename)
 	if cacheStat, err := os.Stat(cachePath); err == nil {
 		if time.Since(cacheStat.ModTime()).Hours() < 1 {
@@ -206,7 +256,7 @@ func MaybeDownload(bazeliskHome, url, filename, description, token string, merge
 	nextURL := url
 	for nextURL != "" {
 		// We could also use go-github here, but I can't get it to build with Bazel's rules_go and it pulls in a lot of dependencies.
-		body, headers, err := ReadRemoteFile(nextURL, token)
+		body, headers, err := ReadRemoteFile(nextURL, auth)
 		if err != nil {
 			return nil, fmt.Errorf("could not download %s: %v", description, err)
 		}
@@ -236,6 +286,6 @@ func getNextURL(headers http.Header) string {
 		if m[2] == "next" {
 			return m[1]
 		}
-	}	
+	}
 	return ""
 }

repositories/github.go

@@ -53,7 +53,7 @@ func (gh *GitHubRepo) getFilteredVersions(bazeliskHome, bazelFork string, wantPr
 	}
 
 	url := fmt.Sprintf("https://api.github.com/repos/%s/bazel/releases", bazelFork)
-	releasesJSON, err := httputil.MaybeDownload(bazeliskHome, url, bazelFork+"-releases.json", "list of Bazel releases from github.com/"+bazelFork, gh.token, merger)
+	releasesJSON, err := httputil.MaybeDownload(bazeliskHome, url, bazelFork+"-releases.json", "list of Bazel releases from github.com/"+bazelFork, fmt.Sprintf("token %s", gh.token), merger)
 	if err != nil {
 		return []string{}, fmt.Errorf("unable to dermine '%s' releases: %v", bazelFork, err)
 	}