tot uses prow config to determine fallback GCS path

prow/cmd/tot/BUILD.bazel

@@ -23,15 +23,20 @@ go_binary(
 go_test(
     name = "go_default_test",
     srcs = ["main_test.go"],
+    data = ["//prow:configs"],
     embed = [":go_default_library"],
+    deps = ["//prow/config:go_default_library"],
 )
 
 go_library(
     name = "go_default_library",
     srcs = ["main.go"],
     importpath = "k8s.io/test-infra/prow/cmd/tot",
     deps = [
+        "//prow/config:go_default_library",
         "//prow/logrusutil:go_default_library",
+        "//prow/pjutil:go_default_library",
+        "//prow/pod-utils/gcs:go_default_library",
         "//vendor/github.com/sirupsen/logrus:go_default_library",
     ],
 )

prow/cmd/tot/main.go

@@ -20,6 +20,7 @@ package main
 
 import (
 	"encoding/json"
+	"errors"
 	"flag"
 	"fmt"
 	"io/ioutil"
@@ -32,7 +33,10 @@ import (
 
 	"github.com/sirupsen/logrus"
 
+	"k8s.io/test-infra/prow/config"
 	"k8s.io/test-infra/prow/logrusutil"
+	"k8s.io/test-infra/prow/pjutil"
+	"k8s.io/test-infra/prow/pod-utils/gcs"
 )
 
 type options struct {
@@ -41,6 +45,9 @@ type options struct {
 
 	useFallback bool
 	fallbackURI string
+
+	configPath     string
+	fallbackBucket string
 }
 
 func gatherOptions() options {
@@ -51,13 +58,28 @@ func gatherOptions() options {
 	flag.BoolVar(&o.useFallback, "fallback", false, "Fallback to GCS bucket for missing builds.")
 	flag.StringVar(&o.fallbackURI, "fallback-url-template",
 		"https://storage.googleapis.com/kubernetes-jenkins/logs/%s/latest-build.txt",
-		"URL template to fallback to for every job that lacks a last vended build number.",
+		"URL template to fallback to for jobs that lack a last vended build number.",
+	)
+
+	flag.StringVar(&o.configPath, "config-path", "", "Path to prow config.")
+	flag.StringVar(&o.fallbackBucket, "fallback-bucket", "",
+		"Fallback to top-level bucket for jobs that lack a last vended build number. The bucket layout is expected to follow https://github.com/kubernetes/test-infra/tree/master/gubernator#gcs-bucket-layout",
 	)
 
 	flag.Parse()
 	return o
 }
 
+func (o *options) Validate() error {
+	if o.configPath != "" && o.fallbackBucket == "" {
+		return errors.New("you need to provide a bucket to fallback to when the prow config is specified")
+	}
+	if o.configPath == "" && o.fallbackBucket != "" {
+		return errors.New("you need to provide the prow config when a fallback bucket is specified")
+	}
+	return nil
+}
+
 type store struct {
 	Number       map[string]int // job name -> last vended build number
 	mutex        sync.Mutex
@@ -159,10 +181,16 @@ func (s *store) handle(w http.ResponseWriter, r *http.Request) {
 
 type fallbackHandler struct {
 	template string
+	// in case a config agent is provided, tot will
+	// determine the GCS path that it needs to use
+	// based on the configured jobs in prow and
+	// bucket.
+	configAgent *config.Agent
+	bucket      string
 }
 
 func (f fallbackHandler) get(jobName string) int {
-	url := fmt.Sprintf(f.template, jobName)
+	url := f.getURL(jobName)
 
 	var body []byte
 
@@ -192,9 +220,49 @@ func (f fallbackHandler) get(jobName string) int {
 	return n
 }
 
+func (f fallbackHandler) getURL(jobName string) string {
+	if f.configAgent == nil {
+		return fmt.Sprintf(f.template, jobName)
+	}
+
+	var spec *pjutil.JobSpec
+	cfg := f.configAgent.Config()
+
+	for _, pre := range cfg.AllPresubmits(nil) {
+		if jobName == pre.Name {
+			spec = pjutil.PresubmitToJobSpec(pre)
+			break
+		}
+	}
+	if spec == nil {
+		for _, post := range cfg.AllPostsubmits(nil) {
+			if jobName == post.Name {
+				spec = pjutil.PostsubmitToJobSpec(post)
+				break
+			}
+		}
+	}
+	if spec == nil {
+		for _, per := range cfg.AllPeriodics() {
+			if jobName == per.Name {
+				spec = pjutil.PeriodicToJobSpec(per)
+				break
+			}
+		}
+	}
+	// If spec is still nil, we know nothing about the requested job.
+	if spec == nil {
+		logrus.Errorf("requested job is unknown to prow: %s", jobName)
+		return ""
+	}
+	return fmt.Sprintf("%s/%s", strings.TrimSuffix(f.bucket, "/"), gcs.LatestBuildForSpec(spec))
+}
+
 func main() {
 	o := gatherOptions()
-
+	if err := o.Validate(); err != nil {
+		logrus.Fatalf("Invalid options: %v", err)
+	}
 	logrus.SetFormatter(
 		logrusutil.NewDefaultFieldsFormatter(nil, logrus.Fields{"component": "tot"}),
 	)
@@ -205,7 +273,19 @@ func main() {
 	}
 
 	if o.useFallback {
-		s.fallbackFunc = fallbackHandler{o.fallbackURI}.get
+		var configAgent *config.Agent
+		if o.configPath != "" {
+			configAgent = &config.Agent{}
+			if err := configAgent.Start(o.configPath); err != nil {
+				logrus.WithError(err).Fatal("Error starting config agent.")
+			}
+		}
+
+		s.fallbackFunc = fallbackHandler{
+			template:    o.fallbackURI,
+			configAgent: configAgent,
+			bucket:      o.fallbackBucket,
+		}.get
 	}
 
 	http.HandleFunc("/vend/", s.handle)

prow/cmd/tot/main_test.go

@@ -25,6 +25,8 @@ import (
 	"reflect"
 	"strings"
 	"testing"
+
+	"k8s.io/test-infra/prow/config"
 )
 
 func expectEqual(t *testing.T, msg string, have interface{}, want interface{}) {
@@ -138,7 +140,7 @@ func TestFallback(t *testing.T) {
 
 	store := makeStore(t)
 	defer os.Remove(store.storagePath)
-	store.fallbackFunc = fallbackHandler{serv.URL + "/logs/%s/latest-build.txt"}.get
+	store.fallbackFunc = fallbackHandler{template: serv.URL + "/logs/%s/latest-build.txt"}.get
 
 	expectEqual(t, "vend foo 1", store.vend("foo"), 201)
 	expectEqual(t, "vend foo 2", store.vend("foo"), 202)
@@ -147,3 +149,101 @@ func TestFallback(t *testing.T) {
 	expectEqual(t, "vend baz", store.vend("baz"), 1)
 	expectEqual(t, "vend quux", store.vend("quux"), 1)
 }
+
+var c *config.Config
+
+func TestMain(m *testing.M) {
+	conf, err := config.Load("../../config.yaml")
+	if err != nil {
+		fmt.Printf("Could not load config: %v", err)
+		os.Exit(1)
+	}
+	c = conf
+	os.Exit(m.Run())
+}
+
+func TestGetURL(t *testing.T) {
+	tests := []struct {
+		name string
+
+		jobName  string
+		template string
+		c        *config.Config
+		bucket   string
+
+		expected string
+	}{
+		{
+			name: "fallback template",
+
+			jobName:  "pull-community-verify",
+			template: "https://storage.googleapis.com/kubernetes-jenkins/logs/%s/latest-build.txt",
+
+			expected: "https://storage.googleapis.com/kubernetes-jenkins/logs/pull-community-verify/latest-build.txt",
+		},
+		{
+			name: "fallback bucket - presubmit",
+
+			jobName: "pull-community-verify",
+			c:       c,
+			bucket:  "https://storage.googleapis.com/kubernetes-jenkins",
+
+			expected: "https://storage.googleapis.com/kubernetes-jenkins/pr-logs/directory/pull-community-verify/latest-build.txt",
+		},
+		{
+			name: "fallback bucket - postsubmit",
+
+			jobName: "ci-federation-release",
+			c:       c,
+			bucket:  "https://storage.googleapis.com/kubernetes-jenkins",
+
+			expected: "https://storage.googleapis.com/kubernetes-jenkins/logs/ci-federation-release/latest-build.txt",
+		},
+		{
+			name: "fallback bucket - periodic",
+
+			jobName: "ci-kubernetes-cross-build",
+			c:       c,
+			bucket:  "https://storage.googleapis.com/kubernetes-jenkins",
+
+			expected: "https://storage.googleapis.com/kubernetes-jenkins/logs/ci-kubernetes-cross-build/latest-build.txt",
+		},
+		{
+			name: "fallback bucket - unknown",
+
+			jobName: "a-name-that-is-what-it-is",
+			c:       c,
+			bucket:  "https://storage.googleapis.com/kubernetes-jenkins",
+
+			expected: "",
+		},
+		{
+			name: "fallback bucket with trailing slash",
+
+			jobName: "pull-community-verify",
+			c:       c,
+			bucket:  "https://storage.googleapis.com/kubernetes-jenkins/",
+
+			expected: "https://storage.googleapis.com/kubernetes-jenkins/pr-logs/directory/pull-community-verify/latest-build.txt",
+		},
+	}
+
+	for _, test := range tests {
+		t.Logf("running scenario %q", test.name)
+
+		var configAgent *config.Agent
+		if test.c != nil {
+			configAgent = new(config.Agent)
+			configAgent.Set(test.c)
+		}
+		f := fallbackHandler{
+			template:    test.template,
+			configAgent: configAgent,
+			bucket:      test.bucket,
+		}
+
+		if got := f.getURL(test.jobName); got != test.expected {
+			t.Errorf("unexpected URL:\n%s\nexpected:\n%s", got, test.expected)
+		}
+	}
+}

prow/pjutil/jobspec.go

@@ -28,6 +28,7 @@ import (
 	"strconv"
 	"time"
 
+	"k8s.io/test-infra/prow/config"
 	"k8s.io/test-infra/prow/kube"
 )
 
@@ -61,6 +62,36 @@ func NewJobSpec(spec kube.ProwJobSpec, buildId, prowJobId string) JobSpec {
 	}
 }
 
+// PresubmitToJobSpec generates a JobSpec out of a Presubmit.
+// Useful for figuring out GCS paths when parsing jobs out
+// of a prow config.
+func PresubmitToJobSpec(pre config.Presubmit) *JobSpec {
+	return &JobSpec{
+		Type: kube.PresubmitJob,
+		Job:  pre.Name,
+	}
+}
+
+// PostsubmitToJobSpec generates a JobSpec out of a Postsubmit.
+// Useful for figuring out GCS paths when parsing jobs out
+// of a prow config.
+func PostsubmitToJobSpec(post config.Postsubmit) *JobSpec {
+	return &JobSpec{
+		Type: kube.PostsubmitJob,
+		Job:  post.Name,
+	}
+}
+
+// PeriodicToJobSpec generates a JobSpec out of a Periodic.
+// Useful for figuring out GCS paths when parsing jobs out
+// of a prow config.
+func PeriodicToJobSpec(periodic config.Periodic) *JobSpec {
+	return &JobSpec{
+		Type: kube.PeriodicJob,
+		Job:  periodic.Name,
+	}
+}
+
 // GetBuildID calls out to `tot` in order
 // to vend build identifier for the job
 func GetBuildID(name, totURL string) (string, error) {