Merge pull request kubernetes#44591 from ixdy/bazel-push-build

Automatic merge from submit-queue (batch tested with PRs 44591, 44549)

Update repo-infra bazel dependency and use new gcs_upload rule

This PR provides similar functionality to push-build.sh entirely within Bazel rules (though it relies on gsutil).

It's an alternative to kubernetes#44306.

Depends on kubernetes/repo-infra#13.

**Release note**:

```release-note
NONE
```

BUILD.bazel

@@ -4,16 +4,32 @@ licenses(["notice"])
 
 load("@io_bazel_rules_go//go:def.bzl", "go_prefix")
 load("@io_kubernetes_build//defs:build.bzl", "gcs_upload")
-load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar")
 
 go_prefix("k8s.io/kubernetes")
 
+filegroup(
+    name = "_binary-artifacts-and-hashes",
+    srcs = [
+        "//build:client-targets-and-hashes",
+        "//build:docker-artifacts-and-hashes",
+        "//build:node-targets-and-hashes",
+        "//build:server-targets-and-hashes",
+    ],
+    visibility = ["//visibility:private"],
+)
+
 gcs_upload(
     name = "ci-artifacts",
     data = [
-        "//build/debs",
-        "//build/release-tars",
+        ":_binary-artifacts-and-hashes",
+        "//build/release-tars:release-tars-and-hashes",
+        "//cluster/gce:gcs-release-artifacts-and-hashes",
     ],
+    upload_paths = {
+        "//:_binary-artifacts-and-hashes": "bin/linux/amd64",
+        "//build/release-tars:release-tars-and-hashes": "",
+        "//cluster/gce:gcs-release-artifacts-and-hashes": "extra/gce",
+    },
 )
 
 filegroup(

WORKSPACE

@@ -1,18 +1,18 @@
 git_repository(
     name = "io_bazel_rules_go",
-    commit = "d57a2abc3b16a3904c159794f0f0a52cf6f061a7",
+    commit = "805fd1566500997379806373feb05e138a4dfe28",
     remote = "https://github.com/bazelbuild/rules_go.git",
 )
 
 git_repository(
     name = "io_kubernetes_build",
-    commit = "685f15b90b454af3086ab071fdea1b6db213d1fb",
+    commit = "684e550a2f006dbe3cf3b3d481d3f19217b228f7",
     remote = "https://github.com/kubernetes/repo-infra.git",
 )
 
 git_repository(
     name = "io_bazel",
-    commit = "3b29803eb528ff525c7024190ffbf4b08c598cf2",
+    commit = "1fe52dd4b2d77a740648bc1509b68acae49deffc",
     remote = "https://github.com/ixdy/bazel.git",
 )
 

build/BUILD

@@ -1,6 +1,7 @@
 package(default_visibility = ["//visibility:public"])
 
 load("@io_bazel//tools/build_defs/docker:docker.bzl", "docker_build")
+load("@io_kubernetes_build//defs:build.bzl", "md5sum", "release_filegroup")
 
 filegroup(
     name = "package-srcs",
@@ -69,13 +70,9 @@ DOCKERIZED_BINARIES = {
     },
 }
 
-[genrule(
-    name = binary + "_docker_tag",
-    srcs = [meta["target"]],
-    outs = [binary + ".docker_tag"],
-    # Currently each target has two outputs, the binary and its library, so hash only the first item (the binary).
-    # This can be made less hacky when we have static linking working.
-    cmd = "md5sum $(locations " + meta["target"] + ") | grep '" + binary + "'$$ | cut -f1 -d' ' | tr -d '\n' > $@",
+[md5sum(
+    name = binary + ".docker_tag",
+    src = meta["target"],
 ) for binary, meta in DOCKERIZED_BINARIES.items()]
 
 [docker_build(
@@ -97,8 +94,78 @@ DOCKERIZED_BINARIES = {
     },
 ) for binary, meta in DOCKERIZED_BINARIES.items()]
 
-filegroup(
+release_filegroup(
     name = "docker-artifacts",
     srcs = [":%s.tar" % binary for binary in DOCKERIZED_BINARIES.keys()] +
            [":%s.docker_tag" % binary for binary in DOCKERIZED_BINARIES.keys()],
 )
+
+# KUBE_CLIENT_TARGETS
+release_filegroup(
+    name = "client-targets",
+    srcs = [
+        "//cmd/kubectl",
+        "//federation/cmd/kubefed",
+    ],
+)
+
+# KUBE_NODE_TARGETS
+release_filegroup(
+    name = "node-targets",
+    srcs = [
+        "//cmd/kube-proxy",
+        "//cmd/kubelet",
+    ],
+)
+
+# KUBE_SERVER_TARGETS
+# No need to duplicate CLIENT_TARGETS or NODE_TARGETS here,
+# since we include them in the actual build rule.
+release_filegroup(
+    name = "server-targets",
+    srcs = [
+        "//cmd/cloud-controller-manager",
+        "//cmd/hyperkube",
+        "//cmd/kube-apiserver",
+        "//cmd/kube-controller-manager",
+        "//cmd/kubeadm",
+        "//plugin/cmd/kube-scheduler",
+        "//vendor/k8s.io/kube-aggregator",
+    ],
+)
+
+# kube::golang::test_targets
+filegroup(
+    name = "test-targets",
+    srcs = [
+        "//cmd/gendocs",
+        "//cmd/genkubedocs",
+        "//cmd/genman",
+        "//cmd/genswaggertypedocs",
+        "//cmd/genyaml",
+        "//cmd/kubemark",  # TODO: server platforms only
+        "//cmd/linkcheck",
+        "//cmd/mungedocs",
+        "//examples/k8petstore/web-server/src",
+        "//federation/cmd/genfeddocs",
+        "//test/e2e:e2e.test",
+        "//test/e2e_node:e2e_node.test",  # TODO: server platforms only
+        "//vendor/github.com/onsi/ginkgo/ginkgo",
+    ],
+)
+
+# KUBE_TEST_PORTABLE
+filegroup(
+    name = "test-portable-targets",
+    srcs = [
+        "//federation/develop:all-srcs",
+        "//hack:e2e.go",
+        "//hack:federated-ginkgo-e2e.sh",
+        "//hack:get-build.sh",
+        "//hack:ginkgo-e2e.sh",
+        "//hack/e2e-internal:all-srcs",
+        "//hack/lib:all-srcs",
+        "//test/e2e/testing-manifests:all-srcs",
+        "//test/kubemark:all-srcs",
+    ],
+)

build/release-tars/BUILD

@@ -1,6 +1,7 @@
 package(default_visibility = ["//visibility:public"])
 
 load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar")
+load("@io_kubernetes_build//defs:build.bzl", "release_filegroup")
 
 filegroup(
     name = "package-srcs",
@@ -52,69 +53,19 @@ pkg_tar(
 # FIXME: this should be configurable/auto-detected
 PLATFORM_ARCH_STRING = "linux-amd64"
 
-# KUBE_CLIENT_TARGETS
-CLIENT_TARGETS = [
-    "//cmd/kubectl",
-    "//federation/cmd/kubefed",
-]
-
-# KUBE_NODE_TARGETS
-NODE_TARGETS = [
-    "//cmd/kube-proxy",
-    "//cmd/kubelet",
-]
-
-# KUBE_SERVER_TARGETS
-# No need to duplicate CLIENT_TARGETS or NODE_TARGETS here,
-# since we include them in the actual build rule.
-SERVER_TARGETS = [
-    "//cmd/cloud-controller-manager",
-    "//cmd/hyperkube",
-    "//cmd/kube-apiserver",
-    "//cmd/kube-controller-manager",
-    "//cmd/kubeadm",
-    "//plugin/cmd/kube-scheduler",
-    "//vendor/k8s.io/kube-aggregator",
-]
-
-# kube::golang::test_targets
-TEST_BINARY_TARGETS = [
-    "//cmd/gendocs",
-    "//cmd/genkubedocs",
-    "//cmd/genman",
-    "//cmd/genswaggertypedocs",
-    "//cmd/genyaml",
-    "//cmd/linkcheck",
-    "//cmd/mungedocs",
-    "//examples/k8petstore/web-server/src",
-    "//federation/cmd/genfeddocs",
-    "//test/e2e:e2e.test",
-    "//vendor/github.com/onsi/ginkgo/ginkgo:ginkgo",
-    "//cmd/kubemark",  # TODO: server platforms only
-    "//test/e2e_node:e2e_node.test",  # TODO: server platforms only
-]
-
-TEST_PORTABLE_TARGETS = [
-    "//federation/develop:all-srcs",
-    "//hack:e2e.go",
-    "//hack:federated-ginkgo-e2e.sh",
-    "//hack:get-build.sh",
-    "//hack:ginkgo-e2e.sh",
-    "//hack/e2e-internal:all-srcs",
-    "//hack/lib:all-srcs",
-    "//test/e2e/testing-manifests:all-srcs",
-    "//test/kubemark:all-srcs",
-]
-
 # Included in node and server tarballs.
-LICENSE_TARGETS = [
-    "//:Godeps/LICENSES",
-    ":kubernetes-src.tar.gz",
-]
+filegroup(
+    name = "license-targets",
+    srcs = [
+        ":kubernetes-src.tar.gz",
+        "//:Godeps/LICENSES",
+    ],
+    visibility = ["//visibility:private"],
+)
 
 pkg_tar(
     name = "_client-bin",
-    files = CLIENT_TARGETS,
+    files = ["//build:client-targets"],
     mode = "0755",
     package_dir = "client/bin",
     visibility = ["//visibility:private"],
@@ -131,7 +82,10 @@ pkg_tar(
 
 pkg_tar(
     name = "_node-bin",
-    files = NODE_TARGETS + CLIENT_TARGETS,
+    files = [
+        "//build:client-targets",
+        "//build:node-targets",
+    ],
     mode = "0755",
     package_dir = "node/bin",
     visibility = ["//visibility:private"],
@@ -140,7 +94,7 @@ pkg_tar(
 pkg_tar(
     name = "kubernetes-node-%s" % PLATFORM_ARCH_STRING,
     extension = "tar.gz",
-    files = LICENSE_TARGETS,
+    files = [":license-targets"],
     mode = "0644",
     package_dir = "kubernetes",
     deps = [
@@ -150,8 +104,11 @@ pkg_tar(
 
 pkg_tar(
     name = "_server-bin",
-    files = SERVER_TARGETS + NODE_TARGETS + CLIENT_TARGETS + [
+    files = [
+        "//build:client-targets",
         "//build:docker-artifacts",
+        "//build:node-targets",
+        "//build:server-targets",
     ],
     mode = "0755",
     package_dir = "server/bin",
@@ -162,6 +119,7 @@ genrule(
     name = "dummy",
     outs = [".dummy"],
     cmd = "touch $@",
+    visibility = ["//visibility:private"],
 )
 
 # Some of the startup scripts fail if there isn't an addons/ directory in the server tarball.
@@ -177,7 +135,7 @@ pkg_tar(
 pkg_tar(
     name = "kubernetes-server-%s" % PLATFORM_ARCH_STRING,
     extension = "tar.gz",
-    files = LICENSE_TARGETS,
+    files = [":license-targets"],
     mode = "0644",
     package_dir = "kubernetes",
     deps = [
@@ -188,7 +146,7 @@ pkg_tar(
 
 pkg_tar(
     name = "_test-bin",
-    files = TEST_BINARY_TARGETS,
+    files = ["//build:test-targets"],
     mode = "0755",
     package_dir = "platforms/" + PLATFORM_ARCH_STRING.replace("-", "/"),
     # TODO: how to make this multiplatform?
@@ -198,7 +156,7 @@ pkg_tar(
 pkg_tar(
     name = "kubernetes-test",
     extension = "tar.gz",
-    files = TEST_PORTABLE_TARGETS,
+    files = ["//build:test-portable-targets"],
     package_dir = "kubernetes",
     strip_prefix = "//",
     deps = [
@@ -255,7 +213,7 @@ pkg_tar(
     ],
 )
 
-filegroup(
+release_filegroup(
     name = "release-tars",
     srcs = [
         ":kubernetes.tar.gz",

cluster/gce/BUILD

@@ -1,6 +1,7 @@
 package(default_visibility = ["//visibility:public"])
 
 load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar")
+load("@io_kubernetes_build//defs:build.bzl", "release_filegroup")
 
 pkg_tar(
     name = "gci-trusty-manifests",
@@ -34,3 +35,16 @@ filegroup(
     srcs = [":package-srcs"],
     tags = ["automanaged"],
 )
+
+# Having the configure-vm.sh script and and trusty code from the GCE cluster
+# deploy hosted with the release is useful for GKE.
+# This list should match the list in kubernetes/release/lib/releaselib.sh.
+release_filegroup(
+    name = "gcs-release-artifacts",
+    srcs = [
+        "configure-vm.sh",
+        "gci/configure.sh",
+        "gci/master.yaml",
+        "gci/node.yaml",
+    ],
+)