diff --git a/Gopkg.lock b/Gopkg.lock index d949c5e23888..e0174308a7bd 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -444,7 +444,7 @@ [[projects]] branch = "master" - digest = "1:f36cb404cd23f6fc1fd4e85e538468658ce991ba73fd1425bd0d1a65de4c3374" + digest = "1:cf1ee21f7f6a2db7859a26961f445536ac2447379ddf8e775ee32c9aa59911d8" name = "github.com/knative/test-infra" packages = [ "scripts", @@ -453,7 +453,7 @@ "tools/testgrid", ] pruneopts = "UT" - revision = "2377fbc40f81501f2bc09a4e1e359a7ec7c3528e" + revision = "8a63be3b3795aa2f7ae9be32d840e096ba877c78" [[projects]] digest = "1:56dbf15e091bf7926cb33a57cb6bdfc658fc6d3498d2f76f10a97ce7856f1fde" @@ -1263,6 +1263,9 @@ "github.com/knative/test-infra/tools/testgrid", "github.com/mattbaird/jsonpatch", "github.com/pkg/errors", + "github.com/prometheus/client_golang/api", + "github.com/prometheus/client_golang/api/prometheus/v1", + "github.com/prometheus/common/model", "go.opencensus.io/exporter/prometheus", "go.opencensus.io/stats", "go.opencensus.io/stats/view", diff --git a/hack/release.md b/hack/release.md index d2882e5e0bd0..ea0f62af8bff 100644 --- a/hack/release.md +++ b/hack/release.md @@ -3,7 +3,8 @@ The `release.sh` script automates the creation of Knative Serving releases, either nightly or versioned ones. -By default, the script creates a nightly release but does not publish anywhere. +By default, the script creates a nightly release but does not publish it +anywhere. ## Common flags for cutting releases @@ -16,12 +17,16 @@ release. - `--tag-release`, `--notag-release` Tag (or not) the generated images with either `vYYYYMMDD-` (for nightly releases) or `vX.Y.Z` for versioned releases. _For versioned releases, a tag is always added._ +- `--release-gcs` Defines the GCS bucket where the manifests will be stored. + By default, this is `knative-nightly/serving`. This flag is ignored if the + release is not being published. +- `--release-gcr` Defines the GCR where the images will be stored. By default, + this is `gcr.io/knative-nightly`. This flag is ignored if the release is not + being published. - `--publish`, `--nopublish` Whether the generated images should be published to a GCR, and the generated manifests written to a GCS bucket or not. If yes, the - destination GCR is defined by the environment variable `$SERVING_RELEASE_GCR` - (defaults to `gcr.io/knative-nightly`) and the destination GCS bucket is - defined by the environment variable `$SERVING_RELEASE_GCS` (defaults to - `knative-nightly/serving`). If no, the images will be pushed to the `ko.local` + `--release-gcs` and `--release-gcr` flags can be used to change the default + values of the GCR/GCS used. If no, the images will be pushed to the `ko.local` registry, and the manifests written to the local disk only (in the repository root directory). diff --git a/hack/release.sh b/hack/release.sh index 0b3ac8121734..259cf2baea5d 100755 --- a/hack/release.sh +++ b/hack/release.sh @@ -16,15 +16,6 @@ source $(dirname $0)/../vendor/github.com/knative/test-infra/scripts/release.sh -# Set default GCS/GCR -: ${SERVING_RELEASE_GCS:="knative-nightly/serving"} -: ${SERVING_RELEASE_GCR:="gcr.io/knative-nightly"} -readonly SERVING_RELEASE_GCS -readonly SERVING_RELEASE_GCR - -# Set the repository -export KO_DOCKER_REPO="${SERVING_RELEASE_GCR}" - # Script entry point initialize $@ @@ -34,6 +25,8 @@ set -o pipefail run_validation_tests ./test/presubmit-tests.sh +# Build the release + banner "Building the release" # Build should not try to deploy anything, use a bogus value for cluster. @@ -41,13 +34,6 @@ export K8S_CLUSTER_OVERRIDE=CLUSTER_NOT_SET export K8S_USER_OVERRIDE=USER_NOT_SET export DOCKER_REPO_OVERRIDE=DOCKER_NOT_SET -echo "- Destination GCR: ${KO_DOCKER_REPO}" -if (( PUBLISH_RELEASE )); then - echo "- Destination GCS: ${SERVING_RELEASE_GCS}" -fi - -# Build the release - # Run `generate-yamls.sh`, which should be versioned with the # branch since the detail of building may change over time. readonly YAML_LIST="$(mktemp)" @@ -55,7 +41,7 @@ $(dirname $0)/generate-yamls.sh "${REPO_ROOT_DIR}" "${YAML_LIST}" readonly YAMLS_TO_PUBLISH=$(cat "${YAML_LIST}" | tr '\n' ' ') readonly RELEASE_YAML="$(head -n1 ${YAML_LIST})" -tag_images_in_yaml "${RELEASE_YAML}" "${KO_DOCKER_REPO}" "${TAG}" +tag_images_in_yaml "${RELEASE_YAML}" echo "New release built successfully" @@ -68,8 +54,7 @@ fi # Publish the release # We publish our own istio.yaml, so users don't need to use helm for yaml in ${YAMLS_TO_PUBLISH}; do - echo "Publishing ${yaml}" - publish_yaml "${yaml}" "${SERVING_RELEASE_GCS}" "${TAG}" + publish_yaml "${yaml}" done branch_release "Knative Serving" "${YAMLS_TO_PUBLISH}" diff --git a/vendor/github.com/knative/test-infra/scripts/README.md b/vendor/github.com/knative/test-infra/scripts/README.md index 0e7ab5726eda..3c6c9aacb5f0 100644 --- a/vendor/github.com/knative/test-infra/scripts/README.md +++ b/vendor/github.com/knative/test-infra/scripts/README.md @@ -163,8 +163,14 @@ also overrides the value of the `TAG` variable as `v`. it's empty and `master` HEAD will be considered the release branch. * `RELEASE_NOTES`: contains the filename with the release notes if `--release-notes` was passed. The release notes is a simple markdown file. + * `RELEASE_GCS_BUCKET`: contains the GCS bucket name to store the manifests if +`--release-gcs` was passed, otherwise the default value `knative-nightly/` will be +used. It is empty if `--publish` was not passed. + * `KO_DOCKER_REPO`: contains the GCR to store the images if `--release-gcr` was +passed, otherwise the default value `gcr.io/knative-nightly` will be used. It is set to +`ko.local` if `--publish` was not passed. * `SKIP_TESTS`: true if `--skip-tests` was passed. This is handled automatically -by the run_validation_tests() function. +by the `run_validation_tests()` function. * `TAG_RELEASE`: true if `--tag-release` was passed. In this case, the environment variable `TAG` will contain the release tag in the form `vYYYYMMDD-`. * `PUBLISH_RELEASE`: true if `--publish` was passed. In this case, the environment @@ -186,14 +192,12 @@ initialize $@ run_validation_tests ./test/presubmit-tests.sh # config/ contains the manifests -KO_DOCKER_REPO=gcr.io/knative-foo ko resolve ${KO_FLAGS} -f config/ > release.yaml -tag_images_in_yaml release.yaml $KO_DOCKER_REPO $TAG +tag_images_in_yaml release.yaml if (( PUBLISH_RELEASE )); then - # gs://knative-foo hosts the manifest - publish_yaml release.yaml knative-foo $TAG + publish_yaml release.yaml fi branch_release "Knative Foo" release.yaml diff --git a/vendor/github.com/knative/test-infra/scripts/library.sh b/vendor/github.com/knative/test-infra/scripts/library.sh index ba267d97cd66..5db1fab6092d 100755 --- a/vendor/github.com/knative/test-infra/scripts/library.sh +++ b/vendor/github.com/knative/test-infra/scripts/library.sh @@ -100,8 +100,8 @@ function wait_until_object_does_not_exist() { fi echo -n "Waiting until ${DESCRIPTION} does not exist" for i in {1..150}; do # timeout after 5 minutes - if kubectl ${KUBECTL_ARGS} > /dev/null 2>&1; then - echo "\n${DESCRIPTION} does not exist" + if ! kubectl ${KUBECTL_ARGS} > /dev/null 2>&1; then + echo -e "\n${DESCRIPTION} does not exist" return 0 fi echo -n "." @@ -265,10 +265,11 @@ function report_go_test() { local field0="${fields[0]}" local field1="${fields[1]}" local name="${fields[2]}" - # Deal with a SIGQUIT log entry (usually a test timeout). + # Deal with a SIGQUIT or panic log entry (usually a test timeout). # This is a fallback in case there's no kill signal log entry. # SIGQUIT: quit - if [[ "${field0}" == "SIGQUIT:" ]]; then + # panic: test timed out after 5m0s + if [[ "${field0}" == "SIGQUIT:" || "${field0}" == "panic:" ]]; then name="${last_run}" field1="FAIL:" error="${fields[@]}" diff --git a/vendor/github.com/knative/test-infra/scripts/release.sh b/vendor/github.com/knative/test-infra/scripts/release.sh index cbde3c64289c..71551d11365c 100755 --- a/vendor/github.com/knative/test-infra/scripts/release.sh +++ b/vendor/github.com/knative/test-infra/scripts/release.sh @@ -25,27 +25,32 @@ function banner() { make_banner "@" "$1" } -# Tag images in the yaml file with a tag. If not tag is passed, does nothing. +# Tag images in the yaml file if $TAG is not empty. +# $KO_DOCKER_REPO is the registry containing the images to tag with $TAG. # Parameters: $1 - yaml file to parse for images. -# $2 - registry where the images are stored. -# $3 - tag to apply (optional). function tag_images_in_yaml() { - [[ -z $3 ]] && return 0 - local src_dir="${GOPATH}/src/" - local BASE_PATH="${REPO_ROOT_DIR/$src_dir}" - echo "Tagging images under '${BASE_PATH}' with $3" - for image in $(grep -o "$2/${BASE_PATH}/[a-z\./-]\+@sha256:[0-9a-f]\+" $1); do - gcloud -q container images add-tag ${image} ${image%%@*}:$3 + [[ -z ${TAG} ]] && return 0 + local SRC_DIR="${GOPATH}/src/" + local DOCKER_BASE="${KO_DOCKER_REPO}/${REPO_ROOT_DIR/$SRC_DIR}" + echo "Tagging images under '${DOCKER_BASE}' with ${TAG}" + for image in $(grep -o "${DOCKER_BASE}/[a-z\./-]\+@sha256:[0-9a-f]\+" $1); do + gcloud -q container images add-tag ${image} ${image%%@*}:${TAG} done } -# Copy the given yaml file to a GCS bucket. Image is tagged :latest, and optionally :$2. +# Copy the given yaml file to the $RELEASE_GCS_BUCKET bucket's "latest" directory. +# If $TAG is not empty, also copy it to $RELEASE_GCS_BUCKET bucket's "previous" directory. # Parameters: $1 - yaml file to copy. -# $2 - destination bucket name. -# $3 - tag to apply (optional). function publish_yaml() { - gsutil cp $1 gs://$2/latest/ - [[ -n $3 ]] && gsutil cp $1 gs://$2/previous/$3/ || true + function verbose_gsutil_cp { + local DEST=gs://${RELEASE_GCS_BUCKET}/$2/ + echo "Publishing $1 to ${DEST}" + gsutil cp $1 ${DEST} + } + verbose_gsutil_cp $1 latest + if [[ -n ${TAG} ]]; then + verbose_gsutil_cp $1 previous/${TAG} + fi } # These are global environment variables. @@ -57,7 +62,9 @@ TAG="" RELEASE_VERSION="" RELEASE_NOTES="" RELEASE_BRANCH="" +RELEASE_GCS_BUCKET="" KO_FLAGS="" +export KO_DOCKER_REPO="" function abort() { echo "error: $@" @@ -71,6 +78,11 @@ function parse_flags() { RELEASE_NOTES="" RELEASE_BRANCH="" KO_FLAGS="-P" + KO_DOCKER_REPO="gcr.io/knative-nightly" + RELEASE_GCS_BUCKET="knative-nightly/$(basename ${REPO_ROOT_DIR})" + local has_gcr_flag=0 + local has_gcs_flag=0 + cd ${REPO_ROOT_DIR} while [[ $# -ne 0 ]]; do local parameter=$1 @@ -80,6 +92,18 @@ function parse_flags() { --notag-release) TAG_RELEASE=0 ;; --publish) PUBLISH_RELEASE=1 ;; --nopublish) PUBLISH_RELEASE=0 ;; + --release-gcr) + shift + [[ $# -ge 1 ]] || abort "missing GCR after --release-gcr" + KO_DOCKER_REPO=$1 + has_gcr_flag=1 + ;; + --release-gcs) + shift + [[ $# -ge 1 ]] || abort "missing GCS bucket after --release-gcs" + RELEASE_GCS_BUCKET=$1 + has_gcs_flag=1 + ;; --version) shift [[ $# -ge 1 ]] || abort "missing version after --version" @@ -105,8 +129,11 @@ function parse_flags() { # Update KO_DOCKER_REPO and KO_FLAGS if we're not publishing. if (( ! PUBLISH_RELEASE )); then + (( has_gcr_flag )) && echo "Not publishing the release, GCR flag is ignored" + (( has_gcs_flag )) && echo "Not publishing the release, GCS flag is ignored" KO_DOCKER_REPO="ko.local" KO_FLAGS="-L ${KO_FLAGS}" + RELEASE_GCS_BUCKET="" fi if (( TAG_RELEASE )); then @@ -131,6 +158,8 @@ function parse_flags() { readonly RELEASE_VERSION readonly RELEASE_NOTES readonly RELEASE_BRANCH + readonly RELEASE_GCS_BUCKET + readonly KO_DOCKER_REPO } # Run tests (unless --skip-tests was passed). Conveniently displays a banner indicating so. @@ -149,6 +178,26 @@ function run_validation_tests() { # Initialize everything (flags, workspace, etc) for a release. function initialize() { parse_flags $@ + + # Log what will be done and where. + banner "Release configuration" + echo "- Destination GCR: ${KO_DOCKER_REPO}" + (( SKIP_TESTS )) && echo "- Tests will NOT be run" || echo "- Tests will be run" + if (( TAG_RELEASE )); then + echo "- Artifacts will tagged '${TAG}'" + else + echo "- Artifacts WILL NOT be tagged" + fi + if (( PUBLISH_RELEASE )); then + echo "- Release WILL BE published to '${RELEASE_GCS_BUCKET}'" + else + echo "- Release will not be published" + fi + if (( BRANCH_RELEASE )); then + echo "- Release WILL BE branched from '${RELEASE_BRANCH}'" + fi + [[ -n "${RELEASE_NOTES}" ]] && echo "- Release notes are generated from '${RELEASE_NOTES}'" + # Checkout specific branch, if necessary if (( BRANCH_RELEASE )); then git checkout upstream/${RELEASE_BRANCH} || abort "cannot checkout branch ${RELEASE_BRANCH}"