Container build improvements (istio#17336)

- Producing all the genersted sources is now done using 'make gen' which runs the build steps within the build-tools container. - 'make fmt' now runs formatting steps from the build-tools container. - Update common files and common protos.
lcdevelop · Sep 25, 2019 · 9e2f4c8 · 9e2f4c8
1 parent 3e7d91e
commit 9e2f4c8
Show file tree

Hide file tree

Showing 118 changed files with 238 additions and 371 deletions.
diff --git a/Makefile b/Makefile
@@ -28,56 +28,55 @@
 # figure out all the tools you need in your environment to make that work.
 export BUILD_WITH_CONTAINER ?= 0
 
-ifeq ($(BUILD_WITH_CONTAINER),1)
-CONTAINER_CLI ?= docker
-DOCKER_SOCKET_MOUNT ?= -v /var/run/docker.sock:/var/run/docker.sock
-IMG ?= gcr.io/istio-testing/build-tools:2019-09-20T15-04-58
-UID = $(shell id -u)
-PWD = $(shell pwd)
-
 LOCAL_ARCH := $(shell uname -m)
 ifeq ($(LOCAL_ARCH),x86_64)
-GOARCH_LOCAL := amd64
+    TARGET_ARCH ?= amd64
 else ifeq ($(shell echo $(LOCAL_ARCH) | head -c 5),armv8)
-GOARCH_LOCAL := arm64
+    TARGET_ARCH ?= arm64
 else ifeq ($(shell echo $(LOCAL_ARCH) | head -c 4),armv)
-GOARCH_LOCAL := arm
+    TARGET_ARCH ?= arm
 else
-GOARCH_LOCAL := $(LOCAL_ARCH)
+   $(error "This system's architecture $(LOCAL_ARCH) isn't recognized/supported")
 endif
 
-GOARCH ?= $(GOARCH_LOCAL)
-
 LOCAL_OS := $(shell uname)
 ifeq ($(LOCAL_OS),Linux)
-   GOOS_LOCAL = linux
-   # Apparently Linux needs a -f flag.
+   TARGET_OS ?= linux
    READLINK_FLAGS="-f"
 else ifeq ($(LOCAL_OS),Darwin)
-   GOOS_LOCAL = darwin
+   TARGET_OS ?= darwin
    READLINK_FLAGS=""
 else
    $(error "This system's OS $(LOCAL_OS) isn't recognized/supported")
 endif
 
+REPO_ROOT = $(shell git rev-parse --show-toplevel)
+REPO_NAME = $(shell basename $(REPO_ROOT))
+TARGET_OUT ?= $(HOME)/istio_out/$(REPO_NAME)
+
+ifeq ($(BUILD_WITH_CONTAINER),1)
+CONTAINER_CLI ?= docker
+DOCKER_SOCKET_MOUNT ?= -v /var/run/docker.sock:/var/run/docker.sock
+IMG ?= gcr.io/istio-testing/build-tools:2019-09-23T12-48-14
+UID = $(shell id -u)
+PWD = $(shell pwd)
+
 # Determine the timezone across various platforms to pass into the
 # docker run operation. This operation assumes zoneinfo is within
 # the path of the file.
 TIMEZONE=`readlink $(READLINK_FLAGS) /etc/localtime | sed -e 's/^.*zoneinfo\///'`
 
-GOOS ?= $(GOOS_LOCAL)
-
 RUN = $(CONTAINER_CLI) run -t -i --sig-proxy=true -u $(UID) --rm \
-	-e GOOS="$(GOOS)" \
-	-e GOARCH="$(GOARCH)" \
 	-e BUILD_WITH_CONTAINER="$(BUILD_WITH_CONTAINER)" \
 	-e TZ="$(TIMEZONE)" \
+	-e TARGET_ARCH="$(TARGET_ARCH)" \
+	-e TARGET_OS="$(TARGET_OS)" \
 	-v /etc/passwd:/etc/passwd:ro \
 	$(DOCKER_SOCKET_MOUNT) \
 	$(CONTAINER_OPTIONS) \
 	--mount type=bind,source="$(PWD)",destination="/work" \
-	--mount type=volume,source=istio-go-mod,destination="/go/pkg/mod" \
-	--mount type=volume,source=istio-go-cache,destination="/gocache" \
+	--mount type=bind,source="$(TARGET_OUT)",destination="/targetout" \
+	--mount type=volume,source=home,destination="/home" \
 	-w /work $(IMG)
 else
 RUN =
@@ -86,9 +85,11 @@ endif
 MAKE = $(RUN) make --no-print-directory -e -f Makefile.core.mk
 
 %:
+	@mkdir -p $(TARGET_OUT)
 	@$(MAKE) $@
 
 default:
+	@mkdir -p $(TARGET_OUT)
 	@$(MAKE)
 
 .PHONY: default
diff --git a/Makefile.core.mk b/Makefile.core.mk
@@ -141,6 +141,7 @@ export ISTIO_OUT:=$(OUT_DIR)/$(GOOS)_$(GOARCH)/$(BUILDTYPE_DIR)
 export ISTIO_OUT_LINUX:=$(OUT_DIR)/linux_amd64/$(BUILDTYPE_DIR)
 export HELM=$(ISTIO_OUT)/helm
 export ARTIFACTS ?= $(ISTIO_OUT)
+export REPO_ROOT := $(shell git rev-parse --show-toplevel)
 
 # scratch dir: this shouldn't be simply 'docker' since that's used for docker.save to store tar.gz files
 ISTIO_DOCKER:=${ISTIO_OUT_LINUX}/docker_temp
@@ -321,12 +322,9 @@ ${GEN_CERT}:
 # If pre-commit script is not used, please run this manually.
 precommit: format lint
 
-format:
-	scripts/run_gofmt.sh
-	go mod tidy
+format: fmt
 
-fmt:
-	scripts/run_gofmt.sh
+fmt: format-go format-python
 	go mod tidy
 
 # Build with -i to store the build caches into $GOPATH/pkg
@@ -379,10 +377,10 @@ MARKDOWN_LINT_WHITELIST=localhost:8080,storage.googleapis.com/istio-artifacts/pi
 
 lint_modern: lint-python lint-copyright-banner lint-scripts lint-dockerfiles lint-markdown lint-yaml
 	@bin/check_helm.sh
-	@GOARCH=amd64 GOOS=linux bin/check_samples.sh
-	@GOARCH=amd64 GOOS=linux bin/check_dashboards.sh
-	@GOARCH=amd64 GOOS=linux go run mixer/tools/adapterlinter/main.go ./mixer/adapter/...
-	@GOARCH=amd64 GOOS=linux bin/check_pilot_codegen.sh
+	@bin/check_samples.sh
+	@bin/check_dashboards.sh
+	@go run mixer/tools/adapterlinter/main.go ./mixer/adapter/...
+	@bin/check_pilot_codegen.sh
 	@golangci-lint run -c ./common/config/.golangci.yml ./galley/...
 	@golangci-lint run -c ./common/config/.golangci.yml ./istioctl/...
 	@golangci-lint run -c ./common/config/.golangci.yml ./mixer/...
@@ -396,6 +394,11 @@ lint_modern: lint-python lint-copyright-banner lint-scripts lint-dockerfiles lin
 	@testlinter
 	@envvarlinter galley istioctl mixer pilot security sidecar-injector
 
+gen:
+	@mkdir -p /tmp/bin
+	@go build -o /tmp/bin/mixgen "${REPO_ROOT}/mixer/tools/mixgen/main.go"
+	@PATH=${PATH}:/tmp/bin go generate ./...
+
 #-----------------------------------------------------------------------------
 # Target: go build
 #-----------------------------------------------------------------------------

diff --git a/bin/counterfeiter.sh b/bin/counterfeiter.sh
diff --git a/bin/mixer_codegen.sh b/bin/mixer_codegen.sh
@@ -63,11 +63,6 @@ while getopts ':f:o:p:i:t:a:d:x:' flag; do
   esac
 done
 
-# Ensure expected GOPATH setup
-if [ "$ROOTDIR" != "${GOPATH-$HOME/go}/src/istio.io/istio" ]; then
-  die "Istio not found in GOPATH/src/istio.io/"
-fi
-
 IMPORTS=(
   "--proto_path=${ROOTDIR}"
   "--proto_path=$optimport"
@@ -144,7 +139,7 @@ if [ "$opttemplate" = true ]; then
     die "template generation failure: $err";
   fi
 
-  go run "$GOPATH/src/istio.io/istio/mixer/tools/mixgen/main.go" api -t "$templateDS" --go_out "$templateHG" --proto_out "$templateHSP" "${TMPL_GEN_MAP[@]}"
+  mixgen api -t "$templateDS" --go_out "$templateHG" --proto_out "$templateHSP" "${TMPL_GEN_MAP[@]}"
 
   err=$($protoc "${IMPORTS[@]}" "$TMPL_PLUGIN" "$templateHSP")
   if [ -n "$err" ]; then
@@ -163,7 +158,7 @@ if [ "$opttemplate" = true ]; then
   fi
 
   templateYaml=${template/.proto/.yaml}
-  go run "$GOPATH/src/istio.io/istio/mixer/tools/mixgen/main.go" template -d "$templateSDS" -o "$templateYaml" -n "$(basename "$(dirname "${template}")")"
+  mixgen template -d "$templateSDS" -o "$templateYaml" -n "$(basename "$(dirname "${template}")")"
 
   rm "$templatePG"
 
@@ -188,7 +183,7 @@ if [ "$optadapter" = true ]; then
   fi
 
   IFS=" " read -r -a extraflags_array <<< "$extraflags"
-  go run "$GOPATH/src/istio.io/istio/mixer/tools/mixgen/main.go" adapter -c "$adapteCfdDS" -o "$(dirname "${file}")" "${extraflags_array[@]}"
+  mixgen adapter -c "$adapteCfdDS" -o "$(dirname "${file}")" "${extraflags_array[@]}"
 
   exit 0
 fi

diff --git a/bin/protoc.sh b/bin/protoc.sh
@@ -19,22 +19,6 @@ if [[ $# -le 0 ]]; then
     exit 1
 fi
 
-SCRIPTPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-ROOTDIR="$(dirname "$SCRIPTPATH")"
-
 api=$(go list -m -f "{{.Dir}}" istio.io/api)
 
-gen_img=gcr.io/istio-testing/build-tools:2019-09-04T21-28-42
-
-docker run \
-  -i \
-  --rm \
-  -v "$ROOTDIR:$ROOTDIR" \
-  -v "${api}:/protos/istio.io/api" \
-  -v "$ROOTDIR"/common-protos:/protos \
-  -w "$(pwd)" \
-  $gen_img \
-  /usr/bin/protoc \
-  -I/protos \
-  -I/protos/istio.io/api \
-  "$@"
+protoc -I"${REPO_ROOT}"/common-protos -I"${api}" "$@"
diff --git a/common-protos/.commonfiles.sha b/common-protos/.commonfiles.sha
@@ -1 +1 @@
-8f666ddd60790734388e89a29932014a632ebada
+495981b2386daebbf27c170f9034193fc88ae8ff
diff --git a/common/.commonfiles.sha b/common/.commonfiles.sha
@@ -1 +1 @@
-8f666ddd60790734388e89a29932014a632ebada
+b911e0db0ccb611ae4fd361d295af2ef5f8089d7
diff --git a/galley/pkg/config/analysis/msg/messages.go b/galley/pkg/config/analysis/msg/messages.go
@@ -15,6 +15,6 @@
 package msg
 
 // Create static initializers file
-//go:generate go run "$GOPATH/src/istio.io/istio/galley/pkg/config/analysis/msg/generate.main.go" messages.yaml messages.gen.go
+//go:generate go run "$REPO_ROOT/galley/pkg/config/analysis/msg/generate.main.go" messages.yaml messages.gen.go
 
-//go:generate goimports -w -local istio.io "$GOPATH/src/istio.io/istio/galley/pkg/config/analysis/msg/messages.gen.go"
+//go:generate goimports -w -local istio.io "$REPO_ROOT/galley/pkg/config/analysis/msg/messages.gen.go"
diff --git a/galley/pkg/config/processor/metadata/gen.go b/galley/pkg/config/processor/metadata/gen.go
@@ -14,15 +14,14 @@
 
 package metadata
 
-// TODO: Switch go-bindata to be scripts/run_gobindata.sh
 // Embed the core metadata file containing the collections as a resource
-//go:generate $GOPATH/src/istio.io/istio/scripts/run_gobindata.sh --nocompress --nometadata --pkg metadata -o metadata.gen.go metadata.yaml
+//go:generate go-bindata --nocompress --nometadata --pkg metadata -o metadata.gen.go metadata.yaml
 
 // Create static initializers file
-//go:generate go run $GOPATH/src/istio.io/istio/galley/pkg/config/schema/codegen/tools/staticinit.main.go metadata metadata.yaml staticinit.gen.go
+//go:generate go run $REPO_ROOT/galley/pkg/config/schema/codegen/tools/staticinit.main.go metadata metadata.yaml staticinit.gen.go
 
 // Create collection constants
-//go:generate go run $GOPATH/src/istio.io/istio/galley/pkg/config/schema/codegen/tools/collections.main.go metadata metadata.yaml collections.gen.go
+//go:generate go run $REPO_ROOT/galley/pkg/config/schema/codegen/tools/collections.main.go metadata metadata.yaml collections.gen.go
 
-//go:generate goimports -w -local istio.io "$GOPATH/src/istio.io/istio/galley/pkg/config/processor/metadata/collections.gen.go"
-//go:generate goimports -w -local istio.io "$GOPATH/src/istio.io/istio/galley/pkg/config/processor/metadata/staticinit.gen.go"
+//go:generate goimports -w -local istio.io "$REPO_ROOT/galley/pkg/config/processor/metadata/collections.gen.go"
+//go:generate goimports -w -local istio.io "$REPO_ROOT/galley/pkg/config/processor/metadata/staticinit.gen.go"
diff --git a/galley/pkg/config/processor/metadata/metadata.gen.go b/galley/pkg/config/processor/metadata/metadata.gen.go
diff --git a/galley/pkg/config/testing/basicmeta/gen.go b/galley/pkg/config/testing/basicmeta/gen.go
@@ -15,13 +15,13 @@
 package basicmeta
 
 // Embed the core metadata file containing the collections as a resource
-//go:generate $GOPATH/src/istio.io/istio/scripts/run_gobindata.sh --nocompress --nometadata --pkg basicmeta -o basicmeta.gen.go basicmeta.yaml basicmeta2.yaml
+//go:generate go-bindata --nocompress --nometadata --pkg basicmeta -o basicmeta.gen.go basicmeta.yaml basicmeta2.yaml
 
 // Create static initializers file
-//go:generate go run $GOPATH/src/istio.io/istio/galley/pkg/config/schema/codegen/tools/staticinit.main.go basicmeta basicmeta.yaml staticinit.gen.go
+//go:generate go run $REPO_ROOT/galley/pkg/config/schema/codegen/tools/staticinit.main.go basicmeta basicmeta.yaml staticinit.gen.go
 
 // Create collection constants
-//go:generate go run $GOPATH/src/istio.io/istio/galley/pkg/config/schema/codegen/tools/collections.main.go basicmeta basicmeta.yaml collections.gen.go
+//go:generate go run $REPO_ROOT/galley/pkg/config/schema/codegen/tools/collections.main.go basicmeta basicmeta.yaml collections.gen.go
 
-//go:generate goimports -w -local istio.io "$GOPATH/src/istio.io/istio/galley/pkg/config/testing/basicmeta/collections.gen.go"
-//go:generate goimports -w -local istio.io "$GOPATH/src/istio.io/istio/galley/pkg/config/testing/basicmeta/staticinit.gen.go"
+//go:generate goimports -w -local istio.io "$REPO_ROOT/galley/pkg/config/testing/basicmeta/collections.gen.go"
+//go:generate goimports -w -local istio.io "$REPO_ROOT/galley/pkg/config/testing/basicmeta/staticinit.gen.go"
diff --git a/galley/pkg/config/testing/data/builtin.go b/galley/pkg/config/testing/data/builtin.go
@@ -13,7 +13,7 @@
 // limitations under the License.
 
 // Embed the core metadata file containing the collections as a resource
-//go:generate $GOPATH/src/istio.io/istio/scripts/run_gobindata.sh --nocompress --nometadata --pkg data -o builtin.gen.go builtin/
+//go:generate go-bindata --nocompress --nometadata --pkg data -o builtin.gen.go builtin/
 
 package data
 

diff --git a/galley/pkg/config/testing/k8smeta/gen.go b/galley/pkg/config/testing/k8smeta/gen.go
@@ -15,13 +15,13 @@
 package k8smeta
 
 // Embed the core metadata file containing the collections as a resource
-//go:generate $GOPATH/src/istio.io/istio/scripts/run_gobindata.sh --nocompress --nometadata --pkg k8smeta -o k8smeta.gen.go  k8smeta.yaml
+//go:generate go-bindata --nocompress --nometadata --pkg k8smeta -o k8smeta.gen.go  k8smeta.yaml
 
 // Create static initializers file
-//go:generate go run $GOPATH/src/istio.io/istio/galley/pkg/config/schema/codegen/tools/staticinit.main.go k8smeta k8smeta.yaml staticinit.gen.go
+//go:generate go run $REPO_ROOT/galley/pkg/config/schema/codegen/tools/staticinit.main.go k8smeta k8smeta.yaml staticinit.gen.go
 
 // Create collection constants
-//go:generate go run $GOPATH/src/istio.io/istio/galley/pkg/config/schema/codegen/tools/collections.main.go k8smeta k8smeta.yaml collections.gen.go
+//go:generate go run $REPO_ROOT/galley/pkg/config/schema/codegen/tools/collections.main.go k8smeta k8smeta.yaml collections.gen.go
 
-//go:generate goimports -w -local istio.io "$GOPATH/src/istio.io/istio/galley/pkg/config/testing/k8smeta/collections.gen.go"
-//go:generate goimports -w -local istio.io "$GOPATH/src/istio.io/istio/galley/pkg/config/testing/k8smeta/staticinit.gen.go"
+//go:generate goimports -w -local istio.io "$REPO_ROOT/galley/pkg/config/testing/k8smeta/collections.gen.go"
+//go:generate goimports -w -local istio.io "$REPO_ROOT/galley/pkg/config/testing/k8smeta/staticinit.gen.go"
diff --git a/galley/pkg/metadata/kube/types.gen.go b/galley/pkg/metadata/kube/types.gen.go
diff --git a/galley/pkg/metadata/types.gen.go b/galley/pkg/metadata/types.gen.go
diff --git a/galley/testdata/conversion/dataset.go b/galley/testdata/conversion/dataset.go
@@ -14,4 +14,4 @@
 
 package conversion
 
-//go:generate $GOPATH/src/istio.io/istio/scripts/run_gobindata.sh --nocompress --nometadata --pkg conversion -o dataset.gen.go dataset/...
+//go:generate go-bindata --nocompress --nometadata --pkg conversion -o dataset.gen.go dataset/...
diff --git a/galley/testdata/validation/dataset.go b/galley/testdata/validation/dataset.go
@@ -14,5 +14,5 @@
 
 package validation
 
-//go:generate $GOPATH/src/istio.io/istio/scripts/run_gobindata.sh --nocompress --nometadata --pkg validation -o dataset.gen.go dataset/...
+//go:generate go-bindata --nocompress --nometadata --pkg validation -o dataset.gen.go dataset/...
 //go:generate goimports -w dataset.gen.go
diff --git a/galley/tools/gen-meta/main.go b/galley/tools/gen-meta/main.go
@@ -206,7 +206,7 @@ func readMetadata(path string) (*metadata, error) {
 const runtimeTemplate = `
 // GENERATED FILE -- DO NOT EDIT
 //
-//go:generate $GOPATH/src/istio.io/istio/galley/tools/gen-meta/gen-meta.sh runtime pkg/metadata/types.gen.go
+//go:generate $REPO_ROOT/galley/tools/gen-meta/gen-meta.sh runtime pkg/metadata/types.gen.go
 //
 
 package metadata
@@ -246,7 +246,7 @@ func init() {
 const kubeTemplate = `
 // GENERATED FILE -- DO NOT EDIT
 //
-//go:generate $GOPATH/src/istio.io/istio/galley/tools/gen-meta/gen-meta.sh kube pkg/metadata/kube/types.go
+//go:generate $REPO_ROOT/galley/tools/gen-meta/gen-meta.sh kube pkg/metadata/kube/types.gen.go
 //
 
 package kube

diff --git a/mixer/adapter/bypass/bypass.go b/mixer/adapter/bypass/bypass.go
@@ -13,7 +13,7 @@
 // limitations under the License.
 
 // nolint: lll
-//go:generate $GOPATH/src/istio.io/istio/bin/mixer_codegen.sh -a mixer/adapter/bypass/config/config.proto -x "-n bypass -t checknothing -t reportnothing -t metric -t quota"
+//go:generate $REPO_ROOT/bin/mixer_codegen.sh -a mixer/adapter/bypass/config/config.proto -x "-n bypass -t checknothing -t reportnothing -t metric -t quota"
 
 package bypass
 

diff --git a/mixer/adapter/bypass/config/bypass.yaml b/mixer/adapter/bypass/config/bypass.yaml
@@ -1,5 +1,5 @@
 # this config is created through command
-# mixgen adapter -c $GOPATH/src/istio.io/istio/mixer/adapter/bypass/config/config.proto_descriptor -o $GOPATH/src/istio.io/istio/mixer/adapter/bypass/config -n bypass -t checknothing -t reportnothing -t metric -t quota
+# mixgen adapter -c $REPO_ROOT/mixer/adapter/bypass/config/config.proto_descriptor -o $REPO_ROOT/mixer/adapter/bypass/config -n bypass -t checknothing -t reportnothing -t metric -t quota
 apiVersion: "config.istio.io/v1alpha2"
 kind: adapter
 metadata:

diff --git a/mixer/adapter/circonus/circonus.go b/mixer/adapter/circonus/circonus.go
@@ -13,7 +13,7 @@
 // limitations under the License.
 
 // nolint: lll
-//go:generate $GOPATH/src/istio.io/istio/bin/mixer_codegen.sh -a mixer/adapter/circonus/config/config.proto -x "-n circonus -t metric"
+//go:generate $REPO_ROOT/bin/mixer_codegen.sh -a mixer/adapter/circonus/config/config.proto -x "-n circonus -t metric"
 
 package circonus
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		8f666ddd60790734388e89a29932014a632ebada
		495981b2386daebbf27c170f9034193fc88ae8ff
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		8f666ddd60790734388e89a29932014a632ebada
		b911e0db0ccb611ae4fd361d295af2ef5f8089d7
Original file line number	Diff line number	Diff line change
Expand Up		@@ -14,4 +14,4 @@

		package conversion

		//go:generate $GOPATH/src/istio.io/istio/scripts/run_gobindata.sh --nocompress --nometadata --pkg conversion -o dataset.gen.go dataset/...
		//go:generate go-bindata --nocompress --nometadata --pkg conversion -o dataset.gen.go dataset/...