Set BUILD_WITH_CONTAINER=1 for this repo. (istio#211)

- Simplify how protos are compiled such that it all works
cleanly from the container. This no longer uses gogo for code
generation of the operator's protos, which makes the code
readily compatible with all the generated protos from k8s.

- Update common files as part of this.

- Rename some generated file to have the .gen.go suffix which
is our convention.

.commonfiles.sha

@@ -1 +1 @@
-20862f1d31af6016f2c99605d9aa59034328348b
+2f9a3a44c34460dca4c77c1db1feebb9153b8692

.golangci.yml

@@ -3,10 +3,10 @@
 # The original version of this file is located in the https://github.com/istio/common-files repo.
 # If you're looking at this file in a different repo and want to make a change, please go to the
 # common-files repo, make the change there and check it in. Then come back to this repo and run
-# "make updatecommon".
+# "make update-common".
 
 service:
-  # When updating this, also update bin/linters.sh accordingly
+  # When updating this, also update the version stored in docker/build-tools/Dockerfile in the istio/tools repo.
   golangci-lint-version: 1.16.x # use the fixed version to not introduce new linters unexpectedly
 run:
   # timeout for analysis, e.g. 30s, 5m, default is 1m

.hadolint.yml

@@ -0,0 +1,12 @@
+# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY
+#
+# The original version of this file is located in the https://github.com/istio/common-files repo.
+# If you're looking at this file in a different repo and want to make a change, please go to the
+# common-files repo, make the change there and check it in. Then come back to this repo and run
+# "make update-common".
+
+ignored:
+
+trustedRegistries:
+  - gcr.io
+  - docker.io

.yamllint.yml

@@ -0,0 +1,29 @@
+# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY
+#
+# The original version of this file is located in the https://github.com/istio/common-files repo.
+# If you're looking at this file in a different repo and want to make a change, please go to the
+# common-files repo, make the change there and check it in. Then come back to this repo and run
+# "make update-common".
+
+rules:
+  braces: disable
+  brackets: disable
+  colons: enable
+  commas: disable
+  comments: disable
+  comments-indentation: disable
+  document-end: disable
+  document-start: disable
+  empty-lines: disable
+  empty-values: enable
+  hyphens: enable
+  indentation: disable
+  key-duplicates: enable
+  key-ordering: disable
+  line-length: disable
+  new-line-at-end-of-file: disable
+  new-lines: enable
+  octal-values: enable
+  quoted-strings: disable
+  trailing-spaces: disable
+  truthy: disable

Makefile

@@ -3,7 +3,7 @@
 # The original version of this file is located in the https://github.com/istio/common-files repo.
 # If you're looking at this file in a different repo and want to make a change, please go to the
 # common-files repo, make the change there and check it in. Then come back to this repo and run
-# "make updatecommon".
+# "make update-common".
 
 # Copyright 2019 Istio Authors
 #
@@ -30,7 +30,7 @@ RUN =
 # figure out all the tools you need in your environment to make that work.
 export BUILD_WITH_CONTAINER ?= 0
 ifeq ($(BUILD_WITH_CONTAINER),1)
-IMG = gcr.io/istio-testing/build-tools:2019-08-16
+IMG = gcr.io/istio-testing/build-tools:2019-08-21T08-35-40
 UID = $(shell id -u)
 PWD = $(shell pwd)
 GOBIN_SOURCE ?= $(GOPATH)/bin
@@ -59,7 +59,7 @@ endif
 
 export GOOS ?= $(GOOS_LOCAL)
 
-RUN = docker run -t --sig-proxy=true -u $(UID) --rm \
+RUN = docker run -t -i --sig-proxy=true -u $(UID) --rm \
 	-e GOOS="$(GOOS)" \
 	-e GOARCH="$(GOARCH)" \
 	-e GOBIN="$(GOBIN)" \

Makefile.common.mk

@@ -3,7 +3,7 @@
 # The original version of this file is located in the https://github.com/istio/common-files repo.
 # If you're looking at this file in a different repo and want to make a change, please go to the
 # common-files repo, make the change there and check it in. Then come back to this repo and run
-# "make updatecommon".
+# "make update-common".
 
 # Copyright 2019 Istio Authors
 #
@@ -19,8 +19,32 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-updatecommon:
+lint-dockerfiles:
+	@find . -path ./vendor -prune -o -type f -name 'Dockerfile*' -print0 | xargs -0 hadolint -c ./.hadolint.yml
+
+lint-scripts:
+	@find . -path ./vendor -prune -o -type f -name '*.sh' -print0 | xargs -0 shellcheck
+
+lint-yaml:
+	@find . -path ./vendor -prune -o -type f -name '*.yml' -print0 | xargs -0 yamllint -c ./.yamllint.yml
+	@find . -path ./vendor -prune -o -type f -name '*.yaml' -print0 | xargs -0 yamllint -c ./.yamllint.yml
+
+lint-copyright-banner:
+	@scripts/lint_copyright_banner.sh
+
+lint-go:
+	@golangci-lint run -j 8 -v ./...
+
+format-go:
+	@goimports -w -local "istio.io" $(shell find . -type f -name '*.go' ! -name '*.gen.go' ! -name '*.pb.go' )
+
+update-common:
 	@git clone --depth 1 --single-branch --branch master https://github.com/istio/common-files
 	@cd common-files ; git rev-parse HEAD >.commonfiles.sha
 	@cp -r common-files/files/* common-files/.commonfiles.sha common-files/files/.[^.]* .
 	@rm -fr common-files
+	@touch Makefile.overrides.mk  # make sure this at least exists
+	# temporary, until cleaned up in all repos
+	@rm -fr scripts/check_license.sh
+
+.PHONY: lint-dockerfiles lint-scripts lint-yaml lint-copyright-banner lint-go format-go update-common

Makefile.core.mk

@@ -1,3 +1,17 @@
+# Copyright 2019 Istio Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 export GO111MODULE=on
 ifeq ($(BUILD_WITH_CONTAINER),0)
 override GOBIN := $(GOPATH)/bin
@@ -8,9 +22,7 @@ pwd := $(shell pwd)
 # make targets
 .PHONY: lint test_with_coverage mandiff build fmt vfsgen update-charts
 
-lint:
-	@scripts/check_license.sh
-	@golangci-lint run -j 8 -v ./...
+lint: lint-copyright-banner lint-go
 
 test:
 	@go test -race ./...
@@ -24,8 +36,7 @@ mandiff: update-charts
 
 build: mesh
 
-fmt:
-	@goimports -w -local "istio.io" $(shell find . -type f -name '*.go' ! -name '*.gen.go' ! -name '*.pb.go' )
+fmt: format-go
 
 update-charts: installer.sha
 	@scripts/run_update_charts.sh `cat installer.sha`
@@ -34,80 +45,64 @@ update-charts: installer.sha
 vfsgen: data/ update-charts
 	go generate ./...
 
-########################
-# protoc_gen_gogo*
-########################
-
-gogofast_plugin_prefix := --gogofast_out=plugins=grpc,
+generate: generate-values generate-types vfsgen
 
-comma := ,
-empty:=
-space := $(empty) $(empty)
+clean: clean-values clean-types
 
-importmaps := \
-	gogoproto/gogo.proto=github.com/gogo/protobuf/gogoproto \
-	google/protobuf/any.proto=github.com/gogo/protobuf/types \
-	google/protobuf/descriptor.proto=github.com/gogo/protobuf/protoc-gen-gogo/descriptor \
-	google/protobuf/duration.proto=github.com/gogo/protobuf/types \
-	google/protobuf/struct.proto=github.com/gogo/protobuf/types \
-	google/protobuf/timestamp.proto=github.com/gogo/protobuf/types \
-	google/protobuf/wrappers.proto=github.com/gogo/protobuf/types \
-	google/rpc/status.proto=github.com/gogo/googleapis/google/rpc \
-	google/rpc/code.proto=github.com/gogo/googleapis/google/rpc \
-	google/rpc/error_details.proto=github.com/gogo/googleapis/google/rpc \
-
-# generate mapping directive with M<proto>:<go pkg>, format for each proto file
-mapping_with_spaces := $(foreach map,$(importmaps),M$(map),)
-gogo_mapping := $(subst $(space),$(empty),$(mapping_with_spaces))
-
-#gofast_plugin := $(gofast_plugin_prefix)$(gogo_mapping):$(out_path)
-gogofast_plugin := $(gogofast_plugin_prefix)$(gogo_mapping):$(out_path)
-
-#####################
-# Generation Rules
-#####################
+default: mesh
 
-api_path := pkg/apis/istio/v1alpha2
-api_protos := $(shell find $(api_path) -type f -name '*.proto' | sort)
-api_pb_gos := $(api_protos:.proto=.pb.go)
+mesh: vfsgen
+	go build -o ${GOBIN}/mesh ./cmd/mesh.go
 
-########################
-# protoc_gen_docs
 ########################
 
-gen_doc_iscp: get_dep_proto
-	protoc -I/tmp/src -I$(pwd) -I/usr/include/protobuf --docs_out=warnings=true,emit_yaml=true,mode=html_page:$(pwd) pkg/apis/istio/v1alpha2/istiocontrolplane_types.proto
+repo_dir := .
+out_path = /tmp
+protoc = protoc -I/usr/include/protobuf -I.
 
-gen_doc_values: get_dep_proto
-	protoc -I/tmp/src -I$(pwd) -I/usr/include/protobuf --docs_out=warnings=true,emit_yaml=true,mode=html_page:$(pwd) pkg/apis/istio/v1alpha2/values/values_types.proto
+go_plugin_prefix := --go_out=plugins=grpc,
+go_plugin := $(go_plugin_prefix):$(out_path)
 
-default: mesh
+python_output_path := python/istio_api
+protoc_gen_python_prefix := --python_out=,
+protoc_gen_python_plugin := $(protoc_gen_python_prefix):$(repo_dir)/$(python_output_path)
 
-generate-api-go: $(api_pb_gos)
-	patch pkg/apis/istio/v1alpha2/istiocontrolplane_types.pb.go < pkg/apis/istio/v1alpha2/fixup_go_structs.patch
+protoc_gen_docs_plugin := --docs_out=warnings=true,mode=html_fragment_with_front_matter:$(repo_dir)/
 
-$(api_pb_gos): $(api_protos)
-	@protoc $(gogofast_plugin) $^
+types_v1alpha2_path := pkg/apis/istio/v1alpha2
+types_v1alpha2_protos := $(wildcard $(types_v1alpha2_path)/*.proto)
+types_v1alpha2_pb_gos := $(types_v1alpha2_protos:.proto=.pb.go)
+types_v1alpha2_pb_pythons := $(patsubst $(types_v1alpha2_path)/%.proto,$(python_output_path)/$(types_v1alpha2_path)/%_pb2.py,$(types_v1alpha2_protos))
+types_v1alpha2_pb_docs := $(types_v1alpha2_protos:.proto=.pb.html)
+types_v1alpha2_openapi := $(types_v1alpha2_protos:.proto=.json)
 
-clean-proto:
-	rm -f $(api_pb_gos)
+$(types_v1alpha2_pb_gos) $(types_v1alpha2_pb_docs) $(types_v1alpha2_pb_pythons): $(types_v1alpha2_protos)
+	@$(protoc) $(go_plugin) $(protoc_gen_docs_plugin)$(types_v1alpha2_path) $(protoc_gen_python_plugin) $^
+	@cp -r /tmp/pkg/* pkg/
+	@sed -i -e 's|github.com/gogo/protobuf/protobuf/google/protobuf|github.com/gogo/protobuf/types|g' $(types_v1alpha2_path)/istiocontrolplane_types.pb.go
+	@patch $(types_v1alpha2_path)/istiocontrolplane_types.pb.go < $(types_v1alpha2_path)/fixup_go_structs.patch
 
-# is all of this needed or is this cruft?
+generate-types: $(types_v1alpha2_pb_gos) $(types_v1alpha2_pb_docs) $(types_v1alpha2_pb_pythons)
 
-get_dep_proto:
-	GO111MODULE=off GOPATH=/tmp go get k8s.io/api/core/v1 k8s.io/api/autoscaling/v2beta1 k8s.io/apimachinery/pkg/apis/meta/v1/
+clean-types:
+	@rm -fr $(types_v1alpha2_pb_gos) $(types_v1alpha2_pb_docs) $(types_v1alpha2_pb_pythons)
 
-proto_iscp: get_dep_proto
-	protoc -I=/tmp/src -I$(pwd) -I=/usr/include/protobuf --gogofast_out=$(pwd) pkg/apis/istio/v1alpha2/istiocontrolplane_types.proto
-	sed -i -e 's|github.com/gogo/protobuf/protobuf/google/protobuf|github.com/gogo/protobuf/types|g' pkg/apis/istio/v1alpha2/istiocontrolplane_types.pb.go
-	patch pkg/apis/istio/v1alpha2/istiocontrolplane_types.pb.go < pkg/apis/istio/v1alpha2/fixup_go_structs.patch
+values_v1alpha2_path := pkg/apis/istio/v1alpha2/values
+values_v1alpha2_protos := $(wildcard $(values_v1alpha2_path)/*.proto)
+values_v1alpha2_pb_gos := $(values_v1alpha2_protos:.proto=.pb.go)
+values_v1alpha2_pb_pythons := $(patsubst $(values_v1alpha2_path)/%.proto,$(python_output_path)/$(values_v1alpha2_path)/%_pb2.py,$(values_v1alpha2_protos))
+values_v1alpha2_pb_docs := $(values_v1alpha2_protos:.proto=.pb.html)
+values_v1alpha2_openapi := $(values_v1alpha2_protos:.proto=.json)
 
-proto_values: get_dep_proto
-	protoc -I=/tmp/src -I$(pwd) -I=/usr/include/protobuf --go_out=$(pwd) pkg/apis/istio/v1alpha2/values/values_types.proto
-	sed -i -e 's|github.com/gogo/protobuf/protobuf/google/protobuf|github.com/gogo/protobuf/types|g' pkg/apis/istio/v1alpha2/values/values_types.pb.go
-	patch pkg/apis/istio/v1alpha2/values/values_types.pb.go < pkg/apis/istio/v1alpha2/values/fix_values_structs.patch
+$(values_v1alpha2_pb_gos) $(values_v1alpha2_pb_docs) $(values_v1alpha2_pb_pythons): $(values_v1alpha2_protos)
+	@$(protoc) $(go_plugin) $(protoc_gen_docs_plugin)$(values_v1alpha2_path) $(protoc_gen_python_plugin) $^
+	@cp -r /tmp/pkg/* pkg/
+	@sed -i -e 's|github.com/gogo/protobuf/protobuf/google/protobuf|github.com/gogo/protobuf/types|g' $(values_v1alpha2_path)/values_types.pb.go
+	@patch $(values_v1alpha2_path)/values_types.pb.go < $(values_v1alpha2_path)/fix_values_structs.patch
 
-mesh: vfsgen
-	go build -o ${GOBIN}/mesh ./cmd/mesh.go
+generate-values: $(values_v1alpha2_pb_gos) $(values_v1alpha2_pb_docs) $(values_v1alpha2_pb_pythons)
+
+clean-values:
+	@rm -fr $(values_v1alpha2_pb_gos) $(values_v1alpha2_pb_docs) $(values_v1alpha2_pb_pythons)
 
 include Makefile.common.mk

Makefile.overrides.mk

@@ -1 +1,18 @@
+# Copyright 2019 Istio Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 CONTAINER_OPTIONS=--mount type=volume,source=istio-installer-cache,destination="${GOPATH}/src/istio.io"
+
+# this repo is on the container plan by default
+BUILD_WITH_CONTAINER ?= 1

go.mod

@@ -2,6 +2,10 @@ module istio.io/operator
 
 go 1.12
 
+replace k8s.io/klog => github.com/istio/klog v0.0.0-20190424230111-fb7481ea8bcf
+
+replace github.com/golang/glog => github.com/istio/glog v0.0.0-20190424172949-d7cfb6fa2ccd
+
 require (
 	cloud.google.com/go v0.37.4 // indirect
 	contrib.go.opencensus.io/exporter/ocagent v0.4.12 // indirect
@@ -43,7 +47,7 @@ require (
 	github.com/go-openapi/spec v0.17.2
 	github.com/go-openapi/swag v0.19.0 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
-	github.com/gogo/protobuf v1.2.1
+	github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect
 	github.com/golang/protobuf v1.3.1