bootstrap: Install protobuf gRPC Python plugin via PyPi and do not co…

…mpile it from a Git checkout.

Tested:
- I've rebuilt bootstrap docker images locally and all tests passed on them.
- I ran `bootstrap.sh` and `make proto` manually on my Linux workstation and my MacOS laptop.

History of changes for references:
- vitessio#3220 - I got rid of compiling gRPC from source. Because of that, the gRPC Python plugin was no longer installed and "make proto" stopped working.
- vitessio#3461 - This PR reverted my changes and changed back to compile gRPC from source again. However, that would not have been necessary in the first place. The "grpcio-tools" package has the protobuf compiler and the protobuf gRPC Python plugin. See: https://grpc.io/docs/quickstart/python.html#install-grpc-tools
- vitessio#3516 - This PR removed the gRPC compilation, but still compiled the protobuf gRPC Python plugin.

Note: As part of the now removed Python plugin compilation, we also had a compiled "protoc" binary. Since we will no longer have this binary, we're using the recommended procedure from the "grpcio-tools" package instead. See: https://grpc.io/docs/quickstart/python.html#generate-grpc-code They provide a Python wrapper script around a shared library of the "protoc" compiler, but not a binary anymore. Therefore, we have to call "python -m grpc_tools.protoc" to invoke the protobuf compiler.

Other changes:
- This protobuf compiler knows the Python plugin by default and does not require the --plugin flag for generating the Python services.
- In contrast, --plugin is now required to pass the gRPC Go plugin.
- The Python plugin changed its flag from --grpc_out to --grpc_python_out.
- The Python plugin emits different files since gRPC 1.7.0. I'll regenerate the files in a separate commit. See: https://github.com/grpc/grpc/releases/tag/v1.7.0
- Removed unlinking and linking back of Homebrew "protobuf" package. It looks to me like an existing package should not interfere with our virtualenv environment.
- Removed cleanup call where we deleted the obsolete $VTROOT/dist/protobuf directory. If users still have that directory on disk, they can delete it manually themselves. Existing directories should not interfere with the new approach.

Signed-off-by: Michael Berlin <[email protected]>

.codeclimate.yml

@@ -42,4 +42,3 @@ exclude_paths:
 - "go/vt/proto/"
 - "go/vt/sqlparser/sql.go"
 - "py/util/grpc_with_metadata.py"
-- "travis/install_grpc.sh"

Makefile

@@ -120,13 +120,15 @@ install_protoc-gen-go:
 	cp -a vendor/github.com/golang/protobuf $${GOPATH}/src/github.com/golang/
 	go install github.com/golang/protobuf/protoc-gen-go
 
-PROTOC_BINARY := $(shell type -p $(VTROOT)/dist/grpc/usr/local/bin/protoc)
-ifeq (,$(PROTOC_BINARY))
-	PROTOC_BINARY := $(shell which protoc)
-endif
-
-ifneq (,$(PROTOC_BINARY))
-	PROTOC_DIR := $(dir $(PROTOC_BINARY))
+# Find protoc compiler.
+# NOTE: We are *not* using the "protoc" binary (as suggested by the grpc Go
+#       quickstart for example). Instead, we run "protoc" via the Python
+#       wrapper script which is provided by the "grpcio-tools" PyPi package.
+#       (The package includes the compiler as library, but not as binary.
+#       Therefore, we have to use the wrapper script they provide.)
+ifneq ($(wildcard $(VTROOT)/dist/grpc/usr/local/lib/python2.7/site-packages/grpc_tools/protoc.py),)
+# IMPORTANT: The next line must not be indented.
+PROTOC_COMMAND := python -m grpc_tools.protoc
 endif
 
 PROTO_SRCS = $(wildcard proto/*.proto)
@@ -139,16 +141,16 @@ PROTO_GO_TEMPS = $(foreach name, $(PROTO_SRC_NAMES), go/vt/.proto.tmp/$(name).pb
 proto: proto_banner $(PROTO_GO_OUTS) $(PROTO_PY_OUTS)
 
 proto_banner:
-ifeq (,$(PROTOC_DIR))
-	$(error "Cannot find protoc binary. Did bootstrap.sh succeed, and did you execute 'source dev.env'?")
+ifeq (,$(PROTOC_COMMAND))
+	$(error "Cannot find protoc compiler. Did bootstrap.sh succeed, and did you execute 'source dev.env'?")
 endif
 
 ifndef NOBANNER
 	echo $$(date): Compiling proto definitions
 endif
 
 $(PROTO_PY_OUTS): py/vtproto/%_pb2.py: proto/%.proto
-	$(PROTOC_DIR)/protoc -Iproto $< --python_out=py/vtproto --grpc_out=py/vtproto --plugin=protoc-gen-grpc=$(PROTOC_DIR)/grpc_python_plugin
+	$(PROTOC_COMMAND) -Iproto $< --python_out=py/vtproto --grpc_python_out=py/vtproto
 
 $(PROTO_GO_OUTS): $(PROTO_GO_TEMPS)
 	for name in $(PROTO_SRC_NAMES); do \
@@ -160,7 +162,7 @@ $(PROTO_GO_TEMPS): install_protoc-gen-go
 
 $(PROTO_GO_TEMPS): go/vt/.proto.tmp/%.pb.go: proto/%.proto
 	mkdir -p go/vt/.proto.tmp
-	$(PROTOC_DIR)/protoc -Iproto $< --go_out=plugins=grpc:go/vt/.proto.tmp
+	$(PROTOC_COMMAND) -Iproto $< --plugin=grpc=protoc-gen-go --go_out=plugins=grpc:go/vt/.proto.tmp
 	sed -i -e 's,import \([a-z0-9_]*\) ".",import \1 "vitess.io/vitess/go/vt/proto/\1",g' $@
 
 # Helper targets for building Docker images.

bootstrap.sh

@@ -101,38 +101,47 @@ else
 fi
 ln -snf $consul_dist/consul $VTROOT/bin/consul
 
-# Install gRPC proto compilers. There is no download for grpc_python_plugin.
-# So, we need to build it.
-export grpc_dist=$VTROOT/dist/grpc
-export grpc_ver="v1.10.0"
-if [[ -f $grpc_dist/.build_finished && "$(cat $grpc_dist/.build_finished)" == "$grpc_ver" ]]; then
-  echo "skipping gRPC build. remove $grpc_dist to force rebuild."
+# Install the gRPC Python library (grpcio) and the protobuf gRPC Python plugin (grpcio-tools) from PyPI.
+# Dependencies like the Python protobuf package will be installed automatically.
+grpc_dist=$VTROOT/dist/grpc
+grpc_ver="1.10.0"
+grpc_version_file="$grpc_dist/.build_finished"
+if [[ -f "$grpc_version_file" && "$(cat "$grpc_version_file")" == "$grpc_ver" ]]; then
+  echo "skipping gRPC build. remove $grpc_dist to force reinstall."
 else
-  echo "installing grpc $grpc_ver"
-  # unlink homebrew's protobuf, to be able to compile the downloaded protobuf package
-  if [[ `uname -s` == "Darwin" && "$(brew list -1 | grep google-protobuf)" ]]; then
-    brew unlink grpc/grpc/google-protobuf
-  fi
+  echo "installing gRPC $grpc_ver"
+  # Cleanup any existing data and re-create the directory.
+  rm -rf "$grpc_dist"
 
-  # protobuf used to be a separate package, now we use the gRPC one.
-  rm -rf $VTROOT/dist/protobuf
+  trap "fail 'gRPC build failed'; exit 1" ERR
+  mkdir -p "$grpc_dist"
+  pushd "$grpc_dist" >/dev/null
 
-  # Cleanup any existing data and re-create the directory.
-  rm -rf $grpc_dist
-  mkdir -p $grpc_dist
+  # Python requires a very recent version of virtualenv.
+  # We also require a recent version of pip, as we use it to
+  # upgrade the other tools.
+  # For instance, setuptools doesn't work with pip 6.0:
+  # https://github.com/pypa/setuptools/issues/945
+  # (and setuptools is used by grpc install).
+  grpc_virtualenv="$grpc_dist/usr/local"
+  $VIRTUALENV -v "$grpc_virtualenv"
+  PIP=$grpc_virtualenv/bin/pip
+  $PIP install --upgrade pip
+  $PIP install --upgrade --ignore-installed virtualenv
+
+  grpcio_ver=$grpc_ver
+  $PIP install --upgrade grpcio==$grpcio_ver grpcio-tools==$grpcio_ver
 
-  ./travis/install_grpc.sh $grpc_dist || fail "gRPC build failed"
-  echo "$grpc_ver" > $grpc_dist/.build_finished
+  popd >/dev/null
+  trap - ERR
 
-  # link homebrew's protobuf back
-  if [[ `uname -s` == "Darwin" && "$(brew list -1 | grep google-protobuf)" ]]; then
-    brew link grpc/grpc/google-protobuf
-  fi
+  echo "$grpc_ver" > "$grpc_version_file"
 
   # Add newly installed Python code to PYTHONPATH such that other Python module
   # installations can reuse it. (Once bootstrap.sh has finished, run
   # source dev.env instead to set the correct PYTHONPATH.)
-  export PYTHONPATH=$(prepend_path $PYTHONPATH $grpc_dist/usr/local/lib/python2.7/dist-packages)
+  PYTHONPATH=$(prepend_path "$PYTHONPATH" "$grpc_virtualenv/lib/python2.7/dist-packages")
+  export PYTHONPATH
 fi
 
 # Install third-party Go tools used as part of the development workflow.

docker/bootstrap/Dockerfile.common

@@ -29,9 +29,6 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins
     libz-dev \
     && rm -rf /var/lib/apt/lists/*
 
-# Not sure if this is needed, but it was part of php init, which we don't do any more.
-RUN mkdir -p /vt/bin
-
 # Install Maven 3.1+
 RUN mkdir -p /vt/dist && \
     cd /vt/dist && \
@@ -61,10 +58,6 @@ COPY tools /vt/src/vitess.io/vitess/tools
 COPY travis /vt/src/vitess.io/vitess/travis
 COPY vendor/vendor.json /vt/src/vitess.io/vitess/vendor/
 
-# grpcio runtime is needed for python tests.
-RUN grpcio_ver="1.7.0" && \
-    pip install --upgrade grpcio==$grpcio_ver
-
 # Download vendored Go dependencies
 RUN cd /vt/src/vitess.io/vitess && \
     go get -u github.com/kardianos/govendor && \