Skip to content

Commit

Permalink
Refactor istio-docker.mk for 30 second builds (istio#9085)
Browse files Browse the repository at this point in the history 
* Refactor istio-docker.mk for 30 second builds

This PR significantly cleans up istio-docker.mk and causes makes
`make docker` to run in ~10% of the time on my gear compared to
master under the same testing circumstances.

Most of this PR's work is done by reducing the size of the build context
sent to the docker daemon.  Note the times below are the `make docker`
operation after `make docker` has already been done.  The first
`make docker` without cached images can take varying times to
build.  The results here simply measure the overhead of building the go
binaries and sending the correct data in the Docker build context.

```
sdake@falkor-07:~/go/src/istio.io/istio$ time make docker
mkdir -p /home/sdake/go/out/logs
bin/gobuild.sh /home/sdake/go/out/linux_amd64/release/pilot-discovery ./pilot/cmd/pilot-discovery

.
.
.

real	0m26.074s
user	0m14.800s
sys	0m7.020s
```

* remove docker.kubectl from list of targets

* Address review coments.

* Add proxy_debug changes to enable versioning
  • Loading branch information
@istio-testing
Steven Dake authored and istio-testing committed Oct 3, 2018
1 parent ae576bb commit 529ebbd
Show file tree
Hide file tree
Showing 2 changed files with 112 additions and 97 deletions.
9 changes: 9 additions & 0 deletions pilot/docker/Dockerfile.proxy_debug
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,17 @@
FROM istionightly/base_debug
ARG proxy_version
ARG istio_version

# Install Envoy.
ADD envoy /usr/local/bin/envoy

# Environment variables indicating this proxy's version/capabilities as opaque string
ENV ISTIO_META_ISTIO_PROXY_VERSION "1.1.0"
# Environment variable indicating the exact proxy sha - for debugging or version-specific configs
ENV ISTIO_META_ISTIO_PROXY_SHA $proxy_version
# Environment variable indicating the exact build, for debugging
ENV ISTIO_META_ISTIO_VERSION $istio_version

ADD pilot-agent /usr/local/bin/pilot-agent

# pilot-agent and envoy may run with effective uid 0 in order to run envoy with
Expand Down
200 changes: 103 additions & 97 deletions tools/istio-docker.mk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,26 +13,22 @@
## limitations under the License.

.PHONY: docker
.PHONY: docker.all
.PHONY: docker.save
.PHONY: docker.push

# Docker target will build the go binaries and package the docker for local testing.
# It does not upload to a registry.
docker: build test-bins docker.all

DOCKER_TARGETS:=docker.pilot docker.proxy_debug docker.proxytproxy docker.proxyv2 docker.app docker.test_policybackend \
docker.proxy_init docker.mixer docker.citadel docker.galley docker.sidecar_injector

$(ISTIO_DOCKER) $(ISTIO_DOCKER_TAR):
mkdir -p $@

.SECONDEXPANSION: #allow $@ to be used in dependency list

# static files/directories that are copied from source tree

NODE_AGENT_TEST_FILES:=security/docker/start_app.sh \
security/docker/app.js


# note that "js" and "force" are directories rather than a file
$(ISTIO_DOCKER)/js $(ISTIO_DOCKER)/force: addons/servicegraph/$$(notdir $$@) | $(ISTIO_DOCKER)
cp -r $< $(@D)

# generated content
$(ISTIO_DOCKER)/istio_ca.crt $(ISTIO_DOCKER)/istio_ca.key: ${GEN_CERT} | ${ISTIO_DOCKER}
${GEN_CERT} --key-size=2048 --out-cert=${ISTIO_DOCKER}/istio_ca.crt \
Expand All @@ -41,58 +37,73 @@ $(ISTIO_DOCKER)/istio_ca.crt $(ISTIO_DOCKER)/istio_ca.key: ${GEN_CERT} | ${ISTIO
$(ISTIO_DOCKER)/node_agent.crt $(ISTIO_DOCKER)/node_agent.key: ${GEN_CERT} $(ISTIO_DOCKER)/istio_ca.crt $(ISTIO_DOCKER)/istio_ca.key
${GEN_CERT} --key-size=2048 --out-cert=${ISTIO_DOCKER}/node_agent.crt \
--out-priv=${ISTIO_DOCKER}/node_agent.key --organization="NodeAgent" \
--mode=signer --host="nodeagent.google.com" --signer-cert=${ISTIO_DOCKER}/istio_ca.crt \
--mode=signer --host="nodeagent.google.com" --signer-cert=${ISTIO_DOCKER}/istio_ca.crt \
--signer-priv=${ISTIO_DOCKER}/istio_ca.key

# directives to copy files to docker scratch directory

# tell make which files are copied form go/out
# tell make which files are copied from $(ISTIO_OUT) and generate rules to copy them to the proper location:
# generates rules like the following:
# $(ISTIO_DOCKER)/pilot-agent: $(ISTIO_OUT)/pilot-agent | $(ISTIO_DOCKER)
# cp $(ISTIO_OUT)/$FILE $(ISTIO_DOCKER)/($FILE)
DOCKER_FILES_FROM_ISTIO_OUT:=pilot-test-client pilot-test-server \
pilot-discovery pilot-agent sidecar-injector servicegraph mixs \
pilot-discovery pilot-agent sidecar-injector mixs \
istio_ca node_agent node_agent_k8s galley
$(foreach FILE,$(DOCKER_FILES_FROM_ISTIO_OUT), \
$(eval $(ISTIO_DOCKER)/$(FILE): $(ISTIO_OUT)/$(FILE) | $(ISTIO_DOCKER); cp $$< $$(@D)))
$(eval $(ISTIO_DOCKER)/$(FILE): $(ISTIO_OUT)/$(FILE) | $(ISTIO_DOCKER); cp $(ISTIO_OUT)/$(FILE) $(ISTIO_DOCKER)/$(FILE)))

# This generates rules like:
#$(ISTIO_DOCKER)/pilot-agent: $(ISTIO_OUT)/pilot-agent | $(ISTIO_DOCKER)
# cp $$< $$(@D))

# tell make which files are copied from the source tree
# tell make which files are copied from the source tree and generate rules to copy them to the proper location:
# TODO(sdake) $(NODE_AGENT_TEST_FILES) $(GRAFANA_FILES)
DOCKER_FILES_FROM_SOURCE:=tools/deb/istio-iptables.sh docker/ca-certificates.tgz \
$(NODE_AGENT_TEST_FILES) $(GRAFANA_FILES) \
pilot/docker/certs/cert.crt pilot/docker/certs/cert.key pilot/docker/certs/cacert.pem
# generates rules like the following:
# $(ISTIO_DOCKER)/tools/deb/istio-iptables.sh: $(ISTIO_OUT)/tools/deb/istio-iptables.sh | $(ISTIO_DOCKER)
# cp $FILE $$(@D))
$(foreach FILE,$(DOCKER_FILES_FROM_SOURCE), \
$(eval $(ISTIO_DOCKER)/$(notdir $(FILE)): $(FILE) | $(ISTIO_DOCKER); cp $(FILE) $$(@D)))

# pilot docker imagesDOCKER_BUILD_TOP

# tell make which files are copied from ISTIO_BIN and generate rules to copy them to the proper location:
# generates rules like the following:
# $(ISTIO_DOCKER)/kubectl: $(ISTIO_BIN)/kubectl | $(ISTIO_DOCKER)
# cp $(ISTIO_BIN)/kubectl $(ISTIO_DOCKER)/kubectl
DOCKER_FILES_FROM_ISTIO_BIN:=kubectl
$(foreach FILE,$(DOCKER_FILES_FROM_ISTIO_BIN), \
$(eval $(ISTIO_DOCKER)/$(FILE): $(ISTIO_BIN)/$(FILE) | $(ISTIO_DOCKER); bin/testEnvLocalK8S.sh getDeps; cp $(ISTIO_BIN)/$(FILE) $(ISTIO_DOCKER)/$(FILE)))

# pilot docker images

docker.proxy_init: pilot/docker/Dockerfile.proxy_init
docker.proxy_init: $(ISTIO_DOCKER)/istio-iptables.sh
docker.sidecar_injector: $(ISTIO_DOCKER)/sidecar-injector
$(DOCKER_RULE)

docker.sidecar_injector: pilot/docker/Dockerfile.sidecar_injector
docker.sidecar_injector:$(ISTIO_DOCKER)/sidecar-injector
$(DOCKER_RULE)

# BUILD_PRE tells $(DOCKER_RULE) to run the command specified before executing a docker build
# BUILD_ARGS tells $(DOCKER_RULE) to execute a docker build with the specified commands

docker.proxy_debug: BUILD_PRE=mv envoy-debug-${PROXY_REPO_SHA} envoy &&
docker.proxy_debug: BUILD_ARGS=--build-arg proxy_version=istio-proxy:${PROXY_REPO_SHA} --build-arg istio_version=${VERSION}
docker.proxy_debug: pilot/docker/Dockerfile.proxy_debug
docker.proxy_debug: tools/deb/envoy_bootstrap_v2.json
docker.proxy_debug: ${ISTIO_ENVOY_DEBUG_PATH}
docker.proxy_debug: $(ISTIO_OUT)/pilot-agent
docker.proxy_debug: pilot/docker/Dockerfile.proxyv2
docker.proxy_debug: pilot/docker/envoy_pilot.yaml.tmpl
docker.proxy_debug: pilot/docker/envoy_policy.yaml.tmpl
docker.proxy_debug: pilot/docker/envoy_telemetry.yaml.tmpl
mkdir -p $(DOCKER_BUILD_TOP)/proxyd
cp ${ISTIO_ENVOY_DEBUG_PATH} $(DOCKER_BUILD_TOP)/proxyd/envoy
cp pilot/docker/*.yaml.tmpl $(DOCKER_BUILD_TOP)/proxyd/
# Not using $^ to avoid 2 copies of envoy
cp tools/deb/envoy_bootstrap_v2.json tools/deb/istio-iptables.sh $(ISTIO_OUT)/pilot-agent pilot/docker/Dockerfile.proxyv2 $(DOCKER_BUILD_TOP)/proxyd/
time (cd $(DOCKER_BUILD_TOP)/proxyd && \
docker build \
--build-arg proxy_version=istio-proxy:${PROXY_REPO_SHA} \
--build-arg istio_version=${VERSION} \
-t $(HUB)/proxy_debug:$(TAG) -f Dockerfile.proxyv2 .)
$(DOCKER_RULE)

# The file must be named 'envoy', depends on the release.
${ISTIO_ENVOY_RELEASE_DIR}/envoy: ${ISTIO_ENVOY_RELEASE_PATH}
mkdir -p $(DOCKER_BUILD_TOP)/proxyv2
cp ${ISTIO_ENVOY_RELEASE_PATH} ${ISTIO_ENVOY_RELEASE_DIR}/envoy

# Default proxy image.
docker.proxyv2: BUILD_ARGS=--build-arg proxy_version=istio-proxy:${PROXY_REPO_SHA} --build-arg istio_version=${VERSION}
docker.proxyv2: tools/deb/envoy_bootstrap_v2.json
docker.proxyv2: $(ISTIO_ENVOY_RELEASE_DIR)/envoy
docker.proxyv2: $(ISTIO_OUT)/pilot-agent
Expand All @@ -101,15 +112,10 @@ docker.proxyv2: pilot/docker/envoy_pilot.yaml.tmpl
docker.proxyv2: pilot/docker/envoy_policy.yaml.tmpl
docker.proxyv2: tools/deb/istio-iptables.sh
docker.proxyv2: pilot/docker/envoy_telemetry.yaml.tmpl
mkdir -p $(DOCKER_BUILD_TOP)/proxyv2
cp $^ $(DOCKER_BUILD_TOP)/proxyv2/
time (cd $(DOCKER_BUILD_TOP)/proxyv2 && \
docker build \
--build-arg proxy_version=istio-proxy:${PROXY_REPO_SHA} \
--build-arg istio_version=${VERSION} \
-t $(HUB)/proxyv2:$(TAG) -f Dockerfile.proxyv2 .)
$(DOCKER_RULE)

# Proxy using TPROXY interception - but no core dumps
docker.proxytproxy: BUILD_ARGS=--build-arg proxy_version=istio-proxy:${PROXY_REPO_SHA} --build-arg istio_version=${VERSION}
docker.proxytproxy: tools/deb/envoy_bootstrap_v2.json
docker.proxytproxy: $(ISTIO_ENVOY_RELEASE_DIR)/envoy
docker.proxytproxy: $(ISTIO_OUT)/pilot-agent
Expand All @@ -118,26 +124,19 @@ docker.proxytproxy: pilot/docker/envoy_pilot.yaml.tmpl
docker.proxytproxy: pilot/docker/envoy_policy.yaml.tmpl
docker.proxytproxy: tools/deb/istio-iptables.sh
docker.proxytproxy: pilot/docker/envoy_telemetry.yaml.tmpl
mkdir -p $(DOCKER_BUILD_TOP)/proxyv2
cp $^ $(DOCKER_BUILD_TOP)/proxyv2/
time (cd $(DOCKER_BUILD_TOP)/proxyv2 && \
docker build \
--build-arg proxy_version=istio-proxy:${PROXY_REPO_SHA} \
--build-arg istio_version=${VERSION} \
-t $(HUB)/proxytproxy:$(TAG) -f Dockerfile.proxytproxy .)

push.proxytproxy: docker.proxytproxy
docker push $(HUB)/proxytproxy:$(TAG)

docker.pilot: $(ISTIO_OUT)/pilot-discovery pilot/docker/certs/cacert.pem pilot/docker/Dockerfile.pilot
mkdir -p $(ISTIO_DOCKER)/pilot
cp $^ $(ISTIO_DOCKER)/pilot/
time (cd $(ISTIO_DOCKER)/pilot && \
docker build -t $(HUB)/pilot:$(TAG) -f Dockerfile.pilot .)
$(DOCKER_RULE)

docker.pilot: $(ISTIO_OUT)/pilot-discovery
docker.pilot: pilot/docker/certs/cacert.pem
docker.pilot: pilot/docker/Dockerfile.pilot
$(DOCKER_RULE)

# Test app for pilot integration
docker.app: $(ISTIO_OUT)/pilot-test-client $(ISTIO_OUT)/pilot-test-server \
pilot/docker/certs/cert.crt pilot/docker/certs/cert.key pilot/docker/Dockerfile.app
docker.app: pilot/docker/Dockerfile.app
docker.app: $(ISTIO_OUT)/pilot-test-client
docker.app: $(ISTIO_OUT)/pilot-test-server
docker.app: pilot/docker/certs/cert.crt
docker.app: pilot/docker/certs/cert.key
mkdir -p $(ISTIO_DOCKER)/pilotapp
cp $^ $(ISTIO_DOCKER)/pilotapp
ifeq ($(DEBUG_IMAGE),1)
Expand All @@ -150,55 +149,67 @@ endif
docker build -t $(HUB)/app:$(TAG) -f Dockerfile.app .)

# Test policy backend for mixer integration
docker.test_policybackend: $(ISTIO_OUT)/mixer-test-policybackend \
mixer/docker/Dockerfile.test_policybackend
mkdir -p $(ISTIO_DOCKER)/test_policybackend
cp $^ $(ISTIO_DOCKER)/test_policybackend
time (cd $(ISTIO_DOCKER)/test_policybackend && \
docker build -t $(HUB)/test_policybackend:$(TAG) -f Dockerfile.test_policybackend .)

PILOT_DOCKER:=docker.proxy_init docker.sidecar_injector
$(PILOT_DOCKER): pilot/docker/Dockerfile$$(suffix $$@) | $(ISTIO_DOCKER)
docker.test_policybackend: mixer/docker/Dockerfile.test_policybackend
docker.test_policybackend: $(ISTIO_OUT)/mixer-test-policybackend
$(DOCKER_RULE)

# addons docker images
#docker.kubectl: docker/Dockerfile$$(suffix $$@) $(ISTIO_BIN)/kubectl
# $(DOCKER_RULE)

SERVICEGRAPH_DOCKER:=docker.servicegraph docker.servicegraph_debug
$(SERVICEGRAPH_DOCKER): addons/servicegraph/docker/Dockerfile$$(suffix $$@) \
$(ISTIO_DOCKER)/servicegraph $(ISTIO_DOCKER)/js $(ISTIO_DOCKER)/force | $(ISTIO_DOCKER)
$(DOCKER_RULE)
# addons docker images

# mixer docker images

MIXER_DOCKER:=docker.mixer docker.mixer_debug
$(MIXER_DOCKER): mixer/docker/Dockerfile$$(suffix $$@) \
$(ISTIO_DOCKER)/ca-certificates.tgz $(ISTIO_DOCKER)/mixs | $(ISTIO_DOCKER)
docker.mixer: mixer/docker/Dockerfile.mixer
docker.mixer: $(ISTIO_DOCKER)/mixs
docker.mixer: $(ISTIO_DOCKER)/ca-certificates.tgz
$(DOCKER_RULE)

# galley docker images

GALLEY_DOCKER:=docker.galley
$(GALLEY_DOCKER): galley/docker/Dockerfile$$(suffix $$@) $(ISTIO_DOCKER)/galley | $(ISTIO_DOCKER)
docker.galley: galley/docker/Dockerfile.galley
docker.galley: $(ISTIO_DOCKER)/galley
$(DOCKER_RULE)

# security docker images

docker.citadel: $(ISTIO_DOCKER)/istio_ca $(ISTIO_DOCKER)/ca-certificates.tgz
docker.citadel-test: $(ISTIO_DOCKER)/istio_ca.crt $(ISTIO_DOCKER)/istio_ca.key
docker.node-agent: $(ISTIO_DOCKER)/node_agent
docker.node-agent-k8s: $(ISTIO_DOCKER)/node_agent_k8s
docker.node-agent-test: $(ISTIO_DOCKER)/node_agent $(ISTIO_DOCKER)/istio_ca.key \
$(ISTIO_DOCKER)/node_agent.crt $(ISTIO_DOCKER)/node_agent.key
$(foreach FILE,$(NODE_AGENT_TEST_FILES),$(eval docker.node-agent-test: $(ISTIO_DOCKER)/$(notdir $(FILE))))
docker.citadel: security/docker/Dockerfile.citadel
docker.citadel: $(ISTIO_DOCKER)/istio_ca
docker.citadel: $(ISTIO_DOCKER)/ca-certificates.tgz
$(DOCKER_RULE)

SECURITY_DOCKER:=docker.citadel docker.citadel-test docker.node-agent docker.node-agent-k8s docker.node-agent-test
$(SECURITY_DOCKER): security/docker/Dockerfile$$(suffix $$@) | $(ISTIO_DOCKER)
docker.citadel-test: security/docker/Dockerfile.citadel-test
docker.citadel-test: $(ISTIO_DOCKER)/istio_ca
docker.citadel-test: $(ISTIO_DOCKER)/istio_ca.crt
docker.citadel-test: $(ISTIO_DOCKER)/istio_ca.key
$(DOCKER_RULE)

DOCKER_TARGETS:=docker.pilot docker.proxy_debug docker.proxytproxy docker.proxyv2 docker.app docker.test_policybackend $(PILOT_DOCKER) $(SERVICEGRAPH_DOCKER) $(MIXER_DOCKER) $(SECURITY_DOCKER) $(GALLEY_DOCKER)
docker.node-agent: security/docker/Dockerfile.node-agent
docker.node-agent: $(ISTIO_DOCKER)/node_agent
$(DOCKER_RULE)

docker.node-agent-k8s: security/docker/Dockerfile.node-agent-k8s
docker.node-agent-k8s: $(ISTIO_DOCKER)/node_agent_k8s
$(DOCKER_RULE)

docker.node-agent-test: security/docker/Dockerfile.node-agent-test
docker.node-agent-test: $(ISTIO_DOCKER)/node_agent
docker.node-agent-test: $(ISTIO_DOCKER)/istio_ca.crt
docker.node-agent-test: $(ISTIO_DOCKER)/node_agent.crt
docker.node-agent-test: $(ISTIO_DOCKER)/node_agent.key
$(DOCKER_RULE)

DOCKER_RULE=time (cp $< $(ISTIO_DOCKER)/ && cd $(ISTIO_DOCKER) && \
docker build -t $(HUB)/$(subst docker.,,$@):$(TAG) -f Dockerfile$(suffix $@) .)
# $@ is the name of the target
# $^ the name of the dependencies for the target
# Rule Steps #
##############
# 1. Make a directory $(DOCKER_BUILD_TOP)/%@
# 2. This rule uses cp to copy all dependency filenames into into $(DOCKER_BUILD_TOP/$@
# 3. This rule then changes directories to $(DOCKER_BUID_TOP)/$@
# 4. This rule runs $(BUILD_PRE) prior to any docker build and only if specified as a dependency variable
# 5. This rule finally runs docker build passing $(BUILD_ARGS) to docker if they are specified as a dependency variable

DOCKER_RULE=time (mkdir -p $(DOCKER_BUILD_TOP)/$@ && cp -r $^ $(DOCKER_BUILD_TOP)/$@ && cd $(DOCKER_BUILD_TOP)/$@ && $(BUILD_PRE) docker build $(BUILD_ARGS) -t $(HUB)/$(subst docker.,,$@):$(TAG) -f Dockerfile$(suffix $@) .)

# This target will package all docker images used in test and release, without re-building
# go binaries. It is intended for CI/CD systems where the build is done in separate job.
Expand All @@ -207,12 +218,14 @@ docker.all: $(DOCKER_TARGETS)
# for each docker.XXX target create a tar.docker.XXX target that says how
# to make a $(ISTIO_OUT)/docker/XXX.tar.gz from the docker XXX image
# note that $(subst docker.,,$(TGT)) strips off the "docker." prefix, leaving just the XXX

# create a DOCKER_TAR_TARGETS that's each of DOCKER_TARGETS with a tar. prefix
DOCKER_TAR_TARGETS:=
$(foreach TGT,$(DOCKER_TARGETS),$(eval tar.$(TGT): $(TGT) | $(ISTIO_DOCKER_TAR) ; \
time (docker save -o ${ISTIO_DOCKER_TAR}/$(subst docker.,,$(TGT)).tar $(HUB)/$(subst docker.,,$(TGT)):$(TAG) && \
gzip ${ISTIO_DOCKER_TAR}/$(subst docker.,,$(TGT)).tar)))

# create a DOCKER_TAR_TARGETS that's each of DOCKER_TARGETS with a tar. prefix
DOCKER_TAR_TARGETS:=
# create a DOCKER_TAR_TARGETS that's each of DOCKER_TARGETS with a tar. prefix DOCKER_TAR_TARGETS:=
$(foreach TGT,$(DOCKER_TARGETS),$(eval DOCKER_TAR_TARGETS+=tar.$(TGT)))

# this target saves a tar.gz of each docker image to ${ISTIO_OUT}/docker/
Expand All @@ -222,19 +235,12 @@ docker.save: $(DOCKER_TAR_TARGETS)
# the local docker image to another hub
# a possible optimization is to use tag.$(TGT) as a dependency to do the tag for us
$(foreach TGT,$(DOCKER_TARGETS),$(eval push.$(TGT): | $(TGT) ; \
time (docker push $(HUB)/$(subst docker.,,$(TGT)):$(TAG))))
time (docker push $(HUB)/$(subst docker.,,$(TGT)):$(TAG))))

# create a DOCKER_PUSH_TARGETS that's each of DOCKER_TARGETS with a push. prefix
DOCKER_PUSH_TARGETS:=
$(foreach TGT,$(DOCKER_TARGETS),$(eval DOCKER_PUSH_TARGETS+=push.$(TGT)))

# This target pushes each docker image to specified HUB and TAG.
# The push scripts support a comma-separated list of HUB(s) and TAG(s),
# but I'm not sure this is worth the added complexity to support.

# Deprecated - just use docker, no need to retag.
docker.tag: docker

# Will build and push docker images.
docker.push: $(DOCKER_PUSH_TARGETS)

Expand Down

0 comments on commit 529ebbd

Please sign in to comment.