Skip to content

Commit

Permalink
Build in a container by default, using go 1.5 and vendored dependancies.
Browse files Browse the repository at this point in the history
- Use submodules for vendoring.
- Don't go clean, now that everything is in .pkg.
- sudo -E as we need the environment variables.
- Install -race version of std so the tests can run without sudo.
- Cache the build image on circle, use the rebuild-image script from tools.
  • Loading branch information
tomwilkie committed Jan 13, 2016
1 parent 6a103ae commit c098aef
Show file tree
Hide file tree
Showing 26 changed files with 154 additions and 196 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ _testmain.go
*.prof
profile.cov
coverage.html
.pkg

# Emacs backup files
*~
Expand Down
48 changes: 48 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
@@ -1,3 +1,51 @@
[submodule "tools"]
path = tools
url = https://github.com/weaveworks/build-tools
[submodule "vendor/github.com/vishvananda/netlink"]
path = vendor/github.com/vishvananda/netlink
url = https://github.com/vishvananda/netlink.git
[submodule "vendor/github.com/davecheney/profile"]
path = vendor/github.com/davecheney/profile
url = https://github.com/davecheney/profile.git
[submodule "vendor/github.com/gorilla/mux"]
path = vendor/github.com/gorilla/mux
url = https://github.com/gorilla/mux.git
[submodule "vendor/github.com/gorilla/context"]
path = vendor/github.com/gorilla/context
url = https://github.com/gorilla/context.git
[submodule "vendor/github.com/google/gopacket"]
path = vendor/github.com/google/gopacket
url = https://github.com/google/gopacket.git
[submodule "vendor/github.com/weaveworks/go-odp"]
path = vendor/github.com/weaveworks/go-odp
url = https://github.com/weaveworks/go-odp.git
[submodule "vendor/github.com/opencontainers/runc"]
path = vendor/github.com/opencontainers/runc
url = https://github.com/opencontainers/runc.git
[submodule "vendor/github.com/fsouza/go-dockerclient"]
path = vendor/github.com/fsouza/go-dockerclient
url = https://github.com/fsouza/go-dockerclient.git
[submodule "vendor/github.com/docker/libnetwork"]
path = vendor/github.com/docker/libnetwork
url = https://github.com/docker/libnetwork.git
[submodule "vendor/github.com/docker/docker"]
path = vendor/github.com/docker/docker
url = https://github.com/docker/docker.git
[submodule "vendor/github.com/Sirupsen/logrus"]
path = vendor/github.com/Sirupsen/logrus
url = https://github.com/Sirupsen/logrus.git
[submodule "vendor/github.com/andybalholm/go-bit"]
path = vendor/github.com/andybalholm/go-bit
url = https://github.com/andybalholm/go-bit.git
[submodule "vendor/github.com/miekg/dns"]
path = vendor/github.com/miekg/dns
url = https://github.com/miekg/dns.git
[submodule "vendor/github.com/stretchr/testify"]
path = vendor/github.com/stretchr/testify
url = https://github.com/stretchr/testify.git
[submodule "vendor/golang.org/x/crypto"]
path = vendor/golang.org/x/crypto
url = https://github.com/golang/crypto.git
[submodule "vendor/github.com/docker/machine"]
path = vendor/github.com/docker/machine
url = https://github.com/docker/machine.git
60 changes: 39 additions & 21 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,11 @@ PUBLISH=publish_weave publish_weaveexec publish_plugin
.PHONY: all exes update tests lint publish $(PUBLISH) clean clean-bin prerequisites build run-smoketests

# If you can use docker without being root, you can do "make SUDO="
SUDO=sudo
SUDO=sudo -E
BUILD_IN_CONTAINER=true
RM=--rm
RUN_FLAGS=-ti
COVERAGE=

DOCKERHUB_USER=weaveworks
WEAVE_VERSION=git-$(shell git rev-parse --short=12 HEAD)
Expand All @@ -22,15 +26,17 @@ TEST_TLS_EXE=test/tls/tls

EXES=$(WEAVER_EXE) $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(WEAVEWAIT_NOMCAST_EXE) $(WEAVEUTIL_EXE) $(DOCKERPLUGIN_EXE) $(TEST_TLS_EXE)

BUILD_UPTODATE=.build.uptodate
WEAVER_UPTODATE=.weaver.uptodate
WEAVEEXEC_UPTODATE=.weaveexec.uptodate
DOCKERPLUGIN_UPTODATE=.dockerplugin.uptodate

IMAGES_UPTODATE=$(WEAVER_UPTODATE) $(WEAVEEXEC_UPTODATE) $(DOCKERPLUGIN_UPTODATE)
IMAGES_UPTODATE=$(WEAVER_UPTODATE) $(WEAVEEXEC_UPTODATE) $(DOCKERPLUGIN_UPTODATE) $(BUILD_UPTODATE)

WEAVER_IMAGE=$(DOCKERHUB_USER)/weave
WEAVEEXEC_IMAGE=$(DOCKERHUB_USER)/weaveexec
DOCKERPLUGIN_IMAGE=$(DOCKERHUB_USER)/plugin
BUILD_IMAGE=$(DOCKERHUB_USER)/weavebuild

IMAGES=$(WEAVER_IMAGE) $(WEAVEEXEC_IMAGE) $(DOCKERPLUGIN_IMAGE)

Expand All @@ -47,17 +53,15 @@ NETGO_CHECK=@strings $@ | grep cgo_stub\\\.go >/dev/null || { \
echo " sudo go install -tags netgo std"; \
false; \
}
BUILD_FLAGS=-ldflags "-extldflags \"-static\" -X main.version $(WEAVE_VERSION)" -tags netgo
BUILD_FLAGS=-i -ldflags "-extldflags \"-static\" -X main.version=$(WEAVE_VERSION)" -tags netgo

PACKAGE_BASE=$(shell go list -e ./)

all: $(WEAVE_EXPORT) $(RUNNER_EXE) $(TEST_TLS_EXE)

exes: $(EXES)

update:
go get -u -f -v -tags netgo $(addprefix ./,$(dir $(EXES)))

$(EXES): $(BUILD_UPTODATE)
$(WEAVER_EXE) $(WEAVEPROXY_EXE) $(WEAVEUTIL_EXE): common/*.go common/*/*.go net/*.go net/*/*.go
$(WEAVER_EXE): router/*.go mesh/*.go ipam/*.go ipam/*/*.go nameserver/*.go prog/weaver/*.go
$(WEAVEPROXY_EXE): proxy/*.go prog/weaveproxy/*.go
Expand All @@ -68,39 +72,60 @@ $(TEST_TLS_EXE): test/tls/*.go
$(WEAVEWAIT_NOOP_EXE): prog/weavewait/*.go
$(WEAVEWAIT_EXE): prog/weavewait/*.go net/*.go
$(WEAVEWAIT_NOMCAST_EXE): prog/weavewait/*.go net/*.go
tests: $(BUILD_UPTODATE) tools/.git
lint: $(BUILD_UPTODATE) tools/.git

ifeq ($(BUILD_IN_CONTAINER),true)

$(EXES) tests lint:
@if [ -z "$$(find vendor -type f -print -quit)" ] ; then git submodule update --init ; fi
@mkdir -p $(shell pwd)/.pkg
$(SUDO) docker run $(RM) $(RUN_FLAGS) \
-v $(shell pwd):/go/src/github.com/weaveworks/weave \
-v $(shell pwd)/.pkg:/go/pkg \
-e GOARCH -e GOOS -e CIRCLECI -e CIRCLE_BUILD_NUM -e CIRCLE_NODE_TOTAL -e CIRCLE_NODE_INDEX -e COVERDIR -e SLOW \
$(BUILD_IMAGE) COVERAGE=$(COVERAGE) WEAVE_VERSION=$(WEAVE_VERSION) $@

else

$(WEAVER_EXE) $(WEAVEPROXY_EXE):
ifeq ($(COVERAGE),true)
$(eval COVERAGE_MODULES := $(shell (go list ./$(@D); go list -f '{{join .Deps "\n"}}' ./$(@D) | grep "^$(PACKAGE_BASE)/") | paste -s -d,))
go get -t -tags netgo ./$(@D)
go test -c -o ./$@ $(BUILD_FLAGS) -v -covermode=atomic -coverpkg $(COVERAGE_MODULES) ./$(@D)/
else
go get -tags netgo ./$(@D)
go build $(BUILD_FLAGS) -o $@ ./$(@D)
endif
$(NETGO_CHECK)

$(WEAVEUTIL_EXE):
go get -tags netgo ./$(@D)
go build $(BUILD_FLAGS) -o $@ ./$(@D)
$(NETGO_CHECK)

$(WEAVEWAIT_EXE):
go get -tags netgo ./$(@D)
go build $(BUILD_FLAGS) -tags "netgo iface mcast" -o $@ ./$(@D)
$(NETGO_CHECK)

$(WEAVEWAIT_NOMCAST_EXE):
go get -tags netgo ./$(@D)
go build $(BUILD_FLAGS) -tags "netgo iface" -o $@ ./$(@D)
$(NETGO_CHECK)

# These programs need a separate rule as they fail the netgo check in
# the main build stanza due to not importing net package
$(SIGPROXY_EXE) $(DOCKERPLUGIN_EXE) $(TEST_TLS_EXE) $(WEAVEWAIT_NOOP_EXE):
go get -tags netgo ./$(@D)
go build $(BUILD_FLAGS) -o $@ ./$(@D)

tests:
./tools/test -no-go-get

lint:
./tools/lint -nocomment -notestpackage .

endif

$(BUILD_UPTODATE): build/*
$(SUDO) docker build -t $(BUILD_IMAGE) build/
touch $@

$(WEAVER_UPTODATE): prog/weaver/Dockerfile $(WEAVER_EXE)
$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker build -t $(WEAVER_IMAGE) prog/weaver
touch $@
Expand All @@ -127,12 +152,6 @@ $(WEAVE_EXPORT): $(IMAGES_UPTODATE)
$(DOCKER_DISTRIB):
curl -o $(DOCKER_DISTRIB) $(DOCKER_DISTRIB_URL)

tests: tools/.git
tools/test

lint: tools/.git
tools/lint -nocomment -notestpackage .

tools/.git:
git submodule update --init

Expand All @@ -149,9 +168,8 @@ endif
publish: $(PUBLISH)

clean-bin:
-$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker rmi $(IMAGES)
go clean -r $(addprefix ./,$(dir $(EXES)))
rm -f $(EXES) $(IMAGES_UPTODATE) $(WEAVE_EXPORT)
-$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker rmi $(IMAGES) $(BUILD_IMAGE)
rm -rf $(EXES) $(IMAGES_UPTODATE) $(WEAVE_EXPORT) .pkg

clean: clean-bin
rm -rf test/tls/*.pem test/coverage.* test/coverage
Expand Down
28 changes: 6 additions & 22 deletions build/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,24 +1,8 @@
FROM ubuntu

RUN apt-get -y update && apt-get -y install --no-install-recommends ca-certificates apt-transport-https
RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9
RUN echo deb https://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/docker.list
RUN apt-get -y update && apt-get -y install --no-install-recommends build-essential git lxc-docker-1.6.2 mercurial libpcap-dev curl make pkg-config gcc bison flex python-requests

# When doing a build in a container, "apt-get update" happens twice,
# which can be a very significant overhead for incremental builds.
# And it's unnecessary, because if any of the prerequisites do change,
# the right thing to do is to rebuild the container image. Rather
# than suppressing the "apt-get update" commands explicitly, we clear
# out sources.list so that 'apt-get update' doesn't do anything.
RUN echo >/etc/apt/sources.list

ENV GO_VERSION 1.4.2
RUN curl -sSL https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz | tar -C /usr/local -xz
ENV PATH /usr/local/go/bin:$PATH

# So that sudo does not trample on PATH
RUN sed -ie '/secure_path/d' /etc/sudoers

FROM golang:1.5.2
ENV GO15VENDOREXPERIMENT 1
RUN apt-get update && apt-get install -y libpcap-dev python-requests time
RUN go get github.com/golang/lint/golint
RUN go clean -i net && go install -tags netgo std
RUN go install -race -tags netgo std
COPY build.sh /
ENTRYPOINT ["sh", "/build.sh"]
55 changes: 19 additions & 36 deletions build/build.sh
Original file line number Diff line number Diff line change
@@ -1,45 +1,28 @@
#!/bin/sh
set -e

GOPATH=/home/go
export GOPATH
set -eu

WEAVE_SRC=$GOPATH/src/github.com/weaveworks/weave

if [ $# -eq 0 -o "$1" = "tests" ] ; then
# No arguments. Expect that the weave repo will be bind-mounted
# into $GOPATH
if ! [ -e $WEAVE_SRC ] ; then
cat 2>&1 <<EOF
No container arguments supplied, and nothing at ${WEAVE_SRC}. Please
either bind-mount the golang workspace containing weave with the
if ! [ -e $WEAVE_SRC ] ; then
cat 2>&1 <<EOF
Nothing at ${WEAVE_SRC}. Please bind-mount the weave repo with the
docker run -v option, e.g.:
$ docker run -v <host gopath>:${GOPATH} \\
-v /var/run/docker.sock:/var/run/docker.sock weaveworks/weave-build
Or supply git clone arguments to retrieve it, e.g.:
$ docker run -v /var/run/docker.sock:/var/run/docker.sock \\
weaveworks/weave-build https://github.com/weaveworks/weave.git
$ cd weave
$ docker run -v $(pwd):/go/src/github.com/weaveworks/weave \\
weaveworks/weave-build
EOF
exit 1
fi

# If we run make directly, any files created on the bind mount
# will have awkward ownership. So we switch to a user with the
# same user and group IDs as source directory. We have to set a
# few things up so that sudo works without complaining later on.
uid=$(stat --format="%u" $WEAVE_SRC)
gid=$(stat --format="%g" $WEAVE_SRC)
echo "weave:x:$uid:$gid::$WEAVE_SRC:/bin/sh" >>/etc/passwd
echo "weave:*:::::::" >>/etc/shadow
echo "weave ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers

su weave -c "PATH=$PATH make -C $WEAVE_SRC build"
else
# There are arguments to pass to git-clone
mkdir -p ${WEAVE_SRC%/*}
git clone "$@" $WEAVE_SRC
make -C $WEAVE_SRC build
fi

# If we run make directly, any files created on the bind mount
# will have awkward ownership. So we switch to a user with the
# same user and group IDs as source directory. We have to set a
# few things up so that sudo works without complaining later on.
uid=$(stat --format="%u" $WEAVE_SRC)
gid=$(stat --format="%g" $WEAVE_SRC)
echo "weave:x:$uid:$gid::$WEAVE_SRC:/bin/sh" >>/etc/passwd
echo "weave:*:::::::" >>/etc/shadow
echo "weave ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers

su weave -c "PATH=$PATH make -C $WEAVE_SRC BUILD_IN_CONTAINER=false $*"
24 changes: 7 additions & 17 deletions circle.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,35 +12,25 @@ machine:
SRCDIR: /home/ubuntu/src/github.com/weaveworks/weave
PATH: $PATH:$HOME/.local/bin:$HOME/google-cloud-sdk/bin/
CLOUDSDK_CORE_DISABLE_PROMPTS: 1
WEAVE_TEST: $HOME/docker/weave-test.tar

dependencies:
cache_directories:
- "~/docker"
override:
- if [[ -e "$WEAVE_TEST" ]]; then
docker load -i $WEAVE_TEST;
else
docker build -t weaveworks/weave-build build;
docker build -t weaveworks/weave-test test/image;
mkdir -p $(dirname "$WEAVE_TEST");
docker save weaveworks/weave-test >$WEAVE_TEST;
fi
post:
- sudo apt-get update && sudo apt-get install bc jq && pip install requests
- "sudo apt-get update && sudo apt-get install bc jq"
- curl https://sdk.cloud.google.com | bash
- bin/setup-circleci-secrets "$SECRET_PASSWORD"
- mkdir -p $(dirname $SRCDIR) && cp -r $(pwd)/ $SRCDIR
- "mkdir -p $(dirname $SRCDIR) && cp -r $(pwd)/ $SRCDIR"
- "cd $SRCDIR; git submodule update --init"
- cd $SRCDIR/test; ./gce.sh make_template:
parallel: false
- "cd $SRCDIR/build; ../tools/rebuild-image weaveworks/weavebuild . Dockerfile build.sh && touch $SRCDIR/.build.uptodate"

test:
override:
- cd $SRCDIR; make lint:
parallel: true
- docker run -v /home/ubuntu:/home/go -e COVERDIR=test/coverage -e SLOW=true -e CIRCLECI -e CIRCLE_BUILD_NUM -e CIRCLE_NODE_TOTAL -e CIRCLE_NODE_INDEX weaveworks/weave-test:
- cd $SRCDIR; COVERDIR=test/coverage make RM= tests:
parallel: true
- docker run -v /var/run/docker.sock:/run/docker.sock -v /home/ubuntu:/home/go -e COVERAGE=true weaveworks/weave-build:
- cd $SRCDIR; make COVERAGE=true RM=:
parallel: true
- cd $SRCDIR/test; ./gce.sh setup && eval $(./gce.sh hosts) && ./setup.sh:
parallel: true
Expand All @@ -49,7 +39,7 @@ test:
timeout: 300
- cd $SRCDIR; make clean-bin:
parallel: true
- docker run -v /var/run/docker.sock:/run/docker.sock -v /home/ubuntu:/home/go weaveworks/weave-build:
- cd $SRCDIR; make RM=:
parallel: true
post:
- cd $SRCDIR/test; ./gce.sh destroy:
Expand Down
Loading

0 comments on commit c098aef

Please sign in to comment.