add test.sh for testing various envs

.gitignore

@@ -13,3 +13,4 @@ kapacitor*.tar
 kapacitor*.zip
 *.pyc
 *.test
+/test-logs

Dockerfile_build_ubuntu32

@@ -0,0 +1,53 @@
+FROM 32bit/ubuntu:14.04
+
+# This dockerfile capabable  of the the minumum required
+# to run the tests and nothing else.
+
+MAINTAINER [email protected]
+
+RUN apt-get update && apt-get install -y \
+    wget \
+    git \
+    mercurial \
+    build-essential \
+    autoconf \
+    automake \
+    libtool \
+    python-setuptools \
+    zip \
+    curl
+
+# Install protobuf3
+ENV PROTO_VERSION 3.0.0-beta-2
+# Download and compile protoc
+RUN wget https://github.com/google/protobuf/archive/v${PROTO_VERSION}.tar.gz && \
+    tar xf v${PROTO_VERSION}.tar.gz && \
+    rm -f v${PROTO_VERSION}.tar.gz && \
+    cd protobuf-${PROTO_VERSION} && \
+    ./autogen.sh && \
+    ./configure --prefix=/usr && \
+    make -j $(nproc) && \
+    make check && \
+    make install
+
+# Install Python Protobuf3
+RUN cd protobuf-${PROTO_VERSION}/python && \
+    python setup.py install;
+
+# Install go
+ENV GOPATH /root/go
+ENV GO_VERSION 1.5.3
+ENV GO_ARCH 386
+RUN wget https://storage.googleapis.com/golang/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz; \
+   tar -C /usr/local/ -xf /go${GO_VERSION}.linux-${GO_ARCH}.tar.gz ; \
+   rm /go${GO_VERSION}.linux-${GO_ARCH}.tar.gz
+ENV PATH /usr/local/go/bin:$PATH
+
+ENV PROJECT_DIR $GOPATH/src/github.com/influxdata/kapacitor
+ENV PATH $GOPATH/bin:$PATH
+RUN mkdir -p $PROJECT_DIR
+WORKDIR $PROJECT_DIR
+
+VOLUME $PROJECT_DIR
+
+ENTRYPOINT [ "/root/go/src/github.com/influxdata/kapacitor/build.py" ]

Dockerfile → Dockerfile_build_ubuntu64

@@ -1,33 +1,33 @@
-# This Dockerfile provides the needed environment to build Kapacitor.
-FROM ubuntu:trusty
+FROM ubuntu:latest
+
+# This dockerfile is capabable of performing all
+# build/test/package/deploy actions needed for Kapacitor.
 
 MAINTAINER [email protected]
 
-# Install deps
-RUN apt-get update
-RUN apt-get install -y \
-    make \
-    wget  \
+RUN apt-get update && apt-get install -y \
+    python-software-properties \
+    software-properties-common \
+    wget \
     git \
     mercurial \
+    make \
     ruby \
     ruby-dev \
     rpm \
     zip \
     python \
-    python-boto
-
-RUN gem install fpm
-
-# Install protobuf3
-RUN apt-get install -y \
+    python-boto \
     build-essential \
     autoconf \
     automake \
     libtool \
     python-setuptools \
     curl
 
+RUN gem install fpm
+
+# Install protobuf3
 ENV PROTO_VERSION 3.0.0-beta-2
 # Download and compile protoc
 RUN wget https://github.com/google/protobuf/archive/v${PROTO_VERSION}.tar.gz && \
@@ -45,29 +45,20 @@ RUN cd protobuf-${PROTO_VERSION}/python && \
     python setup.py install;
 
 
-
 # Install go
+ENV GOPATH /root/go
 ENV GO_VERSION 1.5.3
 ENV GO_ARCH amd64
 RUN wget https://storage.googleapis.com/golang/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz; \
    tar -C /usr/local/ -xf /go${GO_VERSION}.linux-${GO_ARCH}.tar.gz ; \
    rm /go${GO_VERSION}.linux-${GO_ARCH}.tar.gz
 ENV PATH /usr/local/go/bin:$PATH
-ENV GOPATH /gopath
-ENV PATH $GOPATH/bin:$PATH
-ENV PROJECT_PATH $GOPATH/src/github.com/influxdata/kapacitor
-RUN mkdir -p $PROJECT_PATH
 
-WORKDIR $PROJECT_PATH
-ENTRYPOINT ["/usr/local/bin/build"]
-CMD []
-
-# Get gogo for golang protobuf
-RUN go get github.com/gogo/protobuf/protoc-gen-gogo
-
-# Initialize git, needed to stash changes before 'go get ./...'
-RUN git config --global user.email "[email protected]"
-RUN git config --global user.name "Docker Builder"
+ENV PROJECT_DIR $GOPATH/src/github.com/influxdata/kapacitor
+ENV PATH $GOPATH/bin:$PATH
+RUN mkdir -p $PROJECT_DIR
+WORKDIR $PROJECT_DIR
 
-ADD ./build.py /usr/local/bin/build
+VOLUME $PROJECT_DIR
 
+ENTRYPOINT [ "/root/go/src/github.com/influxdata/kapacitor/build.py" ]

Dockerfile_build_ubuntu64_git

@@ -0,0 +1,62 @@
+FROM ubuntu:latest
+
+# This dockerfile capabable  of the the minumum required
+# to run the tests and nothing else.
+
+MAINTAINER [email protected]
+
+RUN apt-get update && apt-get install -y \
+    wget \
+    git \
+    mercurial \
+    build-essential \
+    autoconf \
+    automake \
+    libtool \
+    python-setuptools \
+    zip \
+    curl
+
+# Install protobuf3
+ENV PROTO_VERSION 3.0.0-beta-2
+# Download and compile protoc
+RUN wget https://github.com/google/protobuf/archive/v${PROTO_VERSION}.tar.gz && \
+    tar xf v${PROTO_VERSION}.tar.gz && \
+    rm -f v${PROTO_VERSION}.tar.gz && \
+    cd protobuf-${PROTO_VERSION} && \
+    ./autogen.sh && \
+    ./configure --prefix=/usr && \
+    make -j $(nproc) && \
+    make check && \
+    make install
+
+# Install Python Protobuf3
+RUN cd protobuf-${PROTO_VERSION}/python && \
+    python setup.py install;
+
+# Setup env
+ENV GOPATH /root/go
+ENV PROJECT_DIR $GOPATH/src/github.com/influxdata/kapacitor
+ENV PATH $GOPATH/bin:$PATH
+RUN mkdir -p $PROJECT_DIR
+
+VOLUME $PROJECT_DIR
+
+
+# Install go
+ENV GO_VERSION 1.4.3
+ENV GO_ARCH amd64
+RUN wget https://storage.googleapis.com/golang/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz; \
+   tar -C /usr/local/ -xf /go${GO_VERSION}.linux-${GO_ARCH}.tar.gz ; \
+   rm /go${GO_VERSION}.linux-${GO_ARCH}.tar.gz
+
+# Clone Go tip for compilation
+ENV GOROOT_BOOTSTRAP /usr/local/go
+RUN git clone https://go.googlesource.com/go
+ENV PATH /go/bin:$PATH
+
+
+# Add script for compiling go
+ENV GO_CHECKOUT master
+ADD ./gobuild.sh /gobuild.sh
+ENTRYPOINT [ "/gobuild.sh" ]

build.py

@@ -115,6 +115,7 @@ def package_scripts(build_root):
 
 def run_generate():
     print "Running generate..."
+    run("go get github.com/gogo/protobuf/protoc-gen-gogo")
     command = "go generate ./..."
     code = os.system(command)
     if code != 0:

build.sh

@@ -9,7 +9,7 @@ cd $DIR
 
 
 # Build new docker image
-docker build -t influxdata/kapacitor-builder $DIR
+docker build -f Dockerfile_build_ubuntu64 -t influxdata/kapacitor-builder $DIR
 echo "Running build.py"
 # Run docker
 docker run --rm \

circle-test.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+#
+# This is the InfluxDB test script for CircleCI, it is a light wrapper around ./test.sh.
+
+# Exit if any command fails
+set -e
+
+# Get dir of script and make it is our working directory.
+DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)
+cd $DIR
+
+
+export OUTPUT_DIR="$CIRCLE_ARTIFACTS"
+# Don't delete the container since CircleCI doesn't have permission to do so.
+export DOCKER_RM="false"
+
+# Get number of test environments.
+count=$(./test.sh count)
+# Check that we aren't wasting CircleCI nodes.
+if [ $CIRCLE_NODE_TOTAL -gt $count ]
+then
+    echo "More CircleCI nodes allocated than tests environments to run!"
+    exit 1
+fi
+
+# Map CircleCI nodes to test environments.
+tests=$(seq 0 $((count - 1)))
+for i in $tests
+do
+    mine=$(( $i % $CIRCLE_NODE_TOTAL ))
+    if [ $mine -eq $CIRCLE_NODE_INDEX ]
+    then
+        echo "Running test env index: $i"
+        ./test.sh $i
+    fi
+done

circle.yml

@@ -1,32 +1,23 @@
-# The forces Circle CI to get a fresh clone of the repo.
-#checkout:
-#  post:
-#    - rm -rf ~/kapacitor
-#    - git clone [email protected]:influxdata/kapacitor.git ~/kapacitor
-
 machine:
-  services:
-    - docker
+    services:
+        - docker
 
 dependencies:
-  pre:
-    # setup ipv6
-    - sudo sysctl -w net.ipv6.conf.lo.disable_ipv6=0 net.ipv6.conf.default.disable_ipv6=0 net.ipv6.conf.all.disable_ipv6=0
-  cache_directories:
-    - "~/docker"
-  override:
-    - if [[ -e ~/docker/kapacitor-builder.tar ]]; then docker load -i ~/docker/kapacitor-builder.tar; fi
-    - docker build -t influxdata/kapacitor-builder .
-    - mkdir -p ~/docker; docker save influxdata/kapacitor-builder > ~/docker/kapacitor-builder.tar
+    pre:
+      # setup ipv6
+      - sudo sysctl -w net.ipv6.conf.lo.disable_ipv6=0 net.ipv6.conf.default.disable_ipv6=0 net.ipv6.conf.all.disable_ipv6=0
+    cache_directories:
+        - "~/docker"
+    override:
+      - ./test.sh save
 
 test:
-  override:
-    - ./build.sh --test --generate --no-stash
-    - ./build.sh --test --generate --no-stash --race
-
+    override:
+        - bash circle-test.sh:
+            parallel: true
 
 deployment:
-  release:
-    tag: /v[0-9]+(\.[0-9]+){2}(-rc[0-9]+)?/
-    commands:
-      - ./build.sh --clean --generate --package --upload --platform=all --arch=all
+    release:
+        tag: /v[0-9]+(\.[0-9]+){2}(-rc[0-9]+)?/
+        commands:
+            - ./build.sh --clean --generate --package --upload --platform=all --arch=all

gobuild.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# This script run inside the Dockerfile_build_ubuntu64_git container and
+# gets the latests Go source code and compiles it.
+# Then passes control over to the normal build.py script
+
+set -e
+
+cd /go/src
+git fetch --all
+git checkout $GO_CHECKOUT
+# Merge in recent changes if we are on a branch
+# if we checked out a tag just ignore the error
+git pull || true
+./make.bash
+
+# Run normal build.py
+cd "$PROJECT_DIR"
+exec ./build.py "$@"