Add support for Hashicorp Configuration Language for templates (Data-…

…Science-Platform#18)

* Changes for adding HCL support

Fixes Issue Data-Science-Platform#17

* Adding tests

--integration tests
--updated version for jinjava

* Fixed formatting for NomadServiceIntegrationSpec

* Trying to fix scaa integration tests

TODO
-- http-api-tests for Broccoli would still fail

* Fixed IT to add the new HCL template

* Fix tls tests
-- Moved tls tests to common http-tests directory
-- Added check_service calls for before_each
-- changed response data for newer version of nomad for edit-parameter-start

* changed path for tls tests in travis

* Changed min nomad version for jobs/parse

* Filter out special characters for docker branch tags

* Improved branch filtering

* Reverted change to node version in .travis.yml

* fixed formatting issues with build.sbt

* Fixed imports

* Fixed InstanceServiceSpec tests

* removed unneeded print statements

* removed json from templaterenderer javadoc

.travis.yml

@@ -35,17 +35,20 @@ jobs:
     - stage: integration
       script:
         - . prepare-docker-builds
-        - docker build -t frosner/cluster-broccoli-test docker/test/
+        # build and test against old nomad
+        - docker build --build-arg nomad=0.4.0 -t frosner/cluster-broccoli-test docker/test/
         - sbt it:test
         - bash <(curl -s https://raw.githubusercontent.com/FRosner/http-api-tester/master/http-api-tester) -v -d http-api-tests/instance-persistence-couchdb
         - bash <(curl -s https://raw.githubusercontent.com/FRosner/http-api-tester/master/http-api-tester) -v -d http-api-tests/broccoli-only
         - bash <(curl -s https://raw.githubusercontent.com/FRosner/http-api-tester/master/http-api-tester) -v -d http-api-tests/broccoli-nomad
         - bash <(curl -s https://raw.githubusercontent.com/FRosner/http-api-tester/master/http-api-tester) -v -d http-api-tests/broccoli-nomad-consul
         - bash <(curl -s https://raw.githubusercontent.com/FRosner/http-api-tester/master/http-api-tester) -v -d http-api-tests/instance-persistence-dir
-        # TLS part, with nomad 
-        - docker build -t frosner/cluster-broccoli-test-tls -f docker/test/Dockerfiletls docker/test/
+        # build and test against new nomad
+        - docker build --build-arg nomad=0.9.5 -t frosner/cluster-broccoli-test docker/test/
         - sbt it:test
-        - bash <(curl -s https://raw.githubusercontent.com/ynux/http-api-tester/master/http-api-tester) -v -o ge -d http-api-tests-tls/broccoli-nomad 
+        # TLS part, with nomad
+        - docker build -t frosner/cluster-broccoli-test-tls -f docker/test/Dockerfiletls docker/test/
+        - bash <(curl -s https://raw.githubusercontent.com/ynux/http-api-tester/master/http-api-tester) -v -o ge -d http-api-tests/broccoli-nomad-tls
     - stage: docker
       script:
         # Publish docker image

build.sbt

@@ -65,7 +65,22 @@ lazy val server = project
            s"${dockerUsername.value.get}/${(packageName in Docker).value}:${(version in Docker).value}"),
       Option(System.getenv("TRAVIS_BRANCH"))
         .map(_.replaceAllLiterally("/", "_"))
-        .map(name => if (name == "master") "latest" else name)
+        .map(name =>
+          if (name == "master") "latest"
+          else {
+            val specials = Set('_', '.', '-')
+            val notAllowedBegin = Set('.', '-)
+            name.zipWithIndex
+              .map {
+                case (c, i) =>
+                  val out = if (c.isLetterOrDigit || specials.contains(c)) c else '_'
+                  i match {
+                    case 0 => if (notAllowedBegin.contains(out)) '_' else out
+                    case _ => out
+                  }
+              }
+              .mkString("")
+        })
         .map { tag =>
           List("-t", s"${dockerUsername.value.get}/${(packageName in Docker).value}:$tag")
         }

docker/test/Dockerfile

@@ -5,6 +5,7 @@ RUN echo "deb http://cdn-fastly.deb.debian.org/debian jessie main" > /etc/apt/so
 RUN echo 'deb http://archive.debian.org/debian jessie-backports main' > /etc/apt/sources.list.d/jessie-backports.list
 RUN sed -i '/deb http:\/\/deb.debian.org\/debian jessie-updates main/d' /etc/apt/sources.list
 RUN echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/05validnocheck
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys AA8E81B4331F7F50
 
 # install docker libraries
 RUN apt-get update && \
@@ -14,13 +15,6 @@ RUN apt-get update && \
 RUN curl -fsSL get.docker.com -o get-docker.sh
 RUN sh get-docker.sh
 
-# put nomad
-RUN curl https://releases.hashicorp.com/nomad/0.4.0/nomad_0.4.0_linux_amd64.zip > nomad.zip
-RUN unzip nomad.zip
-RUN echo "#!/bin/bash" > /usr/bin/nomad && \
-  echo "exec /nomad agent -dev" >> /usr/bin/nomad && \
-  chmod 777 /usr/bin/nomad
-
 # put consul
 RUN curl https://releases.hashicorp.com/consul/0.6.4/consul_0.6.4_linux_amd64.zip > consul.zip
 RUN unzip consul.zip
@@ -36,3 +30,11 @@ ADD templates /cluster-broccoli-dist/templates
 RUN mkdir /cluster-broccoli-dist/instances
 RUN ln -s /cluster-broccoli-dist/bin/cluster-broccoli /usr/bin/cluster-broccoli
 
+# put nomad
+ARG nomad="0.4.0"
+ENV NOMAD_VERSION=$nomad
+RUN curl https://releases.hashicorp.com/nomad/$nomad/nomad_${nomad}_linux_amd64.zip > nomad.zip
+RUN unzip nomad.zip
+RUN echo "#!/bin/bash" > /usr/bin/nomad && \
+  echo "exec /nomad agent -dev" >> /usr/bin/nomad && \
+  chmod 777 /usr/bin/nomad

docker/test/Dockerfiletls

@@ -1,42 +1,11 @@
-FROM java:8
+FROM frosner/cluster-broccoli-test:latest
 
-# Fix for java:8 image
-RUN echo "deb http://cdn-fastly.deb.debian.org/debian jessie main" > /etc/apt/sources.list.d/jessie.list
-RUN echo 'deb http://archive.debian.org/debian jessie-backports main' > /etc/apt/sources.list.d/jessie-backports.list
-RUN sed -i '/deb http:\/\/deb.debian.org\/debian jessie-updates main/d' /etc/apt/sources.list
-RUN echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/05validnocheck
-
-ENV NOMAD_VERSION 0.8.7
-# install docker libraries
-RUN apt-get update && \
-  apt-get install -y apt-transport-https ca-certificates && \
-  apt-get clean all
-
-RUN curl -fsSL get.docker.com -o get-docker.sh && \
-  sh get-docker.sh
-
-# put nomad
-RUN curl https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_linux_amd64.zip > nomad.zip && \
-  unzip nomad.zip
-RUN mkdir /etc/nomad.d/
 ## nomad certificates and configs
 COPY ./nomad-server-config/ /etc/nomad.d/
 
 RUN echo "#!/bin/bash" > /usr/bin/nomad && \
   echo "exec /nomad agent -dev -config=/etc/nomad.d/" >> /usr/bin/nomad && \
   chmod 777 /usr/bin/nomad
 
-# put consul
-RUN curl https://releases.hashicorp.com/consul/0.6.4/consul_0.6.4_linux_amd64.zip > consul.zip
-RUN unzip consul.zip
-RUN echo "#!/bin/bash" > /usr/bin/consul && \
-  echo "exec /consul agent -dev -node travis -bind 127.0.0.1" >> /usr/bin/consul && \
-  chmod 777 /usr/bin/consul
-
-# put cluster broccoli
-ADD cluster-broccoli-dist /cluster-broccoli-dist
-ADD templates /cluster-broccoli-dist/templates
-RUN mkdir /cluster-broccoli-dist/instances
-RUN ln -s /cluster-broccoli-dist/bin/cluster-broccoli /usr/bin/cluster-broccoli
 ## cluster-broccoli certificates and configs
 COPY ./cluster-broccoli-files/ /

http-api-tests-tls/broccoli-nomad/api-v1-instances-delete/before → http-api-tests/broccoli-nomad-tls/api-v1-instances-delete/before

@@ -8,7 +8,11 @@ curl -H 'Content-Type: application/json' \
     'http://localhost:9000/api/v1/instances/test-http'
 sleep $BROCCOLI_SLEEP_SHORT
 curl -X DELETE 'http://localhost:9000/api/v1/instances/test-http'
+echo -e "\nSleeping after deleting instance"
 sleep $BROCCOLI_SLEEP_SHORT
-curl -k -X PUT https://localhost:4646/v1/system/gc
+curl -k -X PUT 'https://localhost:4646/v1/system/gc'
+sleep $BROCCOLI_SLEEP_SHORT
+# For some reason gc does not work in the first try inside nomad 0.9.5
+curl -k -X PUT 'https://localhost:4646/v1/system/gc'
 sleep $BROCCOLI_SLEEP_SHORT