Docker image for integration testing (apache#1275)

* Docker image for integration testing

A docker image built with the pulsar distribution tarball. The same
image can be used to run zookeeper, bookkeeper, the broker or a
proxy. Scripts are provided to run each. There is also a script to
initialize the cluster.

All services are run with supervisor, so that tests will be able to
start and stop the services.

* Fix build

docker/pom.xml

@@ -32,7 +32,6 @@
   <artifactId>docker-images</artifactId>
   <name>Apache Pulsar :: Docker Images</name>
   <properties>
-    <dockerfile-maven.version>1.3.7</dockerfile-maven.version>
     <docker.organization>apachepulsar</docker.organization>
   </properties>
   <modules>

pom.xml

@@ -97,6 +97,8 @@ flexible messaging model and an intuitive client API.</description>
     <module>pulsar-client-kafka-compat</module>
     <module>pulsar-zookeeper</module>
     <module>all</module>
+    <module>docker</module>
+    <module>tests</module>
   </modules>
 
   <issueManagement>
@@ -123,6 +125,7 @@ flexible messaging model and an intuitive client API.</description>
     <bouncycastle.version>1.55</bouncycastle.version>
     <jackson.version>2.8.4</jackson.version>
     <puppycrawl.checkstyle.version>6.19</puppycrawl.checkstyle.version>
+    <dockerfile-maven.version>1.3.7</dockerfile-maven.version>
   </properties>
 
   <dependencyManagement>
@@ -982,6 +985,9 @@ flexible messaging model and an intuitive client API.</description>
         </plugins>
       </reporting>
     </profile>
+    <profile>
+      <id>docker</id>
+    </profile>
   </profiles>
 
   <repositories>

tests/docker-images/latest-version-image/Dockerfile

@@ -0,0 +1,39 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+FROM apachepulsar/pulsar:latest
+
+RUN apt-get update && apt-get install -y supervisor
+
+RUN mkdir -p /var/log/pulsar && mkdir -p /var/run/supervisor/
+
+COPY conf/supervisord.conf /etc/supervisord.conf
+COPY conf/global-zk.conf /etc/supervisord/conf.d/global-zk.conf
+COPY conf/local-zk.conf /etc/supervisord/conf.d/local-zk.conf
+COPY conf/bookie.conf /etc/supervisord/conf.d/bookie.conf
+COPY conf/broker.conf /etc/supervisord/conf.d/broker.conf
+COPY conf/proxy.conf /etc/supervisord/conf.d/proxy.conf
+
+COPY scripts/init-cluster.sh /pulsar/bin
+COPY scripts/run-global-zk.sh /pulsar/bin
+COPY scripts/run-local-zk.sh /pulsar/bin
+COPY scripts/run-bookie.sh /pulsar/bin
+COPY scripts/run-broker.sh /pulsar/bin
+COPY scripts/run-proxy.sh /pulsar/bin
+

tests/docker-images/latest-version-image/conf/bookie.conf

@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+[program:bookie]
+autostart=false
+redirect_stderr=true
+stdout_logfile=/var/log/pulsar/bookie.log
+directory=/pulsar
+command=/pulsar/bin/pulsar bookie
+

tests/docker-images/latest-version-image/conf/broker.conf

@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+[program:broker]
+autostart=false
+redirect_stderr=true
+stdout_logfile=/var/log/pulsar/broker.log
+directory=/pulsar
+command=/pulsar/bin/pulsar broker
+

tests/docker-images/latest-version-image/conf/global-zk.conf

@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+[program:global-zk]
+autostart=false
+redirect_stderr=true
+stdout_logfile=/var/log/pulsar/global-zk.log
+directory=/pulsar
+command=/pulsar/bin/pulsar global-zookeeper
+

tests/docker-images/latest-version-image/conf/local-zk.conf

@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+[program:local-zk]
+autostart=false
+redirect_stderr=true
+stdout_logfile=/var/log/pulsar/local-zk.log
+directory=/pulsar
+command=/pulsar/bin/pulsar zookeeper
+

tests/docker-images/latest-version-image/conf/proxy.conf

@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+[program:proxy]
+autostart=false
+redirect_stderr=true
+stdout_logfile=/var/log/pulsar/proxy.log
+directory=/pulsar
+command=/pulsar/bin/pulsar proxy
+

tests/docker-images/latest-version-image/conf/supervisord.conf

@@ -0,0 +1,40 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+[supervisord]
+nodaemon=true
+logfile=/var/log/supervisord.log
+logfile_maxbytes=50MB
+logfile_backups=10
+loglevel=info
+pidfile=/var/run/supervisord.pid
+minfds=1024
+minprocs=200
+
+[unix_http_server]
+file=/var/run/supervisor/supervisor.sock
+
+[supervisorctl]
+serverurl=unix:///var/run/supervisor/supervisor.sock
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[include]
+files = /etc/supervisord/conf.d/*.conf

tests/docker-images/latest-version-image/pom.xml

@@ -0,0 +1,85 @@
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <parent>
+    <groupId>org.apache.pulsar.tests</groupId>
+    <artifactId>docker-images</artifactId>
+    <version>2.0.0-incubating-SNAPSHOT</version>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.apache.pulsar.tests</groupId>
+  <artifactId>latest-version-image</artifactId>
+  <name>Apache Pulsar :: Tests :: Docker Images :: Latest Version Testing</name>
+  <packaging>pom</packaging>
+
+  <profiles>
+    <profile>
+      <id>docker</id>
+      <activation>
+        <property>
+          <name>integrationTests</name>
+        </property>
+      </activation>
+      <dependencies>
+        <dependency>
+          <groupId>org.apache.pulsar</groupId>
+          <artifactId>pulsar-docker-image</artifactId>
+          <version>${project.parent.version}</version>
+          <classifier>docker-info</classifier>
+        </dependency>
+      </dependencies>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>com.spotify</groupId>
+            <artifactId>dockerfile-maven-plugin</artifactId>
+            <version>${dockerfile-maven.version}</version>
+            <executions>
+              <execution>
+                <id>default</id>
+                <goals>
+                  <goal>build</goal>
+                </goals>
+              </execution>
+              <execution>
+                <id>add-latest-tag</id>
+                <goals>
+                  <goal>tag</goal>
+                </goals>
+                <configuration>
+                  <repository>apachepulsar/pulsar-test-latest-version</repository>
+                  <tag>latest</tag>
+                </configuration>
+              </execution>
+            </executions>
+            <configuration>
+              <repository>apachepulsar/pulsar-test-latest-version</repository>
+              <tag>${project.version}</tag>
+              <pullNewerImage>false</pullNewerImage>
+              <noCache>true</noCache>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+  </profiles>
+</project>

tests/docker-images/latest-version-image/scripts/init-cluster.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+ZNODE="/initialized"
+
+bin/watch-znode.py -z $zkServers -p / -w
+
+bin/watch-znode.py -z $zkServers -p $ZNODE -e
+if [ $? != 0 ]; then
+    echo Initializing cluster
+    bin/apply-config-from-env.py conf/bookkeeper.conf &&
+        bin/pulsar initialize-cluster-metadata --cluster $cluster --zookeeper $zkServers \
+                   --global-zookeeper $globalZkServers --web-service-url http://$pulsarNode:8080/ \
+                   --broker-service-url http://$pulsarNode:6650/ &&
+        bin/watch-znode.py -z $zkServers -p $ZNODE -c
+    echo Initialized
+else
+    echo Already Initialized
+fi