From 9c54e73247cdc9cb33b2fe77287a377798f6b7c7 Mon Sep 17 00:00:00 2001 From: Ivan Kelly Date: Wed, 28 Feb 2018 18:10:56 +0100 Subject: [PATCH] Docker image for integration testing (#1275) * Docker image for integration testing A docker image built with the pulsar distribution tarball. The same image can be used to run zookeeper, bookkeeper, the broker or a proxy. Scripts are provided to run each. There is also a script to initialize the cluster. All services are run with supervisor, so that tests will be able to start and stop the services. * Fix build --- docker/pom.xml | 1 - pom.xml | 6 ++ .../latest-version-image/Dockerfile | 39 +++++++++ .../latest-version-image/conf/bookie.conf | 26 ++++++ .../latest-version-image/conf/broker.conf | 26 ++++++ .../latest-version-image/conf/global-zk.conf | 26 ++++++ .../latest-version-image/conf/local-zk.conf | 26 ++++++ .../latest-version-image/conf/proxy.conf | 26 ++++++ .../conf/supervisord.conf | 40 +++++++++ .../latest-version-image/pom.xml | 85 +++++++++++++++++++ .../scripts/init-cluster.sh | 36 ++++++++ .../scripts/run-bookie.sh | 30 +++++++ .../scripts/run-broker.sh | 30 +++++++ .../scripts/run-global-zk.sh | 29 +++++++ .../scripts/run-local-zk.sh | 29 +++++++ .../latest-version-image/scripts/run-proxy.sh | 30 +++++++ tests/docker-images/pom.xml | 37 ++++++++ tests/pom.xml | 48 +++++++++++ 18 files changed, 569 insertions(+), 1 deletion(-) create mode 100644 tests/docker-images/latest-version-image/Dockerfile create mode 100644 tests/docker-images/latest-version-image/conf/bookie.conf create mode 100644 tests/docker-images/latest-version-image/conf/broker.conf create mode 100644 tests/docker-images/latest-version-image/conf/global-zk.conf create mode 100644 tests/docker-images/latest-version-image/conf/local-zk.conf create mode 100644 tests/docker-images/latest-version-image/conf/proxy.conf create mode 100644 tests/docker-images/latest-version-image/conf/supervisord.conf create mode 100644 tests/docker-images/latest-version-image/pom.xml create mode 100755 tests/docker-images/latest-version-image/scripts/init-cluster.sh create mode 100755 tests/docker-images/latest-version-image/scripts/run-bookie.sh create mode 100755 tests/docker-images/latest-version-image/scripts/run-broker.sh create mode 100755 tests/docker-images/latest-version-image/scripts/run-global-zk.sh create mode 100755 tests/docker-images/latest-version-image/scripts/run-local-zk.sh create mode 100755 tests/docker-images/latest-version-image/scripts/run-proxy.sh create mode 100644 tests/docker-images/pom.xml create mode 100644 tests/pom.xml diff --git a/docker/pom.xml b/docker/pom.xml index 2693bf766f7c1..ec4f8ca5724fa 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -32,7 +32,6 @@ docker-images Apache Pulsar :: Docker Images - 1.3.7 apachepulsar diff --git a/pom.xml b/pom.xml index 57222c6b759a4..c3b1406d3a8ac 100644 --- a/pom.xml +++ b/pom.xml @@ -97,6 +97,8 @@ flexible messaging model and an intuitive client API. pulsar-client-kafka-compat pulsar-zookeeper all + docker + tests @@ -123,6 +125,7 @@ flexible messaging model and an intuitive client API. 1.55 2.8.4 6.19 + 1.3.7 @@ -982,6 +985,9 @@ flexible messaging model and an intuitive client API. + + docker + diff --git a/tests/docker-images/latest-version-image/Dockerfile b/tests/docker-images/latest-version-image/Dockerfile new file mode 100644 index 0000000000000..e15c28943d358 --- /dev/null +++ b/tests/docker-images/latest-version-image/Dockerfile @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +FROM apachepulsar/pulsar:latest + +RUN apt-get update && apt-get install -y supervisor + +RUN mkdir -p /var/log/pulsar && mkdir -p /var/run/supervisor/ + +COPY conf/supervisord.conf /etc/supervisord.conf +COPY conf/global-zk.conf /etc/supervisord/conf.d/global-zk.conf +COPY conf/local-zk.conf /etc/supervisord/conf.d/local-zk.conf +COPY conf/bookie.conf /etc/supervisord/conf.d/bookie.conf +COPY conf/broker.conf /etc/supervisord/conf.d/broker.conf +COPY conf/proxy.conf /etc/supervisord/conf.d/proxy.conf + +COPY scripts/init-cluster.sh /pulsar/bin +COPY scripts/run-global-zk.sh /pulsar/bin +COPY scripts/run-local-zk.sh /pulsar/bin +COPY scripts/run-bookie.sh /pulsar/bin +COPY scripts/run-broker.sh /pulsar/bin +COPY scripts/run-proxy.sh /pulsar/bin + diff --git a/tests/docker-images/latest-version-image/conf/bookie.conf b/tests/docker-images/latest-version-image/conf/bookie.conf new file mode 100644 index 0000000000000..53fd06ecdc8e8 --- /dev/null +++ b/tests/docker-images/latest-version-image/conf/bookie.conf @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +[program:bookie] +autostart=false +redirect_stderr=true +stdout_logfile=/var/log/pulsar/bookie.log +directory=/pulsar +command=/pulsar/bin/pulsar bookie + diff --git a/tests/docker-images/latest-version-image/conf/broker.conf b/tests/docker-images/latest-version-image/conf/broker.conf new file mode 100644 index 0000000000000..ca8843f03ddd7 --- /dev/null +++ b/tests/docker-images/latest-version-image/conf/broker.conf @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +[program:broker] +autostart=false +redirect_stderr=true +stdout_logfile=/var/log/pulsar/broker.log +directory=/pulsar +command=/pulsar/bin/pulsar broker + diff --git a/tests/docker-images/latest-version-image/conf/global-zk.conf b/tests/docker-images/latest-version-image/conf/global-zk.conf new file mode 100644 index 0000000000000..f589adecd3081 --- /dev/null +++ b/tests/docker-images/latest-version-image/conf/global-zk.conf @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +[program:global-zk] +autostart=false +redirect_stderr=true +stdout_logfile=/var/log/pulsar/global-zk.log +directory=/pulsar +command=/pulsar/bin/pulsar global-zookeeper + diff --git a/tests/docker-images/latest-version-image/conf/local-zk.conf b/tests/docker-images/latest-version-image/conf/local-zk.conf new file mode 100644 index 0000000000000..f5daba03e331d --- /dev/null +++ b/tests/docker-images/latest-version-image/conf/local-zk.conf @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +[program:local-zk] +autostart=false +redirect_stderr=true +stdout_logfile=/var/log/pulsar/local-zk.log +directory=/pulsar +command=/pulsar/bin/pulsar zookeeper + diff --git a/tests/docker-images/latest-version-image/conf/proxy.conf b/tests/docker-images/latest-version-image/conf/proxy.conf new file mode 100644 index 0000000000000..7ab7e8fe90d1e --- /dev/null +++ b/tests/docker-images/latest-version-image/conf/proxy.conf @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +[program:proxy] +autostart=false +redirect_stderr=true +stdout_logfile=/var/log/pulsar/proxy.log +directory=/pulsar +command=/pulsar/bin/pulsar proxy + diff --git a/tests/docker-images/latest-version-image/conf/supervisord.conf b/tests/docker-images/latest-version-image/conf/supervisord.conf new file mode 100644 index 0000000000000..ee48be3265813 --- /dev/null +++ b/tests/docker-images/latest-version-image/conf/supervisord.conf @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +[supervisord] +nodaemon=true +logfile=/var/log/supervisord.log +logfile_maxbytes=50MB +logfile_backups=10 +loglevel=info +pidfile=/var/run/supervisord.pid +minfds=1024 +minprocs=200 + +[unix_http_server] +file=/var/run/supervisor/supervisor.sock + +[supervisorctl] +serverurl=unix:///var/run/supervisor/supervisor.sock + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[include] +files = /etc/supervisord/conf.d/*.conf diff --git a/tests/docker-images/latest-version-image/pom.xml b/tests/docker-images/latest-version-image/pom.xml new file mode 100644 index 0000000000000..9acad9e30a2ea --- /dev/null +++ b/tests/docker-images/latest-version-image/pom.xml @@ -0,0 +1,85 @@ + + + + org.apache.pulsar.tests + docker-images + 2.0.0-incubating-SNAPSHOT + + 4.0.0 + org.apache.pulsar.tests + latest-version-image + Apache Pulsar :: Tests :: Docker Images :: Latest Version Testing + pom + + + + docker + + + integrationTests + + + + + org.apache.pulsar + pulsar-docker-image + ${project.parent.version} + docker-info + + + + + + com.spotify + dockerfile-maven-plugin + ${dockerfile-maven.version} + + + default + + build + + + + add-latest-tag + + tag + + + apachepulsar/pulsar-test-latest-version + latest + + + + + apachepulsar/pulsar-test-latest-version + ${project.version} + false + true + + + + + + + diff --git a/tests/docker-images/latest-version-image/scripts/init-cluster.sh b/tests/docker-images/latest-version-image/scripts/init-cluster.sh new file mode 100755 index 0000000000000..e4403b3af5907 --- /dev/null +++ b/tests/docker-images/latest-version-image/scripts/init-cluster.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +ZNODE="/initialized" + +bin/watch-znode.py -z $zkServers -p / -w + +bin/watch-znode.py -z $zkServers -p $ZNODE -e +if [ $? != 0 ]; then + echo Initializing cluster + bin/apply-config-from-env.py conf/bookkeeper.conf && + bin/pulsar initialize-cluster-metadata --cluster $cluster --zookeeper $zkServers \ + --global-zookeeper $globalZkServers --web-service-url http://$pulsarNode:8080/ \ + --broker-service-url http://$pulsarNode:6650/ && + bin/watch-znode.py -z $zkServers -p $ZNODE -c + echo Initialized +else + echo Already Initialized +fi diff --git a/tests/docker-images/latest-version-image/scripts/run-bookie.sh b/tests/docker-images/latest-version-image/scripts/run-bookie.sh new file mode 100755 index 0000000000000..adaaced6096a0 --- /dev/null +++ b/tests/docker-images/latest-version-image/scripts/run-bookie.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +bin/apply-config-from-env.py conf/bookkeeper.conf && \ + bin/apply-config-from-env.py conf/pulsar_env.sh + +if [ -z "$NO_AUTOSTART" ]; then + sed -i 's/autostart=.*/autostart=true/' /etc/supervisord/conf.d/bookie.conf +fi + +bin/watch-znode.py -z $zkServers -p /initialized -w +exec /usr/bin/supervisord -c /etc/supervisord.conf + diff --git a/tests/docker-images/latest-version-image/scripts/run-broker.sh b/tests/docker-images/latest-version-image/scripts/run-broker.sh new file mode 100755 index 0000000000000..cef8b65c0dc9b --- /dev/null +++ b/tests/docker-images/latest-version-image/scripts/run-broker.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +bin/apply-config-from-env.py conf/broker.conf && \ + bin/apply-config-from-env.py conf/pulsar_env.sh + +if [ -z "$NO_AUTOSTART" ]; then + sed -i 's/autostart=.*/autostart=true/' /etc/supervisord/conf.d/broker.conf +fi + +bin/watch-znode.py -z $zookeeperServers -p /initialized -w +exec /usr/bin/supervisord -c /etc/supervisord.conf + diff --git a/tests/docker-images/latest-version-image/scripts/run-global-zk.sh b/tests/docker-images/latest-version-image/scripts/run-global-zk.sh new file mode 100755 index 0000000000000..783ef2d46802a --- /dev/null +++ b/tests/docker-images/latest-version-image/scripts/run-global-zk.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +bin/apply-config-from-env.py conf/zookeeper.conf && \ + bin/apply-config-from-env.py conf/pulsar_env.sh && \ + bin/generate-zookeeper-config.sh conf/global_zookeeper.conf + +if [ -z "$NO_AUTOSTART" ]; then + sed -i 's/autostart=.*/autostart=true/' /etc/supervisord/conf.d/global-zk.conf +fi + +exec /usr/bin/supervisord -c /etc/supervisord.conf diff --git a/tests/docker-images/latest-version-image/scripts/run-local-zk.sh b/tests/docker-images/latest-version-image/scripts/run-local-zk.sh new file mode 100755 index 0000000000000..b27212b6ba9d3 --- /dev/null +++ b/tests/docker-images/latest-version-image/scripts/run-local-zk.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +bin/apply-config-from-env.py conf/zookeeper.conf && \ + bin/apply-config-from-env.py conf/pulsar_env.sh && \ + bin/generate-zookeeper-config.sh conf/zookeeper.conf + +if [ -z "$NO_AUTOSTART" ]; then + sed -i 's/autostart=.*/autostart=true/' /etc/supervisord/conf.d/local-zk.conf +fi + +exec /usr/bin/supervisord -c /etc/supervisord.conf diff --git a/tests/docker-images/latest-version-image/scripts/run-proxy.sh b/tests/docker-images/latest-version-image/scripts/run-proxy.sh new file mode 100755 index 0000000000000..05b38b00aafa2 --- /dev/null +++ b/tests/docker-images/latest-version-image/scripts/run-proxy.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +bin/apply-config-from-env.py conf/proxy.conf && \ + bin/apply-config-from-env.py conf/pulsar_env.sh + +if [ -z "$NO_AUTOSTART" ]; then + sed -i 's/autostart=.*/autostart=true/' /etc/supervisord/conf.d/proxy.conf +fi + +bin/watch-znode.py -z $zookeeperServers -p /initialized -w +exec /usr/bin/supervisord -c /etc/supervisord.conf + diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml new file mode 100644 index 0000000000000..9fbbc2d3feb2d --- /dev/null +++ b/tests/docker-images/pom.xml @@ -0,0 +1,37 @@ + + + + pom + 4.0.0 + + org.apache.pulsar.tests + tests-parent + 2.0.0-incubating-SNAPSHOT + + org.apache.pulsar.tests + docker-images + Apache Pulsar :: Tests :: Docker Images + + latest-version-image + + diff --git a/tests/pom.xml b/tests/pom.xml new file mode 100644 index 0000000000000..ea6f20b598447 --- /dev/null +++ b/tests/pom.xml @@ -0,0 +1,48 @@ + + + + pom + 4.0.0 + + org.apache.pulsar + pulsar + 2.0.0-incubating-SNAPSHOT + + org.apache.pulsar.tests + tests-parent + Apache Pulsar :: Tests + + docker-images + + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + +