diff --git a/.gitignore b/.gitignore
index a2e04fa15aba..1e9336fbe869 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,3 +14,4 @@ nb-configuration.xml
*.iws
*~
+.vscode/
diff --git a/nifi-docker/dockerhub/Dockerfile b/nifi-docker/dockerhub/Dockerfile
index 48a2dcd82b2f..91b180e1d7dc 100644
--- a/nifi-docker/dockerhub/Dockerfile
+++ b/nifi-docker/dockerhub/Dockerfile
@@ -25,19 +25,26 @@ ARG GID=1000
ARG NIFI_VERSION=1.7.0
ARG MIRROR=https://archive.apache.org/dist
-ENV NIFI_BASE_DIR /opt/nifi
+ENV NIFI_BASE_DIR /opt/nifi
ENV NIFI_HOME=${NIFI_BASE_DIR}/nifi-${NIFI_VERSION} \
NIFI_BINARY_URL=/nifi/${NIFI_VERSION}/nifi-${NIFI_VERSION}-bin.tar.gz
+ENV NIFI_PID_DIR=${NIFI_HOME}/run
+ENV NIFI_LOG_DIR=${NIFI_HOME}/logs
-ADD sh/ /opt/nifi/scripts/
+ADD sh/ ${NIFI_BASE_DIR}/scripts/
-# Setup NiFi user
+# Setup NiFi user and create necessary directories
RUN groupadd -g ${GID} nifi || groupmod -n nifi `getent group ${GID} | cut -d: -f1` \
&& useradd --shell /bin/bash -u ${UID} -g ${GID} -m nifi \
&& mkdir -p ${NIFI_HOME}/conf/templates \
+ && mkdir -p $NIFI_BASE_DIR/data \
+ && mkdir -p $NIFI_BASE_DIR/flowfile_repository \
+ && mkdir -p $NIFI_BASE_DIR/content_repository \
+ && mkdir -p $NIFI_BASE_DIR/provenance_repository \
+ && mkdir -p $NIFI_LOG_DIR \
&& chown -R nifi:nifi ${NIFI_BASE_DIR} \
&& apt-get update \
- && apt-get install -y jq xmlstarlet
+ && apt-get install -y jq xmlstarlet procps
USER nifi
@@ -45,8 +52,10 @@ USER nifi
RUN curl -fSL ${MIRROR}/${NIFI_BINARY_URL} -o ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}-bin.tar.gz \
&& echo "$(curl https://archive.apache.org/dist/${NIFI_BINARY_URL}.sha256) *${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}-bin.tar.gz" | sha256sum -c - \
&& tar -xvzf ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}-bin.tar.gz -C ${NIFI_BASE_DIR} \
- && rm ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}-bin.tar.gz \
- && chown -R nifi:nifi ${NIFI_HOME}
+ && rm ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}-bin.tar.gz
+
+# Clear nifi-env.sh in favour of configuring all environment variables in the Dockerfile
+RUN echo "#!/bin/sh\n" > $NIFI_HOME/bin/nifi-env.sh
# Web HTTP(s) & Socket Site-to-Site Ports
EXPOSE 8080 8443 10000
@@ -54,4 +63,12 @@ EXPOSE 8080 8443 10000
WORKDIR ${NIFI_HOME}
# Apply configuration and start NiFi
-CMD ${NIFI_BASE_DIR}/scripts/start.sh
+#
+# We need to use the exec form to avoid running our command in a subshell and omitting signals,
+# thus being unable to shut down gracefully:
+# https://docs.docker.com/engine/reference/builder/#entrypoint
+#
+# Also we need to use relative path, because the exec form does not invoke a command shell,
+# thus normal shell processing does not happen:
+# https://docs.docker.com/engine/reference/builder/#exec-form-entrypoint-example
+ENTRYPOINT ["../scripts/start.sh"]
diff --git a/nifi-docker/dockerhub/pom.xml b/nifi-docker/dockerhub/pom.xml
new file mode 100644
index 000000000000..4cf324c325cd
--- /dev/null
+++ b/nifi-docker/dockerhub/pom.xml
@@ -0,0 +1,76 @@
+
+
+
+ 4.0.0
+
+
+ org.apache.nifi
+ nifi-docker
+ 1.7.0-SNAPSHOT
+
+
+ dockerhub
+
+
+
+ docker
+
+
+
+ com.spotify
+ dockerfile-maven-plugin
+ 1.3.5
+
+
+ default
+
+ build
+
+
+
+ 1000
+ 1000
+ 1.6.0
+
+ apache/nifi
+
+ ${project.version}-dockerhub
+
+
+
+
+
+ exec-maven-plugin
+ org.codehaus.mojo
+
+
+ Docker integration tests
+ integration-test
+
+ exec
+
+
+
+ ${project.version}-dockerhub
+ 1.6.0
+
+ ${project.basedir}/../dockermaven/integration-test.sh
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/nifi-docker/dockermaven/Dockerfile b/nifi-docker/dockermaven/Dockerfile
index 62dd03c2fc28..19d14a95d9b6 100644
--- a/nifi-docker/dockermaven/Dockerfile
+++ b/nifi-docker/dockermaven/Dockerfile
@@ -26,23 +26,43 @@ ARG NIFI_BINARY
ENV NIFI_BASE_DIR /opt/nifi
ENV NIFI_HOME $NIFI_BASE_DIR/nifi-$NIFI_VERSION
+ENV NIFI_PID_DIR=${NIFI_HOME}/run
+ENV NIFI_LOG_DIR=${NIFI_HOME}/logs
-# Setup NiFi user
-RUN groupadd -g $GID nifi || groupmod -n nifi `getent group $GID | cut -d: -f1` \
- && useradd --shell /bin/bash -u $UID -g $GID -m nifi \
- && mkdir -p $NIFI_HOME/conf/templates \
- && chown -R nifi:nifi $NIFI_BASE_DIR
+ADD sh/ ${NIFI_BASE_DIR}/scripts/
ADD $NIFI_BINARY $NIFI_BASE_DIR
-RUN chown -R nifi:nifi $NIFI_HOME
+
+# Setup NiFi user and create necessary directories
+RUN groupadd -g ${GID} nifi || groupmod -n nifi `getent group ${GID} | cut -d: -f1` \
+ && useradd --shell /bin/bash -u ${UID} -g ${GID} -m nifi \
+ && mkdir -p ${NIFI_HOME}/conf/templates \
+ && mkdir -p $NIFI_BASE_DIR/data \
+ && mkdir -p $NIFI_BASE_DIR/flowfile_repository \
+ && mkdir -p $NIFI_BASE_DIR/content_repository \
+ && mkdir -p $NIFI_BASE_DIR/provenance_repository \
+ && mkdir -p $NIFI_LOG_DIR \
+ && chown -R nifi:nifi ${NIFI_BASE_DIR} \
+ && apt-get update \
+ && apt-get install -y jq xmlstarlet procps
USER nifi
-# Web HTTP Port & Remote Site-to-Site Ports
-EXPOSE 8080 8181
+# Clear nifi-env.sh in favour of configuring all environment variables in the Dockerfile
+RUN echo "#!/bin/sh\n" > $NIFI_HOME/bin/nifi-env.sh
+
+# Web HTTP(s) & Socket Site-to-Site Ports
+EXPOSE 8080 8443 10000
-WORKDIR $NIFI_HOME
+WORKDIR ${NIFI_HOME}
-# Startup NiFi
-ENTRYPOINT ["bin/nifi.sh"]
-CMD ["run"]
+# Apply configuration and start NiFi
+#
+# We need to use the exec form to avoid running our command in a subshell and omitting signals,
+# thus being unable to shut down gracefully:
+# https://docs.docker.com/engine/reference/builder/#entrypoint
+#
+# Also we need to use relative path, because the exec form does not invoke a command shell,
+# thus normal shell processing does not happen:
+# https://docs.docker.com/engine/reference/builder/#exec-form-entrypoint-example
+ENTRYPOINT ["../scripts/start.sh"]
diff --git a/nifi-docker/dockermaven/integration-test.sh b/nifi-docker/dockermaven/integration-test.sh
new file mode 100755
index 000000000000..e1eedda2c1cc
--- /dev/null
+++ b/nifi-docker/dockermaven/integration-test.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -exuo pipefail
+
+TAG=$1
+VERSION=$2
+
+trap "{ docker rm -f nifi-${TAG}-integration-test; }" EXIT
+
+echo "Checking that all files are owned by NiFi"
+test -z $(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c "find /opt/nifi ! -user nifi")
+
+echo "Checking environment variables"
+test "/opt/nifi/nifi-${VERSION}" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c 'echo -n $NIFI_HOME')"
+test "/opt/nifi/nifi-${VERSION}/logs" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c 'echo -n $NIFI_LOG_DIR')"
+test "/opt/nifi/nifi-${VERSION}/run" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c 'echo -n $NIFI_PID_DIR')"
+test "/opt/nifi" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c 'echo -n $NIFI_BASE_DIR')"
+
+echo "Starting NiFi container..."
+docker run -d --name nifi-${TAG}-integration-test apache/nifi:${TAG}
+
+IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' nifi-${TAG}-integration-test)
+
+for i in $(seq 1 10) :; do
+ if docker exec nifi-${TAG}-integration-test bash -c "ss -ntl | grep 8080"; then
+ break
+ fi
+ sleep 10
+done
+
+echo "Checking system diagnostics"
+test ${VERSION} = $(docker exec nifi-${TAG}-integration-test bash -c "curl -s $IP:8080/nifi-api/system-diagnostics | jq .systemDiagnostics.aggregateSnapshot.versionInfo.niFiVersion -r")
+
+echo "Stopping NiFi container"
+time docker stop nifi-${TAG}-integration-test
\ No newline at end of file
diff --git a/nifi-docker/dockermaven/pom.xml b/nifi-docker/dockermaven/pom.xml
index a5395525ab02..fee654297d60 100644
--- a/nifi-docker/dockermaven/pom.xml
+++ b/nifi-docker/dockermaven/pom.xml
@@ -43,8 +43,7 @@
target/nifi-${nifi.version}-bin.tar.gz
apache/nifi
- ${project.version}
- latest
+ ${project.version}-dockermaven
@@ -72,6 +71,26 @@
+
+ exec-maven-plugin
+ org.codehaus.mojo
+
+
+ Docker integration tests
+ integration-test
+
+ exec
+
+
+
+ ${project.version}-dockermaven
+ ${project.version}
+
+ ${project.basedir}/integration-test.sh
+
+
+
+
diff --git a/nifi-docker/dockermaven/sh/common.sh b/nifi-docker/dockermaven/sh/common.sh
new file mode 100755
index 000000000000..a0a65501bedd
--- /dev/null
+++ b/nifi-docker/dockermaven/sh/common.sh
@@ -0,0 +1,28 @@
+#!/bin/sh -e
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# 1 - value to search for
+# 2 - value to replace
+# 3 - file to perform replacement inline
+prop_replace () {
+ target_file=${3:-${nifi_props_file}}
+ echo 'replacing target file ' ${target_file}
+ sed -i -e "s|^$1=.*$|$1=$2|" ${target_file}
+}
+
+# NIFI_HOME is defined by an ENV command in the backing Dockerfile
+export nifi_props_file=${NIFI_HOME}/conf/nifi.properties
+export hostname=$(hostname)
diff --git a/nifi-docker/dockermaven/sh/secure.sh b/nifi-docker/dockermaven/sh/secure.sh
new file mode 100644
index 000000000000..5ff56e48ebba
--- /dev/null
+++ b/nifi-docker/dockermaven/sh/secure.sh
@@ -0,0 +1,64 @@
+#!/bin/sh -e
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+scripts_dir='/opt/nifi/scripts'
+
+[ -f "${scripts_dir}/common.sh" ] && . "${scripts_dir}/common.sh"
+
+# Perform idempotent changes of configuration to support secure environments
+echo 'Configuring environment with SSL settings'
+
+: ${KEYSTORE_PATH:?"Must specify an absolute path to the keystore being used."}
+if [ ! -f "${KEYSTORE_PATH}" ]; then
+ echo "Keystore file specified (${KEYSTORE_PATH}) does not exist."
+ exit 1
+fi
+: ${KEYSTORE_TYPE:?"Must specify the type of keystore (JKS, PKCS12, PEM) of the keystore being used."}
+: ${KEYSTORE_PASSWORD:?"Must specify the password of the keystore being used."}
+
+: ${TRUSTSTORE_PATH:?"Must specify an absolute path to the truststore being used."}
+if [ ! -f "${TRUSTSTORE_PATH}" ]; then
+ echo "Keystore file specified (${TRUSTSTORE_PATH}) does not exist."
+ exit 1
+fi
+: ${TRUSTSTORE_TYPE:?"Must specify the type of truststore (JKS, PKCS12, PEM) of the truststore being used."}
+: ${TRUSTSTORE_PASSWORD:?"Must specify the password of the truststore being used."}
+
+prop_replace 'nifi.security.keystore' "${KEYSTORE_PATH}"
+prop_replace 'nifi.security.keystoreType' "${KEYSTORE_TYPE}"
+prop_replace 'nifi.security.keystorePasswd' "${KEYSTORE_PASSWORD}"
+prop_replace 'nifi.security.truststore' "${TRUSTSTORE_PATH}"
+prop_replace 'nifi.security.truststoreType' "${TRUSTSTORE_TYPE}"
+prop_replace 'nifi.security.truststorePasswd' "${TRUSTSTORE_PASSWORD}"
+
+# Disable HTTP and enable HTTPS
+prop_replace 'nifi.web.http.port' ''
+prop_replace 'nifi.web.http.host' ''
+prop_replace 'nifi.web.https.port' "${NIFI_WEB_HTTPS_PORT:-8443}"
+prop_replace 'nifi.web.https.host' "${NIFI_WEB_HTTPS_HOST:-$HOSTNAME}"
+prop_replace 'nifi.remote.input.secure' 'true'
+
+# Check if the user has specified a nifi.web.proxy.host setting and handle appropriately
+if [ -z "${NIFI_WEB_PROXY_HOST}" ]; then
+ echo 'NIFI_WEB_PROXY_HOST was not set but NiFi is configured to run in a secure mode. The NiFi UI may be inaccessible if using port mapping.'
+else
+ prop_replace 'nifi.web.proxy.host' "${NIFI_WEB_PROXY_HOST}"
+fi
+
+# Establish initial user and an associated admin identity
+sed -i -e 's||'"${INITIAL_ADMIN_IDENTITY}"'|' ${NIFI_HOME}/conf/authorizers.xml
+sed -i -e 's||'"${INITIAL_ADMIN_IDENTITY}"'|' ${NIFI_HOME}/conf/authorizers.xml
diff --git a/nifi-docker/dockermaven/sh/start.sh b/nifi-docker/dockermaven/sh/start.sh
new file mode 100755
index 000000000000..2775bc73b71e
--- /dev/null
+++ b/nifi-docker/dockermaven/sh/start.sh
@@ -0,0 +1,59 @@
+#!/bin/sh -e
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+scripts_dir='/opt/nifi/scripts'
+
+[ -f "${scripts_dir}/common.sh" ] && . "${scripts_dir}/common.sh"
+
+# Establish baseline properties
+prop_replace 'nifi.web.http.port' "${NIFI_WEB_HTTP_PORT:-8080}"
+prop_replace 'nifi.web.http.host' "${NIFI_WEB_HTTP_HOST:-$HOSTNAME}"
+prop_replace 'nifi.remote.input.host' "${NIFI_REMOTE_INPUT_HOST:-$HOSTNAME}"
+prop_replace 'nifi.remote.input.socket.port' "${NIFI_REMOTE_INPUT_SOCKET_PORT:-10000}"
+prop_replace 'nifi.remote.input.secure' 'false'
+
+# Check if we are secured or unsecured
+case ${AUTH} in
+ tls)
+ echo 'Enabling Two-Way SSL user authentication'
+ . "${scripts_dir}/secure.sh"
+ ;;
+ ldap)
+ echo 'Enabling LDAP user authentication'
+ # Reference ldap-provider in properties
+ prop_replace 'nifi.security.user.login.identity.provider' 'ldap-provider'
+ prop_replace 'nifi.security.needClientAuth' 'WANT'
+
+ . "${scripts_dir}/secure.sh"
+ . "${scripts_dir}/update_login_providers.sh"
+ ;;
+ *)
+ if [ ! -z "${NIFI_WEB_PROXY_HOST}" ]; then
+ echo 'NIFI_WEB_PROXY_HOST was set but NiFi is not configured to run in a secure mode. Will not update nifi.web.proxy.host.'
+ fi
+ ;;
+esac
+
+# Continuously provide logs so that 'docker logs' can produce them
+tail -F "${NIFI_HOME}/logs/nifi-app.log" &
+"${NIFI_HOME}/bin/nifi.sh" run &
+nifi_pid="$!"
+
+trap "echo Received trapped signal, beginning shutdown...;" KILL TERM HUP INT EXIT;
+
+echo NiFi running with PID ${nifi_pid}.
+wait ${nifi_pid}
\ No newline at end of file
diff --git a/nifi-docker/dockermaven/sh/update_login_providers.sh b/nifi-docker/dockermaven/sh/update_login_providers.sh
new file mode 100755
index 000000000000..e124960eec4a
--- /dev/null
+++ b/nifi-docker/dockermaven/sh/update_login_providers.sh
@@ -0,0 +1,47 @@
+#!/bin/sh -e
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+login_providers_file=${NIFI_HOME}/conf/login-identity-providers.xml
+property_xpath='//loginIdentityProviders/provider/property'
+
+# Update a given property in the login-identity-providers file if a value is specified
+edit_property() {
+ property_name=$1
+ property_value=$2
+
+ if [ -n "${property_value}" ]; then
+ xmlstarlet ed --inplace -u "${property_xpath}[@name='${property_name}']" -v "${property_value}" "${login_providers_file}"
+ fi
+}
+
+# Remove comments to enable the ldap-provider
+sed -i '/To enable the ldap-provider remove/d' "${login_providers_file}"
+
+edit_property 'Authentication Strategy' "${LDAP_AUTHENTICATION_STRATEGY}"
+edit_property 'Manager DN' "${LDAP_MANAGER_DN}"
+edit_property 'Manager Password' "${LDAP_MANAGER_PASSWORD}"
+edit_property 'TLS - Keystore' "${LDAP_TLS_KEYSTORE}"
+edit_property 'TLS - Keystore Password' "${LDAP_TLS_KEYSTORE_PASSWORD}"
+edit_property 'TLS - Keystore Type' "${LDAP_TLS_KEYSTORE_TYPE}"
+edit_property 'TLS - Truststore' "${LDAP_TLS_TRUSTSTORE}"
+edit_property 'TLS - Truststore Password' "${LDAP_TLS_TRUSTSTORE_PASSWORD}"
+edit_property 'TLS - Truststore Type' "${LDAP_TLS_TRUSTSTORE_TYPE}"
+edit_property 'TLS - Protocol' "${LDAP_TLS_PROTOCOL}"
+edit_property 'Url' "${LDAP_URL}"
+edit_property 'User Search Base' "${LDAP_USER_SEARCH_BASE}"
+edit_property 'User Search Filter' "${LDAP_USER_SEARCH_FILTER}"
+edit_property 'Identity Strategy' "${LDAP_IDENTITY_STRATEGY}"
\ No newline at end of file
diff --git a/nifi-docker/pom.xml b/nifi-docker/pom.xml
index 3404e6c51904..ab9b28d5a2f5 100644
--- a/nifi-docker/pom.xml
+++ b/nifi-docker/pom.xml
@@ -29,6 +29,7 @@ language governing permissions and limitations under the License. -->
dockermaven
+ dockerhub