From ae423e1d1e591d54cd4da32ffe8835b5fea05ed0 Mon Sep 17 00:00:00 2001
From: Stephan Ewen <sewen@apache.org>
Date: Thu, 11 May 2017 17:00:03 +0200
Subject: [PATCH] [FLINK-6514] [build] Create a proper separate Hadoop uber jar
 for 'flink-dist' assembly

This closes #3876
---
 flink-dist/src/main/assemblies/bin.xml        |   8 +-
 flink-dist/src/main/assemblies/opt.xml        |   7 -
 .../flink-shaded-hadoop2-uber/pom.xml         | 129 ++++++++++++++++++
 flink-shaded-hadoop/pom.xml                   |  38 ++++--
 4 files changed, 157 insertions(+), 25 deletions(-)
 create mode 100644 flink-shaded-hadoop/flink-shaded-hadoop2-uber/pom.xml

diff --git a/flink-dist/src/main/assemblies/bin.xml b/flink-dist/src/main/assemblies/bin.xml
index cefb477297cdc..45fd79238f478 100644
--- a/flink-dist/src/main/assemblies/bin.xml
+++ b/flink-dist/src/main/assemblies/bin.xml
@@ -33,9 +33,9 @@ under the License.
 		<dependencySet>
 			<outputDirectory>lib</outputDirectory>
 			<unpack>false</unpack>
-			<useTransitiveDependencies>true</useTransitiveDependencies>
 			<useProjectArtifact>false</useProjectArtifact>
 			<useProjectAttachments>false</useProjectAttachments>
+			<useTransitiveDependencies>true</useTransitiveDependencies>
 			<useTransitiveFiltering>true</useTransitiveFiltering>
 
 			<includes>
@@ -54,11 +54,11 @@ under the License.
 			<fileMode>0644</fileMode>
 		</file>
 
-		<!-- copy the Hadoop fat jar -->
+		<!-- copy the Hadoop uber jar -->
 		<file>
-			<source>../flink-shaded-hadoop/flink-shaded-hadoop2/target/flink-shaded-hadoop2-${project.version}.jar</source>
+			<source>../flink-shaded-hadoop/flink-shaded-hadoop2-uber/target/flink-shaded-hadoop2-uber-${project.version}.jar</source>
 			<outputDirectory>lib/</outputDirectory>
-			<destName>flink-shaded-hadoop2-${project.version}.jar</destName>
+			<destName>flink-shaded-hadoop2-uber-${project.version}.jar</destName>
 			<fileMode>0644</fileMode>
 		</file>
 
diff --git a/flink-dist/src/main/assemblies/opt.xml b/flink-dist/src/main/assemblies/opt.xml
index 23846be45150d..bb04d286d3e39 100644
--- a/flink-dist/src/main/assemblies/opt.xml
+++ b/flink-dist/src/main/assemblies/opt.xml
@@ -110,12 +110,5 @@
 			<destName>flink-metrics-datadog-${project.version}.jar</destName>
 			<fileMode>0644</fileMode>
 		</file>
-
-		<file>
-			<source>../flink-shaded-hadoop/flink-shaded-hadoop2/target/flink-shaded-hadoop2-${project.version}.jar</source>
-			<outputDirectory>opt/</outputDirectory>
-			<destName>flink-shaded-hadoop2-${project.version}.jar</destName>
-			<fileMode>0644</fileMode>
-		</file>
 	</files>
 </assembly>
diff --git a/flink-shaded-hadoop/flink-shaded-hadoop2-uber/pom.xml b/flink-shaded-hadoop/flink-shaded-hadoop2-uber/pom.xml
new file mode 100644
index 0000000000000..1ccff32b618c3
--- /dev/null
+++ b/flink-shaded-hadoop/flink-shaded-hadoop2-uber/pom.xml
@@ -0,0 +1,129 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+	<modelVersion>4.0.0</modelVersion>
+
+	<parent>
+		<groupId>org.apache.flink</groupId>
+		<artifactId>flink-shaded-hadoop</artifactId>
+		<version>1.4-SNAPSHOT</version>
+		<relativePath>..</relativePath>
+	</parent>
+
+	<artifactId>flink-shaded-hadoop2-uber</artifactId>
+	<name>flink-shaded-hadoop2-uber</name>
+
+	<packaging>jar</packaging>
+
+	<!--
+		the only dependency of the 'flink-shaded-hadoop2' artifact, out
+		of which we build a fat jar
+	-->
+
+	<dependencies>
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-shaded-hadoop2</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+	</dependencies>
+
+	
+
+	<build>
+		<plugins>
+
+			<!-- 
+				Don't deploy this uber-jar. It is not referenced by any other artifact.
+				Its sole purpose is to be included in the 'flink-dist' build.
+			-->
+ 
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-deploy-plugin</artifactId>
+				<configuration>
+					<skip>true</skip>
+				</configuration>
+			</plugin>
+
+			<!-- 
+				Build an uber jar of the shaded-hadoop-dependency
+				and all its transitive dependencies
+			-->
+
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-shade-plugin</artifactId>
+				<executions>
+					<execution>
+						<id>shade-hadoop</id>
+						<phase>package</phase>
+						<goals>
+							<goal>shade</goal>
+						</goals>
+						<configuration>
+							<shadedArtifactAttached>false</shadedArtifactAttached>
+							<createDependencyReducedPom>true</createDependencyReducedPom>
+							<dependencyReducedPomLocation>${project.basedir}/target/dependency-reduced-pom.xml</dependencyReducedPomLocation>
+							<promoteTransitiveDependencies>true</promoteTransitiveDependencies>
+
+							<filters>
+								<!-- Exclude signatures -->
+								<filter>
+									<artifact>*</artifact>
+									<excludes>
+										<exclude>META-INF/*.SF</exclude>
+										<exclude>META-INF/*.DSA</exclude>
+										<exclude>META-INF/*.RSA</exclude>
+									</excludes>
+								</filter>
+							</filters>
+
+							<artifactSet>
+								<includes>
+									<include>*:*</include>
+								</includes>
+								<excludes>
+									<exclude>org.slf4j:*</exclude>
+									<exclude>log4j:*</exclude>
+								</excludes>
+							</artifactSet>
+
+							<transformers>
+								<!-- The service transformer is needed to merge META-INF/services files -->
+								<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+								<!-- The ApacheNoticeResourceTransformer collects and aggregates NOTICE files -->
+								<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer"/>
+								<!-- The ApacheLicenseResourceTransformer prevents duplicate Apache Licenses -->
+								<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
+							</transformers>
+
+						</configuration>
+					</execution>
+				</executions>
+			</plugin>
+
+		</plugins>
+	</build>
+
+</project>
diff --git a/flink-shaded-hadoop/pom.xml b/flink-shaded-hadoop/pom.xml
index 1d957ab3b220e..a549ae99ee1ca 100644
--- a/flink-shaded-hadoop/pom.xml
+++ b/flink-shaded-hadoop/pom.xml
@@ -36,15 +36,31 @@ under the License.
 
 	<modules>
 		<module>flink-shaded-hadoop2</module>
+		<module>flink-shaded-hadoop2-uber</module>
 	</modules>
 
 	<dependencies>
-		<!-- Flink already includes JSR 305. Setting this to provided excludes it from the dependencies-->
+		<!-- drop dependencies that are already provided by Flink from the shaded / uber jars -->
 		<dependency>
 			<groupId>com.google.code.findbugs</groupId>
 			<artifactId>jsr305</artifactId>
 			<scope>provided</scope>
 		</dependency>
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-api</artifactId>
+			<scope>provided</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.zookeeper</groupId>
+			<artifactId>zookeeper</artifactId>
+			<scope>provided</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.avro</groupId>
+			<artifactId>avro</artifactId>
+			<scope>provided</scope>
+		</dependency>
 	</dependencies>
 
 	<profiles>
@@ -92,24 +108,14 @@ under the License.
 										<exclude>META-INF/*.RSA</exclude>
 									</excludes>
 								</filter>
-								<filter>
-									<artifact>org.slf4j:*</artifact>
-									<excludes>
-										<exclude>org/slf4j/impl/**</exclude>
-									</excludes>
-								</filter>
-								<!-- Exclude Hadoop's log4j. Hadoop can use Flink's log4j dependency -->
-								<filter>
-									<artifact>log4j:*</artifact>
-									<excludes>
-										<exclude>org/apache/log4j/**</exclude>
-									</excludes>
-								</filter>
 							</filters>
 							<transformers>
 								<!-- The service transformer is needed to merge META-INF/services files -->
 								<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+								<!-- The ApacheNoticeResourceTransformer collects and aggregates NOTICE files -->
 								<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer"/>
+								<!-- The ApacheLicenseResourceTransformer prevents duplicate Apache Licenses -->
+								<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
 							</transformers>
 							<artifactSet>
 								<includes>
@@ -127,6 +133,10 @@ under the License.
 									<include>org.apache.httpcomponents:*</include>
 									<include>commons-httpclient:commons-httpclient</include>
 								</includes>
+								<excludes>
+									<exclude>org.slf4j:*</exclude>
+									<exclude>log4j:*</exclude>
+								</excludes>
 							</artifactSet>
 							<relocations>
 								<relocation>