[FLINK-1637] Reduce number of files in uberjar for java 6

This closes apache#450

.travis.yml

@@ -14,13 +14,13 @@ language: java
 #See https://issues.apache.org/jira/browse/FLINK-1072
 matrix:
   include:
-    - jdk: "openjdk6"
-      env: PROFILE="-Dhadoop.profile=1"
     - jdk: "oraclejdk7"
       env: PROFILE="-Dhadoop.profile=1"
-    - jdk: "openjdk6"
+    - jdk: "openjdk6" # this will also deploy a uberjar to s3 at some point
+      env: PROFILE="-Dhadoop.profile=1"
+    - jdk: "openjdk7"
       env: PROFILE="-P!include-yarn -Dhadoop.version=2.0.0-alpha"
-    - jdk: "oraclejdk7"
+    - jdk: "openjdk6" # we must use openjdk6 here to deploy a java6 compatible uber-jar for YARN
       env: PROFILE="-Dhadoop.version=2.2.0"
     - jdk: "oraclejdk8"
       env: PROFILE="-Dhadoop.version=2.6.0"

flink-dist/pom.xml

@@ -113,12 +113,6 @@ under the License.
 			<version>${project.version}</version>
 		</dependency>
 
-		<dependency>
-			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-connectors</artifactId>
-			<version>${project.version}</version>
-		</dependency>
-
 		<dependency>
 			<groupId>org.apache.flink</groupId>
 			<artifactId>flink-streaming-examples</artifactId>
@@ -235,23 +229,6 @@ under the License.
 			</build>
 		</profile>
 
-		<profile>
-			<id>hadoop-2</id>
-			<activation>
-				<property>
-					<!-- Please do not remove the 'hadoop2' comment. See ./tools/generate_specific_pom.sh -->
-					<!--hadoop2--><name>!hadoop.profile</name>
-				</property>
-			</activation>
-			<dependencies>
-				<dependency>
-					<groupId>org.apache.flink</groupId>
-					<artifactId>flink-hbase</artifactId>
-					<version>${project.version}</version>
-				</dependency>
-			</dependencies>
-		</profile>
-
 		<profile>
 			<id>debian-package</id>
 			<build>

flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/wordcount/WordCount.java

@@ -53,7 +53,8 @@ public class WordCount {
 	// *************************************************************************
 
 	public static void main(String[] args) throws Exception {
-
+
+
 		if(!parseParameters(args)) {
 			return;
 		}

tools/create_release_files.sh

@@ -28,6 +28,11 @@
 #
 #  https://github.com/apache/spark/blob/master/dev/create-release/create-release.sh
 #
+
+# Added by rmetzger
+echo "Please make sure to use java 6 OPENJDK (not oracle) to build the release binaries"
+exit 1
+
 CURR_DIR=`pwd`
 if [[ `basename $CURR_DIR` != "tools" ]] ; then
   echo "You have to call the script from the tools/ dir"

tools/deploy_to_maven.sh

@@ -27,9 +27,7 @@
 # 4. deploy snapshot and s3 (hadoop2 - 2.2.0)
 # 5. Nothing (hadoop2 - 2.5.1)
 
-# Changes (since travis changed the id assignment)
-# switched 2. with 3.
-# switched 5. with 6.
+
 
 echo "install lifecylce mapping fake plugin"
 git clone https://github.com/mfriedenhagen/dummy-lifecycle-mapping-plugin.git
@@ -47,9 +45,11 @@ function getVersion() {
 	fi
 	flink_home="`dirname \"$here\"`"
 	cd $flink_home
-	echo `mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version|grep -Ev '(^\[|Download\w+:)'`
+	echo `mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -E '^([0-9]+.[0-9]+(.[0-9]+)?(-[a-zA-Z0-9]+)?)$'`
 }
 
+pwd
+
 # this will take a while
 CURRENT_FLINK_VERSION=`getVersion`
 if [[ "$CURRENT_FLINK_VERSION" == *-SNAPSHOT ]]; then
@@ -60,6 +60,7 @@ fi
 
 echo "detected current version as: '$CURRENT_FLINK_VERSION' ; hadoop1: $CURRENT_FLINK_VERSION_HADOOP1 "
 
+
 # Check if push/commit is eligible for pushing
 echo "Job: $TRAVIS_JOB_NUMBER ; isPR: $TRAVIS_PULL_REQUEST"
 if [[ $TRAVIS_PULL_REQUEST == "false" ]] ; then

tools/travis_mvn_watchdog.sh

@@ -179,5 +179,24 @@ rm $MVN_EXIT
 
 upload_artifacts_s3
 
+# Check the number of files in the uber jar and fail the build if there are too many files (see: FLINK-1637)
+
+# since we are in flink/tools/artifacts
+# we are going back to
+cd ../../
+
+
+UBERJAR=`find . | grep uberjar | head -n 1`
+if [ -z "$UBERJAR" ] ; then
+	echo "Uberjar not found. Assuming failed build";
+else 
+	jar tf $UBERJAR | wc -l > num_files_in_uberjar
+	NUM_FILES_IN_UBERJAR=`cat num_files_in_uberjar`
+	echo "Files in uberjar: $NUM_FILES_IN_UBERJAR. Uberjar: $UBERJAR"
+	if [ "$NUM_FILES_IN_UBERJAR" -ge "65536" ] ; then
+		echo "WARN: The number of files in the uberjar ($NUM_FILES_IN_UBERJAR) exceeds the maximum number of possible files for Java 6 (65536)"
+	fi
+fi
+
 # Exit code for Travis build success/failure
 exit $EXIT_CODE