From 8c852c2abf3a1e597dd1d139197d3420861a516c Mon Sep 17 00:00:00 2001 From: Robert Metzger Date: Thu, 27 Aug 2015 18:13:08 +0200 Subject: [PATCH] [FLINK-2584] Check for unshaded classes in fat jar and shade curator This closes #1076 --- .../flink-storm-compatibility-core/pom.xml | 5 ++ flink-runtime/pom.xml | 5 +- flink-shaded-curator/pom.xml | 89 +++++++++++++++++++ .../flink-connector-elasticsearch/pom.xml | 6 ++ .../flink-connector-kafka/pom.xml | 6 ++ pom.xml | 1 + tools/travis_mvn_watchdog.sh | 23 +++++ 7 files changed, 133 insertions(+), 2 deletions(-) create mode 100644 flink-shaded-curator/pom.xml diff --git a/flink-contrib/flink-storm-compatibility/flink-storm-compatibility-core/pom.xml b/flink-contrib/flink-storm-compatibility/flink-storm-compatibility-core/pom.xml index 92be9db2d9c14..cced6784f8baa 100644 --- a/flink-contrib/flink-storm-compatibility/flink-storm-compatibility-core/pom.xml +++ b/flink-contrib/flink-storm-compatibility/flink-storm-compatibility-core/pom.xml @@ -57,6 +57,11 @@ under the License. + + com.google.guava + guava + ${guava.version} + diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml index c5706a1fae8e4..5b8d0caf82082 100644 --- a/flink-runtime/pom.xml +++ b/flink-runtime/pom.xml @@ -193,8 +193,9 @@ under the License. - org.apache.curator - curator-recipes + org.apache.flink + flink-shaded-curator + ${project.version} diff --git a/flink-shaded-curator/pom.xml b/flink-shaded-curator/pom.xml new file mode 100644 index 0000000000000..3e3894a63fa48 --- /dev/null +++ b/flink-shaded-curator/pom.xml @@ -0,0 +1,89 @@ + + + + + 4.0.0 + + + org.apache.flink + flink-parent + 0.10-SNAPSHOT + .. + + + flink-shaded-curator + flink-shaded-curator + + jar + + + + + org.apache.curator + curator-recipes + ${curator.version} + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-flink + package + + shade + + + false + true + ${project.basedir}/target/dependency-reduced-pom.xml + + + + + + + + org.apache.curator:* + com.google.guava:* + + + + + com.google + org.apache.curator.shaded.com.google + + + + + + + + + diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-elasticsearch/pom.xml b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-elasticsearch/pom.xml index ccce7466d9824..d4a009ffec612 100644 --- a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-elasticsearch/pom.xml +++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-elasticsearch/pom.xml @@ -54,6 +54,12 @@ under the License. ${elasticsearch.version} + + com.google.guava + guava + ${guava.version} + + org.apache.flink flink-streaming-core diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/pom.xml b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/pom.xml index bbbd2be53c3a1..98d7f0d05d7fb 100644 --- a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/pom.xml +++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/pom.xml @@ -92,6 +92,12 @@ under the License. + + com.google.guava + guava + ${guava.version} + + org.apache.curator curator-test diff --git a/pom.xml b/pom.xml index 0e7f995164744..cdee6f6bd6e94 100644 --- a/pom.xml +++ b/pom.xml @@ -51,6 +51,7 @@ under the License. flink-shaded-hadoop + flink-shaded-curator flink-core flink-java flink-scala diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_mvn_watchdog.sh index 08cf6efa415d9..629c44c12a58a 100755 --- a/tools/travis_mvn_watchdog.sh +++ b/tools/travis_mvn_watchdog.sh @@ -156,6 +156,27 @@ watchdog () { done } +# Check the final fat jar for illegal artifacts +check_shaded_artifacts() { + jar tf build-target/lib/flink-dist-*.jar > allClasses + ASM=`cat allClasses | grep '^org/objectweb/asm/' | wc -l` + if [ $ASM != "0" ]; then + echo "==============================================================================" + echo "Detected $ASM asm dependencies in fat jar" + echo "==============================================================================" + exit 1 + fi + + GUAVA=`cat allClasses | grep '^com/google/common' | wc -l` + if [ $GUAVA != "0" ]; then + echo "==============================================================================" + echo "Detected $GUAVA guava dependencies in fat jar" + echo "==============================================================================" + exit 1 + fi + +} + # ============================================================================= # WATCHDOG # ============================================================================= @@ -189,6 +210,8 @@ echo "MVN exited with EXIT CODE: ${EXIT_CODE}." rm $MVN_PID rm $MVN_EXIT +check_shaded_artifacts + put_yarn_logs_to_artifacts upload_artifacts_s3