NUTCH-2631: Moving dependencies from root ivy file to indexer-kafka p…

…lugin.

default.properties

@@ -198,6 +198,7 @@ plugins.indexer=\
    org.apache.nutch.indexwriter.elastic*:\
    org.apache.nutch.indexwriter.elasticrest*:\
    org.apache.nutch.indexwriter.rabbit*:\
+   org.apache.nutch.indexwriter.kafka*:\
    org.apache.nutch.indexwriter.solr*
 
 #

ivy/ivy.xml

@@ -139,16 +139,9 @@
 			<exclude org="org.json"/>
 		</dependency>
 
-
 		<!-- RabbitMQ dependencies -->
 		<dependency org="com.rabbitmq" name="amqp-client" rev="5.2.0" conf="*->default" />
 
-		<!-- Kafka Dependencies -->
-		<!--TODO refactor these to indexer-kafka plugin somehow, currently doesn't resolve correctly-->
-		<dependency org="org.apache.kafka" name="kafka_2.12" rev="1.1.0"/>
-		<dependency org="org.apache.kafka" name="connect-json" rev="1.1.0"/>
-
-
 		<!--Added Because of Elasticsearch JEST client-->
 		<!--TODO refactor these to indexer-elastic-rest plugin somehow, currently doesn't resolve correctly-->
 		<dependency org="org.apache.httpcomponents" name="httpcore-nio" rev="4.4.9"/>

src/plugin/indexer-kafka/ivy.xml

@@ -27,7 +27,7 @@
   </info>
 
   <configurations>
-    <include file="../../..//ivy/ivy-configurations.xml"/>
+    <include file="../../../ivy/ivy-configurations.xml"/>
   </configurations>
 
   <publications>
@@ -36,8 +36,8 @@
   </publications>
 
   <dependencies>
-    <!-- <dependency org="org.apache.kafka" name="kafka_2.12" rev="1.1.0"/>
-	<dependency org="org.apache.kafka" name="connect-json" rev="1.1.0"/> -->
+      <dependency org="org.apache.kafka" name="kafka_2.12" rev="1.1.0"/>
+      <dependency org="org.apache.kafka" name="connect-json" rev="1.1.0"/>
   </dependencies>
 
 </ivy-module>

src/plugin/indexer-kafka/plugin.xml

@@ -22,19 +22,22 @@
         <library name="indexer-kafka.jar">
             <export name="*"/>
         </library>
-
-        <library name="commons-codec-1.9.jar"/>
-        <library name="commons-lang3-3.4.jar"/>
-        <library name="commons-logging-1.2.jar"/>
-        <library name="gson-2.6.2.jar"/>
-        <library name="guava-19.0.jar"/>
-        <library name="httpasyncclient-4.1.1.jar"/>
-        <library name="httpclient-4.5.2.jar"/>
-        <library name="httpcore-4.4.4.jar"/>
-        <library name="httpcore-nio-4.4.4.jar"/>
-        <library name="jest-2.0.3.jar"/>
-        <library name="jest-common-2.0.3.jar"/>
-
+        <library name="kafka_2.12-1.1.0.jar"/>
+        <library name="connect-json-1.1.0.jar"/>
+        <library name="connect-api-1.1.0.jar"/>
+        <library name="jackson-annotations-2.9.0.jar"/>
+        <library name="jackson-core-2.9.4.jar"/>
+        <library name="jackson-databind-2.9.4.jar"/>
+        <library name="jopt-simple-5.0.4.jar"/>
+        <library name="kafka-clients-1.1.0.jar"/>
+        <library name="lz4-java-1.4.jar"/>
+        <library name="metrics-core-2.2.0.jar"/>
+        <library name="scala-library-2.12.4.jar"/>
+        <library name="scala-logging_2.12-3.7.2.jar"/>
+        <library name="scala-reflect-2.12.4.jar"/>
+        <library name="snappy-java-1.1.7.1.jar"/>
+        <library name="zkclient-0.10.jar"/>
+        <library name="zookeeper-3.4.10.jar"/>
     </runtime>
 
     <requires>

src/plugin/indexer-kafka/src/java/org/apache/nutch/indexwriter/kafka/KafkaIndexWriter.java

@@ -14,8 +14,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
-//TODO refactor the dependencies out of root ivy file
 package org.apache.nutch.indexwriter.kafka;
 
 import org.apache.commons.lang.StringUtils;
@@ -101,7 +99,8 @@ public void open(IndexWriterParams params) throws IOException {
         keySerializer);
     configProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,
         valueSerializer);
-
+
+    Thread.currentThread().setContextClassLoader(this.getClass().getClassLoader());
     producer = new KafkaProducer<String, JsonNode>(configProperties);
   }