Skip to content

Commit

Permalink
Provide unshaded pulsar-kafka-client (apache#1566)
Browse files Browse the repository at this point in the history
* Provide unshaded pulsar-kafka-client

* update doc

* comment on classes
  • Loading branch information
sijie authored and merlimat committed Apr 13, 2018
1 parent fc67fe2 commit 31dae05
Show file tree
Hide file tree
Showing 4 changed files with 217 additions and 147 deletions.
1 change: 1 addition & 0 deletions pulsar-client-kafka-compat/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@

<modules>
<module>pulsar-client-kafka</module>
<module>pulsar-client-kafka-shaded</module>
<module>pulsar-client-kafka-tests</module>
</modules>
</project>
196 changes: 196 additions & 0 deletions pulsar-client-kafka-compat/pulsar-client-kafka-shaded/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,196 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<project
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>org.apache.pulsar</groupId>
<artifactId>pulsar-client-kafka-compat</artifactId>
<version>2.0.0-incubating-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>

<artifactId>pulsar-client-kafka</artifactId>
<name>Pulsar Kafka compatibility :: API</name>

<description>Drop-in replacement for Kafka client library that publishes and consumes
messages on Pulsar topics</description>

<properties>
<kafka.version>0.10.2.1</kafka.version>
</properties>

<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>pulsar-client-kafka-original</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<createDependencyReducedPom>true</createDependencyReducedPom>
<promoteTransitiveDependencies>true</promoteTransitiveDependencies>
<artifactSet>
<includes>
<include>org.apache.kafka:kafka-clients</include>
<include>org.apache.pulsar:pulsar-client-kafka-original</include>
<include>org.apache.pulsar:pulsar-client-original</include>
<include>org.apache.commons:commons-lang3</include>
<include>commons-codec:commons-codec</include>
<include>commons-collections:commons-collections</include>
<include>org.asynchttpclient:*</include>
<include>io.netty:netty-codec-http</include>
<include>io.netty:netty-transport-native-epoll</include>
<include>org.reactivestreams:reactive-streams</include>
<include>com.typesafe.netty:netty-reactive-streams</include>
<include>org.javassist:javassist</include>
<include>com.google.protobuf:protobuf-java</include>
<include>com.google.guava:guava</include>
<include>com.google.code.gson:gson</include>
<include>com.fasterxml.jackson.core</include>
<include>io.netty:netty</include>
<include>io.netty:netty-all</include>
<include>org.apache.pulsar:pulsar-common</include>
<include>org.apache.bookkeeper:circe-checksum</include>
<include>net.jpountz.lz4:lz4</include>
<include>com.yahoo.datasketches:sketches-core</include>
<include>org.apache.httpcomponents:httpclient</include>
<include>commons-logging:commons-logging</include>
<include>org.apache.httpcomponents:httpcore</include>
</includes>
</artifactSet>
<filters>
<filter>
<artifact>commons-logging:commons-logging</artifact>
<includes>
<include>**</include>
</includes>
</filter>
</filters>
<relocations>
<relocation>
<pattern>org.apache.kafka.clients.producer.KafkaProducer</pattern>
<shadedPattern>org.apache.kafka.clients.producer.OriginalKafkaProducer</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.kafka.clients.producer.PulsarKafkaProducer</pattern>
<shadedPattern>org.apache.kafka.clients.producer.KafkaProducer</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.kafka.clients.consumer.KafkaConsumer</pattern>
<shadedPattern>org.apache.kafka.clients.consumer.OriginalKafkaConsumer</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.kafka.clients.consumer.PulsarKafkaConsumer</pattern>
<shadedPattern>org.apache.kafka.clients.consumer.KafkaConsumer</shadedPattern>
</relocation>

<!-- General relocation rules for Pulsar client dependencies -->

<relocation>
<pattern>org.asynchttpclient</pattern>
<shadedPattern>org.apache.pulsar.shade.org.asynchttpclient</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.commons</pattern>
<shadedPattern>org.apache.pulsar.shade.org.apache.commons</shadedPattern>
</relocation>
<relocation>
<pattern>com.google</pattern>
<shadedPattern>org.apache.pulsar.shade.com.google</shadedPattern>
</relocation>
<relocation>
<pattern>com.fasterxml.jackson</pattern>
<shadedPattern>org.apache.pulsar.shade.com.fasterxml.jackson</shadedPattern>
</relocation>
<relocation>
<pattern>io.netty</pattern>
<shadedPattern>org.apache.pulsar.shade.io.netty</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.pulsar.common</pattern>
<shadedPattern>org.apache.pulsar.shade.org.apache.pulsar.common</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.pulsar.policies</pattern>
<shadedPattern>org.apache.pulsar.shade.org.apache.pulsar.policies</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.pulsar.checksum</pattern>
<shadedPattern>org.apache.pulsar.shade.org.apache.pulsar.checksum</shadedPattern>
</relocation>
<relocation>
<pattern>com.scurrilous.circe</pattern>
<shadedPattern>org.apache.pulsar.shade.com.scurrilous.circe</shadedPattern>
</relocation>
<relocation>
<pattern>net.jpountz</pattern>
<shadedPattern>org.apache.pulsar.shade.net.jpountz</shadedPattern>
</relocation>
<relocation>
<pattern>com.yahoo.datasketches</pattern>
<shadedPattern>org.apache.pulsar.shade.com.yahoo.datasketches</shadedPattern>
</relocation>
<relocation>
<pattern>com.yahoo.sketches</pattern>
<shadedPattern>org.apache.pulsar.shade.com.yahoo.sketches</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.http</pattern>
<shadedPattern>org.apache.pulsar.shade.org.apache.http</shadedPattern>
</relocation>
</relocations>
<filters>
<filter>
<artifact>net.jpountz.lz4:lz4</artifact>
<includes>
<include>**</include>
</includes>
</filter>
<filter>
<artifact>org.apache.pulsar:pulsar-client-original</artifact>
<includes>
<include>**</include>
</includes>
</filter>
</filters>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
150 changes: 3 additions & 147 deletions pulsar-client-kafka-compat/pulsar-client-kafka/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,10 @@
<relativePath>..</relativePath>
</parent>

<artifactId>pulsar-client-kafka</artifactId>
<name>Pulsar Kafka compatibility :: API</name>
<artifactId>pulsar-client-kafka-original</artifactId>
<name>Pulsar Kafka compatibility :: API (original)</name>

<description>Drop-in replacement for Kafka client library that publishes and consumes
messages on Pulsar topics</description>
<description>Kafka client library that publishes and consumes messages on Pulsar topics</description>

<dependencies>
<dependency>
Expand Down Expand Up @@ -84,147 +83,4 @@
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<createDependencyReducedPom>true</createDependencyReducedPom>
<promoteTransitiveDependencies>true</promoteTransitiveDependencies>
<artifactSet>
<includes>
<include>org.apache.kafka:kafka-clients</include>
<include>org.apache.pulsar:pulsar-client-original</include>
<include>org.apache.commons:commons-lang3</include>
<include>commons-codec:commons-codec</include>
<include>commons-collections:commons-collections</include>
<include>org.asynchttpclient:*</include>
<include>io.netty:netty-codec-http</include>
<include>io.netty:netty-transport-native-epoll</include>
<include>org.reactivestreams:reactive-streams</include>
<include>com.typesafe.netty:netty-reactive-streams</include>
<include>org.javassist:javassist</include>
<include>com.google.protobuf:protobuf-java</include>
<include>com.google.guava:guava</include>
<include>com.google.code.gson:gson</include>
<include>com.fasterxml.jackson.core</include>
<include>io.netty:netty</include>
<include>io.netty:netty-all</include>
<include>org.apache.pulsar:pulsar-common</include>
<include>org.apache.bookkeeper:circe-checksum</include>
<include>net.jpountz.lz4:lz4</include>
<include>com.yahoo.datasketches:sketches-core</include>
<include>org.apache.httpcomponents:httpclient</include>
<include>commons-logging:commons-logging</include>
<include>org.apache.httpcomponents:httpcore</include>
</includes>
</artifactSet>
<filters>
<filter>
<artifact>commons-logging:commons-logging</artifact>
<includes>
<include>**</include>
</includes>
</filter>
</filters>
<relocations>
<relocation>
<pattern>org.apache.kafka.clients.producer.KafkaProducer</pattern>
<shadedPattern>org.apache.kafka.clients.producer.OriginalKafkaProducer</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.kafka.clients.producer.PulsarKafkaProducer</pattern>
<shadedPattern>org.apache.kafka.clients.producer.KafkaProducer</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.kafka.clients.consumer.KafkaConsumer</pattern>
<shadedPattern>org.apache.kafka.clients.consumer.OriginalKafkaConsumer</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.kafka.clients.consumer.PulsarKafkaConsumer</pattern>
<shadedPattern>org.apache.kafka.clients.consumer.KafkaConsumer</shadedPattern>
</relocation>

<!-- General relocation rules for Pulsar client dependencies -->

<relocation>
<pattern>org.asynchttpclient</pattern>
<shadedPattern>org.apache.pulsar.shade.org.asynchttpclient</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.commons</pattern>
<shadedPattern>org.apache.pulsar.shade.org.apache.commons</shadedPattern>
</relocation>
<relocation>
<pattern>com.google</pattern>
<shadedPattern>org.apache.pulsar.shade.com.google</shadedPattern>
</relocation>
<relocation>
<pattern>com.fasterxml.jackson</pattern>
<shadedPattern>org.apache.pulsar.shade.com.fasterxml.jackson</shadedPattern>
</relocation>
<relocation>
<pattern>io.netty</pattern>
<shadedPattern>org.apache.pulsar.shade.io.netty</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.pulsar.common</pattern>
<shadedPattern>org.apache.pulsar.shade.org.apache.pulsar.common</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.pulsar.policies</pattern>
<shadedPattern>org.apache.pulsar.shade.org.apache.pulsar.policies</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.pulsar.checksum</pattern>
<shadedPattern>org.apache.pulsar.shade.org.apache.pulsar.checksum</shadedPattern>
</relocation>
<relocation>
<pattern>com.scurrilous.circe</pattern>
<shadedPattern>org.apache.pulsar.shade.com.scurrilous.circe</shadedPattern>
</relocation>
<relocation>
<pattern>net.jpountz</pattern>
<shadedPattern>org.apache.pulsar.shade.net.jpountz</shadedPattern>
</relocation>
<relocation>
<pattern>com.yahoo.datasketches</pattern>
<shadedPattern>org.apache.pulsar.shade.com.yahoo.datasketches</shadedPattern>
</relocation>
<relocation>
<pattern>com.yahoo.sketches</pattern>
<shadedPattern>org.apache.pulsar.shade.com.yahoo.sketches</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.http</pattern>
<shadedPattern>org.apache.pulsar.shade.org.apache.http</shadedPattern>
</relocation>
</relocations>
<filters>
<filter>
<artifact>net.jpountz.lz4:lz4</artifact>
<includes>
<include>**</include>
</includes>
</filter>
<filter>
<artifact>org.apache.pulsar:pulsar-client-original</artifact>
<includes>
<include>**</include>
</includes>
</filter>
</filters>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
17 changes: 17 additions & 0 deletions site/docs/latest/adaptors/KafkaWrapper.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,23 @@ thing that needs to be adjusted is the configuration, to make sure to point the
producers and consumers to Pulsar service rather than Kafka and to use a particular
Pulsar topic.

## Using the Pulsar Kafka compatibility wrapper together with existing kafka client.

When migrating from Kafka to Pulsar, the application might have to use the original kafka client
and the pulsar kafka wrapper together during migration. Then you should consider using the
unshaded pulsar kafka client wrapper.

```xml
<dependency>
<groupId>org.apache.pulsar</groupId>
<artifactId>pulsar-client-kafka-original</artifactId>
<version>{{ site.current_version }}</version>
</dependency>
```

When using this dependency, you need to construct producer using `org.apache.kafka.clients.producer.PulsarKafkaProducer`
instead of `org.apache.kafka.clients.producer.KafkaProducer` and `org.apache.kafka.clients.producer.PulsarKafkaConsumer` for consumers.

## Producer example

```java
Expand Down

0 comments on commit 31dae05

Please sign in to comment.