Skip to content

Commit 6cade31

Browse files
legendtklruanwenjun
legendtkl
and
ruanwenjun
authored
[Feature][Flink-SQL-connector] add flink sql connector kafka and docs (apache#1878)
* [Feature][Flink-SQL-connector] add flink sql connector kafka and docs * address review comment Co-authored-by: ruanwenjun <[email protected]>
1 parent 91ddc2e commit 6cade31

File tree

13 files changed

+136
-25
lines changed

13 files changed

+136
-25
lines changed

‎docs/en/connector/flink-sql/Kafka.md

+74
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
# Flink SQL Kafka Connector
2+
3+
## Description
4+
5+
With kafka connector, we can read data from kafka and write data to kafka using Flink SQL. Refer to the [Kafka connector](https://nightlies.apache.org/flink/flink-docs-release-1.13/docs/connectors/table/kafka/) for more details.
6+
7+
8+
## Usage
9+
Let us have a brief example to show how to use the connector from end to end.
10+
11+
### 1. kafka prepare
12+
Please refer to the [Kafka QuickStart](https://kafka.apache.org/quickstart) to prepare kafka environment and produce data like following:
13+
14+
```bash
15+
$ bin/kafka-console-producer.sh --topic <topic-name> --bootstrap-server localhost:9092
16+
```
17+
18+
After executing the command, we will come to the interactive mode. Print the following message to send data to kafka.
19+
```bash
20+
{"id":1,"name":"abc"}
21+
>{"id":2,"name":"def"}
22+
>{"id":3,"name":"dfs"}
23+
>{"id":4,"name":"eret"}
24+
>{"id":5,"name":"yui"}
25+
```
26+
27+
### 2. prepare seatunnel configuration
28+
Here is a simple example of seatunnel configuration.
29+
```sql
30+
SET table.dml-sync = true;
31+
32+
CREATE TABLE events (
33+
id INT,
34+
name STRING
35+
) WITH (
36+
'connector' = 'kafka',
37+
'topic'='<topic-name>',
38+
'properties.bootstrap.servers' = 'localhost:9092',
39+
'properties.group.id' = 'testGroup',
40+
'scan.startup.mode' = 'earliest-offset',
41+
'format' = 'json'
42+
);
43+
44+
CREATE TABLE print_table (
45+
id INT,
46+
name STRING
47+
) WITH (
48+
'connector' = 'print',
49+
'sink.parallelism' = '1'
50+
);
51+
52+
INSERT INTO print_table SELECT * FROM events;
53+
```
54+
55+
### 3. start flink local cluster
56+
```bash
57+
$ ${FLINK_HOME}/bin/start-cluster.sh
58+
```
59+
60+
### 4. start Flink SQL job
61+
Execute the following command in seatunnel home path to start the Flink SQL job.
62+
```bash
63+
$ bin/start-seatunnel-sql.sh -c config/kafka.sql.conf
64+
```
65+
66+
### 5. verify result
67+
After the job submitted, we can see the data printing by connector 'print' in taskmanager's log .
68+
```text
69+
+I[1, abc]
70+
+I[2, def]
71+
+I[3, dfs]
72+
+I[4, eret]
73+
+I[5, yui]
74+
```

‎pom.xml

+19
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,7 @@
173173
<guava.version>19.0</guava.version>
174174
<auto-service.version>1.0.1</auto-service.version>
175175
<jmockdata.version>4.3.0</jmockdata.version>
176+
<snappy-java.version>1.1.8.3</snappy-java.version>
176177
</properties>
177178

178179
<dependencyManagement>
@@ -611,6 +612,24 @@
611612
<artifactId>jmockdata</artifactId>
612613
<version>${jmockdata.version}</version>
613614
</dependency>
615+
616+
<dependency>
617+
<groupId>org.slf4j</groupId>
618+
<artifactId>slf4j-api</artifactId>
619+
<version>${slf4j.version}</version>
620+
</dependency>
621+
622+
<dependency>
623+
<groupId>org.slf4j</groupId>
624+
<artifactId>slf4j-log4j12</artifactId>
625+
<version>${slf4j.version}</version>
626+
</dependency>
627+
628+
<dependency>
629+
<groupId>org.xerial.snappy</groupId>
630+
<artifactId>snappy-java</artifactId>
631+
<version>${snappy-java.version}</version>
632+
</dependency>
614633
</dependencies>
615634
</dependencyManagement>
616635

‎seatunnel-connectors/seatunnel-connectors-flink-sql-dist/pom.xml

+6
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,12 @@
3131
<artifactId>flink-sql-connector-jdbc</artifactId>
3232
<version>${project.version}</version>
3333
</dependency>
34+
35+
<dependency>
36+
<groupId>org.apache.seatunnel</groupId>
37+
<artifactId>flink-sql-connector-kafka</artifactId>
38+
<version>${project.version}</version>
39+
</dependency>
3440
</dependencies>
3541

3642
<build>

‎seatunnel-connectors/seatunnel-connectors-flink-sql/flink-sql-connector-kafka/pom.xml

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one or more
4+
contributor license agreements. See the NOTICE file distributed with
5+
this work for additional information regarding copyright ownership.
6+
The ASF licenses this file to You under the Apache License, Version 2.0
7+
(the "License"); you may not use this file except in compliance with
8+
the License. You may obtain a copy of the License at
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
-->
16+
<project xmlns="http://maven.apache.org/POM/4.0.0"
17+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
18+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
19+
<parent>
20+
<artifactId>seatunnel-connectors-flink-sql</artifactId>
21+
<groupId>org.apache.seatunnel</groupId>
22+
<version>${revision}</version>
23+
</parent>
24+
<modelVersion>4.0.0</modelVersion>
25+
26+
<artifactId>flink-sql-connector-kafka</artifactId>
27+
28+
<dependencies>
29+
<dependency>
30+
<groupId>org.apache.flink</groupId>
31+
<artifactId>flink-connector-kafka_${scala.binary.version}</artifactId>
32+
</dependency>
33+
34+
</dependencies>
35+
36+
</project>

‎seatunnel-connectors/seatunnel-connectors-flink-sql/pom.xml

+1
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828

2929
<modules>
3030
<module>flink-sql-connector-jdbc</module>
31+
<module>flink-sql-connector-kafka</module>
3132
</modules>
3233

3334

‎seatunnel-dist/release-docs/LICENSE

-12
Original file line numberDiff line numberDiff line change
@@ -816,7 +816,6 @@ The text of each license is the standard Apache 2.0 license.
816816
(The Apache Software License, Version 2.0) Phoenix Core (org.apache.phoenix:phoenix-core:5.0.0-HBase-2.0 - http://www.apache.org/phoenix/phoenix-core/)
817817
(The Apache Software License, Version 2.0) Plexus Interpolation API (org.codehaus.plexus:plexus-interpolation:1.19 - http://plexus.codehaus.org/plexus-components/plexus-interpolation)
818818
(The Apache Software License, Version 2.0) Retrofit (com.squareup.retrofit2:retrofit:2.9.0 - https://github.com/square/retrofit)
819-
(The Apache Software License, Version 2.0) Snappy for Java (org.xerial.snappy:snappy-java:1.0.5 - http://github.com/xerial/snappy-java/)
820819
(The Apache Software License, Version 2.0) SparseBitSet (com.zaxxer:SparseBitSet:1.2 - https://github.com/brettwooldridge/SparseBitSet)
821820
(The Apache Software License, Version 2.0) Spymemcached (net.spy:spymemcached:2.12.3 - http://www.couchbase.org/code/couchbase/java)
822821
(The Apache Software License, Version 2.0) StAX API (stax:stax-api:1.0.1 - http://stax.codehaus.org/)
@@ -884,10 +883,6 @@ The text of each license is the standard Apache 2.0 license.
884883
(The Apache Software License, Version 2.0) secure-sm (org.elasticsearch:elasticsearch-secure-sm:6.3.1 - https://github.com/elastic/elasticsearch)
885884
(The Apache Software License, Version 2.0) server (org.elasticsearch:elasticsearch:6.3.1 - https://github.com/elastic/elasticsearch)
886885
(The Apache Software License, Version 2.0) server (org.elasticsearch:elasticsearch:7.5.1 - https://github.com/elastic/elasticsearch)
887-
(The Apache Software License, Version 2.0) snappy-java (org.xerial.snappy:snappy-java:1.1.2.6 - https://github.com/xerial/snappy-java)
888-
(The Apache Software License, Version 2.0) snappy-java (org.xerial.snappy:snappy-java:1.1.4 - https://github.com/xerial/snappy-java)
889-
(The Apache Software License, Version 2.0) snappy-java (org.xerial.snappy:snappy-java:1.1.7.1 - https://github.com/xerial/snappy-java)
890-
(The Apache Software License, Version 2.0) snappy-java (org.xerial.snappy:snappy-java:1.1.7.3 - https://github.com/xerial/snappy-java)
891886
(The Apache Software License, Version 2.0) transport (org.elasticsearch.client:transport:6.3.1 - https://github.com/elastic/elasticsearch)
892887
(The Apache Software License, Version 2.0) transport (org.elasticsearch.client:transport:7.5.1 - https://github.com/elastic/elasticsearch)
893888
(The Apache Software License, Version 2.0) transport-netty4 (org.elasticsearch.plugin:transport-netty4-client:6.3.1 - https://github.com/elastic/elasticsearch)
@@ -911,14 +906,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
911906
(MIT License) Joni (org.jruby.joni:joni:2.1.11 - http://nexus.sonatype.org/oss-repository-hosting.html/joni)
912907
(MIT License) Joni (org.jruby.joni:joni:2.1.2 - http://nexus.sonatype.org/oss-repository-hosting.html/joni)
913908
(MIT License) Joni (org.jruby.joni:joni:2.1.27 - http://nexus.sonatype.org/oss-repository-hosting.html/joni)
914-
(MIT License) SLF4J API Module (org.slf4j:slf4j-api:1.6.4 - http://www.slf4j.org)
915-
(MIT License) SLF4J API Module (org.slf4j:slf4j-api:1.7.15 - http://www.slf4j.org)
916-
(MIT License) SLF4J API Module (org.slf4j:slf4j-api:1.7.16 - http://www.slf4j.org)
917909
(MIT License) SLF4J API Module (org.slf4j:slf4j-api:1.7.25 - http://www.slf4j.org)
918-
(MIT License) SLF4J API Module (org.slf4j:slf4j-api:1.7.30 - http://www.slf4j.org)
919-
(MIT License) SLF4J LOG4J-12 Binding (org.slf4j:slf4j-log4j12:1.7.10 - http://www.slf4j.org)
920-
(MIT License) SLF4J LOG4J-12 Binding (org.slf4j:slf4j-log4j12:1.7.15 - http://www.slf4j.org)
921-
(MIT License) SLF4J LOG4J-12 Binding (org.slf4j:slf4j-log4j12:1.7.16 - http://www.slf4j.org)
922910
(MIT License) SLF4J LOG4J-12 Binding (org.slf4j:slf4j-log4j12:1.7.25 - http://www.slf4j.org)
923911
(MIT License) pyrolite (net.razorvine:pyrolite:4.13 - https://github.com/irmen/Pyrolite)
924912
(MIT License) scopt (com.github.scopt:scopt_2.11:3.5.0 - https://github.com/scopt/scopt)

‎seatunnel-e2e/seatunnel-flink-e2e/pom.xml

-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@
3636
<dependency>
3737
<groupId>org.slf4j</groupId>
3838
<artifactId>slf4j-log4j12</artifactId>
39-
<version>${slf4j.version}</version>
4039
<scope>test</scope>
4140
</dependency>
4241

‎seatunnel-e2e/seatunnel-flink-sql-e2e/pom.xml

-1
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@
3535
<dependency>
3636
<groupId>org.slf4j</groupId>
3737
<artifactId>slf4j-log4j12</artifactId>
38-
<version>1.7.25</version>
3938
<scope>test</scope>
4039
</dependency>
4140

‎seatunnel-e2e/seatunnel-spark-e2e/pom.xml

-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@
3030
<dependency>
3131
<groupId>org.slf4j</groupId>
3232
<artifactId>slf4j-log4j12</artifactId>
33-
<version>${slf4j.version}</version>
3433
<scope>test</scope>
3534
</dependency>
3635
<dependency>

‎seatunnel-examples/seatunnel-flink-examples/pom.xml

-1
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,6 @@
9393
<dependency>
9494
<groupId>org.slf4j</groupId>
9595
<artifactId>slf4j-log4j12</artifactId>
96-
<version>${slf4j.version}</version>
9796
</dependency>
9897
</dependencies>
9998

‎seatunnel-examples/seatunnel-flink-sql-examples/pom.xml

-1
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,6 @@
9494
<dependency>
9595
<groupId>org.slf4j</groupId>
9696
<artifactId>slf4j-log4j12</artifactId>
97-
<version>${slf4j.version}</version>
9897
</dependency>
9998
</dependencies>
10099
</project>

‎seatunnel-examples/seatunnel-spark-examples/pom.xml

-1
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,6 @@
8888
<dependency>
8989
<groupId>org.slf4j</groupId>
9090
<artifactId>slf4j-log4j12</artifactId>
91-
<version>${slf4j.version}</version>
9291
</dependency>
9392
</dependencies>
9493
</project>

‎tools/dependencies/known-dependencies.txt

-7
Original file line numberDiff line numberDiff line change
@@ -633,18 +633,11 @@ shims-0.9.0.jar
633633
shims-0.9.22.jar
634634
sigar-1.6.5.132.jar
635635
sketches-core-0.9.0.jar
636-
slf4j-api-1.7.15.jar
637-
slf4j-api-1.7.16.jar
638-
slf4j-api-1.7.21.jar
639636
slf4j-api-1.7.25.jar
640-
slf4j-log4j12-1.7.15.jar
641-
slf4j-log4j12-1.7.16.jar
642637
slf4j-log4j12-1.7.25.jar
643638
snakeyaml-1.17.jar
644639
snakeyaml-1.24.jar
645640
snappy-0.3.jar
646-
snappy-java-1.1.4.jar
647-
snappy-java-1.1.7.1.jar
648641
snappy-java-1.1.8.3.jar
649642
spark-catalyst_2.11-2.4.0.jar
650643
spark-hive-thriftserver_2.11-2.3.4.jar

0 commit comments

Comments
 (0)