Skip to content

Commit 12e1e1b

Browse files
garyelephantgaryelephant
authored
Merge pull request apache#738 from yx91490/issue-715
[Feature#715] add quick-start Dockerfile
2 parents 2e5ac31 + 4d8f914 commit 12e1e1b

File tree

5 files changed

+242
-3
lines changed

5 files changed

+242
-3
lines changed

‎pom.xml

+11-1
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@
111111
<maven.compiler.target>${java.version}</maven.compiler.target>
112112
<spark.version>2.4.0</spark.version>
113113
<flink.version>1.9.0</flink.version>
114+
<hadoop.binary.version>2.7</hadoop.binary.version>
114115
<hadoop.version>2.7.5</hadoop.version>
115116
<fastjson.version>1.2.60</fastjson.version>
116117
<lombok.version>1.18.0</lombok.version>
@@ -385,7 +386,16 @@
385386
<build>
386387
<pluginManagement>
387388
<plugins>
388-
389+
<plugin>
390+
<groupId>org.apache.maven.plugins</groupId>
391+
<artifactId>maven-assembly-plugin</artifactId>
392+
<version>2.4</version>
393+
</plugin>
394+
<plugin>
395+
<groupId>io.fabric8</groupId>
396+
<artifactId>docker-maven-plugin</artifactId>
397+
<version>0.38.0</version>
398+
</plugin>
389399
<plugin>
390400
<groupId>org.sonatype.plugins</groupId>
391401
<artifactId>nexus-staging-maven-plugin</artifactId>

‎seatunnel-dist/docker/README.md

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
# SeaTunnel Docker Images
2+
3+
## Quick Start Image
4+
5+
### Introduction
6+
7+
For quick start usage, packaged with Apache Spark and Apache Flink releases, only run in standalone mode.
8+
9+
the spark home is in `/opt/spark`, the flink home is in `/opt/flink`, the seatunnel home is in `/opt/seatunnel`.
10+
11+
### Build Image
12+
13+
Change directory to project root, then run maven package command with `docker-quick-start` profile enabled(disabled by default):
14+
15+
```shell
16+
mvn clean package -Pdocker-quick-start
17+
```
18+
19+
you can choose whether to package spark or flink within the image:
20+
21+
```shell
22+
mvn clean package -Pdocker-quick-start\
23+
-Dinstall.spark=true\
24+
-Dinstall.flink=false\
25+
-Dimage.name='seatunnel:${project.version}'
26+
```
27+
28+
you can specify the mirror url to speed up the download process:
29+
30+
```shell
31+
mvn clean package -Pdocker-quick-start\
32+
-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-2.4.0\
33+
-Dspark.archive.name=spark-2.4.0-bin-hadoop2.7.tgz\
34+
-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.9.0\
35+
-Dflink.archive.name=flink-1.9.0-bin-scala_2.11.tgz
36+
```
37+
38+
you'll find the image tagged with name `seatunnel:<version>` if success.
39+
40+
### Run Container
41+
42+
Run command:
43+
44+
```shell
45+
docker run \
46+
-it \
47+
--name seatunnel-demo \
48+
--net host \
49+
-v <host_job_conf_dir>:<container_job_conf_dir> \
50+
seatunnel[:<version>] \
51+
[params...]
52+
```
53+
54+
parameters:
55+
56+
| Option | Candidates | Explain |
57+
| -------- | ------------ | --------------------------------------------------- |
58+
| --engine | spark, flink | specify which engine to run on |
59+
| --config | * | path to the job config file within docker container |
60+
61+
you can specify `--engine=spark` to run on Spark engine, or specify `--engine=flink` to run on Flink engine.
62+
63+
if you run on spark engine, you can find spark master's UI on http://localhost:8080.
64+
65+
if you run on flink engine, you can find flink UI on http://localhost:8081.
66+
67+
the default job config file for spark engine is `config/spark.batch.conf.template`, and for flink is `config/flink.streaming.conf.template`.
68+
69+
other parameters will be passed to corresponding bootstrap script: `start-seatunnel-spark.sh` or `start-seatunnel-flink.sh`.
70+

‎seatunnel-dist/docker/quick-start/Dockerfile

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
FROM openjdk:8-jre-buster
17+
ENV FLINK_HOME /opt/flink
18+
ENV SPARK_HOME /opt/spark
19+
ENV SEATUNNEL_HOME /opt/seatunnel
20+
ENTRYPOINT ["/seatunnel/entrypoint.sh"]
21+
COPY entrypoint.sh /seatunnel/entrypoint.sh
22+
RUN if [ "${install.spark}" = "true" ]; then\
23+
mkdir -p /opt/spark &&\
24+
curl ${spark.archive.mirror}/${spark.archive.name}\
25+
| tar xzf - --strip-components 1 -C /opt/spark; fi;\
26+
if [ "${install.flink}" = "true" ]; then\
27+
mkdir -p /opt/flink &&\
28+
curl ${flink.archive.mirror}/${flink.archive.name}\
29+
| tar xzf - --strip-components 1 -C /opt/flink; fi;
30+
COPY maven /opt/seatunnel

‎seatunnel-dist/docker/quick-start/entrypoint.sh

+83
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
#!/bin/bash
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
set -eux
18+
19+
log() {
20+
echo ">>> $@"
21+
}
22+
23+
start_flink() {
24+
log "Start Flink cluster..."
25+
${FLINK_HOME}/bin/start-cluster.sh
26+
}
27+
28+
main() {
29+
local PARAMS=""
30+
local ENGINE=""
31+
local CONFIG=""
32+
local MASTER="local[2]"
33+
local DEPLOY_MODE="client"
34+
while [ $# -ne 0 ]; do
35+
case "$1" in
36+
--engine)
37+
ENGINE=$2
38+
shift 2
39+
;;
40+
-m|--master)
41+
MASTER=$2
42+
shift 2
43+
;;
44+
-e|--deploy-mode)
45+
DEPLOY_MODE=$2
46+
shift 2
47+
;;
48+
-c|--config)
49+
CONFIG=$2
50+
shift 2
51+
;;
52+
*)
53+
PARAMS="$PARAMS $1"
54+
shift
55+
;;
56+
esac
57+
done
58+
if [ ${ENGINE} = "spark" ]; then
59+
if [ ! -d ${SPARK_HOME} ]; then
60+
log "Spark not deployed!"
61+
exit 2
62+
fi
63+
set -- "-m ${MASTER} -e ${DEPLOY_MODE} ${PARAMS}"
64+
CONFIG=${CONFIG:-${SEATUNNEL_HOME}/config/spark.batch.conf.template}
65+
log "Run SeaTunnel job on spark..."
66+
exec ${SEATUNNEL_HOME}/bin/start-seatunnel-spark.sh --config ${CONFIG} $@
67+
elif [ ${ENGINE} = "flink" ]; then
68+
if [ ! -d ${FLINK_HOME} ]; then
69+
log "Flink not deployed!"
70+
exit 3
71+
fi
72+
set -- "${PARAMS}"
73+
CONFIG=${CONFIG:-${SEATUNNEL_HOME}/config/flink.streaming.conf.template}
74+
start_flink
75+
log "Run SeaTunnel job on flink..."
76+
exec ${SEATUNNEL_HOME}/bin/start-seatunnel-flink.sh --config ${CONFIG} $@
77+
else
78+
log "Engine not support: ${ENGINE}"
79+
exit 1
80+
fi
81+
}
82+
83+
main $@

‎seatunnel-dist/pom.xml

+48-2
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
</resources>
4646
<plugins>
4747
<plugin>
48+
<groupId>org.apache.maven.plugins</groupId>
4849
<artifactId>maven-assembly-plugin</artifactId>
4950
<executions>
5051
<execution>
@@ -60,9 +61,54 @@
6061
<descriptor>assembly.xml</descriptor>
6162
</descriptors>
6263
</configuration>
63-
6464
</plugin>
6565
</plugins>
6666
</build>
67-
67+
<profiles>
68+
<profile>
69+
<id>docker-quick-start</id>
70+
<properties>
71+
<install.spark>true</install.spark>
72+
<install.flink>true</install.flink>
73+
<image.name>seatunnel:${project.version}</image.name>
74+
<spark.archive.name>spark-${spark.version}-bin-hadoop${hadoop.binary.version}.tgz</spark.archive.name>
75+
<spark.archive.mirror>https://archive.apache.org/dist/spark/spark-${spark.version}
76+
</spark.archive.mirror>
77+
<flink.archive.name>flink-${flink.version}-bin-scala_${scala.binary.version}.tgz</flink.archive.name>
78+
<flink.archive.mirror>https://archive.apache.org/dist/flink/flink-${flink.version}
79+
</flink.archive.mirror>
80+
</properties>
81+
<build>
82+
<plugins>
83+
<plugin>
84+
<groupId>io.fabric8</groupId>
85+
<artifactId>docker-maven-plugin</artifactId>
86+
<executions>
87+
<execution>
88+
<phase>package</phase>
89+
<goals>
90+
<goal>build</goal>
91+
</goals>
92+
</execution>
93+
</executions>
94+
<configuration>
95+
<verbose>true</verbose>
96+
<images>
97+
<image>
98+
<alias>seatunnel</alias>
99+
<name>${image.name}</name>
100+
<build>
101+
<contextDir>${project.basedir}/docker/quick-start</contextDir>
102+
<assembly>
103+
<descriptor>${project.basedir}/assembly.xml</descriptor>
104+
</assembly>
105+
</build>
106+
</image>
107+
</images>
108+
</configuration>
109+
</plugin>
110+
</plugins>
111+
</build>
112+
</profile>
113+
</profiles>
68114
</project>

0 commit comments

Comments
 (0)