forked from apache/kafka
-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathbase.sh
executable file
·207 lines (182 loc) · 7.04 KB
/
base.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
# The version of Kibosh to use for testing.
# If you update this, also update tests/docker/Dockerfile
export KIBOSH_VERSION=8841dd392e6fbf02986e2fb1f1ebf04df344b65a
path_to_jdk_cache() {
jdk_version=$1
echo "/tmp/jdk-${jdk_version}.tar.gz"
}
fetch_jdk_tgz() {
jdk_version=$1
path=$(path_to_jdk_cache $jdk_version)
if [ ! -e $path ]; then
mkdir -p $(dirname $path)
curl --retry 5 -s -L "https://s3-us-west-2.amazonaws.com/kafka-packages/jdk-${jdk_version}.tar.gz" -o $path
fi
}
JDK_MAJOR="${JDK_MAJOR:-17}"
JDK_FULL="${JDK_FULL:-17-linux-x64}"
if [ -z `which javac` ]; then
apt-get -y update
apt-get install -y software-properties-common python-software-properties binutils java-common
echo "===> Installing JDK..."
mkdir -p /opt/jdk
cd /opt/jdk
rm -rf $JDK_MAJOR
mkdir -p $JDK_MAJOR
cd $JDK_MAJOR
fetch_jdk_tgz $JDK_FULL
tar x --strip-components=1 -zf $(path_to_jdk_cache $JDK_FULL)
for bin in /opt/jdk/$JDK_MAJOR/bin/* ; do
name=$(basename $bin)
update-alternatives --install /usr/bin/$name $name $bin 1081 && update-alternatives --set $name $bin
done
echo -e "export JAVA_HOME=/opt/jdk/$JDK_MAJOR\nexport PATH=\$PATH:\$JAVA_HOME/bin" > /etc/profile.d/jdk.sh
echo "JDK installed: $(javac -version 2>&1)"
fi
chmod a+rw /opt
if [ -h /opt/kafka-dev ]; then
# reset symlink
rm /opt/kafka-dev
fi
ln -s /vagrant /opt/kafka-dev
get_kafka() {
version=$1
scala_version=$2
kafka_dir=/opt/kafka-$version
url=https://s3-us-west-2.amazonaws.com/kafka-packages/kafka_$scala_version-$version.tgz
# the .tgz above does not include the streams test jar hence we need to get it separately
url_streams_test=https://s3-us-west-2.amazonaws.com/kafka-packages/kafka-streams-$version-test.jar
if [ ! -d /opt/kafka-$version ]; then
pushd /tmp
curl --retry 5 -O $url
curl --retry 5 -O $url_streams_test || true
file_tgz=`basename $url`
file_streams_jar=`basename $url_streams_test` || true
tar -xzf $file_tgz
rm -rf $file_tgz
file=`basename $file_tgz .tgz`
mv $file $kafka_dir
mv $file_streams_jar $kafka_dir/libs || true
popd
fi
}
# Install Kibosh
apt-get update -y && apt-get install -y git cmake pkg-config libfuse-dev
pushd /opt
rm -rf /opt/kibosh
git clone -q https://github.com/confluentinc/kibosh.git
pushd "/opt/kibosh"
git reset --hard $KIBOSH_VERSION
mkdir "/opt/kibosh/build"
pushd "/opt/kibosh/build"
../configure && make -j 2
popd
popd
popd
# Install iperf
apt-get install -y iperf traceroute
# Test multiple Kafka versions
# We want to use the latest Scala version per Kafka version
# Previously we could not pull in Scala 2.12 builds, because Scala 2.12 requires Java 8 and we were running the system
# tests with Java 7. We have since switched to Java 8, so 2.0.0 and later use Scala 2.12.
# The versions between 0.11.0.3 and 2.0.1 are used to run client code, because zookeeper in these versions is not compatible with JDK 17.
# See KAFKA-17888 for more details.
get_kafka 0.11.0.3 2.11
chmod a+rw /opt/kafka-0.11.0.3
get_kafka 1.0.2 2.11
chmod a+rw /opt/kafka-1.0.2
get_kafka 1.1.1 2.11
chmod a+rw /opt/kafka-1.1.1
get_kafka 2.0.1 2.12
chmod a+rw /opt/kafka-2.0.1
get_kafka 2.1.1 2.12
chmod a+rw /opt/kafka-2.1.1
get_kafka 2.2.2 2.12
chmod a+rw /opt/kafka-2.2.2
get_kafka 2.3.1 2.12
chmod a+rw /opt/kafka-2.3.1
get_kafka 2.4.1 2.12
chmod a+rw /opt/kafka-2.4.1
get_kafka 2.5.1 2.12
chmod a+rw /opt/kafka-2.5.1
get_kafka 2.6.3 2.12
chmod a+rw /opt/kafka-2.6.3
get_kafka 2.7.2 2.12
chmod a+rw /opt/kafka-2.7.2
get_kafka 2.8.2 2.12
chmod a+rw /opt/kafka-2.8.2
get_kafka 3.0.2 2.12
chmod a+rw /opt/kafka-3.0.2
get_kafka 3.1.2 2.12
chmod a+rw /opt/kafka-3.1.2
get_kafka 3.2.3 2.12
chmod a+rw /opt/kafka-3.2.3
get_kafka 3.3.2 2.12
chmod a+rw /opt/kafka-3.3.2
get_kafka 3.4.1 2.12
chmod a+rw /opt/kafka-3.4.1
get_kafka 3.5.2 2.12
chmod a+rw /opt/kafka-3.5.2
get_kafka 3.6.2 2.12
chmod a+rw /opt/kafka-3.6.2
get_kafka 3.7.2 2.12
chmod a+rw /opt/kafka-3.7.2
get_kafka 3.8.1 2.12
chmod a+rw /opt/kafka-3.8.1
get_kafka 3.9.0 2.12
chmod a+rw /opt/kafka-3.9.0
# To ensure the Kafka cluster starts successfully under JDK 17, we need to update the Zookeeper
# client from version 3.4.x to 3.5.7 in Kafka versions 2.1.1, 2.2.2, and 2.3.1, as the older Zookeeper
# client is incompatible with JDK 17. See KAFKA-17888 for more details.
curl -s "https://repo1.maven.org/maven2/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar" -o /opt/zookeeper-3.5.7.jar
curl -s "https://repo1.maven.org/maven2/org/apache/zookeeper/zookeeper-jute/3.5.7/zookeeper-jute-3.5.7.jar" -o /opt/zookeeper-jute-3.5.7.jar
rm -f /opt/kafka-2.1.1/libs/zookeeper-*
rm -f /opt/kafka-2.2.2/libs/zookeeper-*
rm -f /opt/kafka-2.3.1/libs/zookeeper-*
cp /opt/zookeeper-3.5.7.jar /opt/kafka-2.1.1/libs/zookeeper-3.5.7.jar
chmod a+rw /opt/kafka-2.1.1/libs/zookeeper-3.5.7.jar
cp /opt/zookeeper-3.5.7.jar /opt/kafka-2.2.2/libs/zookeeper-3.5.7.jar
chmod a+rw /opt/kafka-2.2.2/libs/zookeeper-3.5.7.jar
cp /opt/zookeeper-3.5.7.jar /opt/kafka-2.3.1/libs/zookeeper-3.5.7.jar
chmod a+rw /opt/kafka-2.3.1/libs/zookeeper-3.5.7.jar
cp /opt/zookeeper-jute-3.5.7.jar /opt/kafka-2.1.1/libs/zookeeper-jute-3.5.7.jar
chmod a+rw /opt/kafka-2.1.1/libs/zookeeper-jute-3.5.7.jar
cp /opt/zookeeper-jute-3.5.7.jar /opt/kafka-2.2.2/libs/zookeeper-jute-3.5.7.jar
chmod a+rw /opt/kafka-2.2.2/libs/zookeeper-jute-3.5.7.jar
cp /opt/zookeeper-jute-3.5.7.jar /opt/kafka-2.3.1/libs/zookeeper-jute-3.5.7.jar
chmod a+rw /opt/kafka-2.3.1/libs/zookeeper-jute-3.5.7.jar
# For EC2 nodes, we want to use /mnt, which should have the local disk. On local
# VMs, we can just create it if it doesn't exist and use it like we'd use
# /tmp. Eventually, we'd like to also support more directories, e.g. when EC2
# instances have multiple local disks.
if [ ! -e /mnt ]; then
mkdir /mnt
fi
chmod a+rwx /mnt
# Run ntpdate once to sync to ntp servers
# use -u option to avoid port collision in case ntp daemon is already running
ntpdate -u pool.ntp.org
# Install ntp daemon - it will automatically start on boot
apt-get -y install ntp
# Increase the ulimit
mkdir -p /etc/security/limits.d
echo "* soft nofile 128000" >> /etc/security/limits.d/nofile.conf
echo "* hard nofile 128000" >> /etc/security/limits.d/nofile.conf
ulimit -Hn 128000
ulimit -Sn 128000