Skip to content

Commit 6762727

Browse files
joschibernd
joschi
authored and
bernd
committed
Migrate to Netty 4.1 (Graylog2#4397)
* Migrate to Netty 4.1 * Revert changes to vendor-module-ids.json and yarn.lock Ceterum censeo JavaScript esse delendam. * Add support for native SSL (OpenSSL) bindings * Wait for transport being started completely in UdpTransportTest * Refactor getBaseChannelHandlers/getFinalChannelHandlers * Upgrade to Netty 4.1.18.Final * Fix throughput and connection counters for Netty-based transports * Use read bytes for CodecAggregator.Result in GelfChunkAggregator * Refactor NettyTransport#getLocalAddress() * Upgrade to Netty 4.1.19.Final * Allow configuring the number of worker threads per Netty transport * Remove support for Netty OIO (old I/O) transports * Remove outdated TODOs * Use locale metric registry for transport-specific metrics
1 parent 08a5962 commit 6762727

File tree

54 files changed

+2954
-1887
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+2954
-1887
lines changed

‎config/forbidden-apis/netty3.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
org.jboss.netty.** @ Migrate to Netty 4.x

‎graylog-project-parent/pom.xml

+21-6
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,25 @@
6767
<type>pom</type>
6868
<scope>import</scope>
6969
</dependency>
70+
<dependency>
71+
<groupId>io.netty</groupId>
72+
<artifactId>netty-bom</artifactId>
73+
<version>${netty.version}</version>
74+
<type>pom</type>
75+
<scope>import</scope>
76+
</dependency>
77+
<dependency>
78+
<groupId>io.netty</groupId>
79+
<artifactId>netty-tcnative-boringssl-static</artifactId>
80+
<version>${netty-tcnative-boringssl-static.version}</version>
81+
<classifier>osx-x86_64</classifier>
82+
</dependency>
83+
<dependency>
84+
<groupId>io.netty</groupId>
85+
<artifactId>netty-tcnative-boringssl-static</artifactId>
86+
<version>${netty-tcnative-boringssl-static.version}</version>
87+
<classifier>linux-x86_64</classifier>
88+
</dependency>
7089

7190
<dependency>
7291
<groupId>com.google.guava</groupId>
@@ -174,11 +193,6 @@
174193
<artifactId>reflections</artifactId>
175194
<version>${reflections.version}</version>
176195
</dependency>
177-
<dependency>
178-
<groupId>io.netty</groupId>
179-
<artifactId>netty</artifactId>
180-
<version>${netty.version}</version>
181-
</dependency>
182196
<dependency>
183197
<groupId>com.jayway.jsonpath</groupId>
184198
<artifactId>json-path</artifactId>
@@ -626,7 +640,7 @@
626640
<!-- Use ALL the cores! -->
627641
<forkCount>1C</forkCount>
628642
<reuseForks>false</reuseForks>
629-
<argLine>-Djava.library.path=${project.basedir}/../lib/sigar-${sigar.version}</argLine>
643+
<argLine>-Djava.library.path=${project.basedir}/../lib/sigar-${sigar.version} -Dio.netty.leakDetectionLevel=paranoid</argLine>
630644
<excludes>
631645
<exclude>**/*IntegrationTest.java</exclude>
632646
<exclude>**/*IT.java</exclude>
@@ -659,6 +673,7 @@
659673
<!--bundledSignature>commons-io-unsafe-${commons-io.version}</bundledSignature-->
660674
</bundledSignatures>
661675
<signaturesFiles>
676+
<signaturesFile>${project.basedir}/../config/forbidden-apis/netty3.txt</signaturesFile>
662677
<signaturesFile>${project.basedir}/../config/forbidden-apis/signatures.txt</signaturesFile>
663678
</signaturesFiles>
664679
</configuration>

‎graylog2-server/pom.xml

+37-1
Original file line numberDiff line numberDiff line change
@@ -404,7 +404,43 @@
404404

405405
<dependency>
406406
<groupId>io.netty</groupId>
407-
<artifactId>netty</artifactId>
407+
<artifactId>netty-common</artifactId>
408+
</dependency>
409+
<dependency>
410+
<groupId>io.netty</groupId>
411+
<artifactId>netty-buffer</artifactId>
412+
</dependency>
413+
<dependency>
414+
<groupId>io.netty</groupId>
415+
<artifactId>netty-handler</artifactId>
416+
</dependency>
417+
<dependency>
418+
<groupId>io.netty</groupId>
419+
<artifactId>netty-codec</artifactId>
420+
</dependency>
421+
<dependency>
422+
<groupId>io.netty</groupId>
423+
<artifactId>netty-codec-http</artifactId>
424+
</dependency>
425+
<dependency>
426+
<groupId>io.netty</groupId>
427+
<artifactId>netty-transport-native-epoll</artifactId>
428+
<classifier>linux-x86_64</classifier>
429+
</dependency>
430+
<dependency>
431+
<groupId>io.netty</groupId>
432+
<artifactId>netty-transport-native-kqueue</artifactId>
433+
<classifier>osx-x86_64</classifier>
434+
</dependency>
435+
<dependency>
436+
<groupId>io.netty</groupId>
437+
<artifactId>netty-tcnative-boringssl-static</artifactId>
438+
<classifier>osx-x86_64</classifier>
439+
</dependency>
440+
<dependency>
441+
<groupId>io.netty</groupId>
442+
<artifactId>netty-tcnative-boringssl-static</artifactId>
443+
<classifier>linux-x86_64</classifier>
408444
</dependency>
409445

410446
<dependency>

‎graylog2-server/src/main/java/org/graylog2/bootstrap/CmdLineTool.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@
4242
import com.google.inject.Module;
4343
import com.google.inject.name.Names;
4444
import com.google.inject.spi.Message;
45+
import io.netty.util.internal.logging.InternalLoggerFactory;
46+
import io.netty.util.internal.logging.Slf4JLoggerFactory;
4547
import org.apache.logging.log4j.Level;
4648
import org.apache.logging.log4j.LogManager;
4749
import org.apache.logging.log4j.core.LoggerContext;
@@ -61,8 +63,6 @@
6163
import org.graylog2.shared.plugins.ChainingClassLoader;
6264
import org.graylog2.shared.plugins.PluginLoader;
6365
import org.graylog2.shared.utilities.ExceptionUtils;
64-
import org.jboss.netty.logging.InternalLoggerFactory;
65-
import org.jboss.netty.logging.Slf4JLoggerFactory;
6666
import org.slf4j.Logger;
6767
import org.slf4j.LoggerFactory;
6868

‎graylog2-server/src/main/java/org/graylog2/bootstrap/ServerBootstrap.java

-2
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
import org.graylog2.plugin.system.NodeId;
3232
import org.graylog2.shared.bindings.GenericBindings;
3333
import org.graylog2.shared.bindings.GenericInitializerBindings;
34-
import org.graylog2.shared.bindings.MessageInputBindings;
3534
import org.graylog2.shared.bindings.SchedulerBindings;
3635
import org.graylog2.shared.bindings.ServerStatusBindings;
3736
import org.graylog2.shared.bindings.SharedPeriodicalBindings;
@@ -199,7 +198,6 @@ protected List<Module> getSharedBindingsModules() {
199198
result.add(new SharedPeriodicalBindings());
200199
result.add(new SchedulerBindings());
201200
result.add(new GenericInitializerBindings());
202-
result.add(new MessageInputBindings());
203201
result.add(new SystemStatsModule(configuration.isDisableSigar()));
204202

205203
return result;

‎graylog2-server/src/main/java/org/graylog2/commands/Server.java

+6-1
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
import org.graylog2.indexer.IndexerBindings;
5555
import org.graylog2.indexer.retention.RetentionStrategyBindings;
5656
import org.graylog2.indexer.rotation.RotationStrategyBindings;
57+
import org.graylog2.inputs.transports.NettyTransportConfiguration;
5758
import org.graylog2.messageprocessors.MessageProcessorModule;
5859
import org.graylog2.migrations.MigrationsModule;
5960
import org.graylog2.notifications.Notification;
@@ -63,6 +64,7 @@
6364
import org.graylog2.plugin.Tools;
6465
import org.graylog2.plugin.system.NodeId;
6566
import org.graylog2.shared.UI;
67+
import org.graylog2.shared.bindings.MessageInputBindings;
6668
import org.graylog2.shared.bindings.ObjectMapperModule;
6769
import org.graylog2.shared.bindings.RestApiBindings;
6870
import org.graylog2.shared.system.activities.Activity;
@@ -93,6 +95,7 @@ public class Server extends ServerBootstrap {
9395
private final MongoDbConfiguration mongoDbConfiguration = new MongoDbConfiguration();
9496
private final VersionCheckConfiguration versionCheckConfiguration = new VersionCheckConfiguration();
9597
private final KafkaJournalConfiguration kafkaJournalConfiguration = new KafkaJournalConfiguration();
98+
private final NettyTransportConfiguration nettyTransportConfiguration = new NettyTransportConfiguration();
9699

97100
public Server() {
98101
super("server", configuration);
@@ -117,6 +120,7 @@ protected List<Module> getCommandBindings() {
117120
new MessageProcessorModule(),
118121
new AlarmCallbackBindings(),
119122
new InitializerBindings(),
123+
new MessageInputBindings(),
120124
new MessageOutputBindings(configuration, chainingClassLoader),
121125
new RotationStrategyBindings(),
122126
new RetentionStrategyBindings(),
@@ -145,7 +149,8 @@ protected List<Object> getCommandConfigurationBeans() {
145149
emailConfiguration,
146150
mongoDbConfiguration,
147151
versionCheckConfiguration,
148-
kafkaJournalConfiguration);
152+
kafkaJournalConfiguration,
153+
nettyTransportConfiguration);
149154
}
150155

151156
@Override

‎graylog2-server/src/main/java/org/graylog2/inputs/codecs/GelfChunkAggregator.java

+8-8
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,12 @@
2121
import com.google.common.annotations.VisibleForTesting;
2222
import com.google.common.base.MoreObjects;
2323
import com.google.common.collect.Maps;
24+
import io.netty.buffer.ByteBuf;
25+
import io.netty.buffer.Unpooled;
2426
import org.graylog2.inputs.codecs.gelf.GELFMessage;
2527
import org.graylog2.inputs.codecs.gelf.GELFMessageChunk;
2628
import org.graylog2.plugin.Tools;
2729
import org.graylog2.plugin.inputs.codecs.CodecAggregator;
28-
import org.jboss.netty.buffer.ChannelBuffer;
29-
import org.jboss.netty.buffer.ChannelBuffers;
3030
import org.slf4j.Logger;
3131
import org.slf4j.LoggerFactory;
3232

@@ -85,13 +85,13 @@ public GelfChunkAggregator(@Named("daemonScheduler") ScheduledExecutorService sc
8585

8686
@Nonnull
8787
@Override
88-
public Result addChunk(ChannelBuffer buffer) {
88+
public Result addChunk(ByteBuf buffer) {
8989
final byte[] readable = new byte[buffer.readableBytes()];
90-
buffer.toByteBuffer().get(readable, buffer.readerIndex(), buffer.readableBytes());
90+
buffer.readBytes(readable, buffer.readerIndex(), buffer.readableBytes());
9191

9292
final GELFMessage msg = new GELFMessage(readable);
9393

94-
final ChannelBuffer aggregatedBuffer;
94+
final ByteBuf aggregatedBuffer;
9595
switch (msg.getGELFType()) {
9696
case CHUNKED:
9797
try {
@@ -108,7 +108,7 @@ public Result addChunk(ChannelBuffer buffer) {
108108
case ZLIB:
109109
case GZIP:
110110
case UNCOMPRESSED:
111-
aggregatedBuffer = buffer;
111+
aggregatedBuffer = Unpooled.wrappedBuffer(readable);
112112
break;
113113
case UNSUPPORTED:
114114
return INVALID_RESULT;
@@ -127,7 +127,7 @@ public Result addChunk(ChannelBuffer buffer) {
127127
* @return null or a {@link org.graylog2.plugin.journal.RawMessage raw message} object
128128
*/
129129
@Nullable
130-
private ChannelBuffer checkForCompletion(GELFMessage gelfMessage) {
130+
private ByteBuf checkForCompletion(GELFMessage gelfMessage) {
131131
if (!chunks.isEmpty() && log.isDebugEnabled()) {
132132
log.debug("Dumping GELF chunk map [chunks for {} messages]:\n{}", chunks.size(), humanReadableChunkMap());
133133
}
@@ -178,7 +178,7 @@ private ChannelBuffer checkForCompletion(GELFMessage gelfMessage) {
178178
}
179179
}
180180
completeMessages.inc();
181-
return ChannelBuffers.wrappedBuffer(allChunks);
181+
return Unpooled.wrappedBuffer(allChunks);
182182
}
183183

184184
// message isn't complete yet, check if we should remove the other parts as well

‎graylog2-server/src/main/java/org/graylog2/inputs/codecs/gelf/GELFMessageChunk.java

+12-23
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,11 @@
1616
*/
1717
package org.graylog2.inputs.codecs.gelf;
1818

19+
import io.netty.buffer.ByteBuf;
20+
import io.netty.buffer.ByteBufUtil;
21+
import io.netty.buffer.Unpooled;
1922
import org.graylog2.plugin.Tools;
2023
import org.graylog2.plugin.inputs.MessageInput;
21-
import org.jboss.netty.buffer.ChannelBuffer;
22-
import org.jboss.netty.buffer.ChannelBuffers;
2324

2425
public final class GELFMessageChunk {
2526

@@ -54,14 +55,14 @@ public final class GELFMessageChunk {
5455
private int sequenceCount = -1;
5556
private long arrival = -1L;
5657

57-
private final ChannelBuffer payload;
58+
private final ByteBuf payload;
5859
private final MessageInput sourceInput;
5960

6061
public GELFMessageChunk(final byte[] payload, MessageInput sourceInput) {
6162
if (payload.length < HEADER_TOTAL_LENGTH) {
6263
throw new IllegalArgumentException("This GELF message chunk is too short. Cannot even contain the required header.");
6364
}
64-
this.payload = ChannelBuffers.wrappedBuffer(payload);
65+
this.payload = Unpooled.wrappedBuffer(payload);
6566
this.sourceInput = sourceInput;
6667
read();
6768
}
@@ -108,12 +109,10 @@ private void read() {
108109
this.arrival = Tools.nowUTC().getMillis();
109110
}
110111

111-
private String extractId() {
112+
private void extractId() {
112113
if (this.id == null) {
113-
this.id = ChannelBuffers.hexDump(payload, HEADER_PART_HASH_START, HEADER_PART_HASH_LENGTH);
114+
this.id = ByteBufUtil.hexDump(payload, HEADER_PART_HASH_START, HEADER_PART_HASH_LENGTH);
114115
}
115-
116-
return this.id;
117116
}
118117

119118
// lol duplication
@@ -152,20 +151,10 @@ private void extractData() {
152151

153152
@Override
154153
public String toString() {
155-
final StringBuilder sb = new StringBuilder();
156-
157-
sb.append("ID: ");
158-
sb.append(this.id);
159-
sb.append("\tSequence: ");
160-
sb.append(this.sequenceNumber + 1); // +1 for readability: 1/2 not 0/2
161-
sb.append("/");
162-
sb.append(this.sequenceCount);
163-
sb.append("\tArrival: ");
164-
sb.append(this.arrival);
165-
sb.append("\tData size: ");
166-
sb.append(this.payload.readableBytes());
167-
168-
return sb.toString();
154+
return "ID: " + this.id +
155+
"\tSequence: " + (this.sequenceNumber + 1) + // +1 for readability: 1/2 not 0/2
156+
"/" + this.sequenceCount +
157+
"\tArrival: " + this.arrival +
158+
"\tData size: " + this.payload.readableBytes();
169159
}
170-
171160
}

‎graylog2-server/src/main/java/org/graylog2/inputs/syslog/tcp/SyslogOctetCountFrameDecoder.java

+22-27
Original file line numberDiff line numberDiff line change
@@ -16,28 +16,30 @@
1616
*/
1717
package org.graylog2.inputs.syslog.tcp;
1818

19-
import org.jboss.netty.buffer.ChannelBuffer;
20-
import org.jboss.netty.channel.Channel;
21-
import org.jboss.netty.channel.ChannelHandlerContext;
22-
import org.jboss.netty.handler.codec.frame.FrameDecoder;
19+
20+
import io.netty.buffer.ByteBuf;
21+
import io.netty.channel.ChannelHandlerContext;
22+
import io.netty.handler.codec.ByteToMessageDecoder;
23+
import io.netty.util.ByteProcessor;
2324

2425
import java.nio.charset.StandardCharsets;
26+
import java.util.List;
2527

2628
/**
27-
* Implements a Netty {@link FrameDecoder} for the Syslog octet counting framing. (RFC6587)
29+
* Implements a Netty {@link ByteToMessageDecoder} for the Syslog octet counting framing. (RFC6587)
2830
*
2931
* @see <a href="http://tools.ietf.org/html/rfc6587#section-3.4.1">RFC6587 Octet Counting</a>
3032
*/
31-
public class SyslogOctetCountFrameDecoder extends FrameDecoder {
33+
public class SyslogOctetCountFrameDecoder extends ByteToMessageDecoder {
34+
private static final ByteProcessor BYTE_PROCESSOR = value -> value != ' ';
35+
3236
@Override
33-
protected Object decode(final ChannelHandlerContext ctx,
34-
final Channel channel,
35-
final ChannelBuffer buffer) throws Exception {
37+
protected void decode(ChannelHandlerContext ctx, ByteBuf buffer, List<Object> out) throws Exception {
3638
final int frameSizeValueLength = findFrameSizeValueLength(buffer);
3739

3840
// We have not found the frame length value byte size yet.
3941
if (frameSizeValueLength <= 0) {
40-
return null;
42+
return;
4143
}
4244

4345
// Convert the frame length value bytes into an integer without mutating the buffer reader index.
@@ -49,18 +51,14 @@ protected Object decode(final ChannelHandlerContext ctx,
4951
// the buffer has enough data to read the complete message.
5052
if (buffer.readableBytes() - skipLength < length) {
5153
// We cannot read the complete frame yet.
52-
return null;
54+
return;
5355
} else {
5456
// Skip the frame length value bytes and the whitespace that follows it.
5557
buffer.skipBytes(skipLength);
5658
}
5759

58-
final ChannelBuffer frame = extractFrame(buffer, buffer.readerIndex(), length);
59-
60-
// Advance the reader index because extractFrame() does not do that.
61-
buffer.skipBytes(length);
62-
63-
return frame;
60+
final ByteBuf frame = buffer.readRetainedSlice(length);
61+
out.add(frame);
6462
}
6563

6664
/**
@@ -69,17 +67,14 @@ protected Object decode(final ChannelHandlerContext ctx,
6967
* @param buffer The channel buffer
7068
* @return The length of the frame length value
7169
*/
72-
private int findFrameSizeValueLength(final ChannelBuffer buffer) {
73-
final int n = buffer.writerIndex();
70+
private int findFrameSizeValueLength(final ByteBuf buffer) {
71+
final int readerIndex = buffer.readerIndex();
72+
int index = buffer.forEachByte(BYTE_PROCESSOR);
7473

75-
for (int i = buffer.readerIndex(); i < n; i ++) {
76-
final byte b = buffer.getByte(i);
77-
78-
if (b == ' ') {
79-
return i - buffer.readerIndex();
80-
}
74+
if (index >= 0) {
75+
return index - readerIndex;
76+
} else {
77+
return -1;
8178
}
82-
83-
return -1; // Not found.
8479
}
8580
}

0 commit comments

Comments
 (0)