From 6ff48dcbe311d2736249b02b31a8745e719d1304 Mon Sep 17 00:00:00 2001 From: Abhijit Sarkar Date: Sat, 13 Jan 2018 22:39:39 -0800 Subject: [PATCH] Fixes #7566 by handling concatenated GZIP streams. Motivation: According to RFC 1952, concatenation of valid gzip streams is also a valid gzip stream. JdkZlibDecoder only processed the first and discarded the rest. Modifications: - Introduced a constructor argument decompressConcatenated that if true, JdkZlibDecoder would continue to process the stream. Result: - If 'decompressConcatenated = true', concatenated streams would be processed in compliance to RFC 1952. - If 'decompressConcatenated = false' (default), existing behavior would remain. --- .../codec/compression/JdkZlibDecoder.java | 26 +++++++-- .../codec/compression/ZlibCodecFactory.java | 4 +- .../codec/compression/JdkZlibTest.java | 55 ++++++++++++++++++ codec/src/test/resources/multiple.gz | Bin 0 -> 46 bytes 4 files changed, 78 insertions(+), 7 deletions(-) create mode 100644 codec/src/test/resources/multiple.gz diff --git a/codec/src/main/java/io/netty/handler/codec/compression/JdkZlibDecoder.java b/codec/src/main/java/io/netty/handler/codec/compression/JdkZlibDecoder.java index 865c0546eb88..c90cc4bd9834 100644 --- a/codec/src/main/java/io/netty/handler/codec/compression/JdkZlibDecoder.java +++ b/codec/src/main/java/io/netty/handler/codec/compression/JdkZlibDecoder.java @@ -39,6 +39,7 @@ public class JdkZlibDecoder extends ZlibDecoder { // GZIP related private final ByteBufChecksum crc; + private final boolean decompressConcatenated; private enum GzipState { HEADER_START, @@ -63,7 +64,7 @@ private enum GzipState { * Creates a new instance with the default wrapper ({@link ZlibWrapper#ZLIB}). */ public JdkZlibDecoder() { - this(ZlibWrapper.ZLIB, null); + this(ZlibWrapper.ZLIB, null, false); } /** @@ -72,7 +73,7 @@ public JdkZlibDecoder() { * supports the preset dictionary. */ public JdkZlibDecoder(byte[] dictionary) { - this(ZlibWrapper.ZLIB, dictionary); + this(ZlibWrapper.ZLIB, dictionary, false); } /** @@ -81,13 +82,22 @@ public JdkZlibDecoder(byte[] dictionary) { * supported atm. */ public JdkZlibDecoder(ZlibWrapper wrapper) { - this(wrapper, null); + this(wrapper, null, false); } - private JdkZlibDecoder(ZlibWrapper wrapper, byte[] dictionary) { + public JdkZlibDecoder(ZlibWrapper wrapper, boolean decompressConcatenated) { + this(wrapper, null, decompressConcatenated); + } + + public JdkZlibDecoder(boolean decompressConcatenated) { + this(ZlibWrapper.GZIP, null, decompressConcatenated); + } + + private JdkZlibDecoder(ZlibWrapper wrapper, byte[] dictionary, boolean decompressConcatenated) { if (wrapper == null) { throw new NullPointerException("wrapper"); } + this.decompressConcatenated = decompressConcatenated; switch (wrapper) { case GZIP: inflater = new Inflater(true); @@ -207,7 +217,13 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) t if (readFooter) { gzipState = GzipState.FOOTER_START; if (readGZIPFooter(in)) { - finished = true; + finished = !decompressConcatenated; + + if (!finished) { + inflater.reset(); + crc.reset(); + gzipState = GzipState.HEADER_START; + } } } } catch (DataFormatException e) { diff --git a/codec/src/main/java/io/netty/handler/codec/compression/ZlibCodecFactory.java b/codec/src/main/java/io/netty/handler/codec/compression/ZlibCodecFactory.java index e913bdb7a54d..a59cad60b0d4 100644 --- a/codec/src/main/java/io/netty/handler/codec/compression/ZlibCodecFactory.java +++ b/codec/src/main/java/io/netty/handler/codec/compression/ZlibCodecFactory.java @@ -113,7 +113,7 @@ public static ZlibDecoder newZlibDecoder() { if (PlatformDependent.javaVersion() < 7 || noJdkZlibDecoder) { return new JZlibDecoder(); } else { - return new JdkZlibDecoder(); + return new JdkZlibDecoder(true); } } @@ -121,7 +121,7 @@ public static ZlibDecoder newZlibDecoder(ZlibWrapper wrapper) { if (PlatformDependent.javaVersion() < 7 || noJdkZlibDecoder) { return new JZlibDecoder(wrapper); } else { - return new JdkZlibDecoder(wrapper); + return new JdkZlibDecoder(wrapper, true); } } diff --git a/codec/src/test/java/io/netty/handler/codec/compression/JdkZlibTest.java b/codec/src/test/java/io/netty/handler/codec/compression/JdkZlibTest.java index 23f178d924f4..54a48a9caeb4 100644 --- a/codec/src/test/java/io/netty/handler/codec/compression/JdkZlibTest.java +++ b/codec/src/test/java/io/netty/handler/codec/compression/JdkZlibTest.java @@ -15,8 +15,20 @@ */ package io.netty.handler.codec.compression; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.embedded.EmbeddedChannel; +import io.netty.util.CharsetUtil; +import io.netty.util.ReferenceCountUtil; +import org.apache.commons.compress.utils.IOUtils; import org.junit.Test; +import java.io.IOException; +import java.util.Arrays; +import java.util.Queue; + +import static org.junit.Assert.*; + public class JdkZlibTest extends ZlibTest { @@ -35,4 +47,47 @@ protected ZlibDecoder createDecoder(ZlibWrapper wrapper) { public void testZLIB_OR_NONE3() throws Exception { super.testZLIB_OR_NONE3(); } + + @Test + // verifies backward compatibility + public void testConcatenatedStreamsReadFirstOnly() throws IOException { + EmbeddedChannel chDecoderGZip = new EmbeddedChannel(createDecoder(ZlibWrapper.GZIP)); + + try { + byte[] bytes = IOUtils.toByteArray(getClass().getResourceAsStream("/multiple.gz")); + + assertTrue(chDecoderGZip.writeInbound(Unpooled.copiedBuffer(bytes))); + Queue messages = chDecoderGZip.inboundMessages(); + assertEquals(1, messages.size()); + + ByteBuf msg = (ByteBuf) messages.poll(); + assertEquals("a", msg.toString(CharsetUtil.UTF_8)); + ReferenceCountUtil.release(msg); + } finally { + assertFalse(chDecoderGZip.finish()); + chDecoderGZip.close(); + } + } + + @Test + public void testConcatenatedStreamsReadFully() throws IOException { + EmbeddedChannel chDecoderGZip = new EmbeddedChannel(new JdkZlibDecoder(true)); + + try { + byte[] bytes = IOUtils.toByteArray(getClass().getResourceAsStream("/multiple.gz")); + + assertTrue(chDecoderGZip.writeInbound(Unpooled.copiedBuffer(bytes))); + Queue messages = chDecoderGZip.inboundMessages(); + assertEquals(2, messages.size()); + + for (String s : Arrays.asList("a", "b")) { + ByteBuf msg = (ByteBuf) messages.poll(); + assertEquals(s, msg.toString(CharsetUtil.UTF_8)); + ReferenceCountUtil.release(msg); + } + } finally { + assertFalse(chDecoderGZip.finish()); + chDecoderGZip.close(); + } + } } diff --git a/codec/src/test/resources/multiple.gz b/codec/src/test/resources/multiple.gz new file mode 100644 index 0000000000000000000000000000000000000000..f5fd0675ee17e4477cb8707b69453ea355812b19 GIT binary patch literal 46 vcmb2|=HSTpIOfN|oXFtK!r;7b`wK<}1_pVca3xqciNTwR;ph8(g&<)794!np literal 0 HcmV?d00001