Skip to content

Commit

Permalink
Fixes for compression codecs
Browse files Browse the repository at this point in the history
Motivation:

Fixed founded mistakes in compression codecs.

Modifications:

- Changed return type of ZlibUtil.inflaterException() from CompressionException to DecompressionException
- Updated @throws in javadoc of JZlibDecoder to throw DecompressionException instead of CompressionException
- Fixed JdkZlibDecoder to throw DecompressionException instead of CompressionException
- Removed unnecessary empty lines in JdkZlibEncoder and JZlibEncoder
- Removed public modifier from Snappy class
- Added MAX_UNCOMPRESSED_DATA_SIZE constant in SnappyFramedDecoder
- Used in.readableBytes() instead of (in.writerIndex() - in.readerIndex()) in SnappyFramedDecoder
- Added private modifier for enum ChunkType in SnappyFramedDecoder
- Fixed potential bug (sum overflow) in Bzip2HuffmanAllocator.first(). For more info, see http://googleresearch.blogspot.ru/2006/06/extra-extra-read-all-about-it-nearly.html

Result:

Fixed sum overflow in Bzip2HuffmanAllocator, improved exceptions in ZlibDecoder implementations, hid Snappy class
idelpivnitskiy authored and Norman Maurer committed Jul 20, 2014

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent dd26d47 commit 37c7766
Showing 8 changed files with 18 additions and 19 deletions.
Original file line number Diff line number Diff line change
@@ -42,7 +42,7 @@ private static int first(final int[] array, int i, final int nodesToMove) {
i = Math.max(nodesToMove - 1, i);

while (k > i + 1) {
int temp = i + k >> 1;
int temp = i + k >>> 1;
if (array[temp] % length > limit) {
k = temp;
} else {
Original file line number Diff line number Diff line change
@@ -31,7 +31,7 @@ public class JZlibDecoder extends ZlibDecoder {
/**
* Creates a new instance with the default wrapper ({@link ZlibWrapper#ZLIB}).
*
* @throws CompressionException if failed to initialize zlib
* @throws DecompressionException if failed to initialize zlib
*/
public JZlibDecoder() {
this(ZlibWrapper.ZLIB);
@@ -40,7 +40,7 @@ public JZlibDecoder() {
/**
* Creates a new instance with the specified wrapper.
*
* @throws CompressionException if failed to initialize zlib
* @throws DecompressionException if failed to initialize zlib
*/
public JZlibDecoder(ZlibWrapper wrapper) {
if (wrapper == null) {
@@ -58,7 +58,7 @@ public JZlibDecoder(ZlibWrapper wrapper) {
* is always {@link ZlibWrapper#ZLIB} because it is the only format that
* supports the preset dictionary.
*
* @throws CompressionException if failed to initialize zlib
* @throws DecompressionException if failed to initialize zlib
*/
public JZlibDecoder(byte[] dictionary) {
if (dictionary == null) {
Original file line number Diff line number Diff line change
@@ -29,7 +29,6 @@

import java.util.concurrent.TimeUnit;


/**
* Compresses a {@link ByteBuf} using the deflate algorithm.
*/
Original file line number Diff line number Diff line change
@@ -24,7 +24,6 @@
import java.util.zip.Deflater;
import java.util.zip.Inflater;


/**
* Decompress a {@link ByteBuf} using the inflate algorithm.
*/
@@ -249,14 +248,14 @@ private boolean readGZIPHeader(ByteBuf in) {
int magic1 = in.readByte();

if (magic0 != 31) {
throw new CompressionException("Input is not in the GZIP format");
throw new DecompressionException("Input is not in the GZIP format");
}
crc.update(magic0);
crc.update(magic1);

int method = in.readUnsignedByte();
if (method != Deflater.DEFLATED) {
throw new CompressionException("Unsupported compression method "
throw new DecompressionException("Unsupported compression method "
+ method + " in the GZIP header");
}
crc.update(method);
@@ -265,7 +264,7 @@ private boolean readGZIPHeader(ByteBuf in) {
crc.update(flags);

if ((flags & FRESERVED) != 0) {
throw new CompressionException(
throw new DecompressionException(
"Reserved flags are set in the GZIP header");
}

@@ -360,7 +359,7 @@ private boolean readGZIPFooter(ByteBuf buf) {
}
int readLength = inflater.getTotalOut();
if (dataLength != readLength) {
throw new CompressionException(
throw new DecompressionException(
"Number of bytes mismatch. Expected: " + dataLength + ", Got: " + readLength);
}
return true;
@@ -373,7 +372,7 @@ private void verifyCrc(ByteBuf in) {
}
long readCrc = crc.getValue();
if (crcValue != readCrc) {
throw new CompressionException(
throw new DecompressionException(
"CRC value missmatch. Expected: " + crcValue + ", Got: " + readCrc);
}
}
Original file line number Diff line number Diff line change
@@ -27,7 +27,6 @@
import java.util.zip.CRC32;
import java.util.zip.Deflater;


/**
* Compresses a {@link ByteBuf} using the deflate algorithm.
*/
Original file line number Diff line number Diff line change
@@ -24,7 +24,7 @@
*
* See http://code.google.com/p/snappy/source/browse/trunk/format_description.txt
*/
public class Snappy {
class Snappy {

private static final int MAX_HT_SIZE = 1 << 14;
private static final int MIN_COMPRESSIBLE_BYTES = 15;
Original file line number Diff line number Diff line change
@@ -36,7 +36,8 @@
* set to {@code true}.
*/
public class SnappyFramedDecoder extends ByteToMessageDecoder {
enum ChunkType {

private enum ChunkType {
STREAM_IDENTIFIER,
COMPRESSED_DATA,
UNCOMPRESSED_DATA,
@@ -45,6 +46,7 @@ enum ChunkType {
}

private static final byte[] SNAPPY = { 's', 'N', 'a', 'P', 'p', 'Y' };
private static final int MAX_UNCOMPRESSED_DATA_SIZE = 65536 + 4;

private final Snappy snappy = new Snappy();
private final boolean validateChecksums;
@@ -83,7 +85,7 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) t

try {
int idx = in.readerIndex();
final int inSize = in.writerIndex() - idx;
final int inSize = in.readableBytes();
if (inSize < 4) {
// We need to be at least able to read the chunk type identifier (one byte),
// and the length of the chunk (3 bytes) in order to proceed
@@ -136,7 +138,7 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) t
if (!started) {
throw new DecompressionException("Received UNCOMPRESSED_DATA tag before STREAM_IDENTIFIER");
}
if (chunkLength > 65536 + 4) {
if (chunkLength > MAX_UNCOMPRESSED_DATA_SIZE) {
throw new DecompressionException("Received UNCOMPRESSED_DATA larger than 65540 bytes");
}

@@ -193,7 +195,7 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) t
* @param type The tag byte extracted from the stream
* @return The appropriate {@link ChunkType}, defaulting to {@link ChunkType#RESERVED_UNSKIPPABLE}
*/
static ChunkType mapChunkType(byte type) {
private static ChunkType mapChunkType(byte type) {
if (type == 0) {
return ChunkType.COMPRESSED_DATA;
} else if (type == 1) {
Original file line number Diff line number Diff line change
@@ -32,8 +32,8 @@ static void fail(Deflater z, String message, int resultCode) {
throw deflaterException(z, message, resultCode);
}

static CompressionException inflaterException(Inflater z, String message, int resultCode) {
return new CompressionException(message + " (" + resultCode + ')' + (z.msg != null? ": " + z.msg : ""));
static DecompressionException inflaterException(Inflater z, String message, int resultCode) {
return new DecompressionException(message + " (" + resultCode + ')' + (z.msg != null? ": " + z.msg : ""));
}

static CompressionException deflaterException(Deflater z, String message, int resultCode) {

0 comments on commit 37c7766

Please sign in to comment.