The following code fragment demonstrates a trivial compression + * and decompression of a string using IntelDeflater and + * Inflater. + * + *
+ * try {
+ * // Encode a String into bytes
+ * String inputString = "blahblahblah";
+ * byte[] input = inputString.getBytes("UTF-8");
+ *
+ * // Compress the bytes
+ * byte[] output = new byte[100];
+ * IntelDeflater compresser = new IntelDeflater();
+ * compresser.setInput(input);
+ * compresser.finish();
+ * int compressedDataLength = compresser.deflate(output);
+ * compresser.end();
+ *
+ * // Decompress the bytes
+ * Inflater decompresser = new Inflater();
+ * decompresser.setInput(output, 0, compressedDataLength);
+ * byte[] result = new byte[100];
+ * int resultLength = decompresser.inflate(result);
+ * decompresser.end();
+ *
+ * // Decode the bytes into a String
+ * String outputString = new String(result, 0, resultLength, "UTF-8");
+ * } catch(java.io.UnsupportedEncodingException ex) {
+ * // handle
+ * } catch (java.util.zip.DataFormatException ex) {
+ * // handle
+ * }
+ * This method uses {@link #NO_FLUSH} as its compression flush mode. + * An invocation of this method of the form {@code deflater.deflate(b, off, len)} + * yields the same result as the invocation of + * {@code deflater.deflate(b, off, len, IntelDeflater.NO_FLUSH)}. + * + * @param b the buffer for the compressed data + * @param off the start offset of the data + * @param len the maximum number of bytes of compressed data + * @return the actual number of bytes of compressed data written to the + * output buffer + */ + @Override + public int deflate(byte[] b, int off, int len) { + return deflate(b, off, len, NO_FLUSH); + } + + /** + * Compresses the input data and fills specified buffer with compressed + * data. Returns actual number of bytes of compressed data. A return value + * of 0 indicates that {@link #needsInput() needsInput} should be called + * in order to determine if more input data is required. + * + *
This method uses {@link #NO_FLUSH} as its compression flush mode. + * An invocation of this method of the form {@code deflater.deflate(b)} + * yields the same result as the invocation of + * {@code deflater.deflate(b, 0, b.length, IntelDeflater.NO_FLUSH)}. + * + * @param b the buffer for the compressed data + * @return the actual number of bytes of compressed data written to the + * output buffer + */ + @Override + public int deflate(byte[] b) { + return deflate(b, 0, b.length, NO_FLUSH); + } + + /** + * Compresses the input data and fills the specified buffer with compressed + * data. Returns actual number of bytes of data compressed. + * + *
Compression flush mode is one of the following three modes: + * + *
In the case of {@link #FULL_FLUSH} or {@link #SYNC_FLUSH}, if + * the return value is {@code len}, the space available in output + * buffer {@code b}, this method should be invoked again with the same + * {@code flush} parameter and more output space. + * + * @param b the buffer for the compressed data + * @param off the start offset of the data + * @param len the maximum number of bytes of compressed data + * @param flush the compression flush mode + * @return the actual number of bytes of compressed data written to + * the output buffer + * + * @throws IllegalArgumentException if the flush mode is invalid + * @since 1.7 + */ + public int deflate(byte[] b, int off, int len, int flush) { + //System.out.println("Inside IntelDeflater\n"); + if (b == null) { + throw new NullPointerException(); + } + if (off < 0 || len < 0 || off > b.length - len) { + throw new ArrayIndexOutOfBoundsException(); + } + synchronized (zsRef) { + ensureOpen(); + if (flush == NO_FLUSH || flush == SYNC_FLUSH || + flush == FULL_FLUSH) + return deflateBytes(zsRef.address(), b, off, len, flush); + throw new IllegalArgumentException(); + } + } + + /** + * Returns the ADLER-32 value of the uncompressed data. + * @return the ADLER-32 value of the uncompressed data + */ + @Override + public int getAdler() { + synchronized (zsRef) { + ensureOpen(); + return getAdler(zsRef.address()); + } + } + + /** + * Returns the total number of uncompressed bytes input so far. + * + *
Since the number of bytes may be greater than + * Integer.MAX_VALUE, the {@link #getBytesRead()} method is now + * the preferred means of obtaining this information.
+ * + * @return the total number of uncompressed bytes input so far + */ + @Override + public int getTotalIn() { + return (int) getBytesRead(); + } + + /** + * Returns the total number of uncompressed bytes input so far. + * + * @return the total (non-negative) number of uncompressed bytes input so far + * @since 1.5 + */ + @Override + public long getBytesRead() { + synchronized (zsRef) { + ensureOpen(); + return getBytesRead(zsRef.address()); + } + } + + /** + * Returns the total number of compressed bytes output so far. + * + *Since the number of bytes may be greater than + * Integer.MAX_VALUE, the {@link #getBytesWritten()} method is now + * the preferred means of obtaining this information.
+ * + * @return the total number of compressed bytes output so far + */ + @Override + public int getTotalOut() { + return (int) getBytesWritten(); + } + + /** + * Returns the total number of compressed bytes output so far. + * + * @return the total (non-negative) number of compressed bytes output so far + * @since 1.5 + */ + @Override + public long getBytesWritten() { + synchronized (zsRef) { + ensureOpen(); + return getBytesWritten(zsRef.address()); + } + } + + /** + * Resets deflater so that a new set of input data can be processed. + * Keeps current compression level and strategy settings. + */ + @Override + public void reset() { + synchronized (zsRef) { + ensureOpen(); + reset(zsRef.address()); + finish = false; + finished = false; + off = len = 0; + } + } + + /** + * Closes the compressor and discards any unprocessed input. + * This method should be called when the compressor is no longer + * being used, but will also be called automatically by the + * finalize() method. Once this method is called, the behavior + * of the IntelDeflater object is undefined. + */ + @Override + public void end() { + synchronized (zsRef) { + long addr = zsRef.address(); + zsRef.clear(); + if (addr != 0) { + end(addr); + buf = null; + } + } + } + + /** + * Closes the compressor when garbage is collected. + */ + protected void finalize() { + end(); + } + + private void ensureOpen() { + assert Thread.holdsLock(zsRef); + if (zsRef.address() == 0) + throw new NullPointerException("IntelDeflater has been closed"); + } + + private static native void initIDs(); + private native static long init(int level, int strategy, boolean nowrap); + private native static void setDictionary(long addr, byte[] b, int off, int len); + private native int deflateBytes(long addr, byte[] b, int off, int len, + int flush); + private native static int getAdler(long addr); + private native static long getBytesRead(long addr); + private native static long getBytesWritten(long addr); + private native static void reset(long addr); + private native static void end(long addr); +} + diff --git a/src/java/net/sf/samtools/util/zip/ZStreamRef.java b/src/java/net/sf/samtools/util/zip/ZStreamRef.java new file mode 100644 index 0000000000..5a49d8a0a6 --- /dev/null +++ b/src/java/net/sf/samtools/util/zip/ZStreamRef.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2009, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package net.sf.samtools.util.zip; + +/** + * Copied from OpenJDK 7. Only change is package name, because IntelDeflater needs this and + * the original class is private to java.util.zip package. + * + * A reference to the native zlib's z_stream structure. + */ + +class ZStreamRef { + + private long address; + ZStreamRef (long address) { + this.address = address; + } + + long address() { + return address; + } + + void clear() { + address = 0; + } +} diff --git a/src/scripts/build_intel_deflater.sh b/src/scripts/build_intel_deflater.sh new file mode 100644 index 0000000000..49d018c30d --- /dev/null +++ b/src/scripts/build_intel_deflater.sh @@ -0,0 +1,64 @@ +#! /bin/bash +# +# The MIT License +# +# Copyright (c) 2013 The Broad Institute +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +# Build libIntelDeflater.so, the JNI library that wraps Intel IPP compression library. +# Note that this is not built as part of standard release process. Rather, it is built manually and then +# copied to Picard-public/lib/jni. + +# Assumes OpenJDK exists at $OPENJDK. I used openjdk-7-fcs-src-b147-27_jun_2011.zip +# Assumes that Picard-public java sources have been compiled +# Assumes IPP8_CODE_SAMPLES_DIR points to Intel IPP sample code built with -fPIC +set -e + +if [ "$OPENJDK" = "" ] +then echo "ERROR: OPENJDK environment variable not defined." >&2 + exit 1 +fi + +if [ "$IPP8_CODE_SAMPLES_DIR" = "" ] +then echo "ERROR: IPP8_CODE_SAMPLES_DIR environment variable not defined." >&2 + exit 1 +fi + +rootdir=$(dirname $(dirname $(dirname $0))) + + +builddir=$rootdir/lib_build +rm -rf $builddir +mkdir -p $builddir + +# Create JNI C header file +javah -jni -classpath $rootdir/classes -d $builddir net.sf.samtools.util.zip.IntelDeflater + +# Compile source and create library. +gcc -I$builddir -I$JAVA_HOME/include/ -I$JAVA_HOME/include/linux/ -I$OPENJDK/jdk/src/share/native/common/ \ +-I$OPENJDK/jdk/src/solaris/native/common/ -c -O3 -fPIC IntelDeflater.c +gcc -shared -o $builddir/libIntelDeflater.so IntelDeflater.o -L${IPP8_CODE_SAMPLES_DIR}/__cmake/data-compression.intel64.make.static.release/__lib/release \ +-lzlib -lstdc++ -Wl,-Bstatic -lbfp754 -ldecimal -liomp5 -liompstubs5 -lipgo -lippac -lippcc -lippch -lippcv \ +-lippdc -lippdi -lippgen -lippi -lippj -lippm -lippr -lippsc -lippvc -lippvm -lirng -lmatmul -lpdbx \ +-lpdbxinst -lsvml -lipps -limf -lirc -lirc_s -lippcore -Wl,-Bdynamic + + +