Merge pull request esheldon#61 from dstndstn/bzip2

Add bzip2 read support

Thanks very much to Dustin

cfitsio3370/drvrfile.c

@@ -20,7 +20,7 @@
 #endif
 
 #ifdef HAVE_FTRUNCATE
-#if defined(unix) || defined(__unix__)  || defined(__unix)
+#if defined(unix) || defined(__unix__)  || defined(__unix) || defined(HAVE_UNISTD_H)
 #include <unistd.h>  /* needed for getcwd prototype on unix machines */
 #endif
 #endif
@@ -772,6 +772,13 @@ int file_is_compressed(char *filename) /* I - FITS file name          */
       strcat(filename,".gz");
       if (file_openfile(filename, 0, &diskfile))
       {
+#if HAVE_BZIP2
+        strcpy(tmpfilename,filename);
+        strcat(filename,".bz2");
+        printf("Trying to open file %s\n", filename);
+        if (file_openfile(filename, 0, &diskfile))
+        {
+#endif
         strcpy(filename, tmpfilename);
         strcat(filename,".Z");
         if (file_openfile(filename, 0, &diskfile))
@@ -799,6 +806,9 @@ int file_is_compressed(char *filename) /* I - FITS file name          */
             }
           }
         }
+#if HAVE_BZIP2
+        }
+#endif
       }
     }
 
@@ -815,7 +825,8 @@ int file_is_compressed(char *filename) /* I - FITS file name          */
          (memcmp(buffer, "\120\113", 2) == 0) ||  /* PKZIP */
          (memcmp(buffer, "\037\036", 2) == 0) ||  /* PACK  */
          (memcmp(buffer, "\037\235", 2) == 0) ||  /* LZW   */
-         (memcmp(buffer, "\037\240", 2) == 0) )   /* LZH   */
+         (memcmp(buffer, "\037\240", 2) == 0) ||  /* LZH   */
+         (memcmp(buffer, "BZ",       2) == 0) )   /* BZip2 */
         {
             return(1);  /* this is a compressed file */
         }

cfitsio3370/drvrmem.c

@@ -9,6 +9,10 @@
 #include <stddef.h>  /* apparently needed to define size_t */
 #include "fitsio2.h"
 
+#if HAVE_BZIP2
+#include "bzlib.h"
+#endif
+
 /* prototype for .Z file uncompression function in zuncompress.c */
 int zuncompress2mem(char *filename, 
              FILE *diskfile, 
@@ -655,6 +659,10 @@ int mem_compress_open(char *filename, int rwmode, int *hdl)
         finalsize = 0;  /* for most methods we can't determine final size */
     else if (memcmp(buffer, "\037\240", 2) == 0)  /* LZH */
         finalsize = 0;  /* for most methods we can't determine final size */
+#if HAVE_BZIP2
+    else if (memcmp(buffer, "BZ", 2) == 0)        /* BZip2 */
+        finalsize = 0;  /* for most methods we can't determine final size */
+#endif
     else
     {
         /* not a compressed file; this should never happen */
@@ -1060,6 +1068,52 @@ int mem_uncompress2mem(char *filename, FILE *diskfile, int hdl)
 		 memTable[hdl].memsizeptr,   /* pointer to size of memory */
 		 realloc,                     /* reallocation function */
 		 &finalsize, &status);        /* returned file size nd status*/
+#if HAVE_BZIP2
+    } else if (strstr(filename, ".bz2")) {
+        BZFILE* b;
+        char buf[8192];
+        int  bzerror;
+        // we read from the bzip stream into "buf" and then copy into
+        // the real memory-file buffer at this "offset".  Could do
+        // this in one shot, with somewhat more memory-file
+        // book-keeping.  Do it the easy way instead, for now.
+        size_t offset = 0;
+
+        b = BZ2_bzReadOpen(&bzerror, diskfile, 0, 0, NULL, 0);
+        if (bzerror != BZ_OK) {
+            BZ2_bzReadClose(&bzerror, b);
+            ffpmsg("failed to bzReadOpen a bzip2 file\n");
+            return 1;
+        }
+        bzerror = BZ_OK;
+        while (bzerror == BZ_OK) {
+            int     nread;
+            nread = BZ2_bzRead(&bzerror, b, buf, sizeof(buf));
+            if (bzerror == BZ_OK || bzerror == BZ_STREAM_END) {
+                char** ptrptr = memTable[hdl].memaddrptr;
+                size_t sz = *(memTable[hdl].memsizeptr);
+                if (offset + nread > sz) {
+                    // realloc the memory file 50% larger
+                    size_t newsize = sz + sz/2;
+                    *ptrptr = realloc(*ptrptr, newsize);
+                    if (*ptrptr == NULL) {
+                        BZ2_bzReadClose(&bzerror, b);
+                        ffpmsg("failed to realloc uncompressing bzip2");
+                        return 1;
+                    }
+                    *(memTable[hdl].memsizeptr) = newsize;
+                }
+                memcpy(*ptrptr + offset, buf, nread);
+                offset += nread;
+            }
+        }
+        BZ2_bzReadClose(&bzerror, b);
+        if (bzerror != BZ_OK) {
+            ffpmsg("failure closing bz2 file with BZ2_bzReadClose()\n");
+            return 1;
+        }
+        finalsize = offset;
+#endif
     } else {
          uncompress2mem(filename, diskfile,
 		 memTable[hdl].memaddrptr,   /* pointer to memory address */

fitsio/fitslib.py

@@ -79,7 +79,7 @@ def read(filename, ext=None, extver=None, **keys):
 
     with FITS(filename, **keys) as fits:
 
-        header=keys.get('header',False)
+        header=keys.pop('header',False)
 
         if ext is None:
             for i in xrange(len(fits)):

fitsio/test.py

@@ -1333,6 +1333,44 @@ def testGZWriteRead(self):
             if os.path.exists(fname):
                 os.remove(fname)
 
+    def testBz2Read(self):
+        '''
+        Write a normal .fits file, run bzip2 on it, then read the bz2
+        file and verify that it's the same as what we put in; we don't
+        [currently support or] test *writing* bzip2.
+        '''
+        fname=tempfile.mktemp(prefix='fitsio-BZ2TableWrite-',suffix='.fits')
+        bzfname = fname + '.bz2'
+
+        try:
+            fits = fitsio.FITS(fname,'rw',clobber=True)
+            fits.write_table(self.data, header=self.keys, extname='mytable')
+            fits.close()
+
+            os.system('bzip2 %s' % fname)
+            f2 = fitsio.FITS(bzfname)
+            d = f2[1].read()
+            self.compare_rec(self.data, d, "bzip2 read")
+
+            h = f2[1].read_header()
+            for entry in self.keys:
+                name=entry['name'].upper()
+                value=entry['value']
+                hvalue = h[name]
+                if isinstance(hvalue,str):
+                    hvalue = hvalue.strip()
+                self.assertEqual(value,hvalue,"testing header key '%s'" % name)
+                if 'comment' in entry:
+                    self.assertEqual(entry['comment'].strip(),
+                                     h.get_comment(name).strip(),
+                                     "testing comment for header key '%s'" % name)
+        except:
+            self.assertTrue(False, 'Exception in testing bzip2 reading')
+        finally:
+            if os.path.exists(fname):
+                os.remove(fname)
+            if os.path.exists(bzfname):
+                os.remove(bzfname)
 
     def testChecksum(self):
         """

setup.py

@@ -54,7 +54,7 @@ def copy_update(dir1,dir2):
 
     def configure_cfitsio():
         os.chdir(cfitsio_build_dir)
-        ret=os.system('sh ./configure')
+        ret=os.system('sh ./configure --with-bzip2')
         if ret != 0:
             raise ValueError("could not configure cfitsio %s" % cfitsio_version)
         os.chdir(package_basedir)
@@ -100,6 +100,7 @@ def compile_cfitsio():
 
 ext=Extension("fitsio._fitsio_wrap", 
               sources,
+              libraries=['bz2'],
               extra_objects=extra_objects,
               extra_compile_args=extra_compile_args, 
               extra_link_args=extra_link_args,