MRF: Add QB3 compression(OSGeo#5824)

Add [QB3](https://github.com/lucianpls/QB3) compression to MRF

QB3 is a new raster specific lossless compression for integer types, lightweight, very fast and very good.

.github/workflows/cmake_builds.yml

@@ -114,6 +114,19 @@ jobs:
             && cd ../.. \
             && rm -rf libjxl
 
+    - name: Buid libQB3
+      run: |
+        # Build libQB3
+        # Used by the MRF driver
+        git clone https://github.com/lucianpls/QB3.git \
+            && mkdir QB3/QB3lib/build \
+            && cd QB3/QB3lib/build \
+            && /usr/bin/cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr .. \
+            && make -j$(nproc) \
+            && sudo make -j$(nproc) install \
+            && cd ../../.. \
+            && rm -rf QB3
+
     - name: Install pdfium
       run: |
         wget -q https://github.com/rouault/pdfium_build_gdal_3_5/releases/download/v1_pdfium_5106/install-ubuntu2004-rev5106.tar.gz \

GDALmake.opt.in

@@ -34,7 +34,7 @@ INSTALL_DIR	= 	$(GDAL_ROOT)/install-sh -d
 LIBS	=	@LIBS@ $(KAK_LIBS) $(DWG_LIBS) $(CURL_LIB) \
 		$(MRSID_LIBS) $(ECW_LIBS) \
 		$(PCIDSK_LIB) $(RASDAMAN_LIB) $(SOSI_LIB) $(BRUNSLI_LIB) \
-		$(OPENCL_LIB) $(LIBICONV) $(FGDB_LIB) $(LIBXML2_LIB) \
+		$(OPENCL_LIB) $(LIBICONV) $(FGDB_LIB) $(LIBXML2_LIB) $(QB3_LIB) \
 		$(MONGOCXXV3_LIBS) $(JNI_LIB) $(HDFS_LIB)
 
 SSEFLAGS = @SSEFLAGS@
@@ -452,6 +452,13 @@ BRUNSLI_ENABLED = @BRUNSLI_ENABLED@
 BRUNSLI_INCLUDE = @BRUNSLI_INCLUDE@
 BRUNSLI_LIB	= @BRUNSLI_LIB@
 
+#
+# QB3
+#
+QB3_ENABLED = @QB3_ENABLED@
+QB3_INCLUDE = @QB3_INCLUDE@
+QB3_LIB = @QB3_LIB@
+
 #
 # PDF stuff
 #

autotest/gdrivers/mrf.py

@@ -35,33 +35,38 @@
 
 pytestmark = pytest.mark.require_driver('MRF')
 
-mrf_list = [
+mrf_tests = (
     ('byte.tif', 4672, [4672], []),
     ('byte.tif', 4672, [4672], ['COMPRESS=ZSTD']),
     ('byte.tif', 4672, [4672], ['COMPRESS=DEFLATE']),
     ('byte.tif', 4672, [4672], ['COMPRESS=NONE']),
     ('byte.tif', 4672, [4672], ['COMPRESS=LERC']),
+    ('byte.tif', 4672, [4672], ['COMPRESS=QB3']),
     ('byte.tif', 4672, [5015], ['COMPRESS=LERC', 'OPTIONS:LERC_PREC=10']),
     ('byte.tif', 4672, [4672], ['COMPRESS=LERC', 'OPTIONS=V1:YES']),
     ('int16.tif', 4672, [4672], []),
     ('int16.tif', 4672, [4672], ['COMPRESS=ZSTD']),
     ('int16.tif', 4672, [4672], ['COMPRESS=DEFLATE']),
     ('int16.tif', 4672, [4672], ['COMPRESS=LERC']),
+    ('int16.tif', 4672, [4672], ['COMPRESS=QB3']),
     ('int16.tif', 4672, [4672], ['COMPRESS=LERC', 'OPTIONS=V1:YES']),
     ('../../gcore/data/uint16.tif', 4672, [4672], []),
     ('../../gcore/data/uint16.tif', 4672, [4672], ['COMPRESS=DEFLATE']),
     ('../../gcore/data/uint16.tif', 4672, [4672], ['COMPRESS=ZSTD']),
     ('../../gcore/data/uint16.tif', 4672, [4672], ['COMPRESS=LERC']),
+    ('../../gcore/data/uint16.tif', 4672, [4672], ['COMPRESS=QB3']),
     ('../../gcore/data/uint16.tif', 4672, [4672], ['COMPRESS=LERC', 'OPTIONS=V1:YES']),
     ('int32.tif', 4672, [4672], ['COMPRESS=DEFLATE']),
     ('int32.tif', 4672, [4672], ['COMPRESS=ZSTD']),
     ('int32.tif', 4672, [4672], ['COMPRESS=TIF']),
     ('int32.tif', 4672, [4672], ['COMPRESS=LERC']),
+    ('int32.tif', 4672, [4672], ['COMPRESS=QB3']),
     ('int32.tif', 4672, [4672], ['COMPRESS=LERC', 'OPTIONS=V1:YES']),
     ('../../gcore/data/uint32.tif', 4672, [4672], ['COMPRESS=DEFLATE']),
     ('../../gcore/data/uint32.tif', 4672, [4672], ['COMPRESS=ZSTD']),
     ('../../gcore/data/uint32.tif', 4672, [4672], ['COMPRESS=TIF']),
     ('../../gcore/data/uint32.tif', 4672, [4672], ['COMPRESS=LERC']),
+    ('../../gcore/data/uint32.tif', 4672, [4672], ['COMPRESS=QB3']),
     ('../../gcore/data/uint32.tif', 4672, [4672], ['COMPRESS=LERC', 'OPTIONS=V1:YES']),
     ('float32.tif', 4672, [4672], ['COMPRESS=DEFLATE']),
     ('float32.tif', 4672, [4672], ['COMPRESS=ZSTD']),
@@ -77,6 +82,9 @@
     ('../../gcore/data/utmsmall.tif', 50054, [50054], []),
     ('small_world.tif', 30111, [30111], ['COMPRESS=ZSTD']),
     ('small_world.tif', 30111, [30111], ['COMPRESS=ZSTD', 'INTERLEAVE=PIXEL']),
+    ('small_world.tif', 30111, [30111], ['COMPRESS=QB3']),
+    ('small_world.tif', 30111, [30111], ['COMPRESS=QB3',  'INTERLEAVE=PIXEL']),
+    ('small_world.tif', 30111, [30111], ['COMPRESS=QB3', 'QUALITY=99']),
     ('small_world.tif', 30111, [30111], ['COMPRESS=LERC', 'INTERLEAVE=PIXEL']),
     ('small_world.tif', 30111, [30111], ['COMPRESS=LERC', 'OPTIONS=V1:1', 'INTERLEAVE=PIXEL']),
     ('small_world_pct.tif', 14890, [14890], ['COMPRESS=PPNG']),
@@ -89,22 +97,20 @@
     ('jpeg/12bit_rose_extract.jpg', 30075, [29650, 29680, 29680, 29650], ['COMPRESS=JPEG']),
     # checksum depends on floating point precision
     ('f32nan_data.tif', 54061, [54052, 54050], ['COMPRESS=LERC', 'OPTIONS=V1:Yes LERC_PREC:0.01']),
-]
+)
 
 @pytest.mark.parametrize(
     'src_filename,chksum,chksum_after_reopening,options',
-    mrf_list,
-    ids=['{0}-{3}'.format(*r) for r in mrf_list],
+    mrf_tests,
+    ids=('{0}-{3}'.format(*r) for r in mrf_tests),
 )
 def test_mrf(src_filename, chksum, chksum_after_reopening, options):
 
     mrf_co = gdal.GetDriverByName('MRF').GetMetadataItem('DMD_CREATIONOPTIONLIST')
 
-    if 'COMPRESS=LERC' in options and 'LERC' not in mrf_co:
-        pytest.skip()
-
-    if 'COMPRESS=ZSTD' in options and 'ZSTD' not in mrf_co:
-        pytest.skip()
+    for comp in "LERC", "ZSTD", "QB3":
+        if ("COMPRESS=" + comp) in options and comp not in mrf_co:
+            pytest.skip()
 
     if 'jpg' in src_filename:
         import jpeg
@@ -127,7 +133,7 @@ def test_mrf(src_filename, chksum, chksum_after_reopening, options):
     return ut.testCreateCopy(check_minmax=check_minmax)
 
 def cleanup(base = '/vsimem/out.'):
-    for ext in ['mrf', 'mrf.aux.xml', 'idx', 'ppg', 'til', 'lrc', 'pjg', 'pzp', 'psz']:
+    for ext in 'mrf', 'mrf.aux.xml', 'idx', 'ppg', 'til', 'lrc', 'pjg', 'pzp', 'psz', 'pq3':
         gdal.Unlink(base + ext)
 
 def test_mrf_zen_test():
@@ -166,9 +172,9 @@ def test_mrf_zen_test():
 def test_mrf_overview_nnb_fact_2():
 
     expected_cs = 1087
-    for dt in [gdal.GDT_Byte, gdal.GDT_Int16, gdal.GDT_UInt16,
+    for dt in (gdal.GDT_Byte, gdal.GDT_Int16, gdal.GDT_UInt16,
                gdal.GDT_Int32, gdal.GDT_UInt32,
-               gdal.GDT_Float32, gdal.GDT_Float64]:
+               gdal.GDT_Float32, gdal.GDT_Float64):
 
         out_ds = gdal.Translate('/vsimem/out.mrf', 'data/byte.tif',
                                 format='MRF',
@@ -230,9 +236,9 @@ def test_mrf_overview_avg_fact_2():
 
 def test_mrf_overview_avg_with_nodata_fact_2():
 
-    for dt in [gdal.GDT_Byte, gdal.GDT_Int16, gdal.GDT_UInt16,
+    for dt in (gdal.GDT_Byte, gdal.GDT_Int16, gdal.GDT_UInt16,
                gdal.GDT_Int32, gdal.GDT_UInt32,
-               gdal.GDT_Float32, gdal.GDT_Float64]:
+               gdal.GDT_Float32, gdal.GDT_Float64):
 
         out_ds = gdal.Translate('/vsimem/out.mrf', 'data/byte.tif',
                                 format='MRF',
@@ -493,7 +499,7 @@ def test_mrf_versioned():
 
 def test_mrf_cleanup():
 
-    files = [
+    files = (
         '12bit_rose_extract.jpg.*',
         'byte.tif.*',
         'int16.tif.*',
@@ -507,9 +513,9 @@ def test_mrf_cleanup():
         'utmsmall.tif.*',
         'cloning.*',
         'f32nan_data.*'
-        ]
+    )
 
-    for f in [fname for n in files for fname in glob.glob('tmp/' + n)]:
+    for f in (fname for n in files for fname in glob.glob('tmp/' + n)):
         gdal.Unlink(f)
 
     cleanup()

cmake/helpers/CheckDependentLibraries.cmake

@@ -417,6 +417,8 @@ gdal_internal_library(LERC)
 
 gdal_check_package(BRUNSLI "Enable BRUNSLI for JPEG packing in MRF" CAN_DISABLE RECOMMENDED)
 
+gdal_check_package(libQB3 "Enable QB3 compression in MRF" CAN_DISABLE RECOMMENDED)
+
 # Disable by default the use of external shapelib, as currently the SAOffset member that holds file offsets in it is a
 # 'unsigned long', hence 32 bit on 32 bit platforms, whereas we can handle DBFs file > 4 GB. Internal shapelib has not
 # this issue

configure.ac

@@ -5148,6 +5148,22 @@ if test "$with_brunsli" != "" -a "$with_brunsli" != "no" ; then
   AC_SUBST(BRUNSLI_LIB, "-lbrunslienc-c -lbrunslidec-c")
 fi
 
+dnl ---------------------------------------------------------------------------
+dnl QB3 support
+dnl ---------------------------------------------------------------------------
+
+AC_ARG_WITH(QB3,[  --with-QB3    Include QB3 support for MRF],,)
+
+QB3_ENABLED=no
+
+if test "$with_QB3" != "" -a "$with_QB3" != "no" ; then
+  QB3_ENABLED="yes"
+  AC_SUBST(QB3_ENABLED, $QB3_ENABLED)
+  AC_SUBST(QB3_INCLUDE, -I$prefix/include)
+  AC_SUBST(QB3_LIB, "-lQB3")
+fi
+
+
 dnl ---------------------------------------------------------------------------
 dnl RDB support
 dnl ---------------------------------------------------------------------------

doc/source/build_hints.rst

@@ -243,6 +243,7 @@ It is used by the :ref:`raster.zarr` driver.
 
     Control whether to use Blosc. Defaults to ON when Blosc is found.
 
+
 BRUNSLI
 *******
 
@@ -1501,6 +1502,17 @@ PROJ
     building Debug releases.
 
 
+QB3
+*******
+
+The `QB3 <https://github.com/lucianpls/QB3>`_ compression, used
+by the :ref:`raster.marfa` driver.
+
+.. option:: GDAL_USE_QB3=ON/OFF
+
+    Control whether to use QB3. Defaults to ON when QB3 is found.
+
+
 QHULL
 *****
 

frmts/mrf/CMakeLists.txt

@@ -84,3 +84,10 @@ if (GDAL_USE_BRUNSLI)
   target_compile_definitions(gdal_MRF PRIVATE -DBRUNSLI)
   gdal_target_link_libraries(gdal_MRF PRIVATE BRUNSLI::ENCODE BRUNSLI::DECODE)
 endif ()
+
+if (GDAL_USE_LIBQB3)
+  target_sources(gdal_MRF PRIVATE QB3_band.cpp)
+  target_compile_definitions(gdal_MRF PRIVATE -DQB3_SUPPORT)
+  gdal_target_link_libraries(gdal_MRF PRIVATE QB3::libQB3)
+endif()
+

frmts/mrf/GNUmakefile

@@ -70,6 +70,11 @@ ifeq ($(ZSTD_SETTING),yes)
 XTRA_OPT		:=	$(XTRA_OPT) -DZSTD_SUPPORT
 endif
 
+ifeq ($(QB3_ENABLED),yes)
+XTRA_OPT	:=	$(XTRA_OPT) -DQB3_SUPPORT $(QB3_INCLUDE)
+OBJ	:=	$(OBJ) QB3_band.o
+endif
+
 CPPFLAGS        :=  $(XTRA_OPT)  $(CPPFLAGS)
 
 default:  $(PREDEPEND)	$(OBJ:.o=.$(OBJ_EXT)) $(SUBLIBS)