Clip composite pixel values to fit into int8 (holoviz#689)

* Clip composite pixel values to fit into int8, avoiding overflow for add
* Fixed tests that previously had incorrect reference data

datashader/composite.py

@@ -1,18 +1,23 @@
+"""
+Binary graphical composition operators
+
+See https://www.cairographics.org/operators/; more could easily be added from there.
+"""
+
 from __future__ import division
 
 import numba as nb
 import numpy as np
 import os
 
-
 __all__ = ('composite_op_lookup', 'over', 'add', 'saturate', 'source')
 
 
 @nb.jit('(uint32,)', nopython=True, nogil=True, cache=True)
 def extract_scaled(x):
     """Extract components as float64 values in [0.0, 1.0]"""
-    r = np.float64((x & 255) / 255)
-    g = np.float64(((x >> 8) & 255) / 255)
+    r = np.float64(( x        & 255) / 255)
+    g = np.float64(((x >>  8) & 255) / 255)
     b = np.float64(((x >> 16) & 255) / 255)
     a = np.float64(((x >> 24) & 255) / 255)
     return r, g, b, a
@@ -22,10 +27,10 @@ def extract_scaled(x):
         nogil=True, cache=True)
 def combine_scaled(r, g, b, a):
     """Combine components in [0, 1] to rgba uint32"""
-    r2 = np.uint32(r * 255)
-    g2 = np.uint32(g * 255)
-    b2 = np.uint32(b * 255)
-    a2 = np.uint32(a * 255)
+    r2 = min(255, np.uint32(r * 255))
+    g2 = min(255, np.uint32(g * 255))
+    b2 = min(255, np.uint32(b * 255))
+    a2 = min(255, np.uint32(a * 255))
     return np.uint32((a2 << 24) | (b2 << 16) | (g2 << 8) | r2)
 
 

datashader/tests/test_composite.py

@@ -7,8 +7,8 @@
                 [0xffff0000, 0xff000000, 0x3a3b3c3d]], dtype='uint32')
 
 clear = np.uint32(0)
-clear_black = np.uint32(0x00ffffff)
-black = np.uint32(0xffffffff)
+clear_white = np.uint32(0x00ffffff)
+white = np.uint32(0xffffffff)
 blue = np.uint32(0xffff0000)
 half_blue = np.uint32(0x7dff0000)
 half_purple = np.uint32(0x7d7d007d)
@@ -18,8 +18,8 @@ def test_source():
     o = src.copy()
     o[0, :2] = clear
     np.testing.assert_equal(source(src, clear), o)
-    o[0, :2] = clear_black
-    np.testing.assert_equal(source(src, clear_black), o)
+    o[0, :2] = clear_white
+    np.testing.assert_equal(source(src, clear_white), o)
     o[0, :2] = half_blue
     np.testing.assert_equal(source(src, half_blue), o)
 
@@ -28,11 +28,11 @@ def test_over():
     o = src.copy()
     o[0, 1] = 0
     np.testing.assert_equal(over(src, clear), o)
-    np.testing.assert_equal(over(src, clear_black), o)
+    np.testing.assert_equal(over(src, clear_white), o)
     o = np.array([[0xffffffff, 0xffffffff, 0xffffffff],
                   [0xffff8282, 0xff82ff82, 0xff8282ff],
                   [0xffff0000, 0xff000000, 0xffd2d2d2]])
-    np.testing.assert_equal(over(src, black), o)
+    np.testing.assert_equal(over(src, white), o)
     o = np.array([[0xffff0000, 0xffff0000, 0xffffffff],
                   [0xffff0000, 0xff827d00, 0xff82007d],
                   [0xffff0000, 0xff000000, 0xffd20d0d]])
@@ -51,32 +51,32 @@ def test_add():
     o = src.copy()
     o[0, 1] = 0
     np.testing.assert_equal(add(src, clear), o)
-    np.testing.assert_equal(add(src, clear_black), o)
-    o = np.array([[0xffffffff, 0xffffffff, 0xfffffffe],
-                  [0xff7cffff, 0xffff7cff, 0xffffff7c],
-                  [0xfffeffff, 0xffffffff, 0xff0d0d0c]])
-    np.testing.assert_equal(add(src, black), o)
-    o = np.array([[0xffff0000, 0xffff0000, 0xfffeffff],
-                  [0xff7c0000, 0xffff7d00, 0xffff007d],
-                  [0xfffe0000, 0xffff0000, 0xff0c0d0d]])
+    np.testing.assert_equal(add(src, clear_white), o)
+    o = np.array([[0xffffffff, 0xffffffff, 0xffffffff],
+                  [0xffffffff, 0xffffffff, 0xffffffff],
+                  [0xffffffff, 0xffffffff, 0xffffffff]])
+    np.testing.assert_equal(add(src, white), o)
+    o = np.array([[0xffff0000, 0xffff0000, 0xffffffff],
+                  [0xffff0000, 0xffff7d00, 0xffff007d],
+                  [0xffff0000, 0xffff0000, 0xffff0d0d]])
     np.testing.assert_equal(add(src, blue), o)
-    o = np.array([[0x7dff0000, 0x7dff0000, 0xff7cffff],
+    o = np.array([[0x7dff0000, 0x7dff0000, 0xffffffff],
                   [0xfaff0000, 0xfa7f7f00, 0xfa7f007f],
-                  [0xff7c0000, 0xff7d0000, 0xb7c01313]])
+                  [0xffff0000, 0xff7d0000, 0xb7c01313]])
     np.testing.assert_equal(add(src, half_blue), o)
-    o = np.array([[0x7d7d007d, 0x7d7d007d, 0xff3cff3c],
+    o = np.array([[0x7d7d007d, 0x7d7d007d, 0xffffffff],
                   [0xfabe003e, 0xfa3e7f3e, 0xfa3e00be],
-                  [0xff3c003d, 0xff3d003d, 0xb7681368]])
+                  [0xffff003d, 0xff3d003d, 0xb7681368]])
     np.testing.assert_equal(add(src, half_purple), o)
 
 
 def test_saturate():
     o = src.copy()
     o[0, 1] = 0
     np.testing.assert_equal(saturate(src, clear), o)
-    np.testing.assert_equal(saturate(src, clear_black), o)
-    o = np.full((3, 3), black, dtype='uint32')
-    np.testing.assert_equal(saturate(src, black), o)
+    np.testing.assert_equal(saturate(src, clear_white), o)
+    o = np.full((3, 3), white, dtype='uint32')
+    np.testing.assert_equal(saturate(src, white), o)
     o = np.full((3, 3), blue, dtype='uint32')
     np.testing.assert_equal(saturate(src, blue), o)
     o = np.array([[0x7dff0000, 0x7dff0000, 0xffff8282],

datashader/tests/test_transfer_functions.py

@@ -310,7 +310,7 @@ def test_stack():
 
     img = tf.stack(img1, img2, how='add')
     out = np.array([[0xff00ffff, 0x00000000],
-                    [0x00000000, 0xff3d3cfa]], dtype='uint32')
+                    [0x00000000, 0xff3dfffa]], dtype='uint32')
     assert (img.x_axis == img1.x_axis).all()
     assert (img.y_axis == img1.y_axis).all()
     np.testing.assert_equal(img.data, out)