[PyFI] Update hypothesis and switch from tp2 (pytorch#41645)

Summary:
Pull Request resolved: pytorch#41645

Pull Request resolved: facebookresearch/pytext#1405

Test Plan: buck test

Reviewed By: thatch

Differential Revision: D20323893

fbshipit-source-id: 54665d589568c4198e96a27f0ed8e5b41df7b86b

caffe2/contrib/fakelowp/test/test_batchmatmul_nnpi_fp16.py

@@ -29,6 +29,7 @@ class TestBatchMatMul(serial.SerializedTestCase):
         trans_b=st.booleans(),
         run_ints=st.booleans()
     )
+    @settings(deadline=10000)
     def test_batch_matmul(self, M, K, N, rand_seed, trans_a, trans_b, run_ints):
         np.random.seed(rand_seed)
         workspace.ResetWorkspace()

caffe2/contrib/fakelowp/test/test_batchnorm_nnpi_fp16.py

@@ -48,7 +48,7 @@ class BatchnormTest(unittest.TestCase):
            size=st.integers(2, 30),
            input_channels=st.integers(2, 40),
            batch_size=st.integers(2, 20))
-    @settings(max_examples=100)
+    @settings(max_examples=100, deadline=1000)
     def test_bn(self, seed, size, input_channels, batch_size):
         workspace.ResetWorkspace()
         np.random.seed(seed)

caffe2/contrib/fakelowp/test/test_fc_nnpi_fp16.py

@@ -24,6 +24,7 @@
 
 class FCTest(serial.SerializedTestCase):
     @given(seed=st.integers(0, 65534))
+    @settings(deadline=1000)
     def test_clip(self, seed):
         np.random.seed(seed)
         m, n, k = 8, 8, 8
@@ -77,6 +78,7 @@ def test_clip(self, seed):
         np.testing.assert_allclose(Y_glow, np.full((m, n), 65504.0, dtype))
 
     @given(seed=st.integers(0, 65534))
+    @settings(deadline=1000)
     def test_fc_exercise(self, seed):
         """ Test that the matmul engine is working, this doesn't test
             precision
@@ -137,6 +139,7 @@ def test_fc_exercise(self, seed):
                 assert(0)
 
     @given(seed=st.integers(0, 65534))
+    @settings(deadline=1000)
     def test_fc_numeric_cases(self, seed):
         """ Test numerics, use examples found from the unit test.
             Use Fp16FCAcc16NNPI as a reference.
@@ -241,12 +244,12 @@ def test_fc_numeric_cases(self, seed):
             if n_offenders > 0:
                 print_test_debug_info("fc",
                     {"seed": seed, "iter": i, "m": m, "k": k, "n": n, "X": X0,
-                     "W0": W0, "b0": b0, "Y_glow": Y_glow, "Y_c2": Y_c2, 
+                     "W0": W0, "b0": b0, "Y_glow": Y_glow, "Y_c2": Y_c2,
                      "diff": diff, "rowdiff": rowdiff})
                 assert(0)
 
+    @settings(max_examples=5, deadline=None)
     @given(seed=st.integers(0, 65535))
-    @settings(max_examples=5)
     def test_fc_num0(self, seed):
         """ Test numerics, fix a dimension and determine the ranges of error.
             Use Fp16FCAcc16 as a reference.
@@ -330,7 +333,7 @@ def test_fc_num0(self, seed):
             if n_offenders > 0:
                 print_test_debug_info("fc",
                     {"seed": seed, "iter": _, "m": m, "k": k, "n": n, "X": X0,
-                     "W0": W0, "b0": b0, "Y_glow": Y_glow, "Y_c2": Y_c2, 
+                     "W0": W0, "b0": b0, "Y_glow": Y_glow, "Y_c2": Y_c2,
                      "diff": diff, "rowdiff": rowdiff})
                 assert(0)
 

caffe2/contrib/fakelowp/test/test_int8_ops_nnpi.py

@@ -20,12 +20,12 @@ def _get_scale_zp(self, tensor):
         zero_point = int(round(np.clip(zero_point, 0, 255.0)))
         return (scale, zero_point)
 
+    @settings(max_examples=30, deadline=None)
     @given(
         n=st.integers(2, 1024),
         rand_seed=st.integers(0, 65534),
         non_zero_offset=st.booleans()
     )
-    @settings(max_examples=100)
     def test_int8_quantize(self, n, rand_seed, non_zero_offset):
         print("n={}, rand_seed={}".format(n, rand_seed))
         np.random.seed(rand_seed)
@@ -125,7 +125,7 @@ def test_int8_quantize(self, n, rand_seed, non_zero_offset):
         rand_seed=st.integers(0, 65534),
         quantize_bias=st.sampled_from([False]),
     )
-    @settings(max_examples=100)
+    @settings(deadline=None, max_examples=30)
     def test_int8_fc(
         self, n, m, k, rand_seed, quantize_bias
     ):
@@ -227,7 +227,7 @@ def test_int8_fc(
         n=st.integers(1, 4),
         rand_seed=st.integers(0, 65534)
     )
-    @settings(max_examples=100)
+    @settings(max_examples=100, deadline=1000)
     def test_int8_small_input(self, n, rand_seed):
         print("n={}, rand_seed={}".format(n, rand_seed))
         np.random.seed(rand_seed)

caffe2/contrib/fakelowp/test/test_layernorm_nnpi_fp16.py

@@ -30,7 +30,7 @@ class LayerNorm(serial.SerializedTestCase):
            size=st.integers(min_value=2, max_value=128),
            epsilon=st.floats(min_value=1e-4, max_value=1e-3),
            elementwise_affine=st.booleans())
-    @settings(max_examples=100)
+    @settings(max_examples=100, deadline=None)
     def test_layernorm(self, seed, batch_size, size, epsilon, elementwise_affine):
         np.random.seed(seed)
         # Reset the workspace

caffe2/contrib/fakelowp/test/test_op_nnpi_fp16.py

@@ -23,6 +23,7 @@
 
 class ArithmeticOpsTest(serial.SerializedTestCase):
     @given(seed=st.integers(0, 65534))
+    @settings(deadline=1000)
     def _test_binary_op_graph(self, name, seed):
         np.random.seed(seed)
         workspace.ResetWorkspace()
@@ -92,8 +93,11 @@ def _test_binary_op_graph(self, name, seed):
             Y_glow[Y_glow == np.Inf] = np.finfo(np.float16).max
             Y_glow[Y_glow == np.NINF] = np.finfo(np.float16).min
 
+            # Ignore mismatches solely due to difference in precision
+            fp16_finite = np.isfinite(A.astype(np.float16) / B.astype(np.float16))
+
             # Results should be identical since we are comparing with the C2 emulation
-            if not np.allclose(Y_c2, Y_glow):
+            if not np.allclose(Y_c2[fp16_finite], Y_glow[fp16_finite]):
                 diff = np.abs((Y_glow - Y_c2) / (Y_c2 + kEpsilon))
                 print_test_debug_info(name, {
                     "dims": dims, "iter": _, "seed": seed, "A": A, "B": B,
@@ -114,6 +118,7 @@ def test_div_graph(self):
 
 
 class UnaryOpTest(serial.SerializedTestCase):
+    @settings(deadline=1000)
     def _test_unary_op(self, opname, X, rtol=1e-5, atol=1e-8):
         workspace.ResetWorkspace()
 
@@ -287,6 +292,7 @@ def test_logit(self):
 
 class ReluTest(serial.SerializedTestCase):
     @given(seed=st.integers(0, 65534))
+    @settings(deadline=10000)
     def relu_test(self, inputs, gc, dc, seed):
         np.random.seed(seed)
         inputs = np.random.rand(1).astype(np.float32)

caffe2/contrib/fakelowp/test/test_sls_4bit_nnpi_fp16.py

@@ -24,6 +24,7 @@
 
 class SparseLengthsSum4BitFakeNNPIFp16Test(serial.SerializedTestCase):
     @given(seed=st.integers(0, 65535))
+    @settings(deadline=10000)
     def test_slws_fused_4bit_rowwise_all_same(self, seed):
         np.random.seed(seed)
         workspace.ResetWorkspace()
@@ -117,7 +118,7 @@ def test_slws_fused_4bit_rowwise_all_same(self, seed):
         batch_size=st.integers(1, 32),
         max_weight=st.integers(0, 1),
     )
-    @settings(max_examples=100)
+    @settings(deadline=10000)
     def test_slws_fused_4bit_rowwise(self, seed, num_rows, embedding_dim, batch_size, max_weight):
         workspace.ResetWorkspace()
         np.random.seed(seed)

caffe2/contrib/fakelowp/test/test_sls_8bit_nnpi_fp16.py

@@ -5,7 +5,7 @@
 # Must happen before importing caffe2.python.*
 import caffe2.python.fakelowp.init_shared_libs  # noqa
 import numpy as np
-from hypothesis import given
+from hypothesis import given, settings
 from hypothesis import strategies as st
 from caffe2.proto import caffe2_pb2
 from caffe2.python import core, workspace
@@ -99,6 +99,7 @@ def Skip_test_SLS_NonQuantized_fp16(self):
             assert 0
 
     @given(seed=st.integers(0, 65535))
+    @settings(deadline=10000)
     def test_slws_fused_8bit_rowwise_all_same(self, seed):
         # Comment out for predictable debugging
         np.random.seed(seed)
@@ -206,6 +207,7 @@ def test_slws_fused_8bit_rowwise_all_same(self, seed):
         batch_size=st.integers(1, 5),
         max_weight=st.integers(0, 100),
     )
+    @settings(deadline=10000)
     def test_slws_fused_8bit_rowwise(self, seed, num_rows, embedding_dim, batch_size, max_weight):
         np.random.seed(seed)
         workspace.ResetWorkspace()
@@ -302,6 +304,7 @@ def test_slws_fused_8bit_rowwise(self, seed, num_rows, embedding_dim, batch_size
     # Simple test to aid debugging order of operations
     # Minimize the case to an SLS that adds two rows
     @given(seed=st.integers(0, 65535))
+    @settings(deadline=10000)
     def test_small_sls(self, seed):
         np.random.seed(seed)
         workspace.ResetWorkspace()

caffe2/contrib/gloo/gloo_test.py

@@ -5,7 +5,7 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from hypothesis import given
+from hypothesis import given, settings
 import hypothesis.strategies as st
 from multiprocessing import Process, Queue
 
@@ -213,10 +213,11 @@ def _test_broadcast(self,
                 workspace.RunNet(net.Name())
 
     @given(comm_size=st.integers(min_value=2, max_value=8),
-           blob_size=st.integers(min_value=1e3, max_value=1e6),
+           blob_size=st.integers(min_value=int(1e3), max_value=int(1e6)),
            num_blobs=st.integers(min_value=1, max_value=4),
            device_option=st.sampled_from([hu.cpu_do]),
            use_float16=st.booleans())
+    @settings(deadline=10000)
     def test_broadcast(self, comm_size, blob_size, num_blobs, device_option,
                        use_float16):
         TestCase.test_counter += 1
@@ -328,10 +329,11 @@ def _test_allreduce_multicw(self,
                     (num_blobs * comm_size) * (num_blobs * comm_size - 1) / 2)
 
     @given(comm_size=st.integers(min_value=2, max_value=8),
-           blob_size=st.integers(min_value=1e3, max_value=1e6),
+           blob_size=st.integers(min_value=int(1e3), max_value=int(1e6)),
            num_blobs=st.integers(min_value=1, max_value=4),
            device_option=st.sampled_from([hu.cpu_do]),
            use_float16=st.booleans())
+    @settings(deadline=10000)
     def test_allreduce(self, comm_size, blob_size, num_blobs, device_option,
                        use_float16):
         TestCase.test_counter += 1
@@ -416,10 +418,11 @@ def _test_reduce_scatter(self,
             workspace.RunNet(net.Name())
 
     @given(comm_size=st.integers(min_value=2, max_value=8),
-           blob_size=st.integers(min_value=1e3, max_value=1e6),
+           blob_size=st.integers(min_value=int(1e3), max_value=int(1e6)),
            num_blobs=st.integers(min_value=1, max_value=4),
            device_option=st.sampled_from([hu.cpu_do]),
            use_float16=st.booleans())
+    @settings(deadline=10000)
     def test_reduce_scatter(self, comm_size, blob_size, num_blobs,
                             device_option, use_float16):
         TestCase.test_counter += 1
@@ -496,10 +499,11 @@ def _test_allgather(self,
             workspace.RunNet(net.Name())
 
     @given(comm_size=st.integers(min_value=2, max_value=8),
-           blob_size=st.integers(min_value=1e3, max_value=1e6),
+           blob_size=st.integers(min_value=int(1e3), max_value=int(1e6)),
            num_blobs=st.integers(min_value=1, max_value=4),
            device_option=st.sampled_from([hu.cpu_do]),
            use_float16=st.booleans())
+    @settings(max_examples=10, deadline=None)
     def test_allgather(self, comm_size, blob_size, num_blobs, device_option,
                        use_float16):
         TestCase.test_counter += 1
@@ -522,6 +526,7 @@ def test_allgather(self, comm_size, blob_size, num_blobs, device_option,
                     use_float16=use_float16)
 
     @given(device_option=st.sampled_from([hu.cpu_do]))
+    @settings(deadline=10000)
     def test_forked_cw(self, device_option):
         TestCase.test_counter += 1
         if os.getenv('COMM_RANK') is not None:
@@ -568,6 +573,7 @@ def _test_barrier(
 
     @given(comm_size=st.integers(min_value=2, max_value=8),
            device_option=st.sampled_from([hu.cpu_do]))
+    @settings(deadline=10000)
     def test_barrier(self, comm_size, device_option):
         TestCase.test_counter += 1
         if os.getenv('COMM_RANK') is not None:
@@ -618,6 +624,7 @@ def _test_close_connection(
 
     @given(comm_size=st.integers(min_value=2, max_value=8),
            device_option=st.sampled_from([hu.cpu_do]))
+    @settings(deadline=10000)
     def test_close_connection(self, comm_size, device_option):
         import time
         start_time = time.time()
@@ -679,6 +686,7 @@ def _test_io_error(
 
     @given(comm_size=st.integers(min_value=2, max_value=8),
            device_option=st.sampled_from([hu.cpu_do]))
+    @settings(deadline=10000)
     def test_io_error(self, comm_size, device_option):
         TestCase.test_counter += 1
         with self.assertRaises(IoError):

caffe2/contrib/nnpack/nnpack_ops_test.py

@@ -48,12 +48,13 @@ class NNPackOpsTest(hu.HypothesisTestCase):
            kernel=st.integers(3, 5),
            size=st.integers(5, 10),
            input_channels=st.integers(1, 8),
-           output_channels=st.integers(1, 8),
            batch_size=st.integers(1, 5),
            groups=st.integers(1, 2))
     def test_convolution_correctness(self, stride, pad, kernel, size,
-                                     input_channels, output_channels,
+                                     input_channels,
                                      batch_size, groups):
+        input_channels *= groups
+        output_channels = int(input_channels / groups)
         assume(input_channels % groups == 0)
         assume(output_channels % groups == 0)
         assume(output_channels == input_channels / groups)
@@ -64,7 +65,7 @@ def test_convolution_correctness(self, stride, pad, kernel, size,
         X = np.random.rand(
             batch_size, input_channels, size, size).astype(np.float32) - 0.5
         w = np.random.rand(
-            output_channels, input_channels, kernel, kernel).astype(np.float32)\
+            input_channels, output_channels, kernel, kernel).astype(np.float32)\
             - 0.5
         b = np.random.rand(output_channels).astype(np.float32) - 0.5
         order = "NCHW"
@@ -175,7 +176,7 @@ def test_leaky_relu_correctness(self, size, input_channels, batch_size,
             atol=1e-4,
             rtol=1e-4)
 
-    @settings(timeout=3600)
+    @settings(deadline=3600)
     @unittest.skipIf(not os.environ.get("CAFFE2_BENCHMARK"), "Benchmark")
     @given(stride=st.integers(1, 1),
            pad=st.integers(0, 2),
@@ -213,7 +214,7 @@ def test_timings(self, stride, pad, kernel, size,
         print("Speedup for NNPACK: {:.2f}".format(
             times[""] / times["NNPACK"]))
 
-    @settings(timeout=3600)
+    @settings(deadline=3600)
     @unittest.skipIf(not os.environ.get("CAFFE2_BENCHMARK"), "Benchmark")
     @given(size=st.integers(30, 90),
            input_channels=st.sampled_from([3, 64, 256]),

caffe2/python/allcompare_test.py

@@ -5,7 +5,7 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from hypothesis import given
+from hypothesis import given, settings
 import hypothesis.strategies as st
 from multiprocessing import Process
 
@@ -60,6 +60,7 @@ class TestAllCompare(hu.HypothesisTestCase):
     @given(
         d=st.integers(1, 5), n=st.integers(2, 11), num_procs=st.integers(1, 8)
     )
+    @settings(deadline=10000)
     def test_allcompare(self, d, n, num_procs):
         dims = []
         for _ in range(d):

caffe2/python/core_gradients_test.py

@@ -4,7 +4,7 @@
 from __future__ import unicode_literals
 
 from future.utils import bytes_to_native_str
-from hypothesis import given
+from hypothesis import given, settings
 import hypothesis.strategies as st
 import unittest
 
@@ -98,6 +98,7 @@ def assertOperatorListEqual(self, operatorDefList1, operatorDefList2):
     @given(device_option=st.sampled_from([
         None,
         core.DeviceOption(workspace.GpuDeviceType, 1)]))
+    @settings(deadline=10000)
     def testDirect(self, device_option):
         operators = [
             CreateOperator('Direct', 'in', 'hidden'),
@@ -283,6 +284,7 @@ def testUseInputButInputHasBeenChanged(self):
     @given(device_option=st.sampled_from([
         None,
         core.DeviceOption(workspace.GpuDeviceType, 1)]))
+    @settings(deadline=10000)
     def testMultiUseInput(self, device_option):
         """Test gradient for the following case:
 

caffe2/python/crf_viterbi_test.py

@@ -8,13 +8,14 @@
 from caffe2.python.crf_predict import crf_update_predictions
 from caffe2.python.test_util import TestCase
 import hypothesis.strategies as st
-from hypothesis import given
+from hypothesis import given, settings
 import numpy as np
 
 
 class TestCrfDecode(TestCase):
 
     @given(num_tags=st.integers(2, 4), num_words=st.integers(2, 15))
+    @settings(deadline=2000)
     def test_crf_viterbi(self, num_tags, num_words):
         model = CNNModelHelper(name='external')
         predictions = np.random.randn(num_words, num_tags).astype(np.float32)