From b4d8158316aeaed0f871782223039b92b25814f2 Mon Sep 17 00:00:00 2001 From: Chinmay Garde Date: Tue, 17 Sep 2019 14:50:51 -0700 Subject: [PATCH] Shuffle test order and repeat test runs once. (#12275) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tests we write must be resilient to the order in which they are run in the harness. That is, they must not rely on global state set by other tests that have already run in the process. Also, these tests must themselves be repeatable. That is, they must correctly clean up after themselves and be able to run successfully again in the same process. This patch adds some safeguards against (but does NOT guarantee) the addition of tests that violate the dictum. Additionally, test failures must be easily reproducible for folks investigating the test failure. Also, tests that assert correctness of unrelated code must not stop progress on the authors patch. This changes does not hinder reproducibility of test failures because the random seed is printed in the logs before running each test. Developers attempting to reproduce the failure locally can do the same via the following invocation `--gtest_shuffle --gtest_repeat= --gtest_random_seed=`. This change does introduce potential burden on patch authors that may see failures in unrelated code as a newly failing shuffle seed is used on their runs. To ameliorate this, we will formulate guidance for them to aggressively mark such tests as disabled and file bugs to enable the same. The test seed is intentionally kept low because it’s purpose is to test that individual tests are repeatable. It must not be used as a replacement for fuzzing. --- .../embedder/tests/embedder_unittests.cc | 1 + testing/run_tests.py | 25 +++++++++++-------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/shell/platform/embedder/tests/embedder_unittests.cc b/shell/platform/embedder/tests/embedder_unittests.cc index ebd53ce6a8918..7e299fb6aa0a0 100644 --- a/shell/platform/embedder/tests/embedder_unittests.cc +++ b/shell/platform/embedder/tests/embedder_unittests.cc @@ -218,6 +218,7 @@ TEST_F(EmbedderTest, CanSpecifyCustomTaskRunner) { kill_latch.Wait(); ASSERT_TRUE(signaled_once); + signaled_once = false; } TEST(EmbedderTestNoFixture, CanGetCurrentTimeInNanoseconds) { diff --git a/testing/run_tests.py b/testing/run_tests.py index a06898e8deac2..421e265952f76 100755 --- a/testing/run_tests.py +++ b/testing/run_tests.py @@ -71,13 +71,18 @@ def RunEngineExecutable(build_dir, executable_name, filter, flags=[], cwd=buildr def RunCCTests(build_dir, filter): print "Running Engine Unit-tests." - RunEngineExecutable(build_dir, 'client_wrapper_glfw_unittests', filter) + shuffle_flags = [ + "--gtest_shuffle", + "--gtest_repeat=2", + ] + + RunEngineExecutable(build_dir, 'client_wrapper_glfw_unittests', filter, shuffle_flags) - RunEngineExecutable(build_dir, 'client_wrapper_unittests', filter) + RunEngineExecutable(build_dir, 'client_wrapper_unittests', filter, shuffle_flags) # https://github.com/flutter/flutter/issues/36294 if not IsWindows(): - RunEngineExecutable(build_dir, 'embedder_unittests', filter) + RunEngineExecutable(build_dir, 'embedder_unittests', filter, shuffle_flags) flow_flags = ['--gtest_filter=-PerformanceOverlayLayer.Gold'] if IsLinux(): @@ -85,25 +90,25 @@ def RunCCTests(build_dir, filter): '--golden-dir=%s' % golden_dir, '--font-file=%s' % roboto_font_path, ] - RunEngineExecutable(build_dir, 'flow_unittests', filter, flow_flags) + RunEngineExecutable(build_dir, 'flow_unittests', filter, flow_flags + shuffle_flags) - RunEngineExecutable(build_dir, 'fml_unittests', filter, [ time_sensitve_test_flag ]) + RunEngineExecutable(build_dir, 'fml_unittests', filter, [ time_sensitve_test_flag ] + shuffle_flags) - RunEngineExecutable(build_dir, 'runtime_unittests', filter) + RunEngineExecutable(build_dir, 'runtime_unittests', filter, shuffle_flags) # https://github.com/flutter/flutter/issues/36295 if not IsWindows(): - RunEngineExecutable(build_dir, 'shell_unittests', filter) + RunEngineExecutable(build_dir, 'shell_unittests', filter, shuffle_flags) - RunEngineExecutable(build_dir, 'ui_unittests', filter) + RunEngineExecutable(build_dir, 'ui_unittests', filter, shuffle_flags) # These unit-tests are Objective-C and can only run on Darwin. if IsMac(): - RunEngineExecutable(build_dir, 'flutter_channels_unittests', filter) + RunEngineExecutable(build_dir, 'flutter_channels_unittests', filter, shuffle_flags) # https://github.com/flutter/flutter/issues/36296 if IsLinux(): - RunEngineExecutable(build_dir, 'txt_unittests', filter, [ fonts_dir_flag ]) + RunEngineExecutable(build_dir, 'txt_unittests', filter, [ fonts_dir_flag ] + shuffle_flags) def RunEngineBenchmarks(build_dir, filter):