Add warning about fuzzers with low number of trials in the report. (g…

…oogle#184)

* Add warning about fuzzers with low number of trials in the report.
* Move warning below plots. Consider `in_progress`.

analysis/benchmark_results.py

@@ -58,6 +58,12 @@ def _benchmark_df(self):
     def _benchmark_snapshot_df(self):
         return data_utils.get_benchmark_snapshot(self._benchmark_df)
 
+    @property
+    def fuzzers_with_not_enough_samples(self):
+        """Fuzzers with not enough samples."""
+        return data_utils.get_fuzzers_with_not_enough_samples(
+            self._benchmark_snapshot_df)
+
     @property
     def summary_table(self):
         """Statistical summary table."""

analysis/data_utils.py

@@ -75,6 +75,23 @@ def get_benchmark_snapshot(benchmark_df,
     return benchmark_snapshot_df
 
 
+_DEFAULT_FUZZER_SAMPLE_NUM_THRESHOLD = 0.8
+
+
+def get_fuzzers_with_not_enough_samples(
+        benchmark_snapshot_df, threshold=_DEFAULT_FUZZER_SAMPLE_NUM_THRESHOLD):
+    """Retruns fuzzers that didn't have enough trials running at snapshot time.
+    It takes a benchmark snapshot and finds the fuzzers that have a sample size
+    smaller than 80% of the largest sample size. Default threshold can be
+    overridden.
+    """
+    samples_per_fuzzer = benchmark_snapshot_df.fuzzer.value_counts()
+    max_samples = samples_per_fuzzer.max()
+    few_sample_criteria = samples_per_fuzzer < threshold * max_samples
+    few_sample_fuzzers = samples_per_fuzzer[few_sample_criteria].index
+    return few_sample_fuzzers.tolist()
+
+
 def get_experiment_snapshots(experiment_df):
     """Finds a good snapshot time for each benchmark in the experiment data.
 

analysis/report_templates/default.html

@@ -124,6 +124,24 @@ <h5 class="center-align">Mean coverage growth over time</h5>
                 </div>
             </div>
 
+            {% if benchmark.fuzzers_with_not_enough_samples and not in_progress %}
+            <div class="card-panel deep-orange lighten-3">
+                <div class="row valign-wrapper">
+                    <div class="col s4 m2">
+                        <i class="medium material-icons">error</i>
+                    </div>
+                    <div class="col s8 m10">
+                        <span class="black-text">
+                            The following fuzzers do not have enough samples:
+                            <strong>
+                                {{ ', '.join(benchmark.fuzzers_with_not_enough_samples) }}.
+                            </strong>
+                        </span>
+                    </div>
+                </div>
+            </div>
+            {% endif %}
+
             <ul class="collapsible">
                 <li>
                     <div class="collapsible-header">