MDL-73938 reportbuilder: helper methods for automating report tests.

Implement stress tester methods for iterating over report columns,
aggregation and conditions. Assert that each works correctly in
isolation, and when used in conjunction with other columns.

admin/tests/reportbuilder/datasource/task_logs_test.php

@@ -221,6 +221,19 @@ public function test_datasource_filters(
         }
     }
 
+    /**
+     * Stress test datasource
+     */
+    public function test_stress_datasource(): void {
+        $this->resetAfterTest();
+
+        $this->generate_task_log_data(true, 3, 2, 1654038000, 1654038060, 'hi', 'core_reportbuilder', 'test', 43);
+
+        $this->datasource_stress_test_columns(task_logs::class);
+        $this->datasource_stress_test_columns_aggregation(task_logs::class);
+        $this->datasource_stress_test_conditions(task_logs::class, 'task_log:name');
+    }
+
     /**
      * Helper to generate some task logs data
      *

badges/tests/reportbuilder/datasource/badges_test.php

@@ -94,4 +94,23 @@ public function test_datasource(): void {
             return array_values($row);
         }, $content));
     }
+
+    /**
+     * Stress test datasource
+     */
+    public function test_stress_datasource(): void {
+        $this->resetAfterTest();
+
+        $course = $this->getDataGenerator()->create_course();
+        $user = $this->getDataGenerator()->create_and_enrol($course);
+
+        /** @var core_badges_generator $generator */
+        $generator = $this->getDataGenerator()->get_plugin_generator('core_badges');
+        $badge = $generator->create_badge(['name' => 'Course badge', 'type' => BADGE_TYPE_COURSE, 'courseid' => $course->id]);
+        $badge->issue($user->id, true);
+
+        $this->datasource_stress_test_columns(badges::class);
+        $this->datasource_stress_test_columns_aggregation(badges::class);
+        $this->datasource_stress_test_conditions(badges::class, 'badge:name');
+    }
 }

cohort/tests/datasource_test.php → cohort/tests/reportbuilder/datasource/cohorts_test.php

@@ -37,7 +37,7 @@
  * @copyright   2021 Paul Holden <[email protected]>
  * @license     http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  */
-class datasource_test extends core_reportbuilder_testcase {
+class cohorts_test extends core_reportbuilder_testcase {
 
     /**
      * Test cohorts datasource
@@ -132,4 +132,19 @@ public function test_cohorts_datasource_user_select(string $username, string $ex
         $contentrow = array_values(reset($content));
         $this->assertEquals([$expectedcohort, $username], $contentrow);
     }
+
+    /**
+     * Stress test datasource
+     */
+    public function test_stress_datasource(): void {
+        $this->resetAfterTest();
+
+        $cohort = $this->getDataGenerator()->create_cohort();
+        $user = $this->getDataGenerator()->create_user();
+        cohort_add_member($cohort->id, $user->id);
+
+        $this->datasource_stress_test_columns(cohorts::class);
+        $this->datasource_stress_test_columns_aggregation(cohorts::class);
+        $this->datasource_stress_test_conditions(cohorts::class, 'cohort:name');
+    }
 }

course/tests/reportbuilder/datasource/courses_test.php

@@ -18,6 +18,7 @@
 
 namespace core_course\reportbuilder\datasource;
 
+use core_customfield_generator;
 use core_reportbuilder_testcase;
 use core_reportbuilder_generator;
 use core_reportbuilder\local\filters\tags;
@@ -192,4 +193,23 @@ public function test_datasource_filters(
             $this->assertEmpty($content);
         }
     }
+
+    /**
+     * Stress test datasource
+     */
+    public function test_stress_datasource(): void {
+        $this->resetAfterTest();
+
+        /** @var core_customfield_generator $generator */
+        $generator = $this->getDataGenerator()->get_plugin_generator('core_customfield');
+        $customfieldcategory = $generator->create_category();
+        $generator->create_field(['categoryid' => $customfieldcategory->get('id'), 'shortname' => 'hi']);
+
+        $category = $this->getDataGenerator()->create_category();
+        $course = $this->getDataGenerator()->create_course(['category' => $category->id, 'customfield_hi' => 'Hello']);
+
+        $this->datasource_stress_test_columns(courses::class);
+        $this->datasource_stress_test_columns_aggregation(courses::class);
+        $this->datasource_stress_test_conditions(courses::class, 'course:idnumber');
+    }
 }

course/tests/reportbuilder/datasource/participants_test.php

@@ -277,4 +277,18 @@ public function test_report_filters(string $filter, array $filtervalues, string
         $this->assertCount(1, $content);
         $this->assertEquals($expected, $content[0]['c0_firstname']);
     }
+
+    /**
+     * Stress test datasource
+     */
+    public function test_stress_datasource(): void {
+        $this->resetAfterTest();
+
+        $course = $this->getDataGenerator()->create_course();
+        $this->getDataGenerator()->create_and_enrol($course);
+
+        $this->datasource_stress_test_columns(participants::class);
+        $this->datasource_stress_test_columns_aggregation(participants::class);
+        $this->datasource_stress_test_conditions(participants::class, 'course:idnumber');
+    }
 }

reportbuilder/tests/helpers.php

@@ -16,6 +16,9 @@
 
 declare(strict_types=1);
 
+use core_reportbuilder\manager;
+use core_reportbuilder\local\helpers\aggregation;
+use core_reportbuilder\local\helpers\report;
 use core_reportbuilder\local\helpers\user_filter_manager;
 use core_reportbuilder\table\custom_report_table_view;
 
@@ -56,4 +59,120 @@ protected function get_custom_report_content(int $reportid, int $pagesize = 30,
 
         return $records;
     }
+
+    /**
+     * Stress test a report source by iterating over all it's columns and asserting we can create a report for each
+     *
+     * @param string $source
+     */
+    protected function datasource_stress_test_columns(string $source): void {
+
+        /** @var core_reportbuilder_generator $generator */
+        $generator = $this->getDataGenerator()->get_plugin_generator('core_reportbuilder');
+
+        $report = $generator->create_report(['name' => 'Stress columns', 'source' => $source, 'default' => 0]);
+        $instance = manager::get_report_from_persistent($report);
+
+        // Iterate over each available column, ensure each works correctly independent of any others.
+        $columnidentifiers = array_keys($instance->get_columns());
+        foreach ($columnidentifiers as $columnidentifier) {
+            $column = report::add_report_column($report->get('id'), $columnidentifier);
+
+            // We are only asserting the report returns content without errors, not the content itself.
+            try {
+                $content = $this->get_custom_report_content($report->get('id'));
+                $this->assertNotEmpty($content);
+            } catch (Throwable $exception) {
+                $this->fail("Error for column '{$columnidentifier}': " . $exception->getMessage());
+            }
+
+            report::delete_report_column($report->get('id'), $column->get('id'));
+        }
+    }
+
+    /**
+     * Stress test a report source by iterating over all columns and asserting we can create a report while aggregating each
+     *
+     * @param string $source
+     */
+    protected function datasource_stress_test_columns_aggregation(string $source): void {
+
+        /** @var core_reportbuilder_generator $generator */
+        $generator = $this->getDataGenerator()->get_plugin_generator('core_reportbuilder');
+
+        $report = $generator->create_report(['name' => 'Stress aggregation', 'source' => $source, 'default' => 0]);
+        $instance = manager::get_report_from_persistent($report);
+
+        // Add every column.
+        $columnidentifiers = array_keys($instance->get_columns());
+        foreach ($columnidentifiers as $columnidentifier) {
+            report::add_report_column($report->get('id'), $columnidentifier);
+        }
+
+        // Now iterate over each column, and apply all suitable aggregation types.
+        foreach ($instance->get_active_columns() as $column) {
+            $aggregations = aggregation::get_column_aggregations($column->get_type(), $column->get_disabled_aggregation());
+            foreach (array_keys($aggregations) as $aggregation) {
+                $column->get_persistent()->set('aggregation', $aggregation)->update();
+
+                // We are only asserting the report returns content without errors, not the content itself.
+                try {
+                    $content = $this->get_custom_report_content($report->get('id'));
+                    $this->assertNotEmpty($content);
+                } catch (Throwable $exception) {
+                    $this->fail("Error for column '{$column->get_unique_identifier()}' with aggregation '{$aggregation}': " .
+                        $exception->getMessage());
+                }
+            }
+
+            // Reset the column aggregation.
+            $column->get_persistent()->set('aggregation', null)->update();
+        }
+    }
+
+    /**
+     * Stress test a report source by iterating over all it's conditions and asserting we can create a report using each
+     *
+     * @param string $source
+     * @param string $columnidentifier Should be a simple column, with as few fields and joins as possible, ideally selected
+     *      from the base table itself
+     */
+    protected function datasource_stress_test_conditions(string $source, string $columnidentifier): void {
+
+        /** @var core_reportbuilder_generator $generator */
+        $generator = $this->getDataGenerator()->get_plugin_generator('core_reportbuilder');
+
+        $report = $generator->create_report(['name' => 'Stress conditions', 'source' => $source, 'default' => 0]);
+        $instance = manager::get_report_from_persistent($report);
+
+        // Add single column only (to ensure no conditions have reliance on any columns).
+        report::add_report_column($report->get('id'), $columnidentifier);
+
+        // Iterate over each available condition, ensure each works correctly independent of any others.
+        $conditionidentifiers = array_keys($instance->get_conditions());
+        foreach ($conditionidentifiers as $conditionidentifier) {
+            $condition = report::add_report_condition($report->get('id'), $conditionidentifier);
+            $conditioninstance = $instance->get_condition($condition->get('uniqueidentifier'));
+
+            /** @var \core_reportbuilder\local\filters\base $conditionclass */
+            $conditionclass = $conditioninstance->get_filter_class();
+
+            // Set report condition values in order to activate it.
+            $conditionvalues = $conditionclass::create($conditioninstance)->get_sample_values();
+            if (empty($conditionvalues)) {
+                debugging("Missing sample values from filter '{$conditionclass}'", DEBUG_DEVELOPER);
+            }
+            $instance->set_condition_values($conditionvalues);
+
+            // We are only asserting the report returns content without errors, not the content itself.
+            try {
+                $content = $this->get_custom_report_content($report->get('id'));
+                $this->assertIsArray($content);
+            } catch (Throwable $exception) {
+                $this->fail("Error for condition '{$conditionidentifier}': " . $exception->getMessage());
+            }
+
+            report::delete_report_condition($report->get('id'), $condition->get('id'));
+        }
+    }
 }

reportbuilder/upgrade.txt

@@ -40,4 +40,8 @@ Information provided here is intended especially for developers.
   - `tags` for reports containing entities with support for core_tag API
   - `autocomplete` for reports that contain pre-defined values for selection.
 * New method `get_sample_values()` added to base filter class, to be overridden in all filter types to support stress testing
+* New test helpers for automated stress testing of report sources:
+  - `datasource_stress_test_columns`
+  - `datasource_stress_test_columns_aggregation`
+  - `datasource_stress_test_conditions`
 * The test helper method `get_custom_report_content()` now accepts a list of filter values and applies them to the report

tag/tests/reportbuilder/datasource/tags_test.php

@@ -258,4 +258,17 @@ public function test_datasource_filters(
             $this->assertEmpty($content);
         }
     }
+
+    /**
+     * Stress test datasource
+     */
+    public function test_stress_datasource(): void {
+        $this->resetAfterTest();
+
+        $this->getDataGenerator()->create_course(['tags' => ['Horses']]);
+
+        $this->datasource_stress_test_columns(tags::class);
+        $this->datasource_stress_test_columns_aggregation(tags::class);
+        $this->datasource_stress_test_conditions(tags::class, 'tag:name');
+    }
 }

user/tests/reportbuilder/datasource/users_test.php

@@ -29,7 +29,7 @@
 require_once("{$CFG->dirroot}/reportbuilder/tests/helpers.php");
 
 /**
- * Unit tests for users datasources
+ * Unit tests for users datasource
  *
  * @package     core_user
  * @covers      \core_user\reportbuilder\datasource\users
@@ -155,4 +155,18 @@ public function test_datasource_filters(
             $this->assertEmpty($content);
         }
     }
+
+    /**
+     * Stress test datasource
+     */
+    public function test_stress_datasource(): void {
+        $this->resetAfterTest();
+
+        $this->getDataGenerator()->create_custom_profile_field(['datatype' => 'text', 'name' => 'Hi', 'shortname' => 'hi']);
+        $user = $this->getDataGenerator()->create_user(['profile_field_hi' => 'Hello']);
+
+        $this->datasource_stress_test_columns(users::class);
+        $this->datasource_stress_test_columns_aggregation(users::class);
+        $this->datasource_stress_test_conditions(users::class, 'user:username');
+    }
 }