MDL-74762 qbank_statistics: improve performance loading the data

This issue greatly improves the performance of displaying statistics in
the question bank.

1. The required quiz statistics are now pre-computed by a scheduled task.

2. Cached statistics in the database are now never cleaned up, so the
   pre-computed stats are always available.

3. The way the cached statistics are loaded for each question
   that is being displayed is now a bit more efficient.

4. Related to that, there is a new callback which activities can implement,
   if they want their question statistics to be included in the ones shown
   in the question bank.

Note, there is still further improvement possible to load the statistics
for all questions being displayed in bulk. However, that must wait for a
future issue, MDL-75576. The other improvements in this issue are
significant and we did not want to delay releasing them.

Co-authored-by: Jonathan Champ <[email protected]>
Co-authored-by: Tim Hunt <[email protected]>

mod/quiz/lib.php

@@ -29,6 +29,7 @@
 defined('MOODLE_INTERNAL') || die();
 
 use mod_quiz\question\bank\custom_view;
+use core_question\statistics\questions\all_calculated_for_qubaid_condition;
 
 require_once($CFG->dirroot . '/calendar/lib.php');
 
@@ -2485,3 +2486,19 @@ function quiz_delete_references($quizid): void {
         $DB->delete_records('question_references', $params);
     }
 }
+
+/**
+ * Implement the calculate_question_stats callback.
+ *
+ * This enables quiz statistics to be shown in statistics columns in the database.
+ *
+ * @param context $context return the statistics related to this context (which will be a quiz context).
+ * @return all_calculated_for_qubaid_condition|null The statistics for this quiz, if any, else null.
+ */
+function mod_quiz_calculate_question_stats(context $context): ?all_calculated_for_qubaid_condition {
+    global $CFG;
+    require_once($CFG->dirroot . '/mod/quiz/report/statistics/report.php');
+    $cm = get_coursemodule_from_id('quiz', $context->instanceid);
+    $report = new quiz_statistics_report();
+    return $report->calculate_questions_stats_for_question_bank($cm->instance);
+}

mod/quiz/report/reportlib.php

@@ -26,6 +26,7 @@
 defined('MOODLE_INTERNAL') || die();
 
 require_once($CFG->dirroot . '/mod/quiz/lib.php');
+require_once($CFG->dirroot . '/mod/quiz/attemptlib.php');
 require_once($CFG->libdir . '/filelib.php');
 require_once($CFG->dirroot . '/mod/quiz/accessmanager.php');
 

mod/quiz/report/statistics/classes/task/recalculate.php

@@ -0,0 +1,88 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace quiz_statistics\task;
+
+use quiz_attempt;
+use quiz;
+use quiz_statistics_report;
+
+defined('MOODLE_INTERNAL') || die();
+
+require_once($CFG->dirroot . '/mod/quiz/locallib.php');
+require_once($CFG->dirroot . '/mod/quiz/report/statistics/statisticslib.php');
+require_once($CFG->dirroot . '/mod/quiz/report/reportlib.php');
+require_once($CFG->dirroot . '/mod/quiz/report/statistics/report.php');
+
+/**
+ * Re-calculate question statistics.
+ *
+ * @package    quiz_statistics
+ * @copyright  2022 Catalyst IT Australia Pty Ltd
+ * @author     Nathan Nguyen <[email protected]>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class recalculate extends \core\task\scheduled_task {
+
+    public function get_name() {
+        return get_string('recalculatetask', 'quiz_statistics');
+    }
+
+    public function execute() {
+        global $DB;
+        // TODO: MDL-75197, add quizid in quiz_statistics so that it is simpler to find quizzes for stats calculation.
+        // Only calculate stats for quizzes which have recently finished attempt.
+        $sql = "
+            SELECT qa.quiz, MAX(qa.timefinish) as timefinish
+              FROM {quiz_attempts} qa
+             WHERE qa.preview = 0
+               AND qa.state = :quizstatefinished
+          GROUP BY qa.quiz
+        ";
+
+        $params = [
+            "quizstatefinished" => quiz_attempt::FINISHED,
+        ];
+
+        $latestattempts = $DB->get_records_sql($sql, $params);
+
+        foreach ($latestattempts as $attempt) {
+            $quizobj = quiz::create($attempt->quiz);
+            $quiz = $quizobj->get_quiz();
+            // Hash code for question stats option in question bank.
+            $qubaids = quiz_statistics_qubaids_condition($quiz->id, new \core\dml\sql_join(), $quiz->grademethod);
+
+            // Check if there is any existing question stats, and it has been calculated after latest quiz attempt.
+            $records = $DB->get_records_select(
+                'quiz_statistics',
+                'hashcode = :hashcode AND timemodified > :timefinish',
+                [
+                    'hashcode' => $qubaids->get_hash_code(),
+                    'timefinish' => $attempt->timefinish
+                ]
+            );
+
+            if (empty($records)) {
+                $report = new quiz_statistics_report();
+                // Clear old cache.
+                $report->clear_cached_data($qubaids);
+                // Calculate new stats.
+                $report->calculate_questions_stats_for_question_bank($quiz->id);
+            }
+        }
+        return true;
+    }
+}

mod/quiz/report/statistics/db/tasks.php

@@ -28,10 +28,10 @@
 
 $tasks = [
     [
-        'classname' => 'quiz_statistics\task\quiz_statistics_cleanup',
+        'classname' => 'quiz_statistics\task\recalculate',
         'blocking' => 0,
         'minute' => 'R',
-        'hour' => '*/5',
+        'hour' => '*/4',
         'day' => '*',
         'dayofweek' => '*',
         'month' => '*'

mod/quiz/report/statistics/lang/en/quiz_statistics.php

@@ -93,7 +93,6 @@
 $string['questionname'] = 'Question name';
 $string['questionnumber'] = 'Q#';
 $string['questionstatistics'] = 'Question statistics';
-$string['quizstatisticscleanuptask'] = 'Clean up old quiz statistics cache records';
 $string['questionstatsfilename'] = 'questionstats';
 $string['questiontype'] = 'Question type';
 $string['quizinformation'] = 'Quiz information';
@@ -104,6 +103,7 @@
 $string['rangeofvalues'] = 'Range of statistics for these questions';
 $string['rangebetween'] = '{$a->min} − {$a->max}';
 $string['recalculatenow'] = 'Recalculate now';
+$string['recalculatetask'] = 'Re-calculate question statistics';
 $string['reportsettings'] = 'Statistics calculation settings';
 $string['response'] = 'Response';
 $string['slotstructureanalysis'] = 'Structural analysis for question number {$a}';

mod/quiz/report/statistics/report.php

@@ -25,10 +25,15 @@
 
 defined('MOODLE_INTERNAL') || die();
 
+use core_question\statistics\questions\all_calculated_for_qubaid_condition;
+
+require_once($CFG->dirroot . '/mod/quiz/report/default.php');
+require_once($CFG->dirroot . '/mod/quiz/report/reportlib.php');
 require_once($CFG->dirroot . '/mod/quiz/report/statistics/statistics_form.php');
 require_once($CFG->dirroot . '/mod/quiz/report/statistics/statistics_table.php');
 require_once($CFG->dirroot . '/mod/quiz/report/statistics/statistics_question_table.php');
 require_once($CFG->dirroot . '/mod/quiz/report/statistics/statisticslib.php');
+
 /**
  * The quiz statistics report provides summary information about each question in
  * a quiz, compared to the whole quiz. It also provides a drill-down to more
@@ -814,7 +819,7 @@ protected function output_caching_info($lastcachetime, $quizid, $groupstudentsjo
      *
      * @param $qubaids qubaid_condition
      */
-    protected function clear_cached_data($qubaids) {
+    public function clear_cached_data($qubaids) {
         global $DB;
         $DB->delete_records('quiz_statistics', array('hashcode' => $qubaids->get_hash_code()));
         $DB->delete_records('question_statistics', array('hashcode' => $qubaids->get_hash_code()));
@@ -921,4 +926,21 @@ protected function output_all_question_response_analysis($qubaids,
             }
         }
     }
+
+    /**
+     * Load question stats for a quiz
+     *
+     * @param int $quizid question usage
+     * @return all_calculated_for_qubaid_condition question stats
+     */
+    public function calculate_questions_stats_for_question_bank(int $quizid): all_calculated_for_qubaid_condition {
+        global $DB;
+        $quiz = $DB->get_record('quiz', ['id' => $quizid], '*', MUST_EXIST);
+        $questions = $this->load_and_initialise_questions_for_calculations($quiz);
+
+        [, $questionstats] = $this->get_all_stats_and_analysis($quiz,
+            $quiz->grademethod, question_attempt::ALL_TRIES, new \core\dml\sql_join(), $questions);
+
+        return $questionstats;
+    }
 }

mod/quiz/report/statistics/version.php

@@ -24,6 +24,6 @@
 
 defined('MOODLE_INTERNAL') || die();
 
-$plugin->version   = 2022041900;
+$plugin->version   = 2022041901;
 $plugin->requires  = 2022041200;
 $plugin->component = 'quiz_statistics';

mod/upgrade.txt

@@ -13,6 +13,9 @@ information provided here is intended especially for developers.
   in returning inaccurate data in this course format, therefore it is advisable to use $settingsnavigation->get_page().
 * A new style of icons has been created for activities. When creating an icon in the new style it should be named
   'monologo' and can site alongside the legacy icon if desired. Only the new logo types will be used.
+* There is a new callback ..._calculate_question_stats which needs to be implemented by components which want
+  to contribute statistics to the display in the question bank. There is an example implementation in mod_quiz.
+  (Added in 4.0.4 / 4.1.)
 
 === 3.9 ===