Skip to content

Commit

Permalink
MDL-82626 hub: Registration collects AI usage data
Browse files Browse the repository at this point in the history
  • Loading branch information
@davewoloszyn
davewoloszyn committed Sep 13, 2024
1 parent ceb30b0 commit 2b1d330
Show file tree
Hide file tree
Showing 3 changed files with 182 additions and 1 deletion.
1 change: 1 addition & 0 deletions lang/en/hub.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
*/
$string['activeparticipantnumberaverage'] = 'Average number of recently active participants ({$a})';
$string['activeusersnumber'] = 'Number of recently active users ({$a})';
$string['aiusagestats'] = 'AI usage stats ({$a->timefrom} - {$a->timeto})';
$string['analyticsactions'] = 'Number of actions taken on generated predictions ({$a})';
$string['analyticsactionsnotuseful'] = 'Number of actions marking a prediction as not useful ({$a})';
$string['analyticsenabledmodels'] = 'Number of enabled prediction models ({$a})';
Expand Down
121 changes: 121 additions & 0 deletions lib/classes/hub/registration.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ class registration {
2023021700 => ['dbtype', 'coursesnodates', 'sitetheme', 'primaryauthtype'],
// Plugin usage added in Moodle 4.5.
2023072300 => ['pluginusage'],
// AI usage added in Moodle 4.5.
2023081200 => ['aiusage'],
];

/** @var string Site privacy: not displayed */
Expand Down Expand Up @@ -191,6 +193,10 @@ public static function get_site_info($defaults = []) {
$siteinfo['sitetheme'] = get_config('core', 'theme');
$siteinfo['pluginusage'] = json_encode(self::get_plugin_usage_data());

// AI usage data.
$aiusagedata = self::get_ai_usage_data();
$siteinfo['aiusage'] = !empty($aiusagedata) ? json_encode($aiusagedata) : '';

// Primary auth type.
$primaryauthsql = 'SELECT auth, count(auth) as tc FROM {user} GROUP BY auth ORDER BY tc DESC';
$siteinfo['primaryauthtype'] = $DB->get_field_sql($primaryauthsql, null, IGNORE_MULTIPLE);
Expand Down Expand Up @@ -278,6 +284,7 @@ public static function get_stats_summary($siteinfo) {
'sitetheme' => get_string('sitetheme', 'hub', $siteinfo['sitetheme']),
'primaryauthtype' => get_string('primaryauthtype', 'hub', $siteinfo['primaryauthtype']),
'pluginusage' => get_string('pluginusagedata', 'hub', $pluginusagelinks),
'aiusage' => get_string('aiusagestats', 'hub', self::get_ai_usage_time_range(true)),
];

foreach ($senddata as $key => $str) {
Expand Down Expand Up @@ -685,4 +692,118 @@ public static function get_plugin_usage_data(): array {

return $data;
}

/**
* Get the time range to use in collected and reporting AI usage data.
*
* @param bool $format Use true to format timestamp.
* @return array
*/
private static function get_ai_usage_time_range(bool $format = false): array {
global $DB;

// We will try and use the last time this site was last registered for our 'from' time.
// Otherwise, default to using one week's worth of data to roughly match the site rego scheduled task.
$timenow = \core\di::get(\core\clock::class)->time();
$defaultfrom = $timenow - WEEKSECS;
$timeto = $timenow;
$params = [
'huburl' => HUB_MOODLEORGHUBURL,
'confirmed' => 1,
];
$lastregistered = $DB->get_field('registration_hubs', 'timemodified', $params);
$timefrom = $lastregistered ? (int)$lastregistered : $defaultfrom;

if ($format) {
$timefrom = userdate($timefrom);
$timeto = userdate($timeto);
}

return [
'timefrom' => $timefrom,
'timeto' => $timeto,
];
}

/**
* Get AI usage data.
*
* @return array
*/
public static function get_ai_usage_data(): array {
global $DB;

$params = self::get_ai_usage_time_range();

$sql = "SELECT aar.*
FROM {ai_action_register} aar
WHERE aar.timecompleted >= :timefrom
AND aar.timecompleted <= :timeto";

$actions = $DB->get_records_sql($sql, $params);

// Build data for site info reporting.
$data = [];

foreach ($actions as $action) {
$provider = $action->provider;
$actionname = $action->actionname;

// Initialise data structure.
if (!isset($data[$provider][$actionname])) {
$data[$provider][$actionname] = [
'success_count' => 0,
'fail_count' => 0,
'times' => [],
'errors' => [],
];
}

if ($action->success === '1') {
$data[$provider][$actionname]['success_count'] += 1;
// Collect AI processing times for averaging.
$data[$provider][$actionname]['times'][] = (int)$action->timecompleted - (int)$action->timecreated;

} else {
$data[$provider][$actionname]['fail_count'] += 1;
// Collect errors for determing the predominant one.
$data[$provider][$actionname]['errors'][] = $action->errorcode;
}
}

// Parse the errors and everage the times, then add them to the data.
foreach ($data as $p => $provider) {
foreach ($provider as $a => $actionname) {
if (isset($data[$p][$a]['errors'])) {
// Create an array with the error codes counted.
$errors = array_count_values($data[$p][$a]['errors']);
if (!empty($errors)) {
// Sort values descending and convert to an array of error codes (most predominant will be at start).
arsort($errors);
$errors = array_keys($errors);
$data[$p][$a]['predominant_error'] = $errors[0];
}
unset($data[$p][$a]['errors']);
}

if (isset($data[$p][$a]['times'])) {
$count = count($data[$p][$a]['times']);
if ($count > 0) {
// Average the time to perform the action (seconds).
$totaltime = array_sum($data[$p][$a]['times']);
$data[$p][$a]['average_time'] = round($totaltime / $count);

}
}
unset($data[$p][$a]['times']);
}
}

// Include the time range used to help interpret the data.
if (!empty($data)) {
$data['time_range'] = $params;
}

return $data;
}
}
61 changes: 60 additions & 1 deletion lib/tests/hub/registration_test.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
/**
* Class containing unit tests for the site registration class.
*
* @package core
* @package core
* @copyright 2023 Matt Porritt <[email protected]>
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
* @covers \core\hub\registration
Expand Down Expand Up @@ -85,4 +85,63 @@ public function test_get_plugin_usage(): void {
$this->assertEquals(0, $pluginusage['mod']['feedback']['enabled']);
$this->assertEquals(1, $pluginusage['mod']['assign']['enabled']);
}

/**
* Test the AI usage data is calculated correctly.
*/
public function test_get_ai_usage(): void {
global $CFG, $DB;
$this->resetAfterTest();
$clock = $this->mock_clock_with_frozen();

// Record some generated text.
$record = new \stdClass();
$record->provider = 'openai';
$record->actionname = 'generate_text';
$record->actionid = 1;
$record->userid = 1;
$record->contextid = 1;
$record->success = true;
$record->timecreated = $clock->time() - 5;
$record->timecompleted = $clock->time();
$DB->insert_record('ai_action_register', $record);

// Record a generated image.
$record->actionname = 'generate_image';
$record->actionid = 2;
$record->timecreated = $clock->time() - 20;
$DB->insert_record('ai_action_register', $record);
// Record another image.
$record->actionid = 3;
$record->timecreated = $clock->time() - 10;
$DB->insert_record('ai_action_register', $record);

// Record some errors.
$record->actionname = 'generate_image';
$record->actionid = 4;
$record->success = false;
$record->errorcode = 403;
$DB->insert_record('ai_action_register', $record);
$record->actionid = 5;
$record->errorcode = 403;
$DB->insert_record('ai_action_register', $record);
$record->actionid = 6;
$record->errorcode = 404;
$DB->insert_record('ai_action_register', $record);

// Get our site info and check the expected calculations are correct.
$siteinfo = registration::get_site_info();
$aisuage = json_decode($siteinfo['aiusage']);
// Check generated text.
$this->assertEquals(1, $aisuage->openai->generate_text->success_count);
$this->assertEquals(0, $aisuage->openai->generate_text->fail_count);
// Check generated images.
$this->assertEquals(2, $aisuage->openai->generate_image->success_count);
$this->assertEquals(3, $aisuage->openai->generate_image->fail_count);
$this->assertEquals(15, $aisuage->openai->generate_image->average_time);
$this->assertEquals(403, $aisuage->openai->generate_image->predominant_error);
// Check time range is set correctly.
$this->assertEquals($clock->time() - WEEKSECS, $aisuage->time_range->timefrom);
$this->assertEquals($clock->time(), $aisuage->time_range->timeto);
}
}

0 comments on commit 2b1d330

Please sign in to comment.