Skip to content

Commit

Permalink
MDL-66004 mlbackend_python: Package installed on a separate server
Browse files Browse the repository at this point in the history
  • Loading branch information
David Monllaó committed Oct 2, 2019
1 parent 9528b1f commit aa5b705
Show file tree
Hide file tree
Showing 13 changed files with 685 additions and 181 deletions.
7 changes: 7 additions & 0 deletions admin/settings/plugins.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,13 @@
$plugin->load_settings($ADMIN, 'antivirussettings', $hassiteconfig);
}

// Machine learning backend plugins.
$ADMIN->add('modules', new admin_category('mlbackendsettings', new lang_string('mlbackendsettings', 'admin')));
$plugins = core_plugin_manager::instance()->get_plugins_of_type('mlbackend');
foreach ($plugins as $plugin) {
$plugin->load_settings($ADMIN, 'mlbackendsettings', $hassiteconfig);
}

/// License types
$ADMIN->add('modules', new admin_category('licensesettings', new lang_string('licenses')));
$temp = new admin_settingpage('managelicenses', new lang_string('managelicenses', 'admin'));
Expand Down
10 changes: 9 additions & 1 deletion analytics/classes/manager.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ class manager {
/**
* @var \core_analytics\predictor[]
*/
protected static $predictionprocessors = null;
protected static $predictionprocessors = [];

/**
* @var \core_analytics\local\target\base[]
Expand Down Expand Up @@ -213,6 +213,14 @@ public static function get_all_prediction_processors() {
return $predictionprocessors;
}

/**
* Resets the cached prediction processors.
* @return null
*/
public static function reset_prediction_processors() {
self::$predictionprocessors = [];
}

/**
* Returns the name of the provided predictions processor.
*
Expand Down
2 changes: 1 addition & 1 deletion analytics/classes/model.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -539,7 +539,7 @@ public function delete() {
debugging('Prediction processor ' . $predictorname . ' is not ready to be used. Model ' .
$this->model->id . ' could not be deleted.');
} else {
$predictor->delete_output_dir($this->get_output_dir(array(), true));
$predictor->delete_output_dir($this->get_output_dir(array(), true), $this->get_unique_id());
}

$DB->delete_records('analytics_models', array('id' => $this->model->id));
Expand Down
3 changes: 2 additions & 1 deletion analytics/classes/predictor.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,9 @@ public function clear_model($uniqueid, $modelversionoutputdir);
* can only be named 'execution', 'evaluation' or 'testing'.
*
* @param string $modeloutputdir The model directory id (parent of all model versions subdirectories).
* @param string $uniqueid
* @return null
*/
public function delete_output_dir($modeloutputdir);
public function delete_output_dir($modeloutputdir, $uniqueid);

}
158 changes: 118 additions & 40 deletions analytics/tests/prediction_test.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,14 @@
/**
* Unit tests for evaluation, training and prediction.
*
* NOTE: in order to execute this test using a separate server for the
* python ML backend you need to define these variables in your config.php file:
*
* define('TEST_MLBACKEND_PYTHON_HOST', '127.0.0.1');
* define('TEST_MLBACKEND_PYTHON_PORT', 5000);
* define('TEST_MLBACKEND_PYTHON_USERNAME', 'default');
* define('TEST_MLBACKEND_PYTHON_PASSWORD', 'sshhhh');
*
* @package core_analytics
* @copyright 2017 David Monllaó {@link http://www.davidmonllao.com}
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
Expand Down Expand Up @@ -109,26 +117,27 @@ public function test_static_prediction() {
* @param int $predictedrangeindex
* @param int $nranges
* @param string $predictionsprocessorclass
* @param array $forcedconfig
* @return void
*/
public function test_ml_training_and_prediction($timesplittingid, $predictedrangeindex, $nranges, $predictionsprocessorclass) {
public function test_ml_training_and_prediction($timesplittingid, $predictedrangeindex, $nranges, $predictionsprocessorclass,
$forcedconfig) {
global $DB;

$this->resetAfterTest(true);

$this->set_forced_config($forcedconfig);
$predictionsprocessor = $this->is_predictions_processor_ready($predictionsprocessorclass);

$this->setAdminuser();
set_config('enabled_stores', 'logstore_standard', 'tool_log');

// Generate training data.
$ncourses = 10;
$this->generate_courses($ncourses);

// We repeat the test for all prediction processors.
$predictionsprocessor = \core_analytics\manager::get_predictions_processor($predictionsprocessorclass, false);
if ($predictionsprocessor->is_ready() !== true) {
$this->markTestSkipped('Skipping ' . $predictionsprocessorclass . ' as the predictor is not ready.');
}

$model = $this->add_perfect_model();

$model->update(true, false, $timesplittingid, get_class($predictionsprocessor));

// No samples trained yet.
Expand Down Expand Up @@ -250,6 +259,17 @@ public function test_ml_training_and_prediction($timesplittingid, $predictedrang
$this->assertCount(2, $fs->get_directory_files(\context_system::instance()->id, 'analytics',
\core_analytics\dataset_manager::UNLABELLED_FILEAREA, $model->get_id(), '/analysable/', true, false));

// Confirm that the files associated to the model are deleted on clear and on delete. The ML backend deletion
// processes will be triggered by these actions and any exception there would result in a failed test.
$model->clear();
$this->assertEquals(0, $DB->count_records('analytics_used_files',
array('modelid' => $model->get_id(), 'action' => 'trained')));
$this->assertCount(0, $fs->get_directory_files(\context_system::instance()->id, 'analytics',
\core_analytics\dataset_manager::LABELLED_FILEAREA, $model->get_id(), '/analysable/', true, false));
$this->assertCount(0, $fs->get_directory_files(\context_system::instance()->id, 'analytics',
\core_analytics\dataset_manager::UNLABELLED_FILEAREA, $model->get_id(), '/analysable/', true, false));
$model->delete();

set_config('enabled_stores', '', 'tool_log');
get_log_manager(true);
}
Expand All @@ -273,25 +293,24 @@ public function provider_ml_training_and_prediction() {
* test_ml_export_import
*
* @param string $predictionsprocessorclass The class name
* @param array $forcedconfig
* @dataProvider provider_ml_processors
*/
public function test_ml_export_import($predictionsprocessorclass) {

public function test_ml_export_import($predictionsprocessorclass, $forcedconfig) {
$this->resetAfterTest(true);

$this->set_forced_config($forcedconfig);
$predictionsprocessor = $this->is_predictions_processor_ready($predictionsprocessorclass);

$this->setAdminuser();
set_config('enabled_stores', 'logstore_standard', 'tool_log');

// Generate training data.
$ncourses = 10;
$this->generate_courses($ncourses);

// We repeat the test for all prediction processors.
$predictionsprocessor = \core_analytics\manager::get_predictions_processor($predictionsprocessorclass, false);
if ($predictionsprocessor->is_ready() !== true) {
$this->markTestSkipped('Skipping ' . $predictionsprocessorclass . ' as the predictor is not ready.');
}

$model = $this->add_perfect_model();

$model->update(true, false, '\core\analytics\time_splitting\quarters', get_class($predictionsprocessor));

$model->train();
Expand Down Expand Up @@ -355,15 +374,14 @@ public function provider_ml_processors() {
* @param int $nsamples
* @param int $classes
* @param string $predictionsprocessorclass
* @param array $forcedconfig
* @return void
*/
public function test_ml_classifiers_return($success, $nsamples, $classes, $predictionsprocessorclass) {
public function test_ml_classifiers_return($success, $nsamples, $classes, $predictionsprocessorclass, $forcedconfig) {
$this->resetAfterTest();

$predictionsprocessor = \core_analytics\manager::get_predictions_processor($predictionsprocessorclass, false);
if ($predictionsprocessor->is_ready() !== true) {
$this->markTestSkipped('Skipping ' . $predictionsprocessorclass . ' as the predictor is not ready.');
}
$this->set_forced_config($forcedconfig);
$predictionsprocessor = $this->is_predictions_processor_ready($predictionsprocessorclass);

if ($nsamples % count($classes) != 0) {
throw new \coding_exception('The number of samples should be divisible by the number of classes');
Expand Down Expand Up @@ -396,7 +414,7 @@ public function test_ml_classifiers_return($success, $nsamples, $classes, $predi

// Training should work correctly if at least 1 sample of each class is included.
$dir = make_request_directory();
$result = $predictionsprocessor->train_classification('whatever', $dataset, $dir);
$result = $predictionsprocessor->train_classification('whatever' . microtime(), $dataset, $dir);

switch ($success) {
case 'yes':
Expand Down Expand Up @@ -441,16 +459,19 @@ public function provider_ml_classifiers_return() {
* @dataProvider provider_test_multi_classifier
* @param string $timesplittingid
* @param string $predictionsprocessorclass
* @param array|null $forcedconfig
* @throws coding_exception
* @throws moodle_exception
*/
public function test_ml_multi_classifier($timesplittingid, $predictionsprocessorclass) {
public function test_ml_multi_classifier($timesplittingid, $predictionsprocessorclass, $forcedconfig) {
global $DB;

$this->resetAfterTest(true);
$this->setAdminuser();
set_config('enabled_stores', 'logstore_standard', 'tool_log');

$this->set_forced_config($forcedconfig);

$predictionsprocessor = \core_analytics\manager::get_predictions_processor($predictionsprocessorclass, false);
if ($predictionsprocessor->is_ready() !== true) {
$this->markTestSkipped('Skipping ' . $predictionsprocessorclass . ' as the predictor is not ready.');
Expand Down Expand Up @@ -483,6 +504,9 @@ public function test_ml_multi_classifier($timesplittingid, $predictionsprocessor
// The range index is not important here, both ranges prediction will be the same.
$this->assertEquals($correct[$sampleid], $predictiondata->prediction);
}

set_config('enabled_stores', '', 'tool_log');
get_log_manager(true);
}

/**
Expand All @@ -508,10 +532,16 @@ public function provider_test_multi_classifier() {
* @param int $ncourses
* @param array $expected
* @param string $predictionsprocessorclass
* @param array $forcedconfig
* @return void
*/
public function test_ml_evaluation_configuration($modelquality, $ncourses, $expected, $predictionsprocessorclass) {
public function test_ml_evaluation_configuration($modelquality, $ncourses, $expected, $predictionsprocessorclass,
$forcedconfig) {
$this->resetAfterTest(true);

$this->set_forced_config($forcedconfig);
$predictionsprocessor = $this->is_predictions_processor_ready($predictionsprocessorclass);

$this->setAdminuser();
set_config('enabled_stores', 'logstore_standard', 'tool_log');

Expand All @@ -530,12 +560,6 @@ public function test_ml_evaluation_configuration($modelquality, $ncourses, $expe
// Generate training data.
$this->generate_courses($ncourses);

// We repeat the test for all prediction processors.
$predictionsprocessor = \core_analytics\manager::get_predictions_processor($predictionsprocessorclass, false);
if ($predictionsprocessor->is_ready() !== true) {
$this->markTestSkipped('Skipping ' . $predictionsprocessorclass . ' as the predictor is not ready.');
}

$model->update(false, false, false, get_class($predictionsprocessor));
$results = $model->evaluate();

Expand Down Expand Up @@ -563,10 +587,15 @@ public function test_ml_evaluation_configuration($modelquality, $ncourses, $expe
* @coversNothing
* @dataProvider provider_ml_processors
* @param string $predictionsprocessorclass
* @param array $forcedconfig
* @return null
*/
public function test_ml_evaluation_trained_model($predictionsprocessorclass) {
public function test_ml_evaluation_trained_model($predictionsprocessorclass, $forcedconfig) {
$this->resetAfterTest(true);

$this->set_forced_config($forcedconfig);
$predictionsprocessor = $this->is_predictions_processor_ready($predictionsprocessorclass);

$this->setAdminuser();
set_config('enabled_stores', 'logstore_standard', 'tool_log');

Expand All @@ -575,12 +604,6 @@ public function test_ml_evaluation_trained_model($predictionsprocessorclass) {
// Generate training data.
$this->generate_courses(50);

// We repeat the test for all prediction processors.
$predictionsprocessor = \core_analytics\manager::get_predictions_processor($predictionsprocessorclass, false);
if ($predictionsprocessor->is_ready() !== true) {
$this->markTestSkipped('Skipping ' . $predictionsprocessorclass . ' as the predictor is not ready.');
}

$model->update(true, false, '\\core\\analytics\\time_splitting\\quarters', get_class($predictionsprocessor));
$model->train();

Expand Down Expand Up @@ -824,6 +847,41 @@ protected function generate_courses_multiclass($ncourses, array $params = []) {
}
}

/**
* Forces some configuration values.
*
* @param array $forcedconfig
*/
protected function set_forced_config($forcedconfig) {
\core_analytics\manager::reset_prediction_processors();

if (empty($forcedconfig)) {
return;
}
foreach ($forcedconfig as $pluginname => $pluginconfig) {
foreach ($pluginconfig as $name => $value) {
set_config($name, $value, $pluginname);
}
}
}

/**
* Is the provided processor ready using the current configuration in the site?
*
* @param string $predictionsprocessorclass
* @return \core_analytics\predictor
*/
protected function is_predictions_processor_ready(string $predictionsprocessorclass) {
// We repeat the test for all prediction processors.
$predictionsprocessor = \core_analytics\manager::get_predictions_processor($predictionsprocessorclass, false);
$ready = $predictionsprocessor->is_ready();
if ($ready !== true) {
$this->markTestSkipped('Skipping ' . $predictionsprocessorclass . ' as the predictor is not ready: ' . $ready);
}

return $predictionsprocessor;
}

/**
* add_prediction_processors
*
Expand All @@ -834,12 +892,32 @@ protected function add_prediction_processors($cases) {

$return = array();

// We need to test all system prediction processors.
if (defined('TEST_MLBACKEND_PYTHON_HOST') && defined('TEST_MLBACKEND_PYTHON_PORT')
&& defined('TEST_MLBACKEND_PYTHON_USERNAME') && defined('TEST_MLBACKEND_PYTHON_USERNAME')) {
$testpythonserver = true;
}

// We need to test all prediction processors in the system.
$predictionprocessors = \core_analytics\manager::get_all_prediction_processors();
foreach ($predictionprocessors as $classfullname => $unused) {
foreach ($predictionprocessors as $classfullname => $predictionsprocessor) {

foreach ($cases as $key => $case) {
$newkey = $key . '-' . $classfullname;
$return[$newkey] = $case + array('predictionsprocessorclass' => $classfullname);
$return[$key . '-' . $classfullname] = $case + ['predictionsprocessor' => $classfullname, 'forcedconfig' => null];
}

if ($predictionsprocessor instanceof \mlbackend_python\processor && !empty($testpythonserver)) {
// We also want to test the python processor using the server.

foreach ($cases as $key => $case) {

// We want the configuration to be forced during the test as things like importing models create new
// instances of ML backend processors during the process.
$forcedconfig = ['mlbackend_python' => ['useserver' => true, 'host' => TEST_MLBACKEND_PYTHON_HOST,
'port' => TEST_MLBACKEND_PYTHON_PORT, 'secure' => false, 'username' => TEST_MLBACKEND_PYTHON_USERNAME,
'password' => TEST_MLBACKEND_PYTHON_PASSWORD]];
$casekey = $key . '-' . $classfullname . '-server';
$return[$casekey] = $case + ['predictionsprocessor' => $classfullname, 'forcedconfig' => $forcedconfig];
}
}
}

Expand Down
1 change: 1 addition & 0 deletions analytics/upgrade.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ information provided here is intended especially for developers.
* A new \core_analytics\local\time_splitting\past_periodic abstract class has been added. Time-splitting
methods extending \core_analytics\local\time_splitting\periodic directly should be extending past_periodic
now. 'periodic' can still be directly extended by implementing get_next_range and get_first_start methods.
* \core_analytics\predictor::delete_output_dir has a new 2nd parameter, $uniquemodelid.

=== 3.7 ===

Expand Down
1 change: 1 addition & 0 deletions lang/en/admin.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -790,6 +790,7 @@
$string['minpasswordnonalphanum'] = 'Non-alphanumeric characters';
$string['minpasswordupper'] = 'Uppercase letters';
$string['misc'] = 'Miscellaneous';
$string['mlbackendsettings'] = 'Machine learning backend settings';
$string['mnetrestore_extusers'] = '<strong>Note:</strong> This backup file contains remote Moodle Network user accounts which will be restored as part of the process.';
$string['mnetrestore_extusers_admin'] = '<strong>Note:</strong> This backup file seems to come from a different Moodle installation and contains remote Moodle Network user accounts. The restore process will try to match the Moodle Network hosts for all created users. Those not matching will be automatically switched to internal authentication (instead of mnet one). The restore log will inform you about that.';
$string['mnetrestore_extusers_mismatch'] = '<strong>Note:</strong> This backup file apparently originates from a different Moodle installation and contains remote Moodle Network user accounts that may fail to restore. This operation is unsupported. If you are certain that it was created on this Moodle installation, or you can ensure that all the needed Moodle Network Hosts are configured, you may want to still try the restore.';
Expand Down
Loading

0 comments on commit aa5b705

Please sign in to comment.