Skip to content

Commit

Permalink
MDL-68729 Search: Allow query on one server while indexing another
Browse files Browse the repository at this point in the history 
To support transitions from one search engine to a different one, or
to a different installation of the same kind, this feature allows for
queries to use a different search engine from indexing. So you can
reindex (and do all other search operation) on one server, while
user queries are unaffected on a different server.

This feature supports changing between search engine types, and also
between two Solr installations.
  • Loading branch information
@sammarshallou
sammarshallou committed Aug 6, 2020
1 parent b580095 commit 679e8d8
Show file tree
Hide file tree
Showing 12 changed files with 284 additions and 18 deletions.
52 changes: 50 additions & 2 deletions admin/settings/plugins.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -549,8 +549,19 @@

// Search engine selection.
$temp->add(new admin_setting_heading('searchengineheading', new lang_string('searchengine', 'admin'), ''));
$temp->add(new admin_setting_configselect('searchengine',
new lang_string('selectsearchengine', 'admin'), '', 'simpledb', $engines));
$searchengineselect = new admin_setting_configselect('searchengine',
new lang_string('selectsearchengine', 'admin'), '', 'simpledb', $engines);
$searchengineselect->set_validate_function(function(string $value): string {
global $CFG;

// Check nobody's setting the indexing and query-only server to the same one.
if ($CFG->searchenginequeryonly === $value) {
return get_string('searchenginequeryonlysame', 'admin');
} else {
return '';
}
});
$temp->add($searchengineselect);
$temp->add(new admin_setting_heading('searchoptionsheading', new lang_string('searchoptions', 'admin'), ''));
$temp->add(new admin_setting_configcheckbox('searchindexwhendisabled',
new lang_string('searchindexwhendisabled', 'admin'), new lang_string('searchindexwhendisabled_desc', 'admin'),
Expand Down Expand Up @@ -590,6 +601,43 @@
new lang_string('searchhideallcategory_desc', 'admin'),
0));

$temp->add(new admin_setting_heading('searchmanagement', new lang_string('searchmanagement', 'admin'),
new lang_string('searchmanagement_desc', 'admin')));

// Get list of search engines including those with alternate settings.
$searchenginequeryonlyselect = new admin_setting_configselect('searchenginequeryonly',
new lang_string('searchenginequeryonly', 'admin'),
new lang_string('searchenginequeryonly_desc', 'admin'), '', function() use($engines) {
$options = ['' => new lang_string('searchenginequeryonly_none', 'admin')];
foreach ($engines as $name => $display) {
$options[$name] = $display;

$classname = '\search_' . $name . '\engine';
$engine = new $classname;
if ($engine->has_alternate_configuration()) {
$options[$name . '-alternate'] =
new lang_string('searchenginealternatesettings', 'admin', $display);
}
}
return $options;
});
$searchenginequeryonlyselect->set_validate_function(function(string $value): string {
global $CFG;

// Check nobody's setting the indexing and query-only server to the same one.
if ($CFG->searchengine === $value) {
return get_string('searchenginequeryonlysame', 'admin');
} else {
return '';
}
});
$temp->add($searchenginequeryonlyselect);
$temp->add(new admin_setting_configcheckbox('searchbannerenable',
new lang_string('searchbannerenable', 'admin'), new lang_string('searchbannerenable_desc', 'admin'),
0));
$temp->add(new admin_setting_confightmleditor('searchbanner',
new lang_string('searchbanner', 'admin'), '', ''));

$ADMIN->add('searchplugins', $temp);
$ADMIN->add('searchplugins', new admin_externalpage('searchareas', new lang_string('searchareas', 'admin'),
new moodle_url('/admin/searchareas.php')));
Expand Down
12 changes: 12 additions & 0 deletions lang/en/admin.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1107,6 +1107,8 @@
$string['searchallavailablecourses_off'] = 'Search within enrolled courses only';
$string['searchallavailablecourses_on'] = 'Search within all courses the user can access';
$string['searchallavailablecourses_desc'] = 'In some situations the search engine may not work when searching across a large number of courses. Set to search only enrolled courses if you need to restrict the number of courses searched.';
$string['searchalternatesettings'] = 'Query-only alternate settings';
$string['searchalternatesettings_desc'] = 'If you complete these settings, you can select \'alternate settings\' for this search engine in the query-only search engine option on the \'Manage global search\' page. This is only useful when moving between two search engines of the same type.';
$string['searchdisplay'] = 'Search results display options';
$string['searchenablecategories'] = 'Display results in separate categories';
$string['searchenablecategories_desc'] = 'If enabled, search results will be displayed in separate categories.';
Expand All @@ -1118,10 +1120,18 @@
$string['searchincludeallcourses'] = 'Include all visible courses';
$string['searchincludeallcourses_desc'] = 'If enabled, search results will include course information (name and summary) of courses which are visible to the user, even if they don\'t have access to the course content.';
$string['searchalldeleted'] = 'All indexed contents have been deleted';
$string['searchbannerenable'] = 'Display search information';
$string['searchbannerenable_desc'] = 'If enabled, the below text will display at the top of the search screen for all users. This can be used to keep users informed while maintenance is being carried out on the search system.';
$string['searchbanner'] = 'Search information';
$string['searchareaenabled'] = 'Search area enabled';
$string['searchareadisabled'] = 'Search area disabled';
$string['searchdeleteindex'] = 'Delete all indexed contents';
$string['searchengine'] = 'Search engine';
$string['searchenginealternatesettings'] = '{$a} (alternate settings)';
$string['searchenginequeryonly'] = 'Query-only search engine';
$string['searchenginequeryonly_desc'] = 'This search engine will be used only for making queries, not indexing. By using this feature you can reindex in a different search engine, while user queries continue to work from this one.';
$string['searchenginequeryonly_none'] = 'None (use main search engine for queries)';
$string['searchenginequeryonlysame'] = 'The query-only search engine and the main search engine cannot be set to the same value.';
$string['searchindexactions'] = 'Index actions';
$string['searchindexdeleted'] = 'Index deleted';
$string['searchindextime'] = 'Indexing time limit';
Expand All @@ -1131,6 +1141,8 @@
$string['searchindexwhendisabled_desc'] = 'Allows the scheduled task to build the search index even when search is disabled. This is useful if you want to build the index before the search facility appears to students.';
$string['searchinsettings'] = 'Search in settings';
$string['searchlastrun'] = 'Last run (time, # docs, # records, # ignores)';
$string['searchmanagement'] = 'Search management';
$string['searchmanagement_desc'] = 'These options are useful when making changes on sites with very large search indexes that take a long time to rebuild.';
$string['searchnotavailable'] = 'Search is not available';
$string['searchpartial'] = '(not yet fully indexed)';
$string['searchoptions'] = 'Search options';
Expand Down
39 changes: 38 additions & 1 deletion search/classes/engine.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,9 +84,13 @@ abstract class engine {
*
* Search engine availability should be checked separately.
*
* The alternate configuration option is only used to construct a special second copy of the
* search engine object, as described in {@see has_alternate_configuration}.
*
* @param bool $alternateconfiguration If true, use alternate configuration settings
* @return void
*/
public function __construct() {
public function __construct(bool $alternateconfiguration = false) {

$classname = get_class($this);
if (strpos($classname, '\\') === false) {
Expand All @@ -102,6 +106,19 @@ public function __construct() {
} else {
$this->config = new stdClass();
}

// For alternate configuration, automatically replace normal configuration values with
// those beginning with 'alternate'.
if ($alternateconfiguration) {
foreach ((array)$this->config as $key => $value) {
if (preg_match('~^alternate(.*)$~', $key, $matches)) {
$this->config->{$matches[1]} = $value;
}
}
}

// Flag just in case engine needs to know it is using the alternate configuration.
$this->config->alternateconfiguration = $alternateconfiguration;
}

/**
Expand Down Expand Up @@ -740,4 +757,24 @@ public function get_batch_max_documents(): int {
public function get_batch_max_content(): int {
return 1024 * 1024;
}

/**
* Checks if the search engine has an alternate configuration.
*
* This is used where the same search engine class supports two different configurations,
* which are both shown on the settings screen. The alternate configuration is selected by
* passing 'true' parameter to the constructor.
*
* The feature is used when a different connection is in use for indexing vs. querying
* the search engine.
*
* This function should only return true if the engine supports an alternate configuration
* and the user has filled in the settings. (We do not need to test they are valid, that will
* happen as normal.)
*
* @return bool True if an alternate configuration is defined
*/
public function has_alternate_configuration(): bool {
return false;
}
}
46 changes: 40 additions & 6 deletions search/classes/manager.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,13 +190,18 @@ public function __construct($engine) {
* parameter provides a way to skip those checks on pages which are used frequently. It has
* no effect if an instance has already been constructed in this request.
*
* The $query parameter indicates that the page is used for queries rather than indexing. If
* configured, this will cause the query-only search engine to be used instead of the 'normal'
* one.
*
* @see \core_search\engine::is_installed
* @see \core_search\engine::is_server_ready
* @param bool $fast Set to true when calling on a page that requires high performance
* @param bool $query Set true on a page that is used for querying
* @throws \core_search\engine_exception
* @return \core_search\manager
*/
public static function instance($fast = false) {
public static function instance(bool $fast = false, bool $query = false) {
global $CFG;

// One per request, this should be purged during testing.
Expand All @@ -208,7 +213,7 @@ public static function instance($fast = false) {
throw new \core_search\engine_exception('enginenotselected', 'search');
}

if (!$engine = static::search_engine_instance()) {
if (!$engine = static::search_engine_instance($query)) {
throw new \core_search\engine_exception('enginenotfound', 'search', '', $CFG->searchengine);
}

Expand Down Expand Up @@ -287,17 +292,46 @@ public static function is_indexing_enabled() {
/**
* Returns an instance of the search engine.
*
* @param bool $query If true, gets the query-only search engine (where configured)
* @return \core_search\engine
*/
public static function search_engine_instance() {
public static function search_engine_instance(bool $query = false) {
global $CFG;

$classname = '\\search_' . $CFG->searchengine . '\\engine';
if ($query && $CFG->searchenginequeryonly) {
return self::search_engine_instance_from_setting($CFG->searchenginequeryonly);
} else {
return self::search_engine_instance_from_setting($CFG->searchengine);
}
}

/**
* Loads a search engine based on the name given in settings, which can optionally
* include '-alternate' to indicate that an alternate version should be used.
*
* @param string $setting
* @return engine|null
*/
protected static function search_engine_instance_from_setting(string $setting): ?engine {
if (preg_match('~^(.*)-alternate$~', $setting, $matches)) {
$enginename = $matches[1];
$alternate = true;
} else {
$enginename = $setting;
$alternate = false;
}

$classname = '\\search_' . $enginename . '\\engine';
if (!class_exists($classname)) {
return false;
return null;
}

return new $classname();
if ($alternate) {
return new $classname(true);
} else {
// Use the constructor with no parameters for compatibility.
return new $classname();
}
}

/**
Expand Down
20 changes: 16 additions & 4 deletions search/engine/solr/classes/engine.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,10 +116,11 @@ class engine extends \core_search\engine {
/**
* Initialises the search engine configuration.
*
* @param bool $alternateconfiguration If true, use alternate configuration settings
* @return void
*/
public function __construct() {
parent::__construct();
public function __construct(bool $alternateconfiguration = false) {
parent::__construct($alternateconfiguration);

$curlversion = curl_version();
if (isset($curlversion['version']) && stripos($curlversion['version'], '7.35.') === 0) {
Expand Down Expand Up @@ -1256,7 +1257,7 @@ public function is_server_ready() {

// Check that the schema is already set up.
try {
$schema = new \search_solr\schema();
$schema = new schema($this);
$schema->validate_setup();
} catch (\moodle_exception $e) {
return $e->getMessage();
Expand Down Expand Up @@ -1466,7 +1467,7 @@ public function supports_group_filtering() {

protected function update_schema($oldversion, $newversion) {
// Construct schema.
$schema = new schema();
$schema = new schema($this);
$cansetup = $schema->can_setup_server();
if ($cansetup !== true) {
return $cansetup;
Expand Down Expand Up @@ -1564,4 +1565,15 @@ public function delete_index_for_course(int $oldcourseid) {
throw new \core_search\engine_exception('error_solr', 'search_solr', '', $e->getMessage());
}
}

/**
* Checks if an alternate configuration has been defined.
*
* @return bool True if alternate configuration is available
*/
public function has_alternate_configuration(): bool {
return !empty($this->config->alternateserver_hostname) &&
!empty($this->config->alternateindexname) &&
!empty($this->config->alternateserver_port);
}
}
5 changes: 3 additions & 2 deletions search/engine/solr/classes/schema.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,11 @@ class schema {
/**
* Constructor.
*
* @param engine $engine Optional engine parameter, if not specified then one will be created
* @throws \moodle_exception
* @return void
*/
public function __construct() {
public function __construct(engine $engine = null) {
if (!$this->config = get_config('search_solr')) {
throw new \moodle_exception('missingconfig', 'search_solr');
}
Expand All @@ -72,7 +73,7 @@ public function __construct() {
throw new \moodle_exception('missingconfig', 'search_solr');
}

$this->engine = new engine();
$this->engine = $engine ?? new engine();
$this->curl = $this->engine->get_curl_object();

// HTTP headers.
Expand Down
36 changes: 36 additions & 0 deletions search/engine/solr/settings.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,42 @@
$settings->add(new admin_setting_configtext('search_solr/maxindexfilekb',
new lang_string('maxindexfilekb', 'search_solr'),
new lang_string('maxindexfilekb_help', 'search_solr'), '2097152', PARAM_INT));

// Alternate connection.
$settings->add(new admin_setting_heading('search_solr_alternatesettings',
new lang_string('searchalternatesettings', 'admin'),
new lang_string('searchalternatesettings_desc', 'admin')));
$settings->add(new admin_setting_configtext('search_solr/alternateserver_hostname',
new lang_string('solrserverhostname', 'search_solr'),
new lang_string('solrserverhostname_desc', 'search_solr'), '127.0.0.1', PARAM_HOST));
$settings->add(new admin_setting_configtext('search_solr/alternateindexname',
new lang_string('solrindexname', 'search_solr'), '', '', PARAM_ALPHANUMEXT));
$settings->add(new admin_setting_configcheckbox('search_solr/alternatesecure',
new lang_string('solrsecuremode', 'search_solr'), '', 0, 1, 0));

$secure = get_config('search_solr', 'alternatesecure');
$defaultport = !empty($secure) ? 8443 : 8983;
$settings->add(new admin_setting_configtext('search_solr/alternateserver_port',
new lang_string('solrhttpconnectionport', 'search_solr'), '', $defaultport, PARAM_INT));
$settings->add(new admin_setting_configtext('search_solr/alternateserver_username',
new lang_string('solrauthuser', 'search_solr'), '', '', PARAM_RAW));
$settings->add(new admin_setting_configpasswordunmask('search_solr/alternateserver_password',
new lang_string('solrauthpassword', 'search_solr'), '', ''));
$settings->add(new admin_setting_configtext('search_solr/alternatessl_cert',
new lang_string('solrsslcert', 'search_solr'),
new lang_string('solrsslcert_desc', 'search_solr'), '', PARAM_RAW));
$settings->add(new admin_setting_configtext('search_solr/alternatessl_key',
new lang_string('solrsslkey', 'search_solr'),
new lang_string('solrsslkey_desc', 'search_solr'), '', PARAM_RAW));
$settings->add(new admin_setting_configpasswordunmask('search_solr/alternatessl_keypassword',
new lang_string('solrsslkeypassword', 'search_solr'),
new lang_string('solrsslkeypassword_desc', 'search_solr'), ''));
$settings->add(new admin_setting_configtext('search_solr/alternatessl_cainfo',
new lang_string('solrsslcainfo', 'search_solr'),
new lang_string('solrsslcainfo_desc', 'search_solr'), '', PARAM_RAW));
$settings->add(new admin_setting_configtext('search_solr/alternatessl_capath',
new lang_string('solrsslcapath', 'search_solr'),
new lang_string('solrsslcapath_desc', 'search_solr'), '', PARAM_RAW));
}
}
}
Loading

0 comments on commit 679e8d8

Please sign in to comment.