MDL-31989 search: Search API and search engine API

Introducing both APIs in moodle along with:
- search_box widget to add a tiny search box
- admin settings with setup steps helper
- cache for search results
- template for a search result
- php unit stuff

Many thanks to Tomasz Muras, Prateek Sachan and Daniel Neis for their contributions, for starting this development
and for pushing for it to be completed. Also thanks to other contributors: Jonathan Harker and eugeneventer.

admin/settings/plugins.php

@@ -457,6 +457,47 @@
     $ADMIN->add('reportplugins', $page);
 }
 
+// Global Search engine plugins.
+$ADMIN->add('modules', new admin_category('searchplugins', new lang_string('search', 'admin')));
+$temp = new admin_settingpage('manageglobalsearch', new lang_string('globalsearchmanage', 'admin'));
+
+$pages = array();
+$engines = array();
+foreach (core_component::get_plugin_list('search') as $engine => $plugindir) {
+    $engines[$engine] = new lang_string('pluginname', 'search_' . $engine);
+    $settingspath = "$plugindir/settings.php";
+    if (file_exists($settingspath)) {
+        $settings = new admin_settingpage('search' . $engine,
+                new lang_string('pluginname', 'search_' . $engine), 'moodle/site:config');
+        include($settingspath);
+        if ($settings) {
+            $pages[] = $settings;
+        }
+    }
+}
+
+// Setup status.
+$temp->add(new admin_setting_searchsetupinfo());
+
+// Search engine selection.
+$temp->add(new admin_setting_heading('searchengineheading', new lang_string('searchengine', 'admin'), ''));
+$temp->add(new admin_setting_configselect('searchengine',
+                            new lang_string('selectsearchengine', 'admin'), '', 'solr', $engines));
+
+// Enable search areas.
+$temp->add(new admin_setting_heading('searchareasheading', new lang_string('availablesearchareas', 'admin'), ''));
+$searchareas = \core_search\manager::get_search_areas_list();
+foreach ($searchareas as $areaid => $searcharea) {
+    list($componentname, $varname) = $searcharea->get_config_var_name();
+    $temp->add(new admin_setting_configcheckbox($componentname . '/enable' . $varname, $searcharea->get_visible_name(true),
+        '', 1, 1, 0));
+}
+$ADMIN->add('searchplugins', $temp);
+
+foreach ($pages as $page) {
+    $ADMIN->add('searchplugins', $page);
+}
+
 /// Add all admin tools
 if ($hassiteconfig) {
     $ADMIN->add('modules', new admin_category('tools', new lang_string('tools', 'admin')));

admin/settings/subsystems.php

@@ -51,4 +51,7 @@
     $optionalsubsystems->add(new admin_setting_configcheckbox('enableplagiarism', new lang_string('enableplagiarism','plagiarism'), new lang_string('configenableplagiarism','plagiarism'), 0));
 
     $optionalsubsystems->add(new admin_setting_configcheckbox('enablebadges', new lang_string('enablebadges', 'badges'), new lang_string('configenablebadges', 'badges'), 1));
+
+    $optionalsubsystems->add(new admin_setting_configcheckbox('enableglobalsearch', new lang_string('enableglobalsearch', 'admin'),
+        new lang_string('enableglobalsearch_desc', 'admin'), 0, 1, 0));
 }

lang/en/admin.php

@@ -66,6 +66,7 @@
 $string['authsettings'] = 'Manage authentication';
 $string['autolang'] = 'Language autodetect';
 $string['autologinguests'] = 'Auto-login guests';
+$string['availablesearchareas'] = 'Available areas for search';
 $string['availableto'] = 'Available to';
 $string['availablelicenses'] = 'Available licences';
 $string['backgroundcolour'] = 'Transparent colour';
@@ -474,12 +475,15 @@
 $string['enablecssoptimiser_desc'] = 'When enabled CSS will be run through an optimisation process before being cached. The optimiser processes the CSS removing duplicate rules and styles, as well as white space removable and reformatting. Please note turning this on at the same time as theme designer mode is awful for performance but will help theme designers create optimised CSS.';
 $string['enabled'] = 'Enabled';
 $string['enabledevicedetection'] = 'Enable device detection';
+$string['enableglobalsearch'] = 'Enable global search';
+$string['enableglobalsearch_desc'] = 'If enabled, data will be indexed and syncronised by a scheduled task.';
 $string['enablegravatar'] = 'Enable Gravatar';
 $string['enablegravatar_help'] = 'When enabled Moodle will attempt to fetch a user profile picture from Gravatar if the user has not uploaded an image.';
 $string['enablemobilewebservice'] = 'Enable web services for mobile devices';
 $string['enablerecordcache'] = 'Enable record cache';
 $string['enablerssfeeds'] = 'Enable RSS feeds';
 $string['enablesafebrowserintegration'] = 'Enable Safe Exam Browser integration';
+$string['enablesearchareas'] = 'Enable search areas';
 $string['enablestats'] = 'Enable statistics';
 $string['enabletrusttext'] = 'Enable trusted content';
 $string['enablewebservices'] = 'Enable web services';
@@ -545,6 +549,8 @@
 $string['generalsettings'] = 'General settings';
 $string['geoipfile'] = 'GeoIP city data file';
 $string['getremoteaddrconf'] = 'Logged IP address source';
+$string['globalsearch'] = 'Global search';
+$string['globalsearchmanage'] = 'Manage global search';
 $string['groupenrolmentkeypolicy'] = 'Group enrolment key policy';
 $string['groupenrolmentkeypolicy_desc'] = 'Turning this on will make Moodle check group enrolment keys against a valid password policy.';
 $string['googlemapkey3'] = 'Google Maps API V3 key';
@@ -586,6 +592,7 @@
 $string['ignore'] = 'Ignore';
 $string['includemoduleuserdata'] = 'Include module user data';
 $string['incompatibleblocks'] = 'Incompatible blocks';
+$string['indexdata'] = 'Index data';
 $string['installhijacked'] = 'Installation must be finished from the original IP address, sorry.';
 $string['installsessionerror'] = 'Can not initialise PHP session, please verify that your browser accepts cookies.';
 $string['intlrecommended'] = 'Intl extension is used to improve internationalization support, such as locale aware sorting.';
@@ -938,12 +945,15 @@
 $string['save'] = 'Save';
 $string['savechanges'] = 'Save changes';
 $string['search'] = 'Search';
+$string['searchengine'] = 'Search engine';
 $string['searchinsettings'] = 'Search in settings';
 $string['searchresults'] = 'Search results';
+$string['searchsetupinfo'] = 'Search setup';
 $string['sectionerror'] = 'Section error!';
 $string['secureforms'] = 'Use additional form security';
 $string['security'] = 'Security';
 $string['selectdevice'] = 'Select device';
+$string['selectsearchengine'] = 'Select search engine';
 $string['selecttheme'] = 'Select theme for {$a} device';
 $string['server'] = 'Server';
 $string['serverchecks'] = 'Server checks';
@@ -957,6 +967,7 @@
 $string['settingfileuploads'] = 'File uploading is required for normal operation, please enable it in PHP configuration.';
 $string['settingmemorylimit'] = 'Insufficient memory detected, please set higher memory limit in PHP settings.';
 $string['settingsafemode'] = 'Moodle is not fully compatible with safe mode, please ask server administrator to turn it off. Running Moodle under safe mode is not supported, please expect various problems if you do so.';
+$string['setupsearchengine'] = 'Setup search engine';
 $string['showcommentscount'] = 'Show comments count';
 $string['showdetails'] = 'Show details';
 $string['showuseridentity'] = 'Show user identity';
@@ -1029,6 +1040,7 @@
 $string['taskdeleteunconfirmedusers'] = 'Delete unconfirmed users';
 $string['taskeventscron'] = 'Background processing for events';
 $string['taskfiletrashcleanup'] = 'Cleanup files in trash';
+$string['taskglobalsearch'] = 'Global search indexing';
 $string['taskgradecron'] = 'Background processing for gradebook';
 $string['tasklegacycron'] = 'Legacy cron processing for plugins';
 $string['taskmessagingcleanup'] = 'Background processing for messaging';

lang/en/cache.php

@@ -59,6 +59,7 @@
 $string['cachedef_plugin_manager'] = 'Plugin info manager';
 $string['cachedef_questiondata'] = 'Question definitions';
 $string['cachedef_repositories'] = 'Repositories instances data';
+$string['cachedef_search_results'] = 'Search results user data';
 $string['cachedef_string'] = 'Language string cache';
 $string['cachedef_tags'] = 'Tags collections and areas';
 $string['cachedef_userselections'] = 'Data used to persist user selections throughout Moodle';

lang/en/moodle.php

@@ -747,6 +747,7 @@
 $string['eventemailfailed'] = 'Email failed to send';
 $string['eventname'] = 'Event name';
 $string['eventrecentactivityviewed'] = 'Recent activity viewed';
+$string['eventsearchindexed'] = 'Search data indexed';
 $string['eventunknownlogged'] = 'Unknown event';
 $string['eventusercreated'] = 'User created';
 $string['eventuserdeleted'] = 'User deleted';

lang/en/search.php

@@ -24,6 +24,7 @@
 
 $string['advancedsearch'] = 'Advanced search';
 $string['all'] = 'All';
+$string['allareas'] = 'All areas';
 $string['author'] = 'Author';
 $string['authorname'] = 'Author name';
 $string['back'] = 'Back';
@@ -35,45 +36,64 @@
 $string['databasestate'] = 'Indexing database state';
 $string['datadirectory'] = 'Data directory';
 $string['deletionsinindex'] = 'Deletions in index';
+$string['docmodifiedon'] = 'Last modified on {$a}';
 $string['doctype'] = 'Doctype';
+$string['doctypenotsupported'] = 'The specified doc type is not yet supported';
 $string['documents'] = 'documents';
 $string['documentsfor'] = 'Documents for';
 $string['documentsindatabase'] = 'Documents in database';
 $string['documentsinindex'] = 'Documents in index';
 $string['duration'] = 'Duration';
 $string['emptydatabaseerror'] = 'Database table is not present, or contains no index records.';
+$string['enginenotfound'] = 'Engine {$a} not found.';
+$string['enginenotinstalled'] = '{$a} not installed.';
+$string['engineserverstatus'] = 'The search engine is not available. Please contact your administrator.';
 $string['enteryoursearchquery'] = 'Enter your search query';
 $string['errors'] = 'Errors';
+$string['errorareanotavailable'] = '{$a} search area is not available.';
 $string['filesinindexdirectory'] = 'Files in index directory';
+$string['filterheader'] = 'Filter';
+$string['fromtime'] = 'Modified after';
+$string['globalsearch'] = 'Global search';
 $string['globalsearchdisabled'] = 'Global searching is not enabled.';
 $string['checkdb'] = 'Check database';
 $string['checkdbadvice'] = 'Check your database for any problems.';
 $string['checkdir'] = 'Check dir';
 $string['checkdiradvice'] = 'Ensure the data directory exists and is writable.';
+$string['incourse'] = 'in course {$a}';
+$string['index'] = 'Index';
 $string['invalidindexerror'] = 'Index directory either contains an invalid index, or nothing at all.';
 $string['ittook'] = 'It took';
 $string['next'] = 'Next';
 $string['noindexmessage'] = 'Admin: There appears to be no search index. Please';
+$string['noresults'] = 'No results';
 $string['normalsearch'] = 'Normal search';
 $string['openedon'] = 'opened on';
+$string['optimize'] = 'Optimize';
+$string['queryerror'] = 'The query you provided could not be parsed by the search engine: {$a}';
 $string['resultsreturnedfor'] = 'results returned for';
 $string['runindexer'] = 'Run indexer (real)';
 $string['runindexertest'] = 'Run indexer test';
 $string['score'] = 'Score';
 $string['search'] = 'Search';
+$string['searcharea'] = 'Search area';
 $string['searching'] = 'Searching in ...';
 $string['searchnotpermitted'] = 'You are not allowed to do a search';
+$string['searchsetupdescription'] = 'The following steps help you to set up Moodle global search.';
 $string['seconds'] = 'seconds';
 $string['solutions'] = 'Solutions';
 $string['statistics'] = 'Statistics';
+$string['step'] = 'Step';
 $string['thesewordshelpimproverank'] = 'These words help improve rank';
 $string['thesewordsmustappear'] = 'These words must appear';
 $string['thesewordsmustnotappear'] = 'These words must not appear';
 $string['title'] = 'Title';
 $string['tofetchtheseresults'] = 'to fetch these results';
 $string['totalsize'] = 'Total size';
+$string['totime'] = 'Modified before';
 $string['type'] = 'Type';
 $string['uncompleteindexingerror'] = 'Indexing was not successfully completed, please restart it.';
 $string['versiontoolow'] = 'Sorry, global search requires PHP 5.0.0 or later';
+$string['viewresultincontext'] = 'View this result in context';
 $string['whichmodulestosearch?'] = 'Which modules to search?';
 $string['wordsintitle'] = 'Words in title';

lib/adminlib.php

@@ -9265,3 +9265,172 @@ public function load_choices() {
         return true;
     }
 }
+
+
+/**
+ * Search setup steps info.
+ *
+ * @package core
+ * @copyright 2016 David Monllao {@link http://www.davidmonllao.com}
+ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class admin_setting_searchsetupinfo extends admin_setting {
+
+    /**
+     * Calls parent::__construct with specific arguments
+     */
+    public function __construct() {
+        $this->nosave = true;
+        parent::__construct('searchsetupinfo', '', '', '');
+    }
+
+    /**
+     * Always returns true, does nothing
+     *
+     * @return true
+     */
+    public function get_setting() {
+        return true;
+    }
+
+    /**
+     * Always returns true, does nothing
+     *
+     * @return true
+     */
+    public function get_defaultsetting() {
+        return true;
+    }
+
+    /**
+     * Always returns '', does not write anything
+     *
+     * @param array $data
+     * @return string Always returns ''
+     */
+    public function write_setting($data) {
+        // Do not write any setting.
+        return '';
+    }
+
+    /**
+     * Builds the HTML to display the control
+     *
+     * @param string $data Unused
+     * @param string $query
+     * @return string
+     */
+    public function output_html($data, $query='') {
+        global $CFG, $OUTPUT;
+
+        $return = '';
+        $brtag = html_writer::empty_tag('br');
+
+        // Available search areas.
+        $searchareas = \core_search\manager::get_search_areas_list();
+        $anyenabled = false;
+        $anyindexed = false;
+        foreach ($searchareas as $areaid => $searcharea) {
+            list($componentname, $varname) = $searcharea->get_config_var_name();
+            if (!$anyenabled) {
+                $anyenabled = get_config($componentname, 'enable' . $varname);
+            }
+            if (!$anyindexed) {
+                $anyindexed = get_config($componentname, $varname . '_indexingstart');
+            }
+            if ($anyenabled && $anyindexed) {
+                break;
+            }
+        }
+
+        $return .= $OUTPUT->heading(get_string('searchsetupinfo', 'admin'), 3, 'main');
+
+        $table = new html_table();
+        $table->head = array(get_string('step', 'search'), get_string('status'));
+        $table->colclasses = array('leftalign step', 'leftalign status');
+        $table->id = 'searchsetup';
+        $table->attributes['class'] = 'admintable generaltable';
+        $table->data = array();
+
+        $return .= $brtag . get_string('searchsetupdescription', 'search') . $brtag . $brtag;
+
+        // Enable global search.
+        $row = array();
+        $url = new moodle_url("/admin/search.php?query=enableglobalsearch");
+        $row[0] = '1. ' . html_writer::tag('a', get_string('enableglobalsearch', 'admin'),
+                        array('href' => $url));
+        $status = html_writer::tag('span', get_string('no'), array('class' => 'statuscritical'));
+        if (\core_search\manager::is_global_search_enabled()) {
+            $status = html_writer::tag('span', get_string('yes'), array('class' => 'statusok'));
+        }
+        $row[1] = $status;
+        $table->data[] = $row;
+
+        // Select a search engine.
+        $row = array();
+        $url = new moodle_url('/admin/settings.php?section=manageglobalsearch#admin-searchengine');
+        $row[0] = '2. ' . html_writer::tag('a', get_string('selectsearchengine', 'admin'),
+                        array('href' => $url));
+
+        $status = html_writer::tag('span', get_string('no'), array('class' => 'statuscritical'));
+        if (!empty($CFG->searchengine)) {
+            $status = html_writer::tag('span', get_string('pluginname', 'search_' . $CFG->searchengine),
+                array('class' => 'statusok'));
+
+        }
+        $row[1] = $status;
+        $table->data[] = $row;
+
+        // Available areas.
+        $row = array();
+        $url = new moodle_url('/admin/settings.php?section=manageglobalsearch#id_s_mod_assign_enablesearch_activity');
+        $row[0] = '3. ' . html_writer::tag('a', get_string('enablesearchareas', 'admin'),
+                        array('href' => $url));
+
+        $status = html_writer::tag('span', get_string('no'), array('class' => 'statuscritical'));
+        if ($anyenabled) {
+            $status = html_writer::tag('span', get_string('yes'), array('class' => 'statusok'));
+
+        }
+        $row[1] = $status;
+        $table->data[] = $row;
+
+        // Setup search engine.
+        $row = array();
+        if (empty($CFG->searchengine)) {
+            $row[0] = '4. ' . get_string('setupsearchengine', 'admin');
+            $row[1] = html_writer::tag('span', get_string('no'), array('class' => 'statuscritical'));
+        } else {
+            $url = new moodle_url('/admin/settings.php?section=search' . $CFG->searchengine);
+            $row[0] = '4. ' . html_writer::tag('a', get_string('setupsearchengine', 'admin'),
+                            array('href' => $url));
+            // Check the engine status.
+            $searchengine = \core_search\manager::search_engine_instance();
+            $serverstatus = $searchengine->is_server_ready();
+            if ($serverstatus === true) {
+                $status = html_writer::tag('span', get_string('yes'), array('class' => 'statusok'));
+            } else {
+                $status = html_writer::tag('span', $serverstatus, array('class' => 'statuscritical'));
+            }
+            $row[1] = $status;
+        }
+        $table->data[] = $row;
+
+        // Indexed data.
+        $row = array();
+        $url = new moodle_url('/report/search/index.php#searchindexform');
+        $row[0] = '5. ' . html_writer::tag('a', get_string('indexdata', 'admin'), array('href' => $url));
+        if ($anyindexed) {
+            $status = html_writer::tag('span', get_string('yes'), array('class' => 'statusok'));
+        } else {
+            $status = html_writer::tag('span', get_string('no'), array('class' => 'statuscritical'));
+        }
+        $row[1] = $status;
+        $table->data[] = $row;
+
+        $return .= html_writer::table($table);
+
+        return highlight($query, $return);
+    }
+
+}