Skip to content

Commit

Permalink
Merge branch 'MDL-53325-master' of git://github.com/merrill-oakland/m…
Browse files Browse the repository at this point in the history 
…oodle
  • Loading branch information
@andrewnicols
andrewnicols committed Mar 9, 2016
2 parents 020d27a + 075fa91 commit bd2e5ef
Show file tree
Hide file tree
Showing 9 changed files with 174 additions and 43 deletions.
3 changes: 2 additions & 1 deletion lang/en/admin.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1028,7 +1028,8 @@
$string['taskdeleteunconfirmedusers'] = 'Delete unconfirmed users';
$string['taskeventscron'] = 'Background processing for events';
$string['taskfiletrashcleanup'] = 'Cleanup files in trash';
$string['taskglobalsearch'] = 'Global search indexing';
$string['taskglobalsearchindex'] = 'Global search indexing';
$string['taskglobalsearchoptimize'] = 'Global search index optimization';
$string['taskgradecron'] = 'Background processing for gradebook';
$string['tasklegacycron'] = 'Legacy cron processing for plugins';
$string['taskmessagingcleanup'] = 'Background processing for messaging';
Expand Down
7 changes: 2 additions & 5 deletions lib/classes/task/search_task.php → lib/classes/task/search_index_task.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,15 @@
* @copyright 2015 David Monllao {@link http://www.davidmonllao.com}
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class search_task extends scheduled_task {
class search_index_task extends scheduled_task {

/**
* Get a descriptive name for this task (shown to admins).
*
* @return string
*/
public function get_name() {
return get_string('taskglobalsearch', 'admin');
return get_string('taskglobalsearchindex', 'admin');
}

/**
Expand All @@ -53,8 +53,5 @@ public function execute() {

// Indexing database records for modules + rich documents of forum.
$globalsearch->index();

// Optimize index at last.
$globalsearch->optimize_index();
}
}
61 changes: 61 additions & 0 deletions lib/classes/task/search_optimize_task.php
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.

/**
* A scheduled task for global search.
*
* @package core
* @copyright 2016 Eric Merrill {@link https://www.merrilldigital.com}
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/

namespace core\task;

defined('MOODLE_INTERNAL') || die();

/**
* Runs search index optimization.
*
* @package core
* @copyright 2016 Eric Merrill {@link https://www.merrilldigital.com}
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class search_optimize_task extends scheduled_task {

/**
* Get a descriptive name for this task (shown to admins).
*
* @return string
*/
public function get_name() {
return get_string('taskglobalsearchoptimize', 'admin');
}

/**
* Do the job.
* Throw exceptions on errors (the job will be retried).
*/
public function execute() {
if (!\core_search\manager::is_global_search_enabled()) {
return;
}

$globalsearch = \core_search\manager::instance();

// Optimize index at last.
$globalsearch->optimize_index();
}
}
11 changes: 10 additions & 1 deletion lib/db/tasks.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -285,14 +285,23 @@
'month' => '*'
),
array(
'classname' => 'core\task\search_task',
'classname' => 'core\task\search_index_task',
'blocking' => 0,
'minute' => '*/30',
'hour' => '*',
'day' => '*',
'dayofweek' => '*',
'month' => '*'
),
array(
'classname' => 'core\task\search_optimize_task',
'blocking' => 0,
'minute' => '15',
'hour' => '*/12',
'day' => '*',
'dayofweek' => '*',
'month' => '*'
),
array(
'classname' => 'core\task\stats_cron_task',
'blocking' => 0,
Expand Down
57 changes: 50 additions & 7 deletions search/classes/engine.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,56 @@ public function get_document_classname() {
return $classname;
}

/**
* Run any pre-indexing operations.
*
* Should be overwritten if the search engine needs to do any pre index preparation.
*
* @param bool $fullindex True if a full index will be performed
* @return void
*/
public function index_starting($fullindex = false) {
// Nothing by default.
}

/**
* Run any post indexing operations.
*
* Should be overwritten if the search engine needs to do any post index cleanup.
*
* @param int $numdocs The number of documents that were added to the index
* @param bool $fullindex True if a full index was performed
* @return void
*/
public function index_complete($numdocs = 0, $fullindex = false) {
// Nothing by default.
}

/**
* Do anything that may need to be done before an area is indexed.
*
* @param \core_search\area\base $searcharea The search area that was complete
* @param bool $fullindex True if a full index is being performed
* @return void
*/
public function area_index_starting($searcharea, $fullindex = false) {
// Nothing by default.
}

/**
* Do any area cleanup needed, and do anything to confirm contents.
*
* Return false to prevent the search area completed time and stats from being updated.
*
* @param \core_search\area\base $searcharea The search area that was complete
* @param int $numdocs The number of documents that were added to the index
* @param bool $fullindex True if a full index is being performed
* @return bool True means that data is considered indexed
*/
public function area_index_complete($searcharea, $numdocs = 0, $fullindex = false) {
return true;
}

/**
* Optimizes the search engine.
*
Expand Down Expand Up @@ -289,13 +339,6 @@ abstract function is_server_ready();
*/
abstract function add_document($doc);

/**
* Commits changes to the server.
*
* @return void
*/
abstract function commit();

/**
* Executes the query on the engine.
*
Expand Down
52 changes: 29 additions & 23 deletions search/classes/manager.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -466,7 +466,10 @@ public function index($fullindex = false) {
// Unlimited time.
\core_php_time_limit::raise();

$anyupdate = false;
// Notify the engine that an index starting.
$this->engine->index_starting($fullindex);

$sumdocs = 0;

$searchareas = $this->get_search_areas_list(true);
foreach ($searchareas as $areaid => $searcharea) {
Expand All @@ -475,6 +478,9 @@ public function index($fullindex = false) {
mtrace('Processing ' . $searcharea->get_visible_name() . ' area');
}

// Notify the engine that an area is starting.
$this->engine->area_index_starting($searcharea, $fullindex);

$indexingstart = time();

// This is used to store this component config.
Expand Down Expand Up @@ -518,38 +524,40 @@ public function index($fullindex = false) {
$numrecords++;
}

if ($numdocs > 0) {
$anyupdate = true;

// Commit all remaining documents.
$this->engine->commit();

if (CLI_SCRIPT && !PHPUNIT_TEST) {
if (CLI_SCRIPT && !PHPUNIT_TEST) {
if ($numdocs > 0) {
mtrace('Processed ' . $numrecords . ' records containing ' . $numdocs . ' documents for ' .
$searcharea->get_visible_name() . ' area. Commits completed.');
$searcharea->get_visible_name() . ' area.');
} else {
mtrace('No new documents to index for ' . $searcharea->get_visible_name() . ' area.');
}
} else if (CLI_SCRIPT && !PHPUNIT_TEST) {
mtrace('No new documents to index for ' . $searcharea->get_visible_name() . ' area.');
}

// Store last index run once documents have been commited to the search engine.
set_config($varname . '_indexingstart', $indexingstart, $componentconfigname);
set_config($varname . '_indexingend', time(), $componentconfigname);
set_config($varname . '_docsignored', $numdocsignored, $componentconfigname);
set_config($varname . '_docsprocessed', $numdocs, $componentconfigname);
set_config($varname . '_recordsprocessed', $numrecords, $componentconfigname);
if ($lastindexeddoc > 0) {
set_config($varname . '_lastindexrun', $lastindexeddoc, $componentconfigname);
// Notify the engine this area is complete, and only mark times if true.
if ($this->engine->area_index_complete($searcharea, $numdocs, $fullindex)) {
$sumdocs += $numdocs;

// Store last index run once documents have been commited to the search engine.
set_config($varname . '_indexingstart', $indexingstart, $componentconfigname);
set_config($varname . '_indexingend', time(), $componentconfigname);
set_config($varname . '_docsignored', $numdocsignored, $componentconfigname);
set_config($varname . '_docsprocessed', $numdocs, $componentconfigname);
set_config($varname . '_recordsprocessed', $numrecords, $componentconfigname);
if ($lastindexeddoc > 0) {
set_config($varname . '_lastindexrun', $lastindexeddoc, $componentconfigname);
}
}
}

if ($anyupdate) {
if ($sumdocs > 0) {
$event = \core\event\search_indexed::create(
array('context' => \context_system::instance()));
$event->trigger();
}

return $anyupdate;
$this->engine->index_complete($sumdocs, $fullindex);

return (bool)$sumdocs;
}

/**
Expand Down Expand Up @@ -598,7 +606,6 @@ public function delete_index($areaid = false) {
$this->engine->delete();
$this->reset_config();
}
$this->engine->commit();
}

/**
Expand All @@ -608,7 +615,6 @@ public function delete_index($areaid = false) {
*/
public function delete_index_by_id($id) {
$this->engine->delete_by_id($id);
$this->engine->commit();
}

/**
Expand Down
20 changes: 19 additions & 1 deletion search/engine/solr/classes/engine.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -317,10 +317,26 @@ public function add_document($doc) {
*
* @return void
*/
public function commit() {
protected function commit() {
$this->get_search_client()->commit();
}

/**
* Do any area cleanup needed, and do anything to confirm contents.
*
* Return false to prevent the search area completed time and stats from being updated.
*
* @param \core_search\area\base $searcharea The search area that was complete
* @param int $numdocs The number of documents that were added to the index
* @param bool $fullindex True if a full index is being performed
* @return bool True means that data is considered indexed
*/
public function area_index_complete($searcharea, $numdocs = 0, $fullindex = false) {
$this->commit();

return true;
}

/**
* Defragments the index.
*
Expand All @@ -338,6 +354,7 @@ public function optimize() {
*/
public function delete_by_id($id) {
$this->get_search_client()->deleteById($id);
$this->commit();
}

/**
Expand All @@ -352,6 +369,7 @@ public function delete($areaid = null) {
} else {
$this->get_search_client()->deleteByQuery('*:*');
}
$this->commit();
}

/**
Expand Down
4 changes: 0 additions & 4 deletions search/tests/fixtures/mock_search_engine.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,6 @@ public function add_document($doc) {
// No need to implement.
}

public function commit() {
// No need to implement.
}

public function execute_query($data, $usercontexts) {
// No need to implement.
}
Expand Down
2 changes: 1 addition & 1 deletion version.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@

defined('MOODLE_INTERNAL') || die();

$version = 2016030400.00; // YYYYMMDD = weekly release date of this DEV branch.
$version = 2016030400.01; // YYYYMMDD = weekly release date of this DEV branch.
// RR = release increments - 00 in DEV branches.
// .XX = incremental changes.

Expand Down

0 comments on commit bd2e5ef

Please sign in to comment.