Skip to content

Commit

Permalink
global search MDL-25009 indexing failures in each module are now hand…
Browse files Browse the repository at this point in the history 
…led so that indexing can proceed to the next module.
  • Loading branch information
@nebgor
nebgor committed Nov 3, 2010
1 parent 737a1ef commit 7ee2741
Show file tree
Hide file tree
Showing 3 changed files with 77 additions and 35 deletions.
42 changes: 32 additions & 10 deletions search/add.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,7 @@
}
$dbcontrol = new IndexDBControl();
$addition_count = 0;
$startindextime = time();

$indexdate = @$CFG->search_indexer_run_date;
$mainstartindextime = time();

mtrace('Starting index update (additions)...');
mtrace('Index size before: '.$CFG->search_index_size."\n");
Expand All @@ -74,6 +72,14 @@
/// append virtual modules onto array

foreach ($mods as $mod) {

$indexdate = 0;
$indexdatestring = 'search_indexer_run_date_'.$mod->name;
$startrundate = time();
if (isset($CFG->$indexdatestring)) {
$indexdate = $CFG->$indexdatestring;
}

//build include file and function names
$class_file = $CFG->dirroot.'/search/documents/'.$mod->name.'_document.php';
$db_names_function = $mod->name.'_db_names';
Expand Down Expand Up @@ -138,21 +144,37 @@
// foreach document, add it to the index and database table
foreach ($additions as $add) {
++$addition_count;
// try the addDocument() so possible dml_write_exception don't block other modules running.
// also we can list all the new documents that are failing.
try {
// object to insert into db
$dbid = $dbcontrol->addDocument($add);

// synchronise db with index
$add->addField(Zend_Search_Lucene_Field::Keyword('dbid', $dbid));

// object to insert into db
$dbid = $dbcontrol->addDocument($add);
$index->addDocument($add);

// synchronise db with index
$add->addField(Zend_Search_Lucene_Field::Keyword('dbid', $dbid));
mtrace(" Add: $add->title (database id = $add->dbid, moodle instance id = $add->docid)");
}

mtrace(" Add: $add->title (database id = $add->dbid, moodle instance id = $add->docid)");
catch (dml_write_exception $e) {
mtrace(" Add: FAILED adding '$add->title' , moodle instance id = $add->docid , Error: $e->error ");
mtrace($e);
}

$index->addDocument($add);
}
}
else{
mtrace("No types to add.\n");
}

//commit changes
$index->commit();

//update index date
set_config($indexdatestring, $startrundate);

mtrace("Finished $mod->name.\n");
}
}
Expand All @@ -165,7 +187,7 @@

/// update index date and size

set_config('search_indexer_run_date', $startindextime);
set_config('search_indexer_run_date', $mainstartindextime);
set_config('search_index_size', (int)$CFG->search_index_size + (int)$addition_count);

/// print some additional info
Expand Down
13 changes: 6 additions & 7 deletions search/indexer.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,12 @@

if ($searchables){
foreach ($searchables as $mod) {

//mark last update times for mods to now.
$indexdatestring = 'search_indexer_update_date_'.$mod->name;
set_config($indexdatestring, time());
$indexdatestring = 'search_indexer_run_date_'.$mod->name;
set_config($indexdatestring, time());

mtrace("starting indexing {$mod->name}\n");

Expand Down Expand Up @@ -211,13 +217,6 @@

set_config('search_indexer_run_date', time());

//mark last update times for mods to now.
if ($mods = search_collect_searchables(false, true)){
foreach($mods as $mod) {
$indexdatestring = 'search_indexer_update_date_'.$mod->name;
set_config($indexdatestring, time());
}
}
/// and the index size

set_config('search_index_size', (int)$index->count());
Expand Down
57 changes: 39 additions & 18 deletions search/update.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,29 +128,50 @@

foreach ($updates as $update) {
++$update_count;

//delete old document
$added_doc = false;

//get old document for deletion later
// change from default text only search to include numerals for this search.
Zend_Search_Lucene_Analysis_Analyzer::setDefault(new Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum_CaseInsensitive());
$doc = $index->find("+docid:{$update->id} +doctype:{$mod->name} +itemtype:{$update->itemtype}");

//get the record, should only be one
foreach ($doc as $thisdoc) {
mtrace(" Delete: $thisdoc->title (database id = $thisdoc->dbid, index id = $thisdoc->id, moodle instance id = $thisdoc->docid)");
$dbcontrol->delDocument($thisdoc);
$index->delete($thisdoc->id);
}

//add new modified document back into index
$add = $get_document_function($update->id, $update->itemtype);

//object to insert into db
$dbid = $dbcontrol->addDocument($add);
try {
//add new modified document back into index
$add = $get_document_function($update->id, $update->itemtype);

//object to insert into db
$dbid = $dbcontrol->addDocument($add);

//synchronise db with index
$add->addField(Zend_Search_Lucene_Field::Keyword('dbid', $dbid));
mtrace(" Add: $add->title (database id = $add->dbid, moodle instance id = $add->docid)");
$index->addDocument($add);
$added_doc = true;
}

catch (dml_write_exception $e) {
mtrace(" Add: FAILED adding '$add->title' , moodle instance id = $add->docid , Error: $e->error ");
mtrace($e);
$added_doc = false;
}

//synchronise db with index
$add->addField(Zend_Search_Lucene_Field::Keyword('dbid', $dbid));
mtrace(" Add: $add->title (database id = $add->dbid, moodle instance id = $add->docid)");
$index->addDocument($add);
if ($added_doc) {
// ok we've successfully added the new document so far
// delete single previous old document
try {
//get the record, should only be one
foreach ($doc as $thisdoc) {
mtrace(" Delete: $thisdoc->title (database id = $thisdoc->dbid, index id = $thisdoc->id, moodle instance id = $thisdoc->docid)");
$dbcontrol->delDocument($thisdoc);
$index->delete($thisdoc->id);
}
}

catch (dml_write_exception $e) {
mtrace(" Delete: FAILED deleting '$thisdoc->title' , moodle instance id = $thisdoc->docid , Error: $e->error ");
mtrace($e);
}
}
}
}
else{
Expand Down

0 comments on commit 7ee2741

Please sign in to comment.