SERVER-16749: handle per index exceptions and continue

Author: scotthernandez

jstests/noPassthrough/ttl_capped.js

@@ -0,0 +1,53 @@
+/**
+ * Test that a capped collection ttl index doesn't cause the server to shut down, 
+ * nor stop processing collections in the db
+ * TODO: Change this test to show that you can't create capped collection ttl index 
+ *       Will need to figure out how to test that failed index ttl processing doesn't block others
+ */
+(function() {
+    "use strict";
+    var baseDir = "jstests_ttl_capped";
+    var port = allocatePorts( 1 )[ 0 ];
+    var dbpath = MongoRunner.dataPath + baseDir + "/";
+    
+    var m = MongoRunner.runMongod({
+                                    dbpath: dbpath,
+                                    port: port,
+                                    setParameter:"ttlMonitorSleepSecs=1"});
+    var db = m.getDB( "test" );
+    
+    // Make sure we have collections before and after the capped one, so we can check they work.
+    var bc = db.ttl_before;
+    var t = db.ttl_capped;
+    var ac = db.ttl_zafter;
+    
+    t.drop();
+    bc.drop();
+    ac.drop();
+    var dt = (new Date()).getTime();
+    jsTest.log("using date on inserted docs: " + tojson(new Date(dt)));
+    
+    // increase logging
+    assert.commandWorked(db.adminCommand({setParameter:1, logLevel:1}));
+
+    bc.ensureIndex( { x : 1 } , { expireAfterSeconds : -1 } );
+    assert.commandWorked(t.runCommand( "create", { capped:true, size:100, maxCount:100} ));
+    t.ensureIndex( { x : 1 } , { expireAfterSeconds : -1 } );
+    ac.ensureIndex( { x : 1 } , { expireAfterSeconds : -1 } );
+    
+    assert.writeOK(bc.insert({x: new Date(dt)}));
+    assert.writeOK(t.insert({x: new Date(dt)}));
+    assert.writeOK(ac.insert({x: new Date(dt)}));
+    
+    assert.eq(bc.count(), 1);
+    assert.eq(t.count(), 1);
+    assert.eq(ac.count(), 1);
+
+    sleep(1.1 * 1000); // 2 second sleep
+    jsTest.log("TTL work should be done.")
+    assert.eq(bc.count(), 0);
+    assert.eq(t.count(), 1);
+    assert.eq(ac.count(), 0);
+    
+    MongoRunner.stopMongod(port);
+})();

src/mongo/db/ttl.cpp

@@ -69,6 +69,7 @@ namespace mongo {
     ServerStatusMetricField<Counter64> ttlDeletedDocumentsDisplay("ttl.deletedDocuments", &ttlDeletedDocuments);
 
     MONGO_EXPORT_SERVER_PARAMETER( ttlMonitorEnabled, bool, true );
+    MONGO_EXPORT_SERVER_PARAMETER( ttlMonitorSleepSecs, int, 60 ); //used for testing
 
     class TTLMonitor : public BackgroundJob {
     public:
@@ -84,7 +85,7 @@ namespace mongo {
             cc().getAuthorizationSession()->grantInternalAuthorization();
 
             while ( ! inShutdown() ) {
-                sleepsecs( 60 );
+                sleepsecs( ttlMonitorSleepSecs );
 
                 LOG(3) << "TTLMonitor thread awake" << endl;
 
@@ -123,8 +124,16 @@ namespace mongo {
                     for ( vector<BSONObj>::const_iterator it = indexes.begin();
                           it != indexes.end(); ++it ) {
 
-                        if ( !doTTLForIndex( &txn, db, *it ) ) {
-                            break;  // stop processing TTL indexes on this database
+                        BSONObj idx = *it;
+                        try {
+                            if ( !doTTLForIndex( &txn, db, idx ) ) {
+                                break;  // stop processing TTL indexes on this database
+                            }
+                        } catch (const DBException& dbex) {
+                            error() << "Error processing ttl index: " << idx
+                                    << " -- " << dbex.toString();
+                            // continue on to the next index
+                            continue;
                         }
                     }
                 }
@@ -191,6 +200,7 @@ namespace mongo {
          */
         bool doTTLForIndex( OperationContext* txn, const string& dbName, const BSONObj& idx ) {
             BSONObj key = idx["key"].Obj();
+            const string ns = idx["ns"].String();
             if ( key.nFields() != 1 ) {
                 error() << "key for ttl index can only have 1 field" << endl;
                 return true;
@@ -210,13 +220,11 @@ namespace mongo {
                 query = BSON( key.firstElement().fieldName() << b.obj() );
             }
 
-            LOG(1) << "TTL: " << key << " \t " << query << endl;
+            LOG(1) << "TTL -- ns: " << ns << "key:" << key << " query: " << query << endl;
 
             long long numDeleted = 0;
             int attempt = 1;
             while (1) {
-                const string ns = idx["ns"].String();
-
                 ScopedTransaction scopedXact(txn, MODE_IX);
                 AutoGetDb autoDb(txn, dbName, MODE_IX);
                 Database* db = autoDb.getDb();