SERVER-13635: hook up mmap_v1 to SortedDataInterface test harness and…

… fix mmap_v1

src/mongo/db/diskloc.h

@@ -179,7 +179,7 @@ namespace mongo {
     // Maximum allowed DiskLoc.  Note that only three bytes are used to represent the file number
     // for consistency with the v1 index DiskLoc storage format, which uses only 7 bytes total.
     // No Record may begin at this location because the minimum size of a Record is larger than one
-    // byte.
-    const DiskLoc maxDiskLoc(0x00ffffff, 0x7fffffff);
+    // byte.  Also, the last bit is not able to be used because mmapv1 uses that for "used".
+    const DiskLoc maxDiskLoc(0x00ffffff, 0x7ffffffe);
 
 } // namespace mongo

src/mongo/db/operation_context.h

@@ -140,20 +140,25 @@ namespace mongo {
     public:
         WriteUnitOfWork(OperationContext* txn)
                  : _txn(txn) {
-            _txn->lockState()->beginWriteUnitOfWork();
+            if ( _txn->lockState() ) {
+                _txn->lockState()->beginWriteUnitOfWork();
+            }
             _txn->recoveryUnit()->beginUnitOfWork();
         }
 
         ~WriteUnitOfWork() {
             _txn->recoveryUnit()->endUnitOfWork();
-            _txn->lockState()->endWriteUnitOfWork();
+            if ( _txn->lockState() ) {
+                _txn->lockState()->endWriteUnitOfWork();
+            }
         }
 
         void commit() {
             _txn->recoveryUnit()->commitUnitOfWork();
-
-            _txn->lockState()->endWriteUnitOfWork();
-            _txn->lockState()->beginWriteUnitOfWork();
+            if ( _txn->lockState() ) {
+                _txn->lockState()->endWriteUnitOfWork();
+                _txn->lockState()->beginWriteUnitOfWork();
+            }
         }
 
     private:

src/mongo/db/storage/heap1/heap1_btree_impl.cpp

@@ -224,15 +224,17 @@ namespace {
             }
 
             virtual bool locate(const BSONObj& keyRaw, const DiskLoc& loc) {
-                // An empty key means we should seek to the front
-                if (keyRaw.isEmpty()) {
-                    _it = _data.begin();
+                const BSONObj key = stripFieldNames(keyRaw);
+                _it = _data.lower_bound(IndexKeyEntry(key, loc)); // lower_bound is >= key
+                if ( _it == _data.end() ) {
                     return false;
                 }
 
-                const BSONObj key = stripFieldNames(keyRaw);
-                _it = _data.lower_bound(IndexKeyEntry(key, loc)); // lower_bound is >= key
-                return _it != _data.end() && (_it->key == key); // intentionally not comparing loc
+                if ( _it->key != key ) {
+                    return false;
+                }
+
+                return _it->loc == loc;
             }
 
             virtual void customLocate(const BSONObj& keyBegin,
@@ -331,16 +333,19 @@ namespace {
             }
 
             virtual bool locate(const BSONObj& keyRaw, const DiskLoc& loc) {
-                // An empty key means we should seek to the seek to the end, 
-                // i.e. one past the lowest key in the iterator
-                if (keyRaw.isEmpty()) {
-                    _it = _data.rbegin();
+                const BSONObj key = stripFieldNames(keyRaw);
+                _it = lower_bound(IndexKeyEntry(key, loc)); // lower_bound is <= query
+
+                if ( _it == _data.rend() ) {
                     return false;
                 }
 
-                const BSONObj key = stripFieldNames(keyRaw);
-                _it = lower_bound(IndexKeyEntry(key, loc)); // lower_bound is <= query
-                return _it != _data.rend() && (_it->key == key); // intentionally not comparing loc
+
+                if ( _it->key != key ) {
+                    return false;
+                }
+
+                return _it->loc == loc;
             }
 
             virtual void customLocate(const BSONObj& keyBegin,

src/mongo/db/storage/heap1/heap1_recovery_unit.cpp

@@ -35,46 +35,48 @@
 namespace mongo {
 
     Heap1RecoveryUnit::~Heap1RecoveryUnit() {
-        invariant( _depth == 0 );
+        invariant( _frames.empty() );
     }
 
     void Heap1RecoveryUnit::beginUnitOfWork() {
-        _depth++;
+        _frames.push_back( Frame() );
     }
 
     void Heap1RecoveryUnit::commitUnitOfWork() {
-        if ( _depth == 1 ) {
+        if ( _frames.size() == 1 ) {
             _rollbackPossible = true;
-            _indexInserts.clear();
-            _indexRemoves.clear();
         }
+        else {
+            size_t last = _frames.size() - 1;
+            size_t next = last - 1;
+            _frames[next].indexMods.insert( _frames[next].indexMods.end(),
+                                            _frames[last].indexMods.begin(),
+                                            _frames[last].indexMods.end() );
+        }
+        _frames.back().indexMods.clear();
     }
 
     void Heap1RecoveryUnit::endUnitOfWork() {
-        _depth--;
-
-        // effectively do a rollback
 
-        invariant( _rollbackPossible );
-
-        for ( size_t i = 0; i < _indexInserts.size(); i++ ) {
-            invariant( _depth == 0 ); // todo: fix me
-            SortedDataInterface* idx = _indexInserts[i].idx;
-            idx->unindex( NULL, _indexInserts[i].obj, _indexInserts[i].loc );
-        }
+        // invariant( _rollbackPossible ); // todo
 
-        for ( size_t i = 0; i < _indexRemoves.size(); i++ ) {
-            invariant( _depth == 0 ); // todo: fix me
-            SortedDataInterface* idx = _indexRemoves[i].idx;
-            idx->insert( NULL, _indexRemoves[i].obj, _indexRemoves[i].loc, true );
+        const Frame& frame = _frames.back();
+        for ( size_t i = frame.indexMods.size() ; i > 0; i-- ) {
+            const IndexInfo& ii = frame.indexMods[i-1];
+            SortedDataInterface* idx = ii.idx;
+            if ( ii.insert )
+                idx->unindex( NULL, ii.obj, ii.loc );
+            else
+                idx->insert( NULL, ii.obj, ii.loc, true );
         }
 
+        _frames.pop_back();
     }
 
-    void Heap1RecoveryUnit::notifyIndexInsert( SortedDataInterface* idx,
-                                               const BSONObj& obj, const DiskLoc& loc ) {
-        IndexInfo ii = { idx, obj, loc };
-        _indexInserts.push_back( ii );
+    void Heap1RecoveryUnit::notifyIndexMod( SortedDataInterface* idx,
+                                            const BSONObj& obj, const DiskLoc& loc, bool insert ) {
+        IndexInfo ii = { idx, obj, loc, insert };
+        _frames.back().indexMods.push_back( ii );
     }
 
     // static
@@ -84,13 +86,7 @@ namespace mongo {
             return;
 
         Heap1RecoveryUnit* ru = dynamic_cast<Heap1RecoveryUnit*>( ctx->recoveryUnit() );
-        ru->notifyIndexInsert( idx, obj, loc );
-    }
-
-    void Heap1RecoveryUnit::notifyIndexRemove( SortedDataInterface* idx,
-                                               const BSONObj& obj, const DiskLoc& loc ) {
-        IndexInfo ii = { idx, obj, loc };
-        _indexRemoves.push_back( ii );
+        ru->notifyIndexMod( idx, obj, loc, true );
     }
 
     // static
@@ -100,7 +96,7 @@ namespace mongo {
             return;
 
         Heap1RecoveryUnit* ru = dynamic_cast<Heap1RecoveryUnit*>( ctx->recoveryUnit() );
-        ru->notifyIndexRemove( idx, obj, loc );
+        ru->notifyIndexMod( idx, obj, loc, false );
     }
 
 

src/mongo/db/storage/heap1/heap1_recovery_unit.h

@@ -42,7 +42,6 @@ namespace mongo {
     class Heap1RecoveryUnit : public RecoveryUnit {
     public:
         Heap1RecoveryUnit() {
-            _depth = 0;
             _rollbackPossible = true;
         }
 
@@ -69,25 +68,29 @@ namespace mongo {
 
         void rollbackImpossible() { _rollbackPossible = false; }
 
-        void notifyIndexInsert( SortedDataInterface* idx, const BSONObj& obj, const DiskLoc& loc );
+        void notifyIndexMod( SortedDataInterface* idx,
+                             const BSONObj& obj, const DiskLoc& loc, bool insert );
+
         static void notifyIndexInsert( OperationContext* ctx, SortedDataInterface* idx,
                                        const BSONObj& obj, const DiskLoc& loc );
-
-        void notifyIndexRemove( SortedDataInterface* idx, const BSONObj& obj, const DiskLoc& loc );
         static void notifyIndexRemove( OperationContext* ctx, SortedDataInterface* idx,
                                        const BSONObj& obj, const DiskLoc& loc );
 
     private:
-        int _depth;
         bool _rollbackPossible;
 
         struct IndexInfo {
             SortedDataInterface* idx;
             BSONObj obj;
             DiskLoc loc;
+            bool insert;
         };
-        std::vector<IndexInfo> _indexInserts;
-        std::vector<IndexInfo> _indexRemoves;
+
+        struct Frame {
+            std::vector<IndexInfo> indexMods;
+        };
+
+        std::vector<Frame> _frames;
     };
 
 }

src/mongo/db/storage/heap1/record_store_heap.cpp

@@ -69,6 +69,9 @@ namespace mongo {
 
     HeapRecordStore::HeapRecord* HeapRecordStore::recordFor(const DiskLoc& loc) const {
         Records::const_iterator it = _records.find(loc);
+        if ( it == _records.end() ) {
+            error() << "HeapRecordStore::recordFor cannot find record for " << ns() << ":" << loc;
+        }
         invariant(it != _records.end());
         return reinterpret_cast<HeapRecord*>(it->second.get());
     }

src/mongo/db/storage/index_entry_comparison.cpp

@@ -57,7 +57,8 @@ namespace mongo {
 
         // Iterate through both BSONObjects, comparing individual elements one by one
         for (unsigned mask = 1; lhsIt.more(); mask <<= 1) {
-             invariant(rhsIt.more());
+            if (!rhsIt.more())
+                return _order.descending(mask) ? -1 : 1;
 
             const BSONElement l = lhsIt.next();
             const BSONElement r = rhsIt.next();
@@ -90,7 +91,8 @@ namespace mongo {
 
         }
 
-        invariant(!rhsIt.more());
+        if(rhsIt.more())
+            return -1;
 
         // This means just look at the key, not the loc.
         if (lhs.loc.isNull() || rhs.loc.isNull())

src/mongo/db/storage/mmap_v1/SConscript

@@ -125,3 +125,13 @@ env.CppUnitTest(
         ]
     )
 
+env.CppUnitTest(
+    target='btree_interface_test',
+    source=['btree/btree_interface_test.cpp'
+            ],
+    LIBDEPS=[
+        'btree_test_help',
+        '$BUILD_DIR/mongo/db/storage/sorted_data_interface_test_harness'
+        ]
+    )
+

src/mongo/db/storage/mmap_v1/btree/btree_interface_test.cpp

@@ -0,0 +1,72 @@
+// btree_interface_test.cpp
+
+/**
+ *    Copyright (C) 2014 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/db/storage/mmap_v1/btree/btree_interface.h"
+#include "mongo/db/storage/mmap_v1/btree/btree_test_help.h"
+#include "mongo/db/storage/sorted_data_interface_test_harness.h"
+#include "mongo/unittest/unittest.h"
+
+namespace mongo {
+
+    class MyHarnessHelper : public HarnessHelper {
+    public:
+        MyHarnessHelper()
+            : _recordStore( "a.b" ),
+              _order( Ordering::make( BSONObj() ) ) {
+        }
+
+        virtual SortedDataInterface* newSortedDataInterface() {
+            auto_ptr<SortedDataInterface> sorted( getMMAPV1Interface( &_headManager,
+                                                                      &_recordStore,
+                                                                      _order,
+                                                                      "a_1",
+                                                                      1,
+                                                                      &_deletionNotification ) );
+            OperationContextNoop op;
+            massertStatusOK( sorted->initAsEmpty( &op ) );
+            return sorted.release();
+        }
+
+        virtual RecoveryUnit* newRecoveryUnit() {
+            return new HeapRecordStoreBtreeRecoveryUnit();
+        }
+
+    private:
+        TestHeadManager _headManager;
+        HeapRecordStoreBtree _recordStore;
+        Ordering _order;
+        BucketDeletionNotification _deletionNotification;
+    };
+
+    HarnessHelper* newHarnessHelper() {
+        return new MyHarnessHelper();
+    }
+
+}

src/mongo/db/storage/mmap_v1/btree/btree_ondisk.cpp

@@ -37,6 +37,7 @@ namespace mongo {
 
     void DiskLoc56Bit::operator=(const DiskLoc& loc) {
         ofs = loc.getOfs();
+        invariant( (ofs & 0x1) == 0 ); // we use the last bit for used
         int la = loc.a();
         invariant( la <= 0xffffff ); // must fit in 3 bytes
         if( la < 0 ) {