From 04de85da22d34e925d6b0691b0b2a0647b212543 Mon Sep 17 00:00:00 2001 From: arhag Date: Mon, 24 Sep 2018 12:47:28 -0400 Subject: [PATCH 1/7] add_index ensures that the undo stack and revision of the added index are consistent with the other indices in the database --- include/chainbase/chainbase.hpp | 48 ++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/include/chainbase/chainbase.hpp b/include/chainbase/chainbase.hpp index 53aeb97..bd4dd33 100644 --- a/include/chainbase/chainbase.hpp +++ b/include/chainbase/chainbase.hpp @@ -481,6 +481,18 @@ namespace chainbase { remove( *val ); } + std::pair undo_stack_revision_range()const { + int64_t begin = -1; + int64_t end = -1; + + if( _stack.size() > 0 ) { + begin = _stack.front().revision; + end = _stack.back().revision; + } + + return {begin, end}; + } + private: bool enabled()const { return _stack.size(); } @@ -582,6 +594,7 @@ namespace chainbase { virtual uint32_t type_id()const = 0; virtual uint64_t row_count()const = 0; virtual const std::string& type_name()const = 0; + virtual std::pair undo_stack_revision_range()const = 0; virtual void remove_object( int64_t id ) = 0; @@ -608,6 +621,7 @@ namespace chainbase { virtual uint32_t type_id()const override { return BaseIndex::value_type::type_id; } virtual uint64_t row_count()const override { return _base.indices().size(); } virtual const std::string& type_name() const override { return BaseIndex_name; } + virtual std::pair undo_stack_revision_range()const override { return _base.undo_stack_revision_range(); } virtual void remove_object( int64_t id ) override { return _base.remove_object( id ); } private: @@ -774,6 +788,39 @@ namespace chainbase { idx_ptr->validate(); + // Ensure the undo stack of added index is consistent with the other indices in the database + if( _index_list.size() > 0 ) { + auto expected_revision_range = _index_list.front()->undo_stack_revision_range(); + auto added_index_revision_range = idx_ptr->undo_stack_revision_range(); + + if( added_index_revision_range.first != expected_revision_range.first || + added_index_revision_range.second != expected_revision_range.second ) { + + if( added_index_revision_range.first >= 0 && idx_ptr->revision() != added_index_revision_range.second ) { + BOOST_THROW_EXCEPTION( std::logic_error( "index for " + type_name + " has an undo stack that is inconsistent with its revision" ) ); + } + + if( added_index_revision_range.second > expected_revision_range.second ) { + BOOST_THROW_EXCEPTION( std::logic_error( "cannot add index for " + type_name + " since it has later undo states than the already added indices in the database; change order in which indices are added" ) ); + } + + if( _read_only ) { + BOOST_THROW_EXCEPTION( std::logic_error( "index for " + type_name + " requires an undo stack that is consistent with other indices in the database; cannot fix in read-only mode" ) ); + } + + idx_ptr->commit( expected_revision_range.first ); + added_index_revision_range = idx_ptr->undo_stack_revision_range(); + + if( added_index_revision_range.first < 0 ) { + idx_ptr->set_revision( expected_revision_range.first ); + } + + while( idx_ptr->revision() < expected_revision_range.second ) { + idx_ptr->start_undo_session(true).push(); + } + } + } + if( type_id >= _index_map.size() ) _index_map.resize( type_id + 1 ); @@ -987,4 +1034,3 @@ namespace chainbase { template using shared_multi_index_container = boost::multi_index_container >; } // namepsace chainbase - From e4ab6df78a1f89bd4b5666f4f38941e8202afa60 Mon Sep 17 00:00:00 2001 From: arhag Date: Tue, 25 Sep 2018 14:44:07 -0400 Subject: [PATCH 2/7] make some methods const so they can be called using a const reference to database instance --- include/chainbase/chainbase.hpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/include/chainbase/chainbase.hpp b/include/chainbase/chainbase.hpp index 53aeb97..cc239c6 100644 --- a/include/chainbase/chainbase.hpp +++ b/include/chainbase/chainbase.hpp @@ -786,6 +786,10 @@ namespace chainbase { return _segment->get_segment_manager(); } + auto get_segment_manager()const -> std::add_const_t< decltype( ((bip::managed_mapped_file*)nullptr)->get_segment_manager() ) > { + return _segment->get_segment_manager(); + } + size_t get_free_memory()const { return _segment->get_segment_manager()->get_free_memory(); @@ -948,7 +952,7 @@ namespace chainbase { return callback(); } - database_index_row_count_multiset row_count_per_index() { + database_index_row_count_multiset row_count_per_index()const { database_index_row_count_multiset ret; for(const auto& ai_ptr : _index_map) { if(!ai_ptr) @@ -987,4 +991,3 @@ namespace chainbase { template using shared_multi_index_container = boost::multi_index_container >; } // namepsace chainbase - From 1395591be0b4309bc2452079310cdaade3ad4d5a Mon Sep 17 00:00:00 2001 From: arhag Date: Tue, 25 Sep 2018 14:49:48 -0400 Subject: [PATCH 3/7] Update README to reflect the fact that we are using C++14 features in chainbase code --- README.md | 51 +++++++++++++++++++++++++-------------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 2bd35e4..c239424 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# ChainBase - a fast version controlled, transactional database +# ChainBase - a fast version controlled, transactional database ChainBase is designed to meet the demanding requirments of blockchain applications, but is suitable for use in any application that requires a robust transactional database with the ability have near-infinate levels of undo @@ -7,16 +7,16 @@ While chainbase was designed for blockchain applications, it is suitable for any program that needs to persist complex application state with the ability to undo. -## Features +## Features - Supports multiple objects (tables) with multiple indicies (based upon boost::multi_index_container) - - State is persistant and sharable among multiple processes + - State is persistant and sharable among multiple processes - Nested Transactional Writes with ability to undo changes -## Dependencies - - - c++11 - - [Boost](http://www.boost.org/) +## Dependencies + + - C++14 + - [Boost](http://www.boost.org/) - CMake Build Process - Supports Linux, Mac OS X (no Windows Support) @@ -28,9 +28,9 @@ enum tables { }; /** - * Defines a "table" for storing books. This table is assigned a + * Defines a "table" for storing books. This table is assigned a * globally unique ID (book_table) and must inherit from chainbase::object<> which - * decorates the book type by defining "id_type" and "type_id" + * decorates the book type by defining "id_type" and "type_id" */ struct book : public chainbase::object { @@ -38,7 +38,7 @@ struct book : public chainbase::object { * members requiring dynamic memory allocation. */ CHAINBASE_DEFAULT_CONSTRUCTOR( book ) - + id_type id; ///< this manditory member is a primary key int pages = 0; int publish_date = 0; @@ -49,15 +49,15 @@ struct by_pages; struct by_date; /** - * This is a relatively standard boost multi_index_container definition that has three + * This is a relatively standard boost multi_index_container definition that has three * requirements to be used withn a chainbase database: - * - it must use chainbase::allocator + * - it must use chainbase::allocator * - the first index must be on the primary key (id) and must be unique (hashed or ordered) */ typedef multi_index_container< book, indexed_by< - ordered_unique< tag, member >, ///< required + ordered_unique< tag, member >, ///< required ordered_non_unique< tag, BOOST_MULTI_INDEX_MEMBER(book,int,pages) >, ordered_non_unique< tag, BOOST_MULTI_INDEX_MEMBER(book,int,publish_date) > >, @@ -71,7 +71,7 @@ typedef multi_index_container< int main( int argc, char** argv ) { chainbase::database db; db.open( "database_dir", database::read_write, 1024*1024*8 ); /// open or create a database with 8MB capacity - db.add_index< book_index >(); /// open or create the book_index + db.add_index< book_index >(); /// open or create the book_index const auto& book_idx = db.get_index().indicies(); @@ -89,7 +89,7 @@ int main( int argc, char** argv ) { /** You modify a book by passing in a lambda that receives a - non-const reference to the book you wish to modify. + non-const reference to the book you wish to modify. */ db.modify( new_book300, [&]( book& b ) { b.pages++; @@ -105,22 +105,22 @@ int main( int argc, char** argv ) { } db.remove( new_book400 ); - + return 0; } ``` -## Concurrent Access +## Concurrent Access -By default ChainBase provides no synchronization and has the same concurrency restrictions as any +By default ChainBase provides no synchronization and has the same concurrency restrictions as any boost::multi_index_container. This means that two or more threads may read the database at the same time, but all writes must be protected by a mutex. Multiple processes may open the same database if care is taken to use interpocess locking on the database. -## Persistance +## Persistance By default data is only flushed to disk upon request or when the program exits. So long as the program does not crash in the middle of a call to db.modify(), or db.create() the content of the @@ -134,25 +134,24 @@ ChainBase was designed to be used with blockchain applications where an append-o to secure state in the event of power loss. This block log can be replayed to regenerate the full database state. Dealing with OS crashes, loss of power, and logs, is beyond the scope of ChainBase. -## Portability +## Portability The contents of the database file is dependent upon the memory layout of the computer and process that created the database. Moving the database to a machine that uses a different compiler, operating system, libraries, or build type (release vs debug) will result in undefined behavior. -If portability is desired, the developer will have to export the database to a suitable format. +If portability is desired, the developer will have to export the database to a suitable format. -## Background +## Background -Blockchain applications depend upon a high performance database capable of millions of read/write +Blockchain applications depend upon a high performance database capable of millions of read/write operations per second. Additionally blockchains operate on the basis of "eventually consistant" which means that any changes made to the database are potentially reversible for an unknown amount of time depending -upon the consenus protocol used. +upon the consenus protocol used. Existing database such as [libbitcoin Database](https://github.com/libbitcoin/libbitcoin-database) achieve high peformance using similar techniques (memory mapped files), but they are heavily specialised and do not implement -the logic necessary for multiple indicies or undo history. +the logic necessary for multiple indicies or undo history. Databases such as LevelDB provide a simple Key/Value database, but suffer from poor performance relative to memory mapped file implementations. - From 7156ca2e00e35b7dbd4d9156087c21beb655dbde Mon Sep 17 00:00:00 2001 From: arhag Date: Tue, 25 Sep 2018 17:27:32 -0400 Subject: [PATCH 4/7] add_index only modifies undo stack of brand new index --- include/chainbase/chainbase.hpp | 66 +++++++++++++++------------------ 1 file changed, 30 insertions(+), 36 deletions(-) diff --git a/include/chainbase/chainbase.hpp b/include/chainbase/chainbase.hpp index f57e0fa..68db004 100644 --- a/include/chainbase/chainbase.hpp +++ b/include/chainbase/chainbase.hpp @@ -768,25 +768,27 @@ namespace chainbase { template void add_index() { - const uint16_t type_id = generic_index::value_type::type_id; - typedef generic_index index_type; - typedef typename index_type::allocator_type index_alloc; + const uint16_t type_id = generic_index::value_type::type_id; + typedef generic_index index_type; + typedef typename index_type::allocator_type index_alloc; - std::string type_name = boost::core::demangle( typeid( typename index_type::value_type ).name() ); + std::string type_name = boost::core::demangle( typeid( typename index_type::value_type ).name() ); - if( !( _index_map.size() <= type_id || _index_map[ type_id ] == nullptr ) ) { - BOOST_THROW_EXCEPTION( std::logic_error( type_name + "::type_id is already in use" ) ); - } + if( !( _index_map.size() <= type_id || _index_map[ type_id ] == nullptr ) ) { + BOOST_THROW_EXCEPTION( std::logic_error( type_name + "::type_id is already in use" ) ); + } - index_type* idx_ptr = nullptr; - if( !_read_only ) { - idx_ptr = _segment->find_or_construct< index_type >( type_name.c_str() )( index_alloc( _segment->get_segment_manager() ) ); - } else { - idx_ptr = _segment->find< index_type >( type_name.c_str() ).first; - if( !idx_ptr ) BOOST_THROW_EXCEPTION( std::runtime_error( "unable to find index for " + type_name + " in read only database" ) ); + index_type* idx_ptr = _segment->find< index_type >( type_name.c_str() ).first; + bool first_time_adding = false; + if( !idx_ptr ) { + if( _read_only ) { + BOOST_THROW_EXCEPTION( std::runtime_error( "unable to find index for " + type_name + " in read only database" ) ); + } + first_time_adding = true; + idx_ptr = _segment->construct< index_type >( type_name.c_str() )( index_alloc( _segment->get_segment_manager() ) ); } - idx_ptr->validate(); + idx_ptr->validate(); // Ensure the undo stack of added index is consistent with the other indices in the database if( _index_list.size() > 0 ) { @@ -796,37 +798,29 @@ namespace chainbase { if( added_index_revision_range.first != expected_revision_range.first || added_index_revision_range.second != expected_revision_range.second ) { - if( added_index_revision_range.first >= 0 && idx_ptr->revision() != added_index_revision_range.second ) { - BOOST_THROW_EXCEPTION( std::logic_error( "index for " + type_name + " has an undo stack that is inconsistent with its revision" ) ); - } - - if( added_index_revision_range.second > expected_revision_range.second ) { - BOOST_THROW_EXCEPTION( std::logic_error( "cannot add index for " + type_name + " since it has later undo states than the already added indices in the database; change order in which indices are added" ) ); - } - - if( _read_only ) { - BOOST_THROW_EXCEPTION( std::logic_error( "index for " + type_name + " requires an undo stack that is consistent with other indices in the database; cannot fix in read-only mode" ) ); - } - - idx_ptr->commit( expected_revision_range.first ); - added_index_revision_range = idx_ptr->undo_stack_revision_range(); - - if( added_index_revision_range.first < 0 ) { - idx_ptr->set_revision( expected_revision_range.first ); + if( !first_time_adding ) { + BOOST_THROW_EXCEPTION( std::logic_error( + "existing index for " + type_name + " has an undo stack (revision range [" + + std::to_string(added_index_revision_range.first) + ", " + std::to_string(added_index_revision_range.second) + + "]) that is inconsistent with other indices in the database (revision range [" + + std::to_string(expected_revision_range.first) + ", " + std::to_string(expected_revision_range.second) + + "]); corrupted database?" + ) ); } + idx_ptr->set_revision( expected_revision_range.first ); while( idx_ptr->revision() < expected_revision_range.second ) { idx_ptr->start_undo_session(true).push(); } } } - if( type_id >= _index_map.size() ) - _index_map.resize( type_id + 1 ); + if( type_id >= _index_map.size() ) + _index_map.resize( type_id + 1 ); - auto new_index = new index( *idx_ptr ); - _index_map[ type_id ].reset( new_index ); - _index_list.push_back( new_index ); + auto new_index = new index( *idx_ptr ); + _index_map[ type_id ].reset( new_index ); + _index_list.push_back( new_index ); } auto get_segment_manager() -> decltype( ((bip::managed_mapped_file*)nullptr)->get_segment_manager()) { From b4446ddaa9fb8e94eb21874fbbcf7c0f19faa29b Mon Sep 17 00:00:00 2001 From: arhag Date: Tue, 25 Sep 2018 17:31:17 -0400 Subject: [PATCH 5/7] make sure database is not in read-only mode before attempting to adjust undo stack of newly added index --- include/chainbase/chainbase.hpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/include/chainbase/chainbase.hpp b/include/chainbase/chainbase.hpp index 68db004..e4774b3 100644 --- a/include/chainbase/chainbase.hpp +++ b/include/chainbase/chainbase.hpp @@ -808,6 +808,13 @@ namespace chainbase { ) ); } + if( _read_only ) { + BOOST_THROW_EXCEPTION( std::logic_error( + "new index for " + type_name + + " requires an undo stack that is consistent with other indices in the database; cannot fix in read-only mode" + ) ); + } + idx_ptr->set_revision( expected_revision_range.first ); while( idx_ptr->revision() < expected_revision_range.second ) { idx_ptr->start_undo_session(true).push(); From 7efc817ccb82557fc8b3345f11db0564261ba66a Mon Sep 17 00:00:00 2001 From: Bart Wyatt Date: Wed, 1 Aug 2018 17:24:45 -0400 Subject: [PATCH 6/7] decrement revision on last squash There is an optimization for squashing the last undo session but it fails to properly update the revision --- include/chainbase/chainbase.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/chainbase/chainbase.hpp b/include/chainbase/chainbase.hpp index cc239c6..6584f4f 100644 --- a/include/chainbase/chainbase.hpp +++ b/include/chainbase/chainbase.hpp @@ -351,6 +351,7 @@ namespace chainbase { if( !enabled() ) return; if( _stack.size() == 1 ) { _stack.pop_front(); + --_revision; return; } From 5ddb55b98a2bbfb7fb0ef62fddb91f9da5638f07 Mon Sep 17 00:00:00 2001 From: arhag Date: Wed, 26 Sep 2018 14:12:58 -0400 Subject: [PATCH 7/7] correct off-by-one error in undo_stack_revision_range --- include/chainbase/chainbase.hpp | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/include/chainbase/chainbase.hpp b/include/chainbase/chainbase.hpp index 07b90f0..f5c22fc 100644 --- a/include/chainbase/chainbase.hpp +++ b/include/chainbase/chainbase.hpp @@ -471,8 +471,13 @@ namespace chainbase { void set_revision( uint64_t revision ) { - if( _stack.size() != 0 ) BOOST_THROW_EXCEPTION( std::logic_error("cannot set revision while there is an existing undo stack") ); - _revision = revision; + if( _stack.size() != 0 ) + BOOST_THROW_EXCEPTION( std::logic_error("cannot set revision while there is an existing undo stack") ); + + if( revision > std::numeric_limits::max() ) + BOOST_THROW_EXCEPTION( std::logic_error("revision to set is too high") ); + + _revision = static_cast(revision); } void remove_object( int64_t id ) @@ -483,11 +488,11 @@ namespace chainbase { } std::pair undo_stack_revision_range()const { - int64_t begin = -1; - int64_t end = -1; + int64_t begin = _revision; + int64_t end = _revision; if( _stack.size() > 0 ) { - begin = _stack.front().revision; + begin = _stack.front().revision - 1; end = _stack.back().revision; } @@ -816,7 +821,7 @@ namespace chainbase { ) ); } - idx_ptr->set_revision( expected_revision_range.first ); + idx_ptr->set_revision( static_cast(expected_revision_range.first) ); while( idx_ptr->revision() < expected_revision_range.second ) { idx_ptr->start_undo_session(true).push(); }