Db headers, locks, pos dirs, and config (Chia-Network#88)

* New database table to store headers
* Loading blockchain only loads headers into memory instead of header blocks (header + proofs), speeds up the startup, and reduces normal operation memory usage by 80%
* Fix deadlock issues with store class
* Make memory store access synchronous, to reduce use of locks, this improves performance of block and unfinished block processing
* Remove locks from reads of DB, only lock for writes
* Cleaner configuration system with argparse
* Log to file by default, configurable to stdout
* Proof of space binary and create_plots scripts allow passing in temp and final directories

CMakeLists.txt

@@ -31,7 +31,7 @@ add_subdirectory(lib/pybind11)
 
 pybind11_add_module(chiapos ${CMAKE_CURRENT_SOURCE_DIR}/python-bindings/chiapos.cpp)
 
-set (CMAKE_CXX_FLAGS "-g -O3 -Wall -msse2 -msse -march=native -std=c++11 -maes")
+set (CMAKE_CXX_FLAGS "-g -O3 -Wall -msse2 -msse -march=native -std=c++1z -maes")
 
 add_executable(ProofOfSpace
     src/cli.cpp
@@ -45,7 +45,15 @@ add_executable(RunTests
     tests/test-main.cpp
     tests/test.cpp
 )
-target_link_libraries(chiapos PRIVATE fse)
-target_link_libraries(ProofOfSpace fse)
-target_link_libraries(HellmanAttacks fse)
-target_link_libraries(RunTests fse)
+
+if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+  target_link_libraries(chiapos PRIVATE fse)
+  target_link_libraries(ProofOfSpace fse)
+  target_link_libraries(HellmanAttacks fse)
+  target_link_libraries(RunTests fse)
+else()
+  target_link_libraries(chiapos PRIVATE fse stdc++fs)
+  target_link_libraries(ProofOfSpace fse stdc++fs)
+  target_link_libraries(HellmanAttacks fse stdc++fs)
+  target_link_libraries(RunTests fse stdc++fs)
+endif()

README.md

@@ -8,8 +8,7 @@ Only runs on 64 bit architectures with AES-NI support. Read the [Proof of Space
 ### Compile
 
 ```bash
-git submodule update --init --recursive
-mkdir build && cd build
+mkdir -p build && cd build
 cmake ../
 cmake --build . -- -j 6
 ```

python-bindings/chiapos.cpp

@@ -34,13 +34,14 @@ PYBIND11_MODULE(chiapos, m) {
 
     py::class_<DiskPlotter>(m, "DiskPlotter")
         .def(py::init<>())
-        .def("create_plot_disk", [](DiskPlotter &dp, const std::string filename, uint8_t k,
+        .def("create_plot_disk", [](DiskPlotter &dp, const std::string tmp_dir, const std::string final_dir,
+                                    const std::string filename, uint8_t k,
                                     const py::bytes &memo, const py::bytes &id) {
             std::string memo_str(memo);
             const uint8_t* memo_ptr = reinterpret_cast<const uint8_t*>(memo_str.data());
             std::string id_str(id);
             const uint8_t* id_ptr = reinterpret_cast<const uint8_t*>(id_str.data());
-            dp.CreatePlotDisk(filename, k, memo_ptr, len(memo), id_ptr, len(id));
+            dp.CreatePlotDisk(tmp_dir, final_dir, filename, k, memo_ptr, len(memo), id_ptr, len(id));
         });
 
     py::class_<DiskProver>(m, "DiskProver")

setup.py

@@ -66,7 +66,7 @@ def build_extension(self, ext):
 
 setup(
     name='chiapos',
-    version='0.2.2',
+    version='0.2.3',
     author='Mariano Sorgente',
     author_email='[email protected]',
     description='Chia proof of space plotting, proving, and verifying (wraps C++)',

src/cli.cpp

@@ -64,13 +64,17 @@ int main(int argc, char *argv[]) {
         // Default values
         uint8_t k = 20;
         string filename = "plot.dat";
+        string tempdir = ".";
+        string finaldir = ".";
         string operation = "help";
         string memo  = "0102030405";
         string id = "022fb42c08c12de3a6af053880199806532e79515f94e83461612101f9412f9e";
 
         options.allow_unrecognised_options()
                .add_options()
                 ("k, size", "Plot size", cxxopts::value<uint8_t>(k))
+                ("t, tempdir", "Temporary directory", cxxopts::value<string>(tempdir))
+                ("d, finaldir", "Final directory", cxxopts::value<string>(finaldir))
                 ("f, file", "Filename", cxxopts::value<string>(filename))
                 ("m, memo", "Memo to insert into the plot", cxxopts::value<string>(memo))
                 ("i, id", "Unique 32-byte seed for the plot", cxxopts::value<string>(id))
@@ -82,6 +86,7 @@ int main(int argc, char *argv[]) {
             HelpAndQuit(options);
         }
         operation = argv[1];
+        std::cout << "operation" << operation << std::endl;
 
         if (operation == "help") {
             HelpAndQuit(options);
@@ -101,7 +106,7 @@ int main(int argc, char *argv[]) {
             HexToBytes(id, id_bytes);
 
             DiskPlotter plotter = DiskPlotter();
-            plotter.CreatePlotDisk(filename, k, memo_bytes, 5, id_bytes, 32);
+            plotter.CreatePlotDisk(tempdir, finaldir, filename, k, memo_bytes, 5, id_bytes, 32);
         } else if (operation == "prove") {
             if (argc < 3) {
                 HelpAndQuit(options);

src/plotter_disk.hpp

@@ -26,6 +26,21 @@
 #include <string>
 #include <utility>
 
+#if __has_include(<filesystem>)
+
+#include <filesystem>
+namespace filesystem = std::filesystem;
+
+#elif __has_include(<experimental/filesystem>)
+
+#include <experimental/filesystem>
+namespace filesystem = std::experimental::filesystem;
+
+#else
+#error "an implementation of filesystem is required!"
+#endif
+
+
 #include "util.hpp"
 #include "encoding.hpp"
 #include "calculate_bucket.hpp"
@@ -71,13 +86,32 @@ class DiskPlotter {
     // This method creates a plot on disk with the filename. A temporary file, "plotting" + filename,
     // is created and will be larger than the final plot file. This file is deleted at the end of
     // the process.
-    void CreatePlotDisk(std::string filename, uint8_t k, const uint8_t* memo,
+    void CreatePlotDisk(std::string tmp_dirname, std::string final_dirname, std::string filename,
+                        uint8_t k, const uint8_t* memo,
                         uint32_t memo_len, const uint8_t* id, uint32_t id_len) {
-        std::cout << std::endl << "Starting plotting progress into file " << filename << "." << std::endl;
+        std::cout << std::endl << "Starting plotting progress into temporary dir " << tmp_dirname << "." << std::endl;
         std::cout << "Memo: " << Util::HexStr(memo, memo_len) << std::endl;
         std::cout << "ID: " << Util::HexStr(id, id_len) << std::endl;
         std::cout << "Plot size is: " << static_cast<int>(k) << std::endl;
 
+        // Cross platform way to concatenate paths, c++17.
+        filesystem::path tmp_1_filename = filesystem::path(tmp_dirname) / filesystem::path(filename + ".tmp");
+        filesystem::path tmp_2_filename = filesystem::path(tmp_dirname) / filesystem::path(filename + ".2.tmp");
+        filesystem::path final_filename = filesystem::path(final_dirname) / filesystem::path(filename);
+
+        // Check if the paths exist
+        if (!filesystem::exists(tmp_dirname)) {
+            std::string err_string = "Directory " + tmp_dirname + " does not exist";
+            std::cerr << err_string << std::endl;
+            throw err_string;
+        }
+
+        if (!filesystem::exists(final_dirname)) {
+            std::string err_string = "Directory " + final_dirname + " does not exist";
+            std::cerr << err_string << std::endl;
+            throw err_string;
+        }
+
         // These variables are used in the WriteParkToFile method. They are preallocatted here
         // to save time.
         first_line_point_bytes = new uint8_t[CalculateLinePointSize(k)];
@@ -88,27 +122,27 @@ class DiskPlotter {
         assert(k >= kMinPlotSize);
         assert(k <= kMaxPlotSize);
 
-        std::string plot_filename = filename + ".tmp";
-
         std::cout << std::endl << "Starting phase 1/4: Forward Propagation... " << Timer::GetNow();
+
         Timer p1;
         Timer all_phases;
-        std::vector<uint64_t> results = WritePlotFile(plot_filename, k, id, memo, memo_len);
+        std::vector<uint64_t> results = WritePlotFile(tmp_1_filename, k, id, memo, memo_len);
         p1.PrintElapsed("Time for phase 1 =");
 
-        std::cout << std::endl << "Starting phase 2/4: Backpropagation... " << Timer::GetNow();
+        std::cout << std::endl << "Starting phase 2/4: Backpropagation into " << tmp_1_filename << " and " << tmp_2_filename << " ..." << Timer::GetNow();
+
         Timer p2;
-        Backpropagate(filename, plot_filename, k, id, memo, memo_len, results);
+        Backpropagate(tmp_2_filename, tmp_1_filename, k, id, memo, memo_len, results);
         p2.PrintElapsed("Time for phase 2 =");
 
         std::cout << std::endl << "Starting phase 3/4: Compression... " << Timer::GetNow();
         Timer p3;
-        Phase3Results res = CompressTables(k, results, filename, plot_filename, id, memo, memo_len);
+        Phase3Results res = CompressTables(k, results, tmp_2_filename, tmp_1_filename, id, memo, memo_len);
         p3.PrintElapsed("Time for phase 3 =");
 
         std::cout << std::endl << "Starting phase 4/4: Write Checkpoint tables... " << Timer::GetNow();
         Timer p4;
-        WriteCTables(k, k + 1, filename, plot_filename, res);
+        WriteCTables(k, k + 1, tmp_2_filename, tmp_1_filename, res);
         p4.PrintElapsed("Time for phase 4 =");
 
         std::cout << "Approximate working space used: " <<
@@ -117,7 +151,14 @@ class DiskPlotter {
                      static_cast<double>(res.final_table_begin_pointers[11])/(1024*1024*1024) << " GB" << std::endl;
         all_phases.PrintElapsed("Total time =");
 
-        remove(plot_filename.c_str());
+        bool removed_1 = filesystem::remove(tmp_1_filename);
+        filesystem::copy(tmp_2_filename, final_filename, filesystem::copy_options::overwrite_existing);
+
+        bool removed_2 = filesystem::remove(tmp_2_filename);
+
+        std::cout << "Removed " << tmp_1_filename << "? " << removed_1 << std::endl;
+        std::cout << "Removed " << tmp_2_filename << "? " << removed_2 << std::endl;
+        std::cout << "Copied final file to " << final_filename << std::endl;
 
         delete[] first_line_point_bytes;
         delete[] park_stubs_bytes;
@@ -885,7 +926,7 @@ class DiskPlotter {
              deltas_bits.ToBytes(park_deltas_bytes);
 
              uint16_t encoded_size = deltas_bits.GetSize() / 8;
-        
+
              assert((uint32_t)(encoded_size + 2) < CalculateMaxDeltasSize(k, table_index));
              writer.write((const char*)&encoded_size, 2);
              writer.write((const char*)park_deltas_bytes, encoded_size);

tests/test.cpp

@@ -364,7 +364,7 @@ void PlotAndTestProofOfSpace(std::string filename, uint32_t iterations, uint8_t
                              uint32_t expected_success) {
         DiskPlotter plotter = DiskPlotter();
         uint8_t memo[5] = {1, 2, 3, 4, 5};
-        plotter.CreatePlotDisk(filename, k, memo, 5, plot_id, 32);
+        plotter.CreatePlotDisk(".", ".", filename, k, memo, 5, plot_id, 32);
         TestProofOfSpace(filename, iterations, k, plot_id, expected_success);
         REQUIRE(remove(filename.c_str()) == 0);
 }
@@ -388,7 +388,7 @@ TEST_CASE("Invalid plot") {
         uint8_t memo[5] = {1, 2, 3, 4, 5};
         string filename = "invalid-plot.dat";
         uint8_t k = 22;
-        plotter.CreatePlotDisk(filename, k, memo, 5, plot_id_1, 32);
+        plotter.CreatePlotDisk(".", ".", filename, k, memo, 5, plot_id_1, 32);
         DiskProver prover(filename);
         uint8_t* proof_data = new uint8_t[8 * k];
         uint8_t challenge[32];

tests/test_python_bindings.py

@@ -13,7 +13,7 @@ def test_k_21(self):
                                   10, 11, 129, 139, 171, 15, 23])
 
         pl = DiskPlotter()
-        pl.create_plot_disk("./myplot.dat", 21, bytes([1, 2, 3, 4, 5]), plot_seed)
+        pl.create_plot_disk(".", ".", "myplot.dat", 21, bytes([1, 2, 3, 4, 5]), plot_seed)
         pr = DiskProver("./myplot.dat")
 
         total_proofs: int = 0
@@ -32,7 +32,7 @@ def test_k_21(self):
         print(f"total proofs {total_proofs} out of {iterations}\
             {total_proofs / iterations}")
         assert total_proofs == 4647
-        os.remove("./myplot.dat")
+        os.remove("myplot.dat")
 
 
 if __name__ == '__main__':