Auto merge of zcash#2670 - arielgabizon:lowmemprover, r=daira

Low memory prover This PR integrates @ebfull 's low memory changes: https://github.com/zcash/zcash/pull/2243/commits on top of @str4d 's work bringing in libsnark as a subtree zcash@4699d0e
wip-alex · Nov 6, 2017 · 6f9f09d · 6f9f09d
2 parents 086c6be + bef1b5c
commit 6f9f09d
Show file tree

Hide file tree

Showing 17 changed files with 281 additions and 226 deletions.
diff --git a/src/gtest/main.cpp b/src/gtest/main.cpp
@@ -4,6 +4,8 @@
 
 #include "libsnark/common/default_types/r1cs_ppzksnark_pp.hpp"
 #include "libsnark/zk_proof_systems/ppzksnark/r1cs_ppzksnark/r1cs_ppzksnark.hpp"
+#include "zcash/JoinSplit.hpp"
+#include "util.h"
 
 struct ECCryptoClosure
 {
@@ -12,11 +14,16 @@ struct ECCryptoClosure
 
 ECCryptoClosure instance_of_eccryptoclosure;
 
+ZCJoinSplit* params;
+
 int main(int argc, char **argv) {
   assert(init_and_check_sodium() != -1);
   libsnark::default_r1cs_ppzksnark_pp::init_public_params();
   libsnark::inhibit_profiling_info = true;
   libsnark::inhibit_profiling_counters = true;
+  boost::filesystem::path pk_path = ZC_GetParamsDir() / "sprout-proving.key";
+  boost::filesystem::path vk_path = ZC_GetParamsDir() / "sprout-verifying.key";
+  params = ZCJoinSplit::Prepared(vk_path.string(), pk_path.string());
 
   testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();

diff --git a/src/gtest/test_joinsplit.cpp b/src/gtest/test_joinsplit.cpp
@@ -5,6 +5,7 @@
 #include <boost/foreach.hpp>
 
 #include "zcash/prf.h"
+#include "util.h"
 
 #include "zcash/JoinSplit.hpp"
 #include "zcash/Note.hpp"
@@ -13,6 +14,8 @@
 
 using namespace libzcash;
 
+extern ZCJoinSplit* params;
+
 void test_full_api(ZCJoinSplit* js)
 {
     // Create verification context.
@@ -219,8 +222,6 @@ void invokeAPIFailure(
 
 TEST(joinsplit, h_sig)
 {
-    auto js = ZCJoinSplit::Unopened();
-
 /*
 // by Taylor Hornby
 
@@ -284,16 +285,14 @@ for test_input in TEST_VECTORS:
     };
 
     BOOST_FOREACH(std::vector<std::string>& v, tests) {
-        auto expected = js->h_sig(
+        auto expected = ZCJoinSplit::h_sig(
             uint256S(v[0]),
             {uint256S(v[1]), uint256S(v[2])},
             uint256S(v[3])
         );
 
         EXPECT_EQ(expected, uint256S(v[4]));
     }
-
-    delete js;
 }
 
 void increment_note_witnesses(
@@ -311,8 +310,6 @@ void increment_note_witnesses(
 
 TEST(joinsplit, full_api_test)
 {
-    auto js = ZCJoinSplit::Generate();
-
     {
         std::vector<ZCIncrementalWitness> witnesses;
         ZCIncrementalMerkleTree tree;
@@ -331,7 +328,7 @@ TEST(joinsplit, full_api_test)
         increment_note_witnesses(note5.cm(), witnesses, tree);
 
         // Should work
-        invokeAPI(js,
+        invokeAPI(params,
         {
             JSInput(),
             JSInput()
@@ -345,7 +342,7 @@ TEST(joinsplit, full_api_test)
         tree.root());
 
         // lhs > MAX_MONEY
-        invokeAPIFailure(js,
+        invokeAPIFailure(params,
         {
             JSInput(),
             JSInput()
@@ -360,7 +357,7 @@ TEST(joinsplit, full_api_test)
         "nonsensical vpub_old value");
 
         // rhs > MAX_MONEY
-        invokeAPIFailure(js,
+        invokeAPIFailure(params,
         {
             JSInput(),
             JSInput()
@@ -375,7 +372,7 @@ TEST(joinsplit, full_api_test)
         "nonsensical vpub_new value");
 
         // input witness for the wrong element
-        invokeAPIFailure(js,
+        invokeAPIFailure(params,
         {
             JSInput(witnesses[0], note1, sk),
             JSInput()
@@ -391,7 +388,7 @@ TEST(joinsplit, full_api_test)
 
         // input witness doesn't match up with
         // real root
-        invokeAPIFailure(js,
+        invokeAPIFailure(params,
         {
             JSInput(witnesses[1], note1, sk),
             JSInput()
@@ -406,7 +403,7 @@ TEST(joinsplit, full_api_test)
         "joinsplit not anchored to the correct root");
 
         // input is in the tree now! this should work
-        invokeAPI(js,
+        invokeAPI(params,
         {
             JSInput(witnesses[1], note1, sk),
             JSInput()
@@ -420,7 +417,7 @@ TEST(joinsplit, full_api_test)
         tree.root());
 
         // Wrong secret key
-        invokeAPIFailure(js,
+        invokeAPIFailure(params,
         {
             JSInput(witnesses[1], note1, SpendingKey::random()),
             JSInput()
@@ -435,7 +432,7 @@ TEST(joinsplit, full_api_test)
         "input note not authorized to spend with given key");
 
         // Absurd input value
-        invokeAPIFailure(js,
+        invokeAPIFailure(params,
         {
             JSInput(witnesses[3], note3, sk),
             JSInput()
@@ -450,7 +447,7 @@ TEST(joinsplit, full_api_test)
         "nonsensical input note value");
 
         // Absurd total input value
-        invokeAPIFailure(js,
+        invokeAPIFailure(params,
         {
             JSInput(witnesses[4], note4, sk),
             JSInput(witnesses[5], note5, sk)
@@ -465,7 +462,7 @@ TEST(joinsplit, full_api_test)
         "nonsensical left hand size of joinsplit balance");
 
         // Absurd output value
-        invokeAPIFailure(js,
+        invokeAPIFailure(params,
         {
             JSInput(),
             JSInput()
@@ -480,7 +477,7 @@ TEST(joinsplit, full_api_test)
         "nonsensical output value");
 
         // Absurd total output value
-        invokeAPIFailure(js,
+        invokeAPIFailure(params,
         {
             JSInput(),
             JSInput()
@@ -495,7 +492,7 @@ TEST(joinsplit, full_api_test)
         "nonsensical right hand side of joinsplit balance");
 
         // Absurd total output value
-        invokeAPIFailure(js,
+        invokeAPIFailure(params,
         {
             JSInput(),
             JSInput()
@@ -510,22 +507,7 @@ TEST(joinsplit, full_api_test)
         "invalid joinsplit balance");
     }
 
-    test_full_api(js);
-
-    js->saveProvingKey("./zcashTest.pk");
-    js->saveVerifyingKey("./zcashTest.vk");
-
-    delete js;
-
-    js = ZCJoinSplit::Unopened();
-
-    js->setProvingKeyPath("./zcashTest.pk");
-    js->loadProvingKey();
-    js->loadVerifyingKey("./zcashTest.vk");
-
-    test_full_api(js);
-
-    delete js;
+    test_full_api(params);
 }
 
 TEST(joinsplit, note_plaintexts)

diff --git a/src/gtest/utils.cpp b/src/gtest/utils.cpp
@@ -1,7 +1,3 @@
-#include "zcash/JoinSplit.hpp"
-
-ZCJoinSplit* params = ZCJoinSplit::Unopened();
-
 int GenZero(int n)
 {
     return 0;

diff --git a/src/init.cpp b/src/init.cpp
@@ -688,18 +688,14 @@ static void ZC_LoadParams()
         return;
     }
 
-    pzcashParams = ZCJoinSplit::Unopened();
-
     LogPrintf("Loading verifying key from %s\n", vk_path.string().c_str());
     gettimeofday(&tv_start, 0);
 
-    pzcashParams->loadVerifyingKey(vk_path.string());
+    pzcashParams = ZCJoinSplit::Prepared(vk_path.string(), pk_path.string());
 
     gettimeofday(&tv_end, 0);
     elapsed = float(tv_end.tv_sec-tv_start.tv_sec) + (tv_end.tv_usec-tv_start.tv_usec)/float(1000000);
     LogPrintf("Loaded verifying key in %fs seconds.\n", elapsed);
-
-    pzcashParams->setProvingKeyPath(pk_path.string());
 }
 
 bool AppInitServers(boost::thread_group& threadGroup)

diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp
@@ -20,9 +20,6 @@ JSDescription::JSDescription(ZCJoinSplit& params,
 {
     boost::array<libzcash::Note, ZC_NUM_JS_OUTPUTS> notes;
 
-    if (computeProof) {
-        params.loadProvingKey();
-    }
     proof = params.prove(
         inputs,
         outputs,

diff --git a/src/snark/src/zk_proof_systems/ppzksnark/r1cs_ppzksnark/r1cs_ppzksnark.hpp b/src/snark/src/zk_proof_systems/ppzksnark/r1cs_ppzksnark/r1cs_ppzksnark.hpp
@@ -397,6 +397,12 @@ r1cs_ppzksnark_proof<ppT> r1cs_ppzksnark_prover(const r1cs_ppzksnark_proving_key
                                                 const r1cs_ppzksnark_auxiliary_input<ppT> &auxiliary_input,
                                                 const r1cs_ppzksnark_constraint_system<ppT> &constraint_system);
 
+template<typename ppT>
+r1cs_ppzksnark_proof<ppT> r1cs_ppzksnark_prover_streaming(std::ifstream &proving_key_file,
+                                                          const r1cs_ppzksnark_primary_input<ppT> &primary_input,
+                                                          const r1cs_ppzksnark_auxiliary_input<ppT> &auxiliary_input,
+                                                          const r1cs_ppzksnark_constraint_system<ppT> &constraint_system);
+
 /*
  Below are four variants of verifier algorithm for the R1CS ppzkSNARK.