validator: multiple bugfixes

Author: ton

crypto/block/block.tlb

@@ -151,6 +151,9 @@ message$_ {X:Type} info:CommonMsgInfo
 message$_ {X:Type} info:CommonMsgInfoRelaxed
   init:(Maybe (Either StateInit ^StateInit))
   body:(Either X ^X) = MessageRelaxed X;
+  
+_ (Message Any) = MessageAny;
+
 //
 interm_addr_regular$0 use_dest_bits:(#<= 96) 
   = IntermediateAddress;

crypto/smartcont/highload-wallet-v2-one.fif

@@ -0,0 +1,112 @@
+#!/usr/bin/fift -s
+"TonUtil.fif" include
+"GetOpt.fif" include
+
+{ show-options-help 1 halt } : usage
+
+"" =: comment  // comment for simple transfers
+true =: allow-bounce
+false =: force-bounce
+3 =: send-mode  // mode for SENDRAWMSG: +1 - sender pays fees, +2 - ignore errors
+60 =: timeout   // external message expires in 60 seconds
+variable extra-currencies
+{ extra-currencies @ cc+ extra-currencies ! } : extra-cc+!
+
+begin-options
+     " <filename-base> <dest-addr> <subwallet-id> <amount> [-x <extra-amount>*<extra-currency-id>] [-n|-b] [-t<timeout>] [-B <body-boc>] [-C <comment>] [<savefile>]" +cr +tab
+    +"Creates one request to highload wallet created by new-highload-wallet-v2.fif, with private key loaded from file <filename-base>.pk "
+    +"and address from <filename-base><subwallet-id>.addr, and saves it into <savefile>.boc ('wallet-query.boc' by default)"
+    disable-digit-options generic-help-setopt
+  "n" "--no-bounce" { false =: allow-bounce } short-long-option
+    "Clears bounce flag" option-help
+  "b" "--force-bounce" { true =: force-bounce } short-long-option
+    "Forces bounce flag" option-help
+  "x" "--extra" { $>xcc extra-cc+! } short-long-option-arg
+    "Indicates the amount of extra currencies to be transfered" option-help
+  "t" "--timeout" { parse-int =: timeout } short-long-option-arg
+    "Sets expiration timeout in seconds (" timeout (.) $+ +" by default)" option-help
+  "B" "--body" { =: body-boc-file } short-long-option-arg
+    "Sets the payload of the transfer message" option-help
+  "C" "--comment" { =: comment } short-long-option-arg
+    "Sets the comment to be sent in the transfer message" option-help
+  "m" "--mode" { parse-int =: send-mode } short-long-option-arg
+    "Sets transfer mode (0..255) for SENDRAWMSG (" send-mode (.) $+ +" by default)"
+    option-help
+  "h" "--help" { usage } short-long-option
+    "Shows a help message" option-help
+parse-options
+
+$# dup 4 < swap 5 > or ' usage if
+5 :$1..n
+
+true constant bounce
+$1 =: file-base
+$2 bounce parse-load-address force-bounce or allow-bounce and =: bounce 2=: dest_addr
+$3 parse-int =: subwallet-id
+$4 $>cc extra-cc+! extra-currencies @ 2=: amount
+$5 "wallet-query" replace-if-null =: savefile
+{ subwallet-id (.) $+ } : +subwallet
+
+file-base +subwallet +".addr" load-address
+2dup 2constant wallet_addr
+."Source wallet address = " 2dup .addr cr 6 .Addr cr
+file-base +".pk" load-keypair nip constant wallet_pk
+
+def? body-boc-file { @' body-boc-file file>B B>boc } { comment simple-transfer-body } cond
+constant body-cell
+
+."Transferring " amount .GR+cc ."to account "
+dest_addr 2dup bounce 7 + .Addr ." = " .addr 
+."subwallet-id=0x" subwallet-id x.
+."timeout=" timeout . ."bounce=" bounce . cr
+."Body of transfer message is " body-cell <s csr. cr
+
+variable orders   dictnew orders !
+variable order#   order# 0!
+// c --
+{ <s order# @ dup 254 >= abort"more than 254 orders"
+  orders @ 16 udict!+ not abort"cannot add order to dictionary"
+  orders ! order# 1+!
+} : add-order
+// b body -- b'
+{ tuck <s 2dup 1 s-fits-with? not rot over 1 i, -rot
+  { drop swap ref, } { s, nip } cond
+} : append-msg-body
+// ng wc addr bounce body -- c
+{ <b b{01} s, rot 1 i, b{000100} s, 2swap addr, rot Gram,
+  0 9 64 32 + + 1+ u, swap append-msg-body b>
+} : create-int-msg
+// ng wc addr bnc --
+{ ."Transferring " 3 roll .GR ."to account " 
+  -rot 2dup 4 pick 7 + .Addr ." = " .addr ." bounce=" . cr
+} : .transfer
+// addr$ ng -- c
+{ swap parse-smc-addr force-bounce or allow-bounce and  // ng wc addr bnc 
+  2over 2over .transfer
+  <b 0 32 u, b> create-int-msg
+} : create-simple-transfer
+// c m -- c'
+{ <b swap 8 u, swap ref, b> } : create-order
+
+// addr$ ng --
+{ create-simple-transfer send-mode create-order add-order } : send
+{ bl word bl word $>GR send } : SEND
+
+// create internal message
+<b b{01} s, bounce 1 i, b{000} s, dest_addr Addr, amount Gram+cc, 0 9 64 32 + + u,
+  body-cell <s 2dup 1 s-fits-with? not rot over 1 i, -rot { drop body-cell ref, } { s, } cond
+b>
+send-mode create-order add-order
+
+// create external message
+now timeout + 32 << <b orders @ dict, b> hashu 32 1<<1- and + =: query_id
+<b subwallet-id 32 i, query_id 64 u, orders @ dict, b>
+dup ."signing message: " <s csr. cr
+dup hashu wallet_pk ed25519_sign_uint
+<b b{1000100} s, wallet_addr addr, 0 Gram, b{00} s,
+   swap B, swap <s s, b>
+dup ."resulting external message: " <s csr. cr
+2 boc+>B dup Bx. cr
+."Query_id is " query_id dup . ."= 0x" X. cr
+savefile +".boc" tuck B>file
+."(Saved to file " type .")" cr

crypto/smartcont/wallet.fif

@@ -26,6 +26,8 @@ begin-options
     "Sets the payload of the transfer message" option-help
   "C" "--comment" { =: comment } short-long-option-arg
     "Sets the comment to be sent in the transfer message" option-help
+  "I" "--with-init" { =: init-file } short-long-option-arg
+    "Indicates filename with BoC containing StateInit for internal message" option-help
   "m" "--mode" { parse-int =: send-mode } short-long-option-arg
     "Sets transfer mode (0..255) for SENDRAWMSG (" send-mode (.) $+ +" by default)"
     option-help
@@ -52,13 +54,17 @@ file-base +".pk" load-keypair nip constant wallet_pk
 def? body-boc-file { @' body-boc-file file>B B>boc } { comment simple-transfer-body } cond
 constant body-cell
 
+def? init-file { @' init-file file>B B>boc <s b{11} swap |_ } { b{0} } cond
+=: state-init
+
 ."Transferring " amount .GR+cc ."to account "
 dest_addr 2dup bounce 7 + .Addr ." = " .addr 
 ."seqno=0x" seqno x. ."bounce=" bounce . cr
 ."Body of transfer message is " body-cell <s csr. cr
+."StateInit is " state-init csr. cr
 
 // create a message
-<b b{01} s, bounce 1 i, b{000} s, dest_addr Addr, amount Gram+cc, 0 9 64 32 + + u,
+<b b{01} s, bounce 1 i, b{000} s, dest_addr Addr, amount Gram+cc, 0 8 64 32 + + u, state-init s,
   body-cell <s 2dup 1 s-fits-with? not rot over 1 i, -rot { drop body-cell ref, } { s, } cond
 b>
 <b seqno 32 u, send-mode 8 u, swap ref, b>

crypto/vm/dict.cpp

@@ -1670,40 +1670,51 @@ Ref<vm::Cell> DictionaryFixed::extract_prefix_subdict_root(td::ConstBitPtr prefi
 }
 
 std::pair<Ref<Cell>, int> DictionaryFixed::dict_filter(Ref<Cell> dict, td::BitPtr key, int n,
-                                                       const DictionaryFixed::filter_func_t& check_leaf) const {
+                                                       const DictionaryFixed::filter_func_t& check_leaf,
+                                                       int& skip_rest) const {
   // std::cerr << "dictionary filter for " << n << "-bit key = " << (key + n - key_bits).to_hex(key_bits - n)
   //           << std::endl;
   if (dict.is_null()) {
     // empty dictionary, return unchanged
     return {{}, 0};
   }
+  if (skip_rest >= 0) {
+    // either drop subtree completely (if skip_rest>0), or retain it completely (if skip_rest=0)
+    return {{}, skip_rest};
+  }
   LabelParser label{std::move(dict), n, label_mode()};
   assert(label.l_bits >= 0 && label.l_bits <= n);
   label.extract_label_to(key);
   key += label.l_bits;
   if (label.l_bits == n) {
     // leaf
     int res = check_leaf(label.remainder.write(), key - key_bits, key_bits);
+    if (res >= (1 << 30)) {
+      // skip all, or retain all
+      res &= (1 << 30) - 1;
+      skip_rest = (res ? 0 : (1 << 30));
+    }
     return {{}, res < 0 ? res : !res};
   }
   // fork, process left and right subtrees
   ++key;
   key[-1] = false;
   int delta = label.l_bits + 1;
   n -= delta;
-  auto left_res = dict_filter(label.remainder->prefetch_ref(0), key, n, check_leaf);
+  auto left_res = dict_filter(label.remainder->prefetch_ref(0), key, n, check_leaf, skip_rest);
   if (left_res.second < 0) {
     return left_res;
   }
   key[-1] = true;
-  auto right_res = dict_filter(label.remainder->prefetch_ref(1), key, n, check_leaf);
+  auto right_res = dict_filter(label.remainder->prefetch_ref(1), key, n, check_leaf, skip_rest);
   if ((left_res.second | right_res.second) <= 0) {
     // error in right, or both left and right unchanged
     return right_res;
   }
   auto left = left_res.second ? std::move(left_res.first) : label.remainder->prefetch_ref(0);
   auto right = right_res.second ? std::move(right_res.first) : label.remainder->prefetch_ref(1);
-  auto changes = left_res.second + right_res.second;
+  // 2^30 is effectively infinity, meaning that we dropped whole branches with unknown # of nodes
+  auto changes = ((left_res.second | right_res.second) & (1 << 30)) ? (1 << 30) : left_res.second + right_res.second;
   label.clear();
   if (left.is_null()) {
     if (right.is_null()) {
@@ -1735,8 +1746,9 @@ std::pair<Ref<Cell>, int> DictionaryFixed::dict_filter(Ref<Cell> dict, td::BitPt
 
 int DictionaryFixed::filter(DictionaryFixed::filter_func_t check_leaf) {
   force_validate();
+  int skip_rest = -1;
   unsigned char buffer[DictionaryFixed::max_key_bytes];
-  auto res = dict_filter(get_root_cell(), td::BitPtr{buffer}, key_bits, check_leaf);
+  auto res = dict_filter(get_root_cell(), td::BitPtr{buffer}, key_bits, check_leaf, skip_rest);
   if (res.second > 0) {
     // std::cerr << "after filter (" << res.second << " changes): new augmented dictionary root is:\n";
     // vm::load_cell_slice(res.first).print_rec(std::cerr);

crypto/vm/dict.h

@@ -293,7 +293,8 @@ class DictionaryFixed : public DictionaryBase {
                                                              bool remove_prefix = false) const;
   bool dict_check_for_each(Ref<Cell> dict, td::BitPtr key_buffer, int n, int total_key_len,
                            const foreach_func_t& foreach_func, bool invert_first = false) const;
-  std::pair<Ref<Cell>, int> dict_filter(Ref<Cell> dict, td::BitPtr key, int n, const filter_func_t& check_leaf) const;
+  std::pair<Ref<Cell>, int> dict_filter(Ref<Cell> dict, td::BitPtr key, int n, const filter_func_t& check_leaf,
+                                        int& skip_rest) const;
   Ref<Cell> dict_combine_with(Ref<Cell> dict1, Ref<Cell> dict2, td::BitPtr key_buffer, int n, int total_key_len,
                               const combine_func_t& combine_func, int mode = 0, int skip1 = 0, int skip2 = 0) const;
   bool dict_scan_diff(Ref<Cell> dict1, Ref<Cell> dict2, td::BitPtr key_buffer, int n, int total_key_len,

doc/ConfigParam-HOWTO

@@ -215,7 +215,7 @@ We see that the list of all active configuration proposals consists of exactly o
 
   [6465...6321 [1586779536 0 [8 C{FDCD...} -1] 1124...2998 () 8646...209 3 0 0]]
 
-Here the first number 6465..6321 is the unique identifier of the configuration proposal, equal to its 256-bit hash. The second component of this pair is a Tuple describing the status of this configuration proposal. The first component of this Tuple is the expiration Unixtime of the configuration proposal (1586779546). The second component (0) is the criticality flag. Next comes the configuration proposal proper, described by triple [8 C{FDCD...} -1], where 8 is the index of the configuration parameter to be modified, C{FDCD...} is the cell with the new value (represented by the hash of this cell), and -1 is the optional hash of the old value of this parameter (-1 means that this hash has not been specified). Next we see a large number 1124...2998 representing the identifier of the current validator set, then an empty list () representing the set of all currently active validators that have voted for this proposal so far, then *weight_remaining* equal to 8646...209 - a number that is positive if the proposal has not yet collected enough validator votes in this round, and negative otherwise. Then we see three numbers 3 0 0. These numbers are *rounds_remaining* (this proposal will survive at most three rounds, i.e., changes of the current validator set), *wins* (the count of rounds where the proposal collected votes of more than 3/4 of all validators by weight) and *losses* (the count of rounds where the proposal failed to collect 3/4 of all validator votes).
+Here the first number 6465..6321 is the unique identifier of the configuration proposal, equal to its 256-bit hash. The second component of this pair is a Tuple describing the status of this configuration proposal. The first component of this Tuple is the expiration Unixtime of the configuration proposal (1586779536). The second component (0) is the criticality flag. Next comes the configuration proposal proper, described by triple [8 C{FDCD...} -1], where 8 is the index of the configuration parameter to be modified, C{FDCD...} is the cell with the new value (represented by the hash of this cell), and -1 is the optional hash of the old value of this parameter (-1 means that this hash has not been specified). Next we see a large number 1124...2998 representing the identifier of the current validator set, then an empty list () representing the set of all currently active validators that have voted for this proposal so far, then *weight_remaining* equal to 8646...209 - a number that is positive if the proposal has not yet collected enough validator votes in this round, and negative otherwise. Then we see three numbers 3 0 0. These numbers are *rounds_remaining* (this proposal will survive at most three rounds, i.e., changes of the current validator set), *wins* (the count of rounds where the proposal collected votes of more than 3/4 of all validators by weight) and *losses* (the count of rounds where the proposal failed to collect 3/4 of all validator votes).
 
 We can inspect the proposed value for configuration parameter #8 by asking the lite-client to expand cell C{FDCD...} using its hash FDCD... or a sufficiently long prefix of this hash to uniquely identify the cell in question:
 

validator-session/validator-session-description.cpp

@@ -14,7 +14,7 @@
     You should have received a copy of the GNU Lesser General Public License
     along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
 
-    Copyright 2017-2019 Telegram Systems LLP
+    Copyright 2017-2020 Telegram Systems LLP
 */
 #include "validator-session.hpp"
 #include "td/utils/Random.h"
@@ -52,10 +52,10 @@ ValidatorSessionDescriptionImpl::ValidatorSessionDescriptionImpl(ValidatorSessio
   self_idx_ = it->second;
 
   pdata_temp_ptr_ = 0;
-  pdata_temp_size_ = 1 << 30;
+  pdata_temp_size_ = 1 << 27;
   pdata_temp_ = new td::uint8[pdata_temp_size_];
 
-  pdata_perm_size_ = 1ull << 30;
+  pdata_perm_size_ = 1ull << 27;
   pdata_perm_ptr_ = 0;
 
   for (auto &el : cache_) {

validator-session/validator-session-description.hpp

@@ -14,7 +14,7 @@
     You should have received a copy of the GNU Lesser General Public License
     along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
 
-    Copyright 2017-2019 Telegram Systems LLP
+    Copyright 2017-2020 Telegram Systems LLP
 */
 #pragma once
 

validator/impl/collator-impl.h

@@ -59,6 +59,7 @@ class Collator final : public td::actor::Actor {
   bool preinit_complete{false};
   bool is_key_block_{false};
   bool block_full_{false};
+  bool outq_cleanup_partial_{false};
   bool inbound_queues_empty_{false};
   bool libraries_changed_{false};
   bool prev_key_block_exists_{false};
@@ -144,6 +145,8 @@ class Collator final : public td::actor::Actor {
   bool ihr_enabled_{false};
   bool create_stats_enabled_{false};
   bool report_version_{false};
+  bool skip_topmsgdescr_{false};
+  bool skip_extmsg_{false};
   td::uint64 overload_history_{0}, underload_history_{0};
   td::uint64 block_size_estimate_{};
   Ref<block::WorkchainInfo> wc_info_;

validator/impl/collator.cpp

@@ -34,6 +34,7 @@
 #include "validator-set.hpp"
 #include "top-shard-descr.hpp"
 #include <ctime>
+#include "td/utils/Random.h"
 
 namespace ton {
 
@@ -1208,6 +1209,9 @@ bool Collator::import_new_shard_top_blocks() {
   if (shard_block_descr_.empty()) {
     return true;
   }
+  if (skip_topmsgdescr_) {
+    return true;
+  }
   auto lt_limit = config_->lt + config_->get_max_lt_growth();
   std::sort(shard_block_descr_.begin(), shard_block_descr_.end(), cmp_shard_block_descr_ref);
   int tb_act = 0;
@@ -1442,6 +1446,37 @@ bool Collator::init_utime() {
         "error initializing unix time for the new block: failed to observe end of fsm_split time interval for this "
         "shard");
   }
+  // check whether masterchain catchain rotation is overdue
+  auto ccvc = config_->get_catchain_validators_config();
+  unsigned lifetime = ccvc.mc_cc_lifetime;
+  if (is_masterchain() && now_ / lifetime > prev_now_ / lifetime && now_ > (prev_now_ / lifetime + 1) * lifetime + 20) {
+    auto overdue = now_ - (prev_now_ / lifetime + 1) * lifetime;
+    // masterchain catchain rotation overdue, skip topsharddescr with some probability
+    skip_topmsgdescr_ = (td::Random::fast(0, 1023) < 256);  // probability 1/4
+    skip_extmsg_ = (td::Random::fast(0, 1023) < 256);       // skip ext msg probability 1/4
+    if (skip_topmsgdescr_) {
+      LOG(WARNING)
+          << "randomly skipping import of new shard data because of overdue masterchain catchain rotation (overdue by "
+          << overdue << " seconds)";
+    }
+    if (skip_extmsg_) {
+      LOG(WARNING)
+          << "randomly skipping external message import because of overdue masterchain catchain rotation (overdue by "
+          << overdue << " seconds)";
+    }
+  } else if (is_masterchain() && now_ > prev_now_ + 60) {
+    auto interval = now_ - prev_now_;
+    skip_topmsgdescr_ = (td::Random::fast(0, 1023) < 128);  // probability 1/8
+    skip_extmsg_ = (td::Random::fast(0, 1023) < 128);       // skip ext msg probability 1/8
+    if (skip_topmsgdescr_) {
+      LOG(WARNING) << "randomly skipping import of new shard data because of overdue masterchain block (last block was "
+                   << interval << " seconds ago)";
+    }
+    if (skip_extmsg_) {
+      LOG(WARNING) << "randomly skipping external message import because of overdue masterchain block (last block was "
+                   << interval << " seconds ago)";
+    }
+  }
   return true;
 }
 
@@ -1742,7 +1777,7 @@ bool Collator::dequeue_message(Ref<vm::Cell> msg_envelope, ton::LogicalTime deli
 }
 
 bool Collator::out_msg_queue_cleanup() {
-  LOG(DEBUG) << "in out_msg_queue_cleanup()";
+  LOG(INFO) << "cleaning outbound queue from messages already imported by neighbors";
   if (verbosity >= 2) {
     auto rt = out_msg_queue_->get_root();
     std::cerr << "old out_msg_queue is ";
@@ -1759,6 +1794,11 @@ bool Collator::out_msg_queue_cleanup() {
   auto res = out_msg_queue_->filter([&](vm::CellSlice& cs, td::ConstBitPtr key, int n) -> int {
     assert(n == 352);
     // LOG(DEBUG) << "key is " << key.to_hex(n);
+    if (block_full_) {
+      LOG(WARNING) << "BLOCK FULL while cleaning up outbound queue, cleanup completed only partially";
+      outq_cleanup_partial_ = true;
+      return (1 << 30) + 1;  // retain all remaining outbound queue entries including this one without processing
+    }
     block::EnqueuedMsgDescr enq_msg_descr;
     unsigned long long created_lt;
     if (!(cs.fetch_ulong_bool(64, created_lt)  // augmentation
@@ -1789,6 +1829,10 @@ bool Collator::out_msg_queue_cleanup() {
                               << enq_msg_descr.hash_.to_hex() << ") by inserting a msg_export_deq record");
         return -1;
       }
+      register_out_msg_queue_op();
+      if (!block_limit_status_->fits(block::ParamLimits::cl_normal)) {
+        block_full_ = true;
+      }
     }
     return !delivered;
   });
@@ -2631,6 +2675,10 @@ bool Collator::process_inbound_internal_messages() {
 }
 
 bool Collator::process_inbound_external_messages() {
+  if (skip_extmsg_) {
+    LOG(INFO) << "skipping processing of inbound external messages";
+    return true;
+  }
   bool full = !block_limit_status_->fits(block::ParamLimits::cl_soft);
   for (auto& ext_msg_pair : ext_msg_list_) {
     if (full) {