Update sources

ton-test-liteclient-full/lite-client/CMakeLists.txt

@@ -12,12 +12,6 @@ add_subdirectory(third-party/crc32c EXCLUDE_FROM_ALL)
 set(CRC32C_FOUND 1)
 set(HAVE_SSE42)
 
-set(WITH_TESTS OFF CACHE BOOL "build with tests")
-set(WITH_TOOLS OFF CACHE BOOL "build with tools")
-set(FAIL_ON_WARNINGS OFF CACHE BOOL "fail on warnings")
-add_subdirectory(third-party/rocksdb EXCLUDE_FROM_ALL)
-set(HAVE_SSE42)
-
 option(USE_LIBRAPTORQ "use libraptorq for tests" OFF)
 if (USE_LIBRAPTORQ)
   set(USE_LZ4 OFF CACHE BOOL "use lz4")

ton-test-liteclient-full/lite-client/adnl/adnl-node-id.hpp

@@ -224,7 +224,7 @@ class AdnlEd25519Priv {
 
  public:
   ~AdnlEd25519Priv() {
-    data_ = td::UInt256::zero();
+    data_.set_zero_s();
   }
   AdnlEd25519Priv(const ton_api::adnl_id_pk_ed25519 &obj) {
     data_ = obj.key_;
@@ -251,7 +251,7 @@ class AdnlAESPriv {
 
  public:
   ~AdnlAESPriv() {
-    data_ = td::UInt256::zero();
+    data_.set_zero_s();
   }
   AdnlAESPriv(const ton_api::adnl_id_pk_aes &obj) {
     data_ = obj.key_;

ton-test-liteclient-full/lite-client/crypto/CMakeLists.txt

@@ -1,5 +1,7 @@
 cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR)
 
+set (BUILD_CREATE_STATE 0)
+
 if (NOT OPENSSL_FOUND)
   find_package(OpenSSL REQUIRED)
 endif()
@@ -245,3 +247,13 @@ if (NOT CMAKE_CROSSCOMPILING)
   )
   add_dependencies(ton_block tlb_generate_block)
 endif()
+
+if (BUILD_CREATE_STATE) 
+  add_executable(create-state block/create-state.cpp)
+  target_include_directories(create-state PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/..>)
+  target_link_libraries(create-state PUBLIC ton_crypto fift-lib ton_block adnl ton_validator validator_disk ton_validator)
+  if (WINGETOPT_FOUND)
+    target_link_libraries_system(create-state wingetopt)
+  endif()
+endif()

ton-test-liteclient-full/lite-client/crypto/block/block.cpp

@@ -226,7 +226,6 @@ MsgProcessedUptoCollection::MsgProcessedUptoCollection(ton::ShardIdFull _owner,
     MsgProcessedUpto& z = list.back();
     z.shard = key.get_uint(64);
     z.mc_seqno = (unsigned)((key + 64).get_uint(32));
-    z.our_end_lt = 0;
     z.last_inmsg_lt = value.write().fetch_ulong(64);
     return value.write().fetch_bits_to(z.last_inmsg_hash) && z.shard && ton::shard_contains(owner.shard, z.shard);
   });
@@ -302,9 +301,63 @@ bool MsgProcessedUptoCollection::pack(vm::CellBuilder& cb) {
   return std::move(dict).append_dict_to_bool(cb);
 }
 
-bool MsgProcessedUpto::already_processed(ton::ShardId msg_next_addr, ton::LogicalTime msg_lt,
-                                         td::ConstBitPtr msg_hash) const {
-  return false;  // TODO: implement this properly
+bool MsgProcessedUpto::already_processed(const EnqueuedMsgDescr& msg) const {
+  // LOG(DEBUG) << "compare msg (" << msg.lt_ << "," << msg.hash_.to_hex() << ") against record's (" << last_inmsg_lt
+  //            << "," << last_inmsg_hash.to_hex() << ")";
+  if (msg.lt_ > last_inmsg_lt) {
+    return false;
+  }
+  if (!ton::shard_contains(shard, msg.next_prefix_.account_id_prefix)) {
+    return false;
+  }
+  if (msg.lt_ == last_inmsg_lt && last_inmsg_hash < msg.hash_) {
+    return false;
+  }
+  auto shard_end_lt = compute_shard_end_lt(msg.cur_prefix_);
+  // LOG(DEBUG) << "enqueued_lt = " << msg.enqueued_lt_ << " , shard_end_lt = " << shard_end_lt;
+  return msg.enqueued_lt_ < shard_end_lt;
+}
+
+bool MsgProcessedUptoCollection::already_processed(const EnqueuedMsgDescr& msg) const {
+  // LOG(DEBUG) << "checking message with cur_addr=" << msg.cur_prefix_.to_str()
+  //            << " next_addr=" << msg.next_prefix_.to_str() << " against ProcessedUpto of neighbor " << owner.to_str();
+  if (!ton::shard_contains(owner, msg.next_prefix_)) {
+    return false;
+  }
+  for (const auto& rec : list) {
+    if (rec.already_processed(msg)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+// unpacks some fields from EnqueuedMsg
+bool EnqueuedMsgDescr::unpack(vm::CellSlice& cs) {
+  block::gen::EnqueuedMsg::Record enq;
+  block::tlb::MsgEnvelope::Record_std env;
+  block::gen::CommonMsgInfo::Record_int_msg_info info;
+  if (!(tlb::unpack(cs, enq) && tlb::unpack_cell(enq.out_msg, env) && tlb::unpack_cell_inexact(env.msg, info))) {
+    return invalidate();
+  }
+  src_prefix_ = block::tlb::t_MsgAddressInt.get_prefix(std::move(info.src));
+  dest_prefix_ = block::tlb::t_MsgAddressInt.get_prefix(std::move(info.dest));
+  if (!(src_prefix_.is_valid() && dest_prefix_.is_valid())) {
+    return invalidate();
+  }
+  cur_prefix_ = interpolate_addr(src_prefix_, dest_prefix_, env.cur_addr);
+  next_prefix_ = interpolate_addr(src_prefix_, dest_prefix_, env.next_addr);
+  lt_ = info.created_lt;
+  enqueued_lt_ = enq.enqueued_lt;
+  hash_ = env.msg->get_hash().bits();
+  msg_ = std::move(env.msg);
+  msg_env_ = std::move(enq.out_msg);
+  return true;
+}
+
+bool EnqueuedMsgDescr::check_key(td::ConstBitPtr key) const {
+  return key.get_int(32) == next_prefix_.workchain && (key + 32).get_uint(64) == next_prefix_.account_id_prefix &&
+         hash_ == key + 96;
 }
 
 namespace tlb {
@@ -348,17 +401,17 @@ bool MsgAddressInt::validate_skip(vm::CellSlice& cs) const {
   return false;
 }
 
-ton::ShardIdFull MsgAddressInt::get_shard(vm::CellSlice&& cs) const {
+ton::AccountIdPrefixFull MsgAddressInt::get_prefix(vm::CellSlice&& cs) const {
   if (!cs.have(3 + 8 + 64)) {
     return {};
   }
   ton::WorkchainId workchain;
-  unsigned long long shard;
+  unsigned long long prefix;
   int t = (int)cs.prefetch_ulong(2 + 1 + 5);
   switch (t >> 5) {
     case 4: {  // addr_std$10, anycast=nothing$0
-      if (cs.advance(3) && cs.fetch_int_to(8, workchain) && cs.fetch_uint_to(64, shard)) {
-        return {workchain, shard};
+      if (cs.advance(3) && cs.fetch_int_to(8, workchain) && cs.fetch_uint_to(64, prefix)) {
+        return {workchain, prefix};
       }
       break;
     }
@@ -367,9 +420,9 @@ ton::ShardIdFull MsgAddressInt::get_shard(vm::CellSlice&& cs) const {
       unsigned long long rewrite;
       if (cs.advance(8) && cs.fetch_uint_to(t, rewrite)  // rewrite_pfx:(bits depth)
           && cs.fetch_int_to(8, workchain)               // workchain_id:int8
-          && cs.fetch_uint_to(64, shard)) {              // address:bits256
+          && cs.fetch_uint_to(64, prefix)) {             // address:bits256
         rewrite <<= 64 - t;
-        return {workchain, (shard & (std::numeric_limits<td::uint64>::max() >> t)) | rewrite};
+        return {workchain, (prefix & (std::numeric_limits<td::uint64>::max() >> t)) | rewrite};
       }
       break;
     }
@@ -378,8 +431,8 @@ ton::ShardIdFull MsgAddressInt::get_shard(vm::CellSlice&& cs) const {
       if (cs.advance(3) && cs.fetch_uint_to(9, len)  // addr_len:(## 9)
           && len >= 64                               // { len >= 64 }
           && cs.fetch_int_to(32, workchain)          // workchain_id:int32
-          && cs.fetch_uint_to(64, shard)) {          // address:(bits addr_len)
-        return {workchain, shard};
+          && cs.fetch_uint_to(64, prefix)) {         // address:(bits addr_len)
+        return {workchain, prefix};
       }
       break;
     }
@@ -391,27 +444,27 @@ ton::ShardIdFull MsgAddressInt::get_shard(vm::CellSlice&& cs) const {
           && cs.fetch_uint_to(9, len)                    // addr_len:(## 9)
           && len >= 64                                   // { len >= 64 }
           && cs.fetch_int_to(32, workchain)              // workchain_id:int32
-          && cs.fetch_uint_to(64, shard)) {              // address:bits256
+          && cs.fetch_uint_to(64, prefix)) {             // address:bits256
         rewrite <<= 64 - t;
-        return {workchain, (shard & (std::numeric_limits<td::uint64>::max() >> t)) | rewrite};
+        return {workchain, (prefix & (std::numeric_limits<td::uint64>::max() >> t)) | rewrite};
       }
       break;
     }
   }
   return {};
 }
 
-ton::ShardIdFull MsgAddressInt::get_shard(const vm::CellSlice& cs) const {
+ton::AccountIdPrefixFull MsgAddressInt::get_prefix(const vm::CellSlice& cs) const {
   vm::CellSlice cs2{cs};
-  return get_shard(std::move(cs2));
+  return get_prefix(std::move(cs2));
 }
 
-ton::ShardIdFull MsgAddressInt::get_shard(Ref<vm::CellSlice> cs_ref) const {
+ton::AccountIdPrefixFull MsgAddressInt::get_prefix(Ref<vm::CellSlice> cs_ref) const {
   if (cs_ref->is_unique()) {
-    return get_shard(std::move(cs_ref.unique_write()));
+    return get_prefix(std::move(cs_ref.unique_write()));
   } else {
     vm::CellSlice cs{*cs_ref};
-    return get_shard(std::move(cs));
+    return get_prefix(std::move(cs));
   }
 }
 
@@ -2349,35 +2402,35 @@ bool sub_extra_currency(Ref<vm::Cell> extra1, Ref<vm::Cell> extra2, Ref<vm::Cell
 }
 
 // combine d bits from dest, remaining 64 - d bits from src
-ton::ShardIdFull interpolate_addr(ton::ShardIdFull src, ton::ShardIdFull dest, int d) {
+ton::AccountIdPrefixFull interpolate_addr(ton::AccountIdPrefixFull src, ton::AccountIdPrefixFull dest, int d) {
   if (d <= 0) {
     return src;
   } else if (d >= 96) {
     return dest;
   } else if (d >= 32) {
     unsigned long long mask = (std::numeric_limits<td::uint64>::max() >> (d - 32));
-    return ton::ShardIdFull{dest.workchain, (dest.shard & ~mask) | (src.shard & mask)};
+    return ton::AccountIdPrefixFull{dest.workchain, (dest.account_id_prefix & ~mask) | (src.account_id_prefix & mask)};
   } else {
     int mask = (-1 >> d);
-    return ton::ShardIdFull{(dest.workchain & ~mask) | (src.workchain & mask), src.shard};
+    return ton::AccountIdPrefixFull{(dest.workchain & ~mask) | (src.workchain & mask), src.account_id_prefix};
   }
 }
 
 // result: (transit_addr_dest_bits, nh_addr_dest_bits)
-std::pair<int, int> perform_hypercube_routing(ton::ShardIdFull src, ton::ShardIdFull dest, ton::ShardIdFull cur,
-                                              int used_dest_bits) {
-  ton::ShardIdFull transit = interpolate_addr(src, dest, used_dest_bits);
+std::pair<int, int> perform_hypercube_routing(ton::AccountIdPrefixFull src, ton::AccountIdPrefixFull dest,
+                                              ton::ShardIdFull cur, int used_dest_bits) {
+  ton::AccountIdPrefixFull transit = interpolate_addr(src, dest, used_dest_bits);
   if (!ton::shard_contains(cur, transit)) {
     return {-1, -1};
   }
   if (transit.workchain != dest.workchain) {
     return {used_dest_bits, 32};
   }
-  if (transit.shard == dest.shard || ton::shard_contains(cur, dest)) {
+  if (transit.account_id_prefix == dest.account_id_prefix || ton::shard_contains(cur, dest)) {
     return {96, 96};
   }
   unsigned long long x = cur.shard & (cur.shard - 1), y = cur.shard | (cur.shard - 1);
-  unsigned long long t = transit.shard, q = dest.shard ^ t;
+  unsigned long long t = transit.account_id_prefix, q = dest.account_id_prefix ^ t;
   int i = (td::count_leading_zeroes64(q) & -4);  // top i bits match, next 4 bits differ
   unsigned long long m = (std::numeric_limits<td::uint64>::max() >> i), h;
   do {

ton-test-liteclient-full/lite-client/crypto/block/block.h

@@ -79,16 +79,40 @@ struct ShardId {
   void init();
 };
 
+struct EnqueuedMsgDescr {
+  ton::AccountIdPrefixFull src_prefix_, cur_prefix_, next_prefix_, dest_prefix_;
+  ton::LogicalTime lt_;
+  ton::LogicalTime enqueued_lt_;
+  ton::Bits256 hash_;
+  Ref<vm::Cell> msg_;
+  Ref<vm::Cell> msg_env_;
+  EnqueuedMsgDescr() = default;
+  EnqueuedMsgDescr(ton::AccountIdPrefixFull cur_pfx, ton::AccountIdPrefixFull next_pfx, ton::LogicalTime lt,
+                   ton::LogicalTime enqueued_lt, td::ConstBitPtr hash)
+      : cur_prefix_(cur_pfx), next_prefix_(next_pfx), lt_(lt), enqueued_lt_(enqueued_lt), hash_(hash) {
+  }
+  bool is_valid() const {
+    return next_prefix_.is_valid();
+  }
+  bool check_key(td::ConstBitPtr key) const;
+  bool invalidate() {
+    next_prefix_.workchain = cur_prefix_.workchain = ton::workchainInvalid;
+    return false;
+  }
+  bool unpack(vm::CellSlice& cs);
+};
+
+using compute_shard_end_lt_func_t = std::function<ton::LogicalTime(ton::AccountIdPrefixFull)>;
+
 struct MsgProcessedUpto {
   ton::ShardId shard;
   ton::BlockSeqno mc_seqno;
   ton::LogicalTime last_inmsg_lt;
   ton::Bits256 last_inmsg_hash;
-  ton::LogicalTime our_end_lt;  // corresponds to end_lt of our block seen from mc_seqno
+  compute_shard_end_lt_func_t compute_shard_end_lt;
   MsgProcessedUpto() = default;
-  MsgProcessedUpto(ton::ShardId _shard, ton::BlockSeqno _mcseqno, ton::LogicalTime _lt, td::ConstBitPtr _hash,
-                   ton::LogicalTime _endlt = 0)
-      : shard(_shard), mc_seqno(_mcseqno), last_inmsg_lt(_lt), last_inmsg_hash(_hash), our_end_lt(_endlt) {
+  MsgProcessedUpto(ton::ShardId _shard, ton::BlockSeqno _mcseqno, ton::LogicalTime _lt, td::ConstBitPtr _hash)
+      : shard(_shard), mc_seqno(_mcseqno), last_inmsg_lt(_lt), last_inmsg_hash(_hash) {
   }
   bool operator<(const MsgProcessedUpto& other) const & {
     return shard < other.shard || (shard == other.shard && mc_seqno < other.mc_seqno);
@@ -97,7 +121,7 @@ struct MsgProcessedUpto {
   bool contains(ton::ShardId other_shard, ton::LogicalTime other_lt, td::ConstBitPtr other_hash,
                 ton::BlockSeqno other_mc_seqno) const &;
   // NB: this is for checking whether we have already imported an internal message
-  bool already_processed(ton::ShardId msg_next_addr, ton::LogicalTime msg_lt, td::ConstBitPtr msg_hash) const;
+  bool already_processed(const EnqueuedMsgDescr& msg) const;
 };
 
 struct MsgProcessedUptoCollection {
@@ -108,9 +132,14 @@ struct MsgProcessedUptoCollection {
   }
   MsgProcessedUptoCollection(ton::ShardIdFull _owner, Ref<vm::CellSlice> cs_ref);
   static std::unique_ptr<MsgProcessedUptoCollection> unpack(ton::ShardIdFull _owner, Ref<vm::CellSlice> cs_ref);
+  bool is_valid() const {
+    return valid;
+  }
   bool insert(ton::LogicalTime last_proc_lt, td::ConstBitPtr last_proc_hash, ton::BlockSeqno mc_seqno);
   bool compactify();
   bool pack(vm::CellBuilder& cb);
+  // NB: this is for checking whether we have already imported an internal message
+  bool already_processed(const EnqueuedMsgDescr& msg) const;
 };
 
 namespace tlb {
@@ -337,9 +366,9 @@ struct MsgAddressInt final : TLB_Complex {
   int get_tag(const vm::CellSlice& cs) const override {
     return (int)cs.prefetch_ulong(2);
   }
-  ton::ShardIdFull get_shard(vm::CellSlice&& cs) const;
-  ton::ShardIdFull get_shard(const vm::CellSlice& cs) const;
-  ton::ShardIdFull get_shard(Ref<vm::CellSlice> cs_ref) const;
+  ton::AccountIdPrefixFull get_prefix(vm::CellSlice&& cs) const;
+  ton::AccountIdPrefixFull get_prefix(const vm::CellSlice& cs) const;
+  ton::AccountIdPrefixFull get_prefix(Ref<vm::CellSlice> cs_ref) const;
   bool extract_std_address(Ref<vm::CellSlice> cs_ref, ton::WorkchainId& workchain, ton::StdSmcAddress& addr,
                            bool rewrite = true) const;
   bool extract_std_address(vm::CellSlice& cs, ton::WorkchainId& workchain, ton::StdSmcAddress& addr,
@@ -875,6 +904,9 @@ struct EnqueuedMsg final : TLB_Complex {
     return cs.advance_ext(0x10040);
   }
   bool validate_skip(vm::CellSlice& cs) const override;
+  bool unpack(vm::CellSlice& cs, EnqueuedMsgDescr& descr) const {
+    return descr.unpack(cs);
+  }
 };
 
 extern const EnqueuedMsg t_EnqueuedMsg;
@@ -1071,10 +1103,11 @@ bool valid_config_data(Ref<vm::Cell> cell, const td::BitArray<256>& addr, bool c
 bool add_extra_currency(Ref<vm::Cell> extra1, Ref<vm::Cell> extra2, Ref<vm::Cell>& res);
 bool sub_extra_currency(Ref<vm::Cell> extra1, Ref<vm::Cell> extra2, Ref<vm::Cell>& res);
 
-ton::ShardIdFull interpolate_addr(ton::ShardIdFull src, ton::ShardIdFull dest, int used_dest_bits);
+ton::AccountIdPrefixFull interpolate_addr(ton::AccountIdPrefixFull src, ton::AccountIdPrefixFull dest,
+                                          int used_dest_bits);
 // result: (transit_addr_dest_bits, nh_addr_dest_bits)
-std::pair<int, int> perform_hypercube_routing(ton::ShardIdFull src, ton::ShardIdFull dest, ton::ShardIdFull cur,
-                                              int used_dest_bits = 0);
+std::pair<int, int> perform_hypercube_routing(ton::AccountIdPrefixFull src, ton::AccountIdPrefixFull dest,
+                                              ton::ShardIdFull cur, int used_dest_bits = 0);
 
 bool unpack_block_prev_blk(Ref<vm::Cell> block_root, ton::BlockIdExt& id, std::vector<ton::BlockIdExt>& prev,
                            ton::BlockIdExt& mc_blkid, bool& after_split);

ton-test-liteclient-full/lite-client/crypto/block/block.tlb

@@ -186,7 +186,7 @@ msg_export_new$001 out_msg:^MsgEnvelope
     transaction:^Transaction = OutMsg;
 msg_export_tr$011  out_msg:^MsgEnvelope 
     imported:^InMsg = OutMsg;
-msg_export_deq$110 out_msg:^MsgEnvelope 
+msg_export_deq$110 out_msg:^MsgEnvelope  // out_msg_hash:bits256 ?
     import_block_lt:uint64 = OutMsg;
 msg_export_tr_req$111 out_msg:^MsgEnvelope 
     imported:^InMsg = OutMsg;