updated liteserver

- new methods for liteserver/liteclient - added ADNL/DHT client-only work mode - fixed crash in ADNL
imtabako · Feb 2, 2020 · 53ec968 · 53ec968
1 parent acf1671
commit 53ec968
Show file tree

Hide file tree

Showing 70 changed files with 816 additions and 322 deletions.
diff --git a/GPLv2 b/GPLv2
@@ -23,5 +23,5 @@
     exception statement from your version. If you delete this exception statement 
     from all source files in the program, then also delete it here.
 
-    Copyright 2017-2019 Telegram Systems LLP
+    Copyright 2017-2020 Telegram Systems LLP
 */
diff --git a/LGPLv2 b/LGPLv2
@@ -14,5 +14,5 @@
     You should have received a copy of the GNU Lesser General Public License
     along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
 
-    Copyright 2017-2019 Telegram Systems LLP
+    Copyright 2017-2020 Telegram Systems LLP
 */
diff --git a/adnl/adnl-address-list.cpp b/adnl/adnl-address-list.cpp
@@ -155,6 +155,16 @@ td::Result<AdnlAddressList> AdnlAddressList::create(const tl_object_ptr<ton_api:
   return A;
 }
 
+td::Status AdnlAddressList::add_udp_address(td::IPAddress addr) {
+  if (addr.is_ipv4()) {
+    auto r = td::make_ref<AdnlAddressUdp>(addr.get_ipv4(), static_cast<td::uint16>(addr.get_port()));
+    addrs_.push_back(std::move(r));
+    return td::Status::OK();
+  } else {
+    return td::Status::Error(ErrorCode::protoviolation, "only works with ipv4");
+  }
+}
+
 }  // namespace adnl
 
 }  // namespace ton
diff --git a/adnl/adnl-address-list.h b/adnl/adnl-address-list.h
@@ -88,6 +88,7 @@ class AdnlAddressList {
   void add_addr(AdnlAddress addr) {
     addrs_.push_back(addr);
   }
+  void update(td::IPAddress addr);
   bool public_only() const;
   td::uint32 size() const {
     return static_cast<td::uint32>(addrs_.size());
@@ -98,6 +99,7 @@ class AdnlAddressList {
   }
 
   static td::Result<AdnlAddressList> create(const tl_object_ptr<ton_api::adnl_addressList> &addr_list);
+  td::Status add_udp_address(td::IPAddress addr);
 };
 
 }  // namespace adnl

diff --git a/adnl/adnl-channel.cpp b/adnl/adnl-channel.cpp
@@ -111,13 +111,15 @@ void AdnlChannelImpl::send_message(td::uint32 priority, td::actor::ActorId<AdnlN
   td::actor::send_closure(conn, &AdnlNetworkConnection::send, local_id_, peer_id_, priority, std::move(B));
 }
 
-void AdnlChannelImpl::receive(td::BufferSlice data) {
+void AdnlChannelImpl::receive(td::IPAddress addr, td::BufferSlice data) {
   auto P = td::PromiseCreator::lambda(
-      [peer = peer_pair_, channel_id = channel_in_id_, id = print_id()](td::Result<AdnlPacket> R) {
+      [peer = peer_pair_, channel_id = channel_in_id_, addr, id = print_id()](td::Result<AdnlPacket> R) {
         if (R.is_error()) {
           VLOG(ADNL_WARNING) << id << ": dropping IN message: can not decrypt: " << R.move_as_error();
         } else {
-          td::actor::send_closure(peer, &AdnlPeerPair::receive_packet_from_channel, channel_id, R.move_as_ok());
+          auto packet = R.move_as_ok();
+          packet.set_remote_addr(addr);
+          td::actor::send_closure(peer, &AdnlPeerPair::receive_packet_from_channel, channel_id, std::move(packet));
         }
       });
 

diff --git a/adnl/adnl-channel.h b/adnl/adnl-channel.h
@@ -35,7 +35,7 @@ class AdnlChannel : public td::actor::Actor {
                                                              AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id,
                                                              AdnlChannelIdShort &out_id, AdnlChannelIdShort &in_id,
                                                              td::actor::ActorId<AdnlPeerPair> peer_pair);
-  virtual void receive(td::BufferSlice data) = 0;
+  virtual void receive(td::IPAddress addr, td::BufferSlice data) = 0;
   virtual void send_message(td::uint32 priority, td::actor::ActorId<AdnlNetworkConnection> conn,
                             td::BufferSlice data) = 0;
   virtual ~AdnlChannel() = default;

diff --git a/adnl/adnl-channel.hpp b/adnl/adnl-channel.hpp
@@ -33,7 +33,7 @@ class AdnlChannelImpl : public AdnlChannel {
                   AdnlChannelIdShort in_id, AdnlChannelIdShort out_id, std::unique_ptr<Encryptor> encryptor,
                   std::unique_ptr<Decryptor> decryptor);
   void decrypt(td::BufferSlice data, td::Promise<AdnlPacket> promise);
-  void receive(td::BufferSlice data) override;
+  void receive(td::IPAddress addr, td::BufferSlice data) override;
   void send_message(td::uint32 priority, td::actor::ActorId<AdnlNetworkConnection> conn, td::BufferSlice data) override;
 
   struct AdnlChannelPrintId {

diff --git a/adnl/adnl-local-id.cpp b/adnl/adnl-local-id.cpp
@@ -40,13 +40,15 @@ AdnlAddressList AdnlLocalId::get_addr_list() const {
   return addr_list_;
 }
 
-void AdnlLocalId::receive(td::BufferSlice data) {
+void AdnlLocalId::receive(td::IPAddress addr, td::BufferSlice data) {
   auto P = td::PromiseCreator::lambda(
-      [peer_table = peer_table_, dst = short_id_, id = print_id()](td::Result<AdnlPacket> R) {
+      [peer_table = peer_table_, dst = short_id_, addr, id = print_id()](td::Result<AdnlPacket> R) {
         if (R.is_error()) {
           VLOG(ADNL_WARNING) << id << ": dropping IN message: cannot decrypt: " << R.move_as_error();
         } else {
-          td::actor::send_closure(peer_table, &AdnlPeerTable::receive_decrypted_packet, dst, R.move_as_ok());
+          auto packet = R.move_as_ok();
+          packet.set_remote_addr(addr);
+          td::actor::send_closure(peer_table, &AdnlPeerTable::receive_decrypted_packet, dst, std::move(packet));
         }
       });
 
@@ -117,7 +119,7 @@ void AdnlLocalId::update_address_list(AdnlAddressList addr_list) {
 }
 
 void AdnlLocalId::publish_address_list() {
-  if (dht_node_.empty() || addr_list_.empty()) {
+  if (dht_node_.empty() || addr_list_.empty() || addr_list_.size() == 0) {
     VLOG(ADNL_NOTICE) << this << ": skipping public addr list, because localid (or dht node) not fully initialized";
     return;
   }
@@ -178,7 +180,8 @@ AdnlLocalId::AdnlLocalId(AdnlNodeIdFull id, AdnlAddressList addr_list, td::actor
   id_ = std::move(id);
   short_id_ = id_.compute_short_id();
   addr_list_ = std::move(addr_list);
-  if (addr_list_.addrs().size() > 0) {
+  if (!addr_list_.empty()) {
+    addr_list_.set_reinit_date(Adnl::adnl_start_time());
     addr_list_.set_version(static_cast<td::int32>(td::Clocks::system()));
   }
   peer_table_ = peer_table;

diff --git a/adnl/adnl-local-id.h b/adnl/adnl-local-id.h
@@ -54,7 +54,7 @@ class AdnlLocalId : public td::actor::Actor {
   void decrypt_message(td::BufferSlice data, td::Promise<td::BufferSlice> promise);
   void deliver(AdnlNodeIdShort src, td::BufferSlice data);
   void deliver_query(AdnlNodeIdShort src, td::BufferSlice data, td::Promise<td::BufferSlice> promise);
-  void receive(td::BufferSlice data);
+  void receive(td::IPAddress addr, td::BufferSlice data);
 
   void subscribe(std::string prefix, std::unique_ptr<AdnlPeerTable::Callback> callback);
   void unsubscribe(std::string prefix);

diff --git a/adnl/adnl-network-manager.cpp b/adnl/adnl-network-manager.cpp
@@ -59,7 +59,7 @@ void AdnlNetworkManagerImpl::receive_udp_message(td::UdpMessage message) {
   }
   received_messages_++;
   if (received_messages_ % 64 == 0) {
-    VLOG(ADNL_DEBUG) << this << ": received " << received_messages_ << "udp messages";
+    VLOG(ADNL_DEBUG) << this << ": received " << received_messages_ << " udp messages";
   }
 
   VLOG(ADNL_EXTRA_DEBUG) << this << ": received message of size " << message.data.size();

diff --git a/adnl/adnl-packet.h b/adnl/adnl-packet.h
@@ -132,6 +132,9 @@ class AdnlPacket {
   auto signature() const {
     return signature_.clone();
   }
+  auto remote_addr() const {
+    return remote_addr_;
+  }
 
   void init_random();
 
@@ -188,6 +191,10 @@ class AdnlPacket {
     flags_ |= Flags::f_reinit_date;
   }
 
+  void set_remote_addr(td::IPAddress addr) {
+    remote_addr_ = addr;
+  }
+
  private:
   td::BufferSlice rand1_;
   td::uint32 flags_{0};
@@ -204,6 +211,8 @@ class AdnlPacket {
   td::int32 dst_reinit_date_{0};
   td::BufferSlice signature_;
   td::BufferSlice rand2_;
+
+  td::IPAddress remote_addr_;
 };
 
 }  // namespace adnl

diff --git a/adnl/adnl-peer-table.cpp b/adnl/adnl-peer-table.cpp
@@ -49,7 +49,7 @@ td::actor::ActorOwn<Adnl> Adnl::create(std::string db, td::actor::ActorId<keyrin
   return td::actor::ActorOwn<Adnl>(td::actor::create_actor<AdnlPeerTableImpl>("PeerTable", db, keyring));
 }
 
-void AdnlPeerTableImpl::receive_packet(td::BufferSlice data) {
+void AdnlPeerTableImpl::receive_packet(td::IPAddress addr, td::BufferSlice data) {
   if (data.size() < 32) {
     VLOG(ADNL_WARNING) << this << ": dropping IN message [?->?]: message too short: len=" << data.size();
     return;
@@ -60,14 +60,14 @@ void AdnlPeerTableImpl::receive_packet(td::BufferSlice data) {
 
   auto it = local_ids_own_.find(dst);
   if (it != local_ids_own_.end()) {
-    td::actor::send_closure(it->second, &AdnlLocalId::receive, std::move(data));
+    td::actor::send_closure(it->second, &AdnlLocalId::receive, addr, std::move(data));
     return;
   }
 
   AdnlChannelIdShort dst_chan_id{dst.pubkey_hash()};
   auto it2 = channels_.find(dst_chan_id);
   if (it2 != channels_.end()) {
-    td::actor::send_closure(it2->second, &AdnlChannel::receive, std::move(data));
+    td::actor::send_closure(it2->second, &AdnlChannel::receive, addr, std::move(data));
     return;
   }
 
@@ -237,7 +237,7 @@ void AdnlPeerTableImpl::register_network_manager(td::actor::ActorId<AdnlNetworkM
   class Cb : public AdnlNetworkManager::Callback {
    public:
     void receive_packet(td::IPAddress addr, td::BufferSlice data) override {
-      td::actor::send_closure(id_, &AdnlPeerTableImpl::receive_packet, std::move(data));
+      td::actor::send_closure(id_, &AdnlPeerTableImpl::receive_packet, addr, std::move(data));
     }
     Cb(td::actor::ActorId<AdnlPeerTableImpl> id) : id_(id) {
     }

diff --git a/adnl/adnl-peer-table.h b/adnl/adnl-peer-table.h
@@ -89,7 +89,7 @@ class AdnlPeerTable : public Adnl {
 
   virtual void answer_query(AdnlNodeIdShort src, AdnlNodeIdShort dst, AdnlQueryId query_id, td::BufferSlice data) = 0;
 
-  virtual void receive_packet(td::BufferSlice data) = 0;
+  virtual void receive_packet(td::IPAddress addr, td::BufferSlice data) = 0;
   virtual void receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket packet) = 0;
   virtual void send_message_in(AdnlNodeIdShort src, AdnlNodeIdShort dst, AdnlMessage message) = 0;
 

diff --git a/adnl/adnl-peer-table.hpp b/adnl/adnl-peer-table.hpp
@@ -43,7 +43,7 @@ class AdnlPeerTableImpl : public AdnlPeerTable {
   void add_peer(AdnlNodeIdShort local_id, AdnlNodeIdFull id, AdnlAddressList addr_list) override;
   void add_static_nodes_from_config(AdnlNodesList nodes) override;
 
-  void receive_packet(td::BufferSlice data) override;
+  void receive_packet(td::IPAddress addr, td::BufferSlice data) override;
   void receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket data) override;
   void send_message_in(AdnlNodeIdShort src, AdnlNodeIdShort dst, AdnlMessage message) override;
   void send_message(AdnlNodeIdShort src, AdnlNodeIdShort dst, td::BufferSlice data) override {

diff --git a/adnl/adnl-peer.cpp b/adnl/adnl-peer.cpp
@@ -131,7 +131,12 @@ void AdnlPeerPairImpl::receive_packet_checked(AdnlPacket packet) {
   }
   if (packet.dst_reinit_date() > 0 && packet.dst_reinit_date() < d) {
     if (!packet.addr_list().empty()) {
-      update_addr_list(packet.addr_list());
+      auto addr_list = packet.addr_list();
+      if (packet.remote_addr().is_valid() && addr_list.size() == 0) {
+        VLOG(ADNL_DEBUG) << "adding implicit address " << packet.remote_addr();
+        addr_list.add_udp_address(packet.remote_addr());
+      }
+      update_addr_list(std::move(addr_list));
     }
     if (!packet.priority_addr_list().empty()) {
       update_addr_list(packet.priority_addr_list());
@@ -174,7 +179,12 @@ void AdnlPeerPairImpl::receive_packet_checked(AdnlPacket packet) {
   }
 
   if (!packet.addr_list().empty()) {
-    update_addr_list(packet.addr_list());
+    auto addr_list = packet.addr_list();
+    if (packet.remote_addr().is_valid() && addr_list.size() == 0) {
+      VLOG(ADNL_DEBUG) << "adding implicit address " << packet.remote_addr();
+      addr_list.add_udp_address(packet.remote_addr());
+    }
+    update_addr_list(std::move(addr_list));
   }
   if (!packet.priority_addr_list().empty()) {
     update_addr_list(packet.priority_addr_list());
@@ -642,7 +652,7 @@ void AdnlPeerPairImpl::update_addr_list(AdnlAddressList addr_list) {
   if (addr_list.empty()) {
     return;
   }
-  CHECK(addr_list.size() > 0);
+  //CHECK(addr_list.size() > 0);
 
   if (addr_list.reinit_date() > td::Clocks::system() + 60) {
     VLOG(ADNL_WARNING) << "dropping addr list with too new reinit date";

diff --git a/blockchain-explorer/blockchain-explorer-http.cpp b/blockchain-explorer/blockchain-explorer-http.cpp
@@ -23,7 +23,7 @@
     exception statement from your version. If you delete this exception statement
     from all source files in the program, then also delete it here.
 
-    Copyright 2017-2019 Telegram Systems LLP
+    Copyright 2017-2020 Telegram Systems LLP
 */
 #include "blockchain-explorer-http.hpp"
 #include "block/block-db.h"
@@ -36,6 +36,8 @@
 #include "block/mc-config.h"
 #include "ton/ton-shard.h"
 
+bool local_scripts{false};
+
 HttpAnswer& HttpAnswer::operator<<(AddressCell addr_c) {
   ton::WorkchainId wc;
   ton::StdSmcAddress addr;
@@ -425,7 +427,7 @@ HttpAnswer& HttpAnswer::operator<<(AccountCell acc_c) {
 
 HttpAnswer& HttpAnswer::operator<<(BlockHeaderCell head_c) {
   *this << "<div>";
-  vm::CellSlice cs{vm::NoVm{}, head_c.root};
+  vm::CellSlice cs{vm::NoVm(), head_c.root};
   auto block_id = head_c.block_id;
   try {
     auto virt_root = vm::MerkleProof::virtualize(head_c.root, 1);
@@ -676,13 +678,17 @@ std::string HttpAnswer::header() {
            "maximum-scale=1.0, user-scalable=no\" />\n"
         << "<meta name=\"format-detection\" content=\"telephone=no\" />\n"
         << "<!-- Latest compiled and minified CSS -->\n"
-        << "<link rel=\"stylesheet\" href=\"https://maxcdn.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css\">\n"
+        << "<link rel=\"stylesheet\" href=\"" << (local_scripts ? "/" : "https://")
+        << "maxcdn.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css\">\n"
         << "<!-- jQuery library -->"
-        << "<script src=\"https://ajax.googleapis.com/ajax/libs/jquery/3.4.0/jquery.min.js\"></script>\n"
+        << "<script src=\"" << (local_scripts ? "/" : "https://")
+        << "ajax.googleapis.com/ajax/libs/jquery/3.4.0/jquery.min.js\"></script>\n"
         << "<!-- Popper JS -->\n"
-        << "<script src=\"https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js\"></script>\n"
+        << "<script src=\"" << (local_scripts ? "/" : "https://")
+        << "cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js\"></script>\n"
         << "<!-- Latest compiled JavaScript -->\n"
-        << "<script src=\"https://maxcdn.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js\"></script>\n"
+        << "<script src=\"" << (local_scripts ? "/" : "https://")
+        << "maxcdn.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js\"></script>\n"
         << "</head><body>\n"
         << "<div class=\"container-fluid\">\n"
         << "<nav class=\"navbar navbar-expand px-0 mt-1 flex-wrap\">\n"

diff --git a/blockchain-explorer/blockchain-explorer-http.hpp b/blockchain-explorer/blockchain-explorer-http.hpp
@@ -33,6 +33,8 @@
 #include "td/utils/Random.h"
 #include "block/block.h"
 
+extern bool local_scripts;
+
 class HttpAnswer {
  public:
   struct MessageCell {

diff --git a/blockchain-explorer/blockchain-explorer.cpp b/blockchain-explorer/blockchain-explorer.cpp
@@ -639,6 +639,10 @@ int main(int argc, char* argv[]) {
     td::actor::send_closure(x, &CoreActor::set_http_port, td::to_integer<td::uint32>(arg));
     return td::Status::OK();
   });
+  p.add_option('L', "local-scripts", "use local copy of ajax/bootstrap/... JS", [&]() {
+    local_scripts = true;
+    return td::Status::OK();
+  });
 #if TD_DARWIN || TD_LINUX
   p.add_option('l', "logname", "log to file", [&](td::Slice fname) {
     auto FileLog = td::FileFd::open(td::CSlice(fname.str().c_str()),

diff --git a/blockchain-explorer/blockchain-explorer.hpp b/blockchain-explorer/blockchain-explorer.hpp
@@ -34,6 +34,8 @@
 
 #define MAX_POST_SIZE (64 << 10)
 
+extern bool local_scripts_;
+
 class CoreActorInterface : public td::actor::Actor {
  public:
   struct RemoteNodeStatus {

diff --git a/crypto/block/check-proof.cpp b/crypto/block/check-proof.cpp
@@ -14,7 +14,7 @@
     You should have received a copy of the GNU Lesser General Public License
     along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
 
-    Copyright 2017-2019 Telegram Systems LLP
+    Copyright 2017-2020 Telegram Systems LLP
 */
 #include "check-proof.h"
 #include "block/block.h"
@@ -219,7 +219,17 @@ td::Status check_account_proof(td::Slice proof, ton::BlockIdExt shard_blk, const
 }
 
 td::Result<AccountState::Info> AccountState::validate(ton::BlockIdExt ref_blk, block::StdAddress addr) const {
-  TRY_RESULT_PREFIX(root, vm::std_boc_deserialize(state.as_slice(), true), "cannot deserialize account state");
+  TRY_RESULT_PREFIX(true_root, vm::std_boc_deserialize(state.as_slice(), true), "cannot deserialize account state");
+  Ref<vm::Cell> root;
+
+  if (is_virtualized && true_root.not_null()) {
+    root = vm::MerkleProof::virtualize(true_root, 1);
+    if (root.is_null()) {
+      return td::Status::Error("account state proof is invalid");
+    }
+  } else {
+    root = true_root;
+  }
 
   if (blk != ref_blk && ref_blk.id.seqno != ~0U) {
     return td::Status::Error(PSLICE() << "obtained getAccountState() for a different reference block " << blk.to_str()
@@ -241,6 +251,7 @@ td::Result<AccountState::Info> AccountState::validate(ton::BlockIdExt ref_blk, b
   TRY_STATUS(block::check_account_proof(proof.as_slice(), shard_blk, addr, root, &res.last_trans_lt,
                                         &res.last_trans_hash, &res.gen_utime, &res.gen_lt));
   res.root = std::move(root);
+  res.true_root = std::move(true_root);
 
   return res;
 }

diff --git a/crypto/block/check-proof.h b/crypto/block/check-proof.h
@@ -14,7 +14,7 @@
     You should have received a copy of the GNU Lesser General Public License
     along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
 
-    Copyright 2017-2019 Telegram Systems LLP
+    Copyright 2017-2020 Telegram Systems LLP
 */
 #pragma once
 
@@ -44,10 +44,11 @@ struct AccountState {
   td::BufferSlice shard_proof;
   td::BufferSlice proof;
   td::BufferSlice state;
+  bool is_virtualized{false};
 
   struct Info {
-    td::Ref<vm::Cell> root;
-    ton::LogicalTime last_trans_lt = 0;
+    td::Ref<vm::Cell> root, true_root;
+    ton::LogicalTime last_trans_lt{0};
     ton::Bits256 last_trans_hash;
     ton::LogicalTime gen_lt{0};
     td::uint32 gen_utime{0};