update tdutils from another project

GitOrigin-RevId: 79b8eb2ba229d733f82dbb17b8bd7f27471c9472

td/telegram/StorageManager.cpp

@@ -340,7 +340,7 @@ void StorageManager::schedule_next_gc() {
   if (next_gc_at > sys_time + GC_EACH) {
     next_gc_at = sys_time + GC_EACH;
   }
-  next_gc_at += Random::fast(GC_DELAY, GC_DELAY + GC_RAND_DELAY);
+  next_gc_at += Random::fast(static_cast<int>(GC_DELAY), static_cast<int>(GC_DELAY + GC_RAND_DELAY));
   CHECK(next_gc_at >= sys_time);
   auto next_gc_in = next_gc_at - sys_time;
 

td/telegram/files/FileStatsWorker.cpp

@@ -102,7 +102,7 @@ struct FsFileInfo {
 template <class CallbackT>
 void scan_fs(CancellationToken &token, CallbackT &&callback) {
   for (int32 i = 0; i < MAX_FILE_TYPE; i++) {
-    int32 main_file_type = static_cast<size_t>(get_main_file_type(static_cast<FileType>(i)));
+    int32 main_file_type = static_cast<int32>(get_main_file_type(static_cast<FileType>(i)));
     if (i != main_file_type) {
       continue;
     }

tdutils/CMakeLists.txt

@@ -1,5 +1,15 @@
 cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR)
 
+option(TDUTILS_MIME_TYPE "Generate mime types conversion (gperf is required)" ON)
+
+if (WIN32)
+  if (WINGETOPT_FOUND)
+    set(TD_HAVE_GETOPT 1)
+  endif()
+else()
+  set(TD_HAVE_GETOPT 1)
+endif()
+
 if (NOT DEFINED CMAKE_INSTALL_LIBDIR)
   set(CMAKE_INSTALL_LIBDIR "lib")
 endif()
@@ -59,6 +69,7 @@ set(TDUTILS_SOURCE
   td/utils/port/Stat.cpp
   td/utils/port/StdStreams.cpp
   td/utils/port/thread_local.cpp
+  td/utils/port/user.cpp
   td/utils/port/UdpSocketFd.cpp
   td/utils/port/uname.cpp
   td/utils/port/user.cpp
@@ -95,7 +106,6 @@ set(TDUTILS_SOURCE
   td/utils/JsonBuilder.cpp
   td/utils/logging.cpp
   td/utils/misc.cpp
-  td/utils/MimeType.cpp
   td/utils/MpmcQueue.cpp
   td/utils/OptionParser.cpp
   td/utils/PathView.cpp
@@ -105,9 +115,10 @@ set(TDUTILS_SOURCE
   td/utils/StackAllocator.cpp
   td/utils/Status.cpp
   td/utils/StringBuilder.cpp
-  td/utils/tests.cpp
   td/utils/Time.cpp
   td/utils/Timer.cpp
+  td/utils/TsFileLog.cpp
+  td/utils/tests.cpp
   td/utils/tl_parsers.cpp
   td/utils/translit.cpp
   td/utils/TsFileLog.cpp
@@ -139,6 +150,7 @@ set(TDUTILS_SOURCE
   td/utils/port/StdStreams.h
   td/utils/port/thread.h
   td/utils/port/thread_local.h
+  td/utils/port/user.h
   td/utils/port/UdpSocketFd.h
   td/utils/port/uname.h
   td/utils/port/user.h
@@ -205,7 +217,6 @@ set(TDUTILS_SOURCE
   td/utils/List.h
   td/utils/logging.h
   td/utils/MemoryLog.h
-  td/utils/MimeType.h
   td/utils/misc.h
   td/utils/MovableValue.h
   td/utils/MpmcQueue.h
@@ -232,6 +243,7 @@ set(TDUTILS_SOURCE
   td/utils/SpinLock.h
   td/utils/StackAllocator.h
   td/utils/Status.h
+  td/utils/StealingQueue.h
   td/utils/Storer.h
   td/utils/StorerBase.h
   td/utils/StringBuilder.h
@@ -241,6 +253,7 @@ set(TDUTILS_SOURCE
   td/utils/Time.h
   td/utils/TimedStat.h
   td/utils/Timer.h
+  td/utils/TsFileLog.h
   td/utils/tl_helpers.h
   td/utils/tl_parsers.h
   td/utils/tl_storers.h
@@ -257,12 +270,19 @@ set(TDUTILS_SOURCE
   td/utils/VectorQueue.h
 )
 
+if (TDUTILS_MIME_TYPE)
+  set(TDUTILS_SOURCE
+    ${TDUTILS_SOURCE}
+    td/utils/MimeType.cpp
+    td/utils/MimeType.h
+  )
+endif()
+
 set(TDUTILS_TEST_SOURCE
   ${CMAKE_CURRENT_SOURCE_DIR}/test/buffer.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/ConcurrentHashMap.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/crypto.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/Enumerator.cpp
-  ${CMAKE_CURRENT_SOURCE_DIR}/test/EpochBasedMemoryReclamation.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/filesystem.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/gzip.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/HazardPointers.cpp
@@ -280,6 +300,7 @@ set(TDUTILS_TEST_SOURCE
   ${CMAKE_CURRENT_SOURCE_DIR}/test/pq.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/SharedObjectPool.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/SharedSlice.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/test/StealingQueue.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/variant.cpp
   PARENT_SCOPE
 )
@@ -293,7 +314,7 @@ if (WIN32)
   # target_link_libraries(tdutils PRIVATE ${WS2_32_LIBRARY} ${MSWSOCK_LIBRARY})
   target_link_libraries(tdutils PRIVATE ws2_32 Mswsock Normaliz psapi)
 endif()
-if (NOT CMAKE_CROSSCOMPILING)
+if (NOT CMAKE_CROSSCOMPILING AND TDUTILS_MIME_TYPE)
   add_dependencies(tdutils tdmime_auto)
 endif()
 
@@ -316,7 +337,11 @@ if (CRC32C_FOUND)
   target_link_libraries(tdutils PRIVATE crc32c)
 endif()
 if (ABSL_FOUND)
-  target_link_libraries(tdutils PUBLIC absl::flat_hash_map absl::flat_hash_set absl::hash)
+  target_link_libraries_system(tdutils absl::flat_hash_map absl::flat_hash_set absl::hash)
+endif()
+
+if (WIN32 AND WINGETOPT_FOUND)
+  target_link_libraries(tdutils PRIVATE wingetopt)
 endif()
 
 if (ANDROID)

tdutils/generate/CMakeLists.txt

@@ -3,6 +3,10 @@ cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR)
 # Generates files for MIME type <-> extension conversions
 # DEPENDS ON: gperf grep bash/powershell
 
+if (NOT TDUTILS_MIME_TYPE)
+  return()
+endif()
+
 file(MAKE_DIRECTORY auto)
 
 set(TDMIME_SOURCE

tdutils/td/utils/BigNum.cpp

@@ -88,7 +88,9 @@ BigNum BigNum::from_binary(Slice str) {
 }
 
 BigNum BigNum::from_le_binary(Slice str) {
-#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER)
+#if defined(OPENSSL_IS_BORINGSSL)
+  return BigNum(make_unique<Impl>(BN_le2bn(str.ubegin(), narrow_cast<int>(str.size()), nullptr)));
+#elif OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER)
   return BigNum(make_unique<Impl>(BN_lebin2bn(str.ubegin(), narrow_cast<int>(str.size()), nullptr)));
 #else
   string str_copy = str.str();
@@ -204,15 +206,19 @@ string BigNum::to_binary(int exact_size) const {
 }
 
 string BigNum::to_le_binary(int exact_size) const {
-#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER)
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER) || defined(OPENSSL_IS_BORINGSSL)
   int num_size = get_num_bytes();
   if (exact_size == -1) {
     exact_size = num_size;
   } else {
     CHECK(exact_size >= num_size);
   }
   string res(exact_size, '\0');
+#if defined(OPENSSL_IS_BORINGSSL)
+  BN_bn2le_padded(MutableSlice(res).ubegin(), exact_size, impl_->big_num);
+#else
   BN_bn2lebinpad(impl_->big_num, MutableSlice(res).ubegin(), exact_size);
+#endif
   return res;
 #else
   string result = to_binary(exact_size);

tdutils/td/utils/BufferedFd.h

@@ -72,6 +72,12 @@ class BufferedFd : public BufferedFdBase<FdT> {
   ~BufferedFd();
 
   void close();
+  size_t left_unread() {
+    return input_reader_.size();
+  }
+  size_t left_unwritten() {
+    return output_reader_.size();
+  }
 
   Result<size_t> flush_read(size_t max_read = std::numeric_limits<size_t>::max()) TD_WARN_UNUSED_RESULT;
   Result<size_t> flush_write() TD_WARN_UNUSED_RESULT;

tdutils/td/utils/CancellationToken.h

@@ -20,8 +20,13 @@ struct RawCancellationToken {
 class CancellationToken {
  public:
   explicit operator bool() const {
+    // Empty CancellationToken is never cancelled
+    if (!token_) {
+      return false;
+    }
     return token_->is_cancelled_.load(std::memory_order_acquire);
   }
+  CancellationToken() = default;
   explicit CancellationToken(std::shared_ptr<detail::RawCancellationToken> token) : token_(std::move(token)) {
   }
 

tdutils/td/utils/FileLog.cpp

@@ -82,7 +82,7 @@ void FileLog::append(CSlice cslice, int log_level) {
     process_fatal_error(cslice);
   }
 
-  if (size_ > rotate_threshold_) {
+  if (size_ > rotate_threshold_ || want_rotate_.load(std::memory_order_relaxed)) {
     auto status = rename(path_, PSLICE() << path_ << ".old");
     if (status.is_error()) {
       process_fatal_error(PSLICE() << status.error() << " in " << __FILE__ << " at " << __LINE__);
@@ -98,9 +98,13 @@ void FileLog::rotate() {
   do_rotate();
 }
 
+void FileLog::lazy_rotate() {
+  want_rotate_ = true;
+}
+
 void FileLog::do_rotate() {
-  auto current_verbosity_level = GET_VERBOSITY_LEVEL();
-  SET_VERBOSITY_LEVEL(std::numeric_limits<int>::min());  // to ensure that nothing will be printed to the closed log
+  want_rotate_ = false;
+  td::ScopedDisableLog disable_log;  // to ensure that nothing will be printed to the closed log
   CHECK(!path_.empty());
   fd_.close();
   auto r_fd = FileFd::open(path_, FileFd::Create | FileFd::Truncate | FileFd::Write);
@@ -112,7 +116,12 @@ void FileLog::do_rotate() {
     fd_.get_native_fd().duplicate(Stderr().get_native_fd()).ignore();
   }
   size_ = 0;
-  SET_VERBOSITY_LEVEL(current_verbosity_level);
+}
+
+Result<td::unique_ptr<LogInterface>> FileLog::create(string path, int64 rotate_threshold, bool redirect_stderr) {
+  auto l = make_unique<FileLog>();
+  TRY_STATUS(l->init(std::move(path), rotate_threshold, redirect_stderr));
+  return std::move(l);
 }
 
 }  // namespace td

tdutils/td/utils/FileLog.h

@@ -18,6 +18,8 @@ class FileLog : public LogInterface {
   static constexpr int64 DEFAULT_ROTATE_THRESHOLD = 10 * (1 << 20);
 
  public:
+  static Result<td::unique_ptr<LogInterface>> create(string path, int64 rotate_threshold = DEFAULT_ROTATE_THRESHOLD,
+                                                     bool redirect_stderr = true);
   Status init(string path, int64 rotate_threshold = DEFAULT_ROTATE_THRESHOLD, bool redirect_stderr = true);
 
   Slice get_path() const;
@@ -32,12 +34,15 @@ class FileLog : public LogInterface {
 
   void rotate() override;
 
+  void lazy_rotate();
+
  private:
   FileFd fd_;
   string path_;
   int64 size_ = 0;
   int64 rotate_threshold_ = 0;
   bool redirect_stderr_ = false;
+  std::atomic<bool> want_rotate_{};
 
   void do_rotate();
 };

tdutils/td/utils/Heap.h

@@ -37,6 +37,10 @@ class KHeap {
     return array_[0].key_;
   }
 
+  HeapNode *top() const {
+    return array_[0].node_;
+  }
+
   HeapNode *pop() {
     CHECK(!empty());
     HeapNode *result = array_[0].node_;

tdutils/td/utils/JsonBuilder.h

@@ -172,7 +172,7 @@ class JsonObjectScope;
 
 class JsonBuilder {
  public:
-  explicit JsonBuilder(StringBuilder &&sb, int32 offset = -1) : sb_(std::move(sb)), offset_(offset) {
+  explicit JsonBuilder(StringBuilder &&sb = {}, int32 offset = -1) : sb_(std::move(sb)), offset_(offset) {
   }
   StringBuilder &string_builder() {
     return sb_;
@@ -350,7 +350,10 @@ class JsonArrayScope : public JsonScope {
   }
   void leave() {
     jb_->dec_offset();
-    jb_->print_offset();
+    if (jb_->is_pretty()) {
+      *sb_ << "\n";
+      jb_->print_offset();
+    }
     *sb_ << "]";
   }
   template <class T>
@@ -369,7 +372,10 @@ class JsonArrayScope : public JsonScope {
     } else {
       is_first_ = true;
     }
-    jb_->print_offset();
+    if (jb_->is_pretty()) {
+      *sb_ << "\n";
+      jb_->print_offset();
+    }
     return jb_->enter_value();
   }
 
@@ -391,7 +397,10 @@ class JsonObjectScope : public JsonScope {
   }
   void leave() {
     jb_->dec_offset();
-    jb_->print_offset();
+    if (jb_->is_pretty()) {
+      *sb_ << "\n";
+      jb_->print_offset();
+    }
     *sb_ << "}";
   }
   template <class T>
@@ -402,7 +411,10 @@ class JsonObjectScope : public JsonScope {
     } else {
       is_first_ = true;
     }
-    jb_->print_offset();
+    if (jb_->is_pretty()) {
+      *sb_ << "\n";
+      jb_->print_offset();
+    }
     jb_->enter_value() << key;
     if (jb_->is_pretty()) {
       *sb_ << " : ";

tdutils/td/utils/MpmcQueue.h

@@ -105,24 +105,28 @@ template <class T>
 class OneValue<T *> {
  public:
   bool set_value(T *value) {
-    T *was = nullptr;
+    T *was = Empty();
     return state_.compare_exchange_strong(was, value, std::memory_order_acq_rel);
   }
   bool get_value(T *&value) {
     value = state_.exchange(Taken(), std::memory_order_acq_rel);
-    return value != nullptr;
+    return value != Empty();
   }
   void reset() {
-    state_ = nullptr;
+    state_ = Empty();
   }
   OneValue() {
   }
 
  private:
-  std::atomic<T *> state_{nullptr};
-  T *Taken() {
-    static T xxx;
-    return &xxx;
+  std::atomic<T *> state_{Empty()};
+  static T *Empty() {
+    static int64 xxx;
+    return reinterpret_cast<T *>(&xxx);
+  }
+  static T *Taken() {
+    static int64 xxx;
+    return reinterpret_cast<T *>(&xxx);
   }
 };