Upstream sync

.github/workflows/build-ion-linux-android-tonlib.yml

@@ -28,5 +28,5 @@ jobs:
     - name: Upload artifacts
       uses: actions/upload-artifact@master
       with:
-        name: tonlib-android
+        name: ton-android-tonlib
         path: artifacts

.github/workflows/build-ion-linux-x86-64-shared.yml

@@ -21,6 +21,12 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev
 
+    - if: matrix.os == 'ubuntu-20.04'
+      run: |
+        sudo apt install -y manpages-dev software-properties-common
+        sudo add-apt-repository ppa:ubuntu-toolchain-r/test
+        sudo apt update && sudo apt install gcc-11 g++-11
+
     - if: matrix.os != 'ubuntu-24.04'
       run: |
         wget https://apt.llvm.org/llvm.sh

.github/workflows/build-ton-macos-15-arm64-shared.yml

@@ -0,0 +1,25 @@
+name: MacOS-15 TON build (shared, arm64)
+
+on: [push,workflow_dispatch,workflow_call]
+
+jobs:
+  build:
+    runs-on: macos-15
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
+
+      - name: Build TON
+        run: |
+          cp assembly/native/build-macos-shared.sh .
+          chmod +x build-macos-shared.sh
+          ./build-macos-shared.sh -t -a
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@master
+        with:
+          name: ton-binaries-macos-15
+          path: artifacts

.github/workflows/build-ton-macos-arm64-shared.yml

@@ -1,4 +1,4 @@
-name: MacOS TON build (shared, arm64)
+name: MacOS-14 TON build (shared, arm64)
 
 on: [push,workflow_dispatch,workflow_call]
 

.github/workflows/create-release.yml

@@ -67,6 +67,14 @@ jobs:
           workflow_conclusion: success
           skip_unpack: true
 
+      - name: Download Android Tonlib artifacts
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: build-ton-linux-android-tonlib.yml
+          path: artifacts
+          workflow_conclusion: success
+          skip_unpack: true
+
       - name: Show all artifacts
         run: |
           tree artifacts
@@ -486,3 +494,11 @@ jobs:
           file: artifacts/ion-wasm-binaries.zip
           asset_name: ion-wasm-binaries.zip
           tag: ${{ steps.tag.outputs.TAG }}
+
+      - name: Upload Android Tonlib artifacts
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ton-android-tonlib.zip
+          asset_name: ton-android-tonlib.zip
+          tag: ${{ steps.tag.outputs.TAG }}

.github/workflows/ion-ccpcheck.yml

@@ -20,7 +20,7 @@ jobs:
         generate report: true
 
     - name: Upload report
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@master
       with:
         name: ion-ccpcheck-report
         path: output

CMake/FindSodium.cmake

@@ -127,10 +127,16 @@ elseif (WIN32)
     endif()
     string(APPEND _PLATFORM_PATH "/$$CONFIG$$")
 
+    message(STATUS "MSVC_VERSION ${MSVC_VERSION}")
     if (MSVC_VERSION LESS 1900)
       math(EXPR _VS_VERSION "${MSVC_VERSION} / 10 - 60")
     else()
-      math(EXPR _VS_VERSION "${MSVC_VERSION} / 10 - 50")
+      if (MSVC_VERSION EQUAL 1941)
+        math(EXPR _VS_VERSION "${MSVC_VERSION} / 10 - 51")
+      else()
+        math(EXPR _VS_VERSION "${MSVC_VERSION} / 10 - 50")
+      endif()
+
     endif()
     string(APPEND _PLATFORM_PATH "/v${_VS_VERSION}")
 

CMakeLists.txt

@@ -79,7 +79,7 @@ else()
   set(HAVE_SSE42 FALSE)
 endif()
 
-set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED TRUE)
 set(CMAKE_CXX_EXTENSIONS FALSE)
 
@@ -333,6 +333,10 @@ add_cxx_compiler_flag("-Wno-sign-conversion")
 add_cxx_compiler_flag("-Qunused-arguments")
 add_cxx_compiler_flag("-Wno-unused-private-field")
 add_cxx_compiler_flag("-Wno-redundant-move")
+
+#add_cxx_compiler_flag("-Wno-unused-function")
+#add_cxx_compiler_flag("-Wno-unused-variable")
+#add_cxx_compiler_flag("-Wno-shorten-64-to-32")
 #add_cxx_compiler_flag("-Werror")
 
 #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -isystem /usr/include/c++/v1")
@@ -484,10 +488,10 @@ target_link_libraries(test-net PRIVATE tdnet tdutils ${CMAKE_THREAD_LIBS_INIT})
 
 #BEGIN tonlib
 add_executable(test-tonlib ${TONLIB_ONLINE_TEST_SOURCE})
-target_link_libraries(test-tonlib tdutils tdactor adnllite tl_api ton_crypto ton_block tl_tonlib_api tonlib)
+target_link_libraries(test-tonlib tdactor adnllite tl_api ton_crypto tl_tonlib_api tonlib)
 
 add_executable(test-tonlib-offline test/test-td-main.cpp ${TONLIB_OFFLINE_TEST_SOURCE})
-target_link_libraries(test-tonlib-offline tdutils tdactor adnllite tl_api ton_crypto ton_block fift-lib tl_tonlib_api tonlib)
+target_link_libraries(test-tonlib-offline tdactor adnllite tl_api ton_crypto fift-lib tl_tonlib_api tonlib)
 
 if (NOT CMAKE_CROSSCOMPILING)
   add_dependencies(test-tonlib-offline gen_fif)
@@ -529,6 +533,8 @@ target_link_libraries(test-rldp2 adnl adnltest dht rldp2 tl_api)
 add_executable(test-validator-session-state test/test-validator-session-state.cpp)
 target_link_libraries(test-validator-session-state adnl dht rldp validatorsession tl_api)
 
+add_executable(test-overlay test/test-overlay.cpp)
+target_link_libraries(test-overlay overlay tdutils tdactor adnl adnltest tl_api dht )
 add_executable(test-catchain test/test-catchain.cpp)
 target_link_libraries(test-catchain overlay tdutils tdactor adnl adnltest rldp tl_api dht
   catchain )

Changelog.md

@@ -1,3 +1,19 @@
+## 2024.10 Update
+
+1. Parallel write to celldb: substantial improvement of sync and GC speed, especially with slow disks.
+2. Decreased network traffic: only first block candidate is sent optimistically.
+3. Improved channel creation and dht lookups, introduction of semi-private overlays
+4. New LS dispatch queue related methods and improvement security
+5. Fixing recursion in TVM continuations
+6. Improved stats for actors, validator sessions, perf counters, overlays, adnl, rocksdb
+7. Migration to C++20
+8. Improved block size estimates: account for depth in various structures
+9. Fix bug with `<<` optimization in FunC
+10. Minor changes of TVM which will be activated by `Config8.version >= 9`
+11. Multiple minor improvements
+
+Besides the work of the core team, this update is based on the efforts of @krigga (emulator), Arayz @ TonBit (LS security, TVM recursion), @ret2happy (UB in BLST).
+
 ## 2024.08 Update
 
 1. Introduction of dispatch queues, message envelopes with transaction chain metadata, and explicitly stored msg_queue size, which will be activated by `Config8.version >= 8` and new `Config8.capabilities` bits: `capStoreOutMsgQueueSize`, `capMsgMetadata`, `capDeferMessages`. 

adnl/CMakeLists.txt

@@ -88,17 +88,15 @@ target_link_libraries(adnl PUBLIC tdactor ton_crypto tl_api tdnet tddb keys keyr
 
 add_executable(adnl-proxy ${ADNL_PROXY_SOURCE})
 target_include_directories(adnl-proxy PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/..>)
-target_link_libraries(adnl-proxy PUBLIC tdactor ton_crypto tl_api tdnet common
-  tl-utils git)
+target_link_libraries(adnl-proxy PUBLIC tdactor ton_crypto tl_api tdnet common tl-utils git)
 
 add_executable(adnl-pong adnl-pong.cpp)
 target_include_directories(adnl-pong PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/..>)
-target_link_libraries(adnl-pong PUBLIC tdactor ton_crypto tl_api tdnet common
-  tl-utils adnl dht git)
+target_link_libraries(adnl-pong PUBLIC tdactor ton_crypto tl_api tdnet common tl-utils adnl dht git)
 
 add_library(adnltest STATIC ${ADNL_TEST_SOURCE})
 target_include_directories(adnltest PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/..>)
-target_link_libraries(adnltest PUBLIC adnl )
+target_link_libraries(adnltest PUBLIC adnl)
 
 install(TARGETS adnl-proxy RUNTIME DESTINATION bin)
 endif()

adnl/adnl-local-id.cpp

@@ -306,7 +306,7 @@ void AdnlLocalId::update_packet(AdnlPacket packet, bool update_id, bool sign, td
   }
 }
 
-void AdnlLocalId::get_stats(td::Promise<tl_object_ptr<ton_api::adnl_stats_localId>> promise) {
+void AdnlLocalId::get_stats(bool all, td::Promise<tl_object_ptr<ton_api::adnl_stats_localId>> promise) {
   auto stats = create_tl_object<ton_api::adnl_stats_localId>();
   stats->short_id_ = short_id_.bits256_value();
   for (auto &[ip, x] : inbound_rate_limiter_) {
@@ -317,22 +317,22 @@ void AdnlLocalId::get_stats(td::Promise<tl_object_ptr<ton_api::adnl_stats_localI
   }
   prepare_packet_stats();
   stats->packets_recent_ = packet_stats_prev_.tl();
-  stats->packets_total_ = packet_stats_total_.tl();
+  stats->packets_total_ = packet_stats_total_.tl(all);
   stats->packets_total_->ts_start_ = (double)Adnl::adnl_start_time();
   stats->packets_total_->ts_end_ = td::Clocks::system();
   promise.set_result(std::move(stats));
 }
 
 void AdnlLocalId::add_decrypted_packet_stats(td::IPAddress addr) {
   prepare_packet_stats();
-  ++packet_stats_cur_.decrypted_packets[addr];
-  ++packet_stats_total_.decrypted_packets[addr];
+  packet_stats_cur_.decrypted_packets[addr].inc();
+  packet_stats_total_.decrypted_packets[addr].inc();
 }
 
 void AdnlLocalId::add_dropped_packet_stats(td::IPAddress addr) {
   prepare_packet_stats();
-  ++packet_stats_cur_.dropped_packets[addr];
-  ++packet_stats_total_.dropped_packets[addr];
+  packet_stats_cur_.dropped_packets[addr].inc();
+  packet_stats_total_.dropped_packets[addr].inc();
 }
 
 void AdnlLocalId::prepare_packet_stats() {
@@ -351,17 +351,22 @@ void AdnlLocalId::prepare_packet_stats() {
   }
 }
 
-tl_object_ptr<ton_api::adnl_stats_localIdPackets> AdnlLocalId::PacketStats::tl() const {
+tl_object_ptr<ton_api::adnl_stats_localIdPackets> AdnlLocalId::PacketStats::tl(bool all) const {
+  double threshold = all ? -1.0 : td::Clocks::system() - 600.0;
   auto obj = create_tl_object<ton_api::adnl_stats_localIdPackets>();
   obj->ts_start_ = ts_start;
   obj->ts_end_ = ts_end;
   for (const auto &[ip, packets] : decrypted_packets) {
-    obj->decrypted_packets_.push_back(create_tl_object<ton_api::adnl_stats_ipPackets>(
-        ip.is_valid() ? PSTRING() << ip.get_ip_str() << ":" << ip.get_port() : "", packets));
+    if (packets.last_packet_ts >= threshold) {
+      obj->decrypted_packets_.push_back(create_tl_object<ton_api::adnl_stats_ipPackets>(
+          ip.is_valid() ? PSTRING() << ip.get_ip_str() << ":" << ip.get_port() : "", packets.packets));
+    }
   }
   for (const auto &[ip, packets] : dropped_packets) {
-    obj->dropped_packets_.push_back(create_tl_object<ton_api::adnl_stats_ipPackets>(
-        ip.is_valid() ? PSTRING() << ip.get_ip_str() << ":" << ip.get_port() : "", packets));
+    if (packets.last_packet_ts >= threshold) {
+      obj->dropped_packets_.push_back(create_tl_object<ton_api::adnl_stats_ipPackets>(
+          ip.is_valid() ? PSTRING() << ip.get_ip_str() << ":" << ip.get_port() : "", packets.packets));
+    }
   }
   return obj;
 }

adnl/adnl-local-id.h

@@ -78,7 +78,7 @@ class AdnlLocalId : public td::actor::Actor {
   void update_packet(AdnlPacket packet, bool update_id, bool sign, td::int32 update_addr_list_if,
                      td::int32 update_priority_addr_list_if, td::Promise<AdnlPacket> promise);
 
-  void get_stats(td::Promise<tl_object_ptr<ton_api::adnl_stats_localId>> promise);
+  void get_stats(bool all, td::Promise<tl_object_ptr<ton_api::adnl_stats_localId>> promise);
 
   td::uint32 get_mode() {
     return mode_;
@@ -111,10 +111,20 @@ class AdnlLocalId : public td::actor::Actor {
   std::map<td::IPAddress, InboundRateLimiter> inbound_rate_limiter_;
   struct PacketStats {
     double ts_start = 0.0, ts_end = 0.0;
-    std::map<td::IPAddress, td::uint64> decrypted_packets;
-    std::map<td::IPAddress, td::uint64> dropped_packets;
 
-    tl_object_ptr<ton_api::adnl_stats_localIdPackets> tl() const;
+    struct Counter {
+      td::uint64 packets = 0;
+      double last_packet_ts = 0.0;
+
+      void inc() {
+        ++packets;
+        last_packet_ts = td::Clocks::system();
+      }
+    };
+    std::map<td::IPAddress, Counter> decrypted_packets;
+    std::map<td::IPAddress, Counter> dropped_packets;
+
+    tl_object_ptr<ton_api::adnl_stats_localIdPackets> tl(bool all = true) const;
   } packet_stats_cur_, packet_stats_prev_, packet_stats_total_;
   void add_decrypted_packet_stats(td::IPAddress addr);
   void add_dropped_packet_stats(td::IPAddress addr);

adnl/adnl-peer-table.cpp

@@ -385,7 +385,7 @@ void AdnlPeerTableImpl::get_conn_ip_str(AdnlNodeIdShort l_id, AdnlNodeIdShort p_
   td::actor::send_closure(it->second, &AdnlPeer::get_conn_ip_str, l_id, std::move(promise));
 }
 
-void AdnlPeerTableImpl::get_stats(td::Promise<tl_object_ptr<ton_api::adnl_stats>> promise) {
+void AdnlPeerTableImpl::get_stats(bool all, td::Promise<tl_object_ptr<ton_api::adnl_stats>> promise) {
   class Cb : public td::actor::Actor {
    public:
     explicit Cb(td::Promise<tl_object_ptr<ton_api::adnl_stats>> promise) : promise_(std::move(promise)) {
@@ -440,7 +440,7 @@ void AdnlPeerTableImpl::get_stats(td::Promise<tl_object_ptr<ton_api::adnl_stats>
 
   for (auto &[id, local_id] : local_ids_) {
     td::actor::send_closure(callback, &Cb::inc_pending);
-    td::actor::send_closure(local_id.local_id, &AdnlLocalId::get_stats,
+    td::actor::send_closure(local_id.local_id, &AdnlLocalId::get_stats, all,
                             [id = id, callback](td::Result<tl_object_ptr<ton_api::adnl_stats_localId>> R) {
                               if (R.is_error()) {
                                 VLOG(ADNL_NOTICE)
@@ -454,7 +454,7 @@ void AdnlPeerTableImpl::get_stats(td::Promise<tl_object_ptr<ton_api::adnl_stats>
   for (auto &[id, peer] : peers_) {
     td::actor::send_closure(callback, &Cb::inc_pending);
     td::actor::send_closure(
-        peer, &AdnlPeer::get_stats,
+        peer, &AdnlPeer::get_stats, all,
         [id = id, callback](td::Result<std::vector<tl_object_ptr<ton_api::adnl_stats_peerPair>>> R) {
           if (R.is_error()) {
             VLOG(ADNL_NOTICE) << "failed to get stats for peer " << id << " : " << R.move_as_error();

adnl/adnl-peer-table.hpp

@@ -108,7 +108,7 @@ class AdnlPeerTableImpl : public AdnlPeerTable {
                      td::Promise<std::pair<td::actor::ActorOwn<AdnlTunnel>, AdnlAddress>> promise) override;
   void get_conn_ip_str(AdnlNodeIdShort l_id, AdnlNodeIdShort p_id, td::Promise<td::string> promise) override;
 
-  void get_stats(td::Promise<tl_object_ptr<ton_api::adnl_stats>> promise) override;
+  void get_stats(bool all, td::Promise<tl_object_ptr<ton_api::adnl_stats>> promise) override;
 
   struct PrintId {};
   PrintId print_id() const {

adnl/adnl-peer.cpp

@@ -808,7 +808,15 @@ void AdnlPeerPairImpl::get_conn_ip_str(td::Promise<td::string> promise) {
   promise.set_value("undefined");
 }
 
-void AdnlPeerPairImpl::get_stats(td::Promise<tl_object_ptr<ton_api::adnl_stats_peerPair>> promise) {
+void AdnlPeerPairImpl::get_stats(bool all, td::Promise<tl_object_ptr<ton_api::adnl_stats_peerPair>> promise) {
+  if (!all) {
+    double threshold = td::Clocks::system() - 600.0;
+    if (last_in_packet_ts_ < threshold && last_out_packet_ts_ < threshold) {
+      promise.set_value(nullptr);
+      return;
+    }
+  }
+
   auto stats = create_tl_object<ton_api::adnl_stats_peerPair>();
   stats->local_id_ = local_id_.bits256_value();
   stats->peer_id_ = peer_id_short_.bits256_value();
@@ -993,15 +1001,17 @@ void AdnlPeerImpl::update_addr_list(AdnlNodeIdShort local_id, td::uint32 local_m
   td::actor::send_closure(it->second, &AdnlPeerPair::update_addr_list, std::move(addr_list));
 }
 
-void AdnlPeerImpl::get_stats(td::Promise<std::vector<tl_object_ptr<ton_api::adnl_stats_peerPair>>> promise) {
+void AdnlPeerImpl::get_stats(bool all, td::Promise<std::vector<tl_object_ptr<ton_api::adnl_stats_peerPair>>> promise) {
   class Cb : public td::actor::Actor {
    public:
     explicit Cb(td::Promise<std::vector<tl_object_ptr<ton_api::adnl_stats_peerPair>>> promise)
         : promise_(std::move(promise)) {
     }
 
     void got_peer_pair_stats(tl_object_ptr<ton_api::adnl_stats_peerPair> peer_pair) {
-      result_.push_back(std::move(peer_pair));
+      if (peer_pair) {
+        result_.push_back(std::move(peer_pair));
+      }
       dec_pending();
     }
 
@@ -1027,7 +1037,7 @@ void AdnlPeerImpl::get_stats(td::Promise<std::vector<tl_object_ptr<ton_api::adnl
 
   for (auto &[local_id, peer_pair] : peer_pairs_) {
     td::actor::send_closure(callback, &Cb::inc_pending);
-    td::actor::send_closure(peer_pair, &AdnlPeerPair::get_stats,
+    td::actor::send_closure(peer_pair, &AdnlPeerPair::get_stats, all,
                             [local_id = local_id, peer_id = peer_id_short_,
                              callback](td::Result<tl_object_ptr<ton_api::adnl_stats_peerPair>> R) {
                               if (R.is_error()) {

adnl/adnl-peer.h

@@ -59,7 +59,7 @@ class AdnlPeerPair : public td::actor::Actor {
   virtual void update_peer_id(AdnlNodeIdFull id) = 0;
   virtual void update_addr_list(AdnlAddressList addr_list) = 0;
   virtual void get_conn_ip_str(td::Promise<td::string> promise) = 0;
-  virtual void get_stats(td::Promise<tl_object_ptr<ton_api::adnl_stats_peerPair>> promise) = 0;
+  virtual void get_stats(bool all, td::Promise<tl_object_ptr<ton_api::adnl_stats_peerPair>> promise) = 0;
 
   static td::actor::ActorOwn<AdnlPeerPair> create(td::actor::ActorId<AdnlNetworkManager> network_manager,
                                                   td::actor::ActorId<AdnlPeerTable> peer_table, td::uint32 local_mode,
@@ -101,7 +101,7 @@ class AdnlPeer : public td::actor::Actor {
                                 td::actor::ActorId<AdnlLocalId> local_actor, AdnlAddressList addr_list) = 0;
   virtual void update_dht_node(td::actor::ActorId<dht::Dht> dht_node) = 0;
   virtual void get_conn_ip_str(AdnlNodeIdShort l_id, td::Promise<td::string> promise) = 0;
-  virtual void get_stats(td::Promise<std::vector<tl_object_ptr<ton_api::adnl_stats_peerPair>>> promise) = 0;
+  virtual void get_stats(bool all, td::Promise<std::vector<tl_object_ptr<ton_api::adnl_stats_peerPair>>> promise) = 0;
 };
 
 }  // namespace adnl