Merge branch 'testnet' into test-ton-crypto

.github/workflows/build-ton-macos-arm64-shared.yml

@@ -0,0 +1,25 @@
+name: MacOS TON build (shared, arm64)
+
+on: [push,workflow_dispatch,workflow_call]
+
+jobs:
+  build:
+    runs-on: macos-14
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
+
+      - name: Build TON
+        run: |
+          cp assembly/native/build-macos-shared.sh .
+          chmod +x build-macos-shared.sh
+          ./build-macos-shared.sh -t -a
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@master
+        with:
+          name: ton-binaries-macos-14
+          path: artifacts

.github/workflows/ton-arm64-macos.yml

@@ -0,0 +1,37 @@
+name: MacOS TON build (portable, arm64)
+
+on: [push,workflow_dispatch,workflow_call]
+
+jobs:
+  build:
+    runs-on: macos-14
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: 'recursive'
+
+    - uses: cachix/install-nix-action@v23
+      with:
+        extra_nix_config: |
+          access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
+
+    - name: Build TON
+      run: |
+        cp assembly/nix/build-macos-nix.sh .
+        chmod +x build-macos-nix.sh
+        ./build-macos-nix.sh -t
+
+    - name: Simple binaries test
+      run: |
+        sudo mv /nix/store /nix/store2
+        artifacts/validator-engine -V
+        artifacts/lite-client -V
+        artifacts/fift -V
+        artifacts/func -V
+
+    - name: Upload artifacts
+      uses: actions/upload-artifact@master
+      with:
+        name: ton-arm64-macos-binaries
+        path: artifacts

.github/workflows/ton_crypto_lib-x86-64-windows.yml

@@ -9,7 +9,7 @@ defaults:
 jobs:
   build:
 
-    runs-on: windows-2022
+    runs-on: windows-2019
 
     steps:
       - name: Get Current OS version

CMake/FindSodium.cmake

@@ -37,12 +37,14 @@ if (NOT DEFINED SODIUM_USE_STATIC_LIBS)
   option(SODIUM_USE_STATIC_LIBS "enable to statically link against sodium" OFF)
 endif()
 if(NOT (SODIUM_USE_STATIC_LIBS EQUAL SODIUM_USE_STATIC_LIBS_LAST))
-  unset(sodium_LIBRARY CACHE)
-  unset(SODIUM_LIBRARY_DEBUG CACHE)
-  unset(SODIUM_LIBRARY_RELEASE CACHE)
-  unset(sodium_DLL_DEBUG CACHE)
-  unset(sodium_DLL_RELEASE CACHE)
-  set(SODIUM_USE_STATIC_LIBS_LAST ${SODIUM_USE_STATIC_LIBS} CACHE INTERNAL "internal change tracking variable")
+  if (NOT SODIUM_LIBRARY_RELEASE)
+    unset(sodium_LIBRARY CACHE)
+    unset(SODIUM_LIBRARY_DEBUG CACHE)
+    unset(SODIUM_LIBRARY_RELEASE CACHE)
+    unset(sodium_DLL_DEBUG CACHE)
+    unset(sodium_DLL_RELEASE CACHE)
+    set(SODIUM_USE_STATIC_LIBS_LAST ${SODIUM_USE_STATIC_LIBS} CACHE INTERNAL "internal change tracking variable")
+  endif()
 endif()
 
 
@@ -295,4 +297,4 @@ else()
         )
     endif()
   endif()
-endif()
+endif()

CMakeLists.txt

@@ -234,7 +234,7 @@ if (THREADS_HAVE_PTHREAD_ARG)
 endif()
 
 if (TON_USE_JEMALLOC)
-  find_package(JeMalloc REQUIRED)
+  find_package(jemalloc REQUIRED)
 endif()
 
 set(MEMPROF "" CACHE STRING "Use one of \"ON\", \"FAST\" or \"SAFE\" to enable memory profiling. \
@@ -539,6 +539,9 @@ target_link_libraries(test-ton-collator overlay tdutils tdactor adnl tl_api dht
 add_executable(test-http test/test-http.cpp)
 target_link_libraries(test-http PRIVATE tonhttp)
 
+add_executable(test-emulator test/test-td-main.cpp emulator/test/emulator-tests.cpp)
+target_link_libraries(test-emulator PRIVATE emulator)
+
 get_directory_property(HAS_PARENT PARENT_DIRECTORY)
 if (HAS_PARENT)
   set(ALL_TEST_SOURCE
@@ -570,6 +573,7 @@ add_test(test-cells test-cells ${TEST_OPTIONS})
 add_test(test-smartcont test-smartcont)
 add_test(test-net test-net)
 add_test(test-actors test-tdactor)
+add_test(test-emulator test-emulator)
 
 #BEGIN tonlib
 add_test(test-tdutils test-tdutils)

Changelog.md

@@ -1,3 +1,16 @@
+## 2024.06 Update
+
+1. Make Jemalloc default allocator
+2. Add candidate broadcasting and caching
+3. Limit per address speed for external messages broadcast by reasonably large number 
+4. Overlay improvements: fix dropping peers in small custom overlays, fix wrong certificate on missed keyblocks
+5. Extended statistics and logs for celldb usage, session stats, persistent state serialization
+6. Tonlib and explorer fixes
+7. Flags for precize control of Celldb: `--celldb-cache-size`, `--celldb-direct-io` and `--celldb-preload-all`
+8. Add valiator-console command to stop persistent state serialization
+9. Use `@` path separator for defining include path in fift and create-state utilities on Windows only.
+
+
 ## 2024.04 Update
 
 1. Emulator: Single call optimized runGetMethod added

Dockerfile

@@ -1,10 +1,15 @@
-FROM ubuntu:22.04 as builder
+FROM ubuntu:22.04 AS builder
 RUN apt-get update && \
-	DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git ninja-build libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev pkg-config autoconf automake libtool && \
-	rm -rf /var/lib/apt/lists/*
-ENV CC clang
-ENV CXX clang++
-ENV CCACHE_DISABLE 1
+        DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git ninja-build libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev pkg-config autoconf automake libtool libjemalloc-dev lsb-release software-properties-common gnupg
+
+RUN wget https://apt.llvm.org/llvm.sh && \
+    chmod +x llvm.sh && \
+    ./llvm.sh 16 all && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV CC=/usr/bin/clang-16
+ENV CXX=/usr/bin/clang++-16
+ENV CCACHE_DISABLE=1
 
 WORKDIR /
 RUN mkdir ton
@@ -13,17 +18,16 @@ WORKDIR /ton
 COPY ./ ./
 
 RUN mkdir build && \
-	cd build && \
-	cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= .. && \
-	ninja storage-daemon storage-daemon-cli tonlibjson fift func validator-engine validator-engine-console generate-random-id dht-server lite-client
+        cd build && \
+        cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DTON_USE_JEMALLOC=ON .. && \
+        ninja storage-daemon storage-daemon-cli tonlibjson fift func validator-engine validator-engine-console generate-random-id dht-server lite-client
 
 FROM ubuntu:22.04
 RUN apt-get update && \
-    apt-get install -y wget libatomic1 openssl libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev && \
+    apt-get install -y wget curl libatomic1 openssl libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev libjemalloc-dev htop net-tools netcat iptraf-ng jq tcpdump pv plzip && \
     rm -rf /var/lib/apt/lists/*
 
-RUN mkdir -p /var/ton-work/db && \
-    mkdir -p /var/ton-work/db/static
+RUN mkdir -p /var/ton-work/db /var/ton-work/scripts
 
 COPY --from=builder /ton/build/storage/storage-daemon/storage-daemon /usr/local/bin/
 COPY --from=builder /ton/build/storage/storage-daemon/storage-daemon-cli /usr/local/bin/
@@ -33,7 +37,7 @@ COPY --from=builder /ton/build/validator-engine-console/validator-engine-console
 COPY --from=builder /ton/build/utils/generate-random-id /usr/local/bin/
 
 WORKDIR /var/ton-work/db
-COPY ./docker/init.sh ./docker/control.template ./
-RUN chmod +x init.sh
+COPY ./docker/init.sh ./docker/control.template /var/ton-work/scripts/
+RUN chmod +x /var/ton-work/scripts/init.sh
 
-ENTRYPOINT ["/var/ton-work/db/init.sh"]
+ENTRYPOINT ["/var/ton-work/scripts/init.sh"]

adnl/adnl-ext-server.cpp

@@ -91,7 +91,7 @@ td::Status AdnlInboundConnection::process_custom_packet(td::BufferSlice &data, b
     auto F = fetch_tl_object<ton_api::tcp_authentificate>(data.clone(), true);
     if (F.is_ok()) {
       if (nonce_.size() > 0 || !remote_id_.is_zero()) {
-        return td::Status::Error(ErrorCode::protoviolation, "duplicate authentificate");
+        return td::Status::Error(ErrorCode::protoviolation, "duplicate authenticate");
       }
       auto f = F.move_as_ok();
       nonce_ = td::SecureString{f->nonce_.size() + 256};

assembly/cicd/jenkins/test-builds.groovy

@@ -1,4 +1,5 @@
 pipeline {
+
     agent none
     stages {
         stage('Run Builds') {
@@ -12,7 +13,7 @@ pipeline {
                             sh '''
                                 cp assembly/native/build-ubuntu-shared.sh .
                                 chmod +x build-ubuntu-shared.sh
-                                ./build-ubuntu-shared.sh -t -a
+                                ./build-ubuntu-shared.sh -a
                             '''
                             sh '''
                                 cd artifacts
@@ -31,7 +32,7 @@ pipeline {
                             sh '''
                                 cp assembly/nix/build-linux-x86-64-nix.sh .
                                 chmod +x build-linux-x86-64-nix.sh
-                                ./build-linux-x86-64-nix.sh -t
+                                ./build-linux-x86-64-nix.sh
                             '''
                             sh '''
                                 cd artifacts
@@ -50,7 +51,7 @@ pipeline {
                             sh '''
                                 cp assembly/native/build-ubuntu-shared.sh .
                                 chmod +x build-ubuntu-shared.sh
-                                ./build-ubuntu-shared.sh -t -a
+                                ./build-ubuntu-shared.sh -a
                             '''
                             sh '''
                                 cd artifacts
@@ -69,7 +70,7 @@ pipeline {
                             sh '''
                                 cp assembly/nix/build-linux-arm64-nix.sh .
                                 chmod +x build-linux-arm64-nix.sh
-                                ./build-linux-arm64-nix.sh -t
+                                ./build-linux-arm64-nix.sh
                             '''
                             sh '''
                                 cd artifacts
@@ -88,7 +89,7 @@ pipeline {
                             sh '''
                                 cp assembly/native/build-macos-shared.sh .
                                 chmod +x build-macos-shared.sh
-                                ./build-macos-shared.sh -t -a
+                                ./build-macos-shared.sh -a
                             '''
                             sh '''
                                 cd artifacts
@@ -107,7 +108,7 @@ pipeline {
                             sh '''
                                 cp assembly/nix/build-macos-nix.sh .
                                 chmod +x build-macos-nix.sh
-                                ./build-macos-nix.sh -t
+                                ./build-macos-nix.sh
                             '''
                             sh '''
                                 cd artifacts
@@ -126,7 +127,7 @@ pipeline {
                             sh '''
                                 cp assembly/native/build-macos-shared.sh .
                                 chmod +x build-macos-shared.sh
-                                ./build-macos-shared.sh -t -a
+                                ./build-macos-shared.sh -a
                             '''
                             sh '''
                                 cd artifacts
@@ -145,7 +146,7 @@ pipeline {
                             sh '''
                                 cp assembly/nix/build-macos-nix.sh .
                                 chmod +x build-macos-nix.sh
-                                ./build-macos-nix.sh -t
+                                ./build-macos-nix.sh
                             '''
                             sh '''
                                 cd artifacts
@@ -164,7 +165,7 @@ pipeline {
                             sh '''
                                 cp assembly/native/build-macos-shared.sh .
                                 chmod +x build-macos-shared.sh
-                                ./build-macos-shared.sh -t -a
+                                ./build-macos-shared.sh -a
                             '''
                             sh '''
                                 cd artifacts
@@ -182,7 +183,7 @@ pipeline {
                         timeout(time: 180, unit: 'MINUTES') {
                             bat '''
                                 copy assembly\\native\\build-windows.bat .
-                                build-windows.bat -t
+                                build-windows.bat
                             '''
                             bat '''
                                 cd artifacts

assembly/native/build-macos-portable.sh

@@ -158,7 +158,7 @@ if [ "$with_tests" = true ]; then
   http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \
   test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont \
   test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \
-  test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state
+  test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state test-emulator
   test $? -eq 0 || { echo "Can't compile ton"; exit 1; }
 else
   ninja storage-daemon storage-daemon-cli blockchain-explorer   \

assembly/native/build-macos-shared.sh

@@ -86,7 +86,7 @@ if [ "$with_tests" = true ]; then
   http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \
   test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont \
   test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \
-  test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state
+  test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state test-emulator
   test $? -eq 0 || { echo "Can't compile ton"; exit 1; }
 else
   ninja storage-daemon storage-daemon-cli blockchain-explorer   \

assembly/native/build-toncryptolib-windows.bat

@@ -1,4 +1,4 @@
-REM execute this script inside elevated (Run as Administrator) console "x64 Native Tools Command Prompt for VS 2022"
+REM execute this script inside elevated (Run as Administrator) console "x64 Native Tools Command Prompt for VS 2019"
 
 echo off
 
@@ -31,7 +31,7 @@ git clone https://github.com/madler/zlib.git
 cd zlib
 git checkout v1.3.1
 cd contrib\vstudio\vc14
-msbuild zlibstat.vcxproj /p:Configuration=ReleaseWithoutAsm /p:platform=x64 -p:PlatformToolset=v143
+msbuild zlibstat.vcxproj /p:Configuration=ReleaseWithoutAsm /p:platform=x64 -p:PlatformToolset=v142
 
 IF %errorlevel% NEQ 0 (
   echo Can't install zlib
@@ -47,8 +47,8 @@ git clone https://github.com/lz4/lz4.git
 cd lz4
 git checkout v1.9.4
 cd build\VS2017\liblz4
-msbuild liblz4.vcxproj /p:Configuration=Release /p:platform=x64 -p:PlatformToolset=v143
-dir /s
+msbuild liblz4.vcxproj /p:Configuration=Release /p:platform=x64 -p:PlatformToolset=v142
+
 IF %errorlevel% NEQ 0 (
   echo Can't install lz4
   exit /b %errorlevel%
@@ -62,7 +62,7 @@ if not exist "secp256k1" (
 git clone https://github.com/bitcoin-core/secp256k1.git
 cd secp256k1
 git checkout v0.3.2
-cmake -G "Visual Studio 17 2022" -A x64 -S . -B build -DSECP256K1_ENABLE_MODULE_RECOVERY=ON -DBUILD_SHARED_LIBS=OFF
+cmake -G "Visual Studio 16 2019" -A x64 -S . -B build -DSECP256K1_ENABLE_MODULE_RECOVERY=ON -DBUILD_SHARED_LIBS=OFF
 IF %errorlevel% NEQ 0 (
   echo Can't configure secp256k1
   exit /b %errorlevel%
@@ -78,8 +78,7 @@ echo Using secp256k1...
 )
 
 
-if not exist "libsodium" (
-curl  -Lo libsodium-1.0.18-stable-msvc.zip https://download.libsodium.org/libsodium/releases/libsodium-1.0.18-stable-msvc.zip
+curl --retry 5 --retry-delay 10 -Lo libsodium-1.0.18-stable-msvc.zip https://download.libsodium.org/libsodium/releases/libsodium-1.0.18-stable-msvc.zip
 IF %errorlevel% NEQ 0 (
   echo Can't download libsodium
   exit /b %errorlevel%

assembly/native/build-ubuntu-portable.sh

@@ -150,7 +150,7 @@ ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli \
       adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \
       test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \
       test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \
-      test-fec test-tddb test-db test-validator-session-state
+      test-fec test-tddb test-db test-validator-session-state test-emulator
       test $? -eq 0 || { echo "Can't compile ton"; exit 1; }
 else
 ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli \