diff --git a/.github/workflows/build-ton-linux-x86-64-shared.yml b/.github/workflows/build-ton-linux-x86-64-shared.yml new file mode 100644 index 000000000..e69de29bb diff --git a/.github/workflows/build-ton-macos-arm64-shared.yml b/.github/workflows/build-ton-macos-arm64-shared.yml new file mode 100644 index 000000000..2a68272cc --- /dev/null +++ b/.github/workflows/build-ton-macos-arm64-shared.yml @@ -0,0 +1,25 @@ +name: MacOS TON build (shared, arm64) + +on: [push,workflow_dispatch,workflow_call] + +jobs: + build: + runs-on: macos-14 + + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Build TON + run: | + cp assembly/native/build-macos-shared.sh . + chmod +x build-macos-shared.sh + ./build-macos-shared.sh -t -a + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-binaries-macos-14 + path: artifacts diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml new file mode 100644 index 000000000..e69de29bb diff --git a/.github/workflows/docker-ubuntu-image.yml b/.github/workflows/docker-ubuntu-image.yml new file mode 100644 index 000000000..e69de29bb diff --git a/.github/workflows/ton-arm64-macos.yml b/.github/workflows/ton-arm64-macos.yml new file mode 100644 index 000000000..9e8302e80 --- /dev/null +++ b/.github/workflows/ton-arm64-macos.yml @@ -0,0 +1,37 @@ +name: MacOS TON build (portable, arm64) + +on: [push,workflow_dispatch,workflow_call] + +jobs: + build: + runs-on: macos-14 + + steps: + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - uses: cachix/install-nix-action@v23 + with: + extra_nix_config: | + access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + + - name: Build TON + run: | + cp assembly/nix/build-macos-nix.sh . + chmod +x build-macos-nix.sh + ./build-macos-nix.sh -t + + - name: Simple binaries test + run: | + sudo mv /nix/store /nix/store2 + artifacts/validator-engine -V + artifacts/lite-client -V + artifacts/fift -V + artifacts/func -V + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-arm64-macos-binaries + path: artifacts diff --git a/.github/workflows/ton_crypto_lib-x86-64-windows.yml b/.github/workflows/ton_crypto_lib-x86-64-windows.yml index 3a00d1aaa..fda872bda 100644 --- a/.github/workflows/ton_crypto_lib-x86-64-windows.yml +++ b/.github/workflows/ton_crypto_lib-x86-64-windows.yml @@ -9,7 +9,7 @@ defaults: jobs: build: - runs-on: windows-2022 + runs-on: windows-2019 steps: - name: Get Current OS version diff --git a/CMake/FindSodium.cmake b/CMake/FindSodium.cmake index 85194ee2f..c11e46f1d 100644 --- a/CMake/FindSodium.cmake +++ b/CMake/FindSodium.cmake @@ -37,12 +37,14 @@ if (NOT DEFINED SODIUM_USE_STATIC_LIBS) option(SODIUM_USE_STATIC_LIBS "enable to statically link against sodium" OFF) endif() if(NOT (SODIUM_USE_STATIC_LIBS EQUAL SODIUM_USE_STATIC_LIBS_LAST)) - unset(sodium_LIBRARY CACHE) - unset(SODIUM_LIBRARY_DEBUG CACHE) - unset(SODIUM_LIBRARY_RELEASE CACHE) - unset(sodium_DLL_DEBUG CACHE) - unset(sodium_DLL_RELEASE CACHE) - set(SODIUM_USE_STATIC_LIBS_LAST ${SODIUM_USE_STATIC_LIBS} CACHE INTERNAL "internal change tracking variable") + if (NOT SODIUM_LIBRARY_RELEASE) + unset(sodium_LIBRARY CACHE) + unset(SODIUM_LIBRARY_DEBUG CACHE) + unset(SODIUM_LIBRARY_RELEASE CACHE) + unset(sodium_DLL_DEBUG CACHE) + unset(sodium_DLL_RELEASE CACHE) + set(SODIUM_USE_STATIC_LIBS_LAST ${SODIUM_USE_STATIC_LIBS} CACHE INTERNAL "internal change tracking variable") + endif() endif() @@ -295,4 +297,4 @@ else() ) endif() endif() -endif() \ No newline at end of file +endif() diff --git a/CMake/FindJeMalloc.cmake b/CMake/Findjemalloc.cmake similarity index 100% rename from CMake/FindJeMalloc.cmake rename to CMake/Findjemalloc.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 89be32383..b92ff6f1b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -234,7 +234,7 @@ if (THREADS_HAVE_PTHREAD_ARG) endif() if (TON_USE_JEMALLOC) - find_package(JeMalloc REQUIRED) + find_package(jemalloc REQUIRED) endif() set(MEMPROF "" CACHE STRING "Use one of \"ON\", \"FAST\" or \"SAFE\" to enable memory profiling. \ @@ -539,6 +539,9 @@ target_link_libraries(test-ton-collator overlay tdutils tdactor adnl tl_api dht add_executable(test-http test/test-http.cpp) target_link_libraries(test-http PRIVATE tonhttp) +add_executable(test-emulator test/test-td-main.cpp emulator/test/emulator-tests.cpp) +target_link_libraries(test-emulator PRIVATE emulator) + get_directory_property(HAS_PARENT PARENT_DIRECTORY) if (HAS_PARENT) set(ALL_TEST_SOURCE @@ -570,6 +573,7 @@ add_test(test-cells test-cells ${TEST_OPTIONS}) add_test(test-smartcont test-smartcont) add_test(test-net test-net) add_test(test-actors test-tdactor) +add_test(test-emulator test-emulator) #BEGIN tonlib add_test(test-tdutils test-tdutils) diff --git a/Changelog.md b/Changelog.md index a02410557..effe339f4 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,16 @@ +## 2024.06 Update + +1. Make Jemalloc default allocator +2. Add candidate broadcasting and caching +3. Limit per address speed for external messages broadcast by reasonably large number +4. Overlay improvements: fix dropping peers in small custom overlays, fix wrong certificate on missed keyblocks +5. Extended statistics and logs for celldb usage, session stats, persistent state serialization +6. Tonlib and explorer fixes +7. Flags for precize control of Celldb: `--celldb-cache-size`, `--celldb-direct-io` and `--celldb-preload-all` +8. Add valiator-console command to stop persistent state serialization +9. Use `@` path separator for defining include path in fift and create-state utilities on Windows only. + + ## 2024.04 Update 1. Emulator: Single call optimized runGetMethod added diff --git a/Dockerfile b/Dockerfile index 76c06b350..cf4187630 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,15 @@ -FROM ubuntu:22.04 as builder +FROM ubuntu:22.04 AS builder RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git ninja-build libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev pkg-config autoconf automake libtool && \ - rm -rf /var/lib/apt/lists/* -ENV CC clang -ENV CXX clang++ -ENV CCACHE_DISABLE 1 + DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git ninja-build libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev pkg-config autoconf automake libtool libjemalloc-dev lsb-release software-properties-common gnupg + +RUN wget https://apt.llvm.org/llvm.sh && \ + chmod +x llvm.sh && \ + ./llvm.sh 16 all && \ + rm -rf /var/lib/apt/lists/* + +ENV CC=/usr/bin/clang-16 +ENV CXX=/usr/bin/clang++-16 +ENV CCACHE_DISABLE=1 WORKDIR / RUN mkdir ton @@ -13,17 +18,16 @@ WORKDIR /ton COPY ./ ./ RUN mkdir build && \ - cd build && \ - cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= .. && \ - ninja storage-daemon storage-daemon-cli tonlibjson fift func validator-engine validator-engine-console generate-random-id dht-server lite-client + cd build && \ + cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DTON_USE_JEMALLOC=ON .. && \ + ninja storage-daemon storage-daemon-cli tonlibjson fift func validator-engine validator-engine-console generate-random-id dht-server lite-client FROM ubuntu:22.04 RUN apt-get update && \ - apt-get install -y wget libatomic1 openssl libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev && \ + apt-get install -y wget curl libatomic1 openssl libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev libjemalloc-dev htop net-tools netcat iptraf-ng jq tcpdump pv plzip && \ rm -rf /var/lib/apt/lists/* -RUN mkdir -p /var/ton-work/db && \ - mkdir -p /var/ton-work/db/static +RUN mkdir -p /var/ton-work/db /var/ton-work/scripts COPY --from=builder /ton/build/storage/storage-daemon/storage-daemon /usr/local/bin/ COPY --from=builder /ton/build/storage/storage-daemon/storage-daemon-cli /usr/local/bin/ @@ -33,7 +37,7 @@ COPY --from=builder /ton/build/validator-engine-console/validator-engine-console COPY --from=builder /ton/build/utils/generate-random-id /usr/local/bin/ WORKDIR /var/ton-work/db -COPY ./docker/init.sh ./docker/control.template ./ -RUN chmod +x init.sh +COPY ./docker/init.sh ./docker/control.template /var/ton-work/scripts/ +RUN chmod +x /var/ton-work/scripts/init.sh -ENTRYPOINT ["/var/ton-work/db/init.sh"] +ENTRYPOINT ["/var/ton-work/scripts/init.sh"] diff --git a/adnl/adnl-ext-server.cpp b/adnl/adnl-ext-server.cpp index ed04469cb..162a53afb 100644 --- a/adnl/adnl-ext-server.cpp +++ b/adnl/adnl-ext-server.cpp @@ -91,7 +91,7 @@ td::Status AdnlInboundConnection::process_custom_packet(td::BufferSlice &data, b auto F = fetch_tl_object(data.clone(), true); if (F.is_ok()) { if (nonce_.size() > 0 || !remote_id_.is_zero()) { - return td::Status::Error(ErrorCode::protoviolation, "duplicate authentificate"); + return td::Status::Error(ErrorCode::protoviolation, "duplicate authenticate"); } auto f = F.move_as_ok(); nonce_ = td::SecureString{f->nonce_.size() + 256}; diff --git a/assembly/cicd/jenkins/test-builds.groovy b/assembly/cicd/jenkins/test-builds.groovy index a959d75ab..47ce52e53 100644 --- a/assembly/cicd/jenkins/test-builds.groovy +++ b/assembly/cicd/jenkins/test-builds.groovy @@ -1,4 +1,5 @@ pipeline { + agent none stages { stage('Run Builds') { @@ -12,7 +13,7 @@ pipeline { sh ''' cp assembly/native/build-ubuntu-shared.sh . chmod +x build-ubuntu-shared.sh - ./build-ubuntu-shared.sh -t -a + ./build-ubuntu-shared.sh -a ''' sh ''' cd artifacts @@ -31,7 +32,7 @@ pipeline { sh ''' cp assembly/nix/build-linux-x86-64-nix.sh . chmod +x build-linux-x86-64-nix.sh - ./build-linux-x86-64-nix.sh -t + ./build-linux-x86-64-nix.sh ''' sh ''' cd artifacts @@ -50,7 +51,7 @@ pipeline { sh ''' cp assembly/native/build-ubuntu-shared.sh . chmod +x build-ubuntu-shared.sh - ./build-ubuntu-shared.sh -t -a + ./build-ubuntu-shared.sh -a ''' sh ''' cd artifacts @@ -69,7 +70,7 @@ pipeline { sh ''' cp assembly/nix/build-linux-arm64-nix.sh . chmod +x build-linux-arm64-nix.sh - ./build-linux-arm64-nix.sh -t + ./build-linux-arm64-nix.sh ''' sh ''' cd artifacts @@ -88,7 +89,7 @@ pipeline { sh ''' cp assembly/native/build-macos-shared.sh . chmod +x build-macos-shared.sh - ./build-macos-shared.sh -t -a + ./build-macos-shared.sh -a ''' sh ''' cd artifacts @@ -107,7 +108,7 @@ pipeline { sh ''' cp assembly/nix/build-macos-nix.sh . chmod +x build-macos-nix.sh - ./build-macos-nix.sh -t + ./build-macos-nix.sh ''' sh ''' cd artifacts @@ -126,7 +127,7 @@ pipeline { sh ''' cp assembly/native/build-macos-shared.sh . chmod +x build-macos-shared.sh - ./build-macos-shared.sh -t -a + ./build-macos-shared.sh -a ''' sh ''' cd artifacts @@ -145,7 +146,7 @@ pipeline { sh ''' cp assembly/nix/build-macos-nix.sh . chmod +x build-macos-nix.sh - ./build-macos-nix.sh -t + ./build-macos-nix.sh ''' sh ''' cd artifacts @@ -164,7 +165,7 @@ pipeline { sh ''' cp assembly/native/build-macos-shared.sh . chmod +x build-macos-shared.sh - ./build-macos-shared.sh -t -a + ./build-macos-shared.sh -a ''' sh ''' cd artifacts @@ -182,7 +183,7 @@ pipeline { timeout(time: 180, unit: 'MINUTES') { bat ''' copy assembly\\native\\build-windows.bat . - build-windows.bat -t + build-windows.bat ''' bat ''' cd artifacts diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh index 32a09e452..b296d3393 100644 --- a/assembly/native/build-macos-portable.sh +++ b/assembly/native/build-macos-portable.sh @@ -158,7 +158,7 @@ if [ "$with_tests" = true ]; then http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \ test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont \ test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \ - test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state + test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state test-emulator test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else ninja storage-daemon storage-daemon-cli blockchain-explorer \ diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh index 0f16eeda6..7574f481a 100644 --- a/assembly/native/build-macos-shared.sh +++ b/assembly/native/build-macos-shared.sh @@ -86,7 +86,7 @@ if [ "$with_tests" = true ]; then http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \ test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont \ test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \ - test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state + test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state test-emulator test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else ninja storage-daemon storage-daemon-cli blockchain-explorer \ diff --git a/assembly/native/build-toncryptolib-windows.bat b/assembly/native/build-toncryptolib-windows.bat index d732aa835..019e173ae 100644 --- a/assembly/native/build-toncryptolib-windows.bat +++ b/assembly/native/build-toncryptolib-windows.bat @@ -1,4 +1,4 @@ -REM execute this script inside elevated (Run as Administrator) console "x64 Native Tools Command Prompt for VS 2022" +REM execute this script inside elevated (Run as Administrator) console "x64 Native Tools Command Prompt for VS 2019" echo off @@ -31,7 +31,7 @@ git clone https://github.com/madler/zlib.git cd zlib git checkout v1.3.1 cd contrib\vstudio\vc14 -msbuild zlibstat.vcxproj /p:Configuration=ReleaseWithoutAsm /p:platform=x64 -p:PlatformToolset=v143 +msbuild zlibstat.vcxproj /p:Configuration=ReleaseWithoutAsm /p:platform=x64 -p:PlatformToolset=v142 IF %errorlevel% NEQ 0 ( echo Can't install zlib @@ -47,8 +47,8 @@ git clone https://github.com/lz4/lz4.git cd lz4 git checkout v1.9.4 cd build\VS2017\liblz4 -msbuild liblz4.vcxproj /p:Configuration=Release /p:platform=x64 -p:PlatformToolset=v143 -dir /s +msbuild liblz4.vcxproj /p:Configuration=Release /p:platform=x64 -p:PlatformToolset=v142 + IF %errorlevel% NEQ 0 ( echo Can't install lz4 exit /b %errorlevel% @@ -62,7 +62,7 @@ if not exist "secp256k1" ( git clone https://github.com/bitcoin-core/secp256k1.git cd secp256k1 git checkout v0.3.2 -cmake -G "Visual Studio 17 2022" -A x64 -S . -B build -DSECP256K1_ENABLE_MODULE_RECOVERY=ON -DBUILD_SHARED_LIBS=OFF +cmake -G "Visual Studio 16 2019" -A x64 -S . -B build -DSECP256K1_ENABLE_MODULE_RECOVERY=ON -DBUILD_SHARED_LIBS=OFF IF %errorlevel% NEQ 0 ( echo Can't configure secp256k1 exit /b %errorlevel% @@ -78,8 +78,7 @@ echo Using secp256k1... ) -if not exist "libsodium" ( -curl -Lo libsodium-1.0.18-stable-msvc.zip https://download.libsodium.org/libsodium/releases/libsodium-1.0.18-stable-msvc.zip +curl --retry 5 --retry-delay 10 -Lo libsodium-1.0.18-stable-msvc.zip https://download.libsodium.org/libsodium/releases/libsodium-1.0.18-stable-msvc.zip IF %errorlevel% NEQ 0 ( echo Can't download libsodium exit /b %errorlevel% diff --git a/assembly/native/build-ubuntu-portable.sh b/assembly/native/build-ubuntu-portable.sh index a3a11f1b2..b5a167626 100644 --- a/assembly/native/build-ubuntu-portable.sh +++ b/assembly/native/build-ubuntu-portable.sh @@ -150,7 +150,7 @@ ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli \ adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \ test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ - test-fec test-tddb test-db test-validator-session-state + test-fec test-tddb test-db test-validator-session-state test-emulator test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli \ diff --git a/assembly/native/build-ubuntu-shared.sh b/assembly/native/build-ubuntu-shared.sh index 7d5326a4f..4ce86d81f 100644 --- a/assembly/native/build-ubuntu-shared.sh +++ b/assembly/native/build-ubuntu-shared.sh @@ -1,7 +1,7 @@ #/bin/bash #sudo apt-get update -#sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev +#sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev with_tests=false with_artifacts=false @@ -42,7 +42,7 @@ else echo "Using compiled openssl_3" fi -cmake -GNinja .. \ +cmake -GNinja -DTON_USE_JEMALLOC=ON .. \ -DCMAKE_BUILD_TYPE=Release \ -DOPENSSL_ROOT_DIR=$opensslPath \ -DOPENSSL_INCLUDE_DIR=$opensslPath/include \ @@ -58,7 +58,7 @@ ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli \ adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \ test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ - test-fec test-tddb test-db test-validator-session-state + test-fec test-tddb test-db test-validator-session-state test-emulator test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli \ @@ -96,6 +96,8 @@ test $? -eq 0 || { echo "Can't strip final binaries"; exit 1; } ./lite-client/lite-client -V || exit 1 ./crypto/fift -V || exit 1 +ldd ./validator-engine/validator-engine || exit 1 + cd .. if [ "$with_artifacts" = true ]; then diff --git a/assembly/native/build-windows-2019.bat b/assembly/native/build-windows-2019.bat new file mode 100644 index 000000000..b528b05a2 --- /dev/null +++ b/assembly/native/build-windows-2019.bat @@ -0,0 +1,221 @@ +REM execute this script inside elevated (Run as Administrator) console "x64 Native Tools Command Prompt for VS 2019" + +echo off + +echo Installing chocolatey windows package manager... +@"%SystemRoot%\System32\WindowsPowerShell\v1.0\powershell.exe" -NoProfile -InputFormat None -ExecutionPolicy Bypass -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" && SET "PATH=%PATH%;%ALLUSERSPROFILE%\chocolatey\bin" +choco -? +IF %errorlevel% NEQ 0 ( + echo Can't install chocolatey + exit /b %errorlevel% +) + +choco feature enable -n allowEmptyChecksums + +echo Installing pkgconfiglite... +choco install -y pkgconfiglite +IF %errorlevel% NEQ 0 ( + echo Can't install pkgconfiglite + exit /b %errorlevel% +) + +echo Installing ninja... +choco install -y ninja +IF %errorlevel% NEQ 0 ( + echo Can't install ninja + exit /b %errorlevel% +) + +if not exist "zlib" ( +git clone https://github.com/madler/zlib.git +cd zlib +git checkout v1.3.1 +cd contrib\vstudio\vc14 +msbuild zlibstat.vcxproj /p:Configuration=ReleaseWithoutAsm /p:platform=x64 -p:PlatformToolset=v142 + +IF %errorlevel% NEQ 0 ( + echo Can't install zlib + exit /b %errorlevel% +) +cd ..\..\..\.. +) else ( +echo Using zlib... +) + +if not exist "lz4" ( +git clone https://github.com/lz4/lz4.git +cd lz4 +git checkout v1.9.4 +cd build\VS2017\liblz4 +msbuild liblz4.vcxproj /p:Configuration=Release /p:platform=x64 -p:PlatformToolset=v142 + +IF %errorlevel% NEQ 0 ( + echo Can't install lz4 + exit /b %errorlevel% +) +cd ..\..\..\.. +) else ( +echo Using lz4... +) + +if not exist "secp256k1" ( +git clone https://github.com/bitcoin-core/secp256k1.git +cd secp256k1 +git checkout v0.3.2 +cmake -G "Visual Studio 16 2019" -A x64 -S . -B build -DSECP256K1_ENABLE_MODULE_RECOVERY=ON -DBUILD_SHARED_LIBS=OFF +IF %errorlevel% NEQ 0 ( + echo Can't configure secp256k1 + exit /b %errorlevel% +) +cmake --build build --config Release +IF %errorlevel% NEQ 0 ( + echo Can't install secp256k1 + exit /b %errorlevel% +) +cd .. +) else ( +echo Using secp256k1... +) + + +curl --retry 5 --retry-delay 10 -Lo libsodium-1.0.18-stable-msvc.zip https://download.libsodium.org/libsodium/releases/libsodium-1.0.18-stable-msvc.zip +IF %errorlevel% NEQ 0 ( + echo Can't download libsodium + exit /b %errorlevel% +) +unzip libsodium-1.0.18-stable-msvc.zip +) else ( +echo Using libsodium... +) + +if not exist "openssl-3.1.4" ( +curl -Lo openssl-3.1.4.zip https://github.com/neodiX42/precompiled-openssl-win64/raw/main/openssl-3.1.4.zip +IF %errorlevel% NEQ 0 ( + echo Can't download OpenSSL + exit /b %errorlevel% +) +unzip -q openssl-3.1.4.zip +) else ( +echo Using openssl... +) + +if not exist "libmicrohttpd-0.9.77-w32-bin" ( +curl -Lo libmicrohttpd-0.9.77-w32-bin.zip https://github.com/neodiX42/precompiled-openssl-win64/raw/main/libmicrohttpd-0.9.77-w32-bin.zip +IF %errorlevel% NEQ 0 ( + echo Can't download libmicrohttpd + exit /b %errorlevel% +) +unzip -q libmicrohttpd-0.9.77-w32-bin.zip +) else ( +echo Using libmicrohttpd... +) + +if not exist "readline-5.0-1-lib" ( +curl -Lo readline-5.0-1-lib.zip https://github.com/neodiX42/precompiled-openssl-win64/raw/main/readline-5.0-1-lib.zip +IF %errorlevel% NEQ 0 ( + echo Can't download readline + exit /b %errorlevel% +) +unzip -q -d readline-5.0-1-lib readline-5.0-1-lib.zip +) else ( +echo Using readline... +) + + +set root=%cd% +echo %root% +set SODIUM_DIR=%root%\libsodium + +mkdir build +cd build +cmake -GNinja -DCMAKE_BUILD_TYPE=Release ^ +-DPORTABLE=1 ^ +-DSODIUM_USE_STATIC_LIBS=1 ^ +-DSECP256K1_FOUND=1 ^ +-DSECP256K1_INCLUDE_DIR=%root%\secp256k1\include ^ +-DSECP256K1_LIBRARY=%root%\secp256k1\build\src\Release\libsecp256k1.lib ^ +-DLZ4_FOUND=1 ^ +-DLZ4_INCLUDE_DIRS=%root%\lz4\lib ^ +-DLZ4_LIBRARIES=%root%\lz4\build\VS2017\liblz4\bin\x64_Release\liblz4_static.lib ^ +-DMHD_FOUND=1 ^ +-DMHD_LIBRARY=%root%\libmicrohttpd-0.9.77-w32-bin\x86_64\VS2019\Release-static\libmicrohttpd.lib ^ +-DMHD_INCLUDE_DIR=%root%\libmicrohttpd-0.9.77-w32-bin\x86_64\VS2019\Release-static ^ +-DZLIB_FOUND=1 ^ +-DZLIB_INCLUDE_DIR=%root%\zlib ^ +-DZLIB_LIBRARIES=%root%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib ^ +-DOPENSSL_FOUND=1 ^ +-DOPENSSL_INCLUDE_DIR=%root%\openssl-3.1.4\x64\include ^ +-DOPENSSL_CRYPTO_LIBRARY=%root%\openssl-3.1.4\x64\lib\libcrypto_static.lib ^ +-DREADLINE_INCLUDE_DIR=%root%\readline-5.0-1-lib\include ^ +-DREADLINE_LIBRARY=%root%\readline-5.0-1-lib\lib\readline.lib ^ +-DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj" .. +IF %errorlevel% NEQ 0 ( + echo Can't configure TON + exit /b %errorlevel% +) + +IF "%1"=="-t" ( +ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson ^ +tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id ^ +json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ +test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont test-net ^ +test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ +test-fec test-tddb test-db test-validator-session-state test-emulator +IF %errorlevel% NEQ 0 ( + echo Can't compile TON + exit /b %errorlevel% +) +) else ( +ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson ^ +tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id ^ +json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator +IF %errorlevel% NEQ 0 ( + echo Can't compile TON + exit /b %errorlevel% +) +) + +copy validator-engine\validator-engine.exe test +IF %errorlevel% NEQ 0 ( + echo validator-engine.exe does not exist + exit /b %errorlevel% +) + +IF "%1"=="-t" ( + echo Running tests... +REM ctest -C Release --output-on-failure -E "test-catchain|test-actors|test-validator-session-state" + ctest -C Release --output-on-failure -E "test-bigint" --timeout 1800 + IF %errorlevel% NEQ 0 ( + echo Some tests failed + exit /b %errorlevel% + ) +) + + +echo Creating artifacts... +cd .. +mkdir artifacts +mkdir artifacts\smartcont +mkdir artifacts\lib + +for %%I in (build\storage\storage-daemon\storage-daemon.exe ^ +build\storage\storage-daemon\storage-daemon-cli.exe ^ +build\blockchain-explorer\blockchain-explorer.exe ^ +build\crypto\fift.exe ^ +build\crypto\tlbc.exe ^ +build\crypto\func.exe ^ +build\crypto\create-state.exe ^ +build\validator-engine-console\validator-engine-console.exe ^ +build\tonlib\tonlib-cli.exe ^ +build\tonlib\tonlibjson.dll ^ +build\http\http-proxy.exe ^ +build\rldp-http-proxy\rldp-http-proxy.exe ^ +build\dht-server\dht-server.exe ^ +build\lite-client\lite-client.exe ^ +build\validator-engine\validator-engine.exe ^ +build\utils\generate-random-id.exe ^ +build\utils\json2tlo.exe ^ +build\adnl\adnl-proxy.exe ^ +build\emulator\emulator.dll) do (strip -g %%I & copy %%I artifacts\) +xcopy /e /k /h /i crypto\smartcont artifacts\smartcont +xcopy /e /k /h /i crypto\fift\lib artifacts\lib diff --git a/assembly/native/build-windows-github-2019.bat b/assembly/native/build-windows-github-2019.bat new file mode 100644 index 000000000..4f7eee056 --- /dev/null +++ b/assembly/native/build-windows-github-2019.bat @@ -0,0 +1,2 @@ +call "C:\Program Files (x86)\Microsoft Visual Studio\2019\%1\VC\Auxiliary\Build\vcvars64.bat" +call build-windows-2019.bat -t diff --git a/assembly/native/build-windows-github.bat b/assembly/native/build-windows-github.bat index 7cad8c7e5..bfa2d3362 100644 --- a/assembly/native/build-windows-github.bat +++ b/assembly/native/build-windows-github.bat @@ -1,2 +1,2 @@ call "C:\Program Files\Microsoft Visual Studio\2022\%1\VC\Auxiliary\Build\vcvars64.bat" -call build-windows.bat -t \ No newline at end of file +call build-windows.bat -t diff --git a/assembly/native/build-windows.bat b/assembly/native/build-windows.bat index 9b7322e1d..a9871aa96 100644 --- a/assembly/native/build-windows.bat +++ b/assembly/native/build-windows.bat @@ -161,7 +161,7 @@ tonlib-cli validator-engine lite-client pow-miner validator-engine-console gener json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont test-net ^ test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ -test-fec test-tddb test-db test-validator-session-state +test-fec test-tddb test-db test-validator-session-state test-emulator IF %errorlevel% NEQ 0 ( echo Can't compile TON exit /b %errorlevel% diff --git a/assembly/nix/build-linux-arm64-nix.sh b/assembly/nix/build-linux-arm64-nix.sh index 8e5c367c9..7e85a8712 100644 --- a/assembly/nix/build-linux-arm64-nix.sh +++ b/assembly/nix/build-linux-arm64-nix.sh @@ -15,8 +15,6 @@ while getopts 't' flag; do done cp assembly/nix/linux-arm64* . -cp assembly/nix/microhttpd.nix . -cp assembly/nix/openssl.nix . export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz if [ "$with_tests" = true ]; then @@ -30,7 +28,9 @@ cp ./result/bin/* artifacts/ test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; } chmod +x artifacts/* rm -rf result + nix-build linux-arm64-tonlib.nix + cp ./result/lib/libtonlibjson.so.0.5 artifacts/libtonlibjson.so cp ./result/lib/libemulator.so artifacts/ cp ./result/lib/fift/* artifacts/lib/ diff --git a/assembly/nix/build-linux-x86-64-nix.sh b/assembly/nix/build-linux-x86-64-nix.sh index 38431ca43..c1f1dcf37 100644 --- a/assembly/nix/build-linux-x86-64-nix.sh +++ b/assembly/nix/build-linux-x86-64-nix.sh @@ -15,8 +15,6 @@ while getopts 't' flag; do done cp assembly/nix/linux-x86-64* . -cp assembly/nix/microhttpd.nix . -cp assembly/nix/openssl.nix . export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz if [ "$with_tests" = true ]; then @@ -30,7 +28,9 @@ cp ./result/bin/* artifacts/ test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; } chmod +x artifacts/* rm -rf result + nix-build linux-x86-64-tonlib.nix + cp ./result/lib/libtonlibjson.so.0.5 artifacts/libtonlibjson.so cp ./result/lib/libemulator.so artifacts/ cp ./result/lib/fift/* artifacts/lib/ diff --git a/assembly/nix/build-macos-nix.sh b/assembly/nix/build-macos-nix.sh index 12977745b..c75ca0428 100644 --- a/assembly/nix/build-macos-nix.sh +++ b/assembly/nix/build-macos-nix.sh @@ -28,7 +28,9 @@ cp ./result-bin/bin/* artifacts/ test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; } chmod +x artifacts/* rm -rf result-bin + nix-build macos-tonlib.nix + cp ./result/lib/libtonlibjson.dylib artifacts/ cp ./result/lib/libemulator.dylib artifacts/ cp ./result/lib/fift/* artifacts/lib/ diff --git a/assembly/nix/linux-arm64-static.nix b/assembly/nix/linux-arm64-static.nix index 8c2749b07..536152d26 100644 --- a/assembly/nix/linux-arm64-static.nix +++ b/assembly/nix/linux-arm64-static.nix @@ -6,9 +6,23 @@ , testing ? false }: let - microhttpdmy = (import ./microhttpd.nix) {}; + staticOptions = pkg: pkg.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ]; + }); + + secp256k1Static = (staticOptions pkgs.secp256k1); + libsodiumStatic = (staticOptions pkgs.libsodium); + jemallocStatic = (staticOptions pkgs.jemalloc); + + microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ]; + }); + in -with import microhttpdmy; stdenv.mkDerivation { pname = "ton"; version = "dev-bin"; @@ -16,31 +30,33 @@ stdenv.mkDerivation { src = ./.; nativeBuildInputs = with pkgs; - [ - cmake ninja git pkg-config - ]; + [ cmake ninja git pkg-config ]; buildInputs = with pkgs; [ - pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 glibc.static pkgsStatic.lz4 + (openssl.override { static = true; }).dev + microhttpdStatic.dev + (zlib.override { shared = false; }).dev + (lz4.override { enableStatic = true; enableShared = false; }).dev + jemallocStatic + secp256k1Static + libsodiumStatic.dev + glibc.static ]; - makeStatic = true; - doCheck = testing; - cmakeFlags = [ "-DTON_USE_ABSEIL=OFF" "-DNIX=ON" "-DBUILD_SHARED_LIBS=OFF" "-DCMAKE_LINK_SEARCH_START_STATIC=ON" "-DCMAKE_LINK_SEARCH_END_STATIC=ON" - "-DMHD_FOUND=1" - "-DMHD_INCLUDE_DIR=${microhttpdmy}/usr/local/include" - "-DMHD_LIBRARY=${microhttpdmy}/usr/local/lib/libmicrohttpd.a" - "-DCMAKE_CTEST_ARGUMENTS=--timeout;1800" + "-DTON_USE_JEMALLOC=ON" ]; + makeStatic = true; + doCheck = testing; + LDFLAGS = [ - "-static-libgcc" "-static-libstdc++" "-static" + "-static-libgcc" "-static-libstdc++" "-static" ]; } diff --git a/assembly/nix/linux-arm64-tonlib.nix b/assembly/nix/linux-arm64-tonlib.nix index ae62ca263..a051e34cd 100644 --- a/assembly/nix/linux-arm64-tonlib.nix +++ b/assembly/nix/linux-arm64-tonlib.nix @@ -5,9 +5,21 @@ , stdenv ? pkgs.stdenv }: let - microhttpdmy = (import ./microhttpd.nix) {}; + staticOptions = pkg: pkg.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ]; + }); + + secp256k1Static = (staticOptions pkgs.secp256k1); + libsodiumStatic = (staticOptions pkgs.libsodium); + + microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ]; + }); in -with import microhttpdmy; pkgs.llvmPackages_16.stdenv.mkDerivation { pname = "ton"; version = "dev-lib"; @@ -21,7 +33,12 @@ pkgs.llvmPackages_16.stdenv.mkDerivation { buildInputs = with pkgs; [ - pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 pkgsStatic.lz4 + (openssl.override { static = true; }).dev + microhttpdStatic.dev + (zlib.override { shared = false; }).dev + (lz4.override { enableStatic = true; enableShared = false; }).dev + secp256k1Static + libsodiumStatic.dev ]; dontAddStaticConfigureFlags = false; @@ -29,9 +46,6 @@ pkgs.llvmPackages_16.stdenv.mkDerivation { cmakeFlags = [ "-DTON_USE_ABSEIL=OFF" "-DNIX=ON" - "-DMHD_FOUND=1" - "-DMHD_INCLUDE_DIR=${microhttpdmy}/usr/local/include" - "-DMHD_LIBRARY=${microhttpdmy}/usr/local/lib/libmicrohttpd.a" ]; LDFLAGS = [ diff --git a/assembly/nix/linux-x86-64-static.nix b/assembly/nix/linux-x86-64-static.nix index 8c2749b07..a96a9e867 100644 --- a/assembly/nix/linux-x86-64-static.nix +++ b/assembly/nix/linux-x86-64-static.nix @@ -6,9 +6,23 @@ , testing ? false }: let - microhttpdmy = (import ./microhttpd.nix) {}; + staticOptions = pkg: pkg.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ]; + }); + + secp256k1Static = (staticOptions pkgs.secp256k1); + libsodiumStatic = (staticOptions pkgs.libsodium); + jemallocStatic = (staticOptions pkgs.jemalloc); + + microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ]; + }); + in -with import microhttpdmy; stdenv.mkDerivation { pname = "ton"; version = "dev-bin"; @@ -16,31 +30,33 @@ stdenv.mkDerivation { src = ./.; nativeBuildInputs = with pkgs; - [ - cmake ninja git pkg-config - ]; + [ cmake ninja git pkg-config ]; buildInputs = with pkgs; [ - pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 glibc.static pkgsStatic.lz4 + (openssl.override { static = true; }).dev + microhttpdStatic.dev + (zlib.override { shared = false; }).dev + (lz4.override { enableStatic = true; enableShared = false; }).dev + jemallocStatic + secp256k1Static + libsodiumStatic.dev + glibc.static ]; - makeStatic = true; - doCheck = testing; - cmakeFlags = [ "-DTON_USE_ABSEIL=OFF" "-DNIX=ON" "-DBUILD_SHARED_LIBS=OFF" "-DCMAKE_LINK_SEARCH_START_STATIC=ON" "-DCMAKE_LINK_SEARCH_END_STATIC=ON" - "-DMHD_FOUND=1" - "-DMHD_INCLUDE_DIR=${microhttpdmy}/usr/local/include" - "-DMHD_LIBRARY=${microhttpdmy}/usr/local/lib/libmicrohttpd.a" - "-DCMAKE_CTEST_ARGUMENTS=--timeout;1800" + "-DTON_USE_JEMALLOC=ON" ]; + makeStatic = true; + doCheck = testing; + LDFLAGS = [ - "-static-libgcc" "-static-libstdc++" "-static" + "-static-libgcc" "-static-libstdc++" "-fPIC" ]; } diff --git a/assembly/nix/linux-x86-64-tonlib.nix b/assembly/nix/linux-x86-64-tonlib.nix index 5a6e43e8f..afcbe3ba6 100644 --- a/assembly/nix/linux-x86-64-tonlib.nix +++ b/assembly/nix/linux-x86-64-tonlib.nix @@ -7,20 +7,35 @@ , stdenv ? pkgs.stdenv }: let - system = builtins.currentSystem; - - nixos1909 = (import (builtins.fetchTarball { - url = "https://channels.nixos.org/nixos-19.09/nixexprs.tar.xz"; - sha256 = "1vp1h2gkkrckp8dzkqnpcc6xx5lph5d2z46sg2cwzccpr8ay58zy"; - }) { inherit system; }); - glibc227 = nixos1909.glibc // { pname = "glibc"; }; - stdenv227 = let - cc = pkgs.wrapCCWith { - cc = nixos1909.buildPackages.gcc-unwrapped; - libc = glibc227; - bintools = pkgs.binutils.override { libc = glibc227; }; - }; - in (pkgs.overrideCC pkgs.stdenv cc); + system = builtins.currentSystem; + + staticOptions = pkg: pkg.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ]; + }); + + secp256k1Static = (staticOptions pkgs.secp256k1); + libsodiumStatic = (staticOptions pkgs.libsodium); + + microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ]; + }); + + nixos1909 = (import (builtins.fetchTarball { + url = "https://channels.nixos.org/nixos-19.09/nixexprs.tar.xz"; + sha256 = "1vp1h2gkkrckp8dzkqnpcc6xx5lph5d2z46sg2cwzccpr8ay58zy"; + }) { inherit system; }); + glibc227 = nixos1909.glibc // { pname = "glibc"; }; + stdenv227 = let + cc = pkgs.wrapCCWith { + cc = nixos1909.buildPackages.gcc-unwrapped; + libc = glibc227; + bintools = pkgs.binutils.override { libc = glibc227; }; + }; + in (pkgs.overrideCC pkgs.stdenv cc); in stdenv227.mkDerivation { @@ -34,7 +49,12 @@ stdenv227.mkDerivation { buildInputs = with pkgs; [ - pkgsStatic.openssl pkgsStatic.zlib pkgsStatic.libmicrohttpd.dev pkgsStatic.libsodium.dev pkgsStatic.secp256k1 pkgsStatic.lz4 + (openssl.override { static = true; }).dev + microhttpdStatic.dev + (zlib.override { shared = false; }).dev + (lz4.override { enableStatic = true; enableShared = false; }).dev + secp256k1Static + libsodiumStatic.dev ]; dontAddStaticConfigureFlags = false; diff --git a/assembly/nix/macos-static.nix b/assembly/nix/macos-static.nix index be15579ca..2fd0b3a66 100644 --- a/assembly/nix/macos-static.nix +++ b/assembly/nix/macos-static.nix @@ -17,7 +17,7 @@ pkgs.llvmPackages_14.stdenv.mkDerivation { buildInputs = with pkgs; lib.forEach [ - secp256k1 libsodium.dev libmicrohttpd.dev gmp.dev nettle.dev libtasn1.dev libidn2.dev libunistring.dev gettext (gnutls.override { withP11-kit = false; }).dev + secp256k1 libsodium.dev libmicrohttpd.dev gmp.dev nettle.dev libtasn1.dev libidn2.dev libunistring.dev gettext jemalloc (gnutls.override { withP11-kit = false; }).dev ] (x: x.overrideAttrs(oldAttrs: rec { configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-shared" "--disable-tests" ]; dontDisableStatic = true; })) ++ [ @@ -38,13 +38,13 @@ pkgs.llvmPackages_14.stdenv.mkDerivation { cmakeFlags = [ "-DTON_USE_ABSEIL=OFF" "-DNIX=ON" + "-DTON_USE_JEMALLOC=ON" "-DCMAKE_CROSSCOMPILING=OFF" "-DCMAKE_LINK_SEARCH_START_STATIC=ON" "-DCMAKE_LINK_SEARCH_END_STATIC=ON" "-DBUILD_SHARED_LIBS=OFF" "-DCMAKE_CXX_FLAGS=-stdlib=libc++" "-DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=11.3" - "-DCMAKE_CTEST_ARGUMENTS=--timeout;1800" ]; LDFLAGS = [ diff --git a/assembly/nix/microhttpd.nix b/assembly/nix/microhttpd.nix deleted file mode 100644 index 4f871425a..000000000 --- a/assembly/nix/microhttpd.nix +++ /dev/null @@ -1,28 +0,0 @@ -{ pkgs ? import { system = builtins.currentSystem; } -, stdenv ? pkgs.stdenv -, fetchgit ? pkgs.fetchgit -}: - -stdenv.mkDerivation rec { - name = "microhttpdmy"; - - - src = fetchgit { - url = "https://git.gnunet.org/libmicrohttpd.git"; - rev = "refs/tags/v0.9.77"; - sha256 = "sha256-x+nfB07PbZwBlFc6kZZFYiRpk0a3QN/ByHB+hC8na/o="; - }; - - nativeBuildInputs = with pkgs; [ automake libtool autoconf texinfo ]; - - buildInputs = with pkgs; [ ]; - - configurePhase = '' - ./autogen.sh - ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic - ''; - - installPhase = '' - make install DESTDIR=$out - ''; -} diff --git a/assembly/nix/openssl.nix b/assembly/nix/openssl.nix deleted file mode 100644 index 8d30aa504..000000000 --- a/assembly/nix/openssl.nix +++ /dev/null @@ -1,30 +0,0 @@ -{ pkgs ? import { system = builtins.currentSystem; } -, stdenv ? pkgs.stdenv -, fetchFromGitHub ? pkgs.fetchFromGitHub -}: - -stdenv.mkDerivation rec { - name = "opensslmy"; - - src = fetchFromGitHub { - owner = "openssl"; - repo = "openssl"; - rev = "refs/tags/openssl-3.1.4"; - sha256 = "sha256-Vvf1wiNb4ikg1lIS9U137aodZ2JzM711tSWMJFYWtWI="; - }; - - nativeBuildInputs = with pkgs; [ perl ]; - - buildInputs = with pkgs; [ ]; - - postPatch = '' - patchShebangs Configure - ''; - - configurePhase = '' - ./Configure no-shared - ''; - installPhase = '' - make install DESTDIR=$out - ''; -} diff --git a/assembly/wasm/fift-func-wasm-build-ubuntu.sh b/assembly/wasm/fift-func-wasm-build-ubuntu.sh index 6daf2d4cd..7687a63f5 100644 --- a/assembly/wasm/fift-func-wasm-build-ubuntu.sh +++ b/assembly/wasm/fift-func-wasm-build-ubuntu.sh @@ -38,22 +38,26 @@ cd .. git clone https://github.com/madler/zlib.git cd zlib +git checkout v1.3.1 ZLIB_DIR=`pwd` cd .. git clone https://github.com/lz4/lz4.git cd lz4 +git checkout v1.9.4 LZ4_DIR=`pwd` cd .. git clone https://github.com/bitcoin-core/secp256k1.git cd secp256k1 +git checkout v0.3.2 ./autogen.sh SECP256K1_DIR=`pwd` cd .. -git clone https://github.com/jedisct1/libsodium --branch stable +git clone https://github.com/jedisct1/libsodium cd libsodium +git checkout 1.0.18-RELEASE SODIUM_DIR=`pwd` cd .. diff --git a/blockchain-explorer/blockchain-explorer-query.cpp b/blockchain-explorer/blockchain-explorer-query.cpp index b53e79696..1808a3c46 100644 --- a/blockchain-explorer/blockchain-explorer-query.cpp +++ b/blockchain-explorer/blockchain-explorer-query.cpp @@ -50,32 +50,6 @@ #include "vm/vm.h" #include "vm/cp0.h" -namespace { - -td::Ref prepare_vm_c7(ton::UnixTime now, ton::LogicalTime lt, td::Ref my_addr, - const block::CurrencyCollection &balance) { - td::BitArray<256> rand_seed; - td::RefInt256 rand_seed_int{true}; - td::Random::secure_bytes(rand_seed.as_slice()); - if (!rand_seed_int.unique_write().import_bits(rand_seed.cbits(), 256, false)) { - return {}; - } - auto tuple = vm::make_tuple_ref(td::make_refint(0x076ef1ea), // [ magic:0x076ef1ea - td::make_refint(0), // actions:Integer - td::make_refint(0), // msgs_sent:Integer - td::make_refint(now), // unixtime:Integer - td::make_refint(lt), // block_lt:Integer - td::make_refint(lt), // trans_lt:Integer - std::move(rand_seed_int), // rand_seed:Integer - balance.as_vm_tuple(), // balance_remaining:[Integer (Maybe Cell)] - my_addr, // myself:MsgAddressInt - vm::StackEntry()); // global_config:(Maybe Cell) ] = SmartContractInfo; - LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple).to_string(); - return vm::make_tuple_ref(std::move(tuple)); -} - -} // namespace - td::Result parse_block_id(std::map &opts, bool allow_empty) { if (allow_empty) { if (opts.count("workchain") == 0 && opts.count("shard") == 0 && opts.count("seqno") == 0) { @@ -1343,111 +1317,71 @@ void HttpQueryRunMethod::start_up_query() { if (R.is_error()) { td::actor::send_closure(SelfId, &HttpQueryRunMethod::abort_query, R.move_as_error_prefix("litequery failed: ")); } else { - td::actor::send_closure(SelfId, &HttpQueryRunMethod::got_account, R.move_as_ok()); + td::actor::send_closure(SelfId, &HttpQueryRunMethod::got_result, R.move_as_ok()); } }); + auto a = ton::create_tl_object(addr_.workchain, addr_.addr); - auto query = ton::serialize_tl_object(ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), std::move(a)), - true); + td::int64 method_id = (td::crc16(td::Slice{method_name_}) & 0xffff) | 0x10000; + + // serialize params + vm::CellBuilder cb; + td::Ref cell; + if (!(vm::Stack{params_}.serialize(cb) && cb.finalize_to(cell))) { + return abort_query(td::Status::Error("cannot serialize stack with get-method parameters")); + } + auto params_serialized = vm::std_boc_serialize(std::move(cell)); + if (params_serialized.is_error()) { + return abort_query(params_serialized.move_as_error_prefix("cannot serialize stack with get-method parameters : ")); + } + + auto query = ton::serialize_tl_object( + ton::create_tl_object( + 0x17, ton::create_tl_lite_block_id(block_id_), std::move(a), method_id, params_serialized.move_as_ok()), + true); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, std::move(query), std::move(P)); } -void HttpQueryRunMethod::got_account(td::BufferSlice data) { - auto F = ton::fetch_tl_object(std::move(data), true); +void HttpQueryRunMethod::got_result(td::BufferSlice data) { + auto F = ton::fetch_tl_object(std::move(data), true); if (F.is_error()) { - abort_query(F.move_as_error()); - return; + return abort_query(F.move_as_error()); } - auto f = F.move_as_ok(); - data_ = std::move(f->state_); - proof_ = std::move(f->proof_); - shard_proof_ = std::move(f->shard_proof_); - block_id_ = ton::create_block_id(f->id_); - res_block_id_ = ton::create_block_id(f->shardblk_); - - finish_query(); -} - -void HttpQueryRunMethod::finish_query() { - if (promise_) { - auto page = [&]() -> std::string { - HttpAnswer A{"account", prefix_}; - A.set_account_id(addr_); - A.set_block_id(res_block_id_); + auto page = [&]() -> std::string { + HttpAnswer A{"account", prefix_}; + A.set_account_id(addr_); + A.set_block_id(ton::create_block_id(f->id_)); + if (f->exit_code_ != 0) { + A.abort(PSTRING() << "VM terminated with error code " << f->exit_code_); + return A.finish(); + } - block::AccountState account_state; - account_state.blk = block_id_; - account_state.shard_blk = res_block_id_; - account_state.shard_proof = std::move(shard_proof_); - account_state.proof = std::move(proof_); - account_state.state = std::move(data_); - auto r_info = account_state.validate(block_id_, addr_); - if (r_info.is_error()) { - A.abort(r_info.move_as_error()); - return A.finish(); - } - auto info = r_info.move_as_ok(); - if (info.root.is_null()) { - A.abort(PSTRING() << "account state of " << addr_ << " is empty (cannot run method `" << method_name_ << "`)"); - return A.finish(); - } - block::gen::Account::Record_account acc; - block::gen::AccountStorage::Record store; - block::CurrencyCollection balance; - if (!(tlb::unpack_cell(info.root, acc) && tlb::csr_unpack(acc.storage, store) && - balance.validate_unpack(store.balance))) { - A.abort("error unpacking account state"); + std::ostringstream os; + os << "result: "; + if (f->result_.empty()) { + os << ""; + } else { + auto r_cell = vm::std_boc_deserialize(f->result_); + if (r_cell.is_error()) { + A.abort(PSTRING() << "cannot deserialize VM result boc: " << r_cell.move_as_error()); return A.finish(); } - int tag = block::gen::t_AccountState.get_tag(*store.state); - switch (tag) { - case block::gen::AccountState::account_uninit: - A.abort(PSTRING() << "account " << addr_ << " not initialized yet (cannot run any methods)"); - return A.finish(); - case block::gen::AccountState::account_frozen: - A.abort(PSTRING() << "account " << addr_ << " frozen (cannot run any methods)"); - return A.finish(); - } - - CHECK(store.state.write().fetch_ulong(1) == 1); // account_init$1 _:StateInit = AccountState; - block::gen::StateInit::Record state_init; - CHECK(tlb::csr_unpack(store.state, state_init)); - auto code = state_init.code->prefetch_ref(); - auto data = state_init.data->prefetch_ref(); - auto stack = td::make_ref(std::move(params_)); - td::int64 method_id = (td::crc16(td::Slice{method_name_}) & 0xffff) | 0x10000; - stack.write().push_smallint(method_id); - long long gas_limit = vm::GasLimits::infty; - // OstreamLogger ostream_logger(ctx.error_stream); - // auto log = create_vm_log(ctx.error_stream ? &ostream_logger : nullptr); - vm::GasLimits gas{gas_limit}; - LOG(DEBUG) << "creating VM"; - vm::VmState vm{code, std::move(stack), gas, 1, data, vm::VmLog()}; - vm.set_c7(prepare_vm_c7(info.gen_utime, info.gen_lt, acc.addr, balance)); // tuple with SmartContractInfo - // vm.incr_stack_trace(1); // enable stack dump after each step - int exit_code = ~vm.run(); - if (exit_code != 0) { - A.abort(PSTRING() << "VM terminated with error code " << exit_code); + auto cs = vm::load_cell_slice(r_cell.move_as_ok()); + td::Ref stack; + if (!(vm::Stack::deserialize_to(cs, stack, 0) && cs.empty_ext())) { + A.abort("VM result boc cannot be deserialized"); return A.finish(); } - stack = vm.get_stack_ref(); - { - std::ostringstream os; - os << "result: "; - stack->dump(os, 3); - - A << HttpAnswer::CodeBlock{os.str()}; - } - - return A.finish(); - }(); - auto R = MHD_create_response_from_buffer(page.length(), const_cast(page.c_str()), MHD_RESPMEM_MUST_COPY); - MHD_add_response_header(R, "Content-Type", "text/html"); - promise_.set_value(std::move(R)); - } + stack->dump(os, 3); + } + A << HttpAnswer::CodeBlock{os.str()}; + return A.finish(); + }(); + auto R = MHD_create_response_from_buffer(page.length(), const_cast(page.c_str()), MHD_RESPMEM_MUST_COPY); + MHD_add_response_header(R, "Content-Type", "text/html"); + promise_.set_value(std::move(R)); stop(); } HttpQueryStatus::HttpQueryStatus(std::string prefix, td::Promise promise) diff --git a/blockchain-explorer/blockchain-explorer-query.hpp b/blockchain-explorer/blockchain-explorer-query.hpp index 0ce52af6e..29501265b 100644 --- a/blockchain-explorer/blockchain-explorer-query.hpp +++ b/blockchain-explorer/blockchain-explorer-query.hpp @@ -311,22 +311,14 @@ class HttpQueryRunMethod : public HttpQueryCommon { std::vector params, std::string prefix, td::Promise promise); HttpQueryRunMethod(std::map opts, std::string prefix, td::Promise promise); - void finish_query(); - void start_up_query() override; - void got_account(td::BufferSlice result); + void got_result(td::BufferSlice result); private: ton::BlockIdExt block_id_; block::StdAddress addr_; - std::string method_name_; std::vector params_; - - td::BufferSlice data_; - td::BufferSlice proof_; - td::BufferSlice shard_proof_; - ton::BlockIdExt res_block_id_; }; class HttpQueryStatus : public HttpQueryCommon { diff --git a/common/global-version.h b/common/global-version.h index a6775ffa4..a3032ebf2 100644 --- a/common/global-version.h +++ b/common/global-version.h @@ -19,6 +19,6 @@ namespace ton { // See doc/GlobalVersions.md -const int SUPPORTED_VERSION = 7; +const int SUPPORTED_VERSION = 8; } diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index c5f1add85..3110b1a85 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -246,7 +246,10 @@ class HardforkCreator : public td::actor::Actor { } void send_shard_block_info(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::BufferSlice data) override { } - void send_broadcast(ton::BlockBroadcast broadcast) override { + void send_block_candidate(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override { + } + void send_broadcast(ton::BlockBroadcast broadcast, bool custom_overlays_only) override { } void download_block(ton::BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) override { diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 39223e12e..c539e6255 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -481,10 +481,17 @@ if (NOT CMAKE_CROSSCOMPILING OR USE_EMSCRIPTEN) OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/${ARG_DEST_FIF} ) set(ARG_DEST_CPP "${ARG_DEST}.cpp") + + if (WIN32) + set(ARG_LIB_DIR "fift/lib@smartcont") + else() + set(ARG_LIB_DIR "fift/lib:smartcont") + endif() + add_custom_command( COMMENT "Generate ${ARG_DEST_CPP}" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - COMMAND fift -Ifift/lib:smartcont -s asm-to-cpp.fif ${ARG_DEST_FIF} ${ARG_DEST_CPP} ${ARG_NAME} + COMMAND fift -I${ARG_LIB_DIR} -s asm-to-cpp.fif ${ARG_DEST_FIF} ${ARG_DEST_CPP} ${ARG_NAME} MAIN_DEPENDENCY ${ARG_SOURCE} DEPENDS fift ${ARG_DEST_FIF} smartcont/asm-to-cpp.fif fift/lib/Asm.fif OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/${ARG_DEST_CPP} diff --git a/crypto/block/block-parse.cpp b/crypto/block/block-parse.cpp index 7d51b2e23..50851c795 100644 --- a/crypto/block/block-parse.cpp +++ b/crypto/block/block-parse.cpp @@ -813,19 +813,45 @@ int IntermediateAddress::get_size(const vm::CellSlice& cs) const { const IntermediateAddress t_IntermediateAddress; bool MsgEnvelope::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { - return cs.fetch_ulong(4) == 4 // msg_envelope#4 - && t_IntermediateAddress.validate_skip(ops, cs, weak) // cur_addr:IntermediateAddress - && t_IntermediateAddress.validate_skip(ops, cs, weak) // next_addr:IntermediateAddress - && t_Grams.validate_skip(ops, cs, weak) // fwd_fee_remaining:Grams - && t_Ref_Message.validate_skip(ops, cs, weak); // msg:^Message + switch (get_tag(cs)) { + case 4: + return cs.fetch_ulong(4) == 4 // msg_envelope#4 + && t_IntermediateAddress.validate_skip(ops, cs, weak) // cur_addr:IntermediateAddress + && t_IntermediateAddress.validate_skip(ops, cs, weak) // next_addr:IntermediateAddress + && t_Grams.validate_skip(ops, cs, weak) // fwd_fee_remaining:Grams + && t_Ref_Message.validate_skip(ops, cs, weak); // msg:^Message + case 5: + return cs.fetch_ulong(4) == 5 // msg_envelope_v2#5 + && t_IntermediateAddress.validate_skip(ops, cs, weak) // cur_addr:IntermediateAddress + && t_IntermediateAddress.validate_skip(ops, cs, weak) // next_addr:IntermediateAddress + && t_Grams.validate_skip(ops, cs, weak) // fwd_fee_remaining:Grams + && t_Ref_Message.validate_skip(ops, cs, weak) // msg:^Message + && Maybe(64).validate_skip(ops, cs, weak) // emitted_lt:(Maybe uint64) + && Maybe().validate_skip(ops, cs, weak); // metadata:(Maybe MsgMetadata) + default: + return false; + } } bool MsgEnvelope::skip(vm::CellSlice& cs) const { - return cs.advance(4) // msg_envelope#4 - && t_IntermediateAddress.skip(cs) // cur_addr:IntermediateAddress - && t_IntermediateAddress.skip(cs) // next_addr:IntermediateAddress - && t_Grams.skip(cs) // fwd_fee_remaining:Grams - && t_Ref_Message.skip(cs); // msg:^Message + switch (get_tag(cs)) { + case 4: + return cs.advance(4) // msg_envelope#4 + && t_IntermediateAddress.skip(cs) // cur_addr:IntermediateAddress + && t_IntermediateAddress.skip(cs) // next_addr:IntermediateAddress + && t_Grams.skip(cs) // fwd_fee_remaining:Grams + && t_Ref_Message.skip(cs); // msg:^Message + case 5: + return cs.advance(4) // msg_envelope_v2#5 + && t_IntermediateAddress.skip(cs) // cur_addr:IntermediateAddress + && t_IntermediateAddress.skip(cs) // next_addr:IntermediateAddress + && t_Grams.skip(cs) // fwd_fee_remaining:Grams + && t_Ref_Message.skip(cs) // msg:^Message + && Maybe(64).skip(cs) // emitted_lt:(Maybe uint64) + && Maybe().skip(cs); // metadata:(Maybe MsgMetadata) + default: + return false; + } } bool MsgEnvelope::extract_fwd_fees_remaining(vm::CellSlice& cs) const { @@ -833,34 +859,101 @@ bool MsgEnvelope::extract_fwd_fees_remaining(vm::CellSlice& cs) const { } bool MsgEnvelope::unpack(vm::CellSlice& cs, MsgEnvelope::Record& data) const { - return cs.fetch_ulong(4) == 4 // msg_envelope#4 - && t_IntermediateAddress.fetch_to(cs, data.cur_addr) // cur_addr:IntermediateAddress - && t_IntermediateAddress.fetch_to(cs, data.next_addr) // next_addr:IntermediateAddress - && t_Grams.fetch_to(cs, data.fwd_fee_remaining) // fwd_fee_remaining:Grams - && cs.fetch_ref_to(data.msg); // msg:^Message + switch (get_tag(cs)) { + case 4: + return cs.fetch_ulong(4) == 4 // msg_envelope#4 + && t_IntermediateAddress.fetch_to(cs, data.cur_addr) // cur_addr:IntermediateAddress + && t_IntermediateAddress.fetch_to(cs, data.next_addr) // next_addr:IntermediateAddress + && t_Grams.fetch_to(cs, data.fwd_fee_remaining) // fwd_fee_remaining:Grams + && cs.fetch_ref_to(data.msg); // msg:^Message + case 5: + return cs.fetch_ulong(4) == 5 // msg_envelope_v2#5 + && t_IntermediateAddress.fetch_to(cs, data.cur_addr) // cur_addr:IntermediateAddress + && t_IntermediateAddress.fetch_to(cs, data.next_addr) // next_addr:IntermediateAddress + && t_Grams.fetch_to(cs, data.fwd_fee_remaining) // fwd_fee_remaining:Grams + && cs.fetch_ref_to(data.msg) // msg:^Message + && Maybe(64).skip(cs) // emitted_lt:(Maybe uint64) + && Maybe().skip(cs); // metadata:(Maybe MsgMetadata) + default: + return false; + } } bool MsgEnvelope::unpack(vm::CellSlice& cs, MsgEnvelope::Record_std& data) const { - return cs.fetch_ulong(4) == 4 // msg_envelope#4 - && t_IntermediateAddress.fetch_regular(cs, data.cur_addr) // cur_addr:IntermediateAddress - && t_IntermediateAddress.fetch_regular(cs, data.next_addr) // next_addr:IntermediateAddress - && t_Grams.as_integer_skip_to(cs, data.fwd_fee_remaining) // fwd_fee_remaining:Grams - && cs.fetch_ref_to(data.msg); // msg:^Message + data.emitted_lt = {}; + data.metadata = {}; + switch (get_tag(cs)) { + case 4: + return cs.fetch_ulong(4) == 4 // msg_envelope#4 + && t_IntermediateAddress.fetch_regular(cs, data.cur_addr) // cur_addr:IntermediateAddress + && t_IntermediateAddress.fetch_regular(cs, data.next_addr) // next_addr:IntermediateAddress + && t_Grams.as_integer_skip_to(cs, data.fwd_fee_remaining) // fwd_fee_remaining:Grams + && cs.fetch_ref_to(data.msg); // msg:^Message + case 5: { + bool with_metadata, with_emitted_lt; + return cs.fetch_ulong(4) == 5 // msg_envelope_v2#5 + && t_IntermediateAddress.fetch_regular(cs, data.cur_addr) // cur_addr:IntermediateAddress + && t_IntermediateAddress.fetch_regular(cs, data.next_addr) // next_addr:IntermediateAddress + && t_Grams.as_integer_skip_to(cs, data.fwd_fee_remaining) // fwd_fee_remaining:Grams + && cs.fetch_ref_to(data.msg) // msg:^Message + && cs.fetch_bool_to(with_emitted_lt) && + (!with_emitted_lt || cs.fetch_uint_to(64, data.emitted_lt.value_force())) // emitted_lt:(Maybe uint64) + && cs.fetch_bool_to(with_metadata) && + (!with_metadata || data.metadata.value_force().unpack(cs)); // metadata:(Maybe MsgMetadata) + } + default: + return false; + } +} + +bool MsgEnvelope::pack(vm::CellBuilder& cb, const Record_std& data) const { + bool v2 = (bool)data.metadata || (bool)data.emitted_lt; + if (!(cb.store_long_bool(v2 ? 5 : 4, 4) && // msg_envelope#4 / msg_envelope_v2#5 + cb.store_long_bool(data.cur_addr, 8) && // cur_addr:IntermediateAddress + cb.store_long_bool(data.next_addr, 8) && // next_addr:IntermediateAddress + t_Grams.store_integer_ref(cb, data.fwd_fee_remaining) && // fwd_fee_remaining:Grams + cb.store_ref_bool(data.msg))) { // msg:^Message + return false; + } + if (v2) { + if (!(cb.store_bool_bool((bool)data.emitted_lt) && + (!data.emitted_lt || cb.store_long_bool(data.emitted_lt.value(), 64)))) { // emitted_lt:(Maybe uint64) + return false; + } + if (!(cb.store_bool_bool((bool)data.metadata) && + (!data.metadata || data.metadata.value().pack(cb)))) { // metadata:(Maybe MsgMetadata) + return false; + } + } + return true; } -bool MsgEnvelope::unpack_std(vm::CellSlice& cs, int& cur_a, int& nhop_a, Ref& msg) const { - return cs.fetch_ulong(4) == 4 // msg_envelope#4 - && t_IntermediateAddress.fetch_regular(cs, cur_a) // cur_addr:IntermediateAddress - && t_IntermediateAddress.fetch_regular(cs, nhop_a) // next_addr:IntermediateAddress - && cs.fetch_ref_to(msg); +bool MsgEnvelope::pack_cell(td::Ref& cell, const Record_std& data) const { + vm::CellBuilder cb; + return pack(cb, data) && cb.finalize_to(cell); } -bool MsgEnvelope::get_created_lt(const vm::CellSlice& cs, unsigned long long& created_lt) const { +bool MsgEnvelope::get_emitted_lt(const vm::CellSlice& cs, unsigned long long& emitted_lt) const { + // Emitted lt is emitted_lt from MsgEnvelope (if present), otherwise created_lt if (!cs.size_refs()) { return false; } + if (get_tag(cs) == 5) { + vm::CellSlice cs2 = cs; + // msg_envelope_v2#5 cur_addr:IntermediateAddress + // next_addr:IntermediateAddress fwd_fee_remaining:Grams + // msg:^(Message Any) emitted_lt:(Maybe uint64) ... + bool have_emitted_lt; + if (!(cs2.skip_first(4) && t_IntermediateAddress.skip(cs2) && t_IntermediateAddress.skip(cs2) && + t_Grams.skip(cs2) && t_Ref_Message.skip(cs2) && cs2.fetch_bool_to(have_emitted_lt))) { + return false; + } + if (have_emitted_lt) { + return cs2.fetch_ulong_bool(64, emitted_lt); + } + } auto msg_cs = load_cell_slice(cs.prefetch_ref()); - return t_Message.get_created_lt(msg_cs, created_lt); + return t_Message.get_created_lt(msg_cs, emitted_lt); } const MsgEnvelope t_MsgEnvelope; @@ -1692,6 +1785,15 @@ bool InMsg::skip(vm::CellSlice& cs) const { && cs.advance(64) // transaction_id:uint64 && t_Grams.skip(cs) // fwd_fee:Grams && t_RefCell.skip(cs); // proof_delivered:^Cell + case msg_import_deferred_fin: + return cs.advance(5) // msg_import_deferred_fin$00100 + && t_Ref_MsgEnvelope.skip(cs) // in_msg:^MsgEnvelope + && t_Ref_Transaction.skip(cs) // transaction:^Transaction + && t_Grams.skip(cs); // fwd_fee:Grams + case msg_import_deferred_tr: + return cs.advance(5) // msg_import_deferred_tr$00101 + && t_Ref_MsgEnvelope.skip(cs) // in_msg:^MsgEnvelope + && t_Ref_MsgEnvelope.skip(cs); // out_msg:^MsgEnvelope } return false; } @@ -1734,12 +1836,22 @@ bool InMsg::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { && cs.advance(64) // transaction_id:uint64 && t_Grams.validate_skip(ops, cs, weak) // fwd_fee:Grams && t_RefCell.validate_skip(ops, cs, weak); // proof_delivered:^Cell + case msg_import_deferred_fin: + return cs.advance(5) // msg_import_deferred_fin$00100 + && t_Ref_MsgEnvelope.validate_skip(ops, cs, weak) // in_msg:^MsgEnvelope + && t_Ref_Transaction.validate_skip(ops, cs, weak) // transaction:^Transaction + && t_Grams.validate_skip(ops, cs, weak); // fwd_fee:Grams + case msg_import_deferred_tr: + return cs.advance(5) // msg_import_deferred_tr$00101 + && t_Ref_MsgEnvelope.validate_skip(ops, cs, weak) // in_msg:^MsgEnvelope + && t_Ref_MsgEnvelope.validate_skip(ops, cs, weak); // out_msg:^MsgEnvelope } return false; } bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const { - switch (get_tag(cs)) { + int tag = get_tag(cs); + switch (tag) { case msg_import_ext: // inbound external message return t_ImportFees.null_value(cb); // external messages have no value and no import fees case msg_import_ihr: // IHR-forwarded internal message to its final destination @@ -1765,8 +1877,9 @@ bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const { && t_CurrencyCollection.null_value(cb); // value_imported := 0 } return false; - case msg_import_fin: // internal message delivered to its final destination in this block - if (cs.advance(3) && cs.size_refs() >= 2) { + case msg_import_fin: // internal message delivered to its final destination in this block + case msg_import_deferred_fin: // internal message from DispatchQueue to its final destination in this block + if (cs.advance(tag == msg_import_fin ? 3 : 5) && cs.size_refs() >= 2) { auto msg_env_cs = load_cell_slice(cs.fetch_ref()); MsgEnvelope::Record in_msg; td::RefInt256 fwd_fee, fwd_fee_remaining, value_grams, ihr_fee; @@ -1787,13 +1900,14 @@ bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const { msg_info.value.write()); // value_imported = msg.value + msg.ihr_fee + fwd_fee_remaining } return false; - case msg_import_tr: // transit internal message - if (cs.advance(3) && cs.size_refs() >= 2) { + case msg_import_tr: // transit internal message + case msg_import_deferred_tr: // internal message from DispatchQueue to OutMsgQueue + if (cs.advance(tag == msg_import_tr ? 3 : 5) && cs.size_refs() >= 2) { auto msg_env_cs = load_cell_slice(cs.fetch_ref()); MsgEnvelope::Record in_msg; - td::RefInt256 transit_fee, fwd_fee_remaining, value_grams, ihr_fee; + td::RefInt256 transit_fee = td::zero_refint(), fwd_fee_remaining, value_grams, ihr_fee; if (!(t_MsgEnvelope.unpack(msg_env_cs, in_msg) && cs.fetch_ref().not_null() && - t_Grams.as_integer_skip_to(cs, transit_fee) && + (tag == msg_import_deferred_tr || t_Grams.as_integer_skip_to(cs, transit_fee)) && (fwd_fee_remaining = t_Grams.as_integer(in_msg.fwd_fee_remaining)).not_null() && cmp(transit_fee, fwd_fee_remaining) <= 0)) { return false; @@ -1871,6 +1985,14 @@ bool OutMsg::skip(vm::CellSlice& cs) const { return cs.advance(3) // msg_export_tr_req$111 && t_Ref_MsgEnvelope.skip(cs) // out_msg:^MsgEnvelope && RefTo{}.skip(cs); // imported:^InMsg + case msg_export_new_defer: + return cs.advance(5) // msg_export_new_defer$10100 + && t_Ref_MsgEnvelope.skip(cs) // out_msg:^MsgEnvelope + && t_Ref_Transaction.skip(cs); // transaction:^Transaction + case msg_export_deferred_tr: + return cs.advance(5) // msg_export_deferred_tr$10101 + && t_Ref_MsgEnvelope.skip(cs) // out_msg:^MsgEnvelope + && RefTo{}.skip(cs); // imported:^InMsg } return false; } @@ -1910,12 +2032,21 @@ bool OutMsg::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { return cs.advance(3) // msg_export_tr_req$111 && t_Ref_MsgEnvelope.validate_skip(ops, cs, weak) // out_msg:^MsgEnvelope && RefTo{}.validate_skip(ops, cs, weak); // imported:^InMsg + case msg_export_new_defer: + return cs.advance(5) // msg_export_new_defer$10100 + && t_Ref_MsgEnvelope.validate_skip(ops, cs, weak) // out_msg:^MsgEnvelope + && t_Ref_Transaction.validate_skip(ops, cs, weak); // transaction:^Transaction + case msg_export_deferred_tr: + return cs.advance(5) // msg_export_deferred_tr$10101 + && t_Ref_MsgEnvelope.validate_skip(ops, cs, weak) // out_msg:^MsgEnvelope + && RefTo{}.validate_skip(ops, cs, weak); // imported:^InMsg } return false; } bool OutMsg::get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs) const { - switch (get_tag(cs)) { + auto tag = get_tag(cs); + switch (tag) { case msg_export_ext: // external outbound message carries no value if (cs.have(3, 2)) { return t_CurrencyCollection.null_value(cb); @@ -1929,10 +2060,13 @@ bool OutMsg::get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs) const { return cs.have(4 + 63, 1) && t_CurrencyCollection.null_value(cb); case msg_export_deq_short: // dequeueing record for outbound message, no exported value return cs.have(4 + 256 + 32 + 64 + 64) && t_CurrencyCollection.null_value(cb); - case msg_export_new: // newly-generated outbound internal message, queued - case msg_export_tr: // transit internal message, queued - case msg_export_tr_req: // transit internal message, re-queued from this shardchain - if (cs.advance(3) && cs.size_refs() >= 2) { + case msg_export_new: // newly-generated outbound internal message, queued + case msg_export_tr: // transit internal message, queued + case msg_export_tr_req: // transit internal message, re-queued from this shardchain + case msg_export_new_defer: // newly-generated outbound internal message, deferred + case msg_export_deferred_tr: // internal message from DispatchQueue, queued + int tag_len = (tag == msg_export_new_defer || tag == msg_export_deferred_tr) ? 5 : 3; + if (cs.advance(tag_len) && cs.size_refs() >= 2) { auto msg_env_cs = load_cell_slice(cs.fetch_ref()); MsgEnvelope::Record out_msg; if (!(cs.fetch_ref().not_null() && t_MsgEnvelope.unpack(msg_env_cs, out_msg))) { @@ -1954,12 +2088,12 @@ bool OutMsg::get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs) const { return false; } -bool OutMsg::get_created_lt(vm::CellSlice& cs, unsigned long long& created_lt) const { +bool OutMsg::get_emitted_lt(vm::CellSlice& cs, unsigned long long& emitted_lt) const { switch (get_tag(cs)) { case msg_export_ext: if (cs.have(3, 1)) { auto msg_cs = load_cell_slice(cs.prefetch_ref()); - return t_Message.get_created_lt(msg_cs, created_lt); + return t_Message.get_created_lt(msg_cs, emitted_lt); } else { return false; } @@ -1970,9 +2104,11 @@ bool OutMsg::get_created_lt(vm::CellSlice& cs, unsigned long long& created_lt) c case msg_export_deq_short: case msg_export_deq_imm: case msg_export_tr_req: + case msg_export_new_defer: + case msg_export_deferred_tr: if (cs.have(3, 1)) { auto out_msg_cs = load_cell_slice(cs.prefetch_ref()); - return t_MsgEnvelope.get_created_lt(out_msg_cs, created_lt); + return t_MsgEnvelope.get_emitted_lt(out_msg_cs, emitted_lt); } else { return false; } @@ -2003,26 +2139,53 @@ bool Aug_OutMsgQueue::eval_empty(vm::CellBuilder& cb) const { bool Aug_OutMsgQueue::eval_leaf(vm::CellBuilder& cb, vm::CellSlice& cs) const { Ref msg_env; - unsigned long long created_lt; - return cs.fetch_ref_to(msg_env) && t_MsgEnvelope.get_created_lt(load_cell_slice(std::move(msg_env)), created_lt) && - cb.store_ulong_rchk_bool(created_lt, 64); + unsigned long long emitted_lt; + return cs.fetch_ref_to(msg_env) && t_MsgEnvelope.get_emitted_lt(load_cell_slice(std::move(msg_env)), emitted_lt) && + cb.store_ulong_rchk_bool(emitted_lt, 64); +} + +bool Aug_DispatchQueue::eval_fork(vm::CellBuilder& cb, vm::CellSlice& left_cs, vm::CellSlice& right_cs) const { + unsigned long long x, y; + return left_cs.fetch_ulong_bool(64, x) && right_cs.fetch_ulong_bool(64, y) && + cb.store_ulong_rchk_bool(std::min(x, y), 64); +} + +bool Aug_DispatchQueue::eval_empty(vm::CellBuilder& cb) const { + return cb.store_long_bool(0, 64); +} + +bool Aug_DispatchQueue::eval_leaf(vm::CellBuilder& cb, vm::CellSlice& cs) const { + Ref messages_root; + if (!cs.fetch_maybe_ref(messages_root)) { + return false; + } + vm::Dictionary messages{std::move(messages_root), 64}; + td::BitArray<64> key_buffer; + td::uint64 key; + if (messages.get_minmax_key(key_buffer.bits(), 64).is_null()) { + key = (td::uint64)-1; + } else { + key = key_buffer.to_ulong(); + } + return cb.store_long_bool(key, 64); } const Aug_OutMsgQueue aug_OutMsgQueue; +const Aug_DispatchQueue aug_DispatchQueue; const OutMsgQueue t_OutMsgQueue; const ProcessedUpto t_ProcessedUpto; const HashmapE t_ProcessedInfo{96, t_ProcessedUpto}; const HashmapE t_IhrPendingInfo{256, t_uint128}; -// _ out_queue:OutMsgQueue proc_info:ProcessedInfo = OutMsgQueueInfo; +// _ out_queue:OutMsgQueue proc_info:ProcessedInfo extra:(Maybe OutMsgQueueExtra) = OutMsgQueueInfo; bool OutMsgQueueInfo::skip(vm::CellSlice& cs) const { - return t_OutMsgQueue.skip(cs) && t_ProcessedInfo.skip(cs) && t_IhrPendingInfo.skip(cs); + return t_OutMsgQueue.skip(cs) && t_ProcessedInfo.skip(cs) && Maybe().skip(cs); } bool OutMsgQueueInfo::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { return t_OutMsgQueue.validate_skip(ops, cs, weak) && t_ProcessedInfo.validate_skip(ops, cs, weak) && - t_IhrPendingInfo.validate_skip(ops, cs, weak); + Maybe().validate_skip(ops, cs, weak); } const OutMsgQueueInfo t_OutMsgQueueInfo; diff --git a/crypto/block/block-parse.h b/crypto/block/block-parse.h index c0b117452..65f8b91fe 100644 --- a/crypto/block/block-parse.h +++ b/crypto/block/block-parse.h @@ -28,6 +28,7 @@ #include "td/utils/bits.h" #include "td/utils/StringBuilder.h" #include "ton/ton-types.h" +#include "block-auto.h" namespace block { @@ -469,11 +470,17 @@ struct MsgEnvelope final : TLB_Complex { int cur_addr, next_addr; td::RefInt256 fwd_fee_remaining; Ref msg; + td::optional emitted_lt; + td::optional metadata; }; bool unpack(vm::CellSlice& cs, Record& data) const; bool unpack(vm::CellSlice& cs, Record_std& data) const; - bool unpack_std(vm::CellSlice& cs, int& cur_a, int& nhop_a, Ref& msg) const; - bool get_created_lt(const vm::CellSlice& cs, unsigned long long& created_lt) const; + bool pack(vm::CellBuilder& cb, const Record_std& data) const; + bool pack_cell(td::Ref& cell, const Record_std& data) const; + bool get_emitted_lt(const vm::CellSlice& cs, unsigned long long& emitted_lt) const; + int get_tag(const vm::CellSlice& cs) const override { + return (int)cs.prefetch_ulong(4); + } }; extern const MsgEnvelope t_MsgEnvelope; @@ -801,12 +808,18 @@ struct InMsg final : TLB_Complex { msg_import_fin = 4, msg_import_tr = 5, msg_discard_fin = 6, - msg_discard_tr = 7 + msg_discard_tr = 7, + msg_import_deferred_fin = 8, + msg_import_deferred_tr = 9 }; bool skip(vm::CellSlice& cs) const override; bool validate_skip(int* ops, vm::CellSlice& cs, bool weak = false) const override; int get_tag(const vm::CellSlice& cs) const override { - return (int)cs.prefetch_ulong(3); + int tag = (int)cs.prefetch_ulong(3); + if (tag != 1) { + return tag; + } + return (int)cs.prefetch_ulong(5) - 0b00100 + 8; } bool get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const; }; @@ -822,16 +835,24 @@ struct OutMsg final : TLB_Complex { msg_export_deq_imm = 4, msg_export_deq = 12, msg_export_deq_short = 13, - msg_export_tr_req = 7 + msg_export_tr_req = 7, + msg_export_new_defer = 20, // 0b10100 + msg_export_deferred_tr = 21 // 0b10101 }; bool skip(vm::CellSlice& cs) const override; bool validate_skip(int* ops, vm::CellSlice& cs, bool weak = false) const override; int get_tag(const vm::CellSlice& cs) const override { int t = (int)cs.prefetch_ulong(3); - return t != 6 ? t : (int)cs.prefetch_ulong(4); + if (t == 6) { + return (int)cs.prefetch_ulong(4); + } + if (t == 5) { + return (int)cs.prefetch_ulong(5); + } + return t; } bool get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs) const; - bool get_created_lt(vm::CellSlice& cs, unsigned long long& created_lt) const; + bool get_emitted_lt(vm::CellSlice& cs, unsigned long long& emitted_lt) const; }; extern const OutMsg t_OutMsg; @@ -909,6 +930,16 @@ struct Aug_OutMsgQueue final : AugmentationCheckData { extern const Aug_OutMsgQueue aug_OutMsgQueue; +struct Aug_DispatchQueue final : AugmentationCheckData { + Aug_DispatchQueue() : AugmentationCheckData(gen::t_AccountDispatchQueue, t_uint64) { + } + bool eval_fork(vm::CellBuilder& cb, vm::CellSlice& left_cs, vm::CellSlice& right_cs) const override; + bool eval_empty(vm::CellBuilder& cb) const override; + bool eval_leaf(vm::CellBuilder& cb, vm::CellSlice& cs) const override; +}; + +extern const Aug_DispatchQueue aug_DispatchQueue; + struct OutMsgQueue final : TLB_Complex { HashmapAugE dict_type; OutMsgQueue() : dict_type(32 + 64 + 256, aug_OutMsgQueue){}; diff --git a/crypto/block/block.cpp b/crypto/block/block.cpp index a4086ea8d..0132e957a 100644 --- a/crypto/block/block.cpp +++ b/crypto/block/block.cpp @@ -27,6 +27,7 @@ #include "td/utils/tl_storers.h" #include "td/utils/misc.h" #include "td/utils/Random.h" +#include "vm/fmt.hpp" namespace block { using namespace std::literals::string_literals; @@ -641,7 +642,11 @@ bool EnqueuedMsgDescr::unpack(vm::CellSlice& cs) { } cur_prefix_ = interpolate_addr(src_prefix_, dest_prefix_, env.cur_addr); next_prefix_ = interpolate_addr(src_prefix_, dest_prefix_, env.next_addr); - lt_ = info.created_lt; + unsigned long long lt; + if (!tlb::t_MsgEnvelope.get_emitted_lt(vm::load_cell_slice(enq.out_msg), lt)) { + return invalidate(); + } + lt_ = lt; enqueued_lt_ = enq.enqueued_lt; hash_ = env.msg->get_hash().bits(); msg_ = std::move(env.msg); @@ -857,12 +862,20 @@ td::Status ShardState::unpack_out_msg_queue_info(Ref out_msg_queue_inf return td::Status::Error( -666, "ProcessedInfo in the state of "s + id_.to_str() + " is invalid according to automated validity checks"); } - if (!block::gen::t_IhrPendingInfo.validate_csr(1024, qinfo.ihr_pending)) { - return td::Status::Error( - -666, "IhrPendingInfo in the state of "s + id_.to_str() + " is invalid according to automated validity checks"); - } processed_upto_ = block::MsgProcessedUptoCollection::unpack(ton::ShardIdFull(id_), std::move(qinfo.proc_info)); - ihr_pending_ = std::make_unique(std::move(qinfo.ihr_pending), 320); + ihr_pending_ = std::make_unique(320); + if (qinfo.extra.write().fetch_long(1)) { + block::gen::OutMsgQueueExtra::Record extra; + if (!block::tlb::csr_unpack(qinfo.extra, extra)) { + return td::Status::Error(-666, "cannot unpack OutMsgQueueExtre in the state of "s + id_.to_str()); + } + dispatch_queue_ = std::make_unique(extra.dispatch_queue, 256, tlb::aug_DispatchQueue); + if (extra.out_queue_size.write().fetch_long(1)) { + out_msg_queue_size_ = extra.out_queue_size->prefetch_ulong(48); + } + } else { + dispatch_queue_ = std::make_unique(256, tlb::aug_DispatchQueue); + } auto shard1 = id_.shard_full(); td::BitArray<64> pfx{(long long)shard1.shard}; int pfx_len = shard_prefix_length(shard1); @@ -993,6 +1006,17 @@ td::Status ShardState::merge_with(ShardState& sib) { underload_history_ = overload_history_ = 0; // 10. compute vert_seqno vert_seqno_ = std::max(vert_seqno_, sib.vert_seqno_); + // 11. merge dispatch_queue (same as account dict) + if (!dispatch_queue_->combine_with(*sib.dispatch_queue_)) { + return td::Status::Error(-666, "cannot merge dispatch queues of the two ancestors"); + } + sib.dispatch_queue_.reset(); + // 11. merge out_msg_queue_size + if (out_msg_queue_size_ && sib.out_msg_queue_size_) { + out_msg_queue_size_.value() += sib.out_msg_queue_size_.value(); + } else { + out_msg_queue_size_ = {}; + } // Anything else? add here // ... @@ -1008,8 +1032,8 @@ td::Status ShardState::merge_with(ShardState& sib) { return td::Status::OK(); } -td::Result> ShardState::compute_split_out_msg_queue(ton::ShardIdFull subshard, - td::uint32* queue_size) { +td::Result> ShardState::compute_split_out_msg_queue( + ton::ShardIdFull subshard) { auto shard = id_.shard_full(); if (!ton::shard_is_parent(shard, subshard)) { return td::Status::Error(-666, "cannot split subshard "s + subshard.to_str() + " from state of " + id_.to_str() + @@ -1017,7 +1041,7 @@ td::Result> ShardState::compute_split_o } CHECK(out_msg_queue_); auto subqueue = std::make_unique(*out_msg_queue_); - int res = block::filter_out_msg_queue(*subqueue, shard, subshard, queue_size); + int res = block::filter_out_msg_queue(*subqueue, shard, subshard); if (res < 0) { return td::Status::Error(-666, "error splitting OutMsgQueue of "s + id_.to_str()); } @@ -1039,7 +1063,7 @@ td::Result> ShardState::compu return std::move(sub_processed_upto); } -td::Status ShardState::split(ton::ShardIdFull subshard, td::uint32* queue_size) { +td::Status ShardState::split(ton::ShardIdFull subshard) { if (!ton::shard_is_parent(id_.shard_full(), subshard)) { return td::Status::Error(-666, "cannot split subshard "s + subshard.to_str() + " from state of " + id_.to_str() + " because it is not a parent"); @@ -1057,10 +1081,12 @@ td::Status ShardState::split(ton::ShardIdFull subshard, td::uint32* queue_size) auto shard1 = id_.shard_full(); CHECK(ton::shard_is_parent(shard1, subshard)); CHECK(out_msg_queue_); - int res1 = block::filter_out_msg_queue(*out_msg_queue_, shard1, subshard, queue_size); + td::uint64 queue_size; + int res1 = block::filter_out_msg_queue(*out_msg_queue_, shard1, subshard, &queue_size); if (res1 < 0) { return td::Status::Error(-666, "error splitting OutMsgQueue of "s + id_.to_str()); } + out_msg_queue_size_ = queue_size; LOG(DEBUG) << "split counters: " << res1; // 3. processed_upto LOG(DEBUG) << "splitting ProcessedUpto"; @@ -1090,6 +1116,11 @@ td::Status ShardState::split(ton::ShardIdFull subshard, td::uint32* queue_size) // NB: if total_fees_extra will be allowed to be non-empty, split it here too // 7. reset overload/underload history overload_history_ = underload_history_ = 0; + // 8. split dispatch_queue (same as account dict) + LOG(DEBUG) << "splitting dispatch_queue"; + CHECK(dispatch_queue_); + CHECK(dispatch_queue_->cut_prefix_subdict(pfx.bits(), pfx_len)); + CHECK(dispatch_queue_->has_common_prefix(pfx.bits(), pfx_len)); // 999. anything else? id_.id.shard = subshard.shard; id_.file_hash.set_zero(); @@ -1098,7 +1129,7 @@ td::Status ShardState::split(ton::ShardIdFull subshard, td::uint32* queue_size) } int filter_out_msg_queue(vm::AugmentedDictionary& out_queue, ton::ShardIdFull old_shard, ton::ShardIdFull subshard, - td::uint32* queue_size) { + td::uint64* queue_size) { if (queue_size) { *queue_size = 0; } @@ -1389,7 +1420,7 @@ bool ValueFlow::store(vm::CellBuilder& cb) const { && exported.store(cb2) // exported:CurrencyCollection && cb.store_ref_bool(cb2.finalize()) // ] && fees_collected.store(cb) // fees_collected:CurrencyCollection - && (burned.is_zero() || burned.store(cb)) // fees_burned:CurrencyCollection + && (burned.is_zero() || burned.store(cb)) // fees_burned:CurrencyCollection && fees_imported.store(cb2) // ^[ fees_imported:CurrencyCollection && recovered.store(cb2) // recovered:CurrencyCollection && created.store(cb2) // created:CurrencyCollection @@ -1418,8 +1449,7 @@ bool ValueFlow::fetch(vm::CellSlice& cs) { from_prev_blk.validate_unpack(std::move(f2.r1.from_prev_blk)) && to_next_blk.validate_unpack(std::move(f2.r1.to_next_blk)) && imported.validate_unpack(std::move(f2.r1.imported)) && exported.validate_unpack(std::move(f2.r1.exported)) && - fees_collected.validate_unpack(std::move(f2.fees_collected)) && - burned.validate_unpack(std::move(f2.burned)) && + fees_collected.validate_unpack(std::move(f2.fees_collected)) && burned.validate_unpack(std::move(f2.burned)) && fees_imported.validate_unpack(std::move(f2.r2.fees_imported)) && recovered.validate_unpack(std::move(f2.r2.recovered)) && created.validate_unpack(std::move(f2.r2.created)) && minted.validate_unpack(std::move(f2.r2.minted))) { @@ -2237,4 +2267,132 @@ bool parse_block_id_ext(td::Slice str, ton::BlockIdExt& blkid) { return parse_block_id_ext(str.begin(), str.end(), blkid); } +bool unpack_account_dispatch_queue(Ref csr, vm::Dictionary& dict, td::uint64& dict_size) { + if (csr.not_null()) { + block::gen::AccountDispatchQueue::Record rec; + if (!block::tlb::csr_unpack(std::move(csr), rec)) { + return false; + } + dict = vm::Dictionary{rec.messages, 64}; + dict_size = rec.count; + if (dict_size == 0 || dict.is_empty()) { + return false; + } + } else { + dict = vm::Dictionary{64}; + dict_size = 0; + } + return true; +} + +Ref pack_account_dispatch_queue(const vm::Dictionary& dict, td::uint64 dict_size) { + if (dict_size == 0) { + return {}; + } + // _ messages:(HashmapE 64 EnqueuedMsg) count:uint48 = AccountDispatchQueue; + vm::CellBuilder cb; + CHECK(dict.append_dict_to_bool(cb)); + cb.store_long(dict_size, 48); + return cb.as_cellslice_ref(); +} + +Ref get_dispatch_queue_min_lt_account(const vm::AugmentedDictionary& dispatch_queue, + ton::StdSmcAddress& addr) { + // TODO: This can be done more effectively + vm::AugmentedDictionary queue{dispatch_queue.get_root(), 256, tlb::aug_DispatchQueue}; + if (queue.is_empty()) { + return {}; + } + auto root_extra = queue.get_root_extra(); + if (root_extra.is_null()) { + return {}; + } + ton::LogicalTime min_lt = root_extra->prefetch_long(64); + while (true) { + td::Bits256 key; + int pfx_len = queue.get_common_prefix(key.bits(), 256); + if (pfx_len < 0) { + return {}; + } + if (pfx_len == 256) { + addr = key; + return queue.lookup(key); + } + key[pfx_len] = false; + vm::AugmentedDictionary queue_cut{queue.get_root(), 256, tlb::aug_DispatchQueue}; + if (!queue_cut.cut_prefix_subdict(key.bits(), pfx_len + 1)) { + return {}; + } + root_extra = queue_cut.get_root_extra(); + if (root_extra.is_null()) { + return {}; + } + ton::LogicalTime cut_min_lt = root_extra->prefetch_long(64); + if (cut_min_lt != min_lt) { + key[pfx_len] = true; + } + if (!queue.cut_prefix_subdict(key.bits(), pfx_len + 1)) { + return {}; + } + } +} + +bool remove_dispatch_queue_entry(vm::AugmentedDictionary& dispatch_queue, const ton::StdSmcAddress& addr, + ton::LogicalTime lt) { + auto account_dispatch_queue = dispatch_queue.lookup(addr); + if (account_dispatch_queue.is_null()) { + return false; + } + vm::Dictionary dict{64}; + td::uint64 dict_size; + if (!unpack_account_dispatch_queue(std::move(account_dispatch_queue), dict, dict_size)) { + return false; + } + td::BitArray<64> key; + key.store_ulong(lt); + auto entry = dict.lookup_delete(key); + if (entry.is_null()) { + return false; + } + --dict_size; + account_dispatch_queue = pack_account_dispatch_queue(dict, dict_size); + if (account_dispatch_queue.not_null()) { + dispatch_queue.set(addr, account_dispatch_queue); + } else { + dispatch_queue.lookup_delete(addr); + } + return true; +} + +bool MsgMetadata::unpack(vm::CellSlice& cs) { + // msg_metadata#0 depth:uint32 initiator_addr:MsgAddressInt initiator_lt:uint64 = MsgMetadata; + int tag; + return cs.fetch_int_to(4, tag) && tag == 0 && cs.fetch_uint_to(32, depth) && + cs.prefetch_ulong(3) == 0b100 && // std address, no anycast + tlb::t_MsgAddressInt.extract_std_address(cs, initiator_wc, initiator_addr) && + cs.fetch_uint_to(64, initiator_lt); +} + +bool MsgMetadata::pack(vm::CellBuilder& cb) const { + // msg_metadata#0 depth:uint32 initiator_addr:MsgAddressInt initiator_lt:uint64 = MsgMetadata; + return cb.store_long_bool(0, 4) && cb.store_long_bool(depth, 32) && + tlb::t_MsgAddressInt.store_std_address(cb, initiator_wc, initiator_addr) && + cb.store_long_bool(initiator_lt, 64); +} + +std::string MsgMetadata::to_str() const { + return PSTRING() << "[ depth=" << depth << " init=" << initiator_wc << ":" << initiator_addr.to_hex() << ":" + << initiator_lt << " ]"; +} + +bool MsgMetadata::operator==(const MsgMetadata& other) const { + return depth == other.depth && initiator_wc == other.initiator_wc && initiator_addr == other.initiator_addr && + initiator_lt == other.initiator_lt; +} + +bool MsgMetadata::operator!=(const MsgMetadata& other) const { + return !(*this == other); +} + + } // namespace block diff --git a/crypto/block/block.h b/crypto/block/block.h index f722542ae..d6abfedb4 100644 --- a/crypto/block/block.h +++ b/crypto/block/block.h @@ -417,6 +417,8 @@ struct ShardState { std::unique_ptr ihr_pending_; std::unique_ptr block_create_stats_; std::shared_ptr processed_upto_; + std::unique_ptr dispatch_queue_; + td::optional out_msg_queue_size_; bool is_valid() const { return id_.is_valid(); @@ -433,11 +435,10 @@ struct ShardState { ton::BlockSeqno prev_mc_block_seqno, bool after_split, bool clear_history, std::function for_each_mcseqno); td::Status merge_with(ShardState& sib); - td::Result> compute_split_out_msg_queue(ton::ShardIdFull subshard, - td::uint32* queue_size = nullptr); + td::Result> compute_split_out_msg_queue(ton::ShardIdFull subshard); td::Result> compute_split_processed_upto( ton::ShardIdFull subshard); - td::Status split(ton::ShardIdFull subshard, td::uint32* queue_size = nullptr); + td::Status split(ton::ShardIdFull subshard); td::Status unpack_out_msg_queue_info(Ref out_msg_queue_info); bool clear_load_history() { overload_history_ = underload_history_ = 0; @@ -658,7 +659,7 @@ class MtCarloComputeShare { }; int filter_out_msg_queue(vm::AugmentedDictionary& out_queue, ton::ShardIdFull old_shard, ton::ShardIdFull subshard, - td::uint32* queue_size = nullptr); + td::uint64* queue_size = nullptr); std::ostream& operator<<(std::ostream& os, const ShardId& shard_id); @@ -745,4 +746,25 @@ bool parse_hex_hash(td::Slice str, td::Bits256& hash); bool parse_block_id_ext(const char* str, const char* end, ton::BlockIdExt& blkid); bool parse_block_id_ext(td::Slice str, ton::BlockIdExt& blkid); +bool unpack_account_dispatch_queue(Ref csr, vm::Dictionary& dict, td::uint64& dict_size); +Ref pack_account_dispatch_queue(const vm::Dictionary& dict, td::uint64 dict_size); +Ref get_dispatch_queue_min_lt_account(const vm::AugmentedDictionary& dispatch_queue, + ton::StdSmcAddress& addr); +bool remove_dispatch_queue_entry(vm::AugmentedDictionary& dispatch_queue, const ton::StdSmcAddress& addr, + ton::LogicalTime lt); + +struct MsgMetadata { + td::uint32 depth; + ton::WorkchainId initiator_wc; + ton::StdSmcAddress initiator_addr; + ton::LogicalTime initiator_lt; + + bool unpack(vm::CellSlice& cs); + bool pack(vm::CellBuilder& cb) const; + std::string to_str() const; + + bool operator==(const MsgMetadata& other) const; + bool operator!=(const MsgMetadata& other) const; +}; + } // namespace block diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 3ae542399..a3684f563 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -172,6 +172,12 @@ interm_addr_ext$11 workchain_id:int32 addr_pfx:uint64 msg_envelope#4 cur_addr:IntermediateAddress next_addr:IntermediateAddress fwd_fee_remaining:Grams msg:^(Message Any) = MsgEnvelope; +msg_metadata#0 depth:uint32 initiator_addr:MsgAddressInt initiator_lt:uint64 = MsgMetadata; +msg_envelope_v2#5 cur_addr:IntermediateAddress + next_addr:IntermediateAddress fwd_fee_remaining:Grams + msg:^(Message Any) + emitted_lt:(Maybe uint64) + metadata:(Maybe MsgMetadata) = MsgEnvelope; // msg_import_ext$000 msg:^(Message Any) transaction:^Transaction = InMsg; @@ -187,6 +193,9 @@ msg_discard_fin$110 in_msg:^MsgEnvelope transaction_id:uint64 fwd_fee:Grams = InMsg; msg_discard_tr$111 in_msg:^MsgEnvelope transaction_id:uint64 fwd_fee:Grams proof_delivered:^Cell = InMsg; +msg_import_deferred_fin$00100 in_msg:^MsgEnvelope + transaction:^Transaction fwd_fee:Grams = InMsg; +msg_import_deferred_tr$00101 in_msg:^MsgEnvelope out_msg:^MsgEnvelope = InMsg; // import_fees$_ fees_collected:Grams value_imported:CurrencyCollection = ImportFees; @@ -210,6 +219,10 @@ msg_export_tr_req$111 out_msg:^MsgEnvelope imported:^InMsg = OutMsg; msg_export_deq_imm$100 out_msg:^MsgEnvelope reimport:^InMsg = OutMsg; +msg_export_new_defer$10100 out_msg:^MsgEnvelope + transaction:^Transaction = OutMsg; +msg_export_deferred_tr$10101 out_msg:^MsgEnvelope + imported:^InMsg = OutMsg; _ enqueued_lt:uint64 out_msg:^MsgEnvelope = EnqueuedMsg; @@ -224,8 +237,15 @@ _ (HashmapE 96 ProcessedUpto) = ProcessedInfo; ihr_pending$_ import_lt:uint64 = IhrPendingSince; _ (HashmapE 320 IhrPendingSince) = IhrPendingInfo; +// key - created_lt +_ messages:(HashmapE 64 EnqueuedMsg) count:uint48 = AccountDispatchQueue; +// key - sender address, aug - min created_lt +_ (HashmapAugE 256 AccountDispatchQueue uint64) = DispatchQueue; + +out_msg_queue_extra#0 dispatch_queue:DispatchQueue out_queue_size:(Maybe uint48) = OutMsgQueueExtra; + _ out_queue:OutMsgQueue proc_info:ProcessedInfo - ihr_pending:IhrPendingInfo = OutMsgQueueInfo; + extra:(Maybe OutMsgQueueExtra) = OutMsgQueueInfo; // storage_used$_ cells:(VarUInteger 7) bits:(VarUInteger 7) public_cells:(VarUInteger 7) = StorageUsed; @@ -781,7 +801,7 @@ size_limits_config#01 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells max_ext_msg_size:uint32 max_ext_msg_depth:uint16 = SizeLimitsConfig; size_limits_config_v2#02 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16 max_ext_msg_size:uint32 max_ext_msg_depth:uint16 max_acc_state_cells:uint32 max_acc_state_bits:uint32 - max_acc_public_libraries:uint32 = SizeLimitsConfig; + max_acc_public_libraries:uint32 defer_out_queue_size_limit:uint32 = SizeLimitsConfig; _ SizeLimitsConfig = ConfigParam 43; // key is [ wc:int32 addr:uint256 ] diff --git a/crypto/block/create-state.cpp b/crypto/block/create-state.cpp index 183da0a73..348377e94 100644 --- a/crypto/block/create-state.cpp +++ b/crypto/block/create-state.cpp @@ -814,11 +814,16 @@ void usage(const char* progname) { void parse_include_path_set(std::string include_path_set, std::vector& res) { td::Parser parser(include_path_set); while (!parser.empty()) { - auto path = parser.read_till_nofail(':'); + #if TD_WINDOWS + auto path_separator = '@'; + #else + auto path_separator = ':'; + #endif + auto path = parser.read_till_nofail(path_separator); if (!path.empty()) { res.push_back(path.str()); } - parser.skip_nofail(':'); + parser.skip_nofail(path_separator); } } diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 2a32ea854..49f711f7c 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -1956,6 +1956,7 @@ td::Result Config::do_get_size_limits_config(td::Ref& rand_seed, const ComputeP // if the smart contract wants to randomize further, it can use RANDOMIZE instruction td::BitArray<256 + 256> data; data.bits().copy_from(cfg.block_rand_seed.cbits(), 256); - (data.bits() + 256).copy_from(account.addr_rewrite.cbits(), 256); + if (cfg.global_version >= 8) { + (data.bits() + 256).copy_from(account.addr.cbits(), 256); + } else { + (data.bits() + 256).copy_from(account.addr_rewrite.cbits(), 256); + } rand_seed.clear(); data.compute_sha256(rand_seed); return true; @@ -1600,12 +1604,22 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { cp.skip_reason = in_msg_state.not_null() ? ComputePhase::sk_bad_state : ComputePhase::sk_no_state; return true; } else if (in_msg_state.not_null()) { + if (cfg.allow_external_unfreeze) { + if (in_msg_extern && account.addr != in_msg_state->get_hash().bits()) { + // only for external messages with non-zero initstate in active accounts + LOG(DEBUG) << "in_msg_state hash mismatch in external message"; + cp.skip_reason = ComputePhase::sk_bad_state; + return true; + } + } unpack_msg_state(cfg, true); // use only libraries } - if (in_msg_extern && in_msg_state.not_null() && account.addr != in_msg_state->get_hash().bits()) { - LOG(DEBUG) << "in_msg_state hash mismatch in external message"; - cp.skip_reason = ComputePhase::sk_bad_state; - return true; + if (!cfg.allow_external_unfreeze) { + if (in_msg_extern && in_msg_state.not_null() && account.addr != in_msg_state->get_hash().bits()) { + LOG(DEBUG) << "in_msg_state hash mismatch in external message"; + cp.skip_reason = ComputePhase::sk_bad_state; + return true; + } } td::optional precompiled; @@ -1826,6 +1840,26 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { for (int i = n - 1; i >= 0; --i) { ap.result_arg = n - 1 - i; if (!block::gen::t_OutListNode.validate_ref(ap.action_list[i])) { + if (cfg.message_skip_enabled) { + // try to read mode from action_send_msg even if out_msg scheme is violated + // action should at least contain 40 bits: 32bit tag and 8 bit mode + // if (mode & 2), that is ignore error mode, skip action even for invalid message + // if there is no (mode & 2) but (mode & 16) presents - enable bounce if possible + bool special = true; + auto cs = load_cell_slice_special(ap.action_list[i], special); + if (!special) { + if ((cs.size() >= 40) && ((int)cs.fetch_ulong(32) == 0x0ec3c86d)) { + int mode = (int)cs.fetch_ulong(8); + if (mode & 2) { + ap.skipped_actions++; + ap.action_list[i] = {}; + continue; + } else if ((mode & 16) && cfg.bounce_on_fail_enabled) { + ap.bounce = true; + } + } + } + } ap.result_code = 34; // action #i invalid or unsupported ap.action_list_invalid = true; LOG(DEBUG) << "invalid action " << ap.result_arg << " found while preprocessing action list: error code " @@ -1835,6 +1869,9 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { } ap.valid = true; for (int i = n - 1; i >= 0; --i) { + if(ap.action_list[i].is_null()) { + continue; + } ap.result_arg = n - 1 - i; vm::CellSlice cs = load_cell_slice(ap.action_list[i]); CHECK(cs.fetch_ref().not_null()); @@ -1872,7 +1909,7 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { ap.no_funds = true; } LOG(DEBUG) << "invalid action " << ap.result_arg << " in action list: error code " << ap.result_code; - // This is reuqired here because changes to libraries are applied even if actipn phase fails + // This is required here because changes to libraries are applied even if actipn phase fails enforce_state_limits(); if (cfg.action_fine_enabled) { ap.action_fine = std::min(ap.action_fine, balance.grams); @@ -2280,6 +2317,15 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, return -1; } bool skip_invalid = (act_rec.mode & 2); + auto check_skip_invalid = [&](unsigned error_code) -> unsigned int { + if (skip_invalid) { + if (cfg.message_skip_enabled) { + ap.skipped_actions++; + } + return 0; + } + return error_code; + }; // try to parse suggested message in act_rec.out_msg td::RefInt256 fwd_fee, ihr_fee; block::gen::MessageRelaxed::Record msg; @@ -2346,8 +2392,12 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, if (!tlb::csr_unpack(msg.info, info) || !block::tlb::t_CurrencyCollection.validate_csr(info.value)) { return -1; } - fwd_fee = block::tlb::t_Grams.as_integer(info.fwd_fee); - ihr_fee = block::tlb::t_Grams.as_integer(info.ihr_fee); + if (cfg.disable_custom_fess) { + fwd_fee = ihr_fee = td::zero_refint(); + } else { + fwd_fee = block::tlb::t_Grams.as_integer(info.fwd_fee); + ihr_fee = block::tlb::t_Grams.as_integer(info.ihr_fee); + } } // set created_at and created_lt to correct values info.created_at = now; @@ -2363,7 +2413,7 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, bool to_mc = false; if (!check_rewrite_dest_addr(info.dest, cfg, &to_mc)) { LOG(DEBUG) << "invalid destination address in a proposed outbound message"; - return skip_invalid ? 0 : 36; // invalid destination address + return check_skip_invalid(36); // invalid destination address } // fetch message pricing info @@ -2378,7 +2428,7 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, if (!ext_msg && !(act_rec.mode & 0x80) && !(act_rec.mode & 1)) { if (!block::tlb::t_CurrencyCollection.validate_csr(info.value)) { LOG(DEBUG) << "invalid value:CurrencyCollection in proposed outbound message"; - return skip_invalid ? 0 : 37; + return check_skip_invalid(37); } block::CurrencyCollection value; CHECK(value.unpack(info.value)); @@ -2395,7 +2445,7 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, if (new_funds->sgn() < 0) { LOG(DEBUG) << "not enough value to transfer with the message: all of the inbound message value has been consumed"; - return skip_invalid ? 0 : 37; + return check_skip_invalid(37); } } funds = std::min(funds, new_funds); @@ -2433,17 +2483,17 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, if (sstat.cells > max_cells && max_cells < cfg.size_limits.max_msg_cells) { LOG(DEBUG) << "not enough funds to process a message (max_cells=" << max_cells << ")"; collect_fine(); - return skip_invalid ? 0 : 40; + return check_skip_invalid(40); } if (sstat.bits > cfg.size_limits.max_msg_bits || sstat.cells > max_cells) { LOG(DEBUG) << "message too large, invalid"; collect_fine(); - return skip_invalid ? 0 : 40; + return check_skip_invalid(40); } if (max_merkle_depth > max_allowed_merkle_depth) { LOG(DEBUG) << "message has too big merkle depth, invalid"; collect_fine(); - return skip_invalid ? 0 : 40; + return check_skip_invalid(40); } LOG(DEBUG) << "storage paid for a message: " << sstat.cells << " cells, " << sstat.bits << " bits"; @@ -2475,7 +2525,7 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, if (!block::tlb::t_CurrencyCollection.validate_csr(info.value)) { LOG(DEBUG) << "invalid value:CurrencyCollection in proposed outbound message"; collect_fine(); - return skip_invalid ? 0 : 37; + return check_skip_invalid(37); } if (info.ihr_disabled) { // if IHR is disabled, IHR fees will be always zero @@ -2502,7 +2552,7 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, LOG(DEBUG) << "not enough value to transfer with the message: all of the inbound message value has been consumed"; collect_fine(); - return skip_invalid ? 0 : 37; + return check_skip_invalid(37); } } } @@ -2518,7 +2568,7 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, LOG(DEBUG) << "not enough value attached to the message to pay forwarding fees : have " << req.grams << ", need " << fees_total; collect_fine(); - return skip_invalid ? 0 : 37; // not enough grams + return check_skip_invalid(37); // not enough grams } else { // decrease message value req.grams -= fees_total; @@ -2529,7 +2579,7 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, LOG(DEBUG) << "not enough grams to transfer with the message : remaining balance is " << ap.remaining_balance.to_str() << ", need " << req_grams_brutto << " (including forwarding fees)"; collect_fine(); - return skip_invalid ? 0 : 37; // not enough grams + return check_skip_invalid(37); // not enough grams } Ref new_extra; @@ -2539,7 +2589,7 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, << block::CurrencyCollection{0, req.extra}.to_str() << " required, only " << block::CurrencyCollection{0, ap.remaining_balance.extra}.to_str() << " available"; collect_fine(); - return skip_invalid ? 0 : 38; // not enough (extra) funds + return check_skip_invalid(38); // not enough (extra) funds } if (ap.remaining_balance.extra.not_null() || req.extra.not_null()) { LOG(DEBUG) << "subtracting extra currencies: " @@ -2563,7 +2613,7 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, LOG(DEBUG) << "outbound message does not fit into a cell after rewriting"; if (redoing == 2) { collect_fine(); - return skip_invalid ? 0 : 39; + return check_skip_invalid(39); } return -2; } @@ -2588,7 +2638,7 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, if (ap.remaining_balance.grams < fwd_fee) { LOG(DEBUG) << "not enough funds to pay for an outbound external message"; collect_fine(); - return skip_invalid ? 0 : 37; // not enough grams + return check_skip_invalid(37); // not enough grams } // repack message // ext_out_msg_info$11 constructor of CommonMsgInfo @@ -2603,7 +2653,7 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, LOG(DEBUG) << "outbound message does not fit into a cell after rewriting"; if (redoing == 2) { collect_fine(); - return (skip_invalid ? 0 : 39); + return check_skip_invalid(39); } return -2; } @@ -3514,7 +3564,7 @@ LtCellRef Transaction::extract_out_msg(unsigned i) { * @returns A triple of the logical time, the extracted output message and the transaction root. */ NewOutMsg Transaction::extract_out_msg_ext(unsigned i) { - return {start_lt + i + 1, std::move(out_msgs.at(i)), root}; + return {start_lt + i + 1, std::move(out_msgs.at(i)), root, i}; } /** @@ -3684,6 +3734,7 @@ td::Status FetchConfigParams::fetch_config_params( compute_phase_cfg->suspended_addresses = config.get_suspended_addresses(now); compute_phase_cfg->size_limits = size_limits; compute_phase_cfg->precompiled_contracts = config.get_precompiled_contracts_config(); + compute_phase_cfg->allow_external_unfreeze = compute_phase_cfg->global_version >= 8; } { // compute action_phase_cfg @@ -3707,6 +3758,8 @@ td::Status FetchConfigParams::fetch_config_params( action_phase_cfg->size_limits = size_limits; action_phase_cfg->action_fine_enabled = config.get_global_version() >= 4; action_phase_cfg->bounce_on_fail_enabled = config.get_global_version() >= 4; + action_phase_cfg->message_skip_enabled = config.get_global_version() >= 8; + action_phase_cfg->disable_custom_fess = config.get_global_version() >= 8; action_phase_cfg->mc_blackhole_addr = config.get_burning_config().blackhole_addr; } { diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index 6d8e8a29f..20d7cb291 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -66,8 +66,11 @@ struct NewOutMsg { ton::LogicalTime lt; Ref msg; Ref trans; - NewOutMsg(ton::LogicalTime _lt, Ref _msg, Ref _trans) - : lt(_lt), msg(std::move(_msg)), trans(std::move(_trans)) { + unsigned msg_idx; + td::optional metadata; + td::Ref msg_env_from_dispatch_queue; // Not null if from dispatch queue; in this case lt is emitted_lt + NewOutMsg(ton::LogicalTime _lt, Ref _msg, Ref _trans, unsigned _msg_idx) + : lt(_lt), msg(std::move(_msg)), trans(std::move(_trans)), msg_idx(_msg_idx) { } bool operator<(const NewOutMsg& other) const& { return lt < other.lt || (lt == other.lt && msg->get_hash() < other.msg->get_hash()); @@ -126,6 +129,7 @@ struct ComputePhaseConfig { bool stop_on_accept_message = false; PrecompiledContractsConfig precompiled_contracts; bool dont_run_precompiled_ = false; + bool allow_external_unfreeze{false}; ComputePhaseConfig() : gas_price(0), gas_limit(0), special_gas_limit(0), gas_credit(0) { compute_threshold(); @@ -163,6 +167,8 @@ struct ActionPhaseConfig { const WorkchainSet* workchains{nullptr}; bool action_fine_enabled{false}; bool bounce_on_fail_enabled{false}; + bool message_skip_enabled{false}; + bool disable_custom_fess{false}; td::optional mc_blackhole_addr; const MsgPrices& fetch_msg_prices(bool is_masterchain) const { return is_masterchain ? fwd_mc : fwd_std; diff --git a/crypto/fift/fift-main.cpp b/crypto/fift/fift-main.cpp index ef833f43e..fd424e8cf 100644 --- a/crypto/fift/fift-main.cpp +++ b/crypto/fift/fift-main.cpp @@ -62,7 +62,7 @@ void usage(const char* progname) { << " [-i] [-n] [-I ] {-L } ...\n"; std::cerr << "\t-n\tDo not preload standard preamble file `Fift.fif`\n" "\t-i\tForce interactive mode even if explicit source file names are indicated\n" - "\t-I\tSets colon-separated library source include path. If not indicated, " + "\t-I\tSets colon-separated (unix) or at-separated (windows) library source include path. If not indicated, " "$FIFTPATH is used instead.\n" "\t-L\tPre-loads a library source file\n" "\t-d\tUse a ton database\n" @@ -75,11 +75,16 @@ void usage(const char* progname) { void parse_include_path_set(std::string include_path_set, std::vector& res) { td::Parser parser(include_path_set); while (!parser.empty()) { - auto path = parser.read_till_nofail(':'); + #if TD_WINDOWS + auto path_separator = '@'; + #else + auto path_separator = ':'; + #endif + auto path = parser.read_till_nofail(path_separator); if (!path.empty()) { res.push_back(path.str()); } - parser.skip_nofail(':'); + parser.skip_nofail(path_separator); } } diff --git a/crypto/smartcont/stdlib.fc b/crypto/smartcont/stdlib.fc index 978b94738..8fb27a7ea 100644 --- a/crypto/smartcont/stdlib.fc +++ b/crypto/smartcont/stdlib.fc @@ -244,7 +244,7 @@ cont bless(slice s) impure asm "BLESS"; ;;; In other words, the current smart contract agrees to buy some gas to finish the current transaction. ;;; This action is required to process external messages, which bring no value (hence no gas) with themselves. ;;; -;;; For more details check [accept_message effects](https://ton.org/docs/#/smart-contracts/accept). +;;; For more details check [accept_message effects](https://docs.ton.org/develop/smart-contracts/guidelines/accept). () accept_message() impure asm "ACCEPT"; ;;; Sets current gas limit `gl` to the minimum of limit and `gm`, and resets the gas credit `gc` to zero. @@ -282,10 +282,10 @@ int abs(int x) asm "ABS"; It is said that a primitive _loads_ some data, if it returns the data and the remainder of the slice - (so it can also be used as [modifying method](https://ton.org/docs/#/func/statements?id=modifying-methods)). + (so it can also be used as [modifying method](https://docs.ton.org/develop/func/statements#modifying-methods)). It is said that a primitive _preloads_ some data, if it returns only the data - (it can be used as [non-modifying method](https://ton.org/docs/#/func/statements?id=non-modifying-methods)). + (it can be used as [non-modifying method](https://docs.ton.org/develop/func/statements#non-modifying-methods)). Unless otherwise stated, loading and preloading primitives read the data from a prefix of the slice. -} @@ -416,7 +416,7 @@ int builder_depth(builder b) asm "BDEPTH"; # Builder primitives It is said that a primitive _stores_ a value `x` into a builder `b` if it returns a modified version of the builder `b'` with the value `x` stored at the end of it. - It can be used as [non-modifying method](https://ton.org/docs/#/func/statements?id=non-modifying-methods). + It can be used as [non-modifying method](https://docs.ton.org/develop/func/statements#non-modifying-methods). All the primitives below first check whether there is enough space in the `builder`, and only then check the range of the value being serialized. diff --git a/crypto/smc-envelope/ManualDns.cpp b/crypto/smc-envelope/ManualDns.cpp index 40ce3bac5..617ab9156 100644 --- a/crypto/smc-envelope/ManualDns.cpp +++ b/crypto/smc-envelope/ManualDns.cpp @@ -105,7 +105,7 @@ td::Result> DnsInterface::EntryData::as_cell() const { return error; } if (res.is_null()) { - return td::Status::Error("Entry data is emtpy"); + return td::Status::Error("Entry data is empty"); } return res; //dns_text#1eda _:Text = DNSRecord; diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index 399ec130a..75071573c 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -17,6 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "td/utils/CancellationToken.h" + #include #include #include "vm/cells.h" diff --git a/crypto/vm/cells/DataCell.cpp b/crypto/vm/cells/DataCell.cpp index cccb11dce..8aac575bf 100644 --- a/crypto/vm/cells/DataCell.cpp +++ b/crypto/vm/cells/DataCell.cpp @@ -113,6 +113,9 @@ td::Result> DataCell::create(td::ConstBitPtr data, unsigned bits, if (bits != 8 + hash_bytes * 8) { return td::Status::Error("Not enouch data for a Library special cell"); } + if (!refs.empty()) { + return td::Status::Error("Library special cell has a cell reference"); + } break; } diff --git a/crypto/vm/large-boc-serializer.cpp b/crypto/vm/large-boc-serializer.cpp index 237af7c8d..d9ef142ee 100644 --- a/crypto/vm/large-boc-serializer.cpp +++ b/crypto/vm/large-boc-serializer.cpp @@ -14,6 +14,9 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ +#include "td/utils/Time.h" +#include "td/utils/Timer.h" + #include #include "vm/large-boc-serializer.h" #include "vm/boc-writers.h" @@ -30,7 +33,9 @@ class LargeBocSerializer { public: using Hash = Cell::Hash; - explicit LargeBocSerializer(std::shared_ptr reader) : reader(std::move(reader)) {} + explicit LargeBocSerializer(std::shared_ptr reader, td::CancellationToken cancellation_token = {}) + : reader(std::move(reader)), cancellation_token(std::move(cancellation_token)) { + } void add_root(Hash root); td::Status import_cells(); @@ -65,7 +70,8 @@ class LargeBocSerializer { std::map cells; std::vector*> cell_list; struct RootInfo { - RootInfo(Hash hash, int idx) : hash(hash), idx(idx) {} + RootInfo(Hash hash, int idx) : hash(hash), idx(idx) { + } Hash hash; int idx; }; @@ -78,6 +84,11 @@ class LargeBocSerializer { void reorder_cells(); int revisit(int cell_idx, int force = 0); td::uint64 compute_sizes(int mode, int& r_size, int& o_size); + + td::CancellationToken cancellation_token; + td::Timestamp log_speed_at_; + size_t processed_cells_ = 0; + static constexpr double LOG_SPEED_PERIOD = 120.0; }; void LargeBocSerializer::add_root(Hash root) { @@ -85,12 +96,16 @@ void LargeBocSerializer::add_root(Hash root) { } td::Status LargeBocSerializer::import_cells() { + td::Timer timer; + log_speed_at_ = td::Timestamp::in(LOG_SPEED_PERIOD); + processed_cells_ = 0; for (auto& root : roots) { TRY_RESULT(idx, import_cell(root.hash)); root.idx = idx; } reorder_cells(); CHECK(!cell_list.empty()); + LOG(ERROR) << "serializer: import_cells took " << timer.elapsed() << "s, " << cell_count << " cells"; return td::Status::OK(); } @@ -98,6 +113,15 @@ td::Result LargeBocSerializer::import_cell(Hash hash, int depth) { if (depth > Cell::max_depth) { return td::Status::Error("error while importing a cell into a bag of cells: cell depth too large"); } + ++processed_cells_; + if (processed_cells_ % 1000 == 0) { + TRY_STATUS(cancellation_token.check()); + } + if (log_speed_at_.is_in_past()) { + log_speed_at_ += LOG_SPEED_PERIOD; + LOG(WARNING) << "serializer: import_cells " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s"; + processed_cells_ = 0; + } auto it = cells.find(hash); if (it != cells.end()) { it->second.should_cache = true; @@ -282,6 +306,7 @@ td::uint64 LargeBocSerializer::compute_sizes(int mode, int& r_size, int& o_size) } td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { + td::Timer timer; using Mode = BagOfCells::Mode; BagOfCells::Info info; if ((mode & Mode::WithCacheBits) && !(mode & Mode::WithIndex)) { @@ -313,13 +338,9 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { return td::Status::Error("bag of cells is too large"); } - boc_writers::FileWriter writer{fd, (size_t) info.total_size}; - auto store_ref = [&](unsigned long long value) { - writer.store_uint(value, info.ref_byte_size); - }; - auto store_offset = [&](unsigned long long value) { - writer.store_uint(value, info.offset_byte_size); - }; + boc_writers::FileWriter writer{fd, (size_t)info.total_size}; + auto store_ref = [&](unsigned long long value) { writer.store_uint(value, info.ref_byte_size); }; + auto store_offset = [&](unsigned long long value) { writer.store_uint(value, info.offset_byte_size); }; writer.store_uint(info.magic, 4); @@ -371,6 +392,8 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { } DCHECK(writer.position() == info.data_offset); size_t keep_position = writer.position(); + log_speed_at_ = td::Timestamp::in(LOG_SPEED_PERIOD); + processed_cells_ = 0; for (int i = 0; i < cell_count; ++i) { auto hash = cell_list[cell_count - 1 - i]->first; const auto& dc_info = cell_list[cell_count - 1 - i]->second; @@ -389,6 +412,15 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { DCHECK(k > i && k < cell_count); store_ref(k); } + ++processed_cells_; + if (processed_cells_ % 1000 == 0) { + TRY_STATUS(cancellation_token.check()); + } + if (log_speed_at_.is_in_past()) { + log_speed_at_ += LOG_SPEED_PERIOD; + LOG(WARNING) << "serializer: serialize " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s"; + processed_cells_ = 0; + } } DCHECK(writer.position() - keep_position == info.data_size); if (info.has_crc32c) { @@ -396,17 +428,23 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { writer.store_uint(td::bswap32(crc), 4); } DCHECK(writer.empty()); - return writer.finalize(); -} + TRY_STATUS(writer.finalize()); + LOG(ERROR) << "serializer: serialize took " << timer.elapsed() << "s, " << cell_count << " cells, " + << writer.position() << " bytes"; + return td::Status::OK(); } +} // namespace -td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, - td::FileFd& fd, int mode) { +td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, + int mode, td::CancellationToken cancellation_token) { + td::Timer timer; CHECK(reader != nullptr) - LargeBocSerializer serializer(reader); + LargeBocSerializer serializer(reader, std::move(cancellation_token)); serializer.add_root(root_hash); TRY_STATUS(serializer.import_cells()); - return serializer.serialize(fd, mode); + TRY_STATUS(serializer.serialize(fd, mode)); + LOG(ERROR) << "serialization took " << timer.elapsed() << "s"; + return td::Status::OK(); } -} +} // namespace vm diff --git a/crypto/vm/large-boc-serializer.h b/crypto/vm/large-boc-serializer.h index 29980cbd3..894d8062f 100644 --- a/crypto/vm/large-boc-serializer.h +++ b/crypto/vm/large-boc-serializer.h @@ -19,7 +19,7 @@ #include "db/DynamicBagOfCellsDb.h" namespace vm { -td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, - td::FileFd& fd, int mode); +td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, + int mode = 0, td::CancellationToken cancellation_token = {}); } // namespace vm diff --git a/dht-server/dht-server.cpp b/dht-server/dht-server.cpp index 37a158ebb..025cf7d51 100644 --- a/dht-server/dht-server.cpp +++ b/dht-server/dht-server.cpp @@ -170,7 +170,7 @@ ton::tl_object_ptr Config::tl() const { return ton::create_tl_object( out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), ton::PublicKeyHash::zero().tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), - nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec)); + nullptr, nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec)); } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index 6c176552f..e649c009a 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -96,4 +96,17 @@ Operations for working with Merkle proofs, where cells can have non-zero level a ### Other changes * `GLOBALID` gets `ConfigParam 19` from the tuple, not from the config dict. This decreases gas usage. -* `SENDMSG` gets `ConfigParam 24/25` (message prices) from the tuple, not from the config dict, and also uses `ConfigParam 43` to get max_msg_cells. \ No newline at end of file +* `SENDMSG` gets `ConfigParam 24/25` (message prices) from the tuple, not from the config dict, and also uses `ConfigParam 43` to get max_msg_cells. + + +## Version 7 + +[Explicitly nullify](https://github.com/ton-blockchain/ton/pull/957/files) `due_payment` after due reimbursment. + +## Version 8 + +- Check mode on invalid `action_send_msg`. Ignore action if `IGNORE_ERROR` (+2) bit is set, bounce if `BOUNCE_ON_FAIL` (+16) bit is set. +- Slightly change random seed generation to fix mix of `addr_rewrite` and `addr`. +- Fill in `skipped_actions` for both invalid and valid messages with `IGNORE_ERROR` mode that can't be sent. +- Allow unfreeze through external messages. +- Don't use user-provided `fwd_fee` and `ihr_fee` for internal messages. \ No newline at end of file diff --git a/docker/README.md b/docker/README.md index fd98374b6..47e109dbf 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,28 +1,525 @@ -# The Open Network Node -Dockerfile for The Open Network Node +# Official TON Docker image -#### Install +1. [Dockerfile](#docker) +2. [Kubernetes deployment on-premises](#deploy-on-premises-with-metallb-load-balancer-) +3. [Kubernetes deployment on AWS](#deploy-on-aws-cloud-amazon-web-services) +4. [Kubernetes deployment on GCP](#deploy-on-gcp-google-cloud-platform) +5. [Kubernetes deployment on AliCloud](#deploy-on-ali-cloud) +6. [Troubleshooting](#troubleshooting) +## Prerequisites + +The TON node, whether it is validator or fullnode, requires a public IP address. +If your server is within an internal network or kubernetes you have to make sure that the required ports are available from the outside. + +Also pay attention at [hardware requirements](https://docs.ton.org/participate/run-nodes/full-node) for TON fullnodes and validators. Pods and StatefulSets in this guide imply these requirements. + +It is recommended to everyone to read Docker chapter first in order to get a better understanding about TON Docker image and its parameters. + +## Docker + +### Installation ```docker pull ghcr.io/ton-blockchain/ton:latest``` -#### Create volume -```docker volume create ton-db``` -#### Run -```docker run -d --name ton-node --mount source=ton-db,target=/var/ton-work/db --network host -e "PUBLIC_IP=" -e "CONSOLE_PORT=" -e "LITESERVER=true" -e "LITE_PORT=" -it ghcr.io/ton-blockchain/ton``` +### Configuration +TON validator-engine supports number of command line parameters, +these parameters can be handed over to the container via environment variables. +Below is the list of supported arguments and their default values: + +| Argument | Description | Mandatory? | Default value | +|:------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:----------:|:-------------------------------------------------------:| +| PUBLIC_IP | This will be a public IP address of your TON node. Normally it is the same IP address as your server's external IP. This also can be your proxy server or load balancer IP address. | yes | | +| GLOBAL_CONFIG_URL | TON global configuration file. Mainnet - https://ton.org/global-config.json, Testnet - https://ton.org/testnet-global.config.json | no | https://api.tontech.io/ton/wallet-mainnet.autoconf.json | +| DUMP_URL | URL to TON dump. Specify dump from https://dump.ton.org. If you are using testnet dump, make sure to download global config for testnet. | no | | +| VALIDATOR_PORT | UDP port that must be available from the outside. Used for communication with other nodes. | no | 30001 | +| CONSOLE_PORT | This TCP port is used to access validator's console. Not necessarily to be opened for external access. | no | 30002 | +| LITE_PORT | Lite-server's TCP port. Used by lite-client. | no | 30003 | +| LITESERVER | true or false. Set to true if you want up and running lite-server. | no | false | +| STATE_TTL | Node's state will be gc'd after this time (in seconds). | no | 86400 | +| ARCHIVE_TTL | Node's archived blocks will be deleted after this time (in seconds). | no | 86400 | +| THREADS | Number of threads used by validator-engine. | no | 8 | +| VERBOSITY | Verbosity level. | no | 3 | +| CUSTOM_ARG | validator-engine might have some undocumented arguments. This is reserved for the test purposes.
For example you can pass **--logname /var/ton-work/log** in order to have log files. | no | | + +### Run the node - the quick way +The below command runs docker container with a TON node, that will start synchronization process. + +Notice **--network host** option, means that the Docker container will use the network namespace of the host machine. +In this case there is no need to map ports between the host and the container. The container will use the same IP address and ports as the host. +This approach simplifies networking configuration for the container, and usually is used on the dedicated server with assigned public IP. + +Keep in mind that this option can also introduce security concerns because the container has access to the host's network interfaces directly, which might not be desirable in a multi-tenant environment. + +Check your firewall configuration and make sure that at least UDP port 43677 is publicly available. +Find out your PUBLIC_IP: +``` +curl -4 ifconfig.me +``` +and replace it in the command below: +``` +docker run -d --name ton-node -v /data/db:/var/ton-work/db \ +-e "PUBLIC_IP=" \ +-e "LITESERVER=true" \ +-e "DUMP_URL=https://dump.ton.org/dumps/latest.tar.lz" \ +--network host \ +-it ghcr.io/ton-blockchain/ton +``` +If you don't need Lite-server, then remove -e "LITESERVER=true". + +### Run the node - isolated way +In production environments it is recommended to use **Port mapping** feature of Docker's default bridge network. +When you use port mapping, Docker allocates a specific port on the host to forward traffic to a port inside the container. +This is ideal for running multiple containers with isolated networks on the same host. +``` +docker run -d --name ton-node -v /data/db:/var/ton-work/db \ +-e "PUBLIC_IP=" \ +-e "DUMP_URL=https://dump.ton.org/dumps/latest.tar.lz" \ +-e "VALIDATOR_PORT=443" \ +-e "CONSOLE_PORT=88" \ +-e "LITE_PORT=443" \ +-e "LITESERVER=true" \ +-p 443:443/udp \ +-p 88:88/tcp \ +-p 443:443/tcp \ +-it ghcr.io/ton-blockchain/ton +``` +Adjust ports per your need. +Check your firewall configuration and make sure that customized ports (443/udp, 88/tcp and 443/tcp in this example) are publicly available. + +### Verify if TON node is operating correctly +After executing above command check the log files: + +```docker logs ton-node``` + +This is totally fine if in the log output for some time (up to 15 minutes) you see messages like: + +```log +failed to download proof link: [Error : 651 : no nodes] +``` + +After some time you should be able to see multiple messages similar to these below: +```log +failed to download key blocks: [Error : 652 : adnl query timeout] +last key block is [ w=-1 s=9223372036854775808 seq=34879845 rcEsfLF3E80PqQPWesW+rlOY2EpXd5UDrW32SzRWgus= C1Hs+q2Vew+WxbGL6PU1P6R2iYUJVJs4032CTS/DQzI= ] +getnextkey: [Error : 651 : not inited] +downloading state (-1,8000000000000000,38585739):9E86E166AE7E24BAA22762766381440C625F47E2B11D72967BB58CE8C90F7EBA:5BFFF759380097DF178325A7151E9C0571C4E452A621441A03A0CECAED970F57: total=1442840576 (71MB/s)downloading state (-1,8000000000000000,38585739):9E86E166AE7E24BAA22762766381440C625F47E2B11D72967BB58CE8C90F7EBA:5BFFF759380097DF178325A7151E9C0571C4E452A621441A03A0CECAED970F57: total=1442840576 (71MB/s) +finished downloading state (-1,8000000000000000,38585739):9E86E166AE7E24BAA22762766381440C625F47E2B11D72967BB58CE8C90F7EBA:5BFFF759380097DF178325A7151E9C0571C4E452A621441A03A0CECAED970F57: total=4520747390 +getnextkey: [Error : 651 : not inited] +getnextkey: [Error : 651 : not inited] +``` +As you noticed we have mounted docker volume to a local folder **/data/db**. +Go inside this folder on your server and check if its size is growing (```sudo du -h .*```) + +Now connect to the running container: +``` +docker exec -ti ton-node /bin/bash +``` +and try to connect and execute **getconfig** command via validator-engine-console: +``` +validator-engine-console -k client -p server.pub -a localhost:$(jq .control[].port <<< cat /var/ton-work/db/config.json) -c getconfig +``` +if you see a json output that means that validator-engine is up, now execute **last** command with a lite-client: +``` +lite-client -a localhost:$(jq .liteservers[].port <<< cat /var/ton-work/db/config.json) -p liteserver.pub -c last +``` +if you see the following output: +``` +conn ready +failed query: [Error : 652 : adnl query timeout] +cannot get server version and time (server too old?) +server version is too old (at least 1.1 with capabilities 1 required), some queries are unavailable +fatal error executing command-line queries, skipping the rest +``` +it means that the lite-server is up, but the node is not synchronized yet. +Once the node is synchronized, the output of **last** command will be similar to this one: + +``` +conn ready +server version is 1.1, capabilities 7 +server time is 1719306580 (delta 0) +last masterchain block is (-1,8000000000000000,20435927):47A517265B25CE4F2C8B3058D46343C070A4B31C5C37745390CE916C7D1CE1C5:279F9AA88C8146257E6C9B537905238C26E37DC2E627F2B6F1D558CB29A6EC82 +server time is 1719306580 (delta 0) +zerostate id set to -1:823F81F306FF02694F935CF5021548E3CE2B86B529812AF6A12148879E95A128:67E20AC184B9E039A62667ACC3F9C00F90F359A76738233379EFA47604980CE8 +``` +If you can't make it working, refer to the [Troubleshooting](#troubleshooting) section below. +### Use validator-engine-console +```docker exec -ti ton-node /bin/bash``` + +```validator-engine-console -k client -p server.pub -a 127.0.0.1:$(jq .control[].port <<< cat /var/ton-work/db/config.json)``` + +### Use lite-client +```docker exec -ti ton-node /bin/bash``` + +```lite-client -p liteserver.pub -a 127.0.0.1:$(jq .liteservers[].port <<< cat /var/ton-work/db/config.json)``` + +If you use lite-client outside the Docker container, copy the **liteserver.pub** from the container: + +```docker cp ton-node:/var/ton-work/db/liteserver.pub /your/path``` + +```lite-client -p /your/path/liteserver.pub -a :``` + +### Stop TON docker container +``` +docker stop ton-node +``` + +## Kubernetes +### Deploy in a quick way (without load balancer) +If the nodes within your kubernetes cluster have external IPs, +make sure that the PUBLIC_IP used for validator-engine matches the node's external IP. +If all Kubernetes nodes are inside DMZ - skip this section. + +#### Prepare +If you are using **flannel** network driver you can find node's IP this way: +```yaml +kubectl get nodes +kubectl describe node | grep public-ip +``` +for **calico** driver use: +```yaml +kubectl describe node | grep IPv4Address +``` +Double check if your Kubernetes node's external IP coincides with the host's IP address: +``` +kubectl run --image=ghcr.io/ton-blockchain/ton:latest validator-engine-pod --env="HOST_IP=1.1.1.1" --env="PUBLIC_IP=1.1.1.1" +kubectl exec -it validator-engine-pod -- curl -4 ifconfig.me +kubectl delete pod validator-engine-pod +``` +If IPs do not match, refer to the sections where load balancers are used. + +Now do the following: +* Add a label to this particular node. +* By this label our pod will know where to be deployed and what storage to use: +``` +kubectl label nodes node_type=ton-validator +``` +* Replace **** (and ports if needed) in file [ton-node-port.yaml](ton-node-port.yaml). +* Replace **** with a real path on host for Persistent Volume. +* If you change the ports, make sure you specify appropriate env vars in Pod section. +* If you want to use dynamic storage provisioning via volumeClaimTemplates, feel free to create own StorageClass. + +#### Install +```yaml +kubectl apply -f ton-node-port.yaml +``` + +this deployment uses host's network stack (**hostNetwork: true**) option and service of **NodePort** type. +Actually you can also use service of type **LoadBalancer**. +This way the service will get public IP assigned to the endpoints. + +#### Verify installation +See if service endpoints were correctly created: + +```yaml +kubectl get endpoints + +NAME ENDPOINTS +validator-engine-srv :30002,:30001,:30003 +``` +Check the logs for the deployment status: +```yaml +kubectl logs validator-engine-pod +``` +or go inside the pod and check if blockchain size is growing: +```yaml +kubectl exec --stdin --tty validator-engine-pod -- /bin/bash +du -h . +``` +### Deploy on-premises with metalLB load balancer + +Often Kubernetes cluster is located in DMZ, is behind corporate firewall and access is controlled via proxy configuration. +In this case we can't use host's network stack (**hostNetwork: true**) within a Pod and must manually proxy the access to the pod. + +A **LoadBalancer** service type automatically provisions an external load balancer (such as those provided by cloud providers like AWS, GCP, Azure) and assigns a public IP address to your service. In a non-cloud environment or in a DMZ setup, you need to manually configure the load balancer. + +If you are running your Kubernetes cluster on-premises or in an environment where an external load balancer is not automatically provided, you can use a load balancer implementation like MetalLB. + +#### Prepare +Select the node where persistent storage will be located for TON validator. +* Add a label to this particular node. By this label our pod will know where to be deployed: +``` +kubectl label nodes node_type=ton-validator +``` +* Replace **** (and ports if needed) in file [ton-metal-lb.yaml](ton-metal-lb.yaml). +* Replace **** with a real path on host for Persistent Volume. +* If you change the ports, make sure you specify appropriate env vars in Pod section. +* If you want to use dynamic storage provisioning via volumeClaimTemplates, feel free to create own StorageClass. + +* Install MetalLB +```yaml +kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.14.5/config/manifests/metallb-native.yaml +``` + +* Configure MetalLB +Create a configuration map to define the IP address range that MetalLB can use for external load balancer services. +```yaml +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: first-pool + namespace: metallb-system +spec: + addresses: + - 10.244.1.0/24 <-- your CIDR address +``` +apply configuration +```yaml +kubectl apply -f metallb-config.yaml +``` +#### Install + +```yaml +kubectl apply -f ton-metal-lb.yaml +``` +We do not use Pod Node Affinity here, since the Pod will remember the host with local storage it was bound to. + +#### Verify installation +Assume your network CIDR (**--pod-network-cidr**) within cluster is 10.244.1.0/24, then you can compare the output with the one below: +```yaml +kubectl get service + +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +kubernetes ClusterIP 443/TCP 28h +validator-engine-srv LoadBalancer 10.244.1.1 30001:30001/UDP,30002:30002/TCP,30003:30003/TCP 60m +``` +you can see that endpoints are pointing to metal-LB subnet: +``` +kubectl get endpoints + +NAME ENDPOINTS +kubernetes :6443 +validator-engine-srv 10.244.1.10:30002,10.244.1.10:30001,10.244.1.10:30003 +``` +and metal-LB itself operates with the right endpoint: +``` +kubectl describe service metallb-webhook-service -n metallb-system + +Name: metallb-webhook-service +Namespace: metallb-system +Selector: component=controller +Type: ClusterIP +IP: +IPs: +Port: 443/TCP +TargetPort: 9443/TCP +Endpoints: 10.244.2.3:9443 <-- CIDR +``` + +Use the commands from the previous chapter to see if node operates properly. + +### Deploy on AWS cloud (Amazon Web Services) + +#### Prepare +* AWS EKS is configured with worker nodes with selected add-ons: + * CoreDNS - Enable service discovery within your cluster. + * kube-proxy - Enable service networking within your cluster. + * Amazon VPC CNI - Enable pod networking within your cluster. +* Allocate Elastic IP. +* Replace **** with the newly created Elastic IP in [ton-aws.yaml](ton-aws.yaml) +* Replace **** with Elastic IP allocation ID (see in AWS console). +* Adjust StorageClass name. Make sure you are providing fast storage. + +#### Install + +```kubectl apply -f ton-aws.yaml``` + +#### Verify installation +Use instructions from the previous sections. + +### Deploy on GCP (Google Cloud Platform) + +#### Prepare +* Kubernetes cluster of type Standard (not Autopilot). +* Premium static IP address. +* Adjust firewall rules and security groups to allow ports 30001/udp, 30002/tcp and 30003/tcp (default ones). +* Replace **** (and ports if needed) in file [ton-gcp.yaml](ton-gcp.yaml). +* Adjust StorageClass name. Make sure you are providing fast storage. + +* Load Balancer will be created automatically according to Kubernetes service in yaml file. + +#### Install +```kubectl apply -f ton-gcp.yaml``` + +#### Verify installation +Use instructions from the previous sections. + +### Deploy on Ali Cloud + +#### Prepare +* AliCloud kubernetes cluster. +* Elastic IP. +* Replace **** with Elastic IP allocation ID (see in AliCloud console). +* Replace **** (and ports if needed) in file [ton-ali.yaml](ton-ali.yaml) with the elastic IP attached to your CLB. +* Adjust StorageClass name. Make sure you are providing fast storage. + +#### Install +```kubectl apply -f ton-ali.yaml``` + +As a result CLB (classic internal Load Balancer) will be created automatically with assigned external IP. + +#### Verify installation +Use instructions from the previous sections. + +## Troubleshooting +## Docker +### TON node cannot synchronize, constantly see messages [Error : 651 : no nodes] in the log + +Start the new container without starting validator-engine: + +``` +docker run -it -v /data/db:/var/ton-work/db \ +-e "HOST_IP=" \ +-e "PUBLIC_IP=" \ +-e "LITESERVER=true" \ +-p 43677:43677/udp \ +-p 43678:43678/tcp \ +-p 43679:43679/tcp \ +--entrypoint /bin/bash \ +ghcr.io/ton-blockchain/ton +``` +identify your PUBLIC_IP: +``` +curl -4 ifconfig.me +``` +compare if resulted IP coincides with your . +If it doesn't, exit container and launch it with the correct public IP. +Then open UDP port (inside the container) you plan to allocate for TON node using netcat utility: +``` +nc -ul 30001 +``` +and from any **other** linux machine check if you can reach this UDP port by sending a test message to that port: +``` +echo "test" | nc -u 30001 +``` +as a result inside the container you have to receive the "test" message. + +If you don't get the message inside the docker container, that means that either your firewall, LoadBalancer, NAT or proxy is blocking it. +Ask your system administrator for assistance. + +In the same way you can check if TCP port is available: + +Execute inside the container ```nc -l 30003``` and test connection from another server +```nc -vz 30003``` + +### Can't connect to lite-server +* check if lite-server was enabled on start by passing **"LITESERVER=true"** argument; +* check if TCP port (LITE_PORT) is available from the outside. From any other linux machine execute: + ``` +nc -vz +``` +### How to see what traffic is generated inside the TON docker container? +There is available a traffic monitoring utility inside the container, just execute: +``` +iptraf-ng +``` +Other tools like **tcpdump**, **nc**, **wget**, **curl**, **ifconfig**, **pv**, **plzip**, **jq** and **netstat** are also available. + +### How to build TON docker image from sources? +``` +git clone --recursive https://github.com/ton-blockchain/ton.git +cd ton +docker build . +``` + +## Kubernetes +### AWS +#### After installing AWS LB, load balancer is still not available (pending): +``` +kubectl get deployment -n kube-system aws-load-balancer-controller +``` +Solution: + +Try to install AWS LoadBalancer using ```Helm``` way. + +--- + +#### After installing AWS LB and running ton node, service shows error: + +```k describe service validator-engine-srv``` + +```log +Failed build model due to unable to resolve at least one subnet (0 match VPC and tags: [kubernetes.io/role/elb]) +``` +Solution: + +You haven't labeled the AWS subnets with the correct resource tags. + +* Public Subnets should be resource tagged with: kubernetes.io/role/elb: 1 +* Private Subnets should be tagged with: kubernetes.io/role/internal-elb: 1 +* Both private and public subnets should be tagged with: kubernetes.io/cluster/${your-cluster-name}: owned +* or if the subnets are also used by non-EKS resources kubernetes.io/cluster/${your-cluster-name}: shared + +So create tags for at least one subnet: +``` +kubernetes.io/role/elb: 1 +kubernetes.io/cluster/: owner +``` +--- +#### AWS Load Balancer works, but I still see ```[no nodes]``` in validator's log +It is required to add the security group for the EC2 instances to the load balancer along with the default security group. +It's a misleading that the default security group has "everything open." + +Add security group (default name is usually something like 'launch-wizard-1'). +And make sure you allow the ports you specified or default ports 30001/udp, 30002/tcp and 30003/tcp. + +You can also set inbound and outbound rules of new security group to allow ALL ports and for ALL protocols and for source CIDR 0.0.0.0/0 for testing purposes. + +--- + +#### Pending PersistentVolumeClaim ```Waiting for a volume to be created either by the external provisioner 'ebs.csi.aws.com' or manually by the system administrator.``` + +Solution: + +Configure Amazon EBS CSI driver for working PersistentVolumes in EKS. + +1. Enable IAM OIDC provider +``` +eksctl utils associate-iam-oidc-provider --region=us-west-2 --cluster=k8s-my --approve +``` + +2. Create Amazon EBS CSI driver IAM role +``` +eksctl create iamserviceaccount \ +--region us-west-2 \ +--name ebs-csi-controller-sa \ +--namespace kube-system \ +--cluster k8s-my \ +--attach-policy-arn arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy \ +--approve \ +--role-only \ +--role-name AmazonEKS_EBS_CSI_DriverRole +``` + +3. Add the Amazon EBS CSI add-on +```yaml +eksctl create addon --name aws-ebs-csi-driver --cluster k8s-my --service-account-role-arn arn:aws:iam::$(aws sts get-caller-identity --query Account --output text):role/AmazonEKS_EBS_CSI_DriverRole --force +``` +### Google Cloud +#### Load Balancer cannot obtain external IP (pending) -If you don't need Liteserver, then remove -e "LITESERVER=true". +``` +kubectl describe service validator-engine-srv -#### Use -```docker exec -ti /bin/bash``` +Events: +Type Reason Age From Message + ---- ------ ---- ---- ------- +Warning LoadBalancerMixedProtocolNotSupported 7m8s g-cloudprovider LoadBalancers with multiple protocols are not supported. +Normal EnsuringLoadBalancer 113s (x7 over 7m8s) service-controller Ensuring load balancer +Warning SyncLoadBalancerFailed 113s (x7 over 7m8s) service-controller Error syncing load balancer: failed to ensure load balancer: mixed protocol is not supported for LoadBalancer +``` +Solution: -```./validator-engine-console -k client -p server.pub -a :``` +Create static IP address of type Premium in GCP console and use it as a value for field ```loadBalancerIP``` in Kubernetes service. -IP:PORT is shown at start of container. +### Ali Cloud -#### Lite-client -To use lite-client you need to get liteserver.pub from container. +#### Validator logs always show +``` +Client got error [PosixError : Connection reset by peer : 104 : Error on [fd:45]] +[!NetworkManager][&ADNL_WARNING] [networkmanager]: received too small proxy packet of size 21 +``` +Solution: -```docker cp :/var/ton-work/db/liteserver.pub /your/path``` +The node is sychnronizing, but very slow though. +Try to use Network Load Balancer (NLB) instead of default CLB. -Then you can connect to it, but be sure you use right port, it's different from fullnode console port. -```lite-client -a : -p liteserver.pub``` diff --git a/docker/control.template b/docker/control.template index 857bcebcd..13b9b6b76 100644 --- a/docker/control.template +++ b/docker/control.template @@ -6,4 +6,4 @@ "permissions" : 15 } ] - } \ No newline at end of file + } diff --git a/docker/init.sh b/docker/init.sh index 1fe4f5e10..2d64690c5 100644 --- a/docker/init.sh +++ b/docker/init.sh @@ -1,30 +1,93 @@ #!/usr/bin/env bash -# global config -if [ ! -z "$GCONFURL" ]; then +if [ ! -z "$TEST" ]; then + echo -e "Running simple validator-engine test..." + validator-engine -h + test $? -eq 2 || { echo "simple validator-engine test failed"; exit 1; } + exit 0; +fi + +# global config +if [ ! -z "$GLOBAL_CONFIG_URL" ]; then echo -e "\e[1;32m[+]\e[0m Downloading provided global config." - wget -q $GCONFURL -O /var/ton-work/db/ton-global.config + wget -q $GLOBAL_CONFIG_URL -O /var/ton-work/db/ton-global.config else - echo -e "\e[1;33m[=]\e[0m No global config provided, downloading default." + echo -e "\e[1;33m[=]\e[0m No global config provided, downloading mainnet default." wget -q https://api.tontech.io/ton/wallet-mainnet.autoconf.json -O /var/ton-work/db/ton-global.config fi +if [ -z "$VALIDATOR_PORT" ]; then + VALIDATOR_PORT=30001 + echo -e "\e[1;33m[=]\e[0m Using default VALIDATOR_PORT $VALIDATOR_PORT udp" +else + echo -e "\e[1;33m[=]\e[0m Using VALIDATOR_PORT $VALIDATOR_PORT udp" +fi + # Init local config with IP:PORT if [ ! -z "$PUBLIC_IP" ]; then - if [ -z "$CONSOLE_PORT" ]; then - CONSOLE_PORT="43678" - fi - echo -e "\e[1;32m[+]\e[0m Using provided IP: $PUBLIC_IP:$CONSOLE_PORT" - validator-engine -C /var/ton-work/db/ton-global.config --db /var/ton-work/db --ip "$PUBLIC_IP:$CONSOLE_PORT" + echo -e "\e[1;32m[+]\e[0m Using provided IP: $PUBLIC_IP:$VALIDATOR_PORT" else - echo -e "\e[1;31m[!]\e[0m No IP:PORT provided, exiting" + echo -e "\e[1;31m[!]\e[0m No PUBLIC_IP provided, exiting..." exit 1 fi +if [ ! -f "/var/ton-work/db/config.json" ]; then + echo -e "\e[1;32m[+]\e[0m Initializing validator-engine:" + echo validator-engine -C /var/ton-work/db/ton-global.config --db /var/ton-work/db --ip "$PUBLIC_IP:$VALIDATOR_PORT" + validator-engine -C /var/ton-work/db/ton-global.config --db /var/ton-work/db --ip "$PUBLIC_IP:$VALIDATOR_PORT" + test $? -eq 0 || { echo "Cannot initialize validator-engine"; exit 2; } +fi + +if [ ! -z "$DUMP_URL" ]; then + echo -e "\e[1;32m[+]\e[0m Using provided dump $DUMP_URL" + if [ ! -f "dump_downloaded" ]; then + echo -e "\e[1;32m[+]\e[0m Downloading dump..." + curl --retry 10 --retry-delay 30 -Ls $DUMP_URL | pv | plzip -d -n8 | tar -xC /var/ton-work/db + touch dump_downloaded + else + echo -e "\e[1;32m[+]\e[0m Dump has been already used." + fi +fi + +if [ -z "$STATE_TTL" ]; then + STATE_TTL=86400 + echo -e "\e[1;33m[=]\e[0m Using default STATE_TTL $STATE_TTL" +else + echo -e "\e[1;33m[=]\e[0m Using STATE_TTL $STATE_TTL" +fi + +if [ -z "$ARCHIVE_TTL" ]; then + ARCHIVE_TTL=86400 + echo -e "\e[1;33m[=]\e[0m Using default ARCHIVE_TTL $ARCHIVE_TTL" +else + echo -e "\e[1;33m[=]\e[0m Using ARCHIVE_TTL $ARCHIVE_TTL" +fi + +if [ -z "$THREADS" ]; then + THREADS=8 + echo -e "\e[1;33m[=]\e[0m Using default THREADS $THREADS" +else + echo -e "\e[1;33m[=]\e[0m Using THREADS $THREADS" +fi + +if [ -z "$VERBOSITY" ]; then + VERBOSITY=3 + echo -e "\e[1;33m[=]\e[0m Using default VERBOSITY $VERBOSITY" +else + echo -e "\e[1;33m[=]\e[0m Using VERBOSITY $VERBOSITY" +fi + +if [ -z "$CONSOLE_PORT" ]; then + CONSOLE_PORT=30002 + echo -e "\e[1;33m[=]\e[0m Using default CONSOLE_PORT $CONSOLE_PORT tcp" +else + echo -e "\e[1;33m[=]\e[0m Using CONSOLE_PORT $CONSOLE_PORT tcp" +fi + # Generating server certificate if [ -f "./server" ]; then echo -e "\e[1;33m[=]\e[0m Found existing server certificate, skipping" -else +else echo -e "\e[1;32m[+]\e[0m Generating and installing server certificate for remote control" read -r SERVER_ID1 SERVER_ID2 <<< $(generate-random-id -m keys -n server) echo "Server IDs: $SERVER_ID1 $SERVER_ID2" @@ -32,16 +95,16 @@ else fi # Generating client certificate -if [ -f "./client" ]; then +if [ -f "./client" ]; then echo -e "\e[1;33m[=]\e[0m Found existing client certificate, skipping" else read -r CLIENT_ID1 CLIENT_ID2 <<< $(generate-random-id -m keys -n client) echo -e "\e[1;32m[+]\e[0m Generated client private certificate $CLIENT_ID1 $CLIENT_ID2" echo -e "\e[1;32m[+]\e[0m Generated client public certificate" # Adding client permissions - sed -e "s/CONSOLE-PORT/\"$(printf "%q" $CONSOLE_PORT)\"/g" -e "s~SERVER-ID~\"$(printf "%q" $SERVER_ID2)\"~g" -e "s~CLIENT-ID~\"$(printf "%q" $CLIENT_ID2)\"~g" control.template > control.new - sed -e "s~\"control\"\ \:\ \[~$(printf "%q" $(cat control.new))~g" config.json > config.json.new - mv config.json.new config.json + sed -e "s/CONSOLE-PORT/\"$(printf "%q" $CONSOLE_PORT)\"/g" -e "s~SERVER-ID~\"$(printf "%q" $SERVER_ID2)\"~g" -e "s~CLIENT-ID~\"$(printf "%q" $CLIENT_ID2)\"~g" /var/ton-work/scripts/control.template > control.new + sed -e "s~\"control\"\ \:\ \[~$(printf "%q" $(cat control.new))~g" /var/ton-work/db/config.json > config.json.new + mv config.json.new /var/ton-work/db/config.json fi # Liteserver @@ -50,20 +113,25 @@ if [ -z "$LITESERVER" ]; then else if [ -f "./liteserver" ]; then echo -e "\e[1;33m[=]\e[0m Found existing liteserver certificate, skipping" - else + else echo -e "\e[1;32m[+]\e[0m Generating and installing liteserver certificate for remote control" read -r LITESERVER_ID1 LITESERVER_ID2 <<< $(generate-random-id -m keys -n liteserver) echo "Liteserver IDs: $LITESERVER_ID1 $LITESERVER_ID2" cp liteserver /var/ton-work/db/keyring/$LITESERVER_ID1 + if [ -z "$LITE_PORT" ]; then - LITE_PORT="43679" + LITE_PORT=30003 + echo -e "\e[1;33m[=]\e[0m Using default LITE_PORT $LITE_PORT tcp" + else + echo -e "\e[1;33m[=]\e[0m Using LITE_PORT $LITE_PORT tcp" fi + LITESERVERS=$(printf "%q" "\"liteservers\":[{\"id\":\"$LITESERVER_ID2\",\"port\":\"$LITE_PORT\"}") - sed -e "s~\"liteservers\"\ \:\ \[~$LITESERVERS~g" config.json > config.json.liteservers - mv config.json.liteservers config.json + sed -e "s~\"liteservers\"\ \:\ \[~$LITESERVERS~g" /var/ton-work/db/config.json > config.json.liteservers + mv config.json.liteservers /var/ton-work/db/config.json fi fi -echo -e "\e[1;32m[+]\e[0m Running validator-engine" - -exec validator-engine -c /var/ton-work/db/config.json -C /var/ton-work/db/ton-global.config --db /var/ton-work/db \ No newline at end of file +echo -e "\e[1;32m[+]\e[0m Starting validator-engine:" +echo validator-engine -c /var/ton-work/db/config.json -C /var/ton-work/db/ton-global.config --db /var/ton-work/db --state-ttl $STATE_TTL --archive-ttl $ARCHIVE_TTL --threads $THREADS --verbosity $VERBOSITY $CUSTOM_ARG +exec validator-engine -c /var/ton-work/db/config.json -C /var/ton-work/db/ton-global.config --db /var/ton-work/db --state-ttl $STATE_TTL --archive-ttl $ARCHIVE_TTL --threads $THREADS --verbosity $VERBOSITY $CUSTOM_ARG diff --git a/docker/ton-ali.yaml b/docker/ton-ali.yaml new file mode 100644 index 000000000..03ffbdb0f --- /dev/null +++ b/docker/ton-ali.yaml @@ -0,0 +1,121 @@ +apiVersion: "apps/v1" +kind: StatefulSet +metadata: + name: validator-engine-pod + labels: + name: validator-engine-pod +spec: + volumeClaimTemplates: + - metadata: + name: validator-engine-pvc + spec: + storageClassName: alicloud-disk-ssd + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 800Gi + serviceName: validator-engine-srv-headless + replicas: 1 + selector: + matchLabels: + name: validator-engine-pod + template: + metadata: + labels: + name: validator-engine-pod + spec: + containers: + - name: validator-engine-container + image: ghcr.io/neodix42/ton:latest + env: + - name: PUBLIC_IP + value: "" + - name: GLOBAL_CONFIG_URL + value: "https://api.tontech.io/ton/wallet-mainnet.autoconf.json" + - name: DUMP_URL + value: "https://dump.ton.org/dumps/latest.tar.lz" + - name: LITESERVER + value: "true" + - name: VALIDATOR_PORT + value: "30001" + - name: CONSOLE_PORT + value: "30002" + - name: LITE_PORT + value: "30003" + - name: STATE_TTL + value: "86400" + - name: ARCHIVE_TTL + value: "86400" + - name: THREADS + value: "8" + - name: VERBOSITY + value: "3" + ports: + - containerPort: 30001 + protocol: UDP + - containerPort: 30002 + protocol: TCP + - containerPort: 30003 + protocol: TCP + volumeMounts: + - mountPath: "/var/ton-work/db" + name: validator-engine-pvc + resources: + requests: + memory: "64Gi" + cpu: "16" + limits: + memory: "128Gi" + cpu: "32" +--- +kind: Service +apiVersion: v1 +metadata: + name: validator-engine-srv + annotations: + service.beta.kubernetes.io/alibaba-cloud-loadbalancer-eip-ids: "" + service.beta.kubernetes.io/alibaba-cloud-loadbalancer-address-type: "intranet" +spec: + type: LoadBalancer + externalTrafficPolicy: Local + ports: + - name: validator-udp + nodePort: 30001 + port: 30001 + targetPort: 30001 + protocol: UDP + - name: console-tcp + nodePort: 30002 + port: 30002 + targetPort: 30002 + protocol: TCP + - name: ls-tcp + nodePort: 30003 + port: 30003 + targetPort: 30003 + protocol: TCP + selector: + name: validator-engine-pod +--- +apiVersion: v1 +kind: Service +metadata: + name: validator-engine-srv-headless +spec: + clusterIP: None + ports: + - name: validator-udp + port: 30001 + targetPort: 30001 + protocol: UDP + - name: console-tcp + port: 30002 + targetPort: 30002 + protocol: TCP + - name: ls-tcp + port: 30003 + targetPort: 30003 + protocol: TCP + selector: + name: validator-engine-pod diff --git a/docker/ton-aws.yaml b/docker/ton-aws.yaml new file mode 100644 index 000000000..da16cbae9 --- /dev/null +++ b/docker/ton-aws.yaml @@ -0,0 +1,122 @@ +apiVersion: "apps/v1" +kind: StatefulSet +metadata: + name: validator-engine-pod + labels: + name: validator-engine-pod +spec: + volumeClaimTemplates: + - metadata: + name: validator-engine-pvc + spec: + storageClassName: gp2 + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 800Gi + serviceName: validator-engine-srv-headless + replicas: 1 + selector: + matchLabels: + name: validator-engine-pod + template: + metadata: + labels: + name: validator-engine-pod + spec: + containers: + - name: validator-engine-container + image: ghcr.io/neodix42/ton:latest + env: + - name: PUBLIC_IP + value: "" + - name: GLOBAL_CONFIG_URL + value: "https://api.tontech.io/ton/wallet-mainnet.autoconf.json" + - name: DUMP_URL + value: "https://dump.ton.org/dumps/latest.tar.lz" + - name: LITESERVER + value: "true" + - name: VALIDATOR_PORT + value: "30001" + - name: CONSOLE_PORT + value: "30002" + - name: LITE_PORT + value: "30003" + - name: STATE_TTL + value: "86400" + - name: ARCHIVE_TTL + value: "86400" + - name: THREADS + value: "8" + - name: VERBOSITY + value: "3" + ports: + - containerPort: 30001 + protocol: UDP + - containerPort: 30002 + protocol: TCP + - containerPort: 30003 + protocol: TCP + volumeMounts: + - mountPath: "/var/ton-work/db" + name: validator-engine-pvc + resources: + requests: + memory: "64Gi" + cpu: "16" + limits: + memory: "128Gi" + cpu: "32" +--- +kind: Service +apiVersion: v1 +metadata: + name: validator-engine-srv + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: external + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing + service.beta.kubernetes.io/aws-load-balancer-eip-allocations: "" # Replace with your EIP allocation ID +spec: + type: LoadBalancer + ports: + - name: validator-udp + nodePort: 30001 + port: 30001 + targetPort: 30001 + protocol: UDP + - name: console-tcp + nodePort: 30002 + port: 30002 + targetPort: 30002 + protocol: TCP + - name: ls-tcp + nodePort: 30003 + port: 30003 + targetPort: 30003 + protocol: TCP + selector: + name: validator-engine-pod +--- +apiVersion: v1 +kind: Service +metadata: + name: validator-engine-srv-headless +spec: + clusterIP: None + ports: + - name: validator-udp + port: 30001 + targetPort: 30001 + protocol: UDP + - name: console-tcp + port: 30002 + targetPort: 30002 + protocol: TCP + - name: ls-tcp + port: 30003 + targetPort: 30003 + protocol: TCP + selector: + name: validator-engine-pod diff --git a/docker/ton-gcp.yaml b/docker/ton-gcp.yaml new file mode 100644 index 000000000..0ded5a794 --- /dev/null +++ b/docker/ton-gcp.yaml @@ -0,0 +1,134 @@ +apiVersion: "apps/v1" +kind: StatefulSet +metadata: + name: validator-engine-pod + labels: + name: validator-engine-pod +spec: + volumeClaimTemplates: + - metadata: + name: validator-engine-pvc + spec: + storageClassName: standard-rwo + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 800Gi + serviceName: validator-engine-srv-headless + replicas: 1 + selector: + matchLabels: + name: validator-engine-pod + template: + metadata: + labels: + name: validator-engine-pod + spec: + containers: + - name: validator-engine-container + image: ghcr.io/neodix42/ton:latest + env: + - name: PUBLIC_IP + value: "" + - name: GLOBAL_CONFIG_URL + value: "https://api.tontech.io/ton/wallet-mainnet.autoconf.json" + - name: DUMP_URL + value: "https://dump.ton.org/dumps/latest.tar.lz" + - name: LITESERVER + value: "true" + - name: VALIDATOR_PORT + value: "30001" + - name: CONSOLE_PORT + value: "30002" + - name: LITE_PORT + value: "30003" + - name: STATE_TTL + value: "86400" + - name: ARCHIVE_TTL + value: "86400" + - name: THREADS + value: "8" + - name: VERBOSITY + value: "3" + ports: + - containerPort: 30001 + protocol: UDP + - containerPort: 30002 + protocol: TCP + - containerPort: 30003 + protocol: TCP + volumeMounts: + - mountPath: "/var/ton-work/db" + name: validator-engine-pvc + resources: + requests: + memory: "64Gi" + cpu: "16" + limits: + memory: "128Gi" + cpu: "32" +--- +kind: Service +apiVersion: v1 +metadata: + name: validator-engine-srv +spec: + type: LoadBalancer + loadBalancerIP: + ports: + - port: 30001 + targetPort: 30001 + protocol: UDP + selector: + name: validator-engine-pod +--- +kind: Service +apiVersion: v1 +metadata: + name: validator-engine-console-srv +spec: + type: LoadBalancer + loadBalancerIP: + ports: + - port: 30002 + targetPort: 30002 + protocol: TCP + selector: + name: validator-engine-pod +--- +kind: Service +apiVersion: v1 +metadata: + name: lite-server-srv +spec: + type: LoadBalancer + loadBalancerIP: + ports: + - port: 30003 + targetPort: 30003 + protocol: TCP + selector: + name: validator-engine-pod +--- +apiVersion: v1 +kind: Service +metadata: + name: validator-engine-srv-headless +spec: + clusterIP: None + ports: + - name: validator-udp + port: 30001 + targetPort: 30001 + protocol: UDP + - name: console-tcp + port: 30002 + targetPort: 30002 + protocol: TCP + - name: ls-tcp + port: 30003 + targetPort: 30003 + protocol: TCP + selector: + name: validator-engine-pod diff --git a/docker/ton-metal-lb.yaml b/docker/ton-metal-lb.yaml new file mode 100644 index 000000000..ceaf3a7c0 --- /dev/null +++ b/docker/ton-metal-lb.yaml @@ -0,0 +1,118 @@ +apiVersion: v1 +kind: Pod +metadata: + name: validator-engine-pod + labels: + name: validator-engine-pod +spec: + volumes: + - name: validator-engine-pv + persistentVolumeClaim: + claimName: validator-engine-pvc + containers: + - name: validator-engine-container + image: ghcr.io/neodix42/ton:latest + env: + - name: PUBLIC_IP + value: "" + - name: GLOBAL_CONFIG_URL + value: "https://api.tontech.io/ton/wallet-mainnet.autoconf.json" + - name: DUMP_URL + value: "https://dump.ton.org/dumps/latest.tar.lz" + - name: LITESERVER + value: "true" + - name: VALIDATOR_PORT + value: "30001" + - name: CONSOLE_PORT + value: "30002" + - name: LITE_PORT + value: "30003" + - name: STATE_TTL + value: "86400" + - name: ARCHIVE_TTL + value: "86400" + - name: THREADS + value: "8" + - name: VERBOSITY + value: "3" + volumeMounts: + - mountPath: "/var/ton-work/db" + name: validator-engine-pv + resources: + requests: + memory: "64Gi" + cpu: "16" + limits: + memory: "128Gi" + cpu: "32" +--- +kind: Service +apiVersion: v1 +metadata: + name: validator-engine-srv + annotations: + metallb.universe.tf/address-pool: first-pool +spec: + type: LoadBalancer + ports: + - name: validator-engine-public-udp-port + nodePort: 30001 + port: 30001 + targetPort: 30001 + protocol: UDP + - name: validator-console-tcp-port + nodePort: 30002 + port: 30002 + targetPort: 30002 + protocol: TCP + - name: lite-server-tcp-port + nodePort: 30003 + port: 30003 + targetPort: 30003 + protocol: TCP + selector: + name: validator-engine-pod +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: local-storage +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: WaitForFirstConsumer +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: validator-engine-pv + labels: + type: local +spec: + storageClassName: local-storage + capacity: + storage: 800Gi + accessModes: + - ReadWriteOnce + - ReadOnlyMany + persistentVolumeReclaimPolicy: Retain + local: + path: + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: node_type + operator: In + values: + - ton-validator +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: validator-engine-pvc +spec: + storageClassName: local-storage + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 800Gi diff --git a/docker/ton-node-port.yaml b/docker/ton-node-port.yaml new file mode 100644 index 000000000..ec594031f --- /dev/null +++ b/docker/ton-node-port.yaml @@ -0,0 +1,126 @@ +apiVersion: v1 +kind: Pod +metadata: + name: validator-engine-pod + labels: + name: validator-engine-pod +spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node_type + operator: In + values: + - ton-validator + hostNetwork: true + volumes: + - name: validator-engine-pv + persistentVolumeClaim: + claimName: validator-engine-pvc + containers: + - name: validator-engine-container + image: ghcr.io/neodix42/ton:latest + env: + - name: PUBLIC_IP + value: "" + - name: GLOBAL_CONFIG_URL + value: "https://api.tontech.io/ton/wallet-mainnet.autoconf.json" + - name: DUMP_URL + value: "https://dump.ton.org/dumps/latest.tar.lz" + - name: LITESERVER + value: "true" + - name: VALIDATOR_PORT + value: "30001" + - name: CONSOLE_PORT + value: "30002" + - name: LITE_PORT + value: "30003" + - name: STATE_TTL + value: "86400" + - name: ARCHIVE_TTL + value: "86400" + - name: THREADS + value: "8" + - name: VERBOSITY + value: "3" + volumeMounts: + - mountPath: "/var/ton-work/db" + name: validator-engine-pv + resources: + requests: + memory: "64Gi" + cpu: "16" + limits: + memory: "128Gi" + cpu: "32" +--- +kind: Service +apiVersion: v1 +metadata: + name: validator-engine-srv +spec: + type: NodePort + ports: + - name: validator-engine-public-udp-port + nodePort: 30001 + port: 30001 + targetPort: 30001 + protocol: UDP + - name: validator-console-tcp-port + nodePort: 30002 + port: 30002 + targetPort: 30002 + protocol: TCP + - name: lite-server-tcp-port + nodePort: 30003 + port: 30003 + targetPort: 30003 + protocol: TCP + selector: + name: validator-engine-pod +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: local-storage +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: WaitForFirstConsumer +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: validator-engine-pv + labels: + type: local +spec: + storageClassName: local-storage + capacity: + storage: 800Gi + accessModes: + - ReadWriteOnce + - ReadOnlyMany + persistentVolumeReclaimPolicy: Retain + local: + path: + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: node_type + operator: In + values: + - ton-validator +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: validator-engine-pvc +spec: + storageClassName: local-storage + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 800Gi diff --git a/emulator/emulator-extern.cpp b/emulator/emulator-extern.cpp index 6d38ae59b..5710d4c8a 100644 --- a/emulator/emulator-extern.cpp +++ b/emulator/emulator-extern.cpp @@ -65,7 +65,18 @@ const char *external_not_accepted_response(std::string&& vm_log, int vm_exit_cod td::Result decode_config(const char* config_boc) { TRY_RESULT_PREFIX(config_params_cell, boc_b64_to_cell(config_boc), "Can't deserialize config params boc: "); - auto global_config = block::Config(config_params_cell, td::Bits256::zero(), block::Config::needWorkchainInfo | block::Config::needSpecialSmc | block::Config::needCapabilities); + auto config_dict = std::make_unique(config_params_cell, 32); + auto config_addr_cell = config_dict->lookup_ref(td::BitArray<32>::zero()); + if (config_addr_cell.is_null()) { + return td::Status::Error("Can't find config address (param 0) is missing in config params"); + } + auto config_addr_cs = vm::load_cell_slice(std::move(config_addr_cell)); + if (config_addr_cs.size() != 0x100) { + return td::Status::Error(PSLICE() << "configuration parameter 0 with config address has wrong size"); + } + ton::StdSmcAddress config_addr; + config_addr_cs.fetch_bits_to(config_addr); + auto global_config = block::Config(config_params_cell, std::move(config_addr), block::Config::needWorkchainInfo | block::Config::needSpecialSmc | block::Config::needCapabilities); TRY_STATUS_PREFIX(global_config.unpack(), "Can't unpack config params: "); return global_config; } @@ -76,8 +87,17 @@ void *transaction_emulator_create(const char *config_params_boc, int vm_log_verb LOG(ERROR) << global_config_res.move_as_error().message(); return nullptr; } + auto global_config = std::make_shared(global_config_res.move_as_ok()); + return new emulator::TransactionEmulator(std::move(global_config), vm_log_verbosity); +} - return new emulator::TransactionEmulator(global_config_res.move_as_ok(), vm_log_verbosity); +void *emulator_config_create(const char *config_params_boc) { + auto config = decode_config(config_params_boc); + if (config.is_error()) { + LOG(ERROR) << "Error decoding config: " << config.move_as_error(); + return nullptr; + } + return new block::Config(config.move_as_ok()); } const char *transaction_emulator_emulate_transaction(void *transaction_emulator, const char *shard_account_boc, const char *message_boc) { @@ -319,7 +339,21 @@ bool transaction_emulator_set_config(void *transaction_emulator, const char* con return false; } - emulator->set_config(global_config_res.move_as_ok()); + emulator->set_config(std::make_shared(global_config_res.move_as_ok())); + + return true; +} + +void config_deleter(block::Config* ptr) { + // We do not delete the config object, since ownership management is delegated to the caller +} + +bool transaction_emulator_set_config_object(void *transaction_emulator, void* config) { + auto emulator = static_cast(transaction_emulator); + + std::shared_ptr config_ptr(static_cast(config), config_deleter); + + emulator->set_config(config_ptr); return true; } @@ -461,6 +495,13 @@ bool tvm_emulator_set_c7(void *tvm_emulator, const char *address, uint32_t unixt return true; } +bool tvm_emulator_set_config_object(void* tvm_emulator, void* config) { + auto emulator = static_cast(tvm_emulator); + auto global_config = std::shared_ptr(static_cast(config), config_deleter); + emulator->set_config(global_config); + return true; +} + bool tvm_emulator_set_prev_blocks_info(void *tvm_emulator, const char* info_boc) { auto emulator = static_cast(tvm_emulator); @@ -672,3 +713,7 @@ const char *tvm_emulator_send_internal_message(void *tvm_emulator, const char *m void tvm_emulator_destroy(void *tvm_emulator) { delete static_cast(tvm_emulator); } + +void emulator_config_destroy(void *config) { + delete static_cast(config); +} diff --git a/emulator/emulator-extern.h b/emulator/emulator-extern.h index b418e5b0d..28d38d78d 100644 --- a/emulator/emulator-extern.h +++ b/emulator/emulator-extern.h @@ -16,6 +16,13 @@ extern "C" { */ EMULATOR_EXPORT void *transaction_emulator_create(const char *config_params_boc, int vm_log_verbosity); +/** + * @brief Creates Config object from base64 encoded BoC + * @param config_params_boc Base64 encoded BoC serialized Config dictionary (Hashmap 32 ^Cell) + * @return Pointer to Config object or nullptr in case of error + */ +EMULATOR_EXPORT void *emulator_config_create(const char *config_params_boc); + /** * @brief Set unixtime for emulation * @param transaction_emulator Pointer to TransactionEmulator object @@ -49,7 +56,7 @@ EMULATOR_EXPORT bool transaction_emulator_set_rand_seed(void *transaction_emulat EMULATOR_EXPORT bool transaction_emulator_set_ignore_chksig(void *transaction_emulator, bool ignore_chksig); /** - * @brief Set unixtime for emulation + * @brief Set config for emulation * @param transaction_emulator Pointer to TransactionEmulator object * @param config_boc Base64 encoded BoC serialized Config dictionary (Hashmap 32 ^Cell) * @return true in case of success, false in case of error @@ -57,7 +64,15 @@ EMULATOR_EXPORT bool transaction_emulator_set_ignore_chksig(void *transaction_em EMULATOR_EXPORT bool transaction_emulator_set_config(void *transaction_emulator, const char* config_boc); /** - * @brief Set unixtime for emulation + * @brief Set config for emulation + * @param transaction_emulator Pointer to TransactionEmulator object + * @param config Pointer to Config object + * @return true in case of success, false in case of error + */ +EMULATOR_EXPORT bool transaction_emulator_set_config_object(void *transaction_emulator, void* config); + +/** + * @brief Set libraries for emulation * @param transaction_emulator Pointer to TransactionEmulator object * @param libs_boc Base64 encoded BoC serialized shared libraries dictionary (HashmapE 256 ^Cell). * @return true in case of success, false in case of error @@ -167,6 +182,14 @@ EMULATOR_EXPORT bool tvm_emulator_set_libraries(void *tvm_emulator, const char * */ EMULATOR_EXPORT bool tvm_emulator_set_c7(void *tvm_emulator, const char *address, uint32_t unixtime, uint64_t balance, const char *rand_seed_hex, const char *config); +/** + * @brief Set config for TVM emulator + * @param tvm_emulator Pointer to TVM emulator + * @param config Pointer to Config object + * @return true in case of success, false in case of error + */ +EMULATOR_EXPORT bool tvm_emulator_set_config_object(void* tvm_emulator, void* config); + /** * @brief Set tuple of previous blocks (13th element of c7) * @param tvm_emulator Pointer to TVM emulator @@ -278,6 +301,13 @@ EMULATOR_EXPORT const char *tvm_emulator_send_internal_message(void *tvm_emulato */ EMULATOR_EXPORT void tvm_emulator_destroy(void *tvm_emulator); +/** + * @brief Destroy Config object + * @param tvm_emulator Pointer to Config object + */ +EMULATOR_EXPORT void emulator_config_destroy(void *config); + + #ifdef __cplusplus } // extern "C" #endif diff --git a/emulator/emulator_export_list b/emulator/emulator_export_list index 93f5dbac8..c719a393d 100644 --- a/emulator/emulator_export_list +++ b/emulator/emulator_export_list @@ -4,6 +4,7 @@ _transaction_emulator_set_lt _transaction_emulator_set_rand_seed _transaction_emulator_set_ignore_chksig _transaction_emulator_set_config +_transaction_emulator_set_config_object _transaction_emulator_set_libs _transaction_emulator_set_debug_enabled _transaction_emulator_set_prev_blocks_info @@ -11,9 +12,12 @@ _transaction_emulator_emulate_transaction _transaction_emulator_emulate_tick_tock_transaction _transaction_emulator_destroy _emulator_set_verbosity_level +_emulator_config_create +_emulator_config_destroy _tvm_emulator_create _tvm_emulator_set_libraries _tvm_emulator_set_c7 +_tvm_emulator_set_config_object _tvm_emulator_set_prev_blocks_info _tvm_emulator_set_gas_limit _tvm_emulator_set_debug_enabled diff --git a/emulator/test/emulator-tests.cpp b/emulator/test/emulator-tests.cpp new file mode 100644 index 000000000..0780be611 --- /dev/null +++ b/emulator/test/emulator-tests.cpp @@ -0,0 +1,400 @@ +#include "td/utils/tests.h" + +#include "block/block-auto.h" +#include "block/block.h" +#include "block/block-parse.h" + +#include "crypto/vm/boc.h" + +#include "td/utils/base64.h" +#include "td/utils/crypto.h" +#include "td/utils/JsonBuilder.h" +#include "td/utils/Time.h" + +#include "smc-envelope/WalletV3.h" + +#include "emulator/emulator-extern.h" + +// testnet config as of 27.06.24 +const char *config_boc = "te6cckICAl8AAQAANecAAAIBIAABAAICAtgAAwAEAgL1AA0ADgIBIAAFAAYCAUgCPgI/AgEgAAcACAIBSAAJAAoCASAAHgAfAgEgAGUAZgIBSAALAAwCAWoA0gDTAQFI" + "AJIBAUgAsgEDpDMADwIBbgAQABEAQDPAueB1cC0DTaIjG28I/scJsoxoIScEE9LNtuiQoYa2AgOuIAASABMBA7LwABoBASAAFAEBIAAYAQHAABUCAWoAFgAXAIm/VzGV" + "o387z8N7BhdH91LBHMMhBLu7nv21jwo9wtTSXQIBABvI0aFLnw2QbZgjMPCLRdtRHxhUyinQudg6sdiohIwgwCAAQ79oJ47o6vzJDO5wV60LQESEyBcI3zuSSKtFQIlz" + "hk86tAMBg+mbgbrrZVY0qEWL8HxF+gYzy9t5jLO50+QkJ2DWbWFHj0Qaw5TPlNDYOnY0A2VNeAnS9bZ98W8X7FTvgVqStlmABAAZAIOgCYiOTH0TnIIa0oSKjkT3CsgH" + "NUU1Iy/5E472ortANeCAAAAAAAAAAAAAAAAROiXXYZuWf8AAi5Oy+xV/i+2JL9ABA6BgABsCASAAHAAdAFur4AAAAAAHGv1JjQAAEeDul1fav9HZ8+939/IsLGZ46E5h" + "3qjR13yIrB8mcfbBAFur/////8AHGv1JjQAAEeDul1fav9HZ8+939/IsLGZ46E5h3qjR13yIrB8mcfbBAgEgACAAIQIBIAAzADQCASAAIgAjAgEgACkAKgIBIAAkACUB" + "AUgAKAEBIAAmAQEgACcAQFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVAEAzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMwBAAQEBAQEBAQEBAQEB" + "AQEBAQEBAQEBAQEBAQEBAQEBAQECASAAKwAsAQFYAC8BASAALQEBIAAuAEDv5x0Thgr6pq6ur2NvkWhIf4DxAxsL+Nk5rknT6n99oABTAf//////////////////////" + "////////////////////gAAAAIAAAAFAAQHAADACASAAMQAyABW+AAADvLNnDcFVUAAVv////7y9GpSiABACASAANQA2AgEgADcAOAIBIABCAEMCASAATgBPAgEgADkA" + "OgIBIAA+AD8BASAAOwEBIAA9AQHAADwAt9BTLudOzwABAnAAKtiftocOhhpk4QsHt8jHSWwV/O7nxvFyZKUf75zoqiN3Bfb/JZk7D9mvTw7EDHU5BlaNBz2ml2s54kRz" + "l0iBoQAAAAAP////+AAAAAAAAAAEABMaQ7msoAEBIB9IAQEgAEABASAAQQAUa0ZVPxAEO5rKAAAgAAAcIAAACWAAAAC0AAADhAEBIABEAQEgAEUAGsQAAAAGAAAAAAAA" + "AC4CA81AAEYARwIBIABVAEgAA6igAgEgAEkASgIBIABLAEwCASAATQBdAgEgAFsAXgIBIABbAFsCAUgAYQBhAQEgAFABASAAYgIBIABRAFICAtkAUwBUAgm3///wYABf" + "AGACASAAVQBWAgFiAFwAXQIBIABgAFcCAc4AYQBhAgEgAFgAWQIBIABaAF4CASAAXgBbAAFYAgEgAGEAYQIBIABeAF4AAdQAAUgAAfwCAdQAYQBhAAEgAgKRAGMAZAAq" + "NgIGAgUAD0JAAJiWgAAAAAEAAAH0ACo2BAcDBQBMS0ABMS0AAAAAAgAAA+gCASAAZwBoAgEgAHoAewIBIABpAGoCASAAcABxAgEgAGsAbAEBSABvAQEgAG0BASAAbgAM" + "AB4AHgADADFgkYTnKgAHEcN5N+CAAGteYg9IAAAB4AAIAE3QZgAAAAAAAAAAAAAAAIAAAAAAAAD6AAAAAAAAAfQAAAAAAAPQkEACASAAcgBzAgEgAHYAdwEBIAB0AQEg" + "AHUAlNEAAAAAAAAAZAAAAAAAD0JA3gAAAAAnEAAAAAAAAAAPQkAAAAAAAhYOwAAAAAAAACcQAAAAAAAmJaAAAAAABfXhAAAAAAA7msoAAJTRAAAAAAAAAGQAAAAAAACc" + "QN4AAAAAAZAAAAAAAAAAD0JAAAAAAAAPQkAAAAAAAAAnEAAAAAAAmJaAAAAAAAX14QAAAAAAO5rKAAEBIAB4AQEgAHkAUF3DAAIAAAAIAAAAEAAAwwAATiAAAYagAAJJ" + "8MMAAAPoAAATiAAAJxAAUF3DAAIAAAAIAAAAEAAAwwAehIAAmJaAATEtAMMAAABkAAATiAAAJxACAUgAfAB9AgEgAIAAgQEBIAB+AQEgAH8AQuoAAAAAAJiWgAAAAAAn" + "EAAAAAAAD0JAAAAAAYAAVVVVVQBC6gAAAAAABhqAAAAAAAGQAAAAAAAAnEAAAAABgABVVVVVAgEgAIIAgwEBWACGAQEgAIQBASAAhQAkwgEAAAD6AAAA+gAAA+gAAAAP" + "AErZAQMAAAfQAAA+gAAAAAMAAAAIAAAABAAgAAAAIAAAAAQAACcQAQHAAIcCASAAiACJAgFIAIoAiwIBagCQAJEAA9+wAgFYAIwAjQIBIACOAI8AQb7c3f6FapnFy4B4" + "QZnAdwvqMfKODXM49zeESA3vRM2QFABBvrMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM4AEG+tWede5qpBXVOzaq9SvpqBpwzTJ067Hk01rWZxT5wQ7gAQb8a" + "Yme1MOiTF+EsYXWNG8wYLwlq/ZXmR6g2PgSXaPOEegBBvzSTEofK4j4twU1E7XMbFoxvESypy3LTYwDOK8PDTfsWASsSZn08y2Z9WOsAEAAQD/////////3AAJMCAswA" + "lACVAgEgAJYAlwIBIACkAKUCASAAmACZAgEgAJ4AnwIBIACaAJsCASAAnACdAJsc46BJ4rulpzksHMZaJjfdtBExV1HRdikp9U7VlmJllrEaW2TYAFmAXnBlZIRH4Sqp" + "CbKkE6v60jyawOEYfVWJDgHg5kDaLMWq7kWQy6AAmxzjoEniuRloX7kgG9FNmRyw/AB/KERuToZdY5v8AHv9JJ8bCIKAWYBecGVkhEt/mk7tOEXbKUWuqIz/1NliY9sm" + "KNHFQimyb79WXudTIACbHOOgSeK0/SaSD6j2aEnWfmW/B7LOQBq2QiiBlnaLIzfq+J2HM0BZgF5wZWSEWPYUSh0McOyjsLL8prcsF5RNab+7jLN/5bOme1r98c8gAJsc" + "46BJ4rT4ptGRb52wRyHzhe/A8y/IQOC/W5R5aC6/l1IM4f/EgFmAXnBlZIRmDW7+WN70SpQsfX5DetODFOpW6zjCBx7cDf6E+rEipKACASAAoAChAgEgAKIAowCbHOOg" + "SeKqqZCAjJ16vfAa2GI9Dcp/I9zBTG2CwPqbx22lq00uLoBZgF5wZWSETeqWp7jqIGPuCYnPZSlQ1fMuSS4e1gF/i9uIeD8GEkNgAJsc46BJ4rugeQAFCtwRUJhvWRbx" + "smlpXTdXCio8SJSBdH/6VPCkAFmAXnBlZIRQPeE6JpjzEwkPI2mvCM1sDTcny96f2dhZ2DcBQmmywCAAmxzjoEnimDpTGClVkh/V+/mJmKVKEpdp4MvFgP5onw6saJRD" + "QApAWYBecGVkhElWAHSIgIhlXt+lUyQjmndd50temeILBd7WJwjjWBeIIACbHOOgSeKtcjPEr2gq3gMraY11K9Ikv1SPcVaj3veDWrY1o4nxKcBZgF5wZWSEabqKQLtX" + "PIkaYDaKvupB8EOxFDWpuMaJJVqafjw4h4sgAgEgAKYApwIBIACsAK0CASAAqACpAgEgAKoAqwCbHOOgSeK8POt5lMj96a3WrXWw7peFtWWh5oi9wsZqXRsrnHM4eoBZ" + "gF5wZWSEXlJk0ILG3LG9zsmxXf+r2OTayqr9FSKLBt9LJAow+aBgAJsc46BJ4qjb23m1w/0EvFl179XCQUUMk32z0kjSh+t6V2jnnqeFwFmAXnBlZIR2KWk8cqZgC06K" + "AphhfzE3VceQWtppAGEbybk06szO9KAAmxzjoEnihVEG74vb19K1l5o8WtWa0dH/gTPfytoA1LsVXR3ztfgAWYBecGVkhEVHN0AzKnDpKLX5P7Tnay/Ogc4rxeoks/yh" + "U3aWhEnGIACbHOOgSeKNl8PpsnZjGIy1CTzi01K8MhvQAEhGlzUDwj2ACC/yFUALGRulQuFOdHw2ulDcYktF860U0mFOYFaQPC7MVNbEeSsk45C9tSPgAgEgAK4ArwIB" + "IACwALEAmxzjoEnivAzuiTw+hkcXtw4XyJGYavfPayk6ehceV8FqrxrzKbQACMou1fGNuRpwF6ilPaS03+BSsz0YID1gpIkGozQp7gRFcQsyZFvVYACbHOOgSeKsoYF9" + "T9f0ArrtFxbViCRmpw2DsDzrllY35uHzP9DEosAICQwVUUQOx01jZ84Uy8ccqQ90Ml6tj5Sw14wOK055ds2sYSPy532gAJsc46BJ4piyhqkrUrk/KUOony6llV0S+DnZ" + "xDLdccZzKJ7bV+XiAAeBJKPSjdajMGMdZwRvewwnwsyc/7uHN718Pd8cHn7VQG1i9BJSeaAAmxzjoEnihY8aTVKeJnW4JHbfVPfkJwElQXxxqG94pNWmN6n9I5jABA51" + "90xtZChBtmQcmPHlOmtU6aLeZ+HBY7/jW6AMz26cNcymYyIuIAErEmZ9WOtmfXULABAAEA/////////3wACzAgLMALQAtQIBIAC2ALcCASAAxADFAgEgALgAuQIBIAC+" + "AL8CASAAugC7AgEgALwAvQCbHOOgSeK5Nyl3TF7AOD2UwhNOh+y3h9P5e0emd2zjffbNatQR1EBS4qdSDsPAZjIVSudNcsvyCAIbiOyNPYmj/MJG5lMjVLkYt4TIEDCg" + "AJsc46BJ4q0qr9PzfnnT+A41FG5Owo+9L+LsuT6PrQkuoR7XsLMzgFLioMqMr4sLf5pO7ThF2ylFrqiM/9TZYmPbJijRxUIpsm+/Vl7nUyAAmxzjoEnisgCK09re8agW" + "Ee8S6q329jm1WbZoHBHjO9oP0q3qItiAUuKgyoyviwfhKqkJsqQTq/rSPJrA4Rh9VYkOAeDmQNosxaruRZDLoACbHOOgSeKeKPVNUBZ96hhTOP8lp1kiAm2wfuT0HIxn" + "lw/0cyISP8BS4qDKjK+LGPYUSh0McOyjsLL8prcsF5RNab+7jLN/5bOme1r98c8gAgEgAMAAwQIBIADCAMMAmxzjoEnip+PTCe8vsapzyPHm88uO5qKBwt9yvn+S6aJW" + "OlcBqeDAUuKgyoyviyYNbv5Y3vRKlCx9fkN604MU6lbrOMIHHtwN/oT6sSKkoACbHOOgSeKwOTDV9phg7jYWvy7bbTD8N773bX9y1P7lxC7vtvdbvsBS4qDKjK+LDeqW" + "p7jqIGPuCYnPZSlQ1fMuSS4e1gF/i9uIeD8GEkNgAJsc46BJ4opGGis7tEqqLAW2742I2ugw5S5lFxeYpc4D9f/qbOMhwFLioMqMr4sQPeE6JpjzEwkPI2mvCM1sDTcn" + "y96f2dhZ2DcBQmmywCAAmxzjoEniqGUvGQXdvzVXTq/g3DpDkom5aqVipETXzq2o+FZdGDfAUuKgyoyviwlWAHSIgIhlXt+lUyQjmndd50temeILBd7WJwjjWBeIIAIB" + "IADGAMcCASAAzADNAgEgAMgAyQIBIADKAMsAmxzjoEnihA6ouVC73YehzpHoNBKL8q3Gp4YbwxOBhJdxpNWePHwAUuKgyoyviym6ikC7VzyJGmA2ir7qQfBDsRQ1qbjG" + "iSVamn48OIeLIACbHOOgSeKr2ACjLl9IlajrtDqvMLD+lfOMRQvmZAaL2NVDooVPYQBS4qDKjK+LHlJk0ILG3LG9zsmxXf+r2OTayqr9FSKLBt9LJAow+aBgAJsc46BJ" + "4oohDH+XJf2EoPKNkp+gv/WG2UonjUWXV+B/IvWUldUuQFLioMqMr4s2KWk8cqZgC06KAphhfzE3VceQWtppAGEbybk06szO9KAAmxzjoEnilP2IvoMbkK7LwTeBBX8u" + "dYI608SRo4nDIg7XUWQf2CYAUuKgyoyviwVHN0AzKnDpKLX5P7Tnay/Ogc4rxeoks/yhU3aWhEnGIAIBIADOAM8CASAA0ADRAJsc46BJ4qS3beCYCuu47Ohag9xU5wk6" + "/1uLtI/5NZ+VaqSyKsGdAApHFgZLFGK0fDa6UNxiS0XzrRTSYU5gVpA8LsxU1sR5KyTjkL21I+AAmxzjoEnivJI7eg6kFGx7dvMX7Xzoog/s5cwHxrcfec5z8/aP/8kA" + "CFtq86KYH4dNY2fOFMvHHKkPdDJerY+UsNeMDitOeXbNrGEj8ud9oACbHOOgSeKlwkl68jfkl6kGCq/tElh6bM85sFBPnt7exnkRJq68iQAG+mnlyjEXYzBjHWcEb3sM" + "J8LMnP+7hze9fD3fHB5+1UBtYvQSUnmgAJsc46BJ4oYswn2e5gWf+Va6NJ+K8sfz4qIHmVG2ryktqCkE9P8hQAPDhRot06toQbZkHJjx5TprVOmi3mfhwWO/41ugDM9u" + "nDXMpmMiLiABASAA1AEBIAD6AQsAtb0+sEAA1QIBIADWANcCA8H4ANgA2QID4fgA+AD5AgEgAPwA/QIBIADaANsCASAA3ADdAgEgAbgBuQIBIAGQAZECASAA3gDfAgEg" + "AOAA4QIBIADqAOsAQb7edpH5xbuqiZNqTG9H7flTOIfNiYtDxI5AH4T6G4tcVAIBIADiAOMAQb6U4RvTn2B6e+8nmlEv/eZoRz1YKr3qyDudETjcrMFgKAIBIADkAOUC" + "ASAA5gDnAgEgAOgA6QBBvgukN4cHaqlFuawJv/TGaxhU3HU2B5iu8cZPVMOseQOgAEG+K7U1xAKEqaBEZoqjpyAnvSx8Z9jfPTeAR/anR5axvmAAQb4tEpbKJaulevOY" + "XQPqlmgiMgHDU6C6X7KRxpFyzPf0YABBvjbzLj0Z1oudyhyW/QhJ0OUxRj9zEM8Y1YUI9Py3ga6gAgFqAOwA7QIBIADuAO8AQb4JmTypqySHVMVJMHWspb3xrs2Lrdy4" + "eJ+M7QxpbS4cIABBvgOb8O+4IZEUWqtnRGQ8JpMkMBocpZyk/do3d/9MYnVgAgEgAPAA8QBBvqQeZ13QP0lszxNKt380fCWuaV94vwC/bfuqmrlg1/fIAgEgAPIA8wIB" + "IAD0APUAQb4G2ph6AS/mD/+cIv4aIYm1z5jAgCW/TTDEr72ygXOP4ABBvhBZkdUWyc1zdg9Fhp9QSsWD+LSyXChKLJOiMF3rVNqgAgEgAPYA9wBBvhsYuojZc90oYnM2" + "WQ+c6cHdiTDRBD2UgxkJlbkZa+mgAEG9wBVbqgGsx1Pog5dkmDyUl4VIe1ZME2BEDY6zMNoQYsAAQb3R4obtqmXfb1H2NxdElqeDuWD4d+Y73ozNJ7dE4jGfQAIBIAHw" + "AfECASACGAIZAQPAwAD7AFWgESjR4FjxyuEAXHMvOQot+HG+D9TtSQavwKbeV09n3G92AAAAAAAAAH0QAgEgAP4A/wIBIAEcAR0CASABAAEBAgEgAR4BHwIBIAECAQMC" + "ASABEAERAgEgAQQBBQIBIAEIAQkCAWIBBgEHAEG+tp/96j2CYcuIRGkfljl5uv/Pilfg3KwCY8xwdr1JdqgAA97wAEG99o5GkuI7pwd5/g4Lt+avHh31l5WoNTndbJgd" + "dTJBicACAUgBCgELAgEgAQwBDQBBvgIKjJdXg0pHrRIfDgYLQ20dIU6mEbDa1FxtUXy9B6rgAEG+Cev2EcR/qY3lMYZ3tIojHR5s+wWySfwNg7XZgP23waACASABDgEP" + "AEG+fZGfOd+cHGx01cd8+xQAwUjfI/VrANsfVPw1jZFJhTAAQb4y2lPdHZUPm695Z+bh0Z1dcta4xXX7fl6dlc2SXOliIABBvhfW5EoZl/I8jARohetHRk6pp1y3mrXR" + "28rFYjHHtJCgAgFqARIBEwIBIAEUARUAQb4zE+Nef80O9dLZy91HfPiOb6EEQ8YqyWKyIU+KeaYLIABBvgPcWeL0jqPxd5IiX7AAYESGqFqZ7o60BjQZJwpPQP1gAgEg" + "ARYBFwBBvofANH7PG2eeTdX5Vr2ZUebxCfwJyzBCE4oriUVRU3jIAgEgARgBGQIBIAEaARsAQb4btDCZEGRAOXaB6WwVqFzYTd1zZgyp15BIuy9n029k4ABBvimf97Kd" + "WV/siLZ3qM/+nVRE+t0X0XdLsOK51DJ6WSPgAEG+CQrglDQDcC3b6lTaIr2tVPRR4RlxVAwxYNcF+6BkvaAAQb4mML93xvUT+iBDJrOfhiRGSs3vOczEy9DJAbuCb7aU" + "4AIBIAFAAUECASABYAFhAgEgASABIQIBIAE0ATUCASABIgEjAgFYAS4BLwIBIAEkASUAQb6L1UE7T5lmGOuEiyPgykuqAW0ENCaxjsi4fdzZq2D0GAICcAEmAScCASAB" + "KAEpAD+9QolK/7nMhu3MO9bzK31P7DqSFoQkLyeYP3RWz5f3KwA/vVaiOV3iXF+2BW0R7uGwqmnXP7y0cjEHibQT6v4MssECASABKgErAgV/rWABLAEtAEG96YUi7d3r" + "hTwVGwv/pocif6dNQ6DcZ3JVzvqdhFltQ0AAQb3zT7C1dlWQlR1QmfrLfaGi5Sj94Guq/gLQXakuFmoVwAA/u8n6yK+GpbUUdG9dja4DHHLGGEu5ZXb6rUHFOFMS7kAA" + "P7v3dUiUhgaZGC+mdUGyJEzagm0IMNe3d2Q1lCRBTK5AAEG+co6LJmQv3h46OSV3KsT2gWyv6MLPKOrfIXFt86dsXVACASABMAExAEG+KQF+kzAAZybpH/1z1zYof09W" + "YAAY6MbQHDj3AO9dCGACASABMgEzAEG9xJZFhUbajV1FgRPu0X8LSHY3DIBRmI4wC6uLpNG5lkAAQb3/+UXNzozn7Eb1PsCLs8NaD2VhG+9qBBlvLJG76KkTQAIBIAE2" + "ATcCASABPgE/AgEgATgBOQIBYgE8AT0AQb5l6UC6/ZmwRTHlWwthzsJcYx+8Vj2vmom9/nu617FmkAIBIAE6ATsAQb4J64Df7Vfb8/jmlGnsZByGAdCsEWA/FfWXyVEU" + "5d6CoABBvhv0Q/VEAfHxjnYRJRxb6xtGetqoO1OgjstzC/3Ok41gAEG964EWqVOQS0JWHUcxnAz6STWs7+BsROmocJCo+xmqe0AAQb3vR9oRALXcwLQPRb70F/gP7SAV" + "WqyMgCIasOqw+b47wABBvpbvxWd5+q2vJUVqR9AlbEIfdFysLR0PXGgVlBf8x5hYAEG+j9bgcxjKxRmfMrJEC6BbHTCQ+WNXqC3H+z591gZw0AgCASABQgFDAgEgAUgB" + "SQIBSAFEAUUAQb7KkreZXaSZXSPGxbgwuJddzpWJly3MFNYwALkyQcIdDABBvnLW0BTZocy0D6h48ehPtgqA0XqNxrqB86bTTks9uvuQAgEgAUYBRwBBvjYzcOXWIfyk" + "HqSDt3m92Hacz/XRoWD5F4yy0AQ/E0ogAEG+AShOVhiiJZ6Itzjs8O75CiiF+eXloz74MSVsHpPAMiACASABSgFLAgEgAVABUQIDeuABTAFNAgFYAU4BTwA/vVuDIbt9" + "1w2Z2FpLSOsyAUPo2ovei28SxaHKDSUdRz0AP71qm4D4evL40x1qJi6AGLh6oOBtxFr5bgc8Xr8jaeWRAEG+HzK7ymUhDh5PL//pLHqwaYidq3sym7hIWC32Rqol+mAA" + "Qb41DOvSox2jnjN40ZFtUSQhSJMCyEWhBRdRERRSltibIAIBIAFSAVMCASABWAFZAgFYAVQBVQIBIAFWAVcAQb3cHJ+brtBSsROnSioWNJqFxZ+5hIGX7ta5KuhleBFn" + "wABBvf/lQA5TJrGDmv6EqacNl5j6ktTzbQOEGqpl45xcekNAAEG+Nve9GdRJhn/t0fgYe7d1pkTBxa2AfiXcWeRYqE1K3yAAQb4jrXHoxDyh1ZYGBdBoQgLaScxW6pZR" + "1hEhJC8BqF+5IAIBIAFaAVsCAVgBXgFfAEG+CdErMSfFYmEK9J9XimJDXyszQjtVELtHIXQt7AvQjKACAUgBXAFdAEC9ivFB4bA7PAP0VXnTs784TO/4CoWLb1QqRdyr" + "0orLAgBAvb5z8xm2yt/HlB1G9TB2Qna4rVgzGxI/n4z3UYr3a7gAQb3f0PQO3/nU5ypuXD5/SaZboj2RhZjd5z47o7VM8AjDwABBvfGIqWXxgi7mCltWrYf4pQa2aRZP" + "FvMA8LBV1hmpauDAAgEgAWIBYwIBIAGAAYECASABZAFlAgEgAXIBcwIBIAFmAWcCAVgBcAFxAgFIAWgBaQIBIAFqAWsAQb33dj2qlHUSOf2DkiVrVwhcqy3SkE9YbBfn" + "zU07vK+uwABBvdxiQ8Yt/Lb9BztkNe9dyXuUyTOcKJRlF9BteI2LK99AAgEgAWwBbQBBvjxAsXZAtTQoMwJV27nrzNCyFum1aU1fbygeFMFuYX9gAgFIAW4BbwBBvdro" + "odCnIayUb5VXYFh23qJGAE4Oed7iqqU/L0iFAPpAAD+9QlUpU0rFnXRmWi3ZnIsFtIIm3JDSdtVPEGqGefBt/wA/vWGl+1GrGASEj3GaAizvMOXDl69yZpcU2YUtCHfG" + "jLUAQb4d/oR88TrfAGcKrMn44T3wBnbh3TWVQWr8rVq0bYTnYABBvhpY6fA3+apwMQXdpEMu8s8uFXf+625mtfciMt0dh4LgAgEgAXQBdQIBIAF4AXkAQb5d0CvPvsyC" + "ZxuTbUe5O2PtTudCwtgc3Ou4DMuX2WizEAIBSAF2AXcAQb3BrlEdo+Hw0uZZJxCgCdxWs/njs6bTHuprY7HtqNl0QABBvcSsc0L20So00ByQZ2oo0aUWf4BlreuHcpYk" + "R/C5Av7AAgEgAXoBewIBIAF+AX8CASABfAF9AEG+ErNElODwkPB+KvEKqCtCz8CS5HCcsC8/VoJGV5f0+uAAQb3FCW/Cy20jtvAS0j4k9eQvRg9tcpaQgFnHc5cB7Fdv" + "wABBvc5nMn9h2c6FeqzonvA74SwaTxZXTgLEXOKOIFOki9BAAEG+NkNRDvICKDQNaqBlpx1LnSn5qpShA00BPg8Tfv+LHaAAQb4+0zsN9j+Lxs1EvbGG0fMwbeeqbWlx" + "TzyjV4LE+0uJYAIBIAGCAYMCAUgBigGLAgEgAYQBhQIBIAGGAYcAQb5O+6O6Y7dWb4HOnMBK4fZ7QNo9woEzBIeKd5+K08xlkABBvlwlLor18dZ5/O3AomXxI5hxYM4o" + "J1Xrrx0JChLVxHpQAgFYAYgBiQBBvn9hAM+g43TTR8vOvZfnhX3kPBCgPp3T0+YF+Ai6RFHwAEG99KmZCgwzysLzIR2TNaJdbyX4lKduOMlCmhCp4L9gJEAAQb3Ntnmm" + "W4yzmAdiAYg7sNjoD8sCiWIvgvkpuYpTXcyiQAIBZgGMAY0CAW4BjgGPAEC9hzviVxD170gIZfsWPGFKfbOB6LCP5YhH7I7fWz7wdwBAvaey9kbu3gkPDYYEraB8b3sF" + "UrCgg4ask3C+O8UJ1mkAQL2wAL6FGQaCTbDdEwGUJ82TDpVMLoNr4ZGZWxcofghZAEC9lqzgehIXoMRj58vAWaHnNAi6UXEU5Ce942dJqf4HawIBIAGSAZMCASABqAGp" + "AgEgAZQBlQIBIAGkAaUCASABlgGXAgEgAZwBnQIBagGYAZkCASABmgGbAEC9syAieemf3vF3umY0lCaQxLhwvbTFuL8eQxPYrpeZ8ABAvbl6reyIsCKH2fq2I8+oEnkS" + "4xYy3RUH/7ka152WrisAQb4CJHgAcs+wQzgf/9IPKdknw/ej0Z+Q+n3BtSEKi0hIoABBvgqovnD/owP5nsA4G62765H5klOyA1TV+7jriGf2CtjgAgFYAZ4BnwIBIAGg" + "AaEAQb3dAG8Nta3/iYiTymgGxV0CfKQlN6UlidHeNgbvtMT9wABBve7An2cFgShRoZx3xA7hUDRtwbcLae0x4dPQQlAH8o3AAEG+HDeG9ZNvkzq3wDDpGt0cb5cHHFQ0" + "itHD3s5R2YHy8eACAWIBogGjAD+9ewqjet2JVaCzHa8NXfnW3ZtLEzEASpk9eicyztCrvwA/vXDzaFNMjF1BnqMojulsIHfT2Dj1ltCTVvoe8wu+GKcCASABpgGnAEG+" + "un2oV7CbmRhYGc7tLiCXj/L40+4ZlzvlmEnZPxyuQrgAQb5ElmikSUchX0lT+0ASVhwF0OBnUB8X4TD4m4/v2Dfl0ABBvlBR7mcUQO8IfN+DkkDYHF1reSJZhv08w6k+" + "JIA6ITiwAgEgAaoBqwIBIAG0AbUCAVgBrAGtAgEgAbIBswBBvhX0m4apMW/GEDxtnd+z0ug75voHd+OibSQbA2+tUPigAgEgAa4BrwIBWAGwAbEAQb3WKikPb9a/J2ti" + "V6yOhNUW5BivimV3gM+EI3VAxst6QAA/vUeSH4ZL+7V8eQBEF/0lm/ouIJ+wQs5QTzBpsSHSXLcAP71t4YT+jYHLpx5Gv3HFoOzL5rhg0Ukud8G3adF8AYlRAEG+Zf0n" + "TrwaPPTPlLjegNsGkoz7UV5wz7oYQet9+SNmRfAAQb5m0tqyXFYp4ntucDLTwJV1gxwoh6JoJL1Y0rfwfLQhUABBvqSCHVak+jIc9ANutTAfHpZNM3YdGky7yaDzsTrg" + "0WhIAgN9eAG2AbcAP70AGCAXHtaQJNqiST0rNTs8mUZSo5H6vM7gvA+3q7+iAD+9FgzFlOZUrfRtonCQzjDSFzrRv4l/94TFs9oi+RQ6kgIBIAG6AbsCASAB1gHXAgEg" + "AbwBvQIBIAHKAcsCASABvgG/AgEgAcQBxQBBvqg93lUVxmlCEks5kL8jTFcqg8lElfAi8dSee8j2jFDIAgEgAcABwQICcwHCAcMAQb5gqEQiOqBKE6++9fJCR6LRVtNC" + "cE9MFknXFlF0leXQMAA/vWDgwPyHRVDvZl2iYgjJ3nWePRW2wjoUWAxrbgzB5a8AP71vi5ua8R9Xas7ZJOxnHw9u9q/5yyOmKiac4YXhpzZdAEG+s1A7ERdFjokIunFC" + "SgeOxki+V8FwbGaF2nFzHDuF3TgCASABxgHHAEG+VoZmB1FqSlGFLPm5r9LBLAX67F6BFQLDlwahNArjz1ACAnIByAHJAD+9QiJtY3MezTL7KB0xvFikeKH4EL/XSXL0" + "b7P1FoVCXwA/vWinW8a2SNxgyMi+e0ML00BiBRy4kZh/JQrAHMZZ3Y0CASABzAHNAgEgAdIB0wIBWAHOAc8CBX+rYAHQAdEAQb4MUGwt25IQd3/yHjI03F71G8Kp2GMa" + "MEv2TiWoTKbs4ABBvjfgYNaJyJijra4RuhLyyPeGUpRcBZhwzdStzQ2MIyDgAD+8XsswC94XkGKDsoUR3B73WxXRX2LdrWSok77uwX/c8AA/vF/xbT+aFbepxFKzgZQ9" + "HbF9uy1KEVspm2/20klhldAAQb6ORoMEHrkmcAR+9ntDkAj0Hq6gLGUT0ceglU8Tm9jfuAIBIAHUAdUAQb5A/TMaqnaKx2BBvcxafTpwUxZYRXcKXTAZj80OapRScABB" + "vm8iGJqmHDhbx34EGjoh2YHhU4mpC/HVkmnz7NBQA0LwAgEgAdgB2QIBIAHmAecCASAB2gHbAgEgAd4B3wIDeqAB3AHdAEG+rC9orZ39Jto92k4zrR5989Z4qySyANXA" + "U8TLG5+0zfgAP71bgmShTXyEATbw0sECEmtwNtuzKI+S3DHEAPCPRhvTAD+9YC74p2ZuEIcz5A4sE69a7MTFuARvrmQnzUDgc7Mo3QIBIAHgAeECA3jgAeQB5QBBvlnO" + "v0cNQ7XgFJEwo9boghCVUHzfZ+urQtJh6esRW5xQAgFqAeIB4wBAvYY1sTf2ZnuWrkRZ+aijWbaH+q5ZMHkghn/Ys+tCZhoAQL2mLfoqMZw77ln7oAn0Cna+Bkp/snNw" + "xHgR2MTl/uqVAD+9XiSecyAvpnbNK3Z28HAfLhXvbXN59PmK+A7M2VDdAwA/vVcEpETq6AblfmVHtN91B7GNEyGglVc2447ooPciTZMCAUgB6AHpAgEgAe4B7wIBIAHq" + "AesAQb5J79ZyWgm+nqrXs6x0I4wkPiKQBH28C7RWNfPTqAfu8ABBvga7i8W/V7fCfyaKf+LLs48ld6A5hMVDltkVnlrlk+IgAgFYAewB7QBAvZIZkLzw7YHDbLe+Scl6" + "3uhdXfRwOUa0JHwJvuhGG3kAQL2a+QtRGkljjF6hjiME0j7LnnMjJkDh6mYBahv3SgufAEG+q3Z1cONnEXUOq6coX7x0RaK8l2WJj/QViIJee2G6qcgAQb6p4a4p479A" + "eC04K9HUR0x8B9TDrIBoSgVyWXe7xEjGWAIBIAHyAfMCASACBAIFAgEgAfQB9QIBIAH6AfsCAUgB9gH3AEG/JvWFCk64ubdT7k9fADlAADZW2oUeE0F//hNAx5vmQ24C" + "ASAB+AH5AEG+ortA8RL/qsRfVCCcmhh9yV+abEsHsmRmSDIyM5jiKZgAQb52rnetuJmLxwetwRXlQ8SwkzMrIHn9f1t+3vxypn8ikABBvlRRrWQUSUCo75+dTtj6fP1U" + "VTmV5DEujv1TIAc3ZLZQAgFYAfwB/QIBIAH+Af8AQb6OgDPbFGfKzqixWPD2Hmgt4G6KWUdQTJBPH3A9K+TZ6ABBvoMGKypw006AeRYqimLjmY2Ufp+SHk8C0ZJBNgVB" + "lzw4AgFqAgACAQIBWAICAgMAQb4FNJ5NJO4+0QwlVAWckUZXdk+PfYDexDZ1+ju9SxhF4ABBvjxQpfN455vPpJ/T+t2rtlKCE9X6KviHFRV802gCPe5gAEG+eMP12XnW" + "n0wTl6XmbgClnjYFM2JY2UAZYhUaknKJf3AAQb5WLKPfVeykQ1NoeXCT+51aWRbOsYTKmyd3AQSzEZ39EAIBIAIGAgcCASACDAINAgFYAggCCQIBIAIKAgsAQb68pxxy" + "oAcWOvpflv3VjfgrRk9v44uazdxMziPqfc1hGABBvqK0CHqoBidcEUJHx4naV3TtgmUv1oEhGpt3DFLGnncoAEG+xnddXOiUNI6DJEK4qY1Cxoa8Hl6iQkWXMWUwTPTo" + "H6wAQb72G1Ke4q6X03mCI87z+qVMO/gd+xvXv6SSwdWpfbnvjAIBIAIOAg8AQb8B8+e/xOcnn+D3yL8SGkEf/SXAx3pRSH/Lf3UDC6zxGgIBIAIQAhEAQb7an34AE4Mg" + "4PeqZAW6F6j/JbgFl8egPBFDGYC5dIgrvABBvpMd78gzSiVsK0zz0AHtEja8x1UoB/NDZMjn+l86NQK4AgFYAhICEwIBIAIUAhUAQb4zj6RBc4mQ6p3ng7mGJ7tp7Mbz" + "ERhe7obkM9A0wnCCIABBvcdlWZEG0Xj7uGgLfagzT4G4zmtS/JDEdPQBzOA0r99AAgEgAhYCFwBAvYD00VNmocZyrS8LPuogdwJgYw9wWC7QCKaicnWos7IAQL2UR4JV" + "cHfZibOIOqdJm+OTPN6Z1z0bykKu09Up+xc/AgEgAhoCGwIBIAIoAikCASACHAIdAgEgAiYCJwIBWAIeAh8CASACJAIlAEG+pJiW3Qo4nq8pKjVzzfs3/0uJxMmWXYyD" + "sduLHtuy8ggCASACIAIhAEG+VOzUzgqzn6yjJdPd2lOP2LQqiZF7O2/LbcmLzMf+hfACAnICIgIjAD+9bmuGAYNACsk0M2FDu866cYUghqLilNK52oLflBoKXQA/vU+c" + "jkDnrb+NojfOEJpwm2m9hlmHmr3HOWwyl4LEIcEAQb7xrpmUHCzHHfaaDbiK66LDRKeKblhi4QoTVRthJ2OzbABBvu6d/bOGE/iiKiKq5AGCvcetA3Izw45ihY196+ey" + "/BbcAEG/IPVJM6fGP9OC+PczMUdiKPNfwkUrt4eslgzXXEY0qCIAQb8FwRfn4LbYMTzpLsSBuEI3vAaLitADflpdxp+M5JVWtgIBIAIqAisCASACNgI3AEG/OXz/ktGT" + "HClb8arzLt3XEjlJTw9LEYxjGvSJNff79loCASACLAItAgFIAi4CLwIBIAIwAjEAQb5bNqQnT8GAdHDnixf9NzTB5VYvmnvaYs6m53KwbxMzsABBvlGslmQWFAphVxFA" + "GGIJvfuk/oBpngdzy0sJ8WxmWNSQAgN+ugIyAjMCAW4CNAI1AD+84Hccb00HqhGM3lRQZIZ3QmOuWlRDBQ9+uXRKu1L+hAA/vOLc2o+R4+ofOAQzeQiU06F6MN1nTGWW" + "J0eurH869zQAQb36Q2nDRQfZx/XsGJ+z0zYtk4S6OXPZcUASOm420y1FQABBvd9bukINCpKmNEXeA+ve7Mnhp8WSt+MPJFDCUYjDLZ1AAgEgAjgCOQBBvzD0lLSsv1Pi" + "WQ0jVDajeXFbJ/TkSakvdy+g0TPR27KGAgFYAjoCOwIBWAI8Aj0AQb53taVCRMwrV1sky/EE45BOJoTTJ0d6vkLZIb6j4k+G0ABBvlKuPPc+sdv9ffRS/Kj+bSQKZFE7" + "fT/jbtog/5dYYCCQAEG+ZZdBcxF7VCWJS+ti78o7J2qY+aXyKipCl2P0CfXeUhAAQb5gdZIvzW7H8KDz4y1oKMiuAzlXY+TF7PGVAwUvGCn0UAIBIAJAAkEBA6DAAkwB" + "AfwCQgIBIAJDAkQBwbnpmKopRu2n8DHZCDhXCHvJdckI7xw0kBvbb0npdd7jjldXaYBVRMxJsrwBE0/IJ4amdSKh5/Ec0+nZhJr583uAAAAAAAAAAAAAAABtiv/XlkR5" + "bE7cmy0osGrcZKJHU0ACRwEB1AJFAQH0AkYBwcaYme1MOiTF+EsYXWNG8wYLwlq/ZXmR6g2PgSXaPOEeN1Z517mqkFdU7Nqr1K+moGnDNMnTrseTTWtZnFPnBDuAAAAA" + "AAAAAAAAAABtiv/XlkR5bE7cmy0osGrcZKJHU0ACRwLFAaUkEAuNdJLBIqJ50rOuJIeLHBBTEnUHFMTTlSvkBfBlTSx/ArBlJBChmMwsWi3fU4ek+WJDvjF7AhFPUcNX" + "4kaAAAAAAAAAAAAAAAAAJ37Hglt9pn14Z9Vgj9pE3L7fXbBAAkcCTgIBIAJIAkkCASACSgJLAIO/z+IwR9x5RqPSfAzguJqFxanKeUhZQgFsmKwj4GuAK2WAAAAAAAAA" + "AAAAAAB7G3oHXwv9lQmh8vd3TonVSERFqMAAgr+jPzrhTYloKgTCsGgEFNx7OdH+sJ98etJnwrIVSsFxHwAAAAAAAAAAAAAAAOsF4basDVdO8s8p/fAcwLo9j5vxAIK/" + "n8LJGSxLhg32E0QLb7fZPphHZGiLJJFDrBMD8NcM15MAAAAAAAAAAAAAAADlTNYxyXvgdnFyrRaQRoiWLQnS/gLFAbUl61s8X25tzWBr7nugeg7IMDUhKEm34FWUmcD2" + "utVNIR8VdL9iPRR4dwjF/dVl4ymiWr+kkJXphEJvGbzwSXSAAAAAAAAAAAAAAAAAWZG0lbam3LV4+pciTNFehvbNeeLAAk0CTgIBIAJPAlAAMEO5rKAEO5rKADehIAPk" + "4cBAX14QA5iWgAIBIAJRAlIAg7/T7quzPdTpPcCght7xTpoi+g9Sw7gtkYDSyaOh0qHc0AAAAAAAAAAAAAAAADavGw+/CvXTnyDIJ6fZU+llAiixQAIBIAJTAlQCASAC" + "WwJcAgEgAlUCVgCBv1wad2ywThLttxU0gcwWuSJSuLNadPm8j3J85ggRzjkGAAAAAAAAAAAAAAAB1xLrLNteGQzkOClxdvv3E/l3M5UAgb8JuDCFQxifbIdTfjd1x7Mq" + "S+Z7dzIUkHtIdVjcVeFT2AAAAAAAAAAAAAAAAiwal03Yl9B7p2fVDSCtlYsZX6m+AgEgAlcCWAIBIAJZAloAgb7jxvbib0yb3DKvQBDcHL/hdg7NjCuqjUQ09t8hgmhV" + "oAAAAAAAAAAAAAAABEGpMZGoNId5F80sBzWgnjo+AP2UAIG+sE8ccijAbmkaBJVfyfgqY5pf4QSO+c5IFGVC9WwlY/AAAAAAAAAAAAAAAAeg08QveVui23B9QhrdMd7a" + "nx/sGACBvqxwYOyAk+H0YGBc70gZFJc6oqUvcHywU+yJNBfSNh+AAAAAAAAAAAAAAAADFU5kDFbQI6mIkEJqJNGncvWjiygCASACXQJeAIG/acxhhr+dznhtppGVCg+k" + "FqjL65rOddHn1mwyRj1rYgQAAAAAAAAAAAAAAACRfpTwfZ9v81WVbRpRYN+1/m9YhwCBvw9fhTm/NqURBT4FuwJczZWe39F575hmpFtt8KVniCwIAAAAAAAAAAAAAAAB" + "DkxuMKeNKjBZpVAjNVjJ/URzwhoAgb8RuD3rFDyNUpuXtBAnWTykKVAuY7UKLrye419st2b25AAAAAAAAAAAAAAAAlUrmS7Amiwb/77tvRUhnpfLLMXeL4vIgQ=="; + + +constexpr td::int64 Ton = 1000000000; + +TEST(Emulator, wallet_int_and_ext_msg) { + td::Ed25519::PrivateKey priv_key = td::Ed25519::generate_private_key().move_as_ok(); + auto pub_key = priv_key.get_public_key().move_as_ok(); + ton::WalletV3::InitData init_data; + init_data.public_key = pub_key.as_octet_string(); + init_data.wallet_id = 239; + auto wallet = ton::WalletV3::create(init_data, 2); + + auto address = wallet->get_address(); + + void *emulator = transaction_emulator_create(config_boc, 3); + const uint64_t lt = 42000000000; + CHECK(transaction_emulator_set_lt(emulator, lt)); + const uint32_t utime = 1337; + transaction_emulator_set_unixtime(emulator, utime); + + std::string shard_account_after_boc_b64; + + // emulate internal message with init state on uninit account + { + td::Ref account_root; + block::gen::Account().cell_pack_account_none(account_root); + auto none_shard_account_cell = vm::CellBuilder().store_ref(account_root).store_bits(td::Bits256::zero().as_bitslice()).store_long(0).finalize(); + auto none_shard_account_boc = td::base64_encode(std_boc_serialize(none_shard_account_cell).move_as_ok()); + + td::Ref int_msg; + { + block::gen::Message::Record message; + block::gen::CommonMsgInfo::Record_int_msg_info msg_info; + msg_info.ihr_disabled = true; + msg_info.bounce = false; + msg_info.bounced = false; + { + block::gen::MsgAddressInt::Record_addr_std src; + src.anycast = vm::CellBuilder().store_zeroes(1).as_cellslice_ref(); + src.workchain_id = 0; + src.address = td::Bits256();; + tlb::csr_pack(msg_info.src, src); + } + { + block::gen::MsgAddressInt::Record_addr_std dest; + dest.anycast = vm::CellBuilder().store_zeroes(1).as_cellslice_ref(); + dest.workchain_id = address.workchain; + dest.address = address.addr; + tlb::csr_pack(msg_info.dest, dest); + } + { + block::CurrencyCollection cc{10 * Ton}; + cc.pack_to(msg_info.value); + } + { + vm::CellBuilder cb; + block::tlb::t_Grams.store_integer_value(cb, td::BigInt256(int(0.03 * Ton))); + msg_info.fwd_fee = cb.as_cellslice_ref(); + } + { + vm::CellBuilder cb; + block::tlb::t_Grams.store_integer_value(cb, td::BigInt256(0)); + msg_info.ihr_fee = cb.as_cellslice_ref(); + } + msg_info.created_lt = 0; + msg_info.created_at = static_cast(td::Time::now()); + tlb::csr_pack(message.info, msg_info); + message.init = vm::CellBuilder() + .store_ones(1) + .store_zeroes(1) + .append_cellslice(vm::load_cell_slice(ton::GenericAccount::get_init_state(wallet->get_state()))) + .as_cellslice_ref(); + message.body = vm::CellBuilder().store_zeroes(1).as_cellslice_ref(); + + tlb::type_pack_cell(int_msg, block::gen::t_Message_Any, message); + } + + CHECK(int_msg.not_null()); + + auto int_msg_boc = td::base64_encode(std_boc_serialize(int_msg).move_as_ok()); + + std::string int_emu_res = transaction_emulator_emulate_transaction(emulator, none_shard_account_boc.c_str(), int_msg_boc.c_str()); + + auto int_result_json = td::json_decode(td::MutableSlice(int_emu_res)); + CHECK(int_result_json.is_ok()); + auto int_result_value = int_result_json.move_as_ok(); + auto& int_result_obj = int_result_value.get_object(); + + auto success_field = td::get_json_object_field(int_result_obj, "success", td::JsonValue::Type::Boolean, false); + CHECK(success_field.is_ok()); + auto success = success_field.move_as_ok().get_boolean(); + CHECK(success); + + auto transaction_field = td::get_json_object_field(int_result_obj, "transaction", td::JsonValue::Type::String, false); + CHECK(transaction_field.is_ok()); + auto transaction_boc_b64 = transaction_field.move_as_ok().get_string(); + auto transaction_boc = td::base64_decode(transaction_boc_b64); + CHECK(transaction_boc.is_ok()); + auto trans_cell = vm::std_boc_deserialize(transaction_boc.move_as_ok()); + CHECK(trans_cell.is_ok()); + td::Bits256 trans_hash = trans_cell.ok()->get_hash().bits(); + block::gen::Transaction::Record trans; + block::gen::TransactionDescr::Record_trans_ord trans_descr; + CHECK(tlb::unpack_cell(trans_cell.move_as_ok(), trans) && tlb::unpack_cell(trans.description, trans_descr)); + CHECK(trans.outmsg_cnt == 0); + CHECK(trans.account_addr == wallet->get_address().addr); + CHECK(trans_descr.aborted == false); + CHECK(trans_descr.destroyed == false); + CHECK(trans.lt == lt); + CHECK(trans.now == utime); + + auto shard_account_field = td::get_json_object_field(int_result_obj, "shard_account", td::JsonValue::Type::String, false); + CHECK(shard_account_field.is_ok()); + auto shard_account_boc_b64 = shard_account_field.move_as_ok().get_string(); + shard_account_after_boc_b64 = shard_account_boc_b64.str(); + auto shard_account_boc = td::base64_decode(shard_account_boc_b64); + CHECK(shard_account_boc.is_ok()); + auto shard_account_cell = vm::std_boc_deserialize(shard_account_boc.move_as_ok()); + CHECK(shard_account_cell.is_ok()); + block::gen::ShardAccount::Record shard_account; + block::gen::Account::Record_account account; + CHECK(tlb::unpack_cell(shard_account_cell.move_as_ok(), shard_account) && tlb::unpack_cell(shard_account.account, account)); + CHECK(shard_account.last_trans_hash == trans_hash); + CHECK(shard_account.last_trans_lt == lt); + ton::WorkchainId wc; + ton::StdSmcAddress addr; + CHECK(block::tlb::t_MsgAddressInt.extract_std_address(account.addr, wc, addr)); + CHECK(address.workchain == wc); + CHECK(address.addr == addr); + } + + // emulate external message + { + auto ext_body = wallet->make_a_gift_message(priv_key, static_cast(td::Time::now()) + 60, {ton::WalletV3::Gift{block::StdAddress(0, ton::StdSmcAddress()), 1 * Ton}}); + CHECK(ext_body.is_ok()); + auto ext_msg = ton::GenericAccount::create_ext_message(address, {}, ext_body.move_as_ok()); + auto ext_msg_boc = td::base64_encode(std_boc_serialize(ext_msg).move_as_ok()); + std::string ext_emu_res = transaction_emulator_emulate_transaction(emulator, shard_account_after_boc_b64.c_str(), ext_msg_boc.c_str()); + + auto ext_result_json = td::json_decode(td::MutableSlice(ext_emu_res)); + CHECK(ext_result_json.is_ok()); + auto ext_result = ext_result_json.move_as_ok(); + auto &ext_result_obj = ext_result.get_object(); + auto ext_success_field = td::get_json_object_field(ext_result_obj, "success", td::JsonValue::Type::Boolean, false); + CHECK(ext_success_field.is_ok()); + auto ext_success = ext_success_field.move_as_ok().get_boolean(); + CHECK(ext_success); + + auto ext_transaction_field = td::get_json_object_field(ext_result_obj, "transaction", td::JsonValue::Type::String, false); + CHECK(ext_transaction_field.is_ok()); + auto ext_transaction_boc_b64 = ext_transaction_field.move_as_ok().get_string(); + auto ext_transaction_boc = td::base64_decode(ext_transaction_boc_b64); + CHECK(ext_transaction_boc.is_ok()); + auto ext_trans_cell = vm::std_boc_deserialize(ext_transaction_boc.move_as_ok()); + CHECK(ext_trans_cell.is_ok()); + td::Bits256 ext_trans_hash = ext_trans_cell.ok()->get_hash().bits(); + block::gen::Transaction::Record ext_trans; + block::gen::TransactionDescr::Record_trans_ord ext_trans_descr; + CHECK(tlb::unpack_cell(ext_trans_cell.move_as_ok(), ext_trans) && tlb::unpack_cell(ext_trans.description, ext_trans_descr)); + CHECK(ext_trans.outmsg_cnt == 1); + CHECK(ext_trans.account_addr == wallet->get_address().addr); + CHECK(ext_trans_descr.aborted == false); + CHECK(ext_trans_descr.destroyed == false); + + auto ext_shard_account_field = td::get_json_object_field(ext_result_obj, "shard_account", td::JsonValue::Type::String, false); + CHECK(ext_shard_account_field.is_ok()); + auto ext_shard_account_boc_b64 = ext_shard_account_field.move_as_ok().get_string(); + auto ext_shard_account_boc = td::base64_decode(ext_shard_account_boc_b64); + CHECK(ext_shard_account_boc.is_ok()); + auto ext_shard_account_cell = vm::std_boc_deserialize(ext_shard_account_boc.move_as_ok()); + CHECK(ext_shard_account_cell.is_ok()); + block::gen::ShardAccount::Record ext_shard_account; + block::gen::Account::Record_account ext_account; + CHECK(tlb::unpack_cell(ext_shard_account_cell.move_as_ok(), ext_shard_account) && tlb::unpack_cell(ext_shard_account.account, ext_account)); + CHECK(ext_shard_account.last_trans_hash == ext_trans_hash); + CHECK(ext_shard_account.last_trans_lt == ext_trans.lt); + ton::WorkchainId wc; + ton::StdSmcAddress addr; + CHECK(block::tlb::t_MsgAddressInt.extract_std_address(ext_account.addr, wc, addr)); + CHECK(address.workchain == wc); + CHECK(address.addr == addr); + } +} + +TEST(Emulator, tvm_emulator) { + td::Ed25519::PrivateKey priv_key = td::Ed25519::generate_private_key().move_as_ok(); + auto pub_key = priv_key.get_public_key().move_as_ok(); + ton::WalletV3::InitData init_data; + init_data.public_key = pub_key.as_octet_string(); + init_data.wallet_id = 239; + init_data.seqno = 1337; + auto wallet = ton::WalletV3::create(init_data, 2); + + auto code = ton::SmartContractCode::get_code(ton::SmartContractCode::Type::WalletV3, 2); + auto code_boc_b64 = td::base64_encode(std_boc_serialize(code).move_as_ok()); + auto data = ton::WalletV3::get_init_data(init_data); + auto data_boc_b64 = td::base64_encode(std_boc_serialize(data).move_as_ok()); + + void *tvm_emulator = tvm_emulator_create(code_boc_b64.c_str(), data_boc_b64.c_str(), 1); + unsigned method_crc = td::crc16("seqno"); + unsigned method_id = (method_crc & 0xffff) | 0x10000; + auto stack = td::make_ref(); + vm::CellBuilder stack_cb; + CHECK(stack->serialize(stack_cb)); + auto stack_cell = stack_cb.finalize(); + auto stack_boc = td::base64_encode(std_boc_serialize(stack_cell).move_as_ok()); + + char addr_buffer[49] = {0}; + CHECK(wallet->get_address().rserialize_to(addr_buffer)); + + auto rand_seed = std::string(64, 'F'); + CHECK(tvm_emulator_set_c7(tvm_emulator, addr_buffer, static_cast(td::Time::now()), 10 * Ton, rand_seed.c_str(), config_boc)); + std::string tvm_res = tvm_emulator_run_get_method(tvm_emulator, method_id, stack_boc.c_str()); + + auto result_json = td::json_decode(td::MutableSlice(tvm_res)); + CHECK(result_json.is_ok()); + auto result = result_json.move_as_ok(); + auto& result_obj = result.get_object(); + + auto success_field = td::get_json_object_field(result_obj, "success", td::JsonValue::Type::Boolean, false); + CHECK(success_field.is_ok()); + auto success = success_field.move_as_ok().get_boolean(); + CHECK(success); + + auto stack_field = td::get_json_object_field(result_obj, "stack", td::JsonValue::Type::String, false); + CHECK(stack_field.is_ok()); + auto stack_val = stack_field.move_as_ok(); + auto& stack_obj = stack_val.get_string(); + auto stack_res_boc = td::base64_decode(stack_obj); + CHECK(stack_res_boc.is_ok()); + auto stack_res_cell = vm::std_boc_deserialize(stack_res_boc.move_as_ok()); + CHECK(stack_res_cell.is_ok()); + td::Ref stack_res; + auto stack_res_cs = vm::load_cell_slice(stack_res_cell.move_as_ok()); + CHECK(vm::Stack::deserialize_to(stack_res_cs, stack_res)); + CHECK(stack_res->depth() == 1); + CHECK(stack_res.write().pop_int()->to_long() == init_data.seqno); +} diff --git a/emulator/transaction-emulator.cpp b/emulator/transaction-emulator.cpp index 2e8ba0374..e87b2dfbb 100644 --- a/emulator/transaction-emulator.cpp +++ b/emulator/transaction-emulator.cpp @@ -25,7 +25,7 @@ td::Result> TransactionEmu utime = (unsigned)std::time(nullptr); } - auto fetch_res = block::FetchConfigParams::fetch_config_params(config_, prev_blocks_info_, &old_mparams, + auto fetch_res = block::FetchConfigParams::fetch_config_params(*config_, prev_blocks_info_, &old_mparams, &storage_prices, &storage_phase_cfg, &rand_seed_, &compute_phase_cfg, &action_phase_cfg, &masterchain_create_fee, @@ -130,17 +130,28 @@ td::Result TransactionEmulator::emulate_t } TRY_RESULT(emulation, emulate_transaction(std::move(account), msg_root, utime, lt, trans_type)); + + if (auto emulation_result_ptr = dynamic_cast(emulation.get())) { + auto& emulation_result = *emulation_result_ptr; + + if (td::Bits256(emulation_result.transaction->get_hash().bits()) != td::Bits256(original_trans->get_hash().bits())) { + return td::Status::Error("transaction hash mismatch"); + } - auto emulation_result = dynamic_cast(*emulation); - if (td::Bits256(emulation_result.transaction->get_hash().bits()) != td::Bits256(original_trans->get_hash().bits())) { - return td::Status::Error("transaction hash mismatch"); - } + if (!check_state_update(emulation_result.account, record_trans)) { + return td::Status::Error("account hash mismatch"); + } - if (!check_state_update(emulation_result.account, record_trans)) { - return td::Status::Error("account hash mismatch"); - } + return emulation_result; - return emulation_result; + } else if (auto emulation_not_accepted_ptr = dynamic_cast(emulation.get())) { + return td::Status::Error( PSTRING() + << "VM Log: " << emulation_not_accepted_ptr->vm_log + << ", VM Exit Code: " << emulation_not_accepted_ptr->vm_exit_code + << ", Elapsed Time: " << emulation_not_accepted_ptr->elapsed_time); + } else { + return td::Status::Error("emulation failed"); + } } td::Result TransactionEmulator::emulate_transactions_chain(block::Account&& account, std::vector>&& original_transactions) { @@ -227,7 +238,9 @@ td::Result> TransactionEmulator return td::Status::Error(-669,"cannot create action phase of a new transaction for smart contract "s + acc->addr.to_hex()); } - if (trans->bounce_enabled && !trans->compute_phase->success && !trans->prepare_bounce_phase(*action_phase_cfg)) { + if (trans->bounce_enabled + && (!trans->compute_phase->success || trans->action_phase->state_exceeds_limits || trans->action_phase->bounce) + && !trans->prepare_bounce_phase(*action_phase_cfg)) { return td::Status::Error(-669,"cannot create bounce phase of a new transaction for smart contract "s + acc->addr.to_hex()); } @@ -250,8 +263,8 @@ void TransactionEmulator::set_ignore_chksig(bool ignore_chksig) { ignore_chksig_ = ignore_chksig; } -void TransactionEmulator::set_config(block::Config &&config) { - config_ = std::forward(config); +void TransactionEmulator::set_config(std::shared_ptr config) { + config_ = std::move(config); } void TransactionEmulator::set_libs(vm::Dictionary &&libs) { diff --git a/emulator/transaction-emulator.h b/emulator/transaction-emulator.h index 8186a3c4a..eae109f40 100644 --- a/emulator/transaction-emulator.h +++ b/emulator/transaction-emulator.h @@ -9,7 +9,7 @@ namespace emulator { class TransactionEmulator { - block::Config config_; + std::shared_ptr config_; vm::Dictionary libraries_; int vm_log_verbosity_; ton::UnixTime unixtime_; @@ -20,7 +20,7 @@ class TransactionEmulator { td::Ref prev_blocks_info_; public: - TransactionEmulator(block::Config&& config, int vm_log_verbosity = 0) : + TransactionEmulator(std::shared_ptr config, int vm_log_verbosity = 0) : config_(std::move(config)), libraries_(256), vm_log_verbosity_(vm_log_verbosity), unixtime_(0), lt_(0), rand_seed_(td::BitArray<256>::zero()), ignore_chksig_(false), debug_enabled_(false) { } @@ -57,7 +57,7 @@ class TransactionEmulator { }; const block::Config& get_config() { - return config_; + return *config_; } ton::UnixTime get_unixtime() { @@ -74,7 +74,7 @@ class TransactionEmulator { void set_lt(ton::LogicalTime lt); void set_rand_seed(td::BitArray<256>& rand_seed); void set_ignore_chksig(bool ignore_chksig); - void set_config(block::Config &&config); + void set_config(std::shared_ptr config); void set_libs(vm::Dictionary &&libs); void set_debug_enabled(bool debug_enabled); void set_prev_blocks_info(td::Ref prev_blocks_info); diff --git a/emulator/tvm-emulator.hpp b/emulator/tvm-emulator.hpp index a9f248b72..413298c99 100644 --- a/emulator/tvm-emulator.hpp +++ b/emulator/tvm-emulator.hpp @@ -24,12 +24,12 @@ class TvmEmulator { } void set_c7(block::StdAddress address, uint32_t unixtime, uint64_t balance, td::BitArray<256> rand_seed, std::shared_ptr config) { - args_.set_address(address); + args_.set_address(std::move(address)); args_.set_now(unixtime); args_.set_balance(balance); - args_.set_rand_seed(rand_seed); + args_.set_rand_seed(std::move(rand_seed)); if (config) { - args_.set_config(config); + args_.set_config(std::move(config)); } } @@ -37,6 +37,10 @@ class TvmEmulator { args_.set_c7(std::move(c7)); } + void set_config(std::shared_ptr config) { + args_.set_config(std::move(config)); + } + void set_prev_blocks_info(td::Ref tuple) { args_.set_prev_blocks_info(std::move(tuple)); } @@ -46,7 +50,8 @@ class TvmEmulator { } Answer run_get_method(int method_id, td::Ref stack) { - return smc_.run_get_method(args_.set_stack(stack).set_method_id(method_id)); + ton::SmartContract::Args args = args_; + return smc_.run_get_method(args.set_stack(stack).set_method_id(method_id)); } Answer send_external_message(td::Ref message_body) { @@ -54,7 +59,8 @@ class TvmEmulator { } Answer send_internal_message(td::Ref message_body, uint64_t amount) { - return smc_.send_internal_message(message_body, args_.set_amount(amount)); + ton::SmartContract::Args args = args_; + return smc_.send_internal_message(message_body, args.set_amount(amount)); } }; } \ No newline at end of file diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index 55d46ad1f..020aca705 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -949,8 +949,8 @@ bool TestNode::show_help(std::string command) { "lasttrans[dump] []\tShows or dumps specified transaction and " "several preceding " "ones\n" - "listblocktrans[rev] [ ]\tLists block transactions, " - "starting immediately after or before the specified one\n" + "listblocktrans[rev][meta] [ ]\tLists block " + "transactions, starting immediately after or before the specified one\n" "blkproofchain[step] []\tDownloads and checks proof of validity of the " "second " "indicated block (or the last known masterchain block) starting from given block\n" @@ -1074,6 +1074,13 @@ bool TestNode::do_parse_line() { return parse_block_id_ext(blkid) && parse_uint32(count) && (seekeoln() || (parse_hash(hash) && parse_lt(lt) && (mode |= 128) && seekeoln())) && get_block_transactions(blkid, mode, count, hash, lt); + } else if (word == "listblocktransmeta" || word == "listblocktransrevmeta") { + lt = 0; + int mode = (word == "listblocktransmeta" ? 7 : 0x47); + mode |= 256; + return parse_block_id_ext(blkid) && parse_uint32(count) && + (seekeoln() || (parse_hash(hash) && parse_lt(lt) && (mode |= 128) && seekeoln())) && + get_block_transactions(blkid, mode, count, hash, lt); } else if (word == "blkproofchain" || word == "blkproofchainstep") { ton::BlockIdExt blkid2{}; return parse_block_id_ext(blkid) && (seekeoln() || parse_block_id_ext(blkid2)) && seekeoln() && @@ -2493,23 +2500,40 @@ bool TestNode::get_block_transactions(ton::BlockIdExt blkid, int mode, unsigned } else { auto f = F.move_as_ok(); std::vector transactions; + std::vector> metadata; for (auto& id : f->ids_) { transactions.emplace_back(id->account_, id->lt_, id->hash_); + metadata.push_back(std::move(id->metadata_)); } td::actor::send_closure_later(Self, &TestNode::got_block_transactions, ton::create_block_id(f->id_), mode, - f->req_count_, f->incomplete_, std::move(transactions), std::move(f->proof_)); + f->req_count_, f->incomplete_, std::move(transactions), std::move(metadata), + std::move(f->proof_)); } }); } -void TestNode::got_block_transactions(ton::BlockIdExt blkid, int mode, unsigned req_count, bool incomplete, - std::vector trans, td::BufferSlice proof) { +void TestNode::got_block_transactions( + ton::BlockIdExt blkid, int mode, unsigned req_count, bool incomplete, std::vector trans, + std::vector> metadata, td::BufferSlice proof) { LOG(INFO) << "got up to " << req_count << " transactions from block " << blkid.to_str(); auto out = td::TerminalIO::out(); int count = 0; - for (auto& t : trans) { + for (size_t i = 0; i < trans.size(); ++i) { + auto& t = trans[i]; out << "transaction #" << ++count << ": account " << t.acc_addr.to_hex() << " lt " << t.trans_lt << " hash " << t.trans_hash.to_hex() << std::endl; + if (mode & 256) { + auto& meta = metadata.at(i); + if (meta == nullptr) { + out << " metadata: " << std::endl; + } else { + out << " metadata: " + << block::MsgMetadata{(td::uint32)meta->depth_, meta->initiator_->workchain_, meta->initiator_->id_, + (ton::LogicalTime)meta->initiator_lt_} + .to_str() + << std::endl; + } + } } out << (incomplete ? "(block transaction list incomplete)" : "(end of block transaction list)") << std::endl; } diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h index 219ba7d5c..17680f448 100644 --- a/lite-client/lite-client.h +++ b/lite-client/lite-client.h @@ -258,7 +258,9 @@ class TestNode : public td::actor::Actor { bool get_block_transactions(ton::BlockIdExt blkid, int mode, unsigned count, ton::Bits256 acc_addr, ton::LogicalTime lt); void got_block_transactions(ton::BlockIdExt blkid, int mode, unsigned req_count, bool incomplete, - std::vector trans, td::BufferSlice proof); + std::vector trans, + std::vector> metadata, + td::BufferSlice proof); bool get_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mode); void got_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mode, td::BufferSlice res); bool get_creator_stats(ton::BlockIdExt blkid, int mode, unsigned req_count, ton::Bits256 start_after, diff --git a/overlay/overlay-fec-broadcast.cpp b/overlay/overlay-fec-broadcast.cpp index aed5248b8..cd030742a 100644 --- a/overlay/overlay-fec-broadcast.cpp +++ b/overlay/overlay-fec-broadcast.cpp @@ -78,7 +78,6 @@ td::Status OverlayFecBroadcastPart::check_signature() { } td::Status OverlayFecBroadcastPart::run_checks() { - TRY_STATUS(check_time()); TRY_STATUS(check_duplicate()); TRY_STATUS(check_source()); @@ -94,14 +93,17 @@ void BroadcastFec::broadcast_checked(td::Result R) { overlay_->deliver_broadcast(get_source().compute_short_id(), data_.clone()); auto manager = overlay_->overlay_manager(); while (!parts_.empty()) { - distribute_part(parts_.begin()->first); + distribute_part(parts_.begin()->first); } + + is_checked_ = true; } // Do we need status here?? -td::Status BroadcastFec::distribute_part(td::uint32 seqno) { +td::Status BroadcastFec::distribute_part(td::uint32 seqno) { auto i = parts_.find(seqno); if (i == parts_.end()) { + VLOG(OVERLAY_WARNING) << "not distibuting empty part " << seqno; // should not get here return td::Status::OK(); } @@ -132,7 +134,6 @@ td::Status BroadcastFec::distribute_part(td::uint32 seqno) { } td::Status OverlayFecBroadcastPart::apply() { - if (!bcast_) { bcast_ = overlay_->get_fec_broadcast(broadcast_hash_); } @@ -165,16 +166,20 @@ td::Status OverlayFecBroadcastPart::apply() { return S; } } else { - if(untrusted_) { + if (untrusted_) { auto P = td::PromiseCreator::lambda( - [id = broadcast_hash_, overlay_id = actor_id(overlay_)](td::Result RR) mutable { - td::actor::send_closure(std::move(overlay_id), &OverlayImpl::broadcast_checked, id, std::move(RR)); - }); + [id = broadcast_hash_, overlay_id = actor_id(overlay_)](td::Result RR) mutable { + td::actor::send_closure(std::move(overlay_id), &OverlayImpl::broadcast_checked, id, std::move(RR)); + }); overlay_->check_broadcast(bcast_->get_source().compute_short_id(), R.move_as_ok(), std::move(P)); } else { overlay_->deliver_broadcast(bcast_->get_source().compute_short_id(), R.move_as_ok()); } } + } else { + bcast_->set_overlay(overlay_); + bcast_->set_src_peer_id(src_peer_id_); + TRY_STATUS(bcast_->add_part(seqno_, data_.clone(), export_serialized_short(), export_serialized())); } return td::Status::OK(); } @@ -304,7 +309,8 @@ td::Status OverlayFecBroadcastPart::create_new(OverlayImpl *overlay, td::actor:: auto B = std::make_unique( broadcast_hash, part_hash, PublicKey{}, overlay->get_certificate(local_id), data_hash, size, flags, - part_data_hash, std::move(part), seqno, std::move(fec_type), date, td::BufferSlice{}, false, nullptr, overlay, adnl::AdnlNodeIdShort::zero()); + part_data_hash, std::move(part), seqno, std::move(fec_type), date, td::BufferSlice{}, false, nullptr, overlay, + adnl::AdnlNodeIdShort::zero()); auto to_sign = B->to_sign(); auto P = td::PromiseCreator::lambda( diff --git a/overlay/overlay-fec-broadcast.hpp b/overlay/overlay-fec-broadcast.hpp index 612af22fb..85de648e3 100644 --- a/overlay/overlay-fec-broadcast.hpp +++ b/overlay/overlay-fec-broadcast.hpp @@ -82,15 +82,15 @@ class BroadcastFec : public td::ListNode { } } - td::Status add_part(td::uint32 seqno, td::BufferSlice data, - td::BufferSlice serialized_fec_part_short, + td::Status add_part(td::uint32 seqno, td::BufferSlice data, td::BufferSlice serialized_fec_part_short, td::BufferSlice serialized_fec_part) { - CHECK(decoder_); - td::fec::Symbol s; - s.id = seqno; - s.data = std::move(data); + if (decoder_) { + td::fec::Symbol s; + s.id = seqno; + s.data = std::move(data); - decoder_->add_symbol(std::move(s)); + decoder_->add_symbol(std::move(s)); + } parts_[seqno] = std::pair(std::move(serialized_fec_part_short), std::move(serialized_fec_part)); @@ -200,8 +200,13 @@ class BroadcastFec : public td::ListNode { td::Status distribute_part(td::uint32 seqno); + bool is_checked() const { + return is_checked_; + } + private: bool ready_ = false; + bool is_checked_ = false; Overlay::BroadcastHash hash_; Overlay::BroadcastDataHash data_hash_; @@ -281,7 +286,7 @@ class OverlayFecBroadcastPart : public td::ListNode { , signature_(std::move(signature)) , is_short_(is_short) , bcast_(bcast) - , overlay_(overlay) + , overlay_(overlay) , src_peer_id_(src_peer_id) { } @@ -300,7 +305,7 @@ class OverlayFecBroadcastPart : public td::ListNode { signature_ = std::move(signature); } void update_overlay(OverlayImpl *overlay); - + tl_object_ptr export_tl(); tl_object_ptr export_tl_short(); td::BufferSlice export_serialized(); @@ -310,14 +315,16 @@ class OverlayFecBroadcastPart : public td::ListNode { td::Status run() { TRY_STATUS(run_checks()); TRY_STATUS(apply()); - if(!untrusted_) { + if (!untrusted_ || bcast_->is_checked()) { TRY_STATUS(distribute()); } return td::Status::OK(); } - static td::Status create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, tl_object_ptr broadcast); - static td::Status create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, tl_object_ptr broadcast); + static td::Status create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, + tl_object_ptr broadcast); + static td::Status create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, + tl_object_ptr broadcast); static td::Status create_new(OverlayImpl *overlay, td::actor::ActorId overlay_actor_id, PublicKeyHash local_id, Overlay::BroadcastDataHash data_hash, td::uint32 size, td::uint32 flags, td::BufferSlice part, td::uint32 seqno, fec::FecType fec_type, diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index 409f09935..e81fecc70 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -229,7 +229,7 @@ void OverlayImpl::update_neighbours(td::uint32 nodes_to_change) { continue; } - if (X->get_version() <= td::Clocks::system() - 600) { + if (public_ && X->get_version() <= td::Clocks::system() - 600) { if (X->is_neighbour()) { bool found = false; for (auto &n : neighbours_) { @@ -301,7 +301,7 @@ void OverlayImpl::get_overlay_random_peers(td::uint32 max_peers, auto t = td::Clocks::system(); while (v.size() < max_peers && v.size() < peers_.size() - bad_peers_.size()) { auto P = peers_.get_random(); - if (P->get_version() + 3600 < t) { + if (public_ && P->get_version() + 3600 < t) { VLOG(OVERLAY_INFO) << this << ": deleting outdated peer " << P->get_id(); del_peer(P->get_id()); } else if (P->is_alive()) { diff --git a/recent_changelog.md b/recent_changelog.md index f0b029eea..25a93c189 100644 --- a/recent_changelog.md +++ b/recent_changelog.md @@ -1,13 +1,11 @@ ## 2024.04 Update -1. Emulator: Single call optimized runGetMethod added -2. Tonlib: a series of proof improvements, also breaking Change in `liteServer.getAllShardsInfo` method (see below) -3. DB: usage statistics now collected, outdated persistent states are not serialized -4. LS: fast `getOutMsgQueueSizes` added, preliminary support of non-final block requests -5. Network: lz4 compression of block candidates (disabled by default). -6. Overlays: add custom overlays -7. Transaction Executor: fixed issue with due_payment collection - -* `liteServer.getAllShardsInfo` method was updated for better efficiency. Previously, field proof contained BoC with two roots: one for BlockState from block's root and another for ShardHashes from BlockState. Now, it returns a single-root proof BoC, specifically the merkle proof of ShardHashes directly from the block's root, streamlining data access and integrity. Checking of the proof requires to check that ShardHashes in the `data` correspond to ShardHashes from the block. - -Besides the work of the core team, this update is based on the efforts of @akifoq (due_payment issue). +1. Make Jemalloc default allocator +2. Add candidate broadcasting and caching +3. Limit per address speed for external messages broadcast by reasonably large number +4. Overlay improvements: fix dropping peers in small custom overlays, fix wrong certificate on missed keyblocks +5. Extended statistics and logs for celldb usage, session stats, persistent state serialization +6. Tonlib and explorer fixes +7. Flags for precize control of Celldb: `--celldb-cache-size`, `--celldb-direct-io` and `--celldb-preload-all` +8. Add valiator-console command to stop persistent state serialization +9. Use `@` path separator for defining include path in fift and create-state utilities on Windows only. diff --git a/tddb/td/db/KeyValue.h b/tddb/td/db/KeyValue.h index 4e0d85384..4f30a272b 100644 --- a/tddb/td/db/KeyValue.h +++ b/tddb/td/db/KeyValue.h @@ -19,6 +19,7 @@ #pragma once #include "td/utils/Status.h" #include "td/utils/logging.h" +#include namespace td { class KeyValueReader { public: @@ -27,6 +28,9 @@ class KeyValueReader { virtual Result get(Slice key, std::string &value) = 0; virtual Result count(Slice prefix) = 0; + virtual Status for_each(std::function f) { + return Status::Error("for_each is not supported"); + } }; class PrefixedKeyValueReader : public KeyValueReader { diff --git a/tddb/td/db/RocksDb.cpp b/tddb/td/db/RocksDb.cpp index a84a804bb..f8688c006 100644 --- a/tddb/td/db/RocksDb.cpp +++ b/tddb/td/db/RocksDb.cpp @@ -56,41 +56,45 @@ RocksDb::~RocksDb() { } RocksDb RocksDb::clone() const { - return RocksDb{db_, statistics_}; + return RocksDb{db_, options_}; } -Result RocksDb::open(std::string path, std::shared_ptr statistics) { +Result RocksDb::open(std::string path, RocksDbOptions options) { rocksdb::OptimisticTransactionDB *db; { - rocksdb::Options options; + rocksdb::Options db_options; - static auto cache = rocksdb::NewLRUCache(1 << 30); + static auto default_cache = rocksdb::NewLRUCache(1 << 30); + if (options.block_cache == nullptr) { + options.block_cache = default_cache; + } rocksdb::BlockBasedTableOptions table_options; - table_options.block_cache = cache; - options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); - - options.manual_wal_flush = true; - options.create_if_missing = true; - options.max_background_compactions = 4; - options.max_background_flushes = 2; - options.bytes_per_sync = 1 << 20; - options.writable_file_max_buffer_size = 2 << 14; - options.statistics = statistics; + table_options.block_cache = options.block_cache; + db_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); + + db_options.use_direct_reads = options.use_direct_reads; + db_options.manual_wal_flush = true; + db_options.create_if_missing = true; + db_options.max_background_compactions = 4; + db_options.max_background_flushes = 2; + db_options.bytes_per_sync = 1 << 20; + db_options.writable_file_max_buffer_size = 2 << 14; + db_options.statistics = options.statistics; rocksdb::OptimisticTransactionDBOptions occ_options; occ_options.validate_policy = rocksdb::OccValidationPolicy::kValidateSerial; - rocksdb::ColumnFamilyOptions cf_options(options); + rocksdb::ColumnFamilyOptions cf_options(db_options); std::vector column_families; column_families.push_back(rocksdb::ColumnFamilyDescriptor(rocksdb::kDefaultColumnFamilyName, cf_options)); std::vector handles; - TRY_STATUS(from_rocksdb( - rocksdb::OptimisticTransactionDB::Open(options, occ_options, std::move(path), column_families, &handles, &db))); + TRY_STATUS(from_rocksdb(rocksdb::OptimisticTransactionDB::Open(db_options, occ_options, std::move(path), + column_families, &handles, &db))); CHECK(handles.size() == 1); // i can delete the handle since DBImpl is always holding a reference to // default column family delete handles[0]; } - return RocksDb(std::shared_ptr(db), std::move(statistics)); + return RocksDb(std::shared_ptr(db), std::move(options)); } std::shared_ptr RocksDb::create_statistics() { @@ -105,6 +109,10 @@ void RocksDb::reset_statistics(const std::shared_ptr statis statistics->Reset(); } +std::shared_ptr RocksDb::create_cache(size_t capacity) { + return rocksdb::NewLRUCache(capacity); +} + std::unique_ptr RocksDb::snapshot() { auto res = std::make_unique(clone()); res->begin_snapshot().ensure(); @@ -116,7 +124,6 @@ std::string RocksDb::stats() const { db_->GetProperty("rocksdb.stats", &out); //db_->GetProperty("rocksdb.cur-size-all-mem-tables", &out); return out; - return statistics_->ToString(); } Result RocksDb::get(Slice key, std::string &value) { @@ -183,6 +190,28 @@ Result RocksDb::count(Slice prefix) { return res; } +Status RocksDb::for_each(std::function f) { + rocksdb::ReadOptions options; + options.snapshot = snapshot_.get(); + std::unique_ptr iterator; + if (snapshot_ || !transaction_) { + iterator.reset(db_->NewIterator(options)); + } else { + iterator.reset(transaction_->GetIterator(options)); + } + + iterator->SeekToFirst(); + for (; iterator->Valid(); iterator->Next()) { + auto key = from_rocksdb(iterator->key()); + auto value = from_rocksdb(iterator->value()); + TRY_STATUS(f(key, value)); + } + if (!iterator->status().ok()) { + return from_rocksdb(iterator->status()); + } + return Status::OK(); +} + Status RocksDb::begin_write_batch() { CHECK(!transaction_); write_batch_ = std::make_unique(); @@ -239,7 +268,7 @@ Status RocksDb::end_snapshot() { return td::Status::OK(); } -RocksDb::RocksDb(std::shared_ptr db, std::shared_ptr statistics) - : db_(std::move(db)), statistics_(std::move(statistics)) { +RocksDb::RocksDb(std::shared_ptr db, RocksDbOptions options) + : db_(std::move(db)), options_(options) { } } // namespace td diff --git a/tddb/td/db/RocksDb.h b/tddb/td/db/RocksDb.h index 1afba4cc4..5efcd0f48 100644 --- a/tddb/td/db/RocksDb.h +++ b/tddb/td/db/RocksDb.h @@ -24,8 +24,10 @@ #include "td/db/KeyValue.h" #include "td/utils/Status.h" +#include "td/utils/optional.h" namespace rocksdb { +class Cache; class OptimisticTransactionDB; class Transaction; class WriteBatch; @@ -34,16 +36,24 @@ class Statistics; } // namespace rocksdb namespace td { + +struct RocksDbOptions { + std::shared_ptr statistics = nullptr; + std::shared_ptr block_cache; // Default - one 1G cache for all RocksDb + bool use_direct_reads = false; +}; + class RocksDb : public KeyValue { public: static Status destroy(Slice path); RocksDb clone() const; - static Result open(std::string path, std::shared_ptr statistics = nullptr); + static Result open(std::string path, RocksDbOptions options = {}); Result get(Slice key, std::string &value) override; Status set(Slice key, Slice value) override; Status erase(Slice key) override; Result count(Slice prefix) override; + Status for_each(std::function f) override; Status begin_write_batch() override; Status commit_write_batch() override; @@ -64,6 +74,8 @@ class RocksDb : public KeyValue { static std::string statistics_to_string(const std::shared_ptr statistics); static void reset_statistics(const std::shared_ptr statistics); + static std::shared_ptr create_cache(size_t capacity); + RocksDb(RocksDb &&); RocksDb &operator=(RocksDb &&); ~RocksDb(); @@ -74,7 +86,7 @@ class RocksDb : public KeyValue { private: std::shared_ptr db_; - std::shared_ptr statistics_; + RocksDbOptions options_; std::unique_ptr transaction_; std::unique_ptr write_batch_; @@ -87,7 +99,6 @@ class RocksDb : public KeyValue { }; std::unique_ptr snapshot_; - explicit RocksDb(std::shared_ptr db, - std::shared_ptr statistics); + explicit RocksDb(std::shared_ptr db, RocksDbOptions options); }; } // namespace td diff --git a/tddb/td/db/utils/StreamInterface.h b/tddb/td/db/utils/StreamInterface.h index 5262f3df5..cf8f1d519 100644 --- a/tddb/td/db/utils/StreamInterface.h +++ b/tddb/td/db/utils/StreamInterface.h @@ -27,7 +27,7 @@ namespace td { // Generic stream interface // Will to hide implementations details. // CyclicBuffer, ChainBuffer, Bounded ChainBuffer, some clever writers. They all should be interchangable -// Most implementaions will assume that reading and writing may happen concurrently +// Most implementations will assume that reading and writing may happen concurrently class StreamReaderInterface { public: diff --git a/tddb/td/db/utils/StreamToFileActor.cpp b/tddb/td/db/utils/StreamToFileActor.cpp index 24202da41..5a3427d38 100644 --- a/tddb/td/db/utils/StreamToFileActor.cpp +++ b/tddb/td/db/utils/StreamToFileActor.cpp @@ -73,7 +73,7 @@ Result StreamToFileActor::do_loop() { // Also it could be useful to check error and stop immediately. TRY_RESULT(is_closed, is_closed()); - // Flush all data that is awailable on the at the beginning of loop + // Flush all data that is available on the at the beginning of loop TRY_STATUS(do_flush_once()); if ((sync_at_ && sync_at_.is_in_past()) || is_closed) { diff --git a/tdutils/td/utils/CancellationToken.h b/tdutils/td/utils/CancellationToken.h index 9f30d204c..7ef304979 100644 --- a/tdutils/td/utils/CancellationToken.h +++ b/tdutils/td/utils/CancellationToken.h @@ -20,6 +20,7 @@ #include #include +#include "Status.h" namespace td { @@ -38,6 +39,12 @@ class CancellationToken { } return token_->is_cancelled_.load(std::memory_order_acquire); } + Status check() const { + if (*this) { + return Status::Error(653, "cancelled"); // cancelled = 653 + } + return Status::OK(); + } CancellationToken() = default; explicit CancellationToken(std::shared_ptr token) : token_(std::move(token)) { } diff --git a/tdutils/td/utils/Timer.cpp b/tdutils/td/utils/Timer.cpp index 0f6a7d6ed..1f72fba96 100644 --- a/tdutils/td/utils/Timer.cpp +++ b/tdutils/td/utils/Timer.cpp @@ -87,4 +87,8 @@ void PerfWarningTimer::reset() { start_at_ = 0; } +double PerfWarningTimer::elapsed() const { + return Time::now() - start_at_; +} + } // namespace td diff --git a/tdutils/td/utils/Timer.h b/tdutils/td/utils/Timer.h index 18ea35d7c..3e0cafbf5 100644 --- a/tdutils/td/utils/Timer.h +++ b/tdutils/td/utils/Timer.h @@ -53,6 +53,7 @@ class PerfWarningTimer { PerfWarningTimer &operator=(PerfWarningTimer &&) = delete; ~PerfWarningTimer(); void reset(); + double elapsed() const; private: string name_; diff --git a/tdutils/td/utils/filesystem.cpp b/tdutils/td/utils/filesystem.cpp index 562a42816..b84b6b3f8 100644 --- a/tdutils/td/utils/filesystem.cpp +++ b/tdutils/td/utils/filesystem.cpp @@ -68,9 +68,14 @@ Result read_file_impl(CSlice path, int64 size, int64 offset) { return Status::Error("Failed to read file: invalid size"); } auto content = create_empty(narrow_cast(size)); - TRY_RESULT(got_size, from_file.pread(as_mutable_slice(content), offset)); - if (got_size != static_cast(size)) { - return Status::Error("Failed to read file"); + MutableSlice slice = as_mutable_slice(content); + while (!slice.empty()) { + TRY_RESULT(got_size, from_file.pread(slice, offset)); + if (got_size == 0) { + return Status::Error("Failed to read file"); + } + offset += got_size; + slice.remove_prefix(got_size); } from_file.close(); return std::move(content); @@ -103,9 +108,15 @@ Status write_file(CSlice to, Slice data, WriteFileOptions options) { TRY_STATUS(to_file.lock(FileFd::LockFlags::Write, to.str(), 10)); TRY_STATUS(to_file.truncate_to_current_position(0)); } - TRY_RESULT(written, to_file.write(data)); - if (written != size) { - return Status::Error(PSLICE() << "Failed to write file: written " << written << " bytes instead of " << size); + size_t total_written = 0; + while (!data.empty()) { + TRY_RESULT(written, to_file.write(data)); + if (written == 0) { + return Status::Error(PSLICE() << "Failed to write file: written " << total_written << " bytes instead of " + << size); + } + total_written += written; + data.remove_prefix(written); } if (options.need_sync) { TRY_STATUS(to_file.sync()); diff --git a/tdutils/td/utils/optional.h b/tdutils/td/utils/optional.h index 44575948c..7723d2c31 100644 --- a/tdutils/td/utils/optional.h +++ b/tdutils/td/utils/optional.h @@ -66,6 +66,12 @@ class optional { DCHECK(*this); return impl_.ok_ref(); } + T &value_force() { + if (!*this) { + *this = T(); + } + return value(); + } T &operator*() { return value(); } @@ -88,6 +94,14 @@ class optional { impl_.emplace(std::forward(args)...); } + bool operator==(const optional& other) const { + return (bool)*this == (bool)other && (!(bool)*this || value() == other.value()); + } + + bool operator!=(const optional& other) const { + return !(*this == other); + } + private: Result impl_; }; diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index 286a5feaf..60476cf8f 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -347,7 +347,10 @@ class TestNode : public td::actor::Actor { } } } - void send_broadcast(ton::BlockBroadcast broadcast) override { + void send_block_candidate(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override { + } + void send_broadcast(ton::BlockBroadcast broadcast, bool custom_overlays_only) override { } void download_block(ton::BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) override { diff --git a/tl-utils/lite-utils.cpp b/tl-utils/lite-utils.cpp index 9ea7756a7..daa3dbaf0 100644 --- a/tl-utils/lite-utils.cpp +++ b/tl-utils/lite-utils.cpp @@ -159,6 +159,7 @@ std::string lite_query_name_by_id(int id) { {lite_api::liteServer_getLibrariesWithProof::ID, "getLibrariesWithProof"}, {lite_api::liteServer_getShardBlockProof::ID, "getShardBlockProof"}, {lite_api::liteServer_getOutMsgQueueSizes::ID, "getOutMsgQueueSizes"}, + {lite_api::liteServer_getBlockOutMsgQueueSize::ID, "getBlockOutMsgQueueSize"}, {lite_api::liteServer_nonfinal_getCandidate::ID, "nonfinal.getCandidate"}, {lite_api::liteServer_nonfinal_getValidatorGroups::ID, "nonfinal.getValidatorGroups"}}; auto it = names.find(id); diff --git a/tl/generate/scheme/lite_api.tl b/tl/generate/scheme/lite_api.tl index d74e89536..879d7ff4a 100644 --- a/tl/generate/scheme/lite_api.tl +++ b/tl/generate/scheme/lite_api.tl @@ -41,7 +41,8 @@ liteServer.shardInfo id:tonNode.blockIdExt shardblk:tonNode.blockIdExt shard_pro liteServer.allShardsInfo id:tonNode.blockIdExt proof:bytes data:bytes = liteServer.AllShardsInfo; liteServer.transactionInfo id:tonNode.blockIdExt proof:bytes transaction:bytes = liteServer.TransactionInfo; liteServer.transactionList ids:(vector tonNode.blockIdExt) transactions:bytes = liteServer.TransactionList; -liteServer.transactionId mode:# account:mode.0?int256 lt:mode.1?long hash:mode.2?int256 = liteServer.TransactionId; +liteServer.transactionMetadata mode:# depth:int initiator:liteServer.accountId initiator_lt:long = liteServer.TransactionMetadata; +liteServer.transactionId#b12f65af mode:# account:mode.0?int256 lt:mode.1?long hash:mode.2?int256 metadata:mode.8?liteServer.transactionMetadata = liteServer.TransactionId; liteServer.transactionId3 account:int256 lt:long = liteServer.TransactionId3; liteServer.blockTransactions id:tonNode.blockIdExt req_count:# incomplete:Bool ids:(vector liteServer.transactionId) proof:bytes = liteServer.BlockTransactions; liteServer.blockTransactionsExt id:tonNode.blockIdExt req_count:# incomplete:Bool transactions:bytes proof:bytes = liteServer.BlockTransactionsExt; @@ -59,6 +60,7 @@ liteServer.shardBlockProof masterchain_id:tonNode.blockIdExt links:(vector liteS liteServer.lookupBlockResult id:tonNode.blockIdExt mode:# mc_block_id:tonNode.blockIdExt client_mc_state_proof:bytes mc_block_proof:bytes shard_links:(vector liteServer.shardBlockLink) header:bytes prev_header:bytes = liteServer.LookupBlockResult; liteServer.outMsgQueueSize id:tonNode.blockIdExt size:int = liteServer.OutMsgQueueSize; liteServer.outMsgQueueSizes shards:(vector liteServer.outMsgQueueSize) ext_msg_queue_size_limit:int = liteServer.OutMsgQueueSizes; +liteServer.blockOutMsgQueueSize mode:# id:tonNode.blockIdExt size:long proof:mode.0?bytes = liteServer.BlockOutMsgQueueSize; liteServer.debug.verbosity value:int = liteServer.debug.Verbosity; @@ -97,8 +99,9 @@ liteServer.getLibraries library_list:(vector int256) = liteServer.LibraryResult; liteServer.getLibrariesWithProof id:tonNode.blockIdExt mode:# library_list:(vector int256) = liteServer.LibraryResultWithProof; liteServer.getShardBlockProof id:tonNode.blockIdExt = liteServer.ShardBlockProof; liteServer.getOutMsgQueueSizes mode:# wc:mode.0?int shard:mode.0?long = liteServer.OutMsgQueueSizes; +liteServer.getBlockOutMsgQueueSize mode:# id:tonNode.blockIdExt want_proof:mode.0?true = liteServer.BlockOutMsgQueueSize; -liteServer.nonfinal.getValidatorGroups mode:# wc:mode.0?int shard:mode.1?long = liteServer.nonfinal.ValidatorGroups; +liteServer.nonfinal.getValidatorGroups mode:# wc:mode.0?int shard:mode.0?long = liteServer.nonfinal.ValidatorGroups; liteServer.nonfinal.getCandidate id:liteServer.nonfinal.candidateId = liteServer.nonfinal.Candidate; liteServer.queryPrefix = Object; diff --git a/tl/generate/scheme/lite_api.tlo b/tl/generate/scheme/lite_api.tlo index ccae66f36..6ece1d20f 100644 Binary files a/tl/generate/scheme/lite_api.tlo and b/tl/generate/scheme/lite_api.tlo differ diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index c61bad858..b33ca5425 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -396,6 +396,11 @@ tonNode.blockBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int valida tonNode.ihrMessageBroadcast message:tonNode.ihrMessage = tonNode.Broadcast; tonNode.externalMessageBroadcast message:tonNode.externalMessage = tonNode.Broadcast; tonNode.newShardBlockBroadcast block:tonNode.newShardBlock = tonNode.Broadcast; +// signature may be empty, at least for now +tonNode.newBlockCandidateBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int + collator_signature:tonNode.blockSignature data:bytes = tonNode.Broadcast; +tonNode.newBlockCandidateBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int + collator_signature:tonNode.blockSignature flags:# compressed:bytes = tonNode.Broadcast; tonNode.shardPublicOverlayId workchain:int shard:long zero_state_file_hash:int256 = tonNode.ShardPublicOverlayId; @@ -587,18 +592,25 @@ engine.dht.config dht:(vector engine.dht) gc:engine.gc = engine.dht.Config; engine.validator.fullNodeMaster port:int adnl:int256 = engine.validator.FullNodeMaster; engine.validator.fullNodeSlave ip:int port:int adnl:PublicKey = engine.validator.FullNodeSlave; engine.validator.fullNodeConfig ext_messages_broadcast_disabled:Bool = engine.validator.FullNodeConfig; -engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) +engine.validator.extraConfig state_serializer_enabled:Bool = engine.validator.ExtraConfig; +engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) dht:(vector engine.dht) validators:(vector engine.validator) fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave) fullnodemasters:(vector engine.validator.fullNodeMaster) fullnodeconfig:engine.validator.fullNodeConfig + extraconfig:engine.validator.extraConfig liteservers:(vector engine.liteServer) control:(vector engine.controlInterface) gc:engine.gc = engine.validator.Config; -engine.validator.customOverlayNode adnl_id:int256 msg_sender:Bool msg_sender_priority:int = engine.validator.CustomOverlayNode; +engine.validator.customOverlayNode adnl_id:int256 msg_sender:Bool msg_sender_priority:int block_sender:Bool = engine.validator.CustomOverlayNode; engine.validator.customOverlay name:string nodes:(vector engine.validator.customOverlayNode) = engine.validator.CustomOverlay; engine.validator.customOverlaysConfig overlays:(vector engine.validator.customOverlay) = engine.validator.CustomOverlaysConfig; +engine.validator.collatorOptions + deferring_enabled:Bool defer_messages_after:int defer_out_queue_size_limit:long + dispatch_phase_2_max_total:int dispatch_phase_3_max_total:int + dispatch_phase_2_max_per_initiator:int dispatch_phase_3_max_per_initiator:int = engine.validator.CollatorOptions; + ---functions--- ---types--- @@ -646,7 +658,7 @@ engine.validator.onePerfTimerStat time:int min:double avg:double max:double = en engine.validator.perfTimerStatsByName name:string stats:(vector engine.validator.OnePerfTimerStat) = engine.validator.PerfTimerStatsByName; engine.validator.perfTimerStats stats:(vector engine.validator.PerfTimerStatsByName) = engine.validator.PerfTimerStats; -engine.validator.shardOutQueueSize size:int = engine.validator.ShardOutQueueSize; +engine.validator.shardOutQueueSize size:long = engine.validator.ShardOutQueueSize; ---functions--- @@ -706,6 +718,11 @@ engine.validator.addCustomOverlay overlay:engine.validator.customOverlay = engin engine.validator.delCustomOverlay name:string = engine.validator.Success; engine.validator.showCustomOverlays = engine.validator.CustomOverlaysConfig; +engine.validator.setStateSerializerEnabled enabled:Bool = engine.validator.Success; + +engine.validator.setCollatorOptionsJson json:string = engine.validator.Success; +engine.validator.getCollatorOptionsJson = engine.validator.JsonConfig; + ---types--- storage.pong = storage.Pong; @@ -752,15 +769,26 @@ http.server.config dhs:(vector http.server.dnsEntry) local_hosts:(vector http.se ---types--- -validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int block_timestamp:long comment:string = validatorSession.StatsProducer; +validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int comment:string + block_timestamp:double is_accepted:Bool is_ours:Bool got_submit_at:double + collation_time:double collated_at:double collation_cached:Bool + validation_time:double validated_at:double validation_cached:Bool + gen_utime:double + approved_weight:long approved_33pct_at:double approved_66pct_at:double + signed_weight:long signed_33pct_at:double signed_66pct_at:double + serialize_time:double deserialize_time:double serialized_size:int = validatorSession.StatsProducer; -validatorSession.statsRound timestamp:long producers:(vector validatorSession.statsProducer) = validatorSession.StatsRound; +validatorSession.statsRound timestamp:double producers:(vector validatorSession.statsProducer) = validatorSession.StatsRound; -validatorSession.stats success:Bool id:tonNode.blockIdExt timestamp:long self:int256 session_id:int256 cc_seqno:int +validatorSession.stats success:Bool id:tonNode.blockIdExt timestamp:double self:int256 session_id:int256 cc_seqno:int creator:int256 total_validators:int total_weight:long signatures:int signatures_weight:long approve_signatures:int approve_signatures_weight:long first_round:int rounds:(vector validatorSession.statsRound) = validatorSession.Stats; +validatorSession.newValidatorGroupStats.node id:int256 weight:long = validatorSession.newValidatorGroupStats.Node; +validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:long cc_seqno:int timestamp:double + self_idx:int nodes:(vector validatorSession.newValidatorGroupStats.node) = validatorSession.NewValidatorGroupStats; + ---functions--- ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index b8f5d5a49..da1aa331d 100644 Binary files a/tl/generate/scheme/ton_api.tlo and b/tl/generate/scheme/ton_api.tlo differ diff --git a/tl/generate/scheme/tonlib_api.tl b/tl/generate/scheme/tonlib_api.tl index a6172376a..6cf40d005 100644 --- a/tl/generate/scheme/tonlib_api.tl +++ b/tl/generate/scheme/tonlib_api.tl @@ -230,6 +230,8 @@ blocks.blockSignatures id:ton.blockIdExt signatures:(vector blocks.signature) = blocks.shardBlockLink id:ton.blockIdExt proof:bytes = blocks.ShardBlockLink; blocks.blockLinkBack to_key_block:Bool from:ton.blockIdExt to:ton.blockIdExt dest_proof:bytes proof:bytes state_proof:bytes = blocks.BlockLinkBack; blocks.shardBlockProof from:ton.blockIdExt mc_id:ton.blockIdExt links:(vector blocks.shardBlockLink) mc_proof:(vector blocks.blockLinkBack) = blocks.ShardBlockProof; +blocks.outMsgQueueSize id:ton.blockIdExt size:int32 = blocks.OutMsgQueueSize; +blocks.outMsgQueueSizes shards:(vector blocks.outMsgQueueSize) ext_msg_queue_size_limit:int32 = blocks.OutMsgQueueSizes; configInfo config:tvm.cell = ConfigInfo; @@ -309,6 +311,7 @@ smc.forget id:int53 = Ok; smc.getCode id:int53 = tvm.Cell; smc.getData id:int53 = tvm.Cell; smc.getState id:int53 = tvm.Cell; +smc.getRawFullAccountState id:int53 = raw.FullAccountState; smc.runGetMethod id:int53 method:smc.MethodId stack:vector = smc.RunResult; smc.getLibraries library_list:(vector int256) = smc.LibraryResult; @@ -331,6 +334,7 @@ blocks.getTransactionsExt id:ton.blockIdExt mode:# count:# after:blocks.accountT blocks.getBlockHeader id:ton.blockIdExt = blocks.Header; blocks.getMasterchainBlockSignatures seqno:int32 = blocks.BlockSignatures; blocks.getShardBlockProof id:ton.blockIdExt mode:# from:mode.0?ton.blockIdExt = blocks.ShardBlockProof; +blocks.getOutMsgQueueSizes mode:# wc:mode.0?int32 shard:mode.0?int64 = blocks.OutMsgQueueSizes; onLiteServerQueryResult id:int64 bytes:bytes = Ok; onLiteServerQueryError id:int64 error:error = Ok; diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index 7657852ea..686bd9181 100644 Binary files a/tl/generate/scheme/tonlib_api.tlo and b/tl/generate/scheme/tonlib_api.tlo differ diff --git a/ton/ton-types.h b/ton/ton-types.h index 24d542599..915682655 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -57,7 +57,10 @@ enum GlobalCapabilities { capBounceMsgBody = 4, capReportVersion = 8, capSplitMergeTransactions = 16, - capShortDequeue = 32 + capShortDequeue = 32, + capStoreOutMsgQueueSize = 64, + capMsgMetadata = 128, + capDeferMessages = 256 }; inline int shard_pfx_len(ShardId shard) { @@ -344,6 +347,10 @@ struct BlockSignature { struct ReceivedBlock { BlockIdExt id; td::BufferSlice data; + + ReceivedBlock clone() const { + return ReceivedBlock{id, data.clone()}; + } }; struct BlockBroadcast { diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 86acbcc37..f62b9ae47 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -1891,7 +1891,9 @@ class RunEmulator : public TonlibQueryActor { if (stopped_) { return; } - get_block_id([self = this](td::Result&& block_id) { self->set_block_id(std::move(block_id)); }); + get_block_id([SelfId = actor_id(this)](td::Result&& block_id) { + td::actor::send_closure(SelfId, &RunEmulator::set_block_id, std::move(block_id)); + }); } void set_block_id(td::Result&& block_id) { @@ -1900,8 +1902,12 @@ class RunEmulator : public TonlibQueryActor { } else { block_id_ = block_id.move_as_ok(); - get_mc_state_root([self = this](td::Result>&& mc_state_root) { self->set_mc_state_root(std::move(mc_state_root)); }); - get_account_state([self = this](td::Result>&& state) { self->set_account_state(std::move(state)); }); + get_mc_state_root([SelfId = actor_id(this)](td::Result>&& mc_state_root) { + td::actor::send_closure(SelfId, &RunEmulator::set_mc_state_root, std::move(mc_state_root)); + }); + get_account_state([SelfId = actor_id(this)](td::Result>&& state) { + td::actor::send_closure(SelfId, &RunEmulator::set_account_state, std::move(state)); + }); check(get_transactions(0)); inc(); @@ -1923,7 +1929,9 @@ class RunEmulator : public TonlibQueryActor { } else { account_state_ = account_state.move_as_ok(); send_query(int_api::ScanAndLoadGlobalLibs{account_state_->get_raw_state()}, - [self = this](td::Result R) { self->set_global_libraries(std::move(R)); }); + [SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(SelfId, &RunEmulator::set_global_libraries, std::move(R)); + }); } } @@ -1963,7 +1971,7 @@ class RunEmulator : public TonlibQueryActor { check(r_config.move_as_error()); return; } - std::unique_ptr config = r_config.move_as_ok(); + std::shared_ptr config = r_config.move_as_ok(); auto r_shard_account = account_state_->to_shardAccountCellSlice(); if (r_shard_account.is_error()) { @@ -1987,7 +1995,7 @@ class RunEmulator : public TonlibQueryActor { return; } vm::Dictionary libraries = global_libraries_; - emulator::TransactionEmulator trans_emulator(std::move(*config)); + emulator::TransactionEmulator trans_emulator(config); trans_emulator.set_prev_blocks_info(prev_blocks_info.move_as_ok()); trans_emulator.set_libs(std::move(libraries)); trans_emulator.set_rand_seed(block_id_.rand_seed); @@ -4352,6 +4360,17 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_getState& request, return td::Status::OK(); } +td::Status TonlibClient::do_request(const tonlib_api::smc_getRawFullAccountState& request, + td::Promise>&& promise) { + auto it = smcs_.find(request.id_); + if (it == smcs_.end()) { + return TonlibError::InvalidSmcId(); + } + auto& acc = it->second; + promise.set_result(acc->to_raw_fullAccountState()); + return td::Status::OK(); +} + bool is_list(vm::StackEntry entry) { while (true) { if (entry.type() == vm::StackEntry::Type::t_null) { @@ -5521,7 +5540,7 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getShards& request, } block::ShardConfig sh_conf; - if (!sh_conf.unpack(mc_extra.shard_hashes)) { + if (!sh_conf.unpack(data_csr)) { return td::Status::Error("cannot extract shard block list from shard configuration"); } auto ids = sh_conf.get_shard_hash_ids(true); @@ -5544,7 +5563,9 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getShards& request, return td::Status::OK(); } -td::Status check_lookup_block_proof(lite_api_ptr& result, int mode, ton::BlockId blkid, ton::BlockIdExt client_mc_blkid, td::uint64 lt, td::uint32 utime); +td::Status check_lookup_block_proof(lite_api_ptr& result, int mode, + ton::BlockId blkid, ton::BlockIdExt client_mc_blkid, td::uint64 lt, + td::uint32 utime); td::Status TonlibClient::do_request(const tonlib_api::blocks_lookupBlock& request, td::Promise>&& promise) { @@ -5553,6 +5574,11 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_lookupBlock& reques client_.with_last_block( [self = this, blkid, lite_block = std::move(lite_block), mode = request.mode_, lt = (td::uint64)request.lt_, utime = (td::uint32)request.utime_, promise = std::move(promise)](td::Result r_last_block) mutable { + if (r_last_block.is_error()) { + promise.set_error(r_last_block.move_as_error_prefix(TonlibError::Internal("get last block failed "))); + return; + } + self->client_.send_query(ton::lite_api::liteServer_lookupBlockWithProof(mode, std::move(lite_block), ton::create_tl_lite_block_id(r_last_block.ok().last_block_id), lt, utime), promise.wrap([blkid, mode, utime, lt, last_block = r_last_block.ok().last_block_id](lite_api_ptr&& result) -> td::Result> { @@ -5730,7 +5756,7 @@ auto to_tonlib_api(const ton::lite_api::liteServer_transactionId& txid) td::Status check_block_transactions_proof(lite_api_ptr& bTxes, int32_t mode, ton::LogicalTime start_lt, td::Bits256 start_addr, td::Bits256 root_hash, int req_count) { - if (mode & ton::lite_api::liteServer_listBlockTransactions::WANT_PROOF_MASK == 0) { + if ((mode & ton::lite_api::liteServer_listBlockTransactions::WANT_PROOF_MASK) == 0) { return td::Status::OK(); } constexpr int max_answer_transactions = 256; @@ -6021,6 +6047,24 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getShardBlockProof& return td::Status::OK(); } +td::Status TonlibClient::do_request(const tonlib_api::blocks_getOutMsgQueueSizes& request, + td::Promise>&& promise) { + client_.send_query(ton::lite_api::liteServer_getOutMsgQueueSizes(request.mode_, request.wc_, request.shard_), + promise.wrap([](lite_api_ptr&& queue_sizes) { + tonlib_api::blocks_outMsgQueueSizes result; + result.ext_msg_queue_size_limit_ = queue_sizes->ext_msg_queue_size_limit_; + for (auto &x : queue_sizes->shards_) { + tonlib_api::blocks_outMsgQueueSize shard; + shard.id_ = to_tonlib_api(*x->id_); + shard.size_ = x->size_; + result.shards_.push_back(tonlib_api::make_object(std::move(shard))); + } + return tonlib_api::make_object(std::move(result)); + })); + + return td::Status::OK(); +} + void TonlibClient::load_libs_from_disk() { LOG(DEBUG) << "loading libraries from disk cache"; auto r_data = kv_->get("tonlib.libcache"); diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index 001df748c..7db443247 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -324,6 +324,8 @@ class TonlibClient : public td::actor::Actor { td::Promise>&& promise); td::Status do_request(const tonlib_api::smc_getState& request, td::Promise>&& promise); + td::Status do_request(const tonlib_api::smc_getRawFullAccountState& request, + td::Promise>&& promise); td::Status do_request(const tonlib_api::smc_runGetMethod& request, td::Promise>&& promise); @@ -390,6 +392,8 @@ class TonlibClient : public td::actor::Actor { td::Promise>&& promise); td::Status do_request(const tonlib_api::blocks_getShardBlockProof& request, td::Promise>&& promise); + td::Status do_request(const tonlib_api::blocks_getOutMsgQueueSizes& request, + td::Promise>&& promise); void get_config_param(int32_t param, int32_t mode, ton::BlockIdExt block, td::Promise>&& promise); diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index 1107a5d04..5a32b50f3 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -1423,7 +1423,8 @@ class TonlibCli : public td::actor::Actor { if (r_cell.is_error()) { sb << ""; } - auto cs = vm::load_cell_slice(r_cell.move_as_ok()); + bool spec = true; + auto cs = vm::load_cell_slice_special(r_cell.move_as_ok(), spec); std::stringstream ss; cs.print_rec(ss); sb << ss.str(); diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 956c23aa0..41721ab96 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1171,11 +1171,95 @@ td::Status ShowCustomOverlaysQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << "Overlay \"" << overlay->name_ << "\": " << overlay->nodes_.size() << " nodes\n"; for (const auto &node : overlay->nodes_) { td::TerminalIO::out() << " " << node->adnl_id_ - << (node->msg_sender_ ? (PSTRING() << " (sender, p=" << node->msg_sender_priority_ << ")") - : "") - << "\n"; + << (node->msg_sender_ + ? (PSTRING() << " (msg sender, p=" << node->msg_sender_priority_ << ")") + : "") + << (node->block_sender_ ? " (block sender)" : "") << "\n"; } td::TerminalIO::out() << "\n"; } return td::Status::OK(); } + +td::Status SetStateSerializerEnabledQuery::run() { + TRY_RESULT(value, tokenizer_.get_token()); + if (value != 0 && value != 1) { + return td::Status::Error("expected 0 or 1"); + } + TRY_STATUS(tokenizer_.check_endl()); + enabled_ = value; + return td::Status::OK(); +} + +td::Status SetStateSerializerEnabledQuery::send() { + auto b = ton::create_serialize_tl_object(enabled_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status SetStateSerializerEnabledQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status SetCollatorOptionsJsonQuery::run() { + TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status SetCollatorOptionsJsonQuery::send() { + TRY_RESULT(data, td::read_file(file_name_)); + auto b = + ton::create_serialize_tl_object(data.as_slice().str()); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status SetCollatorOptionsJsonQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status ResetCollatorOptionsQuery::run() { + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status ResetCollatorOptionsQuery::send() { + auto b = ton::create_serialize_tl_object("{}"); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ResetCollatorOptionsQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status GetCollatorOptionsJsonQuery::run() { + TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status GetCollatorOptionsJsonQuery::send() { + auto b = + ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status GetCollatorOptionsJsonQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + TRY_STATUS(td::write_file(file_name_, f->data_)); + td::TerminalIO::out() << "saved config to " << file_name_ << "\n"; + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 34b516d6c..08ac1572a 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1207,3 +1207,88 @@ class ShowCustomOverlaysQuery : public Query { return get_name(); } }; + +class SetStateSerializerEnabledQuery : public Query { + public: + SetStateSerializerEnabledQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "setstateserializerenabled"; + } + static std::string get_help() { + return "setstateserializerenabled \tdisable or enable persistent state serializer; value is 0 or 1"; + } + std::string name() const override { + return get_name(); + } + + private: + bool enabled_; +}; + +class SetCollatorOptionsJsonQuery : public Query { + public: + SetCollatorOptionsJsonQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "setcollatoroptionsjson"; + } + static std::string get_help() { + return "setcollatoroptionsjson \tset collator options from file "; + } + std::string name() const override { + return get_name(); + } + + private: + std::string file_name_; +}; + +class ResetCollatorOptionsQuery : public Query { + public: + ResetCollatorOptionsQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "resetcollatoroptions"; + } + static std::string get_help() { + return "resetcollatoroptions\tset collator options to default values"; + } + std::string name() const override { + return get_name(); + } +}; + +class GetCollatorOptionsJsonQuery : public Query { + public: + GetCollatorOptionsJsonQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "getcollatoroptionsjson"; + } + static std::string get_help() { + return "getcollatoroptionsjson \tsave current collator options to file "; + } + std::string name() const override { + return get_name(); + } + + private: + std::string file_name_; +}; diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 263bca3c0..d8a230801 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -146,6 +146,10 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 8ba991782..b75bdd8af 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -73,6 +73,7 @@ #include "block-parse.h" #include "common/delay.h" #include "block/precompiled-smc/PrecompiledSmartContract.h" +#include "interfaces/validator-manager.h" Config::Config() { out_port = 3278; @@ -155,6 +156,11 @@ Config::Config(ton::ton_api::engine_validator_config &config) { if (config.fullnodeconfig_) { full_node_config = ton::validator::fullnode::FullNodeConfig(config.fullnodeconfig_); } + if (config.extraconfig_) { + state_serializer_enabled = config.extraconfig_->state_serializer_enabled_; + } else { + state_serializer_enabled = true; + } for (auto &serv : config.liteservers_) { config_add_lite_server(ton::PublicKeyHash{serv->id_}, serv->port_).ensure(); @@ -231,6 +237,12 @@ ton::tl_object_ptr Config::tl() const { full_node_config_obj = full_node_config.tl(); } + ton::tl_object_ptr extra_config_obj = {}; + if (!state_serializer_enabled) { + // Non-default values + extra_config_obj = ton::create_tl_object(state_serializer_enabled); + } + std::vector> liteserver_vec; for (auto &x : liteservers) { liteserver_vec.push_back(ton::create_tl_object(x.second.tl(), x.first)); @@ -253,7 +265,7 @@ ton::tl_object_ptr Config::tl() const { return ton::create_tl_object( out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(full_node_config_obj), - std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec)); + std::move(extra_config_obj), std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec)); } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, @@ -1369,6 +1381,17 @@ td::Status ValidatorEngine::load_global_config() { validator_options_.write().set_archive_preload_period(archive_preload_period_); validator_options_.write().set_disable_rocksdb_stats(disable_rocksdb_stats_); validator_options_.write().set_nonfinal_ls_queries_enabled(nonfinal_ls_queries_enabled_); + if (celldb_cache_size_) { + validator_options_.write().set_celldb_cache_size(celldb_cache_size_.value()); + } + if (!celldb_cache_size_ || celldb_cache_size_.value() < (30ULL << 30)) { + celldb_direct_io_ = false; + } + validator_options_.write().set_celldb_direct_io(celldb_direct_io_); + validator_options_.write().set_celldb_preload_all(celldb_preload_all_); + if (catchain_max_block_delay_) { + validator_options_.write().set_catchain_max_block_delay(catchain_max_block_delay_.value()); + } std::vector h; for (auto &x : conf.validator_->hardforks_) { @@ -1799,6 +1822,9 @@ void ValidatorEngine::started_overlays() { void ValidatorEngine::start_validator() { validator_options_.write().set_allow_blockchain_init(config_.validators.size() > 0); + validator_options_.write().set_state_serializer_enabled(config_.state_serializer_enabled); + load_collator_options(); + validator_manager_ = ton::validator::ValidatorManagerFactory::create( validator_options_, db_root_, keyring_.get(), adnl_.get(), rldp_.get(), overlay_manager_.get()); @@ -2357,16 +2383,9 @@ void ValidatorEngine::load_custom_overlays_config() { } for (auto &overlay : custom_overlays_config_->overlays_) { - std::vector nodes; - std::map senders; - for (const auto &node : overlay->nodes_) { - nodes.emplace_back(node->adnl_id_); - if (node->msg_sender_) { - senders[ton::adnl::AdnlNodeIdShort{node->adnl_id_}] = node->msg_sender_priority_; - } - } - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_ext_msg_overlay, std::move(nodes), - std::move(senders), overlay->name_, [](td::Result R) { R.ensure(); }); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_custom_overlay, + ton::validator::fullnode::CustomOverlayParams::fetch(*overlay), + [](td::Result R) { R.ensure(); }); } } @@ -2396,6 +2415,69 @@ void ValidatorEngine::del_custom_overlay_from_config(std::string name, td::Promi promise.set_error(td::Status::Error(PSTRING() << "no overlay \"" << name << "\" in config")); } +static td::Result> parse_collator_options(td::MutableSlice json_str) { + td::Ref ref{true}; + ton::validator::CollatorOptions& opts = ref.write(); + + // Set default values (from_json leaves missing fields as is) + ton::ton_api::engine_validator_collatorOptions f; + f.deferring_enabled_ = opts.deferring_enabled; + f.defer_out_queue_size_limit_ = opts.defer_out_queue_size_limit; + f.defer_messages_after_ = opts.defer_messages_after; + f.dispatch_phase_2_max_total_ = opts.dispatch_phase_2_max_total; + f.dispatch_phase_3_max_total_ = opts.dispatch_phase_3_max_total; + f.dispatch_phase_2_max_per_initiator_ = opts.dispatch_phase_2_max_per_initiator; + f.dispatch_phase_3_max_per_initiator_ = + opts.dispatch_phase_3_max_per_initiator ? opts.dispatch_phase_3_max_per_initiator.value() : -1; + + TRY_RESULT_PREFIX(json, td::json_decode(json_str), "failed to parse json: "); + TRY_STATUS_PREFIX(ton::ton_api::from_json(f, json.get_object()), "json does not fit TL scheme: "); + + if (f.defer_messages_after_ <= 0) { + return td::Status::Error("defer_messages_after should be positive"); + } + if (f.defer_out_queue_size_limit_ < 0) { + return td::Status::Error("defer_out_queue_size_limit should be non-negative"); + } + if (f.dispatch_phase_2_max_total_ < 0) { + return td::Status::Error("dispatch_phase_2_max_total should be non-negative"); + } + if (f.dispatch_phase_3_max_total_ < 0) { + return td::Status::Error("dispatch_phase_3_max_total should be non-negative"); + } + if (f.dispatch_phase_2_max_per_initiator_ < 0) { + return td::Status::Error("dispatch_phase_2_max_per_initiator should be non-negative"); + } + + opts.deferring_enabled = f.deferring_enabled_; + opts.defer_messages_after = f.defer_messages_after_; + opts.defer_out_queue_size_limit = f.defer_out_queue_size_limit_; + opts.dispatch_phase_2_max_total = f.dispatch_phase_2_max_total_; + opts.dispatch_phase_3_max_total = f.dispatch_phase_3_max_total_; + opts.dispatch_phase_2_max_per_initiator = f.dispatch_phase_2_max_per_initiator_; + if (f.dispatch_phase_3_max_per_initiator_ >= 0) { + opts.dispatch_phase_3_max_per_initiator = f.dispatch_phase_3_max_per_initiator_; + } else { + opts.dispatch_phase_3_max_per_initiator = {}; + } + + return ref; +} + +void ValidatorEngine::load_collator_options() { + auto r_data = td::read_file(collator_options_file()); + if (r_data.is_error()) { + return; + } + td::BufferSlice data = r_data.move_as_ok(); + auto r_collator_options = parse_collator_options(data.as_slice()); + if (r_collator_options.is_error()) { + LOG(ERROR) << "Failed to read collator options from file: " << r_collator_options.move_as_error(); + return; + } + validator_options_.write().set_collator_options(r_collator_options.move_as_ok()); +} + void ValidatorEngine::check_key(ton::PublicKeyHash id, td::Promise promise) { if (keys_.count(id) == 1) { promise.set_value(td::Unit()); @@ -3475,7 +3557,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getShardO if (!dest) { td::actor::send_closure( manager, &ton::validator::ValidatorManagerInterface::get_out_msg_queue_size, handle->id(), - [promise = std::move(promise)](td::Result R) mutable { + [promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error_prefix("failed to get queue size: "))); } else { @@ -3571,11 +3653,10 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCustom senders[ton::adnl::AdnlNodeIdShort{node->adnl_id_}] = node->msg_sender_priority_; } } - std::string name = overlay->name_; + auto params = ton::validator::fullnode::CustomOverlayParams::fetch(*query.overlay_); td::actor::send_closure( - full_node_, &ton::validator::fullnode::FullNode::add_ext_msg_overlay, std::move(nodes), std::move(senders), - std::move(name), - [SelfId = actor_id(this), overlay = std::move(overlay), + full_node_, &ton::validator::fullnode::FullNode::add_custom_overlay, std::move(params), + [SelfId = actor_id(this), overlay = std::move(query.overlay_), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); @@ -3605,7 +3686,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCustom return; } td::actor::send_closure( - full_node_, &ton::validator::fullnode::FullNode::del_ext_msg_overlay, query.name_, + full_node_, &ton::validator::fullnode::FullNode::del_custom_overlay, query.name_, [SelfId = actor_id(this), name = query.name_, promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); @@ -3639,6 +3720,81 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCusto custom_overlays_config_, true)); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setStateSerializerEnabled &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + if (query.enabled_ == validator_options_->get_state_serializer_enabled()) { + promise.set_value(ton::create_serialize_tl_object()); + return; + } + validator_options_.write().set_state_serializer_enabled(query.enabled_); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + config_.state_serializer_enabled = query.enabled_; + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value(ton::create_serialize_tl_object()); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setCollatorOptionsJson &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + auto r_collator_options = parse_collator_options(query.json_); + if (r_collator_options.is_error()) { + promise.set_value(create_control_query_error(r_collator_options.move_as_error_prefix("failed to parse json: "))); + return; + } + auto S = td::write_file(collator_options_file(), query.json_); + if (S.is_error()) { + promise.set_value(create_control_query_error(r_collator_options.move_as_error_prefix("failed to write file: "))); + return; + } + validator_options_.write().set_collator_options(r_collator_options.move_as_ok()); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + promise.set_value(ton::create_serialize_tl_object()); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getCollatorOptionsJson &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + auto r_data = td::read_file(collator_options_file()); + if (r_data.is_error()) { + promise.set_value(ton::create_serialize_tl_object("{}")); + } else { + promise.set_value( + ton::create_serialize_tl_object(r_data.ok().as_slice().str())); + } +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { @@ -3853,7 +4009,7 @@ int main(int argc, char *argv[]) { acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_max_mempool_num, v); }); return td::Status::OK(); }); - p.add_checked_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=7*86400", + p.add_checked_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=86400", [&](td::Slice fname) { auto v = td::to_double(fname); if (v <= 0) { @@ -3863,7 +4019,7 @@ int main(int argc, char *argv[]) { return td::Status::OK(); }); p.add_checked_option( - 'A', "archive-ttl", "archived blocks will be deleted after this time (in seconds) default=365*86400", + 'A', "archive-ttl", "archived blocks will be deleted after this time (in seconds) default=7*86400", [&](td::Slice fname) { auto v = td::to_double(fname); if (v <= 0) { @@ -3975,6 +4131,33 @@ int main(int argc, char *argv[]) { p.add_option('\0', "nonfinal-ls", "enable special LS queries to non-finalized blocks", [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_nonfinal_ls_queries_enabled); }); }); + p.add_checked_option( + '\0', "celldb-cache-size", "block cache size for RocksDb in CellDb, in bytes (default: 1G)", + [&](td::Slice s) -> td::Status { + TRY_RESULT(v, td::to_integer_safe(s)); + if (v == 0) { + return td::Status::Error("celldb-cache-size should be positive"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_cache_size, v); }); + return td::Status::OK(); + }); + p.add_option( + '\0', "celldb-direct-io", "enable direct I/O mode for RocksDb in CellDb (doesn't apply when celldb cache is < 30G)", + [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_direct_io, true); }); }); + p.add_option( + '\0', "celldb-preload-all", + "preload all cells from CellDb on startup (recommended to use with big enough celldb-cache-size and celldb-direct-io)", + [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_preload_all, true); }); }); + p.add_checked_option( + '\0', "catchain-max-block-delay", "delay before creating a new catchain block, in seconds (default: 0.5)", + [&](td::Slice s) -> td::Status { + auto v = td::to_double(s); + if (v < 0) { + return td::Status::Error("catchain-max-block-delay should be non-negative"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_catchain_max_block_delay, v); }); + return td::Status::OK(); + }); auto S = p.run(argc, argv); if (S.is_error()) { LOG(ERROR) << "failed to parse options: " << S.move_as_error(); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 0a99dfcc8..436706ff0 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -90,6 +90,8 @@ struct Config { std::map controls; std::set gc; + bool state_serializer_enabled = true; + void decref(ton::PublicKeyHash key); void incref(ton::PublicKeyHash key) { keys_refcnt[key]++; @@ -209,6 +211,10 @@ class ValidatorEngine : public td::actor::Actor { double archive_preload_period_ = 0.0; bool disable_rocksdb_stats_ = false; bool nonfinal_ls_queries_enabled_ = false; + td::optional celldb_cache_size_ = 1LL << 30; + bool celldb_direct_io_ = false; + bool celldb_preload_all_ = false; + td::optional catchain_max_block_delay_; bool read_config_ = false; bool started_keyring_ = false; bool started_ = false; @@ -281,6 +287,18 @@ class ValidatorEngine : public td::actor::Actor { void set_nonfinal_ls_queries_enabled() { nonfinal_ls_queries_enabled_ = true; } + void set_celldb_cache_size(td::uint64 value) { + celldb_cache_size_ = value; + } + void set_celldb_direct_io(bool value) { + celldb_direct_io_ = value; + } + void set_celldb_preload_all(bool value) { + celldb_preload_all_ = value; + } + void set_catchain_max_block_delay(double value) { + catchain_max_block_delay_ = value; + } void start_up() override; ValidatorEngine() { } @@ -366,12 +384,16 @@ class ValidatorEngine : public td::actor::Actor { std::string custom_overlays_config_file() const { return db_root_ + "/custom-overlays.json"; } + std::string collator_options_file() const { + return db_root_ + "/collator-options.json"; + } void load_custom_overlays_config(); td::Status write_custom_overlays_config(); void add_custom_overlay_to_config( ton::tl_object_ptr overlay, td::Promise promise); void del_custom_overlay_from_config(std::string name, td::Promise promise); + void load_collator_options(); void check_key(ton::PublicKeyHash id, td::Promise promise); @@ -457,6 +479,12 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_showCustomOverlays &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_setStateSerializerEnabled &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_setCollatorOptionsJson &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_getCollatorOptionsJson &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator-session/candidate-serializer.cpp b/validator-session/candidate-serializer.cpp index c220b4a95..a85c4b72e 100644 --- a/validator-session/candidate-serializer.cpp +++ b/validator-session/candidate-serializer.cpp @@ -14,7 +14,6 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#pragma once #include "candidate-serializer.h" #include "tl-utils/tl-utils.hpp" #include "vm/boc.h" diff --git a/validator-session/validator-session-types.h b/validator-session/validator-session-types.h index bcbaa8f71..e13c36d24 100644 --- a/validator-session/validator-session-types.h +++ b/validator-session/validator-session-types.h @@ -76,11 +76,60 @@ struct ValidatorSessionStats { PublicKeyHash id = PublicKeyHash::zero(); ValidatorSessionCandidateId candidate_id = ValidatorSessionCandidateId::zero(); int block_status = status_none; - td::uint64 block_timestamp = 0; + double block_timestamp = -1.0; std::string comment; + + bool is_accepted = false; + bool is_ours = false; + double got_submit_at = -1.0; + double collation_time = -1.0; + double validation_time = -1.0; + double collated_at = -1.0; + double validated_at = -1.0; + bool collation_cached = false; + bool validation_cached = false; + double gen_utime = -1.0; + + std::vector approvers, signers; + ValidatorWeight approved_weight = 0; + ValidatorWeight signed_weight = 0; + double approved_33pct_at = -1.0; + double approved_66pct_at = -1.0; + double signed_33pct_at = -1.0; + double signed_66pct_at = -1.0; + + double serialize_time = -1.0; + double deserialize_time = -1.0; + td::int32 serialized_size = -1; + + void set_approved_by(td::uint32 id, ValidatorWeight weight, ValidatorWeight total_weight) { + if (!approvers.at(id)) { + approvers.at(id) = true; + approved_weight += weight; + if (approved_33pct_at <= 0.0 && approved_weight >= total_weight / 3 + 1) { + approved_33pct_at = td::Clocks::system(); + } + if (approved_66pct_at <= 0.0 && approved_weight >= (total_weight * 2) / 3 + 1) { + approved_66pct_at = td::Clocks::system(); + } + } + } + + void set_signed_by(td::uint32 id, ValidatorWeight weight, ValidatorWeight total_weight) { + if (!signers.at(id)) { + signers.at(id) = true; + signed_weight += weight; + if (signed_33pct_at <= 0.0 && signed_weight >= total_weight / 3 + 1) { + signed_33pct_at = td::Clocks::system(); + } + if (signed_66pct_at <= 0.0 && signed_weight >= (total_weight * 2) / 3 + 1) { + signed_66pct_at = td::Clocks::system(); + } + } + } }; struct Round { - td::uint64 timestamp = 0; + double timestamp = -1.0; std::vector producers; }; @@ -90,7 +139,7 @@ struct ValidatorSessionStats { bool success = false; ValidatorSessionId session_id = ValidatorSessionId::zero(); CatchainSeqno cc_seqno = 0; - td::uint64 timestamp = 0; + double timestamp = -1.0; PublicKeyHash self = PublicKeyHash::zero(); PublicKeyHash creator = PublicKeyHash::zero(); td::uint32 total_validators = 0; @@ -101,6 +150,20 @@ struct ValidatorSessionStats { ValidatorWeight approve_signatures_weight = 0; }; +struct NewValidatorGroupStats { + struct Node { + PublicKeyHash id = PublicKeyHash::zero(); + ValidatorWeight weight = 0; + }; + + ValidatorSessionId session_id = ValidatorSessionId::zero(); + ShardIdFull shard{masterchainId}; + CatchainSeqno cc_seqno = 0; + double timestamp = -1.0; + td::uint32 self_idx = 0; + std::vector nodes; +}; + } // namespace validatorsession } // namespace ton diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 6b328b2bd..46dd44403 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -20,6 +20,7 @@ #include "td/utils/Random.h" #include "td/utils/crypto.h" #include "candidate-serializer.h" +#include "td/utils/overloaded.h" namespace ton { @@ -86,6 +87,7 @@ void ValidatorSessionImpl::process_blocks(std::vector for (auto &msg : msgs) { VLOG(VALIDATOR_SESSION_INFO) << this << ": applying action: " << msg.get(); + stats_process_action(local_idx(), *msg); real_state_ = ValidatorSessionState::action(description(), real_state_, local_idx(), att, msg.get()); } @@ -167,6 +169,7 @@ void ValidatorSessionImpl::preprocess_block(catchain::CatChainBlock *block) { for (auto &msg : B->actions_) { VLOG(VALIDATOR_SESSION_INFO) << this << "[node " << description().get_source_id(block->source()) << "][block " << block->hash() << "]: applying action " << msg.get(); + stats_process_action(block->source(), *msg); state = ValidatorSessionState::action(description(), state, block->source(), att, msg.get()); } state = ValidatorSessionState::make_all(description(), state, block->source(), att); @@ -222,9 +225,11 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice // Note: src is not necessarily equal to the sender of this message: // If requested using get_broadcast_p2p, src is the creator of the block, sender possibly is some other node. auto src_idx = description().get_source_idx(src); + td::Timer deserialize_timer; auto R = deserialize_candidate(data, compress_block_candidates_, description().opts().max_block_size + description().opts().max_collated_data_size + 1024); + double deserialize_time = deserialize_timer.elapsed(); if (R.is_error()) { VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice()) << "]: failed to parse: " << R.move_as_error(); @@ -255,6 +260,18 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice return; } + auto stat = stats_get_candidate_stat(block_round, src, block_id); + if (stat) { + if (stat->block_status == ValidatorSessionStats::status_none) { + stat->block_status = ValidatorSessionStats::status_received; + } + if (stat->block_timestamp <= 0.0) { + stat->block_timestamp = td::Clocks::system(); + } + stat->deserialize_time = deserialize_time; + stat->serialized_size = data.size(); + } + if ((td::int32)block_round < (td::int32)cur_round_ - MAX_PAST_ROUND_BLOCK || block_round >= cur_round_ + MAX_FUTURE_ROUND_BLOCK) { VLOG(VALIDATOR_SESSION_NOTICE) << this << "[node " << src << "][broadcast " << block_id @@ -291,7 +308,6 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice CHECK(!pending_reject_.count(block_id)); CHECK(!rejected_.count(block_id)); - stats_set_candidate_status(cur_round_, src, block_id, ValidatorSessionStats::status_received); auto v = virtual_state_->choose_blocks_to_approve(description(), local_idx()); for (auto &b : v) { if (b && SentBlock::get_block_id(b) == block_id) { @@ -363,9 +379,16 @@ void ValidatorSessionImpl::process_query(PublicKeyHash src, td::BufferSlice data } void ValidatorSessionImpl::candidate_decision_fail(td::uint32 round, ValidatorSessionCandidateId hash, - std::string result, td::uint32 src, td::BufferSlice proof) { - stats_set_candidate_status(round, description().get_source_id(src), hash, ValidatorSessionStats::status_rejected, - result); + std::string result, td::uint32 src, td::BufferSlice proof, + double validation_time, bool validation_cached) { + auto stat = stats_get_candidate_stat(round, description().get_source_id(src), hash); + if (stat) { + stat->block_status = ValidatorSessionStats::status_rejected; + stat->comment = result; + stat->validation_time = validation_time; + stat->validated_at = td::Clocks::system(); + stat->validation_cached = validation_cached; + } if (round != cur_round_) { return; } @@ -379,9 +402,17 @@ void ValidatorSessionImpl::candidate_decision_fail(td::uint32 round, ValidatorSe } void ValidatorSessionImpl::candidate_decision_ok(td::uint32 round, ValidatorSessionCandidateId hash, RootHash root_hash, - FileHash file_hash, td::uint32 src, td::uint32 ok_from) { - stats_set_candidate_status(round, description().get_source_id(src), hash, ValidatorSessionStats::status_approved, - PSTRING() << "ts=" << ok_from); + FileHash file_hash, td::uint32 src, td::uint32 ok_from, + double validation_time, bool validation_cached) { + auto stat = stats_get_candidate_stat(round, description().get_source_id(src), hash); + if (stat) { + stat->block_status = ValidatorSessionStats::status_approved; + stat->comment = PSTRING() << "ts=" << ok_from; + stat->validation_time = validation_time; + stat->gen_utime = (double)ok_from; + stat->validated_at = td::Clocks::system(); + stat->validation_cached = validation_cached; + } if (round != cur_round_) { return; } @@ -418,10 +449,8 @@ void ValidatorSessionImpl::candidate_approved_signed(td::uint32 round, Validator } void ValidatorSessionImpl::generated_block(td::uint32 round, ValidatorSessionCandidateId root_hash, - td::BufferSlice data, td::BufferSlice collated_data) { - if (round != cur_round_) { - return; - } + td::BufferSlice data, td::BufferSlice collated_data, double collation_time, + bool collation_cached) { if (data.size() > description().opts().max_block_size || collated_data.size() > description().opts().max_collated_data_size) { LOG(ERROR) << this << ": generated candidate is too big. Dropping. size=" << data.size() << " " @@ -430,13 +459,27 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, ValidatorSessionCan } auto file_hash = sha256_bits256(data.as_slice()); auto collated_data_file_hash = sha256_bits256(collated_data.as_slice()); + auto block_id = description().candidate_id(local_idx(), root_hash, file_hash, collated_data_file_hash); + auto stat = stats_get_candidate_stat(round, local_id(), block_id); + if (stat) { + stat->block_status = ValidatorSessionStats::status_received; + stat->collation_time = collation_time; + stat->collated_at = td::Clocks::system(); + stat->block_timestamp = td::Clocks::system(); + stat->collation_cached = collation_cached; + } + if (round != cur_round_) { + return; + } + td::Timer serialize_timer; auto b = create_tl_object(local_id().tl(), round, root_hash, std::move(data), std::move(collated_data)); - auto B = serialize_candidate(b, compress_block_candidates_).move_as_ok(); - - auto block_id = description().candidate_id(local_idx(), root_hash, file_hash, collated_data_file_hash); + if (stat) { + stat->serialize_time = serialize_timer.elapsed(); + stat->serialized_size = B.size(); + } td::actor::send_closure(catchain_, &catchain::CatChain::send_broadcast, std::move(B)); @@ -496,11 +539,11 @@ void ValidatorSessionImpl::check_generate_slot() { td::PerfWarningTimer timer{"too long block generation", 1.0}; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), print_id = print_id(), timer = std::move(timer), - round = cur_round_](td::Result R) { + round = cur_round_](td::Result R) { if (R.is_ok()) { - auto c = R.move_as_ok(); + auto c = std::move(R.ok_ref().candidate); td::actor::send_closure(SelfId, &ValidatorSessionImpl::generated_block, round, c.id.root_hash, - c.data.clone(), c.collated_data.clone()); + c.data.clone(), c.collated_data.clone(), timer.elapsed(), R.ok().is_cached); } else { LOG(WARNING) << print_id << ": failed to generate block candidate: " << R.move_as_error(); } @@ -550,6 +593,16 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { it->second->round_ = std::max(it->second->round_, cur_round_); td::PerfWarningTimer timer{"too long block validation", 1.0}; auto &B = it->second; + auto stat = stats_get_candidate_stat(B->round_, PublicKeyHash{B->src_}); + if (stat) { + // Can happen if block is cached from previous round + if (stat->block_status == ValidatorSessionStats::status_none) { + stat->block_status = ValidatorSessionStats::status_received; + } + if (stat->block_timestamp <= 0.0) { + stat->block_timestamp = td::Clocks::system(); + } + } auto P = td::PromiseCreator::lambda([round = cur_round_, hash = block_id, root_hash = block->get_root_hash(), file_hash = block->get_file_hash(), timer = std::move(timer), @@ -563,10 +616,10 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { auto R = res.move_as_ok(); if (R.is_ok()) { td::actor::send_closure(SelfId, &ValidatorSessionImpl::candidate_decision_ok, round, hash, root_hash, - file_hash, src, R.ok_from()); + file_hash, src, R.ok_from(), timer.elapsed(), R.is_cached()); } else { td::actor::send_closure(SelfId, &ValidatorSessionImpl::candidate_decision_fail, round, hash, R.reason(), - src, R.proof()); + src, R.proof(), timer.elapsed(), R.is_cached()); } }); pending_approve_.insert(block_id); @@ -754,8 +807,8 @@ void ValidatorSessionImpl::request_new_block(bool now) { } else { double lambda = 10.0 / description().get_total_nodes(); double x = -1 / lambda * log(td::Random::fast(1, 999) * 0.001); - if (x > 0.5) { - x = 0.5; + if (x > catchain_max_block_delay_) { // default = 0.5 + x = catchain_max_block_delay_; } td::actor::send_closure(catchain_, &catchain::CatChain::need_new_block, td::Timestamp::in(x)); } @@ -818,28 +871,40 @@ void ValidatorSessionImpl::on_new_round(td::uint32 round) { callback_->on_block_skipped(cur_round_); } else { cur_stats_.success = true; - cur_stats_.timestamp = (td::uint64)td::Clocks::system(); + cur_stats_.timestamp = td::Clocks::system(); cur_stats_.signatures = (td::uint32)export_sigs.size(); cur_stats_.signatures_weight = signatures_weight; cur_stats_.approve_signatures = (td::uint32)export_approve_sigs.size(); cur_stats_.approve_signatures_weight = approve_signatures_weight; cur_stats_.creator = description().get_source_id(block->get_src_idx()); + auto stat = stats_get_candidate_stat(cur_round_, cur_stats_.creator); + if (stat) { + stat->is_accepted = true; + } + auto stats = cur_stats_; + while (!stats.rounds.empty() && stats.rounds.size() + stats.first_round - 1 > cur_round_) { + stats.rounds.pop_back(); + } if (it == blocks_.end()) { callback_->on_block_committed(cur_round_, description().get_source_public_key(block->get_src_idx()), block->get_root_hash(), block->get_file_hash(), td::BufferSlice(), - std::move(export_sigs), std::move(export_approve_sigs), std::move(cur_stats_)); + std::move(export_sigs), std::move(export_approve_sigs), std::move(stats)); } else { callback_->on_block_committed(cur_round_, description().get_source_public_key(block->get_src_idx()), block->get_root_hash(), block->get_file_hash(), it->second->data_.clone(), - std::move(export_sigs), std::move(export_approve_sigs), std::move(cur_stats_)); + std::move(export_sigs), std::move(export_approve_sigs), std::move(stats)); } } cur_round_++; if (have_block) { stats_init(); } else { - stats_add_round(); + size_t round_idx = cur_round_ - cur_stats_.first_round; + while (round_idx >= cur_stats_.rounds.size()) { + stats_add_round(); + } + cur_stats_.rounds[round_idx].timestamp = td::Clocks::system(); } auto it2 = blocks_.begin(); while (it2 != blocks_.end()) { @@ -929,9 +994,7 @@ void ValidatorSessionImpl::destroy() { } void ValidatorSessionImpl::get_current_stats(td::Promise promise) { - ValidatorSessionStats stats = cur_stats_; - stats.timestamp = (td::uint64)td::Clocks::system(); - promise.set_result(std::move(stats)); + promise.set_result(cur_stats_); } void ValidatorSessionImpl::get_validator_group_info_for_litequery( @@ -997,52 +1060,149 @@ void ValidatorSessionImpl::start_up() { } void ValidatorSessionImpl::stats_init() { + auto old_rounds = std::move(cur_stats_.rounds); + if (stats_inited_ && cur_stats_.first_round + old_rounds.size() > cur_round_) { + old_rounds.erase(old_rounds.begin(), old_rounds.end() - (cur_stats_.first_round + old_rounds.size() - cur_round_)); + } else { + old_rounds.clear(); + } cur_stats_ = ValidatorSessionStats(); + cur_stats_.rounds = std::move(old_rounds); cur_stats_.first_round = cur_round_; cur_stats_.session_id = unique_hash_; cur_stats_.total_validators = description().get_total_nodes(); cur_stats_.total_weight = description().get_total_weight(); cur_stats_.self = description().get_source_id(local_idx()); - stats_add_round(); + + for (auto it = stats_pending_approve_.begin(); it != stats_pending_approve_.end(); ) { + if (it->first.first < cur_round_) { + it = stats_pending_approve_.erase(it); + } else { + ++it; + } + } + for (auto it = stats_pending_sign_.begin(); it != stats_pending_sign_.end(); ) { + if (it->first.first < cur_round_) { + it = stats_pending_sign_.erase(it); + } else { + ++it; + } + } + + if (cur_stats_.rounds.empty()) { + stats_add_round(); + } + cur_stats_.rounds[0].timestamp = td::Clocks::system(); + stats_inited_ = true; } void ValidatorSessionImpl::stats_add_round() { + td::uint32 round = cur_stats_.first_round + cur_stats_.rounds.size(); cur_stats_.rounds.emplace_back(); - auto& round = cur_stats_.rounds.back(); - round.timestamp = (td::uint64)td::Clocks::system(); - round.producers.resize(description().get_max_priority() + 1); + auto& stat = cur_stats_.rounds.back(); + stat.producers.resize(description().get_max_priority() + 1); for (td::uint32 i = 0; i < description().get_total_nodes(); i++) { - td::int32 priority = description().get_node_priority(i, cur_round_); + td::int32 priority = description().get_node_priority(i, round); if (priority >= 0) { - CHECK((size_t)priority < round.producers.size()); - round.producers[priority].id = description().get_source_id(i); + CHECK((size_t)priority < stat.producers.size()); + stat.producers[priority].id = description().get_source_id(i); + stat.producers[priority].is_ours = (local_idx() == i); + stat.producers[priority].approvers.resize(description().get_total_nodes(), false); + stat.producers[priority].signers.resize(description().get_total_nodes(), false); } } - while (!round.producers.empty() && round.producers.back().id.is_zero()) { - round.producers.pop_back(); + while (!stat.producers.empty() && stat.producers.back().id.is_zero()) { + stat.producers.pop_back(); } } -void ValidatorSessionImpl::stats_set_candidate_status(td::uint32 round, PublicKeyHash src, - ValidatorSessionCandidateId candidate_id, int status, - std::string comment) { - if (round < cur_stats_.first_round || round - cur_stats_.first_round >= cur_stats_.rounds.size()) { - return; +ValidatorSessionStats::Producer *ValidatorSessionImpl::stats_get_candidate_stat( + td::uint32 round, PublicKeyHash src, ValidatorSessionCandidateId candidate_id) { + if (round < cur_stats_.first_round || round > cur_round_ + 5) { + return nullptr; + } + while (round - cur_stats_.first_round >= cur_stats_.rounds.size()) { + stats_add_round(); } auto &stats_round = cur_stats_.rounds[round - cur_stats_.first_round]; auto it = std::find_if(stats_round.producers.begin(), stats_round.producers.end(), [&](const ValidatorSessionStats::Producer &p) { return p.id == src; }); if (it == stats_round.producers.end()) { - return; + return nullptr; } - it->candidate_id = candidate_id; - if (it->block_status == ValidatorSessionStats::status_none) { - it->block_timestamp = (td::uint64)td::Clocks::system(); + if (!candidate_id.is_zero()) { + it->candidate_id = candidate_id; } - it->block_status = status; - if (!comment.empty()) { - it->comment = std::move(comment); + auto it2 = stats_pending_approve_.find({round, it->candidate_id}); + if (it2 != stats_pending_approve_.end()) { + for (td::uint32 node_id : it2->second) { + it->set_approved_by(node_id, description().get_node_weight(node_id), description().get_total_weight()); + } + stats_pending_approve_.erase(it2); } + it2 = stats_pending_sign_.find({round, it->candidate_id}); + if (it2 != stats_pending_sign_.end()) { + for (td::uint32 node_id : it2->second) { + it->set_signed_by(node_id, description().get_node_weight(node_id), description().get_total_weight()); + } + stats_pending_sign_.erase(it2); + } + return &*it; +} + +ValidatorSessionStats::Producer *ValidatorSessionImpl::stats_get_candidate_stat_by_id( + td::uint32 round, ValidatorSessionCandidateId candidate_id) { + if (round < cur_stats_.first_round || round > cur_round_ + 5) { + return nullptr; + } + while (round - cur_stats_.first_round >= cur_stats_.rounds.size()) { + stats_add_round(); + } + auto &stats_round = cur_stats_.rounds[round - cur_stats_.first_round]; + auto it = std::find_if(stats_round.producers.begin(), stats_round.producers.end(), + [&](const ValidatorSessionStats::Producer &p) { return p.candidate_id == candidate_id; }); + if (it == stats_round.producers.end()) { + return nullptr; + } + return &*it; +} + +void ValidatorSessionImpl::stats_process_action(td::uint32 node_id, ton_api::validatorSession_round_Message &action) { + ton_api::downcast_call(action, td::overloaded( + [&](const ton_api::validatorSession_message_submittedBlock &obj) { + auto candidate_id = description().candidate_id( + node_id, obj.root_hash_, obj.file_hash_, obj.collated_data_file_hash_); + auto stat = stats_get_candidate_stat( + obj.round_, description().get_source_id(node_id), candidate_id); + if (stat && stat->got_submit_at <= 0.0) { + stat->got_submit_at = td::Clocks::system(); + } + }, + [&](const ton_api::validatorSession_message_approvedBlock &obj) { + if (obj.candidate_ == skip_round_candidate_id()) { + return; + } + auto stat = stats_get_candidate_stat_by_id(obj.round_, obj.candidate_); + if (stat) { + stat->set_approved_by(node_id, description().get_node_weight(node_id), + description().get_total_weight()); + } else { + stats_pending_approve_[{obj.round_, obj.candidate_}].push_back(node_id); + } + }, + [&](const ton_api::validatorSession_message_commit &obj) { + if (obj.candidate_ == skip_round_candidate_id()) { + return; + } + auto stat = stats_get_candidate_stat_by_id(obj.round_, obj.candidate_); + if (stat) { + stat->set_signed_by(node_id, description().get_node_weight(node_id), + description().get_total_weight()); + } else { + stats_pending_sign_[{obj.round_, obj.candidate_}].push_back(node_id); + } + }, + [](const auto &) {})); } td::actor::ActorOwn ValidatorSession::create( diff --git a/validator-session/validator-session.h b/validator-session/validator-session.h index 6300eaa49..0870f6718 100644 --- a/validator-session/validator-session.h +++ b/validator-session/validator-session.h @@ -56,6 +56,12 @@ class ValidatorSession : public td::actor::Actor { td::BufferSlice proof() const { return proof_.clone(); } + bool is_cached() const { + return is_cached_; + } + void set_is_cached(bool value = true) { + is_cached_ = value; + } CandidateDecision(td::uint32 ok_from) { ok_ = true; ok_from_ = ok_from; @@ -69,6 +75,12 @@ class ValidatorSession : public td::actor::Actor { td::uint32 ok_from_ = 0; std::string reason_; td::BufferSlice proof_; + bool is_cached_ = false; + }; + + struct GeneratedCandidate { + BlockCandidate candidate; + bool is_cached = false; }; class Callback { @@ -76,7 +88,7 @@ class ValidatorSession : public td::actor::Actor { virtual void on_candidate(td::uint32 round, PublicKey source, ValidatorSessionRootHash root_hash, td::BufferSlice data, td::BufferSlice collated_data, td::Promise promise) = 0; - virtual void on_generate_slot(td::uint32 round, td::Promise promise) = 0; + virtual void on_generate_slot(td::uint32 round, td::Promise promise) = 0; virtual void on_block_committed(td::uint32 round, PublicKey source, ValidatorSessionRootHash root_hash, ValidatorSessionFileHash file_hash, td::BufferSlice data, std::vector> signatures, @@ -96,6 +108,7 @@ class ValidatorSession : public td::actor::Actor { virtual void get_validator_group_info_for_litequery( td::uint32 cur_round, td::Promise>> promise) = 0; + virtual void set_catchain_max_block_delay(double value) = 0; static td::actor::ActorOwn create( catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 619f7178a..580582824 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -90,6 +90,8 @@ class ValidatorSessionImpl : public ValidatorSession { td::actor::ActorOwn catchain_; std::unique_ptr description_; + double catchain_max_block_delay_ = 0.4; + void on_new_round(td::uint32 round); void on_catchain_started(); void check_vote_for_slot(td::uint32 att); @@ -159,10 +161,19 @@ class ValidatorSessionImpl : public ValidatorSession { bool compress_block_candidates_ = false; ValidatorSessionStats cur_stats_; + bool stats_inited_ = false; + std::map, std::vector> + stats_pending_approve_; // round, candidate_id -> approvers + std::map, std::vector> + stats_pending_sign_; // round, candidate_id -> signers void stats_init(); void stats_add_round(); - void stats_set_candidate_status(td::uint32 round, PublicKeyHash src, ValidatorSessionCandidateId candidate_id, - int status, std::string comment = ""); + ValidatorSessionStats::Producer *stats_get_candidate_stat( + td::uint32 round, PublicKeyHash src, + ValidatorSessionCandidateId candidate_id = ValidatorSessionCandidateId::zero()); + ValidatorSessionStats::Producer *stats_get_candidate_stat_by_id(td::uint32 round, + ValidatorSessionCandidateId candidate_id); + void stats_process_action(td::uint32 node_id, ton_api::validatorSession_round_Message &action); public: ValidatorSessionImpl(catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, @@ -179,6 +190,9 @@ class ValidatorSessionImpl : public ValidatorSession { void get_validator_group_info_for_litequery( td::uint32 cur_round, td::Promise>> promise) override; + void set_catchain_max_block_delay(double value) override { + catchain_max_block_delay_ = value; + } void process_blocks(std::vector blocks); void finished_processing(); @@ -190,17 +204,16 @@ class ValidatorSessionImpl : public ValidatorSession { void process_query(PublicKeyHash src, td::BufferSlice data, td::Promise promise); void try_approve_block(const SentBlock *block); - void try_sign(); - void candidate_decision_fail(td::uint32 round, ValidatorSessionCandidateId hash, std::string result, - td::uint32 src, td::BufferSlice proof); + void candidate_decision_fail(td::uint32 round, ValidatorSessionCandidateId hash, std::string result, td::uint32 src, + td::BufferSlice proof, double validation_time, bool validation_cached); void candidate_decision_ok(td::uint32 round, ValidatorSessionCandidateId hash, RootHash root_hash, FileHash file_hash, - td::uint32 src, td::uint32 ok_from); + td::uint32 src, td::uint32 ok_from, double validation_time, bool validation_cached); void candidate_approved_signed(td::uint32 round, ValidatorSessionCandidateId hash, td::uint32 ok_from, td::BufferSlice signature); void generated_block(td::uint32 round, ValidatorSessionRootHash root_hash, td::BufferSlice data, - td::BufferSlice collated); + td::BufferSlice collated, double collation_time, bool collation_cached); void signed_block(td::uint32 round, ValidatorSessionCandidateId hash, td::BufferSlice signature); void end_request(td::uint32 round, ValidatorSessionCandidateId block_id) { diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index a25ac7e1f..20462caeb 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -25,6 +25,8 @@ set(VALIDATOR_DB_SOURCE db/statedb.cpp db/staticfilesdb.cpp db/staticfilesdb.hpp + db/db-utils.cpp + db/db-utils.h db/package.hpp db/package.cpp diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index c806f4e54..14d3ec469 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -310,14 +310,17 @@ void ArchiveManager::get_file(ConstBlockHandle handle, FileReference ref_id, td: get_file_short_cont(std::move(ref_id), get_max_temp_file_desc_idx(), std::move(promise)); } -void ArchiveManager::written_perm_state(FileReferenceShort id) { - perm_states_.emplace(id.hash(), id); +void ArchiveManager::register_perm_state(FileReferenceShort id) { + BlockSeqno masterchain_seqno = 0; + id.ref().visit(td::overloaded( + [&](const fileref::PersistentStateShort &x) { masterchain_seqno = x.masterchain_seqno; }, [&](const auto &) {})); + perm_states_[{masterchain_seqno, id.hash()}] = id; } void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, td::Promise promise) { auto id = FileReference{fileref::ZeroState{block_id}}; auto hash = id.hash(); - if (perm_states_.find(hash) != perm_states_.end()) { + if (perm_states_.find({0, hash}) != perm_states_.end()) { promise.set_value(td::Unit()); return; } @@ -328,7 +331,7 @@ void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, t if (R.is_error()) { promise.set_error(R.move_as_error()); } else { - td::actor::send_closure(SelfId, &ArchiveManager::written_perm_state, id); + td::actor::send_closure(SelfId, &ArchiveManager::register_perm_state, id); promise.set_value(td::Unit()); } }); @@ -357,12 +360,13 @@ void ArchiveManager::add_persistent_state_gen(BlockIdExt block_id, BlockIdExt ma add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer)); } -void ArchiveManager::add_persistent_state_impl(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise, - std::function)> create_writer) { +void ArchiveManager::add_persistent_state_impl( + BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise, + std::function)> create_writer) { auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; + BlockSeqno masterchain_seqno = masterchain_block_id.seqno(); auto hash = id.hash(); - if (perm_states_.find(hash) != perm_states_.end()) { + if (perm_states_.find({masterchain_seqno, hash}) != perm_states_.end()) { promise.set_value(td::Unit()); return; } @@ -373,7 +377,7 @@ void ArchiveManager::add_persistent_state_impl(BlockIdExt block_id, BlockIdExt m if (R.is_error()) { promise.set_error(R.move_as_error()); } else { - td::actor::send_closure(SelfId, &ArchiveManager::written_perm_state, id); + td::actor::send_closure(SelfId, &ArchiveManager::register_perm_state, id); promise.set_value(td::Unit()); } }); @@ -383,7 +387,7 @@ void ArchiveManager::add_persistent_state_impl(BlockIdExt block_id, BlockIdExt m void ArchiveManager::get_zero_state(BlockIdExt block_id, td::Promise promise) { auto id = FileReference{fileref::ZeroState{block_id}}; auto hash = id.hash(); - if (perm_states_.find(hash) == perm_states_.end()) { + if (perm_states_.find({0, hash}) == perm_states_.end()) { promise.set_error(td::Status::Error(ErrorCode::notready, "zerostate not in db")); return; } @@ -395,18 +399,38 @@ void ArchiveManager::get_zero_state(BlockIdExt block_id, td::Promise promise) { auto id = FileReference{fileref::ZeroState{block_id}}; auto hash = id.hash(); - if (perm_states_.find(hash) == perm_states_.end()) { + if (perm_states_.find({0, hash}) == perm_states_.end()) { promise.set_result(false); return; } promise.set_result(true); } +void ArchiveManager::get_previous_persistent_state_files( + BlockSeqno cur_mc_seqno, td::Promise>> promise) { + auto it = perm_states_.lower_bound({cur_mc_seqno, FileHash::zero()}); + if (it == perm_states_.begin()) { + promise.set_value({}); + return; + } + --it; + BlockSeqno mc_seqno = it->first.first; + std::vector> files; + while (it->first.first == mc_seqno) { + files.emplace_back(db_root_ + "/archive/states/" + it->second.filename_short(), it->second.shard()); + if (it == perm_states_.begin()) { + break; + } + --it; + } + promise.set_value(std::move(files)); +} + void ArchiveManager::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) { auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; auto hash = id.hash(); - if (perm_states_.find(hash) == perm_states_.end()) { + if (perm_states_.find({masterchain_block_id.seqno(), hash}) == perm_states_.end()) { promise.set_error(td::Status::Error(ErrorCode::notready, "state file not in db")); return; } @@ -419,7 +443,7 @@ void ArchiveManager::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt td::int64 max_size, td::Promise promise) { auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; auto hash = id.hash(); - if (perm_states_.find(hash) == perm_states_.end()) { + if (perm_states_.find({masterchain_block_id.seqno(), hash}) == perm_states_.end()) { promise.set_error(td::Status::Error(ErrorCode::notready, "state file not in db")); return; } @@ -432,7 +456,7 @@ void ArchiveManager::check_persistent_state(BlockIdExt block_id, BlockIdExt mast td::Promise promise) { auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; auto hash = id.hash(); - if (perm_states_.find(hash) == perm_states_.end()) { + if (perm_states_.find({masterchain_block_id.seqno(), hash}) == perm_states_.end()) { promise.set_result(false); return; } @@ -832,7 +856,10 @@ void ArchiveManager::start_up() { if (!opts_->get_disable_rocksdb_stats()) { statistics_.init(); } - index_ = std::make_shared(td::RocksDb::open(db_root_ + "/files/globalindex", statistics_.rocksdb_statistics).move_as_ok()); + td::RocksDbOptions db_options; + db_options.statistics = statistics_.rocksdb_statistics; + index_ = std::make_shared( + td::RocksDb::open(db_root_ + "/files/globalindex", std::move(db_options)).move_as_ok()); std::string value; auto v = index_->get(create_serialize_tl_object().as_slice(), value); v.ensure(); @@ -881,13 +908,11 @@ void ArchiveManager::start_up() { R = FileReferenceShort::create(newfname); R.ensure(); } - auto f = R.move_as_ok(); - auto hash = f.hash(); - perm_states_[hash] = std::move(f); + register_perm_state(R.move_as_ok()); } }).ensure(); - persistent_state_gc(FileHash::zero()); + persistent_state_gc({0, FileHash::zero()}); double open_since = td::Clocks::system() - opts_->get_archive_preload_period(); for (auto it = files_.rbegin(); it != files_.rend(); ++it) { @@ -973,11 +998,12 @@ void ArchiveManager::run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl } } -void ArchiveManager::persistent_state_gc(FileHash last) { - if (perm_states_.size() == 0) { +void ArchiveManager::persistent_state_gc(std::pair last) { + if (perm_states_.empty()) { delay_action( - [hash = FileHash::zero(), SelfId = actor_id(this)]() { - td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, hash); + [SelfId = actor_id(this)]() { + td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, + std::pair{0, FileHash::zero()}); }, td::Timestamp::in(1.0)); return; @@ -990,12 +1016,12 @@ void ArchiveManager::persistent_state_gc(FileHash last) { it = perm_states_.begin(); } + auto key = it->first; auto &F = it->second; - auto hash = F.hash(); int res = 0; BlockSeqno seqno = 0; - F.ref().visit(td::overloaded([&](const fileref::ZeroStateShort &x) { res = 1; }, + F.ref().visit(td::overloaded([&](const fileref::ZeroStateShort &) { res = 1; }, [&](const fileref::PersistentStateShort &x) { res = 0; seqno = x.masterchain_seqno; @@ -1007,24 +1033,41 @@ void ArchiveManager::persistent_state_gc(FileHash last) { perm_states_.erase(it); } if (res != 0) { - delay_action([hash, SelfId = actor_id( - this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, hash); }, + delay_action([key, SelfId = actor_id( + this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); }, + td::Timestamp::in(1.0)); + return; + } + CHECK(seqno == key.first); + + // Do not delete the most recent fully serialized state + bool allow_delete = false; + auto it2 = perm_states_.lower_bound({seqno + 1, FileHash::zero()}); + if (it2 != perm_states_.end()) { + it2 = perm_states_.lower_bound({it2->first.first + 1, FileHash::zero()}); + if (it2 != perm_states_.end()) { + allow_delete = true; + } + } + if (!allow_delete) { + delay_action([key, SelfId = actor_id( + this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); }, td::Timestamp::in(1.0)); return; } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), hash](td::Result R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), key](td::Result R) { if (R.is_error()) { - td::actor::send_closure(SelfId, &ArchiveManager::got_gc_masterchain_handle, nullptr, hash); + td::actor::send_closure(SelfId, &ArchiveManager::got_gc_masterchain_handle, nullptr, key); } else { - td::actor::send_closure(SelfId, &ArchiveManager::got_gc_masterchain_handle, R.move_as_ok(), hash); + td::actor::send_closure(SelfId, &ArchiveManager::got_gc_masterchain_handle, R.move_as_ok(), key); } }); get_block_by_seqno(AccountIdPrefixFull{masterchainId, 0}, seqno, std::move(P)); } -void ArchiveManager::got_gc_masterchain_handle(ConstBlockHandle handle, FileHash hash) { +void ArchiveManager::got_gc_masterchain_handle(ConstBlockHandle handle, std::pair key) { bool to_del = false; if (!handle || !handle->inited_unix_time() || !handle->unix_time()) { to_del = true; @@ -1032,15 +1075,15 @@ void ArchiveManager::got_gc_masterchain_handle(ConstBlockHandle handle, FileHash auto ttl = ValidatorManager::persistent_state_ttl(handle->unix_time()); to_del = ttl < td::Clocks::system(); } - auto it = perm_states_.find(hash); + auto it = perm_states_.find(key); CHECK(it != perm_states_.end()); auto &F = it->second; if (to_del) { td::unlink(db_root_ + "/archive/states/" + F.filename_short()).ignore(); perm_states_.erase(it); } - delay_action([hash, SelfId = actor_id( - this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, hash); }, + delay_action([key, SelfId = actor_id( + this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); }, td::Timestamp::in(1.0)); } diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index a1ed97022..622969ec5 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -54,6 +54,8 @@ class ArchiveManager : public td::actor::Actor { td::int64 max_size, td::Promise promise); void check_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise); void check_zero_state(BlockIdExt block_id, td::Promise promise); + void get_previous_persistent_state_files(BlockSeqno cur_mc_seqno, + td::Promise>> promise); void truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise promise); //void truncate_continue(BlockSeqno masterchain_seqno, td::Promise promise); @@ -180,7 +182,7 @@ class ArchiveManager : public td::actor::Actor { return p.key ? key_files_ : p.temp ? temp_files_ : files_; } - std::map perm_states_; + std::map, FileReferenceShort> perm_states_; // Mc block seqno, hash -> state void load_package(PackageId seqno); void delete_package(PackageId seqno, td::Promise promise); @@ -207,10 +209,10 @@ class ArchiveManager : public td::actor::Actor { void add_persistent_state_impl(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise, std::function)> create_writer); - void written_perm_state(FileReferenceShort id); + void register_perm_state(FileReferenceShort id); - void persistent_state_gc(FileHash last); - void got_gc_masterchain_handle(ConstBlockHandle handle, FileHash hash); + void persistent_state_gc(std::pair last); + void got_gc_masterchain_handle(ConstBlockHandle handle, std::pair key); std::string db_root_; td::Ref opts_; diff --git a/validator/db/archive-slice.cpp b/validator/db/archive-slice.cpp index 86c2ee932..d392431a9 100644 --- a/validator/db/archive-slice.cpp +++ b/validator/db/archive-slice.cpp @@ -24,6 +24,7 @@ #include "td/utils/port/path.h" #include "common/delay.h" #include "files-async.hpp" +#include "db-utils.h" namespace ton { @@ -41,16 +42,14 @@ class PackageStatistics { void record_read(double time, uint64_t bytes) { read_bytes.fetch_add(bytes, std::memory_order_relaxed); - std::lock_guard guard(read_mutex); + std::lock_guard guard(read_mutex); read_time.insert(time); - read_time_sum += time; } void record_write(double time, uint64_t bytes) { write_bytes.fetch_add(bytes, std::memory_order_relaxed); - std::lock_guard guard(write_mutex); + std::lock_guard guard(write_mutex); write_time.insert(time); - write_time_sum += time; } std::string to_string_and_reset() { @@ -64,68 +63,35 @@ class PackageStatistics { ss << "ton.pack.read.bytes COUNT : " << read_bytes.exchange(0, std::memory_order_relaxed) << "\n"; ss << "ton.pack.write.bytes COUNT : " << write_bytes.exchange(0, std::memory_order_relaxed) << "\n"; - std::multiset temp_read_time; - double temp_read_time_sum; + PercentileStats temp_read_time; { - std::lock_guard guard(read_mutex); + std::lock_guard guard(read_mutex); temp_read_time = std::move(read_time); read_time.clear(); - temp_read_time_sum = read_time_sum; - read_time_sum = 0; } - auto read_stats = calculate_statistics(temp_read_time); - ss << "ton.pack.read.micros P50 : " << read_stats[0] << - " P95 : " << read_stats[1] << - " P99 : " << read_stats[2] << - " P100 : " << read_stats[3] << - " COUNT : " << temp_read_time.size() << - " SUM : " << temp_read_time_sum << "\n"; - - std::multiset temp_write_time; - double temp_write_time_sum; + ss << "ton.pack.read.micros " << temp_read_time.to_string() << "\n"; + + PercentileStats temp_write_time; { - std::lock_guard guard(write_mutex); + std::lock_guard guard(write_mutex); temp_write_time = std::move(write_time); write_time.clear(); - temp_write_time_sum = write_time_sum; - write_time_sum = 0; } - auto write_stats = calculate_statistics(temp_write_time); - ss << "ton.pack.write.micros P50 : " << write_stats[0] << - " P95 : " << write_stats[1] << - " P99 : " << write_stats[2] << - " P100 : " << write_stats[3] << - " COUNT : " << temp_write_time.size() << - " SUM : " << temp_write_time_sum << "\n"; + ss << "ton.pack.write.micros " << temp_write_time.to_string() << "\n"; return ss.str(); } private: - std::atomic_uint64_t open_count; - std::atomic_uint64_t close_count; - std::multiset read_time; - std::atomic_uint64_t read_bytes; - std::multiset write_time; - std::atomic_uint64_t write_bytes; - double read_time_sum; - double write_time_sum; + std::atomic_uint64_t open_count{0}; + std::atomic_uint64_t close_count{0}; + PercentileStats read_time; + std::atomic_uint64_t read_bytes{0}; + PercentileStats write_time; + std::atomic_uint64_t write_bytes{0}; mutable std::mutex read_mutex; mutable std::mutex write_mutex; - - std::vector calculate_statistics(const std::multiset& data) const { - if (data.empty()) return {0, 0, 0, 0}; - - auto size = data.size(); - auto calc_percentile = [&](double p) -> double { - auto it = data.begin(); - std::advance(it, static_cast(std::ceil(p * double(size)) - 1)); - return *it; - }; - - return {calc_percentile(0.5), calc_percentile(0.95), calc_percentile(0.99), *data.rbegin()}; - } }; void DbStatistics::init() { @@ -588,7 +554,9 @@ void ArchiveSlice::get_archive_id(BlockSeqno masterchain_seqno, td::Promise(td::RocksDb::open(db_path_, statistics_.rocksdb_statistics).move_as_ok()); + td::RocksDbOptions db_options; + db_options.statistics = statistics_.rocksdb_statistics; + kv_ = std::make_unique(td::RocksDb::open(db_path_, std::move(db_options)).move_as_ok()); std::string value; auto R2 = kv_->get("status", value); R2.ensure(); diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index d9803cbe2..dfbee0a1a 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -88,8 +88,15 @@ void CellDbIn::start_up() { statistics_ = td::RocksDb::create_statistics(); statistics_flush_at_ = td::Timestamp::in(60.0); } - cell_db_ = std::make_shared(td::RocksDb::open(path_, statistics_).move_as_ok()); - + td::RocksDbOptions db_options; + db_options.statistics = statistics_; + if (opts_->get_celldb_cache_size()) { + db_options.block_cache = td::RocksDb::create_cache(opts_->get_celldb_cache_size().value()); + LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(opts_->get_celldb_cache_size().value()); + } + db_options.use_direct_reads = opts_->get_celldb_direct_io(); + cell_db_ = std::make_shared(td::RocksDb::open(path_, std::move(db_options)).move_as_ok()); + boc_ = vm::DynamicBagOfCellsDb::create(); boc_->set_celldb_compress_depth(opts_->get_celldb_compress_depth()); @@ -105,6 +112,27 @@ void CellDbIn::start_up() { set_block(empty, std::move(e)); cell_db_->commit_write_batch().ensure(); } + + if (opts_->get_celldb_preload_all()) { + // Iterate whole DB in a separate thread + delay_action([snapshot = cell_db_->snapshot()]() { + LOG(WARNING) << "CellDb: pre-loading all keys"; + td::uint64 total = 0; + td::Timer timer; + auto S = snapshot->for_each([&](td::Slice, td::Slice) { + ++total; + if (total % 1000000 == 0) { + LOG(INFO) << "CellDb: iterated " << total << " keys"; + } + return td::Status::OK(); + }); + if (S.is_error()) { + LOG(ERROR) << "CellDb: pre-load failed: " << S.move_as_error(); + } else { + LOG(WARNING) << "CellDb: iterated " << total << " keys in " << timer.elapsed() << "s"; + } + }, td::Timestamp::now()); + } } void CellDbIn::load_cell(RootHash hash, td::Promise> promise) { @@ -155,6 +183,9 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); promise.set_result(boc_->load_cell(cell->get_hash().as_slice())); + if (!opts_->get_disable_rocksdb_stats()) { + cell_db_statistics_.store_cell_time_.insert(timer.elapsed() * 1e6); + } } void CellDbIn::get_cell_db_reader(td::Promise> promise) { @@ -162,8 +193,9 @@ void CellDbIn::get_cell_db_reader(td::Promise> } void CellDbIn::flush_db_stats() { - auto stats = td::RocksDb::statistics_to_string(statistics_); - auto to_file_r = td::FileFd::open(path_ + "/db_stats.txt", td::FileFd::Truncate | td::FileFd::Create | td::FileFd::Write, 0644); + auto stats = td::RocksDb::statistics_to_string(statistics_) + cell_db_statistics_.to_string(); + auto to_file_r = + td::FileFd::open(path_ + "/db_stats.txt", td::FileFd::Truncate | td::FileFd::Create | td::FileFd::Write, 0644); if (to_file_r.is_error()) { LOG(ERROR) << "Failed to open db_stats.txt: " << to_file_r.move_as_error(); return; @@ -176,6 +208,7 @@ void CellDbIn::flush_db_stats() { return; } td::RocksDb::reset_statistics(statistics_); + cell_db_statistics_.clear(); } void CellDbIn::alarm() { @@ -278,6 +311,9 @@ void CellDbIn::gc_cont2(BlockHandle handle) { td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); DCHECK(get_block(key_hash).is_error()); + if (!opts_->get_disable_rocksdb_stats()) { + cell_db_statistics_.gc_cell_time_.insert(timer.elapsed() * 1e6); + } } void CellDbIn::skip_gc() { @@ -441,6 +477,14 @@ td::BufferSlice CellDbIn::DbEntry::release() { return create_serialize_tl_object(create_tl_block_id(block_id), prev, next, root_hash); } +std::string CellDbIn::CellDbStatistics::to_string() { + td::StringBuilder ss; + ss << "ton.celldb.store_cell.micros " << store_cell_time_.to_string() << "\n"; + ss << "ton.celldb.gc_cell.micros " << gc_cell_time_.to_string() << "\n"; + ss << "ton.celldb.total_time.micros : " << (td::Timestamp::now().at() - stats_start_time_.at()) * 1e6 << "\n"; + return ss.as_cslice().str(); +} + } // namespace validator } // namespace ton diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index 573d4b995..7dc1fa781 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -26,6 +26,7 @@ #include "interfaces/block-handle.h" #include "auto/tl/ton_api.h" #include "validator.h" +#include "db-utils.h" namespace rocksdb { class Statistics; @@ -42,9 +43,11 @@ class CellDbAsyncExecutor; class CellDbBase : public td::actor::Actor { public: - virtual void start_up(); + void start_up() override; + protected: std::shared_ptr async_executor; + private: void execute_sync(std::function f); friend CellDbAsyncExecutor; @@ -76,8 +79,7 @@ class CellDbIn : public CellDbBase { RootHash root_hash; DbEntry(tl_object_ptr entry); - DbEntry() { - } + DbEntry() = default; DbEntry(BlockIdExt block_id, KeyHash prev, KeyHash next, RootHash root_hash) : block_id(block_id), prev(prev), next(next), root_hash(root_hash) { } @@ -109,8 +111,6 @@ class CellDbIn : public CellDbBase { std::unique_ptr boc_; std::shared_ptr cell_db_; - std::shared_ptr statistics_; - td::Timestamp statistics_flush_at_ = td::Timestamp::never(); std::function on_load_callback_; std::set cells_to_migrate_; @@ -127,6 +127,21 @@ class CellDbIn : public CellDbBase { }; std::unique_ptr migration_stats_; + struct CellDbStatistics { + PercentileStats store_cell_time_; + PercentileStats gc_cell_time_; + td::Timestamp stats_start_time_ = td::Timestamp::now(); + + std::string to_string(); + void clear() { + *this = CellDbStatistics{}; + } + }; + + std::shared_ptr statistics_; + CellDbStatistics cell_db_statistics_; + td::Timestamp statistics_flush_at_ = td::Timestamp::never(); + public: class MigrationProxy : public td::actor::Actor { public: diff --git a/validator/db/db-utils.cpp b/validator/db/db-utils.cpp new file mode 100644 index 000000000..9375f97e6 --- /dev/null +++ b/validator/db/db-utils.cpp @@ -0,0 +1,54 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "db-utils.h" + +#include "td/utils/logging.h" + +#include + +namespace ton::validator { + +void PercentileStats::insert(double value) { + values_.insert(value); +} + +std::string PercentileStats::to_string() const { + double percentiles[4] = {0.0, 0.0, 0.0, 0.0}; + double sum = 0.0; + size_t size = values_.size(); + if (!values_.empty()) { + size_t indices[4] = {(size_t)std::ceil(0.5 * (double)size) - 1, (size_t)std::ceil(0.95 * (double)size) - 1, + (size_t)std::ceil(0.99 * (double)size) - 1, size - 1}; + size_t i = 0; + for (auto it = values_.begin(); it != values_.end(); ++it, ++i) { + for (size_t j = 0; j < 4; ++j) { + if (indices[j] == i) { + percentiles[j] = *it; + } + } + sum += *it; + } + } + return PSTRING() << "P50 : " << percentiles[0] << " P95 : " << percentiles[1] << " P99 : " << percentiles[2] + << " P100 : " << percentiles[3] << " COUNT : " << size << " SUM : " << sum; +} + +void PercentileStats::clear() { + values_.clear(); +} + +} // namespace ton::validator \ No newline at end of file diff --git a/validator/db/db-utils.h b/validator/db/db-utils.h new file mode 100644 index 000000000..8ffbf5c97 --- /dev/null +++ b/validator/db/db-utils.h @@ -0,0 +1,33 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include +#include + +namespace ton::validator { + +class PercentileStats { + public: + void insert(double value); + std::string to_string() const; + void clear(); + + private: + std::multiset values_; +}; + +} // namespace ton::validator \ No newline at end of file diff --git a/validator/db/files-async.hpp b/validator/db/files-async.hpp index bcb7fa8b7..c85089963 100644 --- a/validator/db/files-async.hpp +++ b/validator/db/files-async.hpp @@ -57,7 +57,7 @@ class WriteFile : public td::actor::Actor { status = file.sync(); } if (status.is_error()) { - td::unlink(old_name); + td::unlink(old_name).ignore(); promise_.set_error(std::move(status)); stop(); return; diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index ff9abae68..93dcfc91f 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -317,6 +317,12 @@ void RootDb::check_zero_state_file_exists(BlockIdExt block_id, td::Promise td::actor::send_closure(archive_db_, &ArchiveManager::check_zero_state, block_id, std::move(promise)); } +void RootDb::get_previous_persistent_state_files( + BlockSeqno cur_mc_seqno, td::Promise>> promise) { + td::actor::send_closure(archive_db_, &ArchiveManager::get_previous_persistent_state_files, cur_mc_seqno, + std::move(promise)); +} + void RootDb::store_block_handle(BlockHandle handle, td::Promise promise) { td::actor::send_closure(archive_db_, &ArchiveManager::update_handle, std::move(handle), std::move(promise)); } diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 97b9550b8..45044e4f8 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -84,6 +84,8 @@ class RootDb : public Db { void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override; void get_zero_state_file(BlockIdExt block_id, td::Promise promise) override; void check_zero_state_file_exists(BlockIdExt block_id, td::Promise promise) override; + void get_previous_persistent_state_files( + BlockSeqno cur_mc_seqno, td::Promise>> promise) override; void try_get_static_file(FileHash file_hash, td::Promise promise) override; diff --git a/validator/downloaders/wait-block-data.cpp b/validator/downloaders/wait-block-data.cpp index c40dba487..220a8a2cf 100644 --- a/validator/downloaders/wait-block-data.cpp +++ b/validator/downloaders/wait-block-data.cpp @@ -17,10 +17,14 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "wait-block-data.hpp" + +#include "block-parse.h" +#include "block-auto.h" #include "fabric.h" #include "adnl/utils.hpp" #include "ton/ton-io.hpp" #include "common/delay.h" +#include "vm/cells/MerkleProof.h" namespace ton { @@ -108,7 +112,7 @@ void WaitBlockData::start() { td::actor::send_closure(SelfId, &WaitBlockData::failed_to_get_block_data_from_net, R.move_as_error_prefix("net error: ")); } else { - td::actor::send_closure(SelfId, &WaitBlockData::got_block_data_from_net, R.move_as_ok()); + td::actor::send_closure(SelfId, &WaitBlockData::got_data_from_net, R.move_as_ok()); } }); @@ -133,13 +137,49 @@ void WaitBlockData::failed_to_get_block_data_from_net(td::Status reason) { td::Timestamp::in(0.1)); } -void WaitBlockData::got_block_data_from_net(ReceivedBlock block) { +void WaitBlockData::got_data_from_net(ReceivedBlock block) { auto X = create_block(std::move(block)); if (X.is_error()) { failed_to_get_block_data_from_net(X.move_as_error_prefix("bad block from net: ")); return; } - data_ = X.move_as_ok(); + got_block_data_from_net(X.move_as_ok()); +} + +void WaitBlockData::got_block_data_from_net(td::Ref block) { + if (data_.not_null()) { + return; + } + data_ = std::move(block); + if (handle_->received()) { + finish_query(); + return; + } + if (!handle_->id().is_masterchain() && !handle_->inited_proof_link()) { + // This can happen if we get block from candidates cache. + // Proof link can be derived from the block (but not for masterchain block). + auto r_proof_link = generate_proof_link(handle_->id(), data_->root_cell()); + if (r_proof_link.is_error()) { + abort_query(r_proof_link.move_as_error_prefix("failed to create proof link for block: ")); + return; + } + td::actor::send_closure(manager_, &ValidatorManager::validate_block_proof_link, handle_->id(), + r_proof_link.move_as_ok(), + [id = handle_->id().id, SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &WaitBlockData::abort_query, + R.move_as_error_prefix("validate proof link error: ")); + return; + } + LOG(DEBUG) << "Created and validated proof link for " << id.to_str(); + td::actor::send_closure(SelfId, &WaitBlockData::checked_proof_link); + }); + return; + } + checked_proof_link(); +} + +void WaitBlockData::checked_proof_link() { CHECK(handle_->id().is_masterchain() ? handle_->inited_proof() : handle_->inited_proof_link()); if (!handle_->received()) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { @@ -198,6 +238,41 @@ void WaitBlockData::got_static_file(td::BufferSlice data) { run_hardfork_accept_block_query(handle_->id(), data_, manager_, std::move(P)); } +td::Result WaitBlockData::generate_proof_link(BlockIdExt id, td::Ref block_root) { + // Creating proof link. Similar to accept-block.cpp + if (id.is_masterchain()) { + return td::Status::Error("cannot create proof link for masterchain block"); + } + auto usage_tree = std::make_shared(); + auto usage_cell = vm::UsageCell::create(block_root, usage_tree->root_ptr()); + + block::gen::Block::Record blk; + block::gen::BlockInfo::Record info; + block::gen::BlockExtra::Record extra; + block::gen::ExtBlkRef::Record mcref{}; // _ ExtBlkRef = BlkMasterInfo; + ShardIdFull shard; + if (!(tlb::unpack_cell(usage_cell, blk) && tlb::unpack_cell(blk.info, info) && !info.version && + block::tlb::t_ShardIdent.unpack(info.shard.write(), shard) && + block::gen::BlkPrevInfo{info.after_merge}.validate_ref(info.prev_ref) && + tlb::unpack_cell(std::move(blk.extra), extra) && block::gen::t_ValueFlow.force_validate_ref(blk.value_flow) && + (!info.not_master || tlb::unpack_cell(info.master_ref, mcref)))) { + return td::Status::Error("cannot unpack block header"); + } + vm::CellSlice upd_cs{vm::NoVmSpec(), blk.state_update}; + + auto proof = vm::MerkleProof::generate(block_root, usage_tree.get()); + vm::CellBuilder cb; + td::Ref bs_cell; + if (!(cb.store_long_bool(0xc3, 8) // block_proof#c3 + && block::tlb::t_BlockIdExt.pack(cb, id) // proof_for:BlockIdExt + && cb.store_ref_bool(std::move(proof)) // proof:^Cell + && cb.store_bool_bool(false) // signatures:(Maybe ^BlockSignatures) + && cb.finalize_to(bs_cell))) { + return td::Status::Error("cannot serialize BlockProof"); + } + return std_boc_serialize(bs_cell, 0); +} + } // namespace validator } // namespace ton diff --git a/validator/downloaders/wait-block-data.hpp b/validator/downloaders/wait-block-data.hpp index 9a03b1cb6..229b4bfc8 100644 --- a/validator/downloaders/wait-block-data.hpp +++ b/validator/downloaders/wait-block-data.hpp @@ -57,11 +57,15 @@ class WaitBlockData : public td::actor::Actor { void set_is_hardfork(bool value); void start(); void got_block_data_from_db(td::Ref data); - void got_block_data_from_net(ReceivedBlock data); + void got_data_from_net(ReceivedBlock data); + void got_block_data_from_net(td::Ref block); + void checked_proof_link(); void failed_to_get_block_data_from_net(td::Status reason); void got_static_file(td::BufferSlice data); + static td::Result generate_proof_link(BlockIdExt id, td::Ref block_root); + private: BlockHandle handle_; diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp index 42f5c791b..0ae82beaa 100644 --- a/validator/downloaders/wait-block-state.cpp +++ b/validator/downloaders/wait-block-state.cpp @@ -192,7 +192,8 @@ void WaitBlockState::got_proof_link(td::BufferSlice data) { td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link); } else { LOG(INFO) << "received bad proof link: " << R.move_as_error(); - td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link); + delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link); }, + td::Timestamp::in(0.1)); } }); run_check_proof_link_query(handle_->id(), R.move_as_ok(), manager_, timeout_, std::move(P)); diff --git a/validator/fabric.h b/validator/fabric.h index 326b17aec..949a6c9ff 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -49,8 +49,8 @@ td::Result>> create_new_shard_bloc td::Ref create_signature_set(std::vector sig_set); -void run_check_external_message(td::BufferSlice data, block::SizeLimitsConfig::ExtMsgLimits limits, - td::actor::ActorId manager, td::Promise> promise); +void run_check_external_message(td::Ref message, td::actor::ActorId manager, + td::Promise> promise); void run_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, @@ -80,8 +80,8 @@ void run_validate_query(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_maste td::Promise promise, bool is_fake = false); void run_collate_query(ShardIdFull shard, td::uint32 min_ts, const BlockIdExt& min_masterchain_block_id, std::vector prev, Ed25519_PublicKey local_id, td::Ref validator_set, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise); + td::Ref collator_opts, td::actor::ActorId manager, + td::Timestamp timeout, td::Promise promise); void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp index a64c0e9b9..e5ea1f0be 100644 --- a/validator/full-node-private-overlay.cpp +++ b/validator/full-node-private-overlay.cpp @@ -17,6 +17,7 @@ #include "full-node-private-overlay.hpp" #include "ton/ton-tl.hpp" #include "common/delay.h" +#include "common/checksum.h" #include "full-node-serializer.hpp" namespace ton::validator::fullnode { @@ -38,17 +39,7 @@ void FullNodePrivateBlockOverlay::process_block_broadcast(PublicKeyHash src, ton } VLOG(FULL_NODE_DEBUG) << "Received block broadcast in private overlay from " << src << ": " << B.ok().block_id.to_str(); - auto P = td::PromiseCreator::lambda([](td::Result R) { - if (R.is_error()) { - if (R.error().code() == ErrorCode::notready) { - LOG(DEBUG) << "dropped broadcast: " << R.move_as_error(); - } else { - LOG(INFO) << "dropped broadcast: " << R.move_as_error(); - } - } - }); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, B.move_as_ok(), - std::move(P)); + td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok()); } void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query) { @@ -59,7 +50,45 @@ void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, ton_api:: query.block_->cc_seqno_, std::move(query.block_->data_)); } +void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcast &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressed &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodePrivateBlockOverlay::process_block_candidate_broadcast(PublicKeyHash src, + ton_api::tonNode_Broadcast &query) { + BlockIdExt block_id; + CatchainSeqno cc_seqno; + td::uint32 validator_set_hash; + td::BufferSlice data; + auto S = deserialize_block_candidate_broadcast(query, block_id, cc_seqno, validator_set_hash, data, + overlay::Overlays::max_fec_broadcast_size()); + if (S.is_error()) { + LOG(DEBUG) << "dropped broadcast: " << S; + return; + } + if (data.size() > FullNode::max_block_size()) { + VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; + return; + } + if (td::sha256_bits256(data.as_slice()) != block_id.file_hash) { + VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; + return; + } + VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate in private overlay from " << src << ": " << block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, cc_seqno, + validator_set_hash, std::move(data)); +} + void FullNodePrivateBlockOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { + if (adnl::AdnlNodeIdShort{src} == local_id_) { + return; + } auto B = fetch_tl_object(std::move(broadcast), true); if (B.is_error()) { return; @@ -84,6 +113,22 @@ void FullNodePrivateBlockOverlay::send_shard_block_info(BlockIdExt block_id, Cat } } +void FullNodePrivateBlockOverlay::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::BufferSlice data) { + if (!inited_) { + return; + } + auto B = + serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true); // compression enabled + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block candidate broadcast: " << B.move_as_error(); + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate in private overlay: " << block_id.to_str(); + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); +} + void FullNodePrivateBlockOverlay::send_broadcast(BlockBroadcast broadcast) { if (!inited_) { return; @@ -169,18 +214,86 @@ void FullNodePrivateBlockOverlay::tear_down() { } } +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { + process_block_broadcast(src, query); +} + +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query) { + process_block_broadcast(src, query); +} + +void FullNodeCustomOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { + if (!block_senders_.count(adnl::AdnlNodeIdShort(src))) { + VLOG(FULL_NODE_DEBUG) << "Dropping block broadcast in private overlay \"" << name_ << "\" from unauthorized sender " + << src; + return; + } + auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); + if (B.is_error()) { + LOG(DEBUG) << "dropped broadcast: " << B.move_as_error(); + return; + } + VLOG(FULL_NODE_DEBUG) << "Received block broadcast in custom overlay \"" << name_ << "\" from " << src << ": " + << B.ok().block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok()); +} + void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query) { - auto it = senders_.find(adnl::AdnlNodeIdShort{src}); - if (it == senders_.end()) { + auto it = msg_senders_.find(adnl::AdnlNodeIdShort{src}); + if (it == msg_senders_.end()) { + VLOG(FULL_NODE_DEBUG) << "Dropping external message broadcast in custom overlay \"" << name_ + << "\" from unauthorized sender " << src; return; } - LOG(FULL_NODE_DEBUG) << "Got external message in private overlay \"" << name_ << "\" from " << src - << " (priority=" << it->second << ")"; + VLOG(FULL_NODE_DEBUG) << "Got external message in custom overlay \"" << name_ << "\" from " << src + << " (priority=" << it->second << ")"; td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_external_message, std::move(query.message_->data_), it->second); } +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressed &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodeCustomOverlay::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { + if (!block_senders_.count(adnl::AdnlNodeIdShort(src))) { + VLOG(FULL_NODE_DEBUG) << "Dropping block candidate broadcast in private overlay \"" << name_ + << "\" from unauthorized sender " << src; + return; + } + BlockIdExt block_id; + CatchainSeqno cc_seqno; + td::uint32 validator_set_hash; + td::BufferSlice data; + auto S = deserialize_block_candidate_broadcast(query, block_id, cc_seqno, validator_set_hash, data, + overlay::Overlays::max_fec_broadcast_size()); + if (S.is_error()) { + LOG(DEBUG) << "dropped broadcast: " << S; + return; + } + if (data.size() > FullNode::max_block_size()) { + VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; + return; + } + if (td::sha256_bits256(data.as_slice()) != block_id.file_hash) { + VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; + return; + } + VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate in custom overlay \"" << name_ << "\" from " << src << ": " + << block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, cc_seqno, + validator_set_hash, std::move(data)); +} + void FullNodeCustomOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { + if (adnl::AdnlNodeIdShort{src} == local_id_) { + return; + } auto B = fetch_tl_object(std::move(broadcast), true); if (B.is_error()) { return; @@ -192,7 +305,7 @@ void FullNodeCustomOverlay::send_external_message(td::BufferSlice data) { if (!inited_ || config_.ext_messages_broadcast_disabled_) { return; } - LOG(FULL_NODE_DEBUG) << "Sending external message to private overlay \"" << name_ << "\""; + VLOG(FULL_NODE_DEBUG) << "Sending external message to custom overlay \"" << name_ << "\""; auto B = create_serialize_tl_object( create_tl_object(std::move(data))); if (B.size() <= overlay::Overlays::max_simple_broadcast_size()) { @@ -204,6 +317,37 @@ void FullNodeCustomOverlay::send_external_message(td::BufferSlice data) { } } +void FullNodeCustomOverlay::send_broadcast(BlockBroadcast broadcast) { + if (!inited_) { + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending block broadcast to custom overlay \"" << name_ + << "\": " << broadcast.block_id.to_str(); + auto B = serialize_block_broadcast(broadcast, true); // compression_enabled = true + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error(); + return; + } + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); +} + +void FullNodeCustomOverlay::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::BufferSlice data) { + if (!inited_) { + return; + } + auto B = + serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true); // compression enabled + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block candidate broadcast: " << B.move_as_error(); + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate in custom overlay \"" << name_ << "\": " << block_id.to_str(); + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); +} + void FullNodeCustomOverlay::start_up() { std::sort(nodes_.begin(), nodes_.end()); nodes_.erase(std::unique(nodes_.begin(), nodes_.end()), nodes_.end()); @@ -234,7 +378,8 @@ void FullNodeCustomOverlay::try_init() { void FullNodeCustomOverlay::init() { LOG(FULL_NODE_WARNING) << "Creating custom overlay \"" << name_ << "\" for adnl id " << local_id_ << " : " - << nodes_.size() << " nodes, overlay_id=" << overlay_id_; + << nodes_.size() << " nodes, " << msg_senders_.size() << " msg senders, " + << block_senders_.size() << " block senders, overlay_id=" << overlay_id_; class Callback : public overlay::Overlays::Callback { public: void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { @@ -256,9 +401,12 @@ void FullNodeCustomOverlay::init() { }; std::map authorized_keys; - for (const auto &sender : senders_) { + for (const auto &sender : msg_senders_) { authorized_keys[sender.first.pubkey_hash()] = overlay::Overlays::max_fec_broadcast_size(); } + for (const auto &sender : block_senders_) { + authorized_keys[sender.pubkey_hash()] = overlay::Overlays::max_fec_broadcast_size(); + } overlay::OverlayPrivacyRules rules{overlay::Overlays::max_fec_broadcast_size(), 0, std::move(authorized_keys)}; td::actor::send_closure( overlays_, &overlay::Overlays::create_private_overlay, local_id_, overlay_id_full_.clone(), nodes_, diff --git a/validator/full-node-private-overlay.hpp b/validator/full-node-private-overlay.hpp index c651acefe..a0022fa03 100644 --- a/validator/full-node-private-overlay.hpp +++ b/validator/full-node-private-overlay.hpp @@ -27,6 +27,11 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); + + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); + void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); + template void process_broadcast(PublicKeyHash, T &) { VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast"; @@ -34,36 +39,35 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { void receive_broadcast(PublicKeyHash src, td::BufferSlice query); void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data); + void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data); void send_broadcast(BlockBroadcast broadcast); void set_config(FullNodeConfig config) { config_ = std::move(config); } - void set_enable_compression(bool value) { - enable_compression_ = value; - } - void start_up() override; void tear_down() override; FullNodePrivateBlockOverlay(adnl::AdnlNodeIdShort local_id, std::vector nodes, - FileHash zero_state_file_hash, FullNodeConfig config, bool enable_compression, + FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, - td::actor::ActorId validator_manager) + td::actor::ActorId validator_manager, + td::actor::ActorId full_node) : local_id_(local_id) , nodes_(std::move(nodes)) , zero_state_file_hash_(zero_state_file_hash) , config_(config) - , enable_compression_(enable_compression) , keyring_(keyring) , adnl_(adnl) , rldp_(rldp) , rldp2_(rldp2) , overlays_(overlays) - , validator_manager_(validator_manager) { + , validator_manager_(validator_manager) + , full_node_(full_node) { } private: @@ -71,7 +75,7 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { std::vector nodes_; FileHash zero_state_file_hash_; FullNodeConfig config_; - bool enable_compression_; + bool enable_compression_ = true; td::actor::ActorId keyring_; td::actor::ActorId adnl_; @@ -79,6 +83,7 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { td::actor::ActorId rldp2_; td::actor::ActorId overlays_; td::actor::ActorId validator_manager_; + td::actor::ActorId full_node_; bool inited_ = false; overlay::OverlayIdFull overlay_id_full_; @@ -90,7 +95,16 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { class FullNodeCustomOverlay : public td::actor::Actor { public: + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query); + void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); + + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); + void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); + template void process_broadcast(PublicKeyHash, T &) { VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast"; @@ -98,6 +112,9 @@ class FullNodeCustomOverlay : public td::actor::Actor { void receive_broadcast(PublicKeyHash src, td::BufferSlice query); void send_external_message(td::BufferSlice data); + void send_broadcast(BlockBroadcast broadcast); + void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data); void set_config(FullNodeConfig config) { config_ = std::move(config); @@ -106,16 +123,17 @@ class FullNodeCustomOverlay : public td::actor::Actor { void start_up() override; void tear_down() override; - FullNodeCustomOverlay(adnl::AdnlNodeIdShort local_id, std::vector nodes, - std::map senders, std::string name, FileHash zero_state_file_hash, + FullNodeCustomOverlay(adnl::AdnlNodeIdShort local_id, CustomOverlayParams params, FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, - td::actor::ActorId validator_manager) + td::actor::ActorId validator_manager, + td::actor::ActorId full_node) : local_id_(local_id) - , nodes_(std::move(nodes)) - , senders_(std::move(senders)) - , name_(std::move(name)) + , name_(std::move(params.name_)) + , nodes_(std::move(params.nodes_)) + , msg_senders_(std::move(params.msg_senders_)) + , block_senders_(std::move(params.block_senders_)) , zero_state_file_hash_(zero_state_file_hash) , config_(config) , keyring_(keyring) @@ -123,14 +141,16 @@ class FullNodeCustomOverlay : public td::actor::Actor { , rldp_(rldp) , rldp2_(rldp2) , overlays_(overlays) - , validator_manager_(validator_manager) { + , validator_manager_(validator_manager) + , full_node_(full_node) { } private: adnl::AdnlNodeIdShort local_id_; - std::vector nodes_; - std::map senders_; std::string name_; + std::vector nodes_; + std::map msg_senders_; + std::set block_senders_; FileHash zero_state_file_hash_; FullNodeConfig config_; @@ -140,6 +160,7 @@ class FullNodeCustomOverlay : public td::actor::Actor { td::actor::ActorId rldp2_; td::actor::ActorId overlays_; td::actor::ActorId validator_manager_; + td::actor::ActorId full_node_; bool inited_ = false; overlay::OverlayIdFull overlay_id_full_; diff --git a/validator/full-node-serializer.cpp b/validator/full-node-serializer.cpp index 42e682864..94dc2155e 100644 --- a/validator/full-node-serializer.cpp +++ b/validator/full-node-serializer.cpp @@ -152,4 +152,63 @@ td::Status deserialize_block_full(ton_api::tonNode_DataFull& obj, BlockIdExt& id return S; } +td::Result serialize_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Slice data, + bool compression_enabled) { + if (!compression_enabled) { + return create_serialize_tl_object( + create_tl_block_id(block_id), cc_seqno, validator_set_hash, + create_tl_object(Bits256::zero(), td::BufferSlice()), td::BufferSlice(data)); + } + TRY_RESULT(root, vm::std_boc_deserialize(data)); + TRY_RESULT(data_new, vm::std_boc_serialize(root, 2)); + td::BufferSlice compressed = td::lz4_compress(data_new); + VLOG(FULL_NODE_DEBUG) << "Compressing block candidate broadcast: " << data.size() << " -> " << compressed.size(); + return create_serialize_tl_object( + create_tl_block_id(block_id), cc_seqno, validator_set_hash, + create_tl_object(Bits256::zero(), td::BufferSlice()), 0, std::move(compressed)); +} + +static td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_newBlockCandidateBroadcast& obj, + BlockIdExt& block_id, CatchainSeqno& cc_seqno, + td::uint32& validator_set_hash, td::BufferSlice& data) { + block_id = create_block_id(obj.id_); + cc_seqno = obj.catchain_seqno_; + validator_set_hash = obj.validator_set_hash_; + data = std::move(obj.data_); + return td::Status::OK(); +} + +static td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_newBlockCandidateBroadcastCompressed& obj, + BlockIdExt& block_id, CatchainSeqno& cc_seqno, + td::uint32& validator_set_hash, td::BufferSlice& data, + int max_decompressed_data_size) { + block_id = create_block_id(obj.id_); + cc_seqno = obj.catchain_seqno_; + validator_set_hash = obj.validator_set_hash_; + TRY_RESULT(decompressed, td::lz4_decompress(obj.compressed_, max_decompressed_data_size)); + TRY_RESULT(root, vm::std_boc_deserialize(decompressed)); + TRY_RESULT_ASSIGN(data, vm::std_boc_serialize(root, 31)); + VLOG(FULL_NODE_DEBUG) << "Decompressing block candidate broadcast: " << obj.compressed_.size() << " -> " + << data.size(); + return td::Status::OK(); +} + +td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_Broadcast& obj, BlockIdExt& block_id, + CatchainSeqno& cc_seqno, td::uint32& validator_set_hash, + td::BufferSlice& data, int max_decompressed_data_size) { + td::Status S; + ton_api::downcast_call(obj, td::overloaded( + [&](ton_api::tonNode_newBlockCandidateBroadcast& f) { + S = deserialize_block_candidate_broadcast(f, block_id, cc_seqno, validator_set_hash, + data); + }, + [&](ton_api::tonNode_newBlockCandidateBroadcastCompressed& f) { + S = deserialize_block_candidate_broadcast(f, block_id, cc_seqno, validator_set_hash, + data, max_decompressed_data_size); + }, + [&](auto&) { S = td::Status::Error("unknown data type"); })); + return S; +} + } // namespace ton::validator::fullnode diff --git a/validator/full-node-serializer.hpp b/validator/full-node-serializer.hpp index a5c73cbc2..f6751689c 100644 --- a/validator/full-node-serializer.hpp +++ b/validator/full-node-serializer.hpp @@ -28,4 +28,11 @@ td::Result serialize_block_full(const BlockIdExt& id, td::Slice td::Status deserialize_block_full(ton_api::tonNode_DataFull& obj, BlockIdExt& id, td::BufferSlice& proof, td::BufferSlice& data, bool& is_proof_link, int max_decompressed_data_size); +td::Result serialize_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Slice data, + bool compression_enabled); +td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_Broadcast& obj, BlockIdExt& block_id, + CatchainSeqno& cc_seqno, td::uint32& validator_set_hash, + td::BufferSlice& data, int max_decompressed_data_size); + } // namespace ton::validator::fullnode diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 433d9469b..fbdbbfd7a 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -17,12 +17,14 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "auto/tl/ton_api.h" +#include "checksum.h" #include "overlays.h" #include "td/utils/SharedSlice.h" #include "full-node-shard.hpp" #include "full-node-shard-queries.hpp" #include "full-node-serializer.hpp" +#include "td/utils/buffer.h" #include "ton/ton-shard.h" #include "ton/ton-tl.hpp" #include "ton/ton-io.hpp" @@ -646,6 +648,35 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ne query.block_->cc_seqno_, std::move(query.block_->data_)); } +void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodeShardImpl::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressed &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodeShardImpl::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { + BlockIdExt block_id; + CatchainSeqno cc_seqno; + td::uint32 validator_set_hash; + td::BufferSlice data; + auto S = deserialize_block_candidate_broadcast(query, block_id, cc_seqno, validator_set_hash, data, + overlay::Overlays::max_fec_broadcast_size()); + if (data.size() > FullNode::max_block_size()) { + VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; + return; + } + if (td::sha256_bits256(data.as_slice()) != block_id.file_hash) { + VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; + return; + } + VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate from " << src << ": " << block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, cc_seqno, + validator_set_hash, std::move(data)); +} + void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { process_block_broadcast(src, query); } @@ -661,17 +692,7 @@ void FullNodeShardImpl::process_block_broadcast(PublicKeyHash src, ton_api::tonN return; } VLOG(FULL_NODE_DEBUG) << "Received block broadcast from " << src << ": " << B.ok().block_id.to_str(); - auto P = td::PromiseCreator::lambda([](td::Result R) { - if (R.is_error()) { - if (R.error().code() == ErrorCode::notready) { - LOG(DEBUG) << "dropped broadcast: " << R.move_as_error(); - } else { - LOG(INFO) << "dropped broadcast: " << R.move_as_error(); - } - } - }); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, B.move_as_ok(), - std::move(P)); + td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok()); } void FullNodeShardImpl::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { @@ -748,6 +769,23 @@ void FullNodeShardImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno } } +void FullNodeShardImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) { + if (!client_.empty()) { + UNREACHABLE(); + return; + } + auto B = + serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true); // compression enabled + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block candidate broadcast: " << B.move_as_error(); + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate: " << block_id.to_str(); + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, adnl_id_, overlay_id_, local_id_, + overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); +} + void FullNodeShardImpl::send_broadcast(BlockBroadcast broadcast) { if (!client_.empty()) { UNREACHABLE(); @@ -794,8 +832,7 @@ void FullNodeShardImpl::download_persistent_state(BlockIdExt id, BlockIdExt mast auto &b = choose_neighbour(); td::actor::create_actor(PSTRING() << "downloadstatereq" << id.id.to_str(), id, masterchain_block_id, adnl_id_, overlay_id_, b.adnl_id, priority, timeout, validator_manager_, - b.use_rldp2() ? (td::actor::ActorId)rldp2_ : rldp_, - overlays_, adnl_, client_, std::move(promise)) + rldp2_, overlays_, adnl_, client_, std::move(promise)) .release(); } @@ -829,10 +866,9 @@ void FullNodeShardImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp t void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) { auto &b = choose_neighbour(); - td::actor::create_actor( - "archive", masterchain_seqno, std::move(tmp_dir), adnl_id_, overlay_id_, b.adnl_id, timeout, validator_manager_, - b.use_rldp2() ? (td::actor::ActorId)rldp2_ : rldp_, overlays_, adnl_, client_, - create_neighbour_promise(b, std::move(promise))) + td::actor::create_actor("archive", masterchain_seqno, std::move(tmp_dir), adnl_id_, overlay_id_, + b.adnl_id, timeout, validator_manager_, rldp2_, overlays_, adnl_, + client_, create_neighbour_promise(b, std::move(promise))) .release(); } @@ -1137,7 +1173,8 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client) + td::actor::ActorId client, + td::actor::ActorId full_node) : shard_(shard) , local_id_(local_id) , adnl_id_(adnl_id) @@ -1149,6 +1186,7 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, , overlays_(overlays) , validator_manager_(validator_manager) , client_(client) + , full_node_(full_node) , config_(config) { } @@ -1157,9 +1195,10 @@ td::actor::ActorOwn FullNodeShard::create( FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client) { + td::actor::ActorId client, td::actor::ActorId full_node) { return td::actor::create_actor("tonnode", shard, local_id, adnl_id, zero_state_file_hash, config, - keyring, adnl, rldp, rldp2, overlays, validator_manager, client); + keyring, adnl, rldp, rldp2, overlays, validator_manager, client, + full_node); } } // namespace fullnode diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 1b742fb98..e89031fe9 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -41,6 +41,8 @@ class FullNodeShard : public td::actor::Actor { virtual void send_ihr_message(td::BufferSlice data) = 0; virtual void send_external_message(td::BufferSlice data) = 0; virtual void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; + virtual void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) = 0; virtual void send_broadcast(BlockBroadcast broadcast) = 0; virtual void sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) = 0; @@ -72,7 +74,7 @@ class FullNodeShard : public td::actor::Actor { FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client); + td::actor::ActorId client, td::actor::ActorId full_node); }; } // namespace fullnode diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index c8b5301ae..a7cf89ac5 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -18,6 +18,7 @@ */ #pragma once +#include "auto/tl/ton_api.h" #include "full-node-shard.h" #include "td/actor/PromiseFuture.h" #include "td/utils/port/Poll.h" @@ -45,10 +46,6 @@ struct Neighbour { void query_failed(); void update_roundtrip(double t); - bool use_rldp2() const { - return std::make_pair(proto_version, capabilities) >= std::make_pair(2, 2); - } - static Neighbour zero; }; @@ -152,12 +149,19 @@ class FullNodeShardImpl : public FullNodeShard { void process_broadcast(PublicKeyHash src, ton_api::tonNode_ihrMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); + + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); + void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); + void receive_broadcast(PublicKeyHash src, td::BufferSlice query); void check_broadcast(PublicKeyHash src, td::BufferSlice query, td::Promise promise); void send_ihr_message(td::BufferSlice data) override; void send_external_message(td::BufferSlice data) override; void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; + void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override; void send_broadcast(BlockBroadcast broadcast) override; void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, @@ -214,7 +218,7 @@ class FullNodeShardImpl : public FullNodeShard { td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client); + td::actor::ActorId client, td::actor::ActorId full_node); private: bool use_new_download() const { @@ -236,6 +240,7 @@ class FullNodeShardImpl : public FullNodeShard { td::actor::ActorId overlays_; td::actor::ActorId validator_manager_; td::actor::ActorId client_; + td::actor::ActorId full_node_; td::uint32 attempt_ = 0; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index affb8bcc8..a72be3ff4 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -36,8 +36,8 @@ void FullNodeImpl::add_permanent_key(PublicKeyHash key, td::Promise pr local_keys_.insert(key); create_private_block_overlay(key); - for (auto &p : private_custom_overlays_) { - update_ext_msg_overlay(p.first, p.second); + for (auto &p : custom_overlays_) { + update_custom_overlay(p.second); } if (!sign_cert_by_.is_zero()) { @@ -64,8 +64,8 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise pr } local_keys_.erase(key); private_block_overlays_.erase(key); - for (auto &p : private_custom_overlays_) { - update_ext_msg_overlay(p.first, p.second); + for (auto &p : custom_overlays_) { + update_custom_overlay(p.second); } if (sign_cert_by_ != key) { @@ -119,8 +119,8 @@ void FullNodeImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise nodes, - std::map senders, std::string name, - td::Promise promise) { - if (nodes.empty()) { +void FullNodeImpl::add_custom_overlay(CustomOverlayParams params, td::Promise promise) { + if (params.nodes_.empty()) { promise.set_error(td::Status::Error("list of nodes is empty")); return; } - if (private_custom_overlays_.count(name)) { - promise.set_error(td::Status::Error(PSTRING() << "duplicate overlay name \"" << name << "\"")); + std::string name = params.name_; + if (custom_overlays_.count(name)) { + promise.set_error(td::Status::Error(PSTRING() << "duplicate custom overlay name \"" << name << "\"")); return; } - VLOG(FULL_NODE_WARNING) << "Adding private overlay for external messages \"" << name << "\", " << nodes.size() - << " nodes"; - auto &p = private_custom_overlays_[name]; - p.nodes_ = nodes; - p.senders_ = senders; - update_ext_msg_overlay(name, p); + VLOG(FULL_NODE_WARNING) << "Adding custom overlay \"" << name << "\", " << params.nodes_.size() << " nodes"; + auto &p = custom_overlays_[name]; + p.params_ = std::move(params); + update_custom_overlay(p); promise.set_result(td::Unit()); } -void FullNodeImpl::del_ext_msg_overlay(std::string name, td::Promise promise) { - auto it = private_custom_overlays_.find(name); - if (it == private_custom_overlays_.end()) { +void FullNodeImpl::del_custom_overlay(std::string name, td::Promise promise) { + auto it = custom_overlays_.find(name); + if (it == custom_overlays_.end()) { promise.set_error(td::Status::Error(PSTRING() << "no such overlay \"" << name << "\"")); return; } - private_custom_overlays_.erase(it); + custom_overlays_.erase(it); promise.set_result(td::Unit()); } @@ -182,8 +179,9 @@ void FullNodeImpl::initial_read_complete(BlockHandle top_handle) { void FullNodeImpl::add_shard(ShardIdFull shard) { while (true) { if (shards_.count(shard) == 0) { - shards_.emplace(shard, FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, config_, keyring_, - adnl_, rldp_, rldp2_, overlays_, validator_manager_, client_)); + shards_.emplace(shard, + FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, config_, keyring_, adnl_, + rldp_, rldp2_, overlays_, validator_manager_, client_, actor_id(this))); if (all_validators_.size() > 0) { td::actor::send_closure(shards_[shard], &FullNodeShard::update_validators, all_validators_, sign_cert_by_); } @@ -221,10 +219,10 @@ void FullNodeImpl::send_ext_message(AccountIdPrefixFull dst, td::BufferSlice dat VLOG(FULL_NODE_WARNING) << "dropping OUT ext message to unknown shard"; return; } - for (auto &private_overlay : private_custom_overlays_) { + for (auto &private_overlay : custom_overlays_) { for (auto &actor : private_overlay.second.actors_) { auto local_id = actor.first; - if (private_overlay.second.senders_.count(local_id)) { + if (private_overlay.second.params_.msg_senders_.count(local_id)) { td::actor::send_closure(actor.second, &FullNodeCustomOverlay::send_external_message, data.clone()); } } @@ -245,13 +243,35 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s td::actor::send_closure(shard, &FullNodeShard::send_shard_block_info, block_id, cc_seqno, std::move(data)); } -void FullNodeImpl::send_broadcast(BlockBroadcast broadcast) { +void FullNodeImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) { + send_block_candidate_broadcast_to_custom_overlays(block_id, cc_seqno, validator_set_hash, data); + auto shard = get_shard(ShardIdFull{masterchainId, shardIdAll}); + if (shard.empty()) { + VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard"; + return; + } + if (!private_block_overlays_.empty()) { + td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_block_candidate, + block_id, cc_seqno, validator_set_hash, data.clone()); + } + if (broadcast_block_candidates_in_public_overlay_) { + td::actor::send_closure(shard, &FullNodeShard::send_block_candidate, block_id, cc_seqno, validator_set_hash, + std::move(data)); + } +} + +void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) { + send_block_broadcast_to_custom_overlays(broadcast); + if (custom_overlays_only) { + return; + } auto shard = get_shard(ShardIdFull{masterchainId}); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; return; } - if (!private_block_overlays_.empty()) { + if (broadcast.block_id.is_masterchain() && !private_block_overlays_.empty()) { td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_broadcast, broadcast.clone()); } @@ -349,14 +369,12 @@ td::actor::ActorId FullNodeImpl::get_shard(AccountIdPrefixFull ds return get_shard(shard_prefix(dst, 60)); } -void FullNodeImpl::got_key_block_state(td::Ref state) { - auto m = td::Ref{std::move(state)}; - +void FullNodeImpl::got_key_block_config(td::Ref config) { PublicKeyHash l = PublicKeyHash::zero(); std::vector keys; std::map current_validators; for (td::int32 i = -1; i <= 1; i++) { - auto r = m->get_total_validator_set(i < 0 ? i : 1 - i); + auto r = config->get_total_validator_set(i < 0 ? i : 1 - i); if (r.not_null()) { auto vec = r->export_vector(); for (auto &el : vec) { @@ -372,16 +390,15 @@ void FullNodeImpl::got_key_block_state(td::Ref state) { } } - set_private_block_overlays_enable_compression(m->get_consensus_config().proto_version >= 3); - if (current_validators != current_validators_) { current_validators_ = std::move(current_validators); update_private_overlays(); } - if (keys == all_validators_) { - return; - } + // Let's turn off this optimization, since keyblocks are rare enough to update on each keyblock + // if (keys == all_validators_) { + // return; + // } all_validators_ = keys; sign_cert_by_ = l; @@ -393,18 +410,57 @@ void FullNodeImpl::got_key_block_state(td::Ref state) { } void FullNodeImpl::new_key_block(BlockHandle handle) { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { - if (R.is_error()) { - VLOG(FULL_NODE_WARNING) << "failed to get key block state: " << R.move_as_error(); - } else { - td::actor::send_closure(SelfId, &FullNodeImpl::got_key_block_state, R.move_as_ok()); - } - }); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_shard_state_from_db, handle, - std::move(P)); + if (handle->id().seqno() == 0) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to get zero state: " << R.move_as_error(); + } else { + auto s = td::Ref{R.move_as_ok()}; + CHECK(s.not_null()); + td::actor::send_closure(SelfId, &FullNodeImpl::got_key_block_config, s->get_config_holder().move_as_ok()); + } + }); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_shard_state_from_db, handle, + std::move(P)); + } else { + CHECK(handle->is_key_block()); + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to get key block proof: " << R.move_as_error(); + } else { + td::actor::send_closure(SelfId, &FullNodeImpl::got_key_block_config, + R.ok()->get_key_block_config().move_as_ok()); + } + }); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_block_proof_link_from_db, handle, + std::move(P)); + } +} + +void FullNodeImpl::process_block_broadcast(BlockBroadcast broadcast) { + send_block_broadcast_to_custom_overlays(broadcast); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, std::move(broadcast), + [](td::Result R) { + if (R.is_error()) { + if (R.error().code() == ErrorCode::notready) { + LOG(DEBUG) << "dropped broadcast: " << R.move_as_error(); + } else { + LOG(INFO) << "dropped broadcast: " << R.move_as_error(); + } + } + }); +} + +void FullNodeImpl::process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::BufferSlice data) { + send_block_candidate_broadcast_to_custom_overlays(block_id, cc_seqno, validator_set_hash, data); + // ignore cc_seqno and validator_hash for now + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_block_candidate, block_id, + std::move(data)); } void FullNodeImpl::start_up() { + add_shard(ShardIdFull{masterchainId}); if (local_id_.is_zero()) { if(adnl_id_.is_zero()) { auto pk = ton::PrivateKey{ton::privkeys::Ed25519::random()}; @@ -435,8 +491,13 @@ void FullNodeImpl::start_up() { void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override { td::actor::send_closure(id_, &FullNodeImpl::send_shard_block_info, block_id, cc_seqno, std::move(data)); } - void send_broadcast(BlockBroadcast broadcast) override { - td::actor::send_closure(id_, &FullNodeImpl::send_broadcast, std::move(broadcast)); + void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override { + td::actor::send_closure(id_, &FullNodeImpl::send_block_candidate, block_id, cc_seqno, validator_set_hash, + std::move(data)); + } + void send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) override { + td::actor::send_closure(id_, &FullNodeImpl::send_broadcast, std::move(broadcast), custom_overlays_only); } void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) override { @@ -488,8 +549,8 @@ void FullNodeImpl::start_up() { } void FullNodeImpl::update_private_overlays() { - for (auto &p : private_custom_overlays_) { - update_ext_msg_overlay(p.first, p.second); + for (auto &p : custom_overlays_) { + update_custom_overlay(p.second); } private_block_overlays_.clear(); @@ -501,16 +562,6 @@ void FullNodeImpl::update_private_overlays() { } } -void FullNodeImpl::set_private_block_overlays_enable_compression(bool value) { - if (private_block_overlays_enable_compression_ == value) { - return; - } - private_block_overlays_enable_compression_ = true; - for (auto &p : private_block_overlays_) { - td::actor::send_closure(p.second, &FullNodePrivateBlockOverlay::set_enable_compression, value); - } -} - void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { CHECK(local_keys_.count(key)); if (current_validators_.count(key)) { @@ -519,29 +570,30 @@ void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { nodes.push_back(p.second); } private_block_overlays_[key] = td::actor::create_actor( - "BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, config_, - private_block_overlays_enable_compression_, keyring_, adnl_, rldp_, rldp2_, overlays_, validator_manager_); + "BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, config_, keyring_, + adnl_, rldp_, rldp2_, overlays_, validator_manager_, actor_id(this)); } } -void FullNodeImpl::update_ext_msg_overlay(const std::string &name, ExtMsgOverlayInfo &overlay) { +void FullNodeImpl::update_custom_overlay(CustomOverlayInfo &overlay) { auto old_actors = std::move(overlay.actors_); overlay.actors_.clear(); + CustomOverlayParams ¶ms = overlay.params_; auto try_local_id = [&](const adnl::AdnlNodeIdShort &local_id) { - if (std::find(overlay.nodes_.begin(), overlay.nodes_.end(), local_id) != overlay.nodes_.end()) { + if (std::find(params.nodes_.begin(), params.nodes_.end(), local_id) != params.nodes_.end()) { auto it = old_actors.find(local_id); if (it != old_actors.end()) { overlay.actors_[local_id] = std::move(it->second); old_actors.erase(it); } else { overlay.actors_[local_id] = td::actor::create_actor( - "CustomOverlay", local_id, overlay.nodes_, overlay.senders_, name, zero_state_file_hash_, config_, - keyring_, adnl_, rldp_, rldp2_, overlays_, validator_manager_); + "CustomOverlay", local_id, params, zero_state_file_hash_, config_, keyring_, adnl_, rldp_, rldp2_, + overlays_, validator_manager_, actor_id(this)); } } }; try_local_id(adnl_id_); - for (const PublicKeyHash &local_key: local_keys_) { + for (const PublicKeyHash &local_key : local_keys_) { auto it = current_validators_.find(local_key); if (it != current_validators_.end()) { try_local_id(it->second); @@ -549,6 +601,48 @@ void FullNodeImpl::update_ext_msg_overlay(const std::string &name, ExtMsgOverlay } } +void FullNodeImpl::send_block_broadcast_to_custom_overlays(const BlockBroadcast& broadcast) { + if (!custom_overlays_sent_broadcasts_.insert(broadcast.block_id).second) { + return; + } + custom_overlays_sent_broadcasts_lru_.push(broadcast.block_id); + if (custom_overlays_sent_broadcasts_lru_.size() > 256) { + custom_overlays_sent_broadcasts_.erase(custom_overlays_sent_broadcasts_lru_.front()); + custom_overlays_sent_broadcasts_lru_.pop(); + } + for (auto &private_overlay : custom_overlays_) { + for (auto &actor : private_overlay.second.actors_) { + auto local_id = actor.first; + if (private_overlay.second.params_.block_senders_.count(local_id)) { + td::actor::send_closure(actor.second, &FullNodeCustomOverlay::send_broadcast, broadcast.clone()); + } + } + } +} + +void FullNodeImpl::send_block_candidate_broadcast_to_custom_overlays(const BlockIdExt &block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, + const td::BufferSlice &data) { + // Same cache of sent broadcasts as in send_block_broadcast_to_custom_overlays + if (!custom_overlays_sent_broadcasts_.insert(block_id).second) { + return; + } + custom_overlays_sent_broadcasts_lru_.push(block_id); + if (custom_overlays_sent_broadcasts_lru_.size() > 256) { + custom_overlays_sent_broadcasts_.erase(custom_overlays_sent_broadcasts_lru_.front()); + custom_overlays_sent_broadcasts_lru_.pop(); + } + for (auto &private_overlay : custom_overlays_) { + for (auto &actor : private_overlay.second.actors_) { + auto local_id = actor.first; + if (private_overlay.second.params_.block_senders_.count(local_id)) { + td::actor::send_closure(actor.second, &FullNodeCustomOverlay::send_block_candidate, block_id, cc_seqno, + validator_set_hash, data.clone()); + } + } + } +} + FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, @@ -569,7 +663,6 @@ FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id , client_(client) , db_root_(db_root) , config_(config) { - add_shard(ShardIdFull{masterchainId}); } td::actor::ActorOwn FullNode::create(ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, @@ -598,6 +691,21 @@ bool FullNodeConfig::operator!=(const FullNodeConfig &rhs) const { return !(*this == rhs); } +CustomOverlayParams CustomOverlayParams::fetch(const ton_api::engine_validator_customOverlay& f) { + CustomOverlayParams c; + c.name_ = f.name_; + for (const auto &node : f.nodes_) { + c.nodes_.emplace_back(node->adnl_id_); + if (node->msg_sender_) { + c.msg_senders_[ton::adnl::AdnlNodeIdShort{node->adnl_id_}] = node->msg_sender_priority_; + } + if (node->block_sender_) { + c.block_senders_.emplace(node->adnl_id_); + } + } + return c; +} + } // namespace fullnode } // namespace validator diff --git a/validator/full-node.h b/validator/full-node.h index 82e0dd5c7..c3719f678 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -55,6 +55,15 @@ struct FullNodeConfig { bool ext_messages_broadcast_disabled_ = false; }; +struct CustomOverlayParams { + std::string name_; + std::vector nodes_; + std::map msg_senders_; + std::set block_senders_; + + static CustomOverlayParams fetch(const ton_api::engine_validator_customOverlay& f); +}; + class FullNode : public td::actor::Actor { public: virtual ~FullNode() = default; @@ -74,10 +83,12 @@ class FullNode : public td::actor::Actor { virtual void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) = 0; virtual void set_config(FullNodeConfig config) = 0; - virtual void add_ext_msg_overlay(std::vector nodes, - std::map senders, std::string name, - td::Promise promise) = 0; - virtual void del_ext_msg_overlay(std::string name, td::Promise promise) = 0; + virtual void add_custom_overlay(CustomOverlayParams params, td::Promise promise) = 0; + virtual void del_custom_overlay(std::string name, td::Promise promise) = 0; + + virtual void process_block_broadcast(BlockBroadcast broadcast) = 0; + virtual void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::BufferSlice data) = 0; static constexpr td::uint32 max_block_size() { return 4 << 20; diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 86664a3b6..3dfa17fdd 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -27,6 +27,7 @@ #include #include +#include namespace ton { @@ -53,9 +54,8 @@ class FullNodeImpl : public FullNode { void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; void set_config(FullNodeConfig config) override; - void add_ext_msg_overlay(std::vector nodes, std::map senders, - std::string name, td::Promise promise) override; - void del_ext_msg_overlay(std::string name, td::Promise promise) override; + void add_custom_overlay(CustomOverlayParams params, td::Promise promise) override; + void del_custom_overlay(std::string name, td::Promise promise) override; void add_shard(ShardIdFull shard); void del_shard(ShardIdFull shard); @@ -66,7 +66,9 @@ class FullNodeImpl : public FullNode { void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data); void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data); void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqnp, td::BufferSlice data); - void send_broadcast(BlockBroadcast broadcast); + void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data); + void send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only); void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); void download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); @@ -80,9 +82,13 @@ class FullNodeImpl : public FullNode { void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise); - void got_key_block_state(td::Ref state); + void got_key_block_config(td::Ref config); void new_key_block(BlockHandle handle); + void process_block_broadcast(BlockBroadcast broadcast) override; + void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override; + void start_up() override; FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, @@ -121,20 +127,22 @@ class FullNodeImpl : public FullNode { FullNodeConfig config_; std::map> private_block_overlays_; - bool private_block_overlays_enable_compression_ = false; + bool broadcast_block_candidates_in_public_overlay_ = false; - struct ExtMsgOverlayInfo { - std::vector nodes_; - std::map senders_; - std::map> - actors_; // our local id -> actor + struct CustomOverlayInfo { + CustomOverlayParams params_; + std::map> actors_; // our local id -> actor }; - std::map private_custom_overlays_; + std::map custom_overlays_; + std::set custom_overlays_sent_broadcasts_; + std::queue custom_overlays_sent_broadcasts_lru_; void update_private_overlays(); - void set_private_block_overlays_enable_compression(bool value); void create_private_block_overlay(PublicKeyHash key); - void update_ext_msg_overlay(const std::string& name, ExtMsgOverlayInfo& overlay); + void update_custom_overlay(CustomOverlayInfo& overlay); + void send_block_broadcast_to_custom_overlays(const BlockBroadcast& broadcast); + void send_block_candidate_broadcast_to_custom_overlays(const BlockIdExt& block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, const td::BufferSlice& data); }; } // namespace fullnode diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index cda1c7871..3da1167aa 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -899,11 +899,6 @@ void AcceptBlockQuery::written_block_info_2() { } void AcceptBlockQuery::applied() { - if (!send_broadcast_) { - finish_query(); - return; - } - BlockBroadcast b; b.data = data_->data(); b.block_id = id_; @@ -923,7 +918,8 @@ void AcceptBlockQuery::applied() { } // do not wait for answer - td::actor::send_closure_later(manager_, &ValidatorManager::send_block_broadcast, std::move(b)); + td::actor::send_closure_later(manager_, &ValidatorManager::send_block_broadcast, std::move(b), + /* custom_overlays_only = */ !send_broadcast_); finish_query(); } diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 055c1aed2..913a0ed87 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -44,7 +44,8 @@ class Collator final : public td::actor::Actor { return SUPPORTED_VERSION; } static constexpr long long supported_capabilities() { - return ton::capCreateStatsEnabled | ton::capBounceMsgBody | ton::capReportVersion | ton::capShortDequeue; + return ton::capCreateStatsEnabled | ton::capBounceMsgBody | ton::capReportVersion | ton::capShortDequeue | + ton::capStoreOutMsgQueueSize | ton::capMsgMetadata | ton::capDeferMessages; } using LtCellRef = block::LtCellRef; using NewOutMsg = block::NewOutMsg; @@ -70,6 +71,7 @@ class Collator final : public td::actor::Actor { std::vector> prev_states; std::vector> prev_block_data; Ed25519_PublicKey created_by_; + Ref collator_opts_; Ref validator_set_; td::actor::ActorId manager; td::Timestamp timeout; @@ -89,7 +91,8 @@ class Collator final : public td::actor::Actor { public: Collator(ShardIdFull shard, bool is_hardfork, td::uint32 min_ts, BlockIdExt min_masterchain_block_id, std::vector prev, Ref validator_set, Ed25519_PublicKey collator_id, - td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); + Ref collator_opts, td::actor::ActorId manager, td::Timestamp timeout, + td::Promise promise); ~Collator() override = default; bool is_busy() const { return busy_; @@ -192,7 +195,10 @@ class Collator final : public td::actor::Actor { std::priority_queue, std::greater> new_msgs; std::pair last_proc_int_msg_, first_unproc_int_msg_; std::unique_ptr in_msg_dict, out_msg_dict, out_msg_queue_, sibling_out_msg_queue_; - td::uint32 out_msg_queue_size_ = 0; + std::map unprocessed_deferred_messages_; // number of messages from dispatch queue in new_msgs + td::uint64 out_msg_queue_size_ = 0; + td::uint64 old_out_msg_queue_size_ = 0; + bool have_out_msg_queue_size_in_state_ = false; std::unique_ptr ihr_pending; std::shared_ptr processed_upto_, sibling_processed_upto_; std::unique_ptr block_create_stats_; @@ -203,6 +209,19 @@ class Collator final : public td::actor::Actor { std::vector> collated_roots_; std::unique_ptr block_candidate; + std::unique_ptr dispatch_queue_; + std::map sender_generated_messages_count_; + unsigned dispatch_queue_ops_{0}; + std::map last_dispatch_queue_emitted_lt_; + bool have_unprocessed_account_dispatch_queue_ = false; + bool dispatch_queue_total_limit_reached_ = false; + td::uint64 defer_out_queue_size_limit_; + td::uint64 hard_defer_out_queue_size_limit_; + + bool msg_metadata_enabled_ = false; + bool deferring_messages_enabled_ = false; + bool store_out_msg_queue_size_ = false; + td::PerfWarningTimer perf_timer_; // block::Account* lookup_account(td::ConstBitPtr addr) const; @@ -231,7 +250,7 @@ class Collator final : public td::actor::Actor { bool fix_one_processed_upto(block::MsgProcessedUpto& proc, const ton::ShardIdFull& owner); bool fix_processed_upto(block::MsgProcessedUptoCollection& upto); void got_neighbor_out_queue(int i, td::Result> res); - void got_out_queue_size(size_t i, td::Result res); + void got_out_queue_size(size_t i, td::Result res); bool adjust_shard_config(); bool store_shard_fees(ShardIdFull shard, const block::CurrencyCollection& fees, const block::CurrencyCollection& created); @@ -249,7 +268,8 @@ class Collator final : public td::actor::Actor { Ref& in_msg); bool create_ticktock_transactions(int mask); bool create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, ton::LogicalTime req_start_lt, int mask); - Ref create_ordinary_transaction(Ref msg_root, bool is_special_tx = false); + Ref create_ordinary_transaction(Ref msg_root, td::optional msg_metadata, + LogicalTime after_lt, bool is_special_tx = false); bool check_cur_validator_set(); bool unpack_last_mc_state(); bool unpack_last_state(); @@ -278,7 +298,7 @@ class Collator final : public td::actor::Actor { int priority); // td::Result register_external_message(td::Slice ext_msg_boc); void register_new_msg(block::NewOutMsg msg); - void register_new_msgs(block::transaction::Transaction& trans); + void register_new_msgs(block::transaction::Transaction& trans, td::optional msg_metadata); bool process_new_messages(bool enqueue_only = false); int process_one_new_message(block::NewOutMsg msg, bool enqueue_only = false, Ref* is_special = nullptr); bool process_inbound_internal_messages(); @@ -286,10 +306,15 @@ class Collator final : public td::actor::Actor { const block::McShardDescr& src_nb); bool process_inbound_external_messages(); int process_external_message(Ref msg); - bool enqueue_message(block::NewOutMsg msg, td::RefInt256 fwd_fees_remaining, ton::LogicalTime enqueued_lt); + bool process_dispatch_queue(); + bool process_deferred_message(Ref enq_msg, StdSmcAddress src_addr, LogicalTime lt, + td::optional& msg_metadata); + bool enqueue_message(block::NewOutMsg msg, td::RefInt256 fwd_fees_remaining, StdSmcAddress src_addr, + bool defer = false); bool enqueue_transit_message(Ref msg, Ref old_msg_env, ton::AccountIdPrefixFull prev_prefix, ton::AccountIdPrefixFull cur_prefix, ton::AccountIdPrefixFull dest_prefix, - td::RefInt256 fwd_fee_remaining); + td::RefInt256 fwd_fee_remaining, td::optional msg_metadata, + td::optional emitted_lt = {}); bool delete_out_msg_queue_msg(td::ConstBitPtr key); bool insert_in_msg(Ref in_msg); bool insert_out_msg(Ref out_msg); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index fd4ddd341..c6dd7caf2 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -1,7 +1,7 @@ /* - This file is part of TON Blockchain Library. + This file is part of TON Blockchain Library. - TON Blockchain Library is free software: you can redistribute it and/or modify + TON Blockchain Library is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation, either version 2 of the License, or (at your option) any later version. @@ -71,20 +71,22 @@ static inline bool dbg(int c) { * @param prev A vector of BlockIdExt representing the previous blocks. * @param validator_set A reference to the ValidatorSet. * @param collator_id The public key of the block creator. + * @param collator_opts A reference to CollatorOptions. * @param manager The ActorId of the ValidatorManager. * @param timeout The timeout for the collator. * @param promise The promise to return the result. */ Collator::Collator(ShardIdFull shard, bool is_hardfork, UnixTime min_ts, BlockIdExt min_masterchain_block_id, - std::vector prev, td::Ref validator_set, Ed25519_PublicKey collator_id, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise) + std::vector prev, Ref validator_set, Ed25519_PublicKey collator_id, + Ref collator_opts, td::actor::ActorId manager, + td::Timestamp timeout, td::Promise promise) : shard_(shard) , is_hardfork_(is_hardfork) , min_ts(min_ts) , min_mc_block_id{min_masterchain_block_id} , prev_blocks(std::move(prev)) , created_by_(collator_id) + , collator_opts_(collator_opts) , validator_set_(std::move(validator_set)) , manager(manager) , timeout(timeout) @@ -551,7 +553,7 @@ bool Collator::preprocess_prev_mc_state() { /** * Callback function called after retrieving the Masterchain state. * - * @param res The retreived masterchain state. + * @param res The retrieved masterchain state. */ void Collator::after_get_mc_state(td::Result, BlockIdExt>> res) { LOG(WARNING) << "in Collator::after_get_mc_state()"; @@ -619,7 +621,7 @@ void Collator::after_get_shard_state(int idx, td::Result> res) { * Callback function called after retrieving block data for a previous block. * * @param idx The index of the previous block (0 or 1). - * @param res The retreived block data. + * @param res The retreved block data. */ void Collator::after_get_block_data(int idx, td::Result> res) { LOG(DEBUG) << "in Collator::after_get_block_data(" << idx << ")"; @@ -694,6 +696,9 @@ bool Collator::unpack_last_mc_state() { create_stats_enabled_ = config_->create_stats_enabled(); report_version_ = config_->has_capability(ton::capReportVersion); short_dequeue_records_ = config_->has_capability(ton::capShortDequeue); + store_out_msg_queue_size_ = config_->has_capability(ton::capStoreOutMsgQueueSize); + msg_metadata_enabled_ = config_->has_capability(ton::capMsgMetadata); + deferring_messages_enabled_ = config_->has_capability(ton::capDeferMessages); shard_conf_ = std::make_unique(*config_); prev_key_block_exists_ = config_->get_last_key_block(prev_key_block_, prev_key_block_lt_); if (prev_key_block_exists_) { @@ -794,19 +799,20 @@ bool Collator::request_neighbor_msg_queues() { } /** - * Requests the size of the outbound message queue from the previous state(s). + * Requests the size of the outbound message queue from the previous state(s) if needed. * * @returns True if the request was successful, false otherwise. */ bool Collator::request_out_msg_queue_size() { - if (after_split_) { - // If block is after split, the size is calculated during split (see Collator::split_last_state) + if (have_out_msg_queue_size_in_state_) { + // if after_split then have_out_msg_queue_size_in_state_ is always true, since the size is calculated during split return true; } + out_msg_queue_size_ = 0; for (size_t i = 0; i < prev_blocks.size(); ++i) { ++pending; send_closure_later(manager, &ValidatorManager::get_out_msg_queue_size, prev_blocks[i], - [self = get_self(), i](td::Result res) { + [self = get_self(), i](td::Result res) { td::actor::send_closure(std::move(self), &Collator::got_out_queue_size, i, std::move(res)); }); } @@ -885,14 +891,14 @@ void Collator::got_neighbor_out_queue(int i, td::Result> res) * @param i The index of the previous block (0 or 1). * @param res The result object containing the size of the queue. */ -void Collator::got_out_queue_size(size_t i, td::Result res) { +void Collator::got_out_queue_size(size_t i, td::Result res) { --pending; if (res.is_error()) { fatal_error( res.move_as_error_prefix(PSTRING() << "failed to get message queue size from prev block #" << i << ": ")); return; } - td::uint32 size = res.move_as_ok(); + td::uint64 size = res.move_as_ok(); LOG(WARNING) << "got outbound queue size from prev block #" << i << ": " << size; out_msg_queue_size_ += size; check_pending(); @@ -1016,7 +1022,7 @@ bool Collator::split_last_state(block::ShardState& ss) { return fatal_error(res2.move_as_error()); } sibling_processed_upto_ = res2.move_as_ok(); - auto res3 = ss.split(shard_, &out_msg_queue_size_); + auto res3 = ss.split(shard_); if (res3.is_error()) { return fatal_error(std::move(res3)); } @@ -1052,7 +1058,12 @@ bool Collator::import_shard_state_data(block::ShardState& ss) { out_msg_queue_ = std::move(ss.out_msg_queue_); processed_upto_ = std::move(ss.processed_upto_); ihr_pending = std::move(ss.ihr_pending_); + dispatch_queue_ = std::move(ss.dispatch_queue_); block_create_stats_ = std::move(ss.block_create_stats_); + if (ss.out_msg_queue_size_) { + have_out_msg_queue_size_in_state_ = true; + out_msg_queue_size_ = ss.out_msg_queue_size_.value(); + } return true; } @@ -1776,6 +1787,7 @@ bool Collator::try_collate() { last_proc_int_msg_.second.set_zero(); first_unproc_int_msg_.first = ~0ULL; first_unproc_int_msg_.second.set_ones(); + old_out_msg_queue_size_ = out_msg_queue_size_; if (is_masterchain()) { LOG(DEBUG) << "getting the list of special smart contracts"; auto res = config_->get_special_smartcontracts(); @@ -1960,6 +1972,10 @@ bool Collator::fetch_config_params() { return fatal_error(res.move_as_error()); } compute_phase_cfg_.libraries = std::make_unique(config_->get_libraries_root(), 256); + defer_out_queue_size_limit_ = std::max(collator_opts_->defer_out_queue_size_limit, + compute_phase_cfg_.size_limits.defer_out_queue_size_limit); + // This one is checked in validate-query + hard_defer_out_queue_size_limit_ = compute_phase_cfg_.size_limits.defer_out_queue_size_limit; return true; } @@ -2090,6 +2106,11 @@ bool Collator::do_collate() { if (!init_value_create()) { return fatal_error("cannot compute the value to be created / minted / recovered"); } + // 2-. take messages from dispatch queue + LOG(INFO) << "process dispatch queue"; + if (!process_dispatch_queue()) { + return fatal_error("cannot process dispatch queue"); + } // 2. tick transactions LOG(INFO) << "create tick transactions"; if (!create_ticktock_transactions(2)) { @@ -2597,7 +2618,7 @@ bool Collator::create_special_transaction(block::CurrencyCollection amount, Ref< } CHECK(block::gen::t_Message_Any.validate_ref(msg)); CHECK(block::tlb::t_Message.validate_ref(msg)); - if (process_one_new_message(block::NewOutMsg{lt, msg, Ref{}}, false, &in_msg) != 1) { + if (process_one_new_message(block::NewOutMsg{lt, msg, Ref{}, 0}, false, &in_msg) != 1) { return fatal_error("cannot generate special transaction for recovering "s + amount.to_str() + " to account " + addr.to_hex()); } @@ -2622,7 +2643,7 @@ bool Collator::create_special_transactions() { * * @param smc_addr The address of the smart contract. * @param req_start_lt The requested start logical time for the transaction. - * @param mask The value indicating wheter the thansaction is tick (mask == 2) or tock (mask == 1). + * @param mask The value indicating whether the thansaction is tick (mask == 2) or tock (mask == 1). * * @returns True if the transaction was created successfully, false otherwise. */ @@ -2639,13 +2660,18 @@ bool Collator::create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, t return true; } req_start_lt = std::max(req_start_lt, start_lt + 1); + auto it = last_dispatch_queue_emitted_lt_.find(acc->addr); + if (it != last_dispatch_queue_emitted_lt_.end()) { + req_start_lt = std::max(req_start_lt, it->second + 1); + } if (acc->last_trans_end_lt_ >= start_lt && acc->transactions.empty()) { return fatal_error(td::Status::Error(-666, PSTRING() << "last transaction time in the state of account " << workchain() << ":" << smc_addr.to_hex() << " is too large")); } std::unique_ptr trans = std::make_unique( - *acc, mask == 2 ? block::transaction::Transaction::tr_tick : block::transaction::Transaction::tr_tock, req_start_lt, now_); + *acc, mask == 2 ? block::transaction::Transaction::tr_tick : block::transaction::Transaction::tr_tock, + req_start_lt, now_); if (!trans->prepare_storage_phase(storage_phase_cfg_, true)) { return fatal_error(td::Status::Error( -666, std::string{"cannot create storage phase of a new transaction for smart contract "} + smc_addr.to_hex())); @@ -2675,7 +2701,8 @@ bool Collator::create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, t td::Status::Error(-666, std::string{"cannot commit new transaction for smart contract "} + smc_addr.to_hex())); } update_max_lt(acc->last_trans_end_lt_); - register_new_msgs(*trans); + block::MsgMetadata new_msg_metadata{0, acc->workchain, acc->addr, trans->start_lt}; + register_new_msgs(*trans, std::move(new_msg_metadata)); return true; } @@ -2683,11 +2710,15 @@ bool Collator::create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, t * Creates an ordinary transaction using a given message. * * @param msg_root The root of the message to be processed serialized using Message TLB-scheme. + * @param msg_metadata Metadata of the inbound message. + * @param after_lt Transaction lt will be grater than after_lt. Used for deferred messages. * @param is_special_tx True if creating a special transaction (mint/recover), false otherwise. * * @returns The root of the serialized transaction, or an empty reference if the transaction creation fails. */ -Ref Collator::create_ordinary_transaction(Ref msg_root, bool is_special_tx) { +Ref Collator::create_ordinary_transaction(Ref msg_root, + td::optional msg_metadata, LogicalTime after_lt, + bool is_special_tx) { ton::StdSmcAddress addr; auto cs = vm::load_cell_slice(msg_root); bool external; @@ -2731,8 +2762,15 @@ Ref Collator::create_ordinary_transaction(Ref msg_root, bool block::Account* acc = acc_res.move_as_ok(); assert(acc); + if (external) { + after_lt = std::max(after_lt, last_proc_int_msg_.first); + } + auto it = last_dispatch_queue_emitted_lt_.find(acc->addr); + if (it != last_dispatch_queue_emitted_lt_.end()) { + after_lt = std::max(after_lt, it->second); + } auto res = impl_create_ordinary_transaction(msg_root, acc, now_, start_lt, &storage_phase_cfg_, &compute_phase_cfg_, - &action_phase_cfg_, external, last_proc_int_msg_.first); + &action_phase_cfg_, external, after_lt); if (res.is_error()) { auto error = res.move_as_error(); if (error.code() == -701) { @@ -2756,7 +2794,14 @@ Ref Collator::create_ordinary_transaction(Ref msg_root, bool return {}; } - register_new_msgs(*trans); + td::optional new_msg_metadata; + if (external || is_special_tx) { + new_msg_metadata = block::MsgMetadata{0, acc->workchain, acc->addr, trans->start_lt}; + } else if (msg_metadata) { + new_msg_metadata = std::move(msg_metadata); + ++new_msg_metadata.value().depth; + } + register_new_msgs(*trans, std::move(new_msg_metadata)); update_max_lt(acc->last_trans_end_lt_); value_flow_.burned += trans->blackhole_burned; return trans_root; @@ -2791,13 +2836,12 @@ td::Result> Collator::impl_crea << ":" << acc->addr.to_hex() << " is too large"); } auto trans_min_lt = lt; - if (external) { - // transactions processing external messages must have lt larger than all processed internal messages - trans_min_lt = std::max(trans_min_lt, after_lt); - } + // transactions processing external messages must have lt larger than all processed internal messages + // if account has deferred message processed in this block, the next transaction should have lt > emitted_lt + trans_min_lt = std::max(trans_min_lt, after_lt); - std::unique_ptr trans = - std::make_unique(*acc, block::transaction::Transaction::tr_ord, trans_min_lt + 1, utime, msg_root); + std::unique_ptr trans = std::make_unique( + *acc, block::transaction::Transaction::tr_ord, trans_min_lt + 1, utime, msg_root); bool ihr_delivered = false; // FIXME if (!trans->unpack_input_msg(ihr_delivered, action_phase_cfg)) { if (external) { @@ -2896,7 +2940,7 @@ bool Collator::update_last_proc_int_msg(const std::pair* is_special) { + bool from_dispatch_queue = msg.msg_env_from_dispatch_queue.not_null(); Ref src, dest; bool enqueue, external; auto cs = load_cell_slice(msg.msg); @@ -2972,7 +3017,7 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R if (!tlb::unpack(cs, info)) { return -1; } - CHECK(info.created_lt == msg.lt && info.created_at == now_); + CHECK(info.created_lt == msg.lt && info.created_at == now_ && !from_dispatch_queue); src = std::move(info.src); enqueue = external = true; break; @@ -2982,7 +3027,7 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R if (!tlb::unpack(cs, info)) { return -1; } - CHECK(info.created_lt == msg.lt && info.created_at == now_); + CHECK(from_dispatch_queue || (info.created_lt == msg.lt && info.created_at == now_)); src = std::move(info.src); dest = std::move(info.dest); fwd_fees = block::tlb::t_Grams.as_integer(info.fwd_fee); @@ -2994,7 +3039,7 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R default: return -1; } - CHECK(is_our_address(std::move(src))); + CHECK(is_our_address(src)); if (external) { // 1. construct a msg_export_ext OutMsg vm::CellBuilder cb; @@ -3006,9 +3051,46 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R // (if ever a structure in the block for listing all external outbound messages appears, insert this message there as well) return 0; } - if (enqueue) { - auto lt = msg.lt; - bool ok = enqueue_message(std::move(msg), std::move(fwd_fees), lt); + + WorkchainId src_wc; + StdSmcAddress src_addr; + CHECK(block::tlb::t_MsgAddressInt.extract_std_address(src, src_wc, src_addr)); + CHECK(src_wc == workchain()); + bool is_special_account = is_masterchain() && config_->is_special_smartcontract(src_addr); + bool defer = false; + if (!from_dispatch_queue) { + if (deferring_messages_enabled_ && collator_opts_->deferring_enabled && !is_special && !is_special_account && + msg.msg_idx != 0) { + if (++sender_generated_messages_count_[src_addr] >= collator_opts_->defer_messages_after || + out_msg_queue_size_ > defer_out_queue_size_limit_) { + defer = true; + } + } + if (dispatch_queue_->lookup(src_addr).not_null() || unprocessed_deferred_messages_.count(src_addr)) { + defer = true; + } + } else { + auto &x = unprocessed_deferred_messages_[src_addr]; + CHECK(x > 0); + if (--x == 0) { + unprocessed_deferred_messages_.erase(src_addr); + } + } + + if (enqueue || defer) { + bool ok; + if (from_dispatch_queue) { + auto msg_env = msg.msg_env_from_dispatch_queue; + block::tlb::MsgEnvelope::Record_std env; + CHECK(block::tlb::unpack_cell(msg_env, env)); + auto src_prefix = block::tlb::MsgAddressInt::get_prefix(src); + auto dest_prefix = block::tlb::MsgAddressInt::get_prefix(dest); + CHECK(env.emitted_lt && env.emitted_lt.value() == msg.lt); + ok = enqueue_transit_message(std::move(msg.msg), std::move(msg_env), src_prefix, src_prefix, dest_prefix, + std::move(env.fwd_fee_remaining), std::move(env.metadata), msg.lt); + } else { + ok = enqueue_message(std::move(msg), std::move(fwd_fees), src_addr, defer); + } return ok ? 0 : -1; } // process message by a transaction in this block: @@ -3019,26 +3101,36 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R return -1; } // 1. create a Transaction processing this Message - auto trans_root = create_ordinary_transaction(msg.msg, is_special != nullptr); + auto trans_root = create_ordinary_transaction(msg.msg, msg.metadata, msg.lt, is_special != nullptr); if (trans_root.is_null()) { fatal_error("cannot create transaction for re-processing output message"); return -1; } // 2. create a MsgEnvelope enveloping this Message - vm::CellBuilder cb; - CHECK(cb.store_long_bool(0x46060, 20) // msg_envelope#4 cur_addr:.. next_addr:.. - && block::tlb::t_Grams.store_integer_ref(cb, fwd_fees) // fwd_fee_remaining:t_Grams - && cb.store_ref_bool(msg.msg)); // msg:^(Message Any) - Ref msg_env = cb.finalize(); + block::tlb::MsgEnvelope::Record_std msg_env_rec{0x60, 0x60, fwd_fees, msg.msg, {}, msg.metadata}; + Ref msg_env; + CHECK(block::tlb::pack_cell(msg_env, msg_env_rec)); if (verbosity > 2) { std::cerr << "new (processed outbound) message envelope: "; block::gen::t_MsgEnvelope.print_ref(std::cerr, msg_env); } // 3. create InMsg, referring to this MsgEnvelope and this Transaction - CHECK(cb.store_long_bool(3, 3) // msg_import_imm$011 - && cb.store_ref_bool(msg_env) // in_msg:^MsgEnvelope - && cb.store_ref_bool(trans_root) // transaction:^Transaction - && block::tlb::t_Grams.store_integer_ref(cb, fwd_fees)); // fwd_fee:Grams + vm::CellBuilder cb; + if (from_dispatch_queue) { + auto msg_env = msg.msg_env_from_dispatch_queue; + block::tlb::MsgEnvelope::Record_std env; + CHECK(block::tlb::unpack_cell(msg_env, env)); + CHECK(env.emitted_lt && env.emitted_lt.value() == msg.lt); + CHECK(cb.store_long_bool(0b00100, 5) // msg_import_deferred_fin$00100 + && cb.store_ref_bool(msg_env) // in_msg:^MsgEnvelope + && cb.store_ref_bool(trans_root) // transaction:^Transaction + && block::tlb::t_Grams.store_integer_ref(cb, env.fwd_fee_remaining)); // fwd_fee:Grams + } else { + CHECK(cb.store_long_bool(3, 3) // msg_import_imm$011 + && cb.store_ref_bool(msg_env) // in_msg:^MsgEnvelope + && cb.store_ref_bool(trans_root) // transaction:^Transaction + && block::tlb::t_Grams.store_integer_ref(cb, fwd_fees)); // fwd_fee:Grams + } // 4. insert InMsg into InMsgDescr Ref in_msg = cb.finalize(); if (!insert_in_msg(in_msg)) { @@ -3049,14 +3141,16 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R *is_special = in_msg; return 1; } - // 5. create OutMsg, referring to this MsgEnvelope and InMsg - CHECK(cb.store_long_bool(2, 3) // msg_export_imm$010 - && cb.store_ref_bool(msg_env) // out_msg:^MsgEnvelope - && cb.store_ref_bool(msg.trans) // transaction:^Transaction - && cb.store_ref_bool(in_msg)); // reimport:^InMsg - // 6. insert OutMsg into OutMsgDescr - if (!insert_out_msg(cb.finalize())) { - return -1; + if (!from_dispatch_queue) { + // 5. create OutMsg, referring to this MsgEnvelope and InMsg + CHECK(cb.store_long_bool(2, 3) // msg_export_imm$010 + && cb.store_ref_bool(msg_env) // out_msg:^MsgEnvelope + && cb.store_ref_bool(msg.trans) // transaction:^Transaction + && cb.store_ref_bool(in_msg)); // reimport:^InMsg + // 6. insert OutMsg into OutMsgDescr + if (!insert_out_msg(cb.finalize())) { + return -1; + } } // 7. check whether the block is full now if (!block_limit_status_->fits(block::ParamLimits::cl_normal)) { @@ -3081,41 +3175,61 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R * @param cur_prefix The account ID prefix for the next hop. * @param dest_prefix The prefix of the destination account ID. * @param fwd_fee_remaining The remaining forward fee. + * @param msg_metadata Metadata of the message. + * @param emitted_lt If present - the message was taken from DispatchQueue, and msg_env will have this emitted_lt. * * @returns True if the transit message is successfully enqueued, false otherwise. */ bool Collator::enqueue_transit_message(Ref msg, Ref old_msg_env, ton::AccountIdPrefixFull prev_prefix, ton::AccountIdPrefixFull cur_prefix, - ton::AccountIdPrefixFull dest_prefix, td::RefInt256 fwd_fee_remaining) { - LOG(DEBUG) << "enqueueing transit message " << msg->get_hash().bits().to_hex(256); - bool requeue = is_our_address(prev_prefix); + ton::AccountIdPrefixFull dest_prefix, td::RefInt256 fwd_fee_remaining, + td::optional msg_metadata, + td::optional emitted_lt) { + bool from_dispatch_queue = (bool)emitted_lt; + if (from_dispatch_queue) { + LOG(DEBUG) << "enqueueing message from dispatch queue " << msg->get_hash().bits().to_hex(256) + << ", emitted_lt=" << emitted_lt.value(); + } else { + LOG(DEBUG) << "enqueueing transit message " << msg->get_hash().bits().to_hex(256); + } + bool requeue = !from_dispatch_queue && is_our_address(prev_prefix) && !from_dispatch_queue; // 1. perform hypercube routing auto route_info = block::perform_hypercube_routing(cur_prefix, dest_prefix, shard_); if ((unsigned)route_info.first > 96 || (unsigned)route_info.second > 96) { return fatal_error("cannot perform hypercube routing for a transit message"); } // 2. compute our part of transit fees - td::RefInt256 transit_fee = action_phase_cfg_.fwd_std.get_next_part(fwd_fee_remaining); + td::RefInt256 transit_fee = + from_dispatch_queue ? td::zero_refint() : action_phase_cfg_.fwd_std.get_next_part(fwd_fee_remaining); fwd_fee_remaining -= transit_fee; CHECK(td::sgn(transit_fee) >= 0 && td::sgn(fwd_fee_remaining) >= 0); // 3. create a new MsgEnvelope - vm::CellBuilder cb; - CHECK(cb.store_long_bool(4, 4) // msg_envelope#4 cur_addr:.. next_addr:.. - && cb.store_long_bool(route_info.first, 8) // cur_addr:IntermediateAddress - && cb.store_long_bool(route_info.second, 8) // next_addr:IntermediateAddress - && block::tlb::t_Grams.store_integer_ref(cb, fwd_fee_remaining) // fwd_fee_remaining:t_Grams - && cb.store_ref_bool(msg)); // msg:^(Message Any) - Ref msg_env = cb.finalize(); + block::tlb::MsgEnvelope::Record_std msg_env_rec{route_info.first, route_info.second, fwd_fee_remaining, msg, + emitted_lt, std::move(msg_metadata)}; + Ref msg_env; + CHECK(block::tlb::t_MsgEnvelope.pack_cell(msg_env, msg_env_rec)); // 4. create InMsg - CHECK(cb.store_long_bool(5, 3) // msg_import_tr$101 - && cb.store_ref_bool(old_msg_env) // in_msg:^MsgEnvelope - && cb.store_ref_bool(msg_env) // out_msg:^MsgEnvelope - && block::tlb::t_Grams.store_integer_ref(cb, transit_fee)); // transit_fee:Grams + vm::CellBuilder cb; + if (from_dispatch_queue) { + CHECK(cb.store_long_bool(0b00101, 5) // msg_import_deferred_tr$00101 + && cb.store_ref_bool(old_msg_env) // in_msg:^MsgEnvelope + && cb.store_ref_bool(msg_env)); // out_msg:^MsgEnvelope + } else { + CHECK(cb.store_long_bool(5, 3) // msg_import_tr$101 + && cb.store_ref_bool(old_msg_env) // in_msg:^MsgEnvelope + && cb.store_ref_bool(msg_env) // out_msg:^MsgEnvelope + && block::tlb::t_Grams.store_integer_ref(cb, transit_fee)); // transit_fee:Grams + } Ref in_msg = cb.finalize(); // 5. create a new OutMsg - CHECK(cb.store_long_bool(requeue ? 7 : 3, 3) // msg_export_tr$011 or msg_export_tr_req$111 - && cb.store_ref_bool(msg_env) // out_msg:^MsgEnvelope - && cb.store_ref_bool(in_msg)); // imported:^InMsg + // msg_export_tr$011 / msg_export_tr_req$111 / msg_export_deferred_tr$10101 + if (from_dispatch_queue) { + CHECK(cb.store_long_bool(0b10101, 5)); + } else { + CHECK(cb.store_long_bool(requeue ? 7 : 3, 3)); + } + CHECK(cb.store_ref_bool(msg_env) // out_msg:^MsgEnvelope + && cb.store_ref_bool(in_msg)); // imported:^InMsg Ref out_msg = cb.finalize(); // 4.1. insert OutMsg into OutMsgDescr if (verbosity > 2) { @@ -3134,8 +3248,8 @@ bool Collator::enqueue_transit_message(Ref msg, Ref old_msg_ return fatal_error("cannot insert a new InMsg into InMsgDescr"); } // 5. create EnqueuedMsg - CHECK(cb.store_long_bool(start_lt) // _ enqueued_lt:uint64 - && cb.store_ref_bool(msg_env)); // out_msg:^MsgEnvelope = EnqueuedMsg; + CHECK(cb.store_long_bool(from_dispatch_queue ? emitted_lt.value() : start_lt) // _ enqueued_lt:uint64 + && cb.store_ref_bool(msg_env)); // out_msg:^MsgEnvelope = EnqueuedMsg; // 6. insert EnqueuedMsg into OutMsgQueue // NB: we use here cur_prefix instead of src_prefix; should we check that route_info.first >= next_addr.use_dest_bits of the old envelope? auto next_hop = block::interpolate_addr(cur_prefix, dest_prefix, route_info.second); @@ -3237,9 +3351,14 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT LOG(ERROR) << "cannot unpack CommonMsgInfo of an inbound internal message"; return false; } - if (info.created_lt != lt) { + if (!env.emitted_lt && info.created_lt != lt) { LOG(ERROR) << "inbound internal message has an augmentation value in source OutMsgQueue distinct from the one in " - "its contents"; + "its contents (CommonMsgInfo)"; + return false; + } + if (env.emitted_lt && env.emitted_lt.value() != lt) { + LOG(ERROR) << "inbound internal message has an augmentation value in source OutMsgQueue distinct from the one in " + "its contents (deferred_it in MsgEnvelope)"; return false; } if (!block::tlb::validate_message_libs(env.msg)) { @@ -3302,7 +3421,8 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT bool our = ton::shard_contains(shard_, cur_prefix); bool to_us = ton::shard_contains(shard_, dest_prefix); - block::EnqueuedMsgDescr enq_msg_descr{cur_prefix, next_prefix, info.created_lt, enqueued_lt, + block::EnqueuedMsgDescr enq_msg_descr{cur_prefix, next_prefix, + env.emitted_lt ? env.emitted_lt.value() : info.created_lt, enqueued_lt, env.msg->get_hash().bits()}; if (processed_upto_->already_processed(enq_msg_descr)) { LOG(DEBUG) << "inbound internal message with lt=" << enq_msg_descr.lt_ << " hash=" << enq_msg_descr.hash_.to_hex() @@ -3319,7 +3439,7 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT // destination is outside our shard, relay transit message // (very similar to enqueue_message()) if (!enqueue_transit_message(std::move(env.msg), std::move(msg_env), cur_prefix, next_prefix, dest_prefix, - std::move(env.fwd_fee_remaining))) { + std::move(env.fwd_fee_remaining), std::move(env.metadata))) { return fatal_error("cannot enqueue transit internal message with key "s + key.to_hex(352)); } return !our || delete_out_msg_queue_msg(key); @@ -3328,7 +3448,7 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT // process the message by an ordinary transaction similarly to process_one_new_message() // // 8. create a Transaction processing this Message - auto trans_root = create_ordinary_transaction(env.msg); + auto trans_root = create_ordinary_transaction(env.msg, env.metadata, 0); if (trans_root.is_null()) { return fatal_error("cannot create transaction for processing inbound message"); } @@ -3368,6 +3488,9 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT * @returns True if the processing was successful, false otherwise. */ bool Collator::process_inbound_internal_messages() { + if (have_unprocessed_account_dispatch_queue_) { + return true; + } while (!block_full_ && !nb_out_msgs_->is_eof()) { block_full_ = !block_limit_status_->fits(block::ParamLimits::cl_normal); if (block_full_) { @@ -3476,7 +3599,7 @@ int Collator::process_external_message(Ref msg) { } // process message by a transaction in this block: // 1. create a Transaction processing this Message - auto trans_root = create_ordinary_transaction(msg); + auto trans_root = create_ordinary_transaction(msg, /* metadata = */ {}, 0); if (trans_root.is_null()) { if (busy_) { // transaction rejected by account @@ -3500,6 +3623,230 @@ int Collator::process_external_message(Ref msg) { return 1; } +/** + * Processes messages from dispatch queue + * + * Messages from dispatch queue are taken in three steps: + * 1. Take one message from each account (in the order of lt) + * 2. Take up to 10 per account (including from p.1), up to 20 per initiator, up to 150 in total + * 3. Take up to X messages per initiator, up to 150 in total. X depends on out msg queue size + * + * @returns True if the processing was successful, false otherwise. + */ +bool Collator::process_dispatch_queue() { + if (out_msg_queue_size_ > defer_out_queue_size_limit_ && old_out_msg_queue_size_ > hard_defer_out_queue_size_limit_) { + return true; + } + have_unprocessed_account_dispatch_queue_ = true; + size_t max_total_count[3] = {1 << 30, collator_opts_->dispatch_phase_2_max_total, + collator_opts_->dispatch_phase_3_max_total}; + size_t max_per_initiator[3] = {1 << 30, collator_opts_->dispatch_phase_2_max_per_initiator, 0}; + if (collator_opts_->dispatch_phase_3_max_per_initiator) { + max_per_initiator[2] = collator_opts_->dispatch_phase_3_max_per_initiator.value(); + } else if (out_msg_queue_size_ <= 256) { + max_per_initiator[2] = 10; + } else if (out_msg_queue_size_ <= 512) { + max_per_initiator[2] = 2; + } else if (out_msg_queue_size_ <= 1500) { + max_per_initiator[2] = 1; + } + for (int iter = 0; iter < 3; ++iter) { + if (max_per_initiator[iter] == 0 || max_total_count[iter] == 0) { + continue; + } + vm::AugmentedDictionary cur_dispatch_queue{dispatch_queue_->get_root(), 256, block::tlb::aug_DispatchQueue}; + std::map, size_t> count_per_initiator; + size_t total_count = 0; + while (!cur_dispatch_queue.is_empty()) { + block_full_ = !block_limit_status_->fits(block::ParamLimits::cl_normal); + if (block_full_) { + LOG(INFO) << "BLOCK FULL, stop processing dispatch queue"; + return true; + } + if (soft_timeout_.is_in_past(td::Timestamp::now())) { + block_full_ = true; + LOG(WARNING) << "soft timeout reached, stop processing dispatch queue"; + return true; + } + StdSmcAddress src_addr; + auto account_dispatch_queue = block::get_dispatch_queue_min_lt_account(cur_dispatch_queue, src_addr); + if (account_dispatch_queue.is_null()) { + return fatal_error("invalid dispatch queue in shard state"); + } + vm::Dictionary dict{64}; + td::uint64 dict_size; + if (!block::unpack_account_dispatch_queue(account_dispatch_queue, dict, dict_size)) { + return fatal_error(PSTRING() << "invalid account dispatch queue for account " << src_addr.to_hex()); + } + td::BitArray<64> key; + Ref enqueued_msg = dict.extract_minmax_key(key.bits(), 64, false, false); + LogicalTime lt = key.to_ulong(); + + td::optional msg_metadata; + if (!process_deferred_message(std::move(enqueued_msg), src_addr, lt, msg_metadata)) { + return fatal_error(PSTRING() << "error processing internal message from dispatch queue: account=" + << src_addr.to_hex() << ", lt=" << lt); + } + + // Remove message from DispatchQueue + bool ok; + if (iter == 0 || + (iter == 1 && sender_generated_messages_count_[src_addr] >= collator_opts_->defer_messages_after)) { + ok = cur_dispatch_queue.lookup_delete(src_addr).not_null(); + } else { + dict.lookup_delete(key); + --dict_size; + account_dispatch_queue = block::pack_account_dispatch_queue(dict, dict_size); + ok = account_dispatch_queue.not_null() ? cur_dispatch_queue.set(src_addr, account_dispatch_queue) + : cur_dispatch_queue.lookup_delete(src_addr).not_null(); + } + if (!ok) { + return fatal_error(PSTRING() << "error processing internal message from dispatch queue: account=" + << src_addr.to_hex() << ", lt=" << lt); + } + if (msg_metadata) { + auto initiator = std::make_tuple(msg_metadata.value().initiator_wc, msg_metadata.value().initiator_addr, + msg_metadata.value().initiator_lt); + size_t initiator_count = ++count_per_initiator[initiator]; + if (initiator_count >= max_per_initiator[iter]) { + cur_dispatch_queue.lookup_delete(src_addr); + } + } + ++total_count; + if (total_count >= max_total_count[iter]) { + dispatch_queue_total_limit_reached_ = true; + break; + } + } + if (iter == 0) { + have_unprocessed_account_dispatch_queue_ = false; + } + } + return true; +} + +/** + * Processes an internal message from DispatchQueue. + * The message may create a transaction or be enqueued. + * + * Similar to Collator::process_inbound_message. + * + * @param enq_msg The internal message serialized using EnqueuedMsg TLB-scheme. + * @param src_addr 256-bit address of the sender. + * @param lt The logical time of the message. + * @param msg_metadata Reference to store msg_metadata + * + * @returns True if the message was processed successfully, false otherwise. + */ +bool Collator::process_deferred_message(Ref enq_msg, StdSmcAddress src_addr, LogicalTime lt, + td::optional& msg_metadata) { + if (!block::remove_dispatch_queue_entry(*dispatch_queue_, src_addr, lt)) { + return fatal_error(PSTRING() << "failed to delete message from DispatchQueue: address=" << src_addr.to_hex() + << ", lt=" << lt); + } + ++dispatch_queue_ops_; + if (!(dispatch_queue_ops_ & 63)) { + if (!block_limit_status_->add_proof(dispatch_queue_->get_root_cell())) { + return false; + } + } + ++sender_generated_messages_count_[src_addr]; + + LogicalTime enqueued_lt = 0; + if (enq_msg.is_null() || enq_msg->size_ext() != 0x10040 || (enqueued_lt = enq_msg->prefetch_ulong(64)) != lt) { + if (enq_msg.not_null()) { + block::gen::t_EnqueuedMsg.print(std::cerr, *enq_msg); + } + LOG(ERROR) << "internal message in DispatchQueue is not a valid EnqueuedMsg (created lt " << lt << ", enqueued " + << enqueued_lt << ")"; + return false; + } + auto msg_env = enq_msg->prefetch_ref(); + CHECK(msg_env.not_null()); + // 0. check MsgEnvelope + if (msg_env->get_level() != 0) { + LOG(ERROR) << "cannot import a message with non-zero level!"; + return false; + } + if (!block::gen::t_MsgEnvelope.validate_ref(msg_env)) { + LOG(ERROR) << "MsgEnvelope from DispatchQueue is invalid according to automated checks"; + return false; + } + if (!block::tlb::t_MsgEnvelope.validate_ref(msg_env)) { + LOG(ERROR) << "MsgEnvelope from DispatchQueue is invalid according to hand-written checks"; + return false; + } + // 1. unpack MsgEnvelope + block::tlb::MsgEnvelope::Record_std env; + if (!tlb::unpack_cell(msg_env, env)) { + LOG(ERROR) << "cannot unpack MsgEnvelope from DispatchQueue"; + return false; + } + // 2. unpack CommonMsgInfo of the message + vm::CellSlice cs{vm::NoVmOrd{}, env.msg}; + if (block::gen::t_CommonMsgInfo.get_tag(cs) != block::gen::CommonMsgInfo::int_msg_info) { + LOG(ERROR) << "internal message from DispatchQueue is not in fact internal!"; + return false; + } + block::gen::CommonMsgInfo::Record_int_msg_info info; + if (!tlb::unpack(cs, info)) { + LOG(ERROR) << "cannot unpack CommonMsgInfo of an internal message from DispatchQueue"; + return false; + } + if (info.created_lt != lt) { + LOG(ERROR) << "internal message has lt in DispatchQueue distinct from the one in " + "its contents"; + return false; + } + if (!block::tlb::validate_message_libs(env.msg)) { + LOG(ERROR) << "internal message in DispatchQueue has invalid StateInit"; + return false; + } + // 2.1. check fwd_fee and fwd_fee_remaining + td::RefInt256 orig_fwd_fee = block::tlb::t_Grams.as_integer(info.fwd_fee); + if (env.fwd_fee_remaining > orig_fwd_fee) { + LOG(ERROR) << "internal message if DispatchQueue has fwd_fee_remaining=" << td::dec_string(env.fwd_fee_remaining) + << " larger than original fwd_fee=" << td::dec_string(orig_fwd_fee); + return false; + } + // 3. extract source and destination shards + auto src_prefix = block::tlb::t_MsgAddressInt.get_prefix(info.src); + auto dest_prefix = block::tlb::t_MsgAddressInt.get_prefix(info.dest); + if (!(src_prefix.is_valid() && dest_prefix.is_valid())) { + LOG(ERROR) << "internal message in DispatchQueue has invalid source or destination address"; + return false; + } + // 4. chech current and next hop shards + if (env.cur_addr != 0 || env.next_addr != 0) { + LOG(ERROR) << "internal message in DispatchQueue is expected to have zero cur_addr and next_addr"; + return false; + } + // 5. calculate emitted_lt + LogicalTime emitted_lt = std::max(start_lt, last_dispatch_queue_emitted_lt_[src_addr]) + 1; + auto it = accounts.find(src_addr); + if (it != accounts.end()) { + emitted_lt = std::max(emitted_lt, it->second->last_trans_end_lt_ + 1); + } + last_dispatch_queue_emitted_lt_[src_addr] = emitted_lt; + update_max_lt(emitted_lt + 1); + + env.emitted_lt = emitted_lt; + if (!block::tlb::pack_cell(msg_env, env)) { + return fatal_error("cannot pack msg envelope"); + } + + // 6. create NewOutMsg + block::NewOutMsg new_msg{emitted_lt, env.msg, {}, 0}; + new_msg.metadata = env.metadata; + new_msg.msg_env_from_dispatch_queue = msg_env; + ++unprocessed_deferred_messages_[src_addr]; + LOG(INFO) << "delivering deferred message from account " << src_addr.to_hex() << ", lt=" << lt + << ", emitted_lt=" << emitted_lt; + register_new_msg(std::move(new_msg)); + msg_metadata = std::move(env.metadata); + return true; +} + /** * Inserts an InMsg into the block's InMsgDescr. * @@ -3517,8 +3864,9 @@ bool Collator::insert_in_msg(Ref in_msg) { return false; } Ref msg = cs.prefetch_ref(); - int tag = (int)cs.prefetch_ulong(3); - if (!(tag == 0 || tag == 2)) { // msg_import_ext$000 or msg_import_ihr$010 contain (Message Any) directly + int tag = block::gen::t_InMsg.get_tag(cs); + // msg_import_ext$000 or msg_import_ihr$010 contain (Message Any) directly + if (!(tag == block::gen::InMsg::msg_import_ext || tag == block::gen::InMsg::msg_import_ihr)) { // extract Message Any from MsgEnvelope to compute correct key auto cs2 = load_cell_slice(std::move(msg)); if (!cs2.size_refs()) { @@ -3599,11 +3947,15 @@ bool Collator::insert_out_msg(Ref out_msg, td::ConstBitPtr msg_hash) { * * @param msg The new outbound message to enqueue. * @param fwd_fees_remaining The remaining forward fees for the message. - * @param enqueued_lt The logical time at which the message is enqueued. + * @param src_addr 256-bit address of the sender + * @param defer Put the message to DispatchQueue * * @returns True if the message was successfully enqueued, false otherwise. */ -bool Collator::enqueue_message(block::NewOutMsg msg, td::RefInt256 fwd_fees_remaining, ton::LogicalTime enqueued_lt) { +bool Collator::enqueue_message(block::NewOutMsg msg, td::RefInt256 fwd_fees_remaining, StdSmcAddress src_addr, + bool defer) { + LogicalTime enqueued_lt = msg.lt; + CHECK(msg.msg_env_from_dispatch_queue.is_null()); // 0. unpack src_addr and dest_addr block::gen::CommonMsgInfo::Record_int_msg_info info; if (!tlb::unpack_cell_inexact(msg.msg, info)) { @@ -3623,18 +3975,24 @@ bool Collator::enqueue_message(block::NewOutMsg msg, td::RefInt256 fwd_fees_rema return fatal_error("cannot perform hypercube routing for a new outbound message"); } // 2. create a new MsgEnvelope - vm::CellBuilder cb; - CHECK(cb.store_long_bool(4, 4) // msg_envelope#4 cur_addr:.. next_addr:.. - && cb.store_long_bool(route_info.first, 8) // cur_addr:IntermediateAddress - && cb.store_long_bool(route_info.second, 8) // next_addr:IntermediateAddress - && block::tlb::t_Grams.store_integer_ref(cb, fwd_fees_remaining) // fwd_fee_remaining:t_Grams - && cb.store_ref_bool(msg.msg)); // msg:^(Message Any) - Ref msg_env = cb.finalize(); + block::tlb::MsgEnvelope::Record_std msg_env_rec{ + defer ? 0 : route_info.first, defer ? 0 : route_info.second, fwd_fees_remaining, msg.msg, {}, msg.metadata}; + Ref msg_env; + CHECK(block::tlb::pack_cell(msg_env, msg_env_rec)); // 3. create a new OutMsg - CHECK(cb.store_long_bool(1, 3) // msg_export_new$001 - && cb.store_ref_bool(msg_env) // out_msg:^MsgEnvelope - && cb.store_ref_bool(msg.trans)); // transaction:^Transaction - Ref out_msg = cb.finalize(); + vm::CellBuilder cb; + Ref out_msg; + if (defer) { + CHECK(cb.store_long_bool(0b10100, 5) // msg_export_new_defer$10100 + && cb.store_ref_bool(msg_env) // out_msg:^MsgEnvelope + && cb.store_ref_bool(msg.trans)); // transaction:^Transaction + out_msg = cb.finalize(); + } else { + CHECK(cb.store_long_bool(1, 3) // msg_export_new$001 + && cb.store_ref_bool(msg_env) // out_msg:^MsgEnvelope + && cb.store_ref_bool(msg.trans)); // transaction:^Transaction + out_msg = cb.finalize(); + } // 4. insert OutMsg into OutMsgDescr if (verbosity > 2) { std::cerr << "OutMsg for a newly-generated message: "; @@ -3646,7 +4004,30 @@ bool Collator::enqueue_message(block::NewOutMsg msg, td::RefInt256 fwd_fees_rema // 5. create EnqueuedMsg CHECK(cb.store_long_bool(enqueued_lt) // _ enqueued_lt:uint64 && cb.store_ref_bool(msg_env)); // out_msg:^MsgEnvelope = EnqueuedMsg; - // 6. insert EnqueuedMsg into OutMsgQueue + + // 6. insert EnqueuedMsg into OutMsgQueue (or DispatchQueue) + if (defer) { + LOG(INFO) << "deferring new message from account " << workchain() << ":" << src_addr.to_hex() << ", lt=" << msg.lt; + vm::Dictionary dispatch_dict{64}; + td::uint64 dispatch_dict_size; + if (!block::unpack_account_dispatch_queue(dispatch_queue_->lookup(src_addr), dispatch_dict, dispatch_dict_size)) { + return fatal_error(PSTRING() << "cannot unpack AccountDispatchQueue for account " << src_addr.to_hex()); + } + td::BitArray<64> key; + key.store_ulong(msg.lt); + if (!dispatch_dict.set_builder(key, cb, vm::Dictionary::SetMode::Add)) { + return fatal_error(PSTRING() << "cannot add message to AccountDispatchQueue for account " << src_addr.to_hex() + << ", lt=" << msg.lt); + } + ++dispatch_dict_size; + dispatch_queue_->set(src_addr, block::pack_account_dispatch_queue(dispatch_dict, dispatch_dict_size)); + ++dispatch_queue_ops_; + if (!(dispatch_queue_ops_ & 63)) { + return block_limit_status_->add_proof(dispatch_queue_->get_root_cell()); + } + return true; + } + auto next_hop = block::interpolate_addr(src_prefix, dest_prefix, route_info.second); td::BitArray<32 + 64 + 256> key; key.bits().store_int(next_hop.workchain, 32); @@ -3680,7 +4061,7 @@ bool Collator::process_new_messages(bool enqueue_only) { block::NewOutMsg msg = new_msgs.top(); new_msgs.pop(); block_limit_status_->extra_out_msgs--; - if (block_full_ && !enqueue_only) { + if ((block_full_ || have_unprocessed_account_dispatch_queue_) && !enqueue_only) { LOG(INFO) << "BLOCK FULL, enqueue all remaining new messages"; enqueue_only = true; } @@ -3713,11 +4094,17 @@ void Collator::register_new_msg(block::NewOutMsg new_msg) { * Registers new messages that were created in the transaction. * * @param trans The transaction containing the messages. + * @param msg_metadata Metadata of the new messages. */ -void Collator::register_new_msgs(block::transaction::Transaction& trans) { +void Collator::register_new_msgs(block::transaction::Transaction& trans, + td::optional msg_metadata) { CHECK(trans.root.not_null()); for (unsigned i = 0; i < trans.out_msgs.size(); i++) { - register_new_msg(trans.extract_out_msg_ext(i)); + block::NewOutMsg msg = trans.extract_out_msg_ext(i); + if (msg_metadata_enabled_) { + msg.metadata = msg_metadata; + } + register_new_msg(std::move(msg)); } } @@ -4274,7 +4661,21 @@ bool Collator::check_block_overload() { << " lt_delta=" << block_limit_status_->cur_lt - block_limit_status_->limits.start_lt << " size_estimate=" << block_size_estimate_; auto cl = block_limit_status_->classify(); - if (cl <= block::ParamLimits::cl_underload) { + if (cl >= block::ParamLimits::cl_soft || dispatch_queue_total_limit_reached_) { + std::string message = "block is overloaded "; + if (cl >= block::ParamLimits::cl_soft) { + message += PSTRING() << "(category " << cl << ")"; + } else { + message += "(long dispatch queue processing)"; + } + if (out_msg_queue_size_ > SPLIT_MAX_QUEUE_SIZE) { + LOG(INFO) << message << ", but don't set overload history because out_msg_queue size is too big to split (" + << out_msg_queue_size_ << " > " << SPLIT_MAX_QUEUE_SIZE << ")"; + } else { + overload_history_ |= 1; + LOG(INFO) << message; + } + } else if (cl <= block::ParamLimits::cl_underload) { if (out_msg_queue_size_ > MERGE_MAX_QUEUE_SIZE) { LOG(INFO) << "block is underloaded, but don't set underload history because out_msg_queue size is too big to merge (" @@ -4283,15 +4684,6 @@ bool Collator::check_block_overload() { underload_history_ |= 1; LOG(INFO) << "block is underloaded"; } - } else if (cl >= block::ParamLimits::cl_soft) { - if (out_msg_queue_size_ > SPLIT_MAX_QUEUE_SIZE) { - LOG(INFO) << "block is overloaded (category " << cl - << "), but don't set overload history because out_msg_queue size is too big to split (" - << out_msg_queue_size_ << " > " << SPLIT_MAX_QUEUE_SIZE << ")"; - } else { - overload_history_ |= 1; - LOG(INFO) << "block is overloaded (category " << cl << ")"; - } } else { LOG(INFO) << "block is loaded normally"; } @@ -4617,9 +5009,27 @@ bool Collator::compute_out_msg_queue_info(Ref& out_msg_queue_info) { rt->print_rec(std::cerr); } vm::CellBuilder cb; + // out_msg_queue_extra#0 dispatch_queue:DispatchQueue out_queue_size:(Maybe uint48) = OutMsgQueueExtra; + // ... extra:(Maybe OutMsgQueueExtra) + if (!dispatch_queue_->is_empty() || store_out_msg_queue_size_) { + if (!(cb.store_long_bool(1, 1) && cb.store_long_bool(0, 4) && dispatch_queue_->append_dict_to_bool(cb))) { + return false; + } + if (!(cb.store_bool_bool(store_out_msg_queue_size_) && + (!store_out_msg_queue_size_ || cb.store_long_bool(out_msg_queue_size_, 48)))) { + return false; + } + } else { + if (!cb.store_long_bool(0, 1)) { + return false; + } + } + vm::CellSlice maybe_extra = cb.as_cellslice(); + cb.reset(); + return register_out_msg_queue_op(true) && out_msg_queue_->append_dict_to_bool(cb) // _ out_queue:OutMsgQueue && processed_upto_->pack(cb) // proc_info:ProcessedInfo - && ihr_pending->append_dict_to_bool(cb) // ihr_pending:IhrPendingInfo + && cb.append_cellslice_bool(maybe_extra) // extra:(Maybe OutMsgQueueExtra) && cb.finalize_to(out_msg_queue_info); } @@ -5012,12 +5422,13 @@ bool Collator::create_block_candidate() { } // 4. save block candidate LOG(INFO) << "saving new BlockCandidate"; - td::actor::send_closure_later(manager, &ValidatorManager::set_block_candidate, block_candidate->id, - block_candidate->clone(), [self = get_self()](td::Result saved) -> void { - LOG(DEBUG) << "got answer to set_block_candidate"; - td::actor::send_closure_later(std::move(self), &Collator::return_block_candidate, - std::move(saved)); - }); + td::actor::send_closure_later( + manager, &ValidatorManager::set_block_candidate, block_candidate->id, block_candidate->clone(), + validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), + [self = get_self()](td::Result saved) -> void { + LOG(DEBUG) << "got answer to set_block_candidate"; + td::actor::send_closure_later(std::move(self), &Collator::return_block_candidate, std::move(saved)); + }); // 5. communicate about bad and delayed external messages if (!bad_ext_msgs_.empty() || !delay_ext_msgs_.empty()) { LOG(INFO) << "sending complete_external_messages() to Manager"; diff --git a/validator/impl/external-message.cpp b/validator/impl/external-message.cpp index 073e7360e..2fdb491bc 100644 --- a/validator/impl/external-message.cpp +++ b/validator/impl/external-message.cpp @@ -86,24 +86,18 @@ td::Result> ExtMessageQ::create_ext_message(td::BufferSlice dat return Ref{true, std::move(data), std::move(ext_msg), dest_prefix, wc, addr}; } -void ExtMessageQ::run_message(td::BufferSlice data, block::SizeLimitsConfig::ExtMsgLimits limits, - td::actor::ActorId manager, +void ExtMessageQ::run_message(td::Ref message, td::actor::ActorId manager, td::Promise> promise) { - auto R = create_ext_message(std::move(data), limits); - if (R.is_error()) { - return promise.set_error(R.move_as_error_prefix("failed to parse external message ")); - } - auto M = R.move_as_ok(); - auto root = M->root_cell(); + auto root = message->root_cell(); block::gen::CommonMsgInfo::Record_ext_in_msg_info info; tlb::unpack_cell_inexact(root, info); // checked in create message - ton::StdSmcAddress addr = M->addr(); - ton::WorkchainId wc = M->wc(); + ton::StdSmcAddress addr = message->addr(); + ton::WorkchainId wc = message->wc(); run_fetch_account_state( wc, addr, manager, - [promise = std::move(promise), msg_root = root, wc, addr, - M](td::Result, UnixTime, LogicalTime, std::unique_ptr>> + [promise = std::move(promise), msg_root = root, wc, addr, message]( + td::Result, UnixTime, LogicalTime, std::unique_ptr>> res) mutable { if (res.is_error()) { promise.set_error(td::Status::Error(PSLICE() << "Failed to get account state")); @@ -120,7 +114,7 @@ void ExtMessageQ::run_message(td::BufferSlice data, block::SizeLimitsConfig::Ext } else { auto status = run_message_on_account(wc, &acc, utime, lt + 1, msg_root, std::move(config)); if (status.is_ok()) { - promise.set_value(std::move(M)); + promise.set_value(std::move(message)); } else { promise.set_error(td::Status::Error(PSLICE() << "External message was not accepted\n" << status.message())); diff --git a/validator/impl/external-message.hpp b/validator/impl/external-message.hpp index d50847617..ad7ecc74e 100644 --- a/validator/impl/external-message.hpp +++ b/validator/impl/external-message.hpp @@ -61,8 +61,7 @@ class ExtMessageQ : public ExtMessage { ton::StdSmcAddress addr); static td::Result> create_ext_message(td::BufferSlice data, block::SizeLimitsConfig::ExtMsgLimits limits); - static void run_message(td::BufferSlice data, block::SizeLimitsConfig::ExtMsgLimits limits, - td::actor::ActorId manager, + static void run_message(td::Ref message, td::actor::ActorId manager, td::Promise> promise); static td::Status run_message_on_account(ton::WorkchainId wc, block::Account* acc, diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index e3478594b..d69492393 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -119,10 +119,9 @@ td::Result> create_ext_message(td::BufferSlice data, return std::move(res); } -void run_check_external_message(td::BufferSlice data, block::SizeLimitsConfig::ExtMsgLimits limits, - td::actor::ActorId manager, +void run_check_external_message(Ref message, td::actor::ActorId manager, td::Promise> promise) { - ExtMessageQ::run_message(std::move(data), limits, std::move(manager), std::move(promise)); + ExtMessageQ::run_message(std::move(message), std::move(manager), std::move(promise)); } td::Result> create_ihr_message(td::BufferSlice data) { @@ -214,8 +213,8 @@ void run_validate_query(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_maste void run_collate_query(ShardIdFull shard, td::uint32 min_ts, const BlockIdExt& min_masterchain_block_id, std::vector prev, Ed25519_PublicKey collator_id, td::Ref validator_set, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise) { + td::Ref collator_opts, td::actor::ActorId manager, + td::Timestamp timeout, td::Promise promise) { BlockSeqno seqno = 0; for (auto& p : prev) { if (p.seqno() > seqno) { @@ -224,7 +223,8 @@ void run_collate_query(ShardIdFull shard, td::uint32 min_ts, const BlockIdExt& m } td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, false, min_ts, min_masterchain_block_id, std::move(prev), std::move(validator_set), - collator_id, std::move(manager), timeout, std::move(promise)) + collator_id, std::move(collator_opts), std::move(manager), timeout, + std::move(promise)) .release(); } @@ -239,7 +239,8 @@ void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_b } td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true, 0, min_masterchain_block_id, std::move(prev), td::Ref{}, - Ed25519_PublicKey{Bits256::zero()}, std::move(manager), timeout, std::move(promise)) + Ed25519_PublicKey{Bits256::zero()}, td::Ref{true}, + std::move(manager), timeout, std::move(promise)) .release(); } diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 7fa6e59e5..d6fad7ee2 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -287,6 +287,9 @@ void LiteQuery::perform() { [&](lite_api::liteServer_getOutMsgQueueSizes& q) { this->perform_getOutMsgQueueSizes(q.mode_ & 1 ? ShardIdFull(q.wc_, q.shard_) : td::optional()); }, + [&](lite_api::liteServer_getBlockOutMsgQueueSize& q) { + this->perform_getBlockOutMsgQueueSize(q.mode_, create_block_id(q.id_)); + }, [&](auto& obj) { this->abort_query(td::Status::Error(ErrorCode::protoviolation, "unknown query")); })); } @@ -2376,6 +2379,45 @@ void LiteQuery::perform_listBlockTransactions(BlockIdExt blkid, int mode, int co request_block_data(blkid); } +static td::Result> get_in_msg_metadata( + const Ref& in_msg_descr_root, const Ref& trans_root) { + vm::AugmentedDictionary in_msg_descr{vm::load_cell_slice_ref(in_msg_descr_root), 256, block::tlb::aug_InMsgDescr}; + block::gen::Transaction::Record transaction; + if (!block::tlb::unpack_cell(trans_root, transaction)) { + return td::Status::Error("invalid Transaction in block"); + } + Ref msg = transaction.r1.in_msg->prefetch_ref(); + if (msg.is_null()) { + return nullptr; + } + td::Bits256 in_msg_hash = msg->get_hash().bits(); + Ref in_msg = in_msg_descr.lookup(in_msg_hash); + if (in_msg.is_null()) { + return td::Status::Error(PSTRING() << "no InMsg in InMsgDescr for message with hash " << in_msg_hash.to_hex()); + } + int tag = block::gen::t_InMsg.get_tag(*in_msg); + if (tag != block::gen::InMsg::msg_import_imm && tag != block::gen::InMsg::msg_import_fin && + tag != block::gen::InMsg::msg_import_deferred_fin) { + return nullptr; + } + Ref msg_env = in_msg->prefetch_ref(); + if (msg_env.is_null()) { + return td::Status::Error(PSTRING() << "no MsgEnvelope in InMsg for message with hash " << in_msg_hash.to_hex()); + } + block::tlb::MsgEnvelope::Record_std env; + if (!block::tlb::unpack_cell(std::move(msg_env), env)) { + return td::Status::Error(PSTRING() << "failed to unpack MsgEnvelope for message with hash " << in_msg_hash.to_hex()); + } + if (!env.metadata) { + return nullptr; + } + block::MsgMetadata& metadata = env.metadata.value(); + return create_tl_object( + 0, metadata.depth, + create_tl_object(metadata.initiator_wc, metadata.initiator_addr), + metadata.initiator_lt); +} + void LiteQuery::finish_listBlockTransactions(int mode, int req_count) { LOG(INFO) << "completing a listBlockTransactions(" << base_blk_id_.to_str() << ", " << mode << ", " << req_count << ", " << acc_addr_.to_hex() << ", " << trans_lt_ << ") liteserver query"; @@ -2395,6 +2437,8 @@ void LiteQuery::finish_listBlockTransactions(int mode, int req_count) { acc_addr_.set_ones(); trans_lt_ = ~0ULL; } + bool with_metadata = mode & 256; + mode &= ~256; std::vector> result; bool eof = false; ton::LogicalTime reverse = (mode & 64) ? ~0ULL : 0; @@ -2448,8 +2492,18 @@ void LiteQuery::finish_listBlockTransactions(int mode, int req_count) { trans_lt_ = reverse; break; } - result.push_back(create_tl_object(mode, cur_addr, cur_trans.to_long(), - tvalue->get_hash().bits())); + tl_object_ptr metadata; + if (with_metadata) { + auto r_metadata = get_in_msg_metadata(extra.in_msg_descr, tvalue); + if (r_metadata.is_error()) { + fatal_error(r_metadata.move_as_error()); + return; + } + metadata = r_metadata.move_as_ok(); + } + result.push_back(create_tl_object( + mode | (metadata ? 256 : 0), cur_addr, cur_trans.to_long(), tvalue->get_hash().bits(), + std::move(metadata))); ++count; } } @@ -2484,6 +2538,36 @@ void LiteQuery::perform_listBlockTransactionsExt(BlockIdExt blkid, int mode, int request_block_data(blkid); } +static td::Status process_all_in_msg_metadata(const Ref& in_msg_descr_root, + const std::vector>& trans_roots) { + vm::AugmentedDictionary in_msg_descr{vm::load_cell_slice_ref(in_msg_descr_root), 256, block::tlb::aug_InMsgDescr}; + for (const Ref& trans_root : trans_roots) { + block::gen::Transaction::Record transaction; + if (!block::tlb::unpack_cell(trans_root, transaction)) { + return td::Status::Error("invalid Transaction in block"); + } + Ref msg = transaction.r1.in_msg->prefetch_ref(); + if (msg.is_null()) { + continue; + } + td::Bits256 in_msg_hash = msg->get_hash().bits(); + Ref in_msg = in_msg_descr.lookup(in_msg_hash); + if (in_msg.is_null()) { + return td::Status::Error(PSTRING() << "no InMsg in InMsgDescr for message with hash " << in_msg_hash.to_hex()); + } + int tag = block::gen::t_InMsg.get_tag(*in_msg); + if (tag == block::gen::InMsg::msg_import_imm || tag == block::gen::InMsg::msg_import_fin || + tag == block::gen::InMsg::msg_import_deferred_fin) { + Ref msg_env = in_msg->prefetch_ref(); + if (msg_env.is_null()) { + return td::Status::Error(PSTRING() << "no MsgEnvelope in InMsg for message with hash " << in_msg_hash.to_hex()); + } + vm::load_cell_slice(msg_env); + } + } + return td::Status::OK(); +} + void LiteQuery::finish_listBlockTransactionsExt(int mode, int req_count) { LOG(INFO) << "completing a listBlockTransactionsExt(" << base_blk_id_.to_str() << ", " << mode << ", " << req_count << ", " << acc_addr_.to_hex() << ", " << trans_lt_ << ") liteserver query"; @@ -2495,6 +2579,10 @@ void LiteQuery::finish_listBlockTransactionsExt(int mode, int req_count) { CHECK(rhash == base_blk_id_.root_hash); vm::MerkleProofBuilder pb; auto virt_root = block_root; + if (mode & 256) { + // with msg metadata in proof + mode |= 32; + } if (mode & 32) { // proof requested virt_root = pb.init(std::move(virt_root)); @@ -2560,6 +2648,13 @@ void LiteQuery::finish_listBlockTransactionsExt(int mode, int req_count) { ++count; } } + if (mode & 256) { + td::Status S = process_all_in_msg_metadata(extra.in_msg_descr, trans_roots); + if (S.is_error()) { + fatal_error(S.move_as_error()); + return; + } + } } catch (vm::VmError err) { fatal_error("error while parsing AccountBlocks of block "s + base_blk_id_.to_str() + " : " + err.get_msg()); return; @@ -3252,7 +3347,7 @@ void LiteQuery::continue_getOutMsgQueueSizes(td::optional shard, Re auto ig = mp.init_guard(); for (size_t i = 0; i < blocks.size(); ++i) { td::actor::send_closure(manager_, &ValidatorManager::get_out_msg_queue_size, blocks[i], - [promise = ig.get_promise(), res, i, id = blocks[i]](td::Result R) mutable { + [promise = ig.get_promise(), res, i, id = blocks[i]](td::Result R) mutable { TRY_RESULT_PROMISE(promise, value, std::move(R)); res->at(i) = create_tl_object( create_tl_lite_block_id(id), value); @@ -3271,6 +3366,73 @@ void LiteQuery::continue_getOutMsgQueueSizes(td::optional shard, Re }); } +void LiteQuery::perform_getBlockOutMsgQueueSize(int mode, BlockIdExt blkid) { + LOG(INFO) << "started a getBlockOutMsgQueueSize(" << blkid.to_str() << ", " << mode << ") liteserver query"; + mode_ = mode; + if (!blkid.is_valid_full()) { + fatal_error("invalid BlockIdExt"); + return; + } + set_continuation([=]() -> void { finish_getBlockOutMsgQueueSize(); }); + request_block_data_state(blkid); +} + +void LiteQuery::finish_getBlockOutMsgQueueSize() { + LOG(INFO) << "completing getBlockOutNsgQueueSize() query"; + bool with_proof = mode_ & 1; + Ref state_root = state_->root_cell(); + vm::MerkleProofBuilder pb; + if (with_proof) { + pb = vm::MerkleProofBuilder{state_root}; + state_root = pb.root(); + } + block::gen::ShardStateUnsplit::Record sstate; + block::gen::OutMsgQueueInfo::Record out_msg_queue_info; + if (!tlb::unpack_cell(state_root, sstate) || !tlb::unpack_cell(sstate.out_msg_queue_info, out_msg_queue_info)) { + fatal_error("cannot unpack shard state"); + return; + } + vm::CellSlice& extra_slice = out_msg_queue_info.extra.write(); + if (extra_slice.fetch_long(1) == 0) { + fatal_error("no out_msg_queue_size in shard state"); + return; + } + block::gen::OutMsgQueueExtra::Record out_msg_queue_extra; + if (!tlb::unpack(extra_slice, out_msg_queue_extra)) { + fatal_error("cannot unpack OutMsgQueueExtra"); + return; + } + vm::CellSlice& size_slice = out_msg_queue_extra.out_queue_size.write(); + if (size_slice.fetch_long(1) == 0) { + fatal_error("no out_msg_queue_size in shard state"); + return; + } + td::uint64 size = size_slice.prefetch_ulong(48); + + td::BufferSlice proof; + if (with_proof) { + Ref proof1, proof2; + if (!make_state_root_proof(proof1)) { + return; + } + if (!pb.extract_proof_to(proof2)) { + fatal_error("unknown error creating Merkle proof"); + return; + } + auto r_proof = vm::std_boc_serialize_multi({std::move(proof1), std::move(proof2)}); + if (r_proof.is_error()) { + fatal_error(r_proof.move_as_error()); + return; + } + proof = r_proof.move_as_ok(); + } + LOG(INFO) << "getBlockOutMsgQueueSize(" << blk_id_.to_str() << ", " << mode_ << ") query completed"; + auto b = ton::create_serialize_tl_object( + mode_, ton::create_tl_lite_block_id(blk_id_), size, std::move(proof)); + finish_query(std::move(b)); +} + + void LiteQuery::perform_nonfinal_getCandidate(td::Bits256 source, BlockIdExt blkid, td::Bits256 collated_data_hash) { LOG(INFO) << "started a nonfinal.getCandidate liteserver query"; td::actor::send_closure_later( diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index 34e569c99..2d75dc61c 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -170,6 +170,8 @@ class LiteQuery : public td::actor::Actor { std::vector> result); void perform_getOutMsgQueueSizes(td::optional shard); void continue_getOutMsgQueueSizes(td::optional shard, Ref state); + void perform_getBlockOutMsgQueueSize(int mode, BlockIdExt blkid); + void finish_getBlockOutMsgQueueSize(); void perform_nonfinal_getCandidate(td::Bits256 source, BlockIdExt blkid, td::Bits256 collated_data_hash); void perform_nonfinal_getValidatorGroups(int mode, ShardIdFull shard); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 8b2723c33..8c39a1ab4 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -895,6 +895,9 @@ bool ValidateQuery::try_unpack_mc_state() { if (!is_masterchain() && !check_this_shard_mc_info()) { return fatal_error("masterchain configuration does not admit creating block "s + id_.to_str()); } + store_out_msg_queue_size_ = config_->has_capability(ton::capStoreOutMsgQueueSize); + msg_metadata_enabled_ = config_->has_capability(ton::capMsgMetadata); + deferring_messages_enabled_ = config_->has_capability(ton::capDeferMessages); } catch (vm::VmError& err) { return fatal_error(-666, err.get_msg()); } catch (vm::VmVirtError& err) { @@ -967,6 +970,7 @@ bool ValidateQuery::fetch_config_params() { compute_phase_cfg_.suspended_addresses = config_->get_suspended_addresses(now_); compute_phase_cfg_.size_limits = size_limits; compute_phase_cfg_.precompiled_contracts = config_->get_precompiled_contracts_config(); + compute_phase_cfg_.allow_external_unfreeze = compute_phase_cfg_.global_version >= 8; } { // compute action_phase_cfg @@ -990,6 +994,8 @@ bool ValidateQuery::fetch_config_params() { action_phase_cfg_.size_limits = size_limits; action_phase_cfg_.action_fine_enabled = config_->get_global_version() >= 4; action_phase_cfg_.bounce_on_fail_enabled = config_->get_global_version() >= 4; + action_phase_cfg_.message_skip_enabled = config_->get_global_version() >= 8; + action_phase_cfg_.disable_custom_fess = config_->get_global_version() >= 8; action_phase_cfg_.mc_blackhole_addr = config_->get_burning_config().blackhole_addr; } { @@ -1702,7 +1708,7 @@ void ValidateQuery::after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result< * @param wc_info The workchain information. * @param ccvc The Catchain validators configuration. * - * @returns True if the validation wasa successful, false othewise. + * @returns True if the validation wasa successful, false otherwise. */ bool ValidateQuery::check_one_shard(const block::McShardHash& info, const block::McShardHash* sibling, const block::WorkchainInfo* wc_info, const block::CatchainValidatorsConfig& ccvc) { @@ -2195,6 +2201,50 @@ bool ValidateQuery::check_utime_lt() { return true; } +/** + * Reads the size of the outbound message queue from the previous state(s), or requests it if needed. + * + * @returns True if the request was successful, false otherwise. + */ +bool ValidateQuery::prepare_out_msg_queue_size() { + if (ps_.out_msg_queue_size_) { + // if after_split then out_msg_queue_size is always present, since it is calculated during split + old_out_msg_queue_size_ = ps_.out_msg_queue_size_.value(); + return true; + } + old_out_msg_queue_size_ = 0; + for (size_t i = 0; i < prev_blocks.size(); ++i) { + ++pending; + send_closure_later(manager, &ValidatorManager::get_out_msg_queue_size, prev_blocks[i], + [self = get_self(), i](td::Result res) { + td::actor::send_closure(std::move(self), &ValidateQuery::got_out_queue_size, i, + std::move(res)); + }); + } + return true; +} + +/** + * Handles the result of obtaining the size of the outbound message queue. + * + * If the block is after merge then the two sizes are added. + * + * @param i The index of the previous block (0 or 1). + * @param res The result object containing the size of the queue. + */ +void ValidateQuery::got_out_queue_size(size_t i, td::Result res) { + --pending; + if (res.is_error()) { + fatal_error( + res.move_as_error_prefix(PSTRING() << "failed to get message queue size from prev block #" << i << ": ")); + return; + } + td::uint64 size = res.move_as_ok(); + LOG(DEBUG) << "got outbound queue size from prev block #" << i << ": " << size; + old_out_msg_queue_size_ += size; + try_validate(); +} + /* * * METHODS CALLED FROM try_validate() stage 1 @@ -3041,6 +3091,7 @@ bool ValidateQuery::precheck_one_message_queue_update(td::ConstBitPtr out_msg_id return reject_query("new EnqueuedMsg with key "s + out_msg_id.to_hex(352) + " is invalid"); } if (new_value.not_null()) { + ++new_out_msg_queue_size_; if (!block::gen::t_EnqueuedMsg.validate_csr(new_value)) { return reject_query("new EnqueuedMsg with key "s + out_msg_id.to_hex(352) + " failed to pass automated validity checks"); @@ -3057,6 +3108,7 @@ bool ValidateQuery::precheck_one_message_queue_update(td::ConstBitPtr out_msg_id } } if (old_value.not_null()) { + --new_out_msg_queue_size_; if (!block::gen::t_EnqueuedMsg.validate_csr(old_value)) { return reject_query("old EnqueuedMsg with key "s + out_msg_id.to_hex(352) + " failed to pass automated validity checks"); @@ -3083,11 +3135,18 @@ bool ValidateQuery::precheck_one_message_queue_update(td::ConstBitPtr out_msg_id " has been changed in the OutMsgQueue, but the key did not change"); } auto q_msg_env = (old_value.not_null() ? old_value : new_value)->prefetch_ref(); - int tag = (int)out_msg_cs->prefetch_ulong(3); - // mode for msg_export_{ext,new,imm,tr,deq_imm,???,deq/deq_short,tr_req} - static const int tag_mode[8] = {0, 2, 0, 2, 1, 0, 1, 3}; - static const char* tag_str[8] = {"ext", "new", "imm", "tr", "deq_imm", "???", "deq", "tr_req"}; - if (tag < 0 || tag >= 8 || !(tag_mode[tag] & mode)) { + int tag = block::tlb::t_OutMsg.get_tag(*out_msg_cs); + if (tag == 12 || tag == 13) { + tag /= 2; + } else if (tag == 20) { + tag = 8; + } else if (tag == 21) { + tag = 9; + } + // mode for msg_export_{ext,new,imm,tr,deq_imm,???,deq/deq_short,tr_req,new_defer,deferred_tr} + static const int tag_mode[10] = {0, 2, 0, 2, 1, 0, 1, 3, 0, 2}; + static const char* tag_str[10] = {"ext", "new", "imm", "tr", "deq_imm", "???", "deq", "tr_req", "new_defer", "deferred_tr"}; + if (tag < 0 || tag >= 10 || !(tag_mode[tag] & mode)) { return reject_query(PSTRING() << "OutMsgDescr corresponding to " << m_str[mode] << "queued message with key " << out_msg_id.to_hex(352) << " has invalid tag " << tag << "(" << tag_str[tag & 7] << ")"); @@ -3202,6 +3261,7 @@ bool ValidateQuery::precheck_message_queue_update() { try { CHECK(ps_.out_msg_queue_ && ns_.out_msg_queue_); CHECK(out_msg_dict_); + new_out_msg_queue_size_ = old_out_msg_queue_size_; if (!ps_.out_msg_queue_->scan_diff( *ns_.out_msg_queue_, [this](td::ConstBitPtr key, int key_len, Ref old_val_extra, @@ -3216,6 +3276,186 @@ bool ValidateQuery::precheck_message_queue_update() { return reject_query("invalid OutMsgQueue dictionary difference between the old and the new state: "s + err.get_msg()); } + LOG(INFO) << "outbound message queue size: " << old_out_msg_queue_size_ << " -> " << new_out_msg_queue_size_; + if (store_out_msg_queue_size_) { + if (!ns_.out_msg_queue_size_) { + return reject_query(PSTRING() << "outbound message queue size in the new state is not correct (expected: " + << new_out_msg_queue_size_ << ", found: none)"); + } + if (ns_.out_msg_queue_size_.value() != new_out_msg_queue_size_) { + return reject_query(PSTRING() << "outbound message queue size in the new state is not correct (expected: " + << new_out_msg_queue_size_ << ", found: " << ns_.out_msg_queue_size_.value() + << ")"); + } + } else { + if (ns_.out_msg_queue_size_) { + return reject_query("outbound message queue size in the new state is present, but shouldn't"); + } + } + return true; +} + +/** + * Performs a check on the difference between the old and new dispatch queues for one account. + * + * @param addr The 256-bit address of the account. + * @param old_queue_csr The old value of the account dispatch queue. + * @param new_queue_csr The new value of the account dispatch queue. + * + * @returns True if the check is successful, false otherwise. + */ +bool ValidateQuery::check_account_dispatch_queue_update(td::Bits256 addr, Ref old_queue_csr, + Ref new_queue_csr) { + vm::Dictionary old_dict{64}; + td::uint64 old_dict_size = 0; + if (!block::unpack_account_dispatch_queue(old_queue_csr, old_dict, old_dict_size)) { + return reject_query(PSTRING() << "invalid AccountDispatchQueue for " << addr.to_hex() << " in the old state"); + } + vm::Dictionary new_dict{64}; + td::uint64 new_dict_size = 0; + if (!block::unpack_account_dispatch_queue(new_queue_csr, new_dict, new_dict_size)) { + return reject_query(PSTRING() << "invalid AccountDispatchQueue for " << addr.to_hex() << " in the new state"); + } + td::uint64 expected_dict_size = old_dict_size; + LogicalTime max_removed_lt = 0; + LogicalTime min_added_lt = (LogicalTime)-1; + bool res = old_dict.scan_diff( + new_dict, [&](td::ConstBitPtr key, int key_len, Ref old_val, Ref new_val) { + CHECK(key_len == 64); + CHECK(old_val.not_null() || new_val.not_null()); + if (old_val.not_null() && new_val.not_null()) { + return false; + } + td::uint64 lt = key.get_uint(64); + block::gen::EnqueuedMsg::Record rec; + if (old_val.not_null()) { + LOG(DEBUG) << "removed message from DispatchQueue: account=" << addr.to_hex() << ", lt=" << lt; + --expected_dict_size; + if (!block::tlb::csr_unpack(old_val, rec)) { + return reject_query(PSTRING() << "invalid EnqueuedMsg in AccountDispatchQueue for " << addr.to_hex()); + } + } else { + LOG(DEBUG) << "added message to DispatchQueue: account=" << addr.to_hex() << ", lt=" << lt; + ++expected_dict_size; + if (!block::tlb::csr_unpack(new_val, rec)) { + return reject_query(PSTRING() << "invalid EnqueuedMsg in AccountDispatchQueue for " << addr.to_hex()); + } + if (is_masterchain() && config_->is_special_smartcontract(addr)) { + return reject_query(PSTRING() << "cannot defer message from a special account -1:" << addr.to_hex()); + } + } + if (lt != rec.enqueued_lt) { + return reject_query(PSTRING() << "invalid EnqueuedMsg in AccountDispatchQueue for " << addr.to_hex() + << ": lt mismatch (" << lt << " != " << rec.enqueued_lt << ")"); + } + block::tlb::MsgEnvelope::Record_std env; + if (!block::gen::t_MsgEnvelope.validate_ref(rec.out_msg) || !block::tlb::unpack_cell(rec.out_msg, env)) { + return reject_query(PSTRING() << "invalid EnqueuedMsg in AccountDispatchQueue for " << addr.to_hex()); + } + if (env.emitted_lt) { + return reject_query(PSTRING() << "invalid EnqueuedMsg in AccountDispatchQueue for " << addr.to_hex() + << ", lt=" << lt << ": unexpected emitted_lt"); + } + unsigned long long created_lt; + vm::CellSlice msg_cs = vm::load_cell_slice(env.msg); + if (!block::tlb::t_Message.get_created_lt(msg_cs, created_lt)) { + return reject_query(PSTRING() << "invalid EnqueuedMsg in AccountDispatchQueue for " << addr.to_hex() + << ": cannot get created_lt"); + } + if (lt != created_lt) { + return reject_query(PSTRING() << "invalid EnqueuedMsg in AccountDispatchQueue for " << addr.to_hex() + << ": lt mismatch (" << lt << " != " << created_lt << ")"); + } + if (old_val.not_null()) { + removed_dispatch_queue_messages_[{addr, lt}] = rec.out_msg; + max_removed_lt = std::max(max_removed_lt, lt); + } else { + new_dispatch_queue_messages_[{addr, lt}] = rec.out_msg; + min_added_lt = std::min(min_added_lt, lt); + } + return true; + }); + if (!res) { + return reject_query(PSTRING() << "invalid AccountDispatchQueue diff for account " << addr.to_hex()); + } + if (expected_dict_size != new_dict_size) { + return reject_query(PSTRING() << "invalid count in AccountDispatchQuery for " << addr.to_hex() + << ": expected=" << expected_dict_size << ", found=" << new_dict_size); + } + if (!new_dict.is_empty()) { + td::BitArray<64> new_min_lt; + CHECK(new_dict.get_minmax_key(new_min_lt).not_null()); + if (new_min_lt.to_ulong() <= max_removed_lt) { + return reject_query(PSTRING() << "invalid AccountDispatchQuery update for " << addr.to_hex() + << ": max removed lt is " << max_removed_lt << ", but lt=" << new_min_lt.to_ulong() + << " is still in queue"); + } + } + if (!old_dict.is_empty()) { + td::BitArray<64> old_max_lt; + CHECK(old_dict.get_minmax_key(old_max_lt, true).not_null()); + if (old_max_lt.to_ulong() >= min_added_lt) { + return reject_query(PSTRING() << "invalid AccountDispatchQuery update for " << addr.to_hex() + << ": min added lt is " << min_added_lt << ", but lt=" << old_max_lt.to_ulong() + << " was present in the queue"); + } + if (max_removed_lt != old_max_lt.to_ulong()) { + // Some old messages are still in DispatchQueue, meaning that all new messages from this account must be deferred + account_expected_defer_all_messages_.insert(addr); + } + } + if (old_dict_size > 0 && max_removed_lt != 0) { + ++processed_account_dispatch_queues_; + } + return true; +} + +/** + * Pre-check the difference between the old and new dispatch queues and put the difference to + * new_dispatch_queue_messages_, old_dispatch_queue_messages_ + * + * @returns True if the pre-check and unpack is successful, false otherwise. + */ +bool ValidateQuery::unpack_dispatch_queue_update() { + LOG(INFO) << "checking the difference between the old and the new dispatch queues"; + try { + CHECK(ps_.dispatch_queue_ && ns_.dispatch_queue_); + CHECK(out_msg_dict_); + bool res = ps_.dispatch_queue_->scan_diff( + *ns_.dispatch_queue_, + [this](td::ConstBitPtr key, int key_len, Ref old_val_extra, Ref new_val_extra) { + CHECK(key_len == 256); + return check_account_dispatch_queue_update(key, ps_.dispatch_queue_->extract_value(std::move(old_val_extra)), + ns_.dispatch_queue_->extract_value(std::move(new_val_extra))); + }, + 3 /* check augmentation of changed nodes */); + if (!res) { + return reject_query("invalid DispatchQueue dictionary in the new state"); + } + + if (old_out_msg_queue_size_ <= compute_phase_cfg_.size_limits.defer_out_queue_size_limit) { + // Check that at least one message was taken from each AccountDispatchQueue + try { + have_unprocessed_account_dispatch_queue_ = false; + td::uint64 total_account_dispatch_queues = 0; + ps_.dispatch_queue_->check_for_each([&](Ref, td::ConstBitPtr, int n) -> bool { + ++total_account_dispatch_queues; + if (total_account_dispatch_queues > processed_account_dispatch_queues_) { + return false; + } + return true; + }); + have_unprocessed_account_dispatch_queue_ = + (total_account_dispatch_queues != processed_account_dispatch_queues_); + } catch (vm::VmVirtError&) { + // VmVirtError can happen if we have only a proof of ShardState + have_unprocessed_account_dispatch_queue_ = true; + } + } + } catch (vm::VmError& err) { + return reject_query("invalid DispatchQueue dictionary difference between the old and the new state: "s + + err.get_msg()); + } return true; } @@ -3343,8 +3583,8 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) CHECK(in_msg.not_null()); int tag = block::gen::t_InMsg.get_tag(*in_msg); CHECK(tag >= 0); // NB: the block has been already checked to be valid TL-B in try_validate() - ton::StdSmcAddress addr; - ton::WorkchainId wc; + ton::StdSmcAddress src_addr, dest_addr; + ton::WorkchainId src_wc, dest_wc; Ref src, dest; Ref transaction; Ref msg, msg_env, tr_msg_env; @@ -3357,6 +3597,7 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) block::gen::CommonMsgInfo::Record_int_msg_info info; ton::AccountIdPrefixFull src_prefix, dest_prefix, cur_prefix, next_prefix; td::RefInt256 fwd_fee, orig_fwd_fee; + bool from_dispatch_queue = false; // initial checks and unpack switch (tag) { case block::gen::InMsg::msg_import_ext: { @@ -3383,7 +3624,7 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) dest_prefix.to_str() + "... not in this shard"); } dest = std::move(info_ext.dest); - if (!block::tlb::t_MsgAddressInt.extract_std_address(dest, wc, addr)) { + if (!block::tlb::t_MsgAddressInt.extract_std_address(dest, dest_wc, dest_addr)) { return reject_query("cannot unpack destination address of inbound external message with hash "s + key.to_hex(256)); } @@ -3395,7 +3636,7 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) block::gen::InMsg::Record_msg_import_imm inp; unsigned long long created_lt = 0; CHECK(tlb::csr_unpack(in_msg, inp) && tlb::unpack_cell(inp.in_msg, env) && - block::tlb::t_MsgEnvelope.get_created_lt(vm::load_cell_slice(inp.in_msg), created_lt) && + block::tlb::t_MsgEnvelope.get_emitted_lt(vm::load_cell_slice(inp.in_msg), created_lt) && (fwd_fee = block::tlb::t_Grams.as_integer(std::move(inp.fwd_fee))).not_null()); transaction = std::move(inp.transaction); msg_env = std::move(inp.in_msg); @@ -3442,9 +3683,42 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) // msg_discard_fin$110 in_msg:^MsgEnvelope transaction_id:uint64 fwd_fee:Grams return reject_query("InMsg with key "s + key.to_hex(256) + " is a msg_discard_fin, but IHR messages are not enabled in this version"); + case block::gen::InMsg::msg_import_deferred_fin: { + from_dispatch_queue = true; + // msg_import_deferredfin$00100 in_msg:^MsgEnvelope transaction:^Transaction fwd_fee:Grams + // importing and processing an internal message from DispatchQueue with destination in this shard + block::gen::InMsg::Record_msg_import_deferred_fin inp; + CHECK(tlb::csr_unpack(in_msg, inp) && tlb::unpack_cell(inp.in_msg, env) && + (fwd_fee = block::tlb::t_Grams.as_integer(std::move(inp.fwd_fee))).not_null()); + transaction = std::move(inp.transaction); + msg_env = std::move(inp.in_msg); + msg = env.msg; + // ... + break; + } + case block::gen::InMsg::msg_import_deferred_tr: { + from_dispatch_queue = true; + // msg_import_deferred_tr$00101 in_msg:^MsgEnvelope out_msg:^MsgEnvelope + // importing and enqueueing internal message from DispatchQueue + block::gen::InMsg::Record_msg_import_deferred_tr inp; + CHECK(tlb::csr_unpack(in_msg, inp) && tlb::unpack_cell(inp.in_msg, env)); + fwd_fee = td::zero_refint(); + msg_env = std::move(inp.in_msg); + msg = env.msg; + tr_msg_env = std::move(inp.out_msg); + // ... + break; + } default: return reject_query(PSTRING() << "InMsg with key " << key.to_hex(256) << " has impossible tag " << tag); } + if (have_unprocessed_account_dispatch_queue_ && tag != block::gen::InMsg::msg_import_ext && + tag != block::gen::InMsg::msg_import_deferred_tr && tag != block::gen::InMsg::msg_import_deferred_fin) { + // Collator is requeired to take at least one message from each AccountDispatchQueue + // (unless the block is full or unless out_msg_queue_size is big) + // If some AccountDispatchQueue is unporcessed then it's not allowed to import other messages except for externals + return reject_query("required DispatchQueue processing is not done, but some other internal messages are imported"); + } // common checks for all (non-external) inbound messages CHECK(msg.not_null()); if (msg->get_hash().as_bitslice() != key) { @@ -3485,27 +3759,34 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) return reject_query("next hop address "s + next_prefix.to_str() + "... of inbound internal message with hash " + key.to_hex(256) + " does not belong to the current block's shard " + shard_.to_str()); } - // next hop may coincide with current address only if destination is already reached - if (next_prefix == cur_prefix && cur_prefix != dest_prefix) { + // next hop may coincide with current address only if destination is already reached (or it is deferred message) + if (!from_dispatch_queue && next_prefix == cur_prefix && cur_prefix != dest_prefix) { return reject_query( "next hop address "s + next_prefix.to_str() + "... of inbound internal message with hash " + key.to_hex(256) + " coincides with its current address, but this message has not reached its final destination " + dest_prefix.to_str() + "... yet"); } + if (from_dispatch_queue && next_prefix != cur_prefix) { + return reject_query( + "next hop address "s + next_prefix.to_str() + "... of deferred internal message with hash " + key.to_hex(256) + + " must coincide with its current prefix "s + cur_prefix.to_str() + "..."s); + } // if a message is processed by a transaction, it must have destination inside the current shard if (transaction.not_null() && !ton::shard_contains(shard_, dest_prefix)) { return reject_query("inbound internal message with hash "s + key.to_hex(256) + " has destination address " + dest_prefix.to_str() + "... not in this shard, but it is processed nonetheless"); } - // if a message is not processed by a transaction, its final destination must be outside this shard - if (transaction.is_null() && ton::shard_contains(shard_, dest_prefix)) { + // if a message is not processed by a transaction, its final destination must be outside this shard, + // or it is a deferred message (dispatch queue -> out msg queue) + if (tag != block::gen::InMsg::msg_import_deferred_tr && transaction.is_null() && + ton::shard_contains(shard_, dest_prefix)) { return reject_query("inbound internal message with hash "s + key.to_hex(256) + " has destination address " + dest_prefix.to_str() + "... in this shard, but it is not processed by a transaction"); } src = std::move(info.src); dest = std::move(info.dest); // unpack complete destination address if it is inside this shard - if (transaction.not_null() && !block::tlb::t_MsgAddressInt.extract_std_address(dest, wc, addr)) { + if (transaction.not_null() && !block::tlb::t_MsgAddressInt.extract_std_address(dest, dest_wc, dest_addr)) { return reject_query("cannot unpack destination address of inbound internal message with hash "s + key.to_hex(256)); } @@ -3517,6 +3798,44 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) td::dec_string(env.fwd_fee_remaining) + " larger than the original (total) forwarding fee " + td::dec_string(orig_fwd_fee)); } + // Unpacr src address + if (!block::tlb::t_MsgAddressInt.extract_std_address(src, src_wc, src_addr)) { + return reject_query("cannot unpack source address of inbound external message with hash "s + key.to_hex(256)); + } + } + + if (from_dispatch_queue) { + // Check that the message was removed from DispatchQueue + LogicalTime lt = info.created_lt; + auto it = removed_dispatch_queue_messages_.find({src_addr, lt}); + if (it == removed_dispatch_queue_messages_.end()) { + return reject_query(PSTRING() << "deferred InMsg with src_addr=" << src_addr.to_hex() << ", lt=" << lt + << " was not removed from the dispatch queue"); + } + // InMsg msg_import_deferred_* has emitted_lt in MessageEnv, but this emitted_lt is not present in DispatchQueue + Ref dispatched_msg_env = it->second; + td::Ref expected_msg_env; + if (!env.emitted_lt) { + return reject_query(PSTRING() << "no dispatch_lt in deferred InMsg with src_addr=" << src_addr.to_hex() + << ", lt=" << lt); + } + auto emitted_lt = env.emitted_lt.value(); + if (emitted_lt < start_lt_ || emitted_lt > end_lt_) { + return reject_query(PSTRING() << "dispatch_lt in deferred InMsg with src_addr=" << src_addr.to_hex() + << ", lt=" << lt << " is not between start and end of the block"); + } + auto env2 = env; + env2.emitted_lt = {}; + CHECK(block::tlb::pack_cell(expected_msg_env, env2)); + if (dispatched_msg_env->get_hash() != expected_msg_env->get_hash()) { + return reject_query(PSTRING() << "deferred InMsg with src_addr=" << src_addr.to_hex() << ", lt=" << lt + << " msg envelope hasg mismatch: " << dispatched_msg_env->get_hash().to_hex() + << " in DispatchQueue, " << expected_msg_env->get_hash().to_hex() << " expected"); + } + removed_dispatch_queue_messages_.erase(it); + if (tag == block::gen::InMsg::msg_import_deferred_fin) { + msg_emitted_lt_.emplace_back(src_addr, lt, env.emitted_lt.value()); + } } if (transaction.not_null()) { @@ -3533,10 +3852,10 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) ton::StdSmcAddress trans_addr; ton::LogicalTime trans_lt; CHECK(block::get_transaction_id(transaction, trans_addr, trans_lt)); - if (addr != trans_addr) { + if (dest_addr != trans_addr) { block::gen::t_InMsg.print(std::cerr, *in_msg); return reject_query(PSTRING() << "InMsg corresponding to inbound message with hash " << key.to_hex(256) - << " and destination address " << addr.to_hex() + << " and destination address " << dest_addr.to_hex() << " claims that the message is processed by transaction " << trans_lt << " of another account " << trans_addr.to_hex()); } @@ -3588,6 +3907,7 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) } case block::gen::InMsg::msg_import_fin: { // msg_import_fin$100 in_msg:^MsgEnvelope transaction:^Transaction fwd_fee:Grams + // msg_import_deferred_fin$00100 in_msg:^MsgEnvelope transaction:^Transaction fwd_fee:Grams // importing and processing an internal message with destination in this shard CHECK(transaction.not_null()); CHECK(shard_contains(shard_, next_prefix)); @@ -3620,22 +3940,39 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) // ... break; } + case block::gen::InMsg::msg_import_deferred_fin: { + // fwd_fee must be equal to the fwd_fee_remaining of this MsgEnvelope + if (*fwd_fee != *env.fwd_fee_remaining) { + return reject_query("msg_import_imm$011 InMsg with hash "s + key.to_hex(256) + + " is invalid because its collected fwd_fee=" + td::dec_string(fwd_fee) + + " is not equal to fwd_fee_remaining=" + td::dec_string(env.fwd_fee_remaining) + + " of this message (envelope)"); + } + // ... + break; + } + case block::gen::InMsg::msg_import_deferred_tr: case block::gen::InMsg::msg_import_tr: { // msg_import_tr$101 in_msg:^MsgEnvelope out_msg:^MsgEnvelope transit_fee:Grams + // msg_import_deferred_tr$00101 in_msg:^MsgEnvelope out_msg:^MsgEnvelope // importing and relaying a (transit) internal message with destination outside this shard - if (cur_prefix == dest_prefix) { + if (cur_prefix == dest_prefix && tag == block::gen::InMsg::msg_import_tr) { return reject_query("inbound internal message with hash "s + key.to_hex(256) + " is a msg_import_tr$101 (a transit message), but its current address " + cur_prefix.to_str() + " is already equal to its final destination"); } + if (cur_prefix != next_prefix && tag == block::gen::InMsg::msg_import_deferred_tr) { + return reject_query("internal message from DispatchQueue with hash "s + key.to_hex(256) + + " is a msg_import_deferred_tr$00101, but its current address " + + cur_prefix.to_str() + " is not equal to next address"); + } CHECK(transaction.is_null()); - CHECK(cur_prefix != next_prefix); auto out_msg_cs = out_msg_dict_->lookup(key, 256); if (out_msg_cs.is_null()) { return reject_query("inbound internal message with hash "s + key.to_hex(256) + " is a msg_import_tr$101 (transit message), but the corresponding OutMsg does not exist"); } - if (shard_contains(shard_, cur_prefix)) { + if (shard_contains(shard_, cur_prefix) && tag == block::gen::InMsg::msg_import_tr) { // we imported this message from our shard! // (very rare situation possible only after merge) tr_req = true; @@ -3648,7 +3985,7 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) } out_msg_env = std::move(out_msg.out_msg); reimport = std::move(out_msg.imported); - } else { + } else if (tag == block::gen::InMsg::msg_import_tr) { block::gen::OutMsg::Record_msg_export_tr out_msg; if (!tlb::csr_unpack_safe(out_msg_cs, out_msg)) { return reject_query( @@ -3662,6 +3999,16 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) if (!check_imported_message(msg_env)) { return false; } + } else { + block::gen::OutMsg::Record_msg_export_deferred_tr out_msg; + if (!tlb::csr_unpack_safe(out_msg_cs, out_msg)) { + return reject_query( + "inbound internal message with hash "s + key.to_hex(256) + + " is a msg_import_deferred_tr$00101 with current address " + cur_prefix.to_str() + + "... outside of our shard, but the corresponding OutMsg is not a valid msg_export_deferred_tr$10101"); + } + out_msg_env = std::move(out_msg.out_msg); + reimport = std::move(out_msg.imported); } // perform hypercube routing for this transit message auto route_info = block::perform_hypercube_routing(next_prefix, dest_prefix, shard_); @@ -3702,6 +4049,18 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) td::dec_string(env.fwd_fee_remaining) + " to " + td::dec_string(tr_env.fwd_fee_remaining) + " in transit"); } + if (tr_env.metadata != env.metadata) { + return reject_query( + PSTRING() << "InMsg for transit message with hash " << key.to_hex(256) << " contains invalid MsgMetadata: " + << (env.metadata ? env.metadata.value().to_str() : "") << " in in_msg, but " + << (tr_env.metadata ? tr_env.metadata.value().to_str() : "") << " in out_msg"); + } + if (tr_env.emitted_lt != env.emitted_lt) { + return reject_query( + PSTRING() << "InMsg for transit message with hash " << key.to_hex(256) << " contains invalid emitted_lt: " + << (env.emitted_lt ? td::to_string(env.emitted_lt.value()) : "") << " in in_msg, but " + << (tr_env.emitted_lt ? td::to_string(tr_env.emitted_lt.value()) : "") << " in out_msg"); + } if (tr_msg_env->get_hash() != out_msg_env->get_hash()) { return reject_query( "InMsg for transit message with hash "s + key.to_hex(256) + @@ -3709,7 +4068,8 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) (tr_req ? "requeued" : "usual") + "transit)"); } // check the amount of the transit fee - td::RefInt256 transit_fee = action_phase_cfg_.fwd_std.get_next_part(env.fwd_fee_remaining); + td::RefInt256 transit_fee = + from_dispatch_queue ? td::zero_refint() : action_phase_cfg_.fwd_std.get_next_part(env.fwd_fee_remaining); if (*transit_fee != *fwd_fee) { return reject_query("InMsg for transit message with hash "s + key.to_hex(256) + " declared collected transit fees to be " + td::dec_string(fwd_fee) + @@ -3735,7 +4095,8 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) " refers to a different reimport InMsg"); } // for transit messages, OutMsg refers to the newly-created outbound messages (not to the re-imported old outbound message) - if (tag != block::gen::InMsg::msg_import_tr && out_msg_env->get_hash() != msg_env->get_hash()) { + if (tag != block::gen::InMsg::msg_import_tr && tag != block::gen::InMsg::msg_import_deferred_tr && + out_msg_env->get_hash() != msg_env->get_hash()) { return reject_query( "InMsg with hash "s + key.to_hex(256) + " is a reimport record, but the corresponding OutMsg exports a MsgEnvelope with a different hash"); @@ -3781,8 +4142,8 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms CHECK(out_msg.not_null()); int tag = block::gen::t_OutMsg.get_tag(*out_msg); CHECK(tag >= 0); // NB: the block has been already checked to be valid TL-B in try_validate() - ton::StdSmcAddress addr; - ton::WorkchainId wc; + ton::StdSmcAddress src_addr; + ton::WorkchainId src_wc; Ref src, dest; Ref transaction; Ref msg, msg_env, tr_msg_env, reimport; @@ -3826,7 +4187,7 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms src_prefix.to_str() + "... not in this shard"); } src = std::move(info_ext.src); - if (!block::tlb::t_MsgAddressInt.extract_std_address(src, wc, addr)) { + if (!block::tlb::t_MsgAddressInt.extract_std_address(src, src_wc, src_addr)) { return reject_query("cannot unpack source address of outbound external message with hash "s + key.to_hex(256)); } break; @@ -3845,7 +4206,7 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms case block::gen::OutMsg::msg_export_new: { block::gen::OutMsg::Record_msg_export_new out; CHECK(tlb::csr_unpack(out_msg, out) && tlb::unpack_cell(out.out_msg, env) && - block::tlb::t_MsgEnvelope.get_created_lt(vm::load_cell_slice(out.out_msg), created_lt)); + block::tlb::t_MsgEnvelope.get_emitted_lt(vm::load_cell_slice(out.out_msg), created_lt)); transaction = std::move(out.transaction); msg_env = std::move(out.out_msg); msg = env.msg; @@ -3908,6 +4269,35 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms // ... break; } + case block::gen::OutMsg::msg_export_new_defer: { + block::gen::OutMsg::Record_msg_export_new_defer out; + CHECK(tlb::csr_unpack(out_msg, out) && tlb::unpack_cell(out.out_msg, env) && + block::tlb::t_MsgEnvelope.get_emitted_lt(vm::load_cell_slice(out.out_msg), created_lt)); + transaction = std::move(out.transaction); + msg_env = std::move(out.out_msg); + msg = env.msg; + // ... + break; + } + case block::gen::OutMsg::msg_export_deferred_tr: { + block::gen::OutMsg::Record_msg_export_deferred_tr out; + CHECK(tlb::csr_unpack(out_msg, out) && tlb::unpack_cell(out.out_msg, env)); + msg_env = std::move(out.out_msg); + msg = env.msg; + reimport = std::move(out.imported); + in_tag = block::gen::InMsg::msg_import_deferred_tr; + mode = 2; // added to OutMsgQueue + if (!env.emitted_lt) { + return reject_query(PSTRING() << "msg_export_deferred_tr for OutMsg with key " << key.to_hex(256) + << " does not have emitted_lt in MsgEnvelope"); + } + if (env.emitted_lt.value() < start_lt_ || env.emitted_lt.value() > end_lt_) { + return reject_query(PSTRING() << "emitted_lt for msg_export_deferred_tr with key " << key.to_hex(256) + << " is not between start and end lt of the block"); + } + // ... + break; + } default: return reject_query(PSTRING() << "OutMsg with key (message hash) " << key.to_hex(256) << " has an unknown tag " << tag); @@ -3942,30 +4332,36 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms return reject_query("destination of outbound internal message with hash "s + key.to_hex(256) + " is an invalid blockchain address"); } - cur_prefix = block::interpolate_addr(src_prefix, dest_prefix, env.cur_addr); - next_prefix = block::interpolate_addr(src_prefix, dest_prefix, env.next_addr); - if (!(cur_prefix.is_valid() && next_prefix.is_valid())) { - return reject_query("cannot compute current and next hop addresses of outbound internal message with hash "s + - key.to_hex(256)); - } - // check that next hop is nearer to the destination than the current address - if (count_matching_bits(dest_prefix, next_prefix) < count_matching_bits(dest_prefix, cur_prefix)) { - return reject_query("next hop address "s + next_prefix.to_str() + "... of outbound internal message with hash " + - key.to_hex(256) + " is further from its destination " + dest_prefix.to_str() + - "... than its current address " + cur_prefix.to_str() + "..."); - } - // current address must belong to this shard (otherwise we should never had exported this message) - if (!ton::shard_contains(shard_, cur_prefix)) { - return reject_query("current address "s + cur_prefix.to_str() + "... of outbound internal message with hash " + - key.to_hex(256) + " does not belong to the current block's shard " + shard_.to_str()); - } - // next hop may coincide with current address only if destination is already reached - if (next_prefix == cur_prefix && cur_prefix != dest_prefix) { - return reject_query( - "next hop address "s + next_prefix.to_str() + "... of outbound internal message with hash " + - key.to_hex(256) + - " coincides with its current address, but this message has not reached its final destination " + - dest_prefix.to_str() + "... yet"); + if (tag == block::gen::OutMsg::msg_export_new_defer) { + if (env.cur_addr != 0 || env.next_addr != 0) { + return reject_query("cur_addr and next_addr of the message in DispatchQueue must be zero"); + } + } else { + cur_prefix = block::interpolate_addr(src_prefix, dest_prefix, env.cur_addr); + next_prefix = block::interpolate_addr(src_prefix, dest_prefix, env.next_addr); + if (!(cur_prefix.is_valid() && next_prefix.is_valid())) { + return reject_query("cannot compute current and next hop addresses of outbound internal message with hash "s + + key.to_hex(256)); + } + // check that next hop is nearer to the destination than the current address + if (count_matching_bits(dest_prefix, next_prefix) < count_matching_bits(dest_prefix, cur_prefix)) { + return reject_query("next hop address "s + next_prefix.to_str() + "... of outbound internal message with hash " + + key.to_hex(256) + " is further from its destination " + dest_prefix.to_str() + + "... than its current address " + cur_prefix.to_str() + "..."); + } + // current address must belong to this shard (otherwise we should never had exported this message) + if (!ton::shard_contains(shard_, cur_prefix)) { + return reject_query("current address "s + cur_prefix.to_str() + "... of outbound internal message with hash " + + key.to_hex(256) + " does not belong to the current block's shard " + shard_.to_str()); + } + // next hop may coincide with current address only if destination is already reached + if (next_prefix == cur_prefix && cur_prefix != dest_prefix) { + return reject_query( + "next hop address "s + next_prefix.to_str() + "... of outbound internal message with hash " + + key.to_hex(256) + + " coincides with its current address, but this message has not reached its final destination " + + dest_prefix.to_str() + "... yet"); + } } // if a message is created by a transaction, it must have source inside the current shard if (transaction.not_null() && !ton::shard_contains(shard_, src_prefix)) { @@ -3976,7 +4372,7 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms src = std::move(info.src); dest = std::move(info.dest); // unpack complete source address if it is inside this shard - if (transaction.not_null() && !block::tlb::t_MsgAddressInt.extract_std_address(src, wc, addr)) { + if (!block::tlb::t_MsgAddressInt.extract_std_address(src, src_wc, src_addr)) { return reject_query("cannot unpack source address of outbound internal message with hash "s + key.to_hex(256) + " created in this shard"); } @@ -4004,10 +4400,10 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms ton::StdSmcAddress trans_addr; ton::LogicalTime trans_lt; CHECK(block::get_transaction_id(transaction, trans_addr, trans_lt)); - if (addr != trans_addr) { + if (src_addr != trans_addr) { block::gen::t_OutMsg.print(std::cerr, *out_msg); return reject_query(PSTRING() << "OutMsg corresponding to outbound message with hash " << key.to_hex(256) - << " and source address " << addr.to_hex() + << " and source address " << src_addr.to_hex() << " claims that the message was created by transaction " << trans_lt << " of another account " << trans_addr.to_hex()); } @@ -4026,43 +4422,64 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms (q_key.bits() + 96).copy_from(key, 256); auto q_entry = ns_.out_msg_queue_->lookup(q_key); auto old_q_entry = ps_.out_msg_queue_->lookup(q_key); - if (old_q_entry.not_null() && q_entry.not_null()) { - return reject_query("OutMsg with key (message hash) "s + key.to_hex(256) + - " should have removed or added OutMsgQueue entry with key " + q_key.to_hex() + - ", but it is present both in the old and in the new output queues"); - } - if (old_q_entry.is_null() && q_entry.is_null() && mode) { - return reject_query("OutMsg with key (message hash) "s + key.to_hex(256) + - " should have removed or added OutMsgQueue entry with key " + q_key.to_hex() + - ", but it is absent both from the old and from the new output queues"); - } - if (!mode && (old_q_entry.not_null() || q_entry.not_null())) { - return reject_query("OutMsg with key (message hash) "s + key.to_hex(256) + - " is a msg_export_imm$010, so the OutMsgQueue entry with key " + q_key.to_hex() + - " should never be created, but it is present in either the old or the new output queue"); - } - // NB: if mode!=0, the OutMsgQueue entry has been changed, so we have already checked some conditions in precheck_one_message_queue_update() - if (mode & 2) { - if (q_entry.is_null()) { - return reject_query("OutMsg with key "s + key.to_hex(256) + - " was expected to create OutMsgQueue entry with key " + q_key.to_hex() + " but it did not"); + + if (tag == block::gen::OutMsg::msg_export_new_defer) { + // check the DispatchQueue update + if (old_q_entry.not_null() || q_entry.not_null()) { + return reject_query("OutMsg with key (message hash) "s + key.to_hex(256) + + " shouldn't exist in the old and the new message queues"); } - if (msg_env_hash != q_entry->prefetch_ref()->get_hash().bits()) { - return reject_query("OutMsg with key "s + key.to_hex(256) + " has created OutMsgQueue entry with key " + - q_key.to_hex() + " containing a different MsgEnvelope"); + auto it = new_dispatch_queue_messages_.find({src_addr, created_lt}); + if (it == new_dispatch_queue_messages_.end()) { + return reject_query(PSTRING() << "new deferred OutMsg with src_addr=" << src_addr.to_hex() + << ", lt=" << created_lt << " was not added to the dispatch queue"); } - // ... - } else if (mode & 1) { - if (old_q_entry.is_null()) { - return reject_query("OutMsg with key "s + key.to_hex(256) + - " was expected to remove OutMsgQueue entry with key " + q_key.to_hex() + - " but it did not exist in the old queue"); + Ref expected_msg_env = it->second; + if (expected_msg_env->get_hash() != msg_env->get_hash()) { + return reject_query(PSTRING() << "new deferred OutMsg with src_addr=" << src_addr.to_hex() << ", lt=" + << created_lt << " msg envelope hasg mismatch: " << msg_env->get_hash().to_hex() + << " in OutMsg, " << expected_msg_env->get_hash().to_hex() << " in DispatchQueue"); } - if (msg_env_hash != old_q_entry->prefetch_ref()->get_hash().bits()) { - return reject_query("OutMsg with key "s + key.to_hex(256) + " has dequeued OutMsgQueue entry with key " + - q_key.to_hex() + " containing a different MsgEnvelope"); + new_dispatch_queue_messages_.erase(it); + } else { + if (old_q_entry.not_null() && q_entry.not_null()) { + return reject_query("OutMsg with key (message hash) "s + key.to_hex(256) + + " should have removed or added OutMsgQueue entry with key " + q_key.to_hex() + + ", but it is present both in the old and in the new output queues"); + } + if (old_q_entry.is_null() && q_entry.is_null() && mode) { + return reject_query("OutMsg with key (message hash) "s + key.to_hex(256) + + " should have removed or added OutMsgQueue entry with key " + q_key.to_hex() + + ", but it is absent both from the old and from the new output queues"); + } + if (!mode && (old_q_entry.not_null() || q_entry.not_null())) { + return reject_query("OutMsg with key (message hash) "s + key.to_hex(256) + + " is a msg_export_imm$010, so the OutMsgQueue entry with key " + q_key.to_hex() + + " should never be created, but it is present in either the old or the new output queue"); + } + // NB: if mode!=0, the OutMsgQueue entry has been changed, so we have already checked some conditions in precheck_one_message_queue_update() + if (mode & 2) { + if (q_entry.is_null()) { + return reject_query("OutMsg with key "s + key.to_hex(256) + + " was expected to create OutMsgQueue entry with key " + q_key.to_hex() + " but it did not"); + } + if (msg_env_hash != q_entry->prefetch_ref()->get_hash().bits()) { + return reject_query("OutMsg with key "s + key.to_hex(256) + " has created OutMsgQueue entry with key " + + q_key.to_hex() + " containing a different MsgEnvelope"); + } + // ... + } else if (mode & 1) { + if (old_q_entry.is_null()) { + return reject_query("OutMsg with key "s + key.to_hex(256) + + " was expected to remove OutMsgQueue entry with key " + q_key.to_hex() + + " but it did not exist in the old queue"); + } + if (msg_env_hash != old_q_entry->prefetch_ref()->get_hash().bits()) { + return reject_query("OutMsg with key "s + key.to_hex(256) + " has dequeued OutMsgQueue entry with key " + + q_key.to_hex() + " containing a different MsgEnvelope"); + } + // ... } - // ... } // check reimport:^InMsg @@ -4090,8 +4507,8 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms int i_tag = block::gen::t_InMsg.get_tag(*in); if (i_tag < 0 || i_tag != in_tag) { return reject_query("OutMsg with key "s + key.to_hex(256) + - " refers to a (re)import InMsg, which is not one of msg_import_imm, msg_import_fin or " - "msg_import_tr as expected"); + " refers to a (re)import InMsg, which is not one of msg_import_imm, msg_import_fin, " + "msg_import_tr or msg_import_deferred_tr as expected"); } } @@ -4151,6 +4568,9 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms // ... break; } + case block::gen::OutMsg::msg_export_new_defer: { + break; + } case block::gen::OutMsg::msg_export_tr: { block::gen::InMsg::Record_msg_import_tr in; block::tlb::MsgEnvelope::Record_std in_env; @@ -4175,6 +4595,24 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms // ... break; } + case block::gen::OutMsg::msg_export_deferred_tr: { + block::gen::InMsg::Record_msg_import_deferred_tr in; + block::tlb::MsgEnvelope::Record_std in_env; + if (!(tlb::unpack_cell(reimport, in) && tlb::unpack_cell(in.in_msg, in_env))) { + return reject_query( + "cannot unpack msg_import_deferred_tr InMsg record corresponding to msg_export_deferred_tr OutMsg record with key "s + + key.to_hex(256)); + } + CHECK(in_env.msg->get_hash() == msg->get_hash()); + auto in_cur_prefix = block::interpolate_addr(src_prefix, dest_prefix, in_env.cur_addr); + if (!shard_contains(shard_, in_cur_prefix)) { + return reject_query( + "msg_export_deferred_tr OutMsg record with key "s + key.to_hex(256) + + " corresponds to msg_import_deferred_tr InMsg record with current imported message address " + + in_cur_prefix.to_str() + " NOT inside the current shard"); + } + break; + } case block::gen::OutMsg::msg_export_deq: case block::gen::OutMsg::msg_export_deq_short: { // check that the message has been indeed processed by a neighbor @@ -4290,6 +4728,24 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms return fatal_error(PSTRING() << "unknown OutMsg tag " << tag); } + if (tag == block::gen::OutMsg::msg_export_imm || tag == block::gen::OutMsg::msg_export_deq_imm || + tag == block::gen::OutMsg::msg_export_new || tag == block::gen::OutMsg::msg_export_deferred_tr) { + if (src_wc != workchain()) { + return true; + } + if (tag == block::gen::OutMsg::msg_export_imm && is_special_in_msg(vm::load_cell_slice(reimport))) { + return true; + } + unsigned long long created_lt; + auto cs = vm::load_cell_slice(env.msg); + if (!block::tlb::t_Message.get_created_lt(cs, created_lt)) { + return reject_query(PSTRING() << "cannot get created_lt for OutMsg with key " << key.to_hex(256) + << ", tag=" << tag); + } + auto emitted_lt = env.emitted_lt ? env.emitted_lt.value() : created_lt; + msg_emitted_lt_.emplace_back(src_addr, created_lt, emitted_lt); + } + return true; } @@ -4377,6 +4833,25 @@ bool ValidateQuery::check_processed_upto() { return true; } +/** + * Check that the difference between the old and new dispatch queues is reflected in OutMsgs and InMsgs + * + * @returns True if the check is successful, false otherwise. + */ +bool ValidateQuery::check_dispatch_queue_update() { + if (!new_dispatch_queue_messages_.empty()) { + auto it = new_dispatch_queue_messages_.begin(); + return reject_query(PSTRING() << "DispatchQueue has a new message with src_addr=" << it->first.first.to_hex() + << ", lt=" << it->first.second << ", but no correseponding OutMsg exists"); + } + if (!removed_dispatch_queue_messages_.empty()) { + auto it = removed_dispatch_queue_messages_.begin(); + return reject_query(PSTRING() << "message with src_addr=" << it->first.first.to_hex() << ", lt=" << it->first.second + << " was removed from DispatchQueue, but no correseponding InMsg exists"); + } + return true; +} + /** * Checks the validity of an outbound message in the neighbor's queue. * Similar to Collator::process_inbound_message. @@ -4691,6 +5166,10 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT // check input message block::CurrencyCollection money_imported(0), money_exported(0); bool is_special_tx = false; // recover/mint transaction + auto td_cs = vm::load_cell_slice(trans.description); + int tag = block::gen::t_TransactionDescr.get_tag(td_cs); + CHECK(tag >= 0); // we have already validated the serialization of all Transactions + td::optional in_msg_metadata; if (in_msg_root.not_null()) { auto in_descr_cs = in_msg_dict_->lookup(in_msg_root->get_hash().as_bitslice()); if (in_descr_cs.is_null()) { @@ -4698,20 +5177,21 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT << " of transaction " << lt << " of account " << addr.to_hex() << " does not have a corresponding InMsg record"); } - auto tag = block::gen::t_InMsg.get_tag(*in_descr_cs); - if (tag != block::gen::InMsg::msg_import_ext && tag != block::gen::InMsg::msg_import_fin && - tag != block::gen::InMsg::msg_import_imm && tag != block::gen::InMsg::msg_import_ihr) { + auto in_msg_tag = block::gen::t_InMsg.get_tag(*in_descr_cs); + if (in_msg_tag != block::gen::InMsg::msg_import_ext && in_msg_tag != block::gen::InMsg::msg_import_fin && + in_msg_tag != block::gen::InMsg::msg_import_imm && in_msg_tag != block::gen::InMsg::msg_import_ihr && + in_msg_tag != block::gen::InMsg::msg_import_deferred_fin) { return reject_query(PSTRING() << "inbound message with hash " << in_msg_root->get_hash().to_hex() << " of transaction " << lt << " of account " << addr.to_hex() << " has an invalid InMsg record (not one of msg_import_ext, msg_import_fin, " - "msg_import_imm or msg_import_ihr)"); + "msg_import_imm, msg_import_ihr or msg_import_deferred_fin)"); } is_special_tx = is_special_in_msg(*in_descr_cs); // once we know there is a InMsg with correct hash, we already know that it contains a message with this hash (by the verification of InMsg), so it is our message // have still to check its destination address and imported value // and that it refers to this transaction Ref dest; - if (tag == block::gen::InMsg::msg_import_ext) { + if (in_msg_tag == block::gen::InMsg::msg_import_ext) { block::gen::CommonMsgInfo::Record_ext_in_msg_info info; CHECK(tlb::unpack_cell_inexact(in_msg_root, info)); dest = std::move(info.dest); @@ -4724,12 +5204,26 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT << " processed inbound message created later at logical time " << info.created_lt); } + LogicalTime emitted_lt = info.created_lt; // See ValidateQuery::check_message_processing_order + if (in_msg_tag == block::gen::InMsg::msg_import_imm || in_msg_tag == block::gen::InMsg::msg_import_fin || + in_msg_tag == block::gen::InMsg::msg_import_deferred_fin) { + block::tlb::MsgEnvelope::Record_std msg_env; + if (!block::tlb::unpack_cell(in_descr_cs->prefetch_ref(), msg_env)) { + return reject_query(PSTRING() << "InMsg record for inbound message with hash " + << in_msg_root->get_hash().to_hex() << " of transaction " << lt + << " of account " << addr.to_hex() << " does not have a valid MsgEnvelope"); + } + in_msg_metadata = std::move(msg_env.metadata); + if (msg_env.emitted_lt) { + emitted_lt = msg_env.emitted_lt.value(); + } + } if (info.created_lt != start_lt_ || !is_special_tx) { - msg_proc_lt_.emplace_back(addr, lt, info.created_lt); + msg_proc_lt_.emplace_back(addr, lt, emitted_lt); } dest = std::move(info.dest); CHECK(money_imported.validate_unpack(info.value)); - ihr_delivered = (tag == block::gen::InMsg::msg_import_ihr); + ihr_delivered = (in_msg_tag == block::gen::InMsg::msg_import_ihr); if (!ihr_delivered) { money_imported += block::tlb::t_Grams.as_integer(info.ihr_fee); } @@ -4751,6 +5245,15 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT } } // check output messages + td::optional new_msg_metadata; + if (msg_metadata_enabled_) { + if (external || is_special_tx || tag != block::gen::TransactionDescr::trans_ord) { + new_msg_metadata = block::MsgMetadata{0, account.workchain, account.addr, (LogicalTime)trans.lt}; + } else if (in_msg_metadata) { + new_msg_metadata = std::move(in_msg_metadata); + ++new_msg_metadata.value().depth; + } + } vm::Dictionary out_dict{trans.r1.out_msgs, 15}; for (int i = 0; i < trans.outmsg_cnt; i++) { auto out_msg_root = out_dict.lookup_ref(td::BitArray<15>{i}); @@ -4763,33 +5266,45 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT } auto tag = block::gen::t_OutMsg.get_tag(*out_descr_cs); if (tag != block::gen::OutMsg::msg_export_ext && tag != block::gen::OutMsg::msg_export_new && - tag != block::gen::OutMsg::msg_export_imm) { - return reject_query( - PSTRING() << "outbound message #" << i + 1 << " with hash " << out_msg_root->get_hash().to_hex() - << " of transaction " << lt << " of account " << addr.to_hex() - << " has an invalid OutMsg record (not one of msg_export_ext, msg_export_new or msg_export_imm)"); + tag != block::gen::OutMsg::msg_export_imm && tag != block::gen::OutMsg::msg_export_new_defer) { + return reject_query(PSTRING() << "outbound message #" << i + 1 << " with hash " + << out_msg_root->get_hash().to_hex() << " of transaction " << lt << " of account " + << addr.to_hex() + << " has an invalid OutMsg record (not one of msg_export_ext, msg_export_new, " + "msg_export_imm or msg_export_new_defer)"); } - // once we know there is an OutMsg with correct hash, we already know that it contains a message with this hash (by the verification of OutMsg), so it is our message + // once we know there is an OutMsg with correct hash, we already know that it contains a message with this hash + // (by the verification of OutMsg), so it is our message // have still to check its source address, lt and imported value // and that it refers to this transaction as its origin Ref src; + LogicalTime message_lt; if (tag == block::gen::OutMsg::msg_export_ext) { block::gen::CommonMsgInfo::Record_ext_out_msg_info info; CHECK(tlb::unpack_cell_inexact(out_msg_root, info)); src = std::move(info.src); + message_lt = info.created_lt; } else { block::gen::CommonMsgInfo::Record_int_msg_info info; CHECK(tlb::unpack_cell_inexact(out_msg_root, info)); src = std::move(info.src); - block::gen::MsgEnvelope::Record msg_env; + message_lt = info.created_lt; + block::tlb::MsgEnvelope::Record_std msg_env; CHECK(tlb::unpack_cell(out_descr_cs->prefetch_ref(), msg_env)); // unpack exported message value (from this transaction) block::CurrencyCollection msg_export_value; CHECK(msg_export_value.unpack(info.value)); msg_export_value += block::tlb::t_Grams.as_integer(info.ihr_fee); - msg_export_value += block::tlb::t_Grams.as_integer(msg_env.fwd_fee_remaining); + msg_export_value += msg_env.fwd_fee_remaining; CHECK(msg_export_value.is_valid()); money_exported += msg_export_value; + if (msg_env.metadata != new_msg_metadata) { + return reject_query(PSTRING() << "outbound message #" << i + 1 << " with hash " + << out_msg_root->get_hash().to_hex() << " of transaction " << lt << " of account " + << addr.to_hex() << " has invalid metadata in an OutMsg record: expected " + << (new_msg_metadata ? new_msg_metadata.value().to_str() : "") << ", found " + << (msg_env.metadata ? msg_env.metadata.value().to_str() : "")); + } } WorkchainId s_wc; StdSmcAddress ss_addr; // s_addr is some macros in Windows @@ -4806,13 +5321,32 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT << out_msg_root->get_hash().to_hex() << " of transaction " << lt << " of account " << addr.to_hex() << " refers to a different processing transaction"); } + if (tag != block::gen::OutMsg::msg_export_ext) { + bool is_deferred = tag == block::gen::OutMsg::msg_export_new_defer; + if (account_expected_defer_all_messages_.count(ss_addr) && !is_deferred) { + return reject_query( + PSTRING() << "outbound message #" << i + 1 << " on account " << workchain() << ":" << ss_addr.to_hex() + << " must be deferred because this account has earlier messages in DispatchQueue"); + } + if (is_deferred) { + LOG(INFO) << "message from account " << workchain() << ":" << ss_addr.to_hex() << " with lt " << message_lt + << " was deferred"; + if (!deferring_messages_enabled_ && !account_expected_defer_all_messages_.count(ss_addr)) { + return reject_query(PSTRING() << "outbound message #" << i + 1 << " on account " << workchain() << ":" + << ss_addr.to_hex() << " is deferred, but deferring messages is disabled"); + } + if (i == 0 && !account_expected_defer_all_messages_.count(ss_addr)) { + return reject_query(PSTRING() << "outbound message #1 on account " << workchain() << ":" << ss_addr.to_hex() + << " must not be deferred (the first message cannot be deferred unless some " + "prevoius messages are deferred)"); + } + account_expected_defer_all_messages_.insert(ss_addr); + } + } } CHECK(money_exported.is_valid()); // check general transaction data block::CurrencyCollection old_balance{account.get_balance()}; - auto td_cs = vm::load_cell_slice(trans.description); - int tag = block::gen::t_TransactionDescr.get_tag(td_cs); - CHECK(tag >= 0); // we have already validated the serialization of all Transactions if (tag == block::gen::TransactionDescr::trans_merge_prepare || tag == block::gen::TransactionDescr::trans_merge_install || tag == block::gen::TransactionDescr::trans_split_prepare || @@ -5271,6 +5805,10 @@ bool ValidateQuery::check_all_ticktock_processed() { * @returns True if the processing order of messages is valid, false otherwise. */ bool ValidateQuery::check_message_processing_order() { + // Old rule: if messages m1 and m2 with the same destination generate transactions t1 and t2, + // then (m1.created_lt < m2.created_lt) => (t1.lt < t2.lt). + // New rule: + // If message was taken from dispatch queue, instead of created_lt use emitted_lt std::sort(msg_proc_lt_.begin(), msg_proc_lt_.end()); for (std::size_t i = 1; i < msg_proc_lt_.size(); i++) { auto &a = msg_proc_lt_[i - 1], &b = msg_proc_lt_[i]; @@ -5282,6 +5820,19 @@ bool ValidateQuery::check_message_processing_order() { << ") processes an earlier message created at logical time " << std::get<2>(b)); } } + + // Check that if messages m1 and m2 with the same source have m1.created_lt < m2.created_lt then + // m1.emitted_lt < m2.emitted_lt. + std::sort(msg_emitted_lt_.begin(), msg_emitted_lt_.end()); + for (std::size_t i = 1; i < msg_emitted_lt_.size(); i++) { + auto &a = msg_emitted_lt_[i - 1], &b = msg_emitted_lt_[i]; + if (std::get<0>(a) == std::get<0>(b) && std::get<2>(a) >= std::get<2>(b)) { + return reject_query(PSTRING() << "incorrect deferred message processing order for sender " + << std::get<0>(a).to_hex() << ": message with created_lt " << std::get<1>(a) + << " has emitted_lt" << std::get<2>(a) << ", but message with created_lt " + << std::get<1>(b) << " has emitted_lt" << std::get<2>(b)); + } + } return true; } @@ -6240,6 +6791,9 @@ bool ValidateQuery::try_validate() { if (!check_utime_lt()) { return reject_query("creation utime/lt of the new block is invalid"); } + if (!prepare_out_msg_queue_size()) { + return reject_query("cannot request out msg queue size"); + } stage_ = 1; if (pending) { return true; @@ -6268,12 +6822,18 @@ bool ValidateQuery::try_validate() { if (!precheck_message_queue_update()) { return reject_query("invalid OutMsgQueue update"); } + if (!unpack_dispatch_queue_update()) { + return reject_query("invalid DispatchQueue update"); + } if (!check_in_msg_descr()) { return reject_query("invalid InMsgDescr"); } if (!check_out_msg_descr()) { return reject_query("invalid OutMsgDescr"); } + if (!check_dispatch_queue_update()) { + return reject_query("invalid OutMsgDescr"); + } if (!check_processed_upto()) { return reject_query("invalid ProcessedInfo"); } @@ -6330,7 +6890,8 @@ bool ValidateQuery::save_candidate() { } }); - td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, id_, block_candidate.clone(), std::move(P)); + td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, id_, block_candidate.clone(), + validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), std::move(P)); return true; } diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 8829ac61f..824afb49d 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -112,7 +112,8 @@ class ValidateQuery : public td::actor::Actor { return SUPPORTED_VERSION; } static constexpr long long supported_capabilities() { - return ton::capCreateStatsEnabled | ton::capBounceMsgBody | ton::capReportVersion | ton::capShortDequeue; + return ton::capCreateStatsEnabled | ton::capBounceMsgBody | ton::capReportVersion | ton::capShortDequeue | + ton::capStoreOutMsgQueueSize | ton::capMsgMetadata | ton::capDeferMessages; } public: @@ -227,9 +228,22 @@ class ValidateQuery : public td::actor::Actor { bool inbound_queues_empty_{false}; std::vector> msg_proc_lt_; + std::vector> msg_emitted_lt_; std::vector> lib_publishers_, lib_publishers2_; + std::map, Ref> removed_dispatch_queue_messages_; + std::map, Ref> new_dispatch_queue_messages_; + std::set account_expected_defer_all_messages_; + td::uint64 old_out_msg_queue_size_ = 0, new_out_msg_queue_size_ = 0; + + bool msg_metadata_enabled_ = false; + bool deferring_messages_enabled_ = false; + bool store_out_msg_queue_size_ = false; + + td::uint64 processed_account_dispatch_queues_ = 0; + bool have_unprocessed_account_dispatch_queue_ = false; + td::PerfWarningTimer perf_timer_; static constexpr td::uint32 priority() { @@ -309,6 +323,8 @@ class ValidateQuery : public td::actor::Actor { bool check_cur_validator_set(); bool check_mc_validator_info(bool update_mc_cc); bool check_utime_lt(); + bool prepare_out_msg_queue_size(); + void got_out_queue_size(size_t i, td::Result res); bool fix_one_processed_upto(block::MsgProcessedUpto& proc, ton::ShardIdFull owner, bool allow_cur = false); bool fix_processed_upto(block::MsgProcessedUptoCollection& upto, bool allow_cur = false); @@ -330,6 +346,9 @@ class ValidateQuery : public td::actor::Actor { bool precheck_one_message_queue_update(td::ConstBitPtr out_msg_id, Ref old_value, Ref new_value); bool precheck_message_queue_update(); + bool check_account_dispatch_queue_update(td::Bits256 addr, Ref old_queue_csr, + Ref new_queue_csr); + bool unpack_dispatch_queue_update(); bool update_max_processed_lt_hash(ton::LogicalTime lt, const ton::Bits256& hash); bool update_min_enqueued_lt_hash(ton::LogicalTime lt, const ton::Bits256& hash); bool check_imported_message(Ref msg_env); @@ -338,6 +357,7 @@ class ValidateQuery : public td::actor::Actor { bool check_in_msg_descr(); bool check_out_msg(td::ConstBitPtr key, Ref out_msg); bool check_out_msg_descr(); + bool check_dispatch_queue_update(); bool check_processed_upto(); bool check_neighbor_outbound_message(Ref enq_msg, ton::LogicalTime lt, td::ConstBitPtr key, const block::McShardDescr& src_nb, bool& unprocessed); diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index 84ea2b366..8bbf7f31f 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -66,6 +66,8 @@ class Db : public td::actor::Actor { virtual void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) = 0; virtual void get_zero_state_file(BlockIdExt block_id, td::Promise promise) = 0; virtual void check_zero_state_file_exists(BlockIdExt block_id, td::Promise promise) = 0; + virtual void get_previous_persistent_state_files( + BlockSeqno cur_mc_seqno, td::Promise>> promise) = 0; virtual void try_get_static_file(FileHash file_hash, td::Promise promise) = 0; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 41412fb17..0e9fab73b 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -93,7 +93,8 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void wait_block_signatures_short(BlockIdExt id, td::Timestamp timeout, td::Promise> promise) = 0; - virtual void set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) = 0; + virtual void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) = 0; virtual void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) = 0; @@ -133,7 +134,7 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void send_external_message(td::Ref message) = 0; virtual void send_ihr_message(td::Ref message) = 0; virtual void send_top_shard_block_description(td::Ref desc) = 0; - virtual void send_block_broadcast(BlockBroadcast broadcast) = 0; + virtual void send_block_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) = 0; virtual void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_shard_client_state(bool from_db, td::Promise promise) = 0; @@ -171,6 +172,7 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) = 0; virtual void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) = 0; + virtual void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) = 0; virtual void get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) = 0; virtual void get_block_data_for_litequery(BlockIdExt block_id, td::Promise> promise) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 42e440811..5678408c6 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -128,8 +128,8 @@ void ValidatorManagerImpl::sync_complete(td::Promise promise) { } Ed25519_PublicKey created_by{td::Bits256::zero()}; td::as(created_by.as_bits256().data() + 32 - 4) = ((unsigned)std::time(nullptr) >> 8); - run_collate_query(shard_id, 0, last_masterchain_block_id_, prev, created_by, val_set, actor_id(this), - td::Timestamp::in(10.0), std::move(P)); + run_collate_query(shard_id, 0, last_masterchain_block_id_, prev, created_by, val_set, td::Ref{true}, + actor_id(this), td::Timestamp::in(10.0), std::move(P)); } void ValidatorManagerImpl::validate_fake(BlockCandidate candidate, std::vector prev, BlockIdExt last, @@ -775,7 +775,8 @@ void ValidatorManagerImpl::set_next_block(BlockIdExt block_id, BlockIdExt next, get_block_handle(block_id, true, std::move(P)); } -void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) { +void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) { td::actor::send_closure(db_, &Db::store_block_candidate, std::move(candidate), std::move(promise)); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index d5a6e909f..a77be2725 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -116,6 +116,10 @@ class ValidatorManagerImpl : public ValidatorManager { td::int64 max_length, td::Promise promise) override { UNREACHABLE(); } + void get_previous_persistent_state_files( + BlockSeqno cur_mc_seqno, td::Promise>> promise) override { + UNREACHABLE(); + } void get_block_proof(BlockHandle handle, td::Promise promise) override; void get_block_proof_link(BlockHandle block_id, td::Promise promise) override { UNREACHABLE(); @@ -130,6 +134,8 @@ class ValidatorManagerImpl : public ValidatorManager { } void new_ihr_message(td::BufferSlice data) override; void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; + void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) override { + } void add_ext_server_id(adnl::AdnlNodeIdShort id) override { UNREACHABLE(); @@ -177,7 +183,8 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_block_signatures_short(BlockIdExt id, td::Timestamp timeout, td::Promise> promise) override; - void set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) override; + void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) override; void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; @@ -251,7 +258,7 @@ class ValidatorManagerImpl : public ValidatorManager { new_ihr_message(message->serialize()); } void send_top_shard_block_description(td::Ref desc) override; - void send_block_broadcast(BlockBroadcast broadcast) override { + void send_block_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) override { } void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; @@ -378,7 +385,10 @@ class ValidatorManagerImpl : public ValidatorManager { void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override { UNREACHABLE(); } - void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { + void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override { + UNREACHABLE(); + } + void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { queue_size_counter_ = td::actor::create_actor("queuesizecounter", td::Ref{}, actor_id(this)); @@ -416,6 +426,10 @@ class ValidatorManagerImpl : public ValidatorManager { promise.set_result(td::Status::Error("not implemented")); } + void update_options(td::Ref opts) override { + opts_ = std::move(opts); + } + private: PublicKeyHash local_id_; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 7937729ce..e7175b77b 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -139,6 +139,10 @@ class ValidatorManagerImpl : public ValidatorManager { td::int64 max_length, td::Promise promise) override { UNREACHABLE(); } + void get_previous_persistent_state_files( + BlockSeqno cur_mc_seqno, td::Promise>> promise) override { + UNREACHABLE(); + } void get_block_proof(BlockHandle handle, td::Promise promise) override; void get_block_proof_link(BlockHandle block_id, td::Promise promise) override; void get_key_block_proof(BlockIdExt block_id, td::Promise promise) override; @@ -152,6 +156,9 @@ class ValidatorManagerImpl : public ValidatorManager { void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override { UNREACHABLE(); } + void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) override { + UNREACHABLE(); + } void add_ext_server_id(adnl::AdnlNodeIdShort id) override { UNREACHABLE(); @@ -215,7 +222,8 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_block_signatures_short(BlockIdExt id, td::Timestamp timeout, td::Promise> promise) override; - void set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) override { + void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) override { promise.set_value(td::Unit()); } @@ -317,7 +325,7 @@ class ValidatorManagerImpl : public ValidatorManager { void send_top_shard_block_description(td::Ref desc) override { UNREACHABLE(); } - void send_block_broadcast(BlockBroadcast broadcast) override { + void send_block_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) override { } void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override { @@ -439,7 +447,10 @@ class ValidatorManagerImpl : public ValidatorManager { void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override { UNREACHABLE(); } - void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { + void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override { + UNREACHABLE(); + } + void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { queue_size_counter_ = td::actor::create_actor("queuesizecounter", td::Ref{}, actor_id(this)); @@ -476,6 +487,9 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override { promise.set_result(td::Status::Error("not implemented")); } + void update_options(td::Ref opts) override { + opts_ = std::move(opts); + } private: td::Ref opts_; diff --git a/validator/manager.cpp b/validator/manager.cpp index e186902ad..eb082d91e 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -17,6 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "manager.hpp" +#include "checksum.h" +#include "td/utils/buffer.h" #include "validator-group.hpp" #include "adnl/utils.hpp" #include "downloaders/wait-block-state.hpp" @@ -306,6 +308,11 @@ void ValidatorManagerImpl::get_persistent_state_slice(BlockIdExt block_id, Block std::move(promise)); } +void ValidatorManagerImpl::get_previous_persistent_state_files( + BlockSeqno cur_mc_seqno, td::Promise>> promise) { + td::actor::send_closure(db_, &Db::get_previous_persistent_state_files, cur_mc_seqno, std::move(promise)); +} + void ValidatorManagerImpl::get_block_proof(BlockHandle handle, td::Promise promise) { auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result> R) mutable { if (R.is_error()) { @@ -410,14 +417,42 @@ void ValidatorManagerImpl::add_external_message(td::Ref msg, int pri ext_messages_hashes_[id.hash] = {priority, id}; } void ValidatorManagerImpl::check_external_message(td::BufferSlice data, td::Promise> promise) { - ++ls_stats_check_ext_messages_; auto state = do_get_last_liteserver_state(); if (state.is_null()) { promise.set_error(td::Status::Error(ErrorCode::notready, "not ready")); return; } - run_check_external_message(std::move(data), state->get_ext_msg_limits(), actor_id(this), - std::move(promise)); + auto R = create_ext_message(std::move(data), state->get_ext_msg_limits()); + if (R.is_error()) { + promise.set_error(R.move_as_error_prefix("failed to parse external message: ")); + return; + } + auto message = R.move_as_ok(); + WorkchainId wc = message->wc(); + StdSmcAddress addr = message->addr(); + if (checked_ext_msg_counter_.get_msg_count(wc, addr) >= max_ext_msg_per_addr()) { + promise.set_error( + td::Status::Error(PSTRING() << "too many external messages to address " << wc << ":" << addr.to_hex())); + return; + } + + promise = [self = this, wc, addr, promise = std::move(promise), + SelfId = actor_id(this)](td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + return; + } + td::actor::send_lambda(SelfId, [=, promise = std::move(promise), message = R.move_as_ok()]() mutable { + if (self->checked_ext_msg_counter_.inc_msg_count(wc, addr) > max_ext_msg_per_addr()) { + promise.set_error( + td::Status::Error(PSTRING() << "too many external messages to address " << wc << ":" << addr.to_hex())); + return; + } + promise.set_result(std::move(message)); + }); + }; + ++ls_stats_check_ext_messages_; + run_check_external_message(std::move(message), actor_id(this), std::move(promise)); } void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { @@ -464,6 +499,17 @@ void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc actor_id(this), td::Timestamp::in(2.0), std::move(P)); } +void ValidatorManagerImpl::new_block_candidate(BlockIdExt block_id, td::BufferSlice data) { + if (!last_masterchain_block_handle_) { + VLOG(VALIDATOR_DEBUG) << "dropping top shard block broadcast: not inited"; + return; + } + if (!started_) { + return; + } + add_cached_block_candidate(ReceivedBlock{block_id, std::move(data)}); +} + void ValidatorManagerImpl::add_shard_block_description(td::Ref desc) { if (desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { auto it = shard_blocks_.find(ShardTopBlockDescriptionId{desc->shard(), desc->catchain_seqno()}); @@ -495,6 +541,36 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refsecond.actor_, &WaitBlockData::got_block_data_from_net, r_block.move_as_ok()); + } + } + } + { + auto it = wait_state_.find(id); + if (it != wait_state_.end()) { + // Proof link is not ready at this point, but this will force WaitBlockState to redo send_get_proof_link_request + td::actor::send_closure(it->second.actor_, &WaitBlockState::after_get_proof_link); + } + } + } + if (cached_block_candidates_lru_.size() > max_cached_candidates()) { + CHECK(cached_block_candidates_.erase(cached_block_candidates_lru_.front())); + cached_block_candidates_lru_.pop_front(); + } +} + void ValidatorManagerImpl::add_ext_server_id(adnl::AdnlNodeIdShort id) { class Cb : public adnl::Adnl::Callback { private: @@ -1189,11 +1265,16 @@ void ValidatorManagerImpl::set_next_block(BlockIdExt block_id, BlockIdExt next, get_block_handle(block_id, true, std::move(P)); } -void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) { +void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) { if (!candidates_buffer_.empty()) { td::actor::send_closure(candidates_buffer_, &CandidatesBuffer::add_new_candidate, id, PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}, candidate.collated_file_hash); } + if (!id.is_masterchain()) { + add_cached_block_candidate(ReceivedBlock{id, candidate.data.clone()}); + callback_->send_block_candidate(id, cc_seqno, validator_set_hash, candidate.data.clone()); + } td::actor::send_closure(db_, &Db::store_block_candidate, std::move(candidate), std::move(promise)); } @@ -1450,6 +1531,13 @@ void ValidatorManagerImpl::get_last_liteserver_state_block( void ValidatorManagerImpl::send_get_block_request(BlockIdExt id, td::uint32 priority, td::Promise promise) { + { + auto it = cached_block_candidates_.find(id); + if (it != cached_block_candidates_.end()) { + LOG(DEBUG) << "send_get_block_request: got result from candidates cache for " << id.to_str(); + return promise.set_value(it->second.clone()); + } + } callback_->download_block(id, priority, td::Timestamp::in(10.0), std::move(promise)); } @@ -1472,6 +1560,20 @@ void ValidatorManagerImpl::send_get_block_proof_request(BlockIdExt block_id, td: void ValidatorManagerImpl::send_get_block_proof_link_request(BlockIdExt block_id, td::uint32 priority, td::Promise promise) { + if (!block_id.is_masterchain()) { + auto it = cached_block_candidates_.find(block_id); + if (it != cached_block_candidates_.end()) { + // Proof link can be created from the cached block candidate + LOG(DEBUG) << "send_get_block_proof_link_request: creating proof link from cached caniddate for " + << block_id.to_str(); + TRY_RESULT_PROMISE_PREFIX(promise, block_root, vm::std_boc_deserialize(it->second.data), + "failed to create proof link: "); + TRY_RESULT_PROMISE_PREFIX(promise, proof_link, WaitBlockData::generate_proof_link(it->second.id, block_root), + "failed to create proof link: "); + promise.set_result(std::move(proof_link)); + return; + } + } callback_->download_block_proof_link(block_id, priority, td::Timestamp::in(10.0), std::move(promise)); } @@ -1503,8 +1605,8 @@ void ValidatorManagerImpl::send_top_shard_block_description(td::Refsend_broadcast(std::move(broadcast)); +void ValidatorManagerImpl::send_block_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) { + callback_->send_broadcast(std::move(broadcast), custom_overlays_only); } void ValidatorManagerImpl::start_up() { @@ -1617,6 +1719,8 @@ void ValidatorManagerImpl::read_gc_list(std::vector list) { serializer_ = td::actor::create_actor("serializer", last_key_block_handle_->id(), opts_, actor_id(this)); + td::actor::send_closure(serializer_, &AsyncStateSerializer::update_last_known_key_block_ts, + last_key_block_handle_->unix_time()); if (last_masterchain_block_handle_->inited_next_left()) { auto b = last_masterchain_block_handle_->one_next(true); @@ -1806,6 +1910,10 @@ void ValidatorManagerImpl::new_masterchain_block() { last_known_key_block_handle_ = last_key_block_handle_; callback_->new_key_block(last_key_block_handle_); } + if (!serializer_.empty()) { + td::actor::send_closure(serializer_, &AsyncStateSerializer::update_last_known_key_block_ts, + last_key_block_handle_->unix_time()); + } } update_shards(); @@ -2135,7 +2243,7 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group auto G = td::actor::create_actor( "validatorgroup", shard, validator_id, session_id, validator_set, opts, keyring_, adnl_, rldp_, overlays_, db_root_, actor_id(this), init_session, - opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno())); + opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_); return G; } } @@ -2418,9 +2526,9 @@ void ValidatorManagerImpl::state_serializer_update(BlockSeqno seqno) { void ValidatorManagerImpl::alarm() { try_advance_gc_masterchain_block(); alarm_timestamp() = td::Timestamp::in(1.0); - if (last_masterchain_block_handle_ && gc_masterchain_handle_) { - td::actor::send_closure(db_, &Db::run_gc, last_masterchain_block_handle_->unix_time(), - gc_masterchain_handle_->unix_time(), static_cast(opts_->archive_ttl())); + if (shard_client_handle_ && gc_masterchain_handle_) { + td::actor::send_closure(db_, &Db::run_gc, shard_client_handle_->unix_time(), gc_masterchain_handle_->unix_time(), + static_cast(opts_->archive_ttl())); } if (log_status_at_.is_in_past()) { if (last_masterchain_block_handle_) { @@ -2523,6 +2631,16 @@ void ValidatorManagerImpl::alarm() { log_ls_stats_at_ = td::Timestamp::in(60.0); } alarm_timestamp().relax(log_ls_stats_at_); + if (cleanup_mempool_at_.is_in_past()) { + if (is_validator()) { + get_external_messages(ShardIdFull{masterchainId, shardIdAll}, + [](td::Result, int>>>) {}); + get_external_messages(ShardIdFull{basechainId, shardIdAll}, + [](td::Result, int>>>) {}); + } + cleanup_mempool_at_ = td::Timestamp::in(250.0); + } + alarm_timestamp().relax(cleanup_mempool_at_); } void ValidatorManagerImpl::update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) { @@ -2714,8 +2832,12 @@ void ValidatorManagerImpl::log_validator_session_stats(BlockIdExt block_id, std::vector> producers; for (const auto &producer : round.producers) { producers.push_back(create_tl_object( - producer.id.bits256_value(), producer.candidate_id, producer.block_status, producer.block_timestamp, - producer.comment)); + producer.id.bits256_value(), producer.candidate_id, producer.block_status, producer.comment, + producer.block_timestamp, producer.is_accepted, producer.is_ours, producer.got_submit_at, + producer.collation_time, producer.collated_at, producer.collation_cached, producer.validation_time, + producer.validated_at, producer.validation_cached, producer.gen_utime, producer.approved_weight, + producer.approved_33pct_at, producer.approved_66pct_at, producer.signed_weight, producer.signed_33pct_at, + producer.signed_66pct_at, producer.serialize_time, producer.deserialize_time, producer.serialized_size)); } rounds.push_back(create_tl_object(round.timestamp, std::move(producers))); } @@ -2725,7 +2847,7 @@ void ValidatorManagerImpl::log_validator_session_stats(BlockIdExt block_id, stats.cc_seqno, stats.creator.bits256_value(), stats.total_validators, stats.total_weight, stats.signatures, stats.signatures_weight, stats.approve_signatures, stats.approve_signatures_weight, stats.first_round, std::move(rounds)); - std::string s = td::json_encode(td::ToJson(*obj.get()), false); + auto s = td::json_encode(td::ToJson(*obj.get()), false); s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); std::ofstream file; @@ -2733,7 +2855,31 @@ void ValidatorManagerImpl::log_validator_session_stats(BlockIdExt block_id, file << s << "\n"; file.close(); - LOG(INFO) << "Writing validator session stats for " << block_id.id; + LOG(INFO) << "Writing validator session stats for " << block_id.id.to_str(); +} + +void ValidatorManagerImpl::log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) { + std::string fname = opts_->get_session_logs_file(); + if (fname.empty()) { + return; + } + std::vector> nodes; + for (const auto &node : stats.nodes) { + nodes.push_back( + create_tl_object(node.id.bits256_value(), node.weight)); + } + auto obj = create_tl_object( + stats.session_id, stats.shard.workchain, stats.shard.shard, stats.cc_seqno, stats.timestamp, stats.self_idx, + std::move(nodes)); + auto s = td::json_encode(td::ToJson(*obj.get()), false); + s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); + + std::ofstream file; + file.open(fname, std::ios_base::app); + file << s << "\n"; + file.close(); + + LOG(INFO) << "Writing new validator group stats for " << stats.shard.to_str(); } void ValidatorManagerImpl::get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) { @@ -2997,6 +3143,20 @@ void ValidatorManagerImpl::get_validator_groups_info_for_litequery( td::actor::create_actor("get-validator-groups-info", std::move(groups), std::move(promise)).release(); } +void ValidatorManagerImpl::update_options(td::Ref opts) { + // Currently options can be updated only to change state_serializer_enabled flag and collator_options + if (!serializer_.empty()) { + td::actor::send_closure(serializer_, &AsyncStateSerializer::update_options, opts); + } + for (auto &group : validator_groups_) { + td::actor::send_closure(group.second.actor, &ValidatorGroup::update_options, opts); + } + for (auto &group : next_validator_groups_) { + td::actor::send_closure(group.second.actor, &ValidatorGroup::update_options, opts); + } + opts_ = std::move(opts); +} + td::actor::ActorOwn ValidatorManagerFactory::create( td::Ref opts, std::string db_root, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, @@ -3005,6 +3165,29 @@ td::actor::ActorOwn ValidatorManagerFactory::create( rldp, overlays); } +size_t ValidatorManagerImpl::CheckedExtMsgCounter::get_msg_count(WorkchainId wc, StdSmcAddress addr) { + before_query(); + auto it1 = counter_cur_.find({wc, addr}); + auto it2 = counter_prev_.find({wc, addr}); + return (it1 == counter_cur_.end() ? 0 : it1->second) + (it2 == counter_prev_.end() ? 0 : it2->second); +} +size_t ValidatorManagerImpl::CheckedExtMsgCounter::inc_msg_count(WorkchainId wc, StdSmcAddress addr) { + before_query(); + auto it2 = counter_prev_.find({wc, addr}); + return (it2 == counter_prev_.end() ? 0 : it2->second) + ++counter_cur_[{wc, addr}]; +} +void ValidatorManagerImpl::CheckedExtMsgCounter::before_query() { + while (cleanup_at_.is_in_past()) { + counter_prev_ = std::move(counter_cur_); + counter_cur_.clear(); + if (counter_prev_.empty()) { + cleanup_at_ = td::Timestamp::in(max_ext_msg_per_addr_time_window() / 2.0); + break; + } + cleanup_at_ += max_ext_msg_per_addr_time_window() / 2.0; + } +} + } // namespace validator } // namespace ton diff --git a/validator/manager.hpp b/validator/manager.hpp index 7e5930d37..12354c634 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -18,10 +18,14 @@ */ #pragma once +#include "common/refcnt.hpp" #include "interfaces/validator-manager.h" #include "interfaces/db.h" #include "td/actor/PromiseFuture.h" +#include "td/utils/SharedSlice.h" +#include "td/utils/buffer.h" #include "td/utils/port/Poll.h" +#include "td/utils/port/StdStreams.h" #include "validator-group.hpp" #include "shard-client.hpp" #include "manager-init.h" @@ -220,6 +224,10 @@ class ValidatorManagerImpl : public ValidatorManager { }; // DATA FOR COLLATOR std::map> shard_blocks_; + + std::map cached_block_candidates_; + std::list cached_block_candidates_lru_; + struct ExtMessages { std::map, std::unique_ptr>> ext_messages_; std::map, std::map>> @@ -233,10 +241,20 @@ class ValidatorManagerImpl : public ValidatorManager { }; std::map ext_msgs_; // priority -> messages std::map>> ext_messages_hashes_; // hash -> priority + td::Timestamp cleanup_mempool_at_; // IHR ? std::map, std::unique_ptr>> ihr_messages_; std::map> ihr_messages_hashes_; + struct CheckedExtMsgCounter { + std::map, size_t> counter_cur_, counter_prev_; + td::Timestamp cleanup_at_ = td::Timestamp::now(); + + size_t get_msg_count(WorkchainId wc, StdSmcAddress addr); + size_t inc_msg_count(WorkchainId wc, StdSmcAddress addr); + void before_query(); + } checked_ext_msg_counter_; + private: // VALIDATOR GROUPS ValidatorSessionId get_validator_set_id(ShardIdFull shard, td::Ref val_set, td::Bits256 opts_hash, @@ -353,6 +371,8 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise promise) override; void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, td::int64 max_length, td::Promise promise) override; + void get_previous_persistent_state_files( + BlockSeqno cur_mc_seqno, td::Promise>> promise) override; void get_block_proof(BlockHandle handle, td::Promise promise) override; void get_block_proof_link(BlockHandle block_id, td::Promise promise) override; void get_key_block_proof(BlockIdExt block_id, td::Promise promise) override; @@ -365,6 +385,7 @@ class ValidatorManagerImpl : public ValidatorManager { void new_ihr_message(td::BufferSlice data) override; void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; + void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) override; void add_ext_server_id(adnl::AdnlNodeIdShort id) override; void add_ext_server_port(td::uint16 port) override; @@ -409,7 +430,8 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_block_signatures_short(BlockIdExt id, td::Timestamp timeout, td::Promise> promise) override; - void set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) override; + void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) override; void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; @@ -473,7 +495,7 @@ class ValidatorManagerImpl : public ValidatorManager { void send_external_message(td::Ref message) override; void send_ihr_message(td::Ref message) override; void send_top_shard_block_description(td::Ref desc) override; - void send_block_broadcast(BlockBroadcast broadcast) override; + void send_block_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) override; void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; @@ -503,6 +525,7 @@ class ValidatorManagerImpl : public ValidatorManager { } void add_shard_block_description(td::Ref desc); + void add_cached_block_candidate(ReceivedBlock block); void register_block_handle(BlockHandle handle); @@ -565,8 +588,11 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) override; void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override; + void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override; - void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { + void update_options(td::Ref opts) override; + + void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { if (last_masterchain_state_.is_null()) { promise.set_error(td::Status::Error(ErrorCode::notready, "not ready")); @@ -663,6 +689,15 @@ class ValidatorManagerImpl : public ValidatorManager { double max_mempool_num() const { return opts_->max_mempool_num(); } + size_t max_cached_candidates() const { + return 128; + } + static double max_ext_msg_per_addr_time_window() { + return 10.0; + } + static size_t max_ext_msg_per_addr() { + return 3 * 10; + } private: std::map> shard_client_waiters_; diff --git a/validator/queue-size-counter.cpp b/validator/queue-size-counter.cpp index 4780f202c..eb8580894 100644 --- a/validator/queue-size-counter.cpp +++ b/validator/queue-size-counter.cpp @@ -23,8 +23,8 @@ namespace ton::validator { -static td::Result calc_queue_size(const td::Ref &state) { - td::uint32 size = 0; +static td::Result calc_queue_size(const td::Ref &state) { + td::uint64 size = 0; TRY_RESULT(outq_descr, state->message_queue()); block::gen::OutMsgQueueInfo::Record qinfo; if (!tlb::unpack_cell(outq_descr->root_cell(), qinfo)) { @@ -41,8 +41,8 @@ static td::Result calc_queue_size(const td::Ref &state) return size; } -static td::Result recalc_queue_size(const td::Ref &state, const td::Ref &prev_state, - td::uint32 prev_size) { +static td::Result recalc_queue_size(const td::Ref &state, const td::Ref &prev_state, + td::uint64 prev_size) { TRY_RESULT(outq_descr, state->message_queue()); block::gen::OutMsgQueueInfo::Record qinfo; if (!tlb::unpack_cell(outq_descr->root_cell(), qinfo)) { @@ -56,7 +56,7 @@ static td::Result recalc_queue_size(const td::Ref &state return td::Status::Error("invalid message queue"); } vm::AugmentedDictionary prev_queue{prev_qinfo.out_queue->prefetch_ref(0), 352, block::tlb::aug_OutMsgQueue}; - td::uint32 add = 0, rem = 0; + td::uint64 add = 0, rem = 0; bool ok = prev_queue.scan_diff( queue, [&](td::ConstBitPtr, int, td::Ref prev_val, td::Ref new_val) -> bool { if (prev_val.not_null()) { @@ -88,11 +88,11 @@ void QueueSizeCounter::start_up() { alarm(); } -void QueueSizeCounter::get_queue_size(BlockIdExt block_id, td::Promise promise) { +void QueueSizeCounter::get_queue_size(BlockIdExt block_id, td::Promise promise) { get_queue_size_ex(block_id, simple_mode_ || is_block_too_old(block_id), std::move(promise)); } -void QueueSizeCounter::get_queue_size_ex(ton::BlockIdExt block_id, bool calc_whole, td::Promise promise) { +void QueueSizeCounter::get_queue_size_ex(ton::BlockIdExt block_id, bool calc_whole, td::Promise promise) { Entry &entry = results_[block_id]; if (entry.done_) { promise.set_result(entry.queue_size_); @@ -152,12 +152,12 @@ void QueueSizeCounter::get_queue_size_cont(BlockHandle handle, td::Refone_prev(true); - get_queue_size(prev_block_id, [=, SelfId = actor_id(this), manager = manager_](td::Result R) { + get_queue_size(prev_block_id, [=, SelfId = actor_id(this), manager = manager_](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &QueueSizeCounter::on_error, state->get_block_id(), R.move_as_error()); return; } - td::uint32 prev_size = R.move_as_ok(); + td::uint64 prev_size = R.move_as_ok(); td::actor::send_closure( manager, &ValidatorManager::wait_block_state_short, prev_block_id, 0, td::Timestamp::in(10.0), [=](td::Result> R) { @@ -171,7 +171,7 @@ void QueueSizeCounter::get_queue_size_cont(BlockHandle handle, td::Ref state, td::Ref prev_state, - td::uint32 prev_size) { + td::uint64 prev_size) { BlockIdExt block_id = state->get_block_id(); Entry &entry = results_[block_id]; CHECK(entry.started_); @@ -252,7 +252,7 @@ void QueueSizeCounter::process_top_shard_blocks_cont(td::Ref s void QueueSizeCounter::get_queue_size_ex_retry(BlockIdExt block_id, bool calc_whole, td::Promise promise) { get_queue_size_ex(block_id, calc_whole, - [=, promise = std::move(promise), SelfId = actor_id(this)](td::Result R) mutable { + [=, promise = std::move(promise), SelfId = actor_id(this)](td::Result R) mutable { if (R.is_error()) { LOG(WARNING) << "Failed to calculate queue size for block " << block_id.to_str() << ": " << R.move_as_error(); diff --git a/validator/queue-size-counter.hpp b/validator/queue-size-counter.hpp index fabb0cec3..4825a43c0 100644 --- a/validator/queue-size-counter.hpp +++ b/validator/queue-size-counter.hpp @@ -26,7 +26,7 @@ class QueueSizeCounter : public td::actor::Actor { } void start_up() override; - void get_queue_size(BlockIdExt block_id, td::Promise promise); + void get_queue_size(BlockIdExt block_id, td::Promise promise); void alarm() override; private: @@ -42,14 +42,14 @@ class QueueSizeCounter : public td::actor::Actor { bool started_ = false; bool done_ = false; bool calc_whole_ = false; - td::uint32 queue_size_ = 0; - std::vector> promises_; + td::uint64 queue_size_ = 0; + std::vector> promises_; }; std::map results_; - void get_queue_size_ex(BlockIdExt block_id, bool calc_whole, td::Promise promise); + void get_queue_size_ex(BlockIdExt block_id, bool calc_whole, td::Promise promise); void get_queue_size_cont(BlockHandle handle, td::Ref state); - void get_queue_size_cont2(td::Ref state, td::Ref prev_state, td::uint32 prev_size); + void get_queue_size_cont2(td::Ref state, td::Ref prev_state, td::uint64 prev_size); void on_error(BlockIdExt block_id, td::Status error); void process_top_shard_blocks(); diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index dfb09663e..b6fb43469 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -22,12 +22,16 @@ #include "ton/ton-io.hpp" #include "common/delay.h" #include "vm/large-boc-serializer.h" +#include "td/utils/filesystem.h" namespace ton { namespace validator { void AsyncStateSerializer::start_up() { + if (!opts_->get_state_serializer_enabled()) { + LOG(ERROR) << "Persistent state serializer is disabled"; + } alarm_timestamp() = td::Timestamp::in(1.0 + td::Random::fast(0, 10) * 1.0); running_ = true; @@ -82,6 +86,20 @@ void AsyncStateSerializer::alarm() { td::actor::send_closure(manager_, &ValidatorManager::get_top_masterchain_block, std::move(P)); } +void AsyncStateSerializer::request_previous_state_files() { + td::actor::send_closure( + manager_, &ValidatorManager::get_previous_persistent_state_files, masterchain_handle_->id().seqno(), + [SelfId = actor_id(this)](td::Result>> R) { + R.ensure(); + td::actor::send_closure(SelfId, &AsyncStateSerializer::got_previous_state_files, R.move_as_ok()); + }); +} + +void AsyncStateSerializer::got_previous_state_files(std::vector> files) { + previous_state_files_ = std::move(files); + request_masterchain_state(); +} + void AsyncStateSerializer::request_masterchain_state() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), manager = manager_](td::Result> R) { if (R.is_error()) { @@ -132,30 +150,42 @@ void AsyncStateSerializer::next_iteration() { CHECK(masterchain_handle_->id() == last_block_id_); if (attempt_ < max_attempt() && last_key_block_id_.id.seqno < last_block_id_.id.seqno && need_serialize(masterchain_handle_)) { - if (!have_masterchain_state_) { - LOG(INFO) << "started serializing persistent state for " << masterchain_handle_->id().id; - // block next attempts immediately, but send actual request later - running_ = true; - delay_action([SelfId = actor_id( - this)]() { td::actor::send_closure(SelfId, &AsyncStateSerializer::request_masterchain_state); }, - td::Timestamp::in(td::Random::fast(0, 3600))); - return; - } - while (next_idx_ < shards_.size()) { - if (!need_monitor(shards_[next_idx_].shard_full())) { - next_idx_++; - } else { - // block next attempts immediately, but send actual request later - running_ = true; - delay_action( - [SelfId = actor_id(this), shard = shards_[next_idx_]]() { td::actor::send_closure(SelfId, &AsyncStateSerializer::request_shard_state, shard); }, - td::Timestamp::in(td::Random::fast(0, 1800))); + if (!have_masterchain_state_ && !opts_->get_state_serializer_enabled()) { + LOG(ERROR) << "skipping serializing persistent state for " << masterchain_handle_->id().id.to_str() + << ": serializer is disabled"; + } else if (!have_masterchain_state_ && have_newer_persistent_state(masterchain_handle_->unix_time())) { + LOG(ERROR) << "skipping serializing persistent state for " << masterchain_handle_->id().id.to_str() + << ": newer key block with ts=" << last_known_key_block_ts_ << " exists"; + } else { + if (!have_masterchain_state_) { + LOG(ERROR) << "started serializing persistent state for " << masterchain_handle_->id().id.to_str(); + // block next attempts immediately, but send actual request later + running_ = true; + double delay = td::Random::fast(0, 3600); + LOG(WARNING) << "serializer delay = " << delay << "s"; + delay_action( + [SelfId = actor_id(this)]() { + td::actor::send_closure(SelfId, &AsyncStateSerializer::request_previous_state_files); + }, + td::Timestamp::in(delay)); return; } + while (next_idx_ < shards_.size()) { + if (!need_monitor(shards_[next_idx_].shard_full())) { + next_idx_++; + } else { + running_ = true; + request_shard_state(shards_[next_idx_]); + return; + } + } + LOG(ERROR) << "finished serializing persistent state for " << masterchain_handle_->id().id.to_str(); } - LOG(INFO) << "finished serializing persistent state for " << masterchain_handle_->id().id; last_key_block_ts_ = masterchain_handle_->unix_time(); last_key_block_id_ = masterchain_handle_->id(); + previous_state_files_ = {}; + previous_state_cache_ = {}; + previous_state_cur_shards_ = {}; } if (!saved_to_db_) { running_ = true; @@ -193,9 +223,97 @@ void AsyncStateSerializer::got_masterchain_handle(BlockHandle handle) { next_iteration(); } +class CachedCellDbReader : public vm::CellDbReader { + public: + CachedCellDbReader(std::shared_ptr parent, + std::shared_ptr>> cache) + : parent_(std::move(parent)), cache_(std::move(cache)) { + } + td::Result> load_cell(td::Slice hash) override { + ++total_reqs_; + DCHECK(hash.size() == 32); + if (cache_) { + auto it = cache_->find(td::Bits256{(const unsigned char*)hash.data()}); + if (it != cache_->end()) { + ++cached_reqs_; + TRY_RESULT(loaded_cell, it->second->load_cell()); + return loaded_cell.data_cell; + } + } + return parent_->load_cell(hash); + } + void print_stats() const { + LOG(WARNING) << "CachedCellDbReader stats : " << total_reqs_ << " reads, " << cached_reqs_ << " cached"; + } + private: + std::shared_ptr parent_; + std::shared_ptr>> cache_; + + td::uint64 total_reqs_ = 0; + td::uint64 cached_reqs_ = 0; +}; + +void AsyncStateSerializer::prepare_previous_state_cache(ShardIdFull shard) { + std::vector prev_shards; + for (const auto& [_, prev_shard] : previous_state_files_) { + if (shard_intersects(shard, prev_shard)) { + prev_shards.push_back(prev_shard); + } + } + if (prev_shards == previous_state_cur_shards_) { + return; + } + previous_state_cur_shards_ = std::move(prev_shards); + previous_state_cache_ = {}; + if (previous_state_cur_shards_.empty()) { + return; + } + td::Timer timer; + LOG(WARNING) << "Preloading previous persistent state for shard " << shard.to_str() << " (" + << previous_state_cur_shards_.size() << " files)"; + std::map> cells; + std::function)> dfs = [&](td::Ref cell) { + td::Bits256 hash = cell->get_hash().bits(); + if (!cells.emplace(hash, cell).second) { + return; + } + bool is_special; + vm::CellSlice cs = vm::load_cell_slice_special(cell, is_special); + for (unsigned i = 0; i < cs.size_refs(); ++i) { + dfs(cs.prefetch_ref(i)); + } + }; + for (const auto& [file, prev_shard] : previous_state_files_) { + if (!shard_intersects(shard, prev_shard)) { + continue; + } + auto r_data = td::read_file(file); + if (r_data.is_error()) { + LOG(INFO) << "Reading " << file << " : " << r_data.move_as_error(); + continue; + } + LOG(INFO) << "Reading " << file << " : " << td::format::as_size(r_data.ok().size()); + auto r_root = vm::std_boc_deserialize(r_data.move_as_ok()); + if (r_root.is_error()) { + LOG(WARNING) << "Deserialize error : " << r_root.move_as_error(); + continue; + } + r_data = {}; + dfs(r_root.move_as_ok()); + } + LOG(WARNING) << "Preloaded previous state: " << cells.size() << " cells in " << timer.elapsed() << "s"; + previous_state_cache_ = std::make_shared>>(std::move(cells)); +} + void AsyncStateSerializer::got_masterchain_state(td::Ref state, std::shared_ptr cell_db_reader) { - LOG(INFO) << "serializing masterchain state " << masterchain_handle_->id().id; + if (!opts_->get_state_serializer_enabled()) { + stored_masterchain_state(); + return; + } + LOG(ERROR) << "serializing masterchain state " << masterchain_handle_->id().id.to_str(); + prepare_previous_state_cache(state->get_shard()); + auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache_); have_masterchain_state_ = true; CHECK(next_idx_ == 0); CHECK(shards_.size() == 0); @@ -205,11 +323,18 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state shards_.push_back(v->top_block_id()); } - auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader](td::FileFd& fd) { - return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31); + auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader = new_cell_db_reader, + cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable { + auto res = vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31, std::move(cancellation_token)); + cell_db_reader->print_stats(); + return res; }; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - R.ensure(); + if (R.is_error() && R.error().code() == cancelled) { + LOG(ERROR) << "Persistent state serialization cancelled"; + } else { + R.ensure(); + } td::actor::send_closure(SelfId, &AsyncStateSerializer::stored_masterchain_state); }); @@ -218,7 +343,7 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state } void AsyncStateSerializer::stored_masterchain_state() { - LOG(INFO) << "finished serializing masterchain state " << masterchain_handle_->id().id; + LOG(ERROR) << "finished serializing masterchain state " << masterchain_handle_->id().id.to_str(); running_ = false; next_iteration(); } @@ -248,13 +373,26 @@ void AsyncStateSerializer::got_shard_handle(BlockHandle handle) { void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref state, std::shared_ptr cell_db_reader) { - LOG(INFO) << "serializing shard state " << handle->id().id; - auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader](td::FileFd& fd) { - return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31); + if (!opts_->get_state_serializer_enabled()) { + success_handler(); + return; + } + LOG(ERROR) << "serializing shard state " << handle->id().id.to_str(); + prepare_previous_state_cache(state->get_shard()); + auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache_); + auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader = new_cell_db_reader, + cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable { + auto res = vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31, std::move(cancellation_token)); + cell_db_reader->print_stats(); + return res; }; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result R) { - R.ensure(); - LOG(INFO) << "finished serializing shard state " << handle->id().id; + if (R.is_error() && R.error().code() == cancelled) { + LOG(ERROR) << "Persistent state serialization cancelled"; + } else { + R.ensure(); + LOG(ERROR) << "finished serializing shard state " << handle->id().id.to_str(); + } td::actor::send_closure(SelfId, &AsyncStateSerializer::success_handler); }); td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, handle->id(), @@ -280,6 +418,14 @@ void AsyncStateSerializer::success_handler() { next_iteration(); } +void AsyncStateSerializer::update_options(td::Ref opts) { + opts_ = std::move(opts); + if (!opts_->get_state_serializer_enabled()) { + cancellation_token_source_.cancel(); + } +} + + bool AsyncStateSerializer::need_monitor(ShardIdFull shard) { return opts_->need_monitor(shard); } @@ -292,6 +438,10 @@ bool AsyncStateSerializer::need_serialize(BlockHandle handle) { ValidatorManager::persistent_state_ttl(handle->unix_time()) > (UnixTime)td::Clocks::system(); } +bool AsyncStateSerializer::have_newer_persistent_state(UnixTime cur_ts) { + return cur_ts / (1 << 17) < last_known_key_block_ts_ / (1 << 17); +} + } // namespace validator } // namespace ton diff --git a/validator/state-serializer.hpp b/validator/state-serializer.hpp index ee2aace01..6d966f930 100644 --- a/validator/state-serializer.hpp +++ b/validator/state-serializer.hpp @@ -37,6 +37,8 @@ class AsyncStateSerializer : public td::actor::Actor { bool saved_to_db_ = true; td::Ref opts_; + td::CancellationTokenSource cancellation_token_source_; + UnixTime last_known_key_block_ts_ = 0; td::actor::ActorId manager_; @@ -46,6 +48,11 @@ class AsyncStateSerializer : public td::actor::Actor { bool have_masterchain_state_ = false; std::vector shards_; + std::vector> previous_state_files_; + std::shared_ptr>> previous_state_cache_; + std::vector previous_state_cur_shards_; + + void prepare_previous_state_cache(ShardIdFull shard); public: AsyncStateSerializer(BlockIdExt block_id, td::Ref opts, @@ -59,12 +66,15 @@ class AsyncStateSerializer : public td::actor::Actor { bool need_serialize(BlockHandle handle); bool need_monitor(ShardIdFull shard); + bool have_newer_persistent_state(UnixTime cur_ts); void alarm() override; void start_up() override; void got_self_state(AsyncSerializerState state); void got_init_handle(BlockHandle handle); + void request_previous_state_files(); + void got_previous_state_files(std::vector> files); void request_masterchain_state(); void request_shard_state(BlockIdExt shard); @@ -80,6 +90,10 @@ class AsyncStateSerializer : public td::actor::Actor { promise.set_result(last_block_id_.id.seqno); } + void update_last_known_key_block_ts(UnixTime ts) { + last_known_key_block_ts_ = std::max(last_known_key_block_ts_, ts); + } + void saved_to_db() { saved_to_db_ = true; running_ = false; @@ -89,6 +103,8 @@ class AsyncStateSerializer : public td::actor::Actor { void fail_handler(td::Status reason); void fail_handler_cont(); void success_handler(); + + void update_options(td::Ref opts); }; } // namespace validator diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 05628ef5d..fc3ebe541 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -27,7 +27,8 @@ namespace ton { namespace validator { -void ValidatorGroup::generate_block_candidate(td::uint32 round_id, td::Promise promise) { +void ValidatorGroup::generate_block_candidate( + td::uint32 round_id, td::Promise promise) { if (round_id > last_known_round_id_) { last_known_round_id_ = round_id; } @@ -37,18 +38,22 @@ void ValidatorGroup::generate_block_candidate(td::uint32 round_id, td::Promiseresult) { - promise.set_result(cached_collated_block_->result.value().clone()); + promise.set_value({cached_collated_block_->result.value().clone(), true}); } else { - cached_collated_block_->promises.push_back(std::move(promise)); + cached_collated_block_->promises.push_back(promise.wrap([](BlockCandidate &&res) { + return validatorsession::ValidatorSession::GeneratedCandidate{std::move(res), true}; + })); } return; } cached_collated_block_ = std::make_shared(); - cached_collated_block_->promises.push_back(std::move(promise)); + cached_collated_block_->promises.push_back(promise.wrap([](BlockCandidate &&res) { + return validatorsession::ValidatorSession::GeneratedCandidate{std::move(res), false}; + })); run_collate_query( shard_, min_ts_, min_masterchain_block_id_, prev_block_ids_, - Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, validator_set_, manager_, td::Timestamp::in(10.0), - [SelfId = actor_id(this), cache = cached_collated_block_](td::Result R) { + Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, validator_set_, opts_->get_collator_options(), manager_, + td::Timestamp::in(10.0), [SelfId = actor_id(this), cache = cached_collated_block_](td::Result R) { td::actor::send_closure(SelfId, &ValidatorGroup::generated_block_candidate, std::move(cache), std::move(R)); }); } @@ -73,7 +78,7 @@ void ValidatorGroup::generated_block_candidate(std::shared_ptr promise) { + td::Promise> promise) { if (round_id > last_known_round_id_) { last_known_round_id_ = round_id; } @@ -88,7 +93,7 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat CacheKey cache_key = block_to_cache_key(block); auto it = approved_candidates_cache_.find(cache_key); if (it != approved_candidates_cache_.end()) { - promise.set_result(it->second); + promise.set_value({it->second, true}); return; } @@ -113,7 +118,7 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat ts); td::actor::send_closure(SelfId, &ValidatorGroup::add_available_block_candidate, block.pubkey.as_bits256(), block.id, block.collated_file_hash); - promise.set_result(ts); + promise.set_value({ts, false}); }, [&](CandidateReject reject) { promise.set_error( @@ -247,15 +252,18 @@ std::unique_ptr ValidatorGroup::ma void on_candidate(td::uint32 round, PublicKey source, validatorsession::ValidatorSessionRootHash root_hash, td::BufferSlice data, td::BufferSlice collated_data, td::Promise promise) override { - auto P = td::PromiseCreator::lambda([id = id_, promise = std::move(promise)](td::Result R) mutable { - if (R.is_ok()) { - promise.set_value(validatorsession::ValidatorSession::CandidateDecision{R.move_as_ok()}); - } else { - auto S = R.move_as_error(); - promise.set_value( - validatorsession::ValidatorSession::CandidateDecision{S.message().c_str(), td::BufferSlice()}); - } - }); + auto P = + td::PromiseCreator::lambda([promise = std::move(promise)](td::Result> R) mutable { + if (R.is_ok()) { + validatorsession::ValidatorSession::CandidateDecision decision(R.ok().first); + decision.set_is_cached(R.ok().second); + promise.set_value(std::move(decision)); + } else { + auto S = R.move_as_error(); + promise.set_value( + validatorsession::ValidatorSession::CandidateDecision{S.message().c_str(), td::BufferSlice()}); + } + }); BlockCandidate candidate{Ed25519_PublicKey{source.ed25519_value().raw()}, BlockIdExt{0, 0, 0, root_hash, sha256_bits256(data.as_slice())}, @@ -264,7 +272,8 @@ std::unique_ptr ValidatorGroup::ma td::actor::send_closure(id_, &ValidatorGroup::validate_block_candidate, round, std::move(candidate), std::move(P)); } - void on_generate_slot(td::uint32 round, td::Promise promise) override { + void on_generate_slot(td::uint32 round, + td::Promise promise) override { td::actor::send_closure(id_, &ValidatorGroup::generate_block_candidate, round, std::move(promise)); } void on_block_committed(td::uint32 round, PublicKey source, validatorsession::ValidatorSessionRootHash root_hash, @@ -339,6 +348,10 @@ void ValidatorGroup::create_session() { << ".", allow_unsafe_self_blocks_resync_); } + if (opts_->get_catchain_max_block_delay()) { + td::actor::send_closure(session_, &validatorsession::ValidatorSession::set_catchain_max_block_delay, + opts_->get_catchain_max_block_delay().value()); + } if (started_) { td::actor::send_closure(session_, &validatorsession::ValidatorSession::start); } @@ -368,6 +381,22 @@ void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterch prev_block_ids_ = std::vector{next_block_id}; } postponed_accept_.clear(); + + validatorsession::NewValidatorGroupStats stats; + stats.session_id = session_id_; + stats.shard = shard_; + stats.cc_seqno = validator_set_->get_catchain_seqno(); + stats.timestamp = td::Clocks::system(); + td::uint32 idx = 0; + for (const auto& node : validator_set_->export_vector()) { + PublicKeyHash id = ValidatorFullId{node.key}.compute_short_id(); + if (id == local_id_) { + stats.self_idx = idx; + } + stats.nodes.push_back(validatorsession::NewValidatorGroupStats::Node{id, node.weight}); + ++idx; + } + td::actor::send_closure(manager_, &ValidatorManager::log_new_validator_group_stats, std::move(stats)); } void ValidatorGroup::destroy() { @@ -381,6 +410,9 @@ void ValidatorGroup::destroy() { return; } auto stats = R.move_as_ok(); + if (stats.rounds.empty()) { + return; + } stats.cc_seqno = cc_seqno; td::actor::send_closure(manager, &ValidatorManager::log_validator_session_stats, block_id, std::move(stats)); diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 2dbff8de2..3499da9d7 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -34,8 +34,10 @@ class ValidatorManager; class ValidatorGroup : public td::actor::Actor { public: - void generate_block_candidate(td::uint32 round_id, td::Promise promise); - void validate_block_candidate(td::uint32 round_id, BlockCandidate block, td::Promise promise); + void generate_block_candidate(td::uint32 round_id, + td::Promise promise); + void validate_block_candidate(td::uint32 round_id, BlockCandidate block, + td::Promise> promise); void accept_block_candidate(td::uint32 round_id, PublicKeyHash src, td::BufferSlice block, RootHash root_hash, FileHash file_hash, std::vector signatures, std::vector approve_signatures, @@ -62,12 +64,16 @@ class ValidatorGroup : public td::actor::Actor { void get_validator_group_info_for_litequery( td::Promise> promise); + void update_options(td::Ref opts) { + opts_ = std::move(opts); + } + ValidatorGroup(ShardIdFull shard, PublicKeyHash local_id, ValidatorSessionId session_id, td::Ref validator_set, validatorsession::ValidatorSessionOptions config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, td::actor::ActorId validator_manager, bool create_session, - bool allow_unsafe_self_blocks_resync) + bool allow_unsafe_self_blocks_resync, td::Ref opts) : shard_(shard) , local_id_(std::move(local_id)) , session_id_(session_id) @@ -80,7 +86,8 @@ class ValidatorGroup : public td::actor::Actor { , db_root_(std::move(db_root)) , manager_(validator_manager) , init_(create_session) - , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) { + , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) + , opts_(std::move(opts)) { } private: @@ -121,6 +128,7 @@ class ValidatorGroup : public td::actor::Actor { bool init_ = false; bool started_ = false; bool allow_unsafe_self_blocks_resync_; + td::Ref opts_; td::uint32 last_known_round_id_ = 0; struct CachedCollatedBlock { diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 1b7c5b09c..9e7767796 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -129,6 +129,24 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool nonfinal_ls_queries_enabled() const override { return nonfinal_ls_queries_enabled_; } + td::optional get_celldb_cache_size() const override { + return celldb_cache_size_; + } + bool get_celldb_direct_io() const override { + return celldb_direct_io_; + } + bool get_celldb_preload_all() const override { + return celldb_preload_all_; + } + td::optional get_catchain_max_block_delay() const override { + return catchain_max_block_delay_; + } + bool get_state_serializer_enabled() const override { + return state_serializer_enabled_; + } + td::Ref get_collator_options() const override { + return collator_options_; + } void set_zero_block_id(BlockIdExt block_id) override { zero_block_id_ = block_id; @@ -197,6 +215,24 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_nonfinal_ls_queries_enabled(bool value) override { nonfinal_ls_queries_enabled_ = value; } + void set_celldb_cache_size(td::uint64 value) override { + celldb_cache_size_ = value; + } + void set_celldb_direct_io(bool value) override { + celldb_direct_io_ = value; + } + void set_celldb_preload_all(bool value) override { + celldb_preload_all_ = value; + } + void set_catchain_max_block_delay(double value) override { + catchain_max_block_delay_ = value; + } + void set_state_serializer_enabled(bool value) override { + state_serializer_enabled_ = value; + } + void set_collator_options(td::Ref value) override { + collator_options_ = std::move(value); + } ValidatorManagerOptionsImpl *make_copy() const override { return new ValidatorManagerOptionsImpl(*this); @@ -244,6 +280,12 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { double archive_preload_period_ = 0.0; bool disable_rocksdb_stats_; bool nonfinal_ls_queries_enabled_ = false; + td::optional celldb_cache_size_; + bool celldb_direct_io_ = false; + bool celldb_preload_all_ = false; + td::optional catchain_max_block_delay_; + bool state_serializer_enabled_ = true; + td::Ref collator_options_{true}; }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index 9ede07110..81f13487e 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -51,6 +51,21 @@ struct PerfTimerStats { std::deque> stats; // }; +struct CollatorOptions : public td::CntObject { + bool deferring_enabled = true; + + // Defer messages from account after Xth message in block (excluding first messages from transactions) + td::uint32 defer_messages_after = 10; + // Defer all messages if out msg queue size is greater than X (excluding first messages from transactions) + td::uint64 defer_out_queue_size_limit = 2048; + + // See Collator::process_dispatch_queue + td::uint32 dispatch_phase_2_max_total = 150; + td::uint32 dispatch_phase_3_max_total = 150; + td::uint32 dispatch_phase_2_max_per_initiator = 20; + td::optional dispatch_phase_3_max_per_initiator; // Default - depends on out msg queue size +}; + struct ValidatorManagerOptions : public td::CntObject { public: enum class ShardCheckMode { m_monitor, m_validate }; @@ -86,6 +101,12 @@ struct ValidatorManagerOptions : public td::CntObject { virtual double get_archive_preload_period() const = 0; virtual bool get_disable_rocksdb_stats() const = 0; virtual bool nonfinal_ls_queries_enabled() const = 0; + virtual td::optional get_celldb_cache_size() const = 0; + virtual bool get_celldb_direct_io() const = 0; + virtual bool get_celldb_preload_all() const = 0; + virtual td::optional get_catchain_max_block_delay() const = 0; + virtual bool get_state_serializer_enabled() const = 0; + virtual td::Ref get_collator_options() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; @@ -110,13 +131,19 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_archive_preload_period(double value) = 0; virtual void set_disable_rocksdb_stats(bool value) = 0; virtual void set_nonfinal_ls_queries_enabled(bool value) = 0; + virtual void set_celldb_cache_size(td::uint64 value) = 0; + virtual void set_celldb_direct_io(bool value) = 0; + virtual void set_celldb_preload_all(bool value) = 0; + virtual void set_catchain_max_block_delay(double value) = 0; + virtual void set_state_serializer_enabled(bool value) = 0; + virtual void set_collator_options(td::Ref value) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id, std::function check_shard = [](ShardIdFull, CatchainSeqno, ShardCheckMode) { return true; }, - bool allow_blockchain_init = false, double sync_blocks_before = 86400, double block_ttl = 86400 * 7, - double state_ttl = 3600, double archive_ttl = 86400 * 365, double key_proof_ttl = 86400 * 3650, + bool allow_blockchain_init = false, double sync_blocks_before = 3600, double block_ttl = 86400, + double state_ttl = 3600, double archive_ttl = 86400 * 7, double key_proof_ttl = 86400 * 3650, double max_mempool_num = 999999, bool initial_sync_disabled = false); }; @@ -134,7 +161,9 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; - virtual void send_broadcast(BlockBroadcast broadcast) = 0; + virtual void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) = 0; + virtual void send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only = false) = 0; virtual void download_block(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; virtual void download_zero_state(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, @@ -188,6 +217,8 @@ class ValidatorManagerInterface : public td::actor::Actor { td::Promise promise) = 0; virtual void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, td::int64 max_length, td::Promise promise) = 0; + virtual void get_previous_persistent_state_files( + BlockSeqno cur_mc_seqno, td::Promise>> promise) = 0; virtual void get_block_proof(BlockHandle handle, td::Promise promise) = 0; virtual void get_block_proof_link(BlockHandle handle, td::Promise promise) = 0; virtual void get_block_handle(BlockIdExt block_id, bool force, td::Promise promise) = 0; @@ -202,6 +233,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void check_external_message(td::BufferSlice data, td::Promise> promise) = 0; virtual void new_ihr_message(td::BufferSlice data) = 0; virtual void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; + virtual void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) = 0; virtual void add_ext_server_id(adnl::AdnlNodeIdShort id) = 0; virtual void add_ext_server_port(td::uint16 port) = 0; @@ -236,8 +268,9 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void prepare_perf_timer_stats(td::Promise> promise) = 0; virtual void add_perf_timer_stat(std::string name, double duration) = 0; - virtual void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) = 0; + virtual void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) = 0; + virtual void update_options(td::Ref opts) = 0; }; } // namespace validator