Merge branch 'optional-evm-simulator' into evm-simulator-server-tests

.github/release-please/manifest.json

@@ -1,5 +1,5 @@
 {
-  "core": "24.24.0",
+  "core": "24.25.0",
   "prover": "16.5.0",
   "zk_toolbox": "0.1.2"
 }

.github/workflows/build-base.yml

@@ -1,4 +1,4 @@
-name: Build base Docker image
+name: Build zksync-build-base Docker image
 on:
   workflow_dispatch:
     inputs:

.github/workflows/build-runtime-base.yml

@@ -0,0 +1,66 @@
+name: Build zksync-runtime-base Docker image
+on:
+  workflow_dispatch:
+    inputs:
+      repo_ref:
+        description: "git reference of the zksync-era to build"
+        required: true
+        default: main
+jobs:
+  build-images:
+    name: Build and Push Docker Images
+    runs-on: matterlabs-ci-runner-high-performance
+    outputs:
+      image_tag_sha: ${{ steps.get-sha.outputs.image_tag_sha }}
+    # Needed to push to Gihub Package Registry
+    permissions:
+      packages: write
+      contents: read
+    env:
+      REPO_REF: ${{ github.event.inputs.repo_ref }}
+    strategy:
+      matrix:
+        name: [ runtime-base ]
+        image_name: [ zksync-runtime-base ]
+
+    steps:
+      - uses: actions/checkout@6d193bf28034eafb982f37bd894289fe649468fc # v4.1.7
+        with:
+          submodules: "recursive"
+
+      - name: Login to google container registry
+        run: |
+          gcloud auth configure-docker us-docker.pkg.dev -q
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get tag
+        id: get-sha
+        run: |
+          echo IMAGE_TAG_SHA=$(git rev-parse --short HEAD) >> $GITHUB_ENV
+          echo image_tag_sha=$(git rev-parse --short HEAD) >> $GITHUB_OUTPUT
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0
+
+      - name: Build and push
+        uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
+        with:
+          push: true
+          context: .
+          platforms: arm64, amd64
+          file: docker/${{ matrix.name }}/Dockerfile
+          labels: |
+            org.opencontainers.image.source=https://github.com/matter-labs/zksync-era
+            org.opencontainers.image.licenses="MIT OR Apache-2.0"
+          tags: |
+            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.image_name }}:latest
+            ghcr.io/${{ github.repository_owner }}/${{ matrix.image_name }}:latest

.github/workflows/vm-perf-to-prometheus.yml

@@ -34,7 +34,7 @@ jobs:
           run_retried docker compose pull zk
           docker compose up -d zk
           ci_run zkt
-          ci_run zk_supervisor contracts all
+          ci_run zk_supervisor contracts
 
       - name: run benchmarks
         run: |

Cargo.toml

@@ -227,7 +227,7 @@ zk_evm_1_4_1 = { package = "zk_evm", version = "0.141" }
 zk_evm_1_5_0 = { package = "zk_evm", version = "=0.150.5" }
 
 # New VM; pinned to a specific commit because of instability
-zksync_vm2 = { git = "https://github.com/matter-labs/vm2.git", rev = "cd6136c42ec56856e0abcf2a98d1a9e120161482" }
+zksync_vm2 = { git = "https://github.com/matter-labs/vm2.git", rev = "74577d9be13b1bff9d1a712389731f669b179e47" }
 
 # Consensus dependencies.
 zksync_concurrency = "=0.1.1"

core/CHANGELOG.md

@@ -1,5 +1,32 @@
 # Changelog
 
+## [24.25.0](https://github.com/matter-labs/zksync-era/compare/core-v24.24.0...core-v24.25.0) (2024-09-19)
+
+
+### Features
+
+* (DB migration) Rename recursion_scheduler_level_vk_hash to snark_wrapper_vk_hash ([#2809](https://github.com/matter-labs/zksync-era/issues/2809)) ([64f9551](https://github.com/matter-labs/zksync-era/commit/64f95514c99f95da2a19a97ff064c29a97efc22f))
+* add da clients ([#2743](https://github.com/matter-labs/zksync-era/issues/2743)) ([9218612](https://github.com/matter-labs/zksync-era/commit/9218612fdb2b63c20841e2e2e5a45bbd23c01fbc))
+* attester committees data extractor (BFT-434) ([#2684](https://github.com/matter-labs/zksync-era/issues/2684)) ([92dde03](https://github.com/matter-labs/zksync-era/commit/92dde039ee8a0bc08e2019b7fa6f243a34d9816f))
+* emit errors in prover API metrics ([#2890](https://github.com/matter-labs/zksync-era/issues/2890)) ([2ac7cc5](https://github.com/matter-labs/zksync-era/commit/2ac7cc5836e69fc82c98df2005fedee01c1084e1))
+* **en:** Resume incomplete snapshot in snapshot creator in more cases ([#2886](https://github.com/matter-labs/zksync-era/issues/2886)) ([f095b4a](https://github.com/matter-labs/zksync-era/commit/f095b4a3223222ac712de53592fe1e68f766600f))
+* make `to` address optional for transaction data ([#2852](https://github.com/matter-labs/zksync-era/issues/2852)) ([8363c1d](https://github.com/matter-labs/zksync-era/commit/8363c1d8697ad9bd2fe5d326218476bc3dad38af))
+* **prover:** Optimize setup keys loading ([#2847](https://github.com/matter-labs/zksync-era/issues/2847)) ([19887ef](https://github.com/matter-labs/zksync-era/commit/19887ef21a8bbd26977353f8ee277b711850dfd2))
+* Selector generator tool ([#2844](https://github.com/matter-labs/zksync-era/issues/2844)) ([b359b08](https://github.com/matter-labs/zksync-era/commit/b359b085895da6582f1d28722107bc5b25f1232c))
+* **tee:** use hex serialization for RPC responses ([#2887](https://github.com/matter-labs/zksync-era/issues/2887)) ([abe0440](https://github.com/matter-labs/zksync-era/commit/abe0440811ae4daf4a0f307922a282e9664308e0))
+* **utils:** Rework locate_workspace, introduce Workspace type ([#2830](https://github.com/matter-labs/zksync-era/issues/2830)) ([d256092](https://github.com/matter-labs/zksync-era/commit/d2560928cc67b40a97a5497ac8542915bf6f91a9))
+* **zk_toolbox:** Add external_node consensus support ([#2821](https://github.com/matter-labs/zksync-era/issues/2821)) ([4a10d7d](https://github.com/matter-labs/zksync-era/commit/4a10d7d9554d6c1aa2f4fc46557d40baaad8ff2f))
+
+
+### Bug Fixes
+
+* count SECP256 precompile to account validation gas limit as well ([#2859](https://github.com/matter-labs/zksync-era/issues/2859)) ([fee0c2a](https://github.com/matter-labs/zksync-era/commit/fee0c2ad08a5ab4a04252765b367eb9fbb1f3db7))
+* **en:** Fix connection starvation during snapshot recovery ([#2836](https://github.com/matter-labs/zksync-era/issues/2836)) ([52f4f76](https://github.com/matter-labs/zksync-era/commit/52f4f763674d25f8a5e7f3a111354a559f798d52))
+* **eth_watch:** fix `get_events_inner` ([#2882](https://github.com/matter-labs/zksync-era/issues/2882)) ([c957dd8](https://github.com/matter-labs/zksync-era/commit/c957dd8011213e0e95fa5962e2310321b29a0d16))
+* handling of HTTP 403 thrown by proxyd ([#2835](https://github.com/matter-labs/zksync-era/issues/2835)) ([2d71c74](https://github.com/matter-labs/zksync-era/commit/2d71c7408a0eed3662fc51f70fa9f525d66e4c6f))
+* **state-keeper:** Restore processed tx metrics in state keeper ([#2815](https://github.com/matter-labs/zksync-era/issues/2815)) ([4d8862b](https://github.com/matter-labs/zksync-era/commit/4d8862b76a55ac78edd481694fefd2107736ffd9))
+* **tee-prover:** fix deserialization of `std::time::Duration` in `envy` config ([#2817](https://github.com/matter-labs/zksync-era/issues/2817)) ([df8641a](https://github.com/matter-labs/zksync-era/commit/df8641a912a8d480ceecff58b0bfaef05e04f0c8))
+
 ## [24.24.0](https://github.com/matter-labs/zksync-era/compare/core-v24.23.0...core-v24.24.0) (2024-09-05)
 
 

core/bin/external_node/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "zksync_external_node"
 description = "Non-validator ZKsync node"
-version = "24.24.0" # x-release-please-version
+version = "24.25.0" # x-release-please-version
 edition.workspace = true
 authors.workspace = true
 homepage.workspace = true

core/lib/basic_types/src/prover_dal.rs

@@ -2,6 +2,7 @@
 use std::{net::IpAddr, ops::Add, str::FromStr};
 
 use chrono::{DateTime, Duration, NaiveDateTime, NaiveTime, Utc};
+use serde::{Deserialize, Serialize};
 use strum::{Display, EnumString};
 
 use crate::{
@@ -27,7 +28,7 @@ pub struct ExtendedJobCountStatistics {
     pub successful: usize,
 }
 
-#[derive(Debug, Clone, Copy, Default)]
+#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)]
 pub struct JobCountStatistics {
     pub queued: usize,
     pub in_progress: usize,

core/lib/config/src/configs/prover_job_monitor.rs

@@ -61,6 +61,8 @@ pub struct ProverJobMonitorConfig {
     /// The interval between runs for Witness Job Queuer.
     #[serde(default = "ProverJobMonitorConfig::default_witness_job_queuer_run_interval_ms")]
     pub witness_job_queuer_run_interval_ms: u64,
+    /// HTTP port of the ProverJobMonitor to send requests to.
+    pub http_port: u16,
 }
 
 impl ProverJobMonitorConfig {

core/lib/config/src/testonly.rs

@@ -1115,6 +1115,7 @@ impl Distribution<configs::prover_job_monitor::ProverJobMonitorConfig> for Encod
             prover_queue_reporter_run_interval_ms: self.sample(rng),
             witness_generator_queue_reporter_run_interval_ms: self.sample(rng),
             witness_job_queuer_run_interval_ms: self.sample(rng),
+            http_port: self.sample(rng),
         }
     }
 }

core/lib/env_config/src/prover_job_monitor.rs

@@ -31,6 +31,7 @@ mod tests {
             prover_queue_reporter_run_interval_ms: 10000,
             witness_generator_queue_reporter_run_interval_ms: 10000,
             witness_job_queuer_run_interval_ms: 10000,
+            http_port: 3074,
         }
     }
 
@@ -55,6 +56,7 @@ mod tests {
     fn from_env_with_default() {
         let config = r#"
             PROVER_JOB_MONITOR_PROMETHEUS_PORT=3317
+            PROVER_JOB_MONITOR_HTTP_PORT=3074
             PROVER_JOB_MONITOR_MAX_DB_CONNECTIONS=9
         "#;
         let mut lock = MUTEX.lock();
@@ -80,6 +82,7 @@ mod tests {
             PROVER_JOB_MONITOR_PROVER_QUEUE_REPORTER_RUN_INTERVAL_MS=10001
             PROVER_JOB_MONITOR_WITNESS_GENERATOR_QUEUE_REPORTER_RUN_INTERVAL_MS=10001
             PROVER_JOB_MONITOR_WITNESS_JOB_QUEUER_RUN_INTERVAL_MS=10001
+            PROVER_JOB_MONITOR_HTTP_PORT=3074
         "#;
         let mut lock = MUTEX.lock();
         lock.set_env(config);

core/lib/multivm/src/versions/vm_fast/circuits_tracer.rs

@@ -1,5 +1,5 @@
 use circuit_sequencer_api_1_5_0::{geometry_config::get_geometry_config, toolset::GeometryConfig};
-use zksync_vm2::{CycleStats, Opcode, OpcodeType, StateInterface, Tracer};
+use zksync_vm2::interface::{CycleStats, Opcode, OpcodeType, StateInterface, Tracer};
 use zksync_vm_interface::CircuitStatistic;
 
 use crate::vm_latest::tracers::circuits_capacity::*;
@@ -17,7 +17,7 @@ pub(crate) struct CircuitsTracer {
     keccak256_cycles: u32,
     ecrecover_cycles: u32,
     sha256_cycles: u32,
-    secp256k1_verify_cycles: u32,
+    secp256r1_verify_cycles: u32,
     transient_storage_checker_cycles: u32,
 }
 
@@ -115,7 +115,7 @@ impl Tracer for CircuitsTracer {
             CycleStats::Keccak256(cycles) => self.keccak256_cycles += cycles,
             CycleStats::Sha256(cycles) => self.sha256_cycles += cycles,
             CycleStats::EcRecover(cycles) => self.ecrecover_cycles += cycles,
-            CycleStats::Secp256k1Verify(cycles) => self.secp256k1_verify_cycles += cycles,
+            CycleStats::Secp256r1Verify(cycles) => self.secp256r1_verify_cycles += cycles,
             CycleStats::Decommit(cycles) => self.code_decommitter_cycles += cycles,
             CycleStats::StorageRead => self.storage_application_cycles += 1,
             CycleStats::StorageWrite => self.storage_application_cycles += 2,
@@ -146,7 +146,7 @@ impl CircuitsTracer {
             ecrecover: self.ecrecover_cycles as f32
                 / GEOMETRY_CONFIG.cycles_per_ecrecover_circuit as f32,
             sha256: self.sha256_cycles as f32 / GEOMETRY_CONFIG.cycles_per_sha256_circuit as f32,
-            secp256k1_verify: self.secp256k1_verify_cycles as f32
+            secp256k1_verify: self.secp256r1_verify_cycles as f32
                 / GEOMETRY_CONFIG.cycles_per_secp256r1_verify_circuit as f32,
             transient_storage_checker: self.transient_storage_checker_cycles as f32
                 / GEOMETRY_CONFIG.cycles_per_transient_storage_sorter as f32,

core/lib/multivm/src/versions/vm_fast/events.rs

@@ -1,6 +1,6 @@
 use zksync_types::{L1BatchNumber, H256};
 use zksync_utils::h256_to_account_address;
-use zksync_vm2::Event;
+use zksync_vm2::interface::Event;
 
 use crate::interface::VmEvent;
 
@@ -23,18 +23,21 @@ impl EventAccumulator {
     }
 }
 
-pub(crate) fn merge_events(events: &[Event], block_number: L1BatchNumber) -> Vec<VmEvent> {
+pub(crate) fn merge_events(
+    events: impl Iterator<Item = Event>,
+    block_number: L1BatchNumber,
+) -> Vec<VmEvent> {
     let mut result = vec![];
     let mut current: Option<(usize, u32, EventAccumulator)> = None;
 
-    for message in events.iter() {
+    for event in events {
         let Event {
             shard_id,
             is_first,
             tx_number,
             key,
             value,
-        } = message.clone();
+        } = event;
 
         if !is_first {
             if let Some((mut remaining_data_length, mut remaining_topics, mut event)) =

core/lib/multivm/src/versions/vm_fast/glue.rs

@@ -1,18 +1,19 @@
 use zksync_types::l2_to_l1_log::{L2ToL1Log, SystemL2ToL1Log};
 use zksync_utils::u256_to_h256;
+use zksync_vm2::interface;
 
 use crate::glue::GlueFrom;
 
-impl GlueFrom<&zksync_vm2::L2ToL1Log> for SystemL2ToL1Log {
-    fn glue_from(value: &zksync_vm2::L2ToL1Log) -> Self {
-        let zksync_vm2::L2ToL1Log {
+impl GlueFrom<interface::L2ToL1Log> for SystemL2ToL1Log {
+    fn glue_from(value: interface::L2ToL1Log) -> Self {
+        let interface::L2ToL1Log {
             key,
             value,
             is_service,
             address,
             shard_id,
             tx_number,
-        } = *value;
+        } = value;
 
         Self(L2ToL1Log {
             shard_id,

core/lib/multivm/src/versions/vm_fast/tests/bootloader.rs

@@ -1,6 +1,6 @@
 use assert_matches::assert_matches;
 use zksync_types::U256;
-use zksync_vm2::HeapId;
+use zksync_vm2::interface::HeapId;
 
 use crate::{
     interface::{ExecutionResult, Halt, TxExecutionMode, VmExecutionMode, VmInterfaceExt},

core/lib/multivm/src/versions/vm_fast/tests/tester/transaction_test_info.rs

@@ -1,6 +1,7 @@
 use std::fmt;
 
 use zksync_types::{ExecuteTransactionCommon, Transaction, H160, U256};
+use zksync_vm2::interface::{Event, StateInterface};
 
 use super::VmTester;
 use crate::{
@@ -181,7 +182,7 @@ impl TransactionTestInfo {
 struct VmStateDump<S> {
     state: S,
     storage_writes: Vec<((H160, U256), U256)>,
-    events: Box<[zksync_vm2::Event]>,
+    events: Box<[Event]>,
 }
 
 impl<S: PartialEq> PartialEq for VmStateDump<S> {
@@ -196,14 +197,8 @@ impl<S: ReadStorage> Vm<S> {
     fn dump_state(&self) -> VmStateDump<impl PartialEq + fmt::Debug> {
         VmStateDump {
             state: self.inner.dump_state(),
-            storage_writes: self
-                .inner
-                .world_diff()
-                .get_storage_state()
-                .iter()
-                .map(|(k, v)| (*k, *v))
-                .collect(),
-            events: self.inner.world_diff().events().into(),
+            storage_writes: self.inner.get_storage_state().collect(),
+            events: self.inner.events().collect(),
         }
     }
 }

core/lib/multivm/src/versions/vm_fast/tests/utils.rs

@@ -10,7 +10,7 @@ use zksync_types::{
     U256,
 };
 use zksync_utils::{bytecode::hash_bytecode, bytes_to_be_words, h256_to_u256, u256_to_h256};
-use zksync_vm2::{HeapId, StateInterface};
+use zksync_vm2::interface::{HeapId, StateInterface};
 
 use crate::interface::storage::ReadStorage;