chore(bench): more determinism (#13603)

Author: DaniPopes

.github/workflows/bench.yml

@@ -10,6 +10,7 @@ on:
 env:
   CARGO_TERM_COLOR: always
   BASELINE: base
+  SEED: reth
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}

crates/primitives/benches/validate_blob_tx.rs

@@ -4,20 +4,12 @@ use alloy_consensus::TxEip4844;
 use alloy_eips::eip4844::{
     env_settings::EnvKzgSettings, BlobTransactionSidecar, MAX_BLOBS_PER_BLOCK,
 };
-use alloy_primitives::hex;
 use criterion::{
     criterion_group, criterion_main, measurement::WallTime, BenchmarkGroup, Criterion,
 };
-use proptest::{
-    prelude::*,
-    strategy::ValueTree,
-    test_runner::{RngAlgorithm, TestRng, TestRunner},
-};
+use proptest::{prelude::*, strategy::ValueTree, test_runner::TestRunner};
 use proptest_arbitrary_interop::arb;
 
-// constant seed to use for the rng
-const SEED: [u8; 32] = hex!("1337133713371337133713371337133713371337133713371337133713371337");
-
 /// Benchmarks EIP-48444 blob validation.
 fn blob_validation(c: &mut Criterion) {
     let mut group = c.benchmark_group("Blob Transaction KZG validation");
@@ -35,9 +27,7 @@ fn validate_blob_tx(
     kzg_settings: EnvKzgSettings,
 ) {
     let setup = || {
-        let config = ProptestConfig::default();
-        let rng = TestRng::from_seed(RngAlgorithm::ChaCha, &SEED);
-        let mut runner = TestRunner::new_with_rng(config, rng);
+        let mut runner = TestRunner::deterministic();
 
         // generate tx and sidecar
         let mut tx = arb::<TxEip4844>().new_tree(&mut runner).unwrap().current();

crates/transaction-pool/benches/truncate.rs

@@ -1,27 +1,20 @@
 #![allow(missing_docs)]
-use alloy_primitives::{hex_literal::hex, Address};
+use alloy_primitives::Address;
 use criterion::{
     criterion_group, criterion_main, measurement::WallTime, BenchmarkGroup, Criterion,
 };
 use pprof::criterion::{Output, PProfProfiler};
-use proptest::{
-    prelude::*,
-    strategy::ValueTree,
-    test_runner::{RngAlgorithm, TestRng, TestRunner},
-};
+use proptest::{prelude::*, strategy::ValueTree, test_runner::TestRunner};
 use reth_transaction_pool::{
     pool::{BasefeeOrd, ParkedPool, PendingPool, QueuedOrd},
     test_utils::{MockOrdering, MockTransaction, MockTransactionFactory},
     SubPoolLimit,
 };
 
-// constant seed to use for the rng
-const SEED: [u8; 32] = hex!("1337133713371337133713371337133713371337133713371337133713371337");
-
 /// Generates a set of `depth` dependent transactions, with the specified sender. Its values are
 /// generated using [Arbitrary].
 fn create_transactions_for_sender(
-    mut runner: TestRunner,
+    runner: &mut TestRunner,
     sender: Address,
     depth: usize,
 ) -> Vec<MockTransaction> {
@@ -32,19 +25,17 @@ fn create_transactions_for_sender(
     assert!(depth > 0);
 
     // make sure these are all post-eip-1559 transactions
-    let mut txs = prop::collection::vec(any::<MockTransaction>(), depth)
-        .new_tree(&mut runner)
-        .unwrap()
-        .current();
+    let mut txs =
+        prop::collection::vec(any::<MockTransaction>(), depth).new_tree(runner).unwrap().current();
 
     for (nonce, tx) in txs.iter_mut().enumerate() {
         // reject pre-eip1559 tx types, if there is a legacy tx, replace it with an eip1559 tx
         if tx.is_legacy() || tx.is_eip2930() {
             *tx = MockTransaction::eip1559();
 
             // set fee values using arbitrary
-            tx.set_priority_fee(any::<u128>().new_tree(&mut runner).unwrap().current());
-            tx.set_max_fee(any::<u128>().new_tree(&mut runner).unwrap().current());
+            tx.set_priority_fee(any::<u128>().new_tree(runner).unwrap().current());
+            tx.set_max_fee(any::<u128>().new_tree(runner).unwrap().current());
         }
 
         tx.set_sender(sender);
@@ -62,9 +53,7 @@ fn create_transactions_for_sender(
 ///
 /// This uses [`create_transactions_for_sender`] to generate the transactions.
 fn generate_many_transactions(senders: usize, max_depth: usize) -> Vec<MockTransaction> {
-    let config = ProptestConfig::default();
-    let rng = TestRng::from_seed(RngAlgorithm::ChaCha, &SEED);
-    let mut runner = TestRunner::new_with_rng(config, rng);
+    let mut runner = TestRunner::deterministic();
 
     let mut txs = Vec::with_capacity(senders);
     for idx in 0..senders {
@@ -79,7 +68,7 @@ fn generate_many_transactions(senders: usize, max_depth: usize) -> Vec<MockTrans
         let addr_slice = [0u8; 12].into_iter().chain(idx_slice.into_iter()).collect::<Vec<_>>();
 
         let sender = Address::from_slice(&addr_slice);
-        txs.extend(create_transactions_for_sender(runner.clone(), sender, depth));
+        txs.extend(create_transactions_for_sender(&mut runner, sender, depth));
     }
 
     txs

crates/trie/common/benches/prefix_set.rs

@@ -2,6 +2,7 @@
 use criterion::{
     criterion_group, criterion_main, measurement::WallTime, BenchmarkGroup, Criterion,
 };
+use prop::test_runner::TestRng;
 use proptest::{
     prelude::*,
     strategy::ValueTree,
@@ -116,7 +117,8 @@ fn generate_test_data(size: usize) -> (Vec<Nibbles>, Vec<Nibbles>, Vec<bool>) {
     use prop::collection::vec;
 
     let config = ProptestConfig { result_cache: basic_result_cache, ..Default::default() };
-    let mut runner = TestRunner::new(config);
+    let rng = TestRng::deterministic_rng(config.rng_algorithm);
+    let mut runner = TestRunner::new_with_rng(config, rng);
 
     let vec_of_nibbles = |range| vec(any_with::<Nibbles>(range), size);
     let mut preload = vec_of_nibbles(32usize.into()).new_tree(&mut runner).unwrap().current();

crates/trie/parallel/benches/root.rs

@@ -78,7 +78,7 @@ pub fn calculate_state_root(c: &mut Criterion) {
 
 fn generate_test_data(size: usize) -> (HashedPostState, HashedPostState) {
     let storage_size = 1_000;
-    let mut runner = TestRunner::new(ProptestConfig::default());
+    let mut runner = TestRunner::deterministic();
 
     use proptest::{collection::hash_map, sample::subsequence};
     let db_state = hash_map(

crates/trie/sparse/benches/rlp_node.rs

@@ -10,7 +10,7 @@ use reth_trie::Nibbles;
 use reth_trie_sparse::RevealedSparseTrie;
 
 fn update_rlp_node_level(c: &mut Criterion) {
-    let mut rng = generators::rng_with_seed(&12345_u16.to_be_bytes());
+    let mut rng = generators::rng();
     let mut group = c.benchmark_group("update rlp node level");
     group.sample_size(20);
 

crates/trie/sparse/benches/root.rs

@@ -169,6 +169,7 @@ fn calculate_root_from_leaves_repeated(c: &mut Criterion) {
                                     trie_updates.finalize(hb, node_iter.walker.take_removed_keys());
                                 }
                             }
+                            (storage, storage_updates, trie_updates)
                         },
                     )
                 });
@@ -205,6 +206,7 @@ fn calculate_root_from_leaves_repeated(c: &mut Criterion) {
                                 }
                                 sparse.root().unwrap();
                             }
+                            sparse
                         },
                     )
                 });

crates/trie/trie/benches/hash_post_state.rs

@@ -51,7 +51,7 @@ fn from_bundle_state_seq(state: &HashMap<Address, BundleAccount>) -> HashedPostS
 
 fn generate_test_data(size: usize) -> HashMap<Address, BundleAccount> {
     let storage_size = 1_000;
-    let mut runner = TestRunner::new(ProptestConfig::default());
+    let mut runner = TestRunner::deterministic();
 
     use proptest::collection::hash_map;
     let state = hash_map(

testing/testing-utils/src/generators.rs

@@ -18,8 +18,7 @@ use reth_primitives::{
 use secp256k1::{Keypair, Secp256k1};
 use std::{
     cmp::{max, min},
-    collections::{hash_map::DefaultHasher, BTreeMap},
-    hash::Hasher,
+    collections::BTreeMap,
     ops::{Range, RangeInclusive},
 };
 
@@ -77,9 +76,9 @@ pub fn rng() -> StdRng {
 
 /// Returns a random number generator from a specific seed, as bytes.
 pub fn rng_with_seed(seed: &[u8]) -> StdRng {
-    let mut hasher = DefaultHasher::new();
-    hasher.write(seed);
-    StdRng::seed_from_u64(hasher.finish())
+    let mut seed_bytes = [0u8; 32];
+    seed_bytes[..seed.len().min(32)].copy_from_slice(seed);
+    StdRng::from_seed(seed_bytes)
 }
 
 /// Generates a range of random [`SealedHeader`]s.