From 86437d72d8de271676de8367e4cb75d3c510ab43 Mon Sep 17 00:00:00 2001 From: LePremierHomme <57456510+LePremierHomme@users.noreply.github.com> Date: Sat, 28 Dec 2024 22:41:33 +0200 Subject: [PATCH 01/14] feat(tests): concurrent materializer tests --- .../testing/materializer/src/concurrency.rs | 13 +++++ fendermint/testing/materializer/src/lib.rs | 1 + .../testing/materializer/src/testnet.rs | 9 ++++ .../testing/materializer/tests/docker.rs | 50 +++++++++++++------ .../materializer/tests/docker_tests/layer2.rs | 3 +- .../tests/docker_tests/root_only.rs | 3 +- .../tests/docker_tests/standalone.rs | 27 +++++----- 7 files changed, 77 insertions(+), 29 deletions(-) create mode 100644 fendermint/testing/materializer/src/concurrency.rs diff --git a/fendermint/testing/materializer/src/concurrency.rs b/fendermint/testing/materializer/src/concurrency.rs new file mode 100644 index 000000000..a8b497fa4 --- /dev/null +++ b/fendermint/testing/materializer/src/concurrency.rs @@ -0,0 +1,13 @@ +pub struct Config { + pub parallelism_level: usize +} + +impl Config { + pub fn with_parallelism_level(v: usize) -> Self { + Self { parallelism_level: v } + } +} + + + + diff --git a/fendermint/testing/materializer/src/lib.rs b/fendermint/testing/materializer/src/lib.rs index 2a0d1965e..f427a47d1 100644 --- a/fendermint/testing/materializer/src/lib.rs +++ b/fendermint/testing/materializer/src/lib.rs @@ -20,6 +20,7 @@ pub mod validation; #[cfg(feature = "arb")] mod arb; +pub mod concurrency; /// An ID identifying a resource within its parent. #[derive(Clone, Serialize, PartialEq, Eq, PartialOrd, Ord)] diff --git a/fendermint/testing/materializer/src/testnet.rs b/fendermint/testing/materializer/src/testnet.rs index 607da9d7f..299d62885 100644 --- a/fendermint/testing/materializer/src/testnet.rs +++ b/fendermint/testing/materializer/src/testnet.rs @@ -140,6 +140,15 @@ where .ok_or_else(|| anyhow!("account {id} does not exist")) } + pub fn account_mod_nth(&self, v: usize) -> &M::Account { + let nth = v % self.accounts.len(); + self.accounts + .iter() + .nth(nth) + .map(|(_, account)| account) + .unwrap() + } + /// Get a node by name. pub fn node(&self, name: &NodeName) -> anyhow::Result<&M::Node> { self.nodes diff --git a/fendermint/testing/materializer/tests/docker.rs b/fendermint/testing/materializer/tests/docker.rs index 04038a183..32ffc4bd4 100644 --- a/fendermint/testing/materializer/tests/docker.rs +++ b/fendermint/testing/materializer/tests/docker.rs @@ -16,13 +16,8 @@ use std::{ use anyhow::{anyhow, Context}; use ethers::providers::Middleware; -use fendermint_materializer::{ - docker::{DockerMaterializer, DockerMaterials}, - manifest::Manifest, - testnet::Testnet, - HasCometBftApi, HasEthApi, TestnetName, -}; -use futures::Future; +use fendermint_materializer::{concurrency, docker::{DockerMaterializer, DockerMaterials}, manifest::Manifest, testnet::Testnet, HasCometBftApi, HasEthApi, TestnetName}; +use futures::{future, Future}; use lazy_static::lazy_static; use tendermint_rpc::Client; @@ -59,13 +54,14 @@ fn read_manifest(file_name: &str) -> anyhow::Result { /// Parse a manifest file in the `manifests` directory, clean up any corresponding /// testnet resources, then materialize a testnet and run some tests. -pub async fn with_testnet(manifest_file_name: &str, alter: G, test: F) -> anyhow::Result<()> +pub async fn with_testnet(manifest_file_name: &str, concurrency: Option, alter: G, test: F) -> anyhow::Result<()> where - // https://users.rust-lang.org/t/function-that-takes-a-closure-with-mutable-reference-that-returns-a-future/54324 - F: for<'a> FnOnce( +// https://users.rust-lang.org/t/function-that-takes-a-closure-with-mutable-reference-that-returns-a-future/54324 + F: for<'a> Fn( &Manifest, - &mut DockerMaterializer, - &'a mut DockerTestnet, + &DockerMaterializer, + &'a DockerTestnet, + usize ) -> Pin> + 'a>>, G: FnOnce(&mut Manifest), { @@ -98,14 +94,40 @@ where .await .context("failed to remove testnet")?; - let mut testnet = Testnet::setup(&mut materializer, &testnet_name, &manifest) + let testnet = Testnet::setup(&mut materializer, &testnet_name, &manifest) .await .context("failed to set up testnet")?; let started = wait_for_startup(&testnet).await?; let res = if started { - test(&manifest, &mut materializer, &mut testnet).await + match concurrency { + None => test(&manifest, &materializer, &testnet, 0).await, + Some(cfg) => { + let mut futures = Vec::new(); + let mut test_ids = Vec::new(); + for i in 0..cfg.parallelism_level { + let test_id = i; + let task = test(&manifest, &materializer, &testnet, test_id); + futures.push(task); + test_ids.push(test_id); + } + + let results: Vec> = future::join_all(futures).await; + let mut err = None; + for (i, result) in results.into_iter().enumerate() { + let test_id = test_ids[i]; + match result { + Ok(_) => println!("test completed successfully (test_id={})", test_id), + Err(e) => { + println!("test failed: {} (test_id={})", e, test_id); + err = Some(e); + }, + } + } + err.map_or(Ok(()), Err) + } + } } else { Err(anyhow!("the startup sequence timed out")) }; diff --git a/fendermint/testing/materializer/tests/docker_tests/layer2.rs b/fendermint/testing/materializer/tests/docker_tests/layer2.rs index acfd0f8f4..80bd44015 100644 --- a/fendermint/testing/materializer/tests/docker_tests/layer2.rs +++ b/fendermint/testing/materializer/tests/docker_tests/layer2.rs @@ -27,6 +27,7 @@ const MAX_RETRIES: u32 = 5; async fn test_topdown_and_bottomup() { with_testnet( MANIFEST, + None, |manifest| { // Try to make sure the bottom-up checkpoint period is quick enough for reasonable test runtime. let subnet = manifest @@ -36,7 +37,7 @@ async fn test_topdown_and_bottomup() { subnet.bottom_up_checkpoint.period = CHECKPOINT_PERIOD; }, - |_, _, testnet| { + |_, _, testnet, _| { let test = async { let brussels = testnet.node(&testnet.root().node("brussels"))?; let london = testnet.node(&testnet.root().subnet("england").node("london"))?; diff --git a/fendermint/testing/materializer/tests/docker_tests/root_only.rs b/fendermint/testing/materializer/tests/docker_tests/root_only.rs index 293f26c55..1c70d8896 100644 --- a/fendermint/testing/materializer/tests/docker_tests/root_only.rs +++ b/fendermint/testing/materializer/tests/docker_tests/root_only.rs @@ -16,8 +16,9 @@ const MANIFEST: &str = "root-only.yaml"; async fn test_full_node_sync() { with_testnet( MANIFEST, + None, |_| {}, - |_, _, testnet| { + |_, _, testnet, _| { let test = async { // Allow a little bit of time for node-2 to catch up with node-1. tokio::time::sleep(Duration::from_secs(5)).await; diff --git a/fendermint/testing/materializer/tests/docker_tests/standalone.rs b/fendermint/testing/materializer/tests/docker_tests/standalone.rs index 805943889..f9724cb8e 100644 --- a/fendermint/testing/materializer/tests/docker_tests/standalone.rs +++ b/fendermint/testing/materializer/tests/docker_tests/standalone.rs @@ -11,10 +11,9 @@ use ethers::{ signers::{Signer, Wallet}, types::{transaction::eip2718::TypedTransaction, Eip1559TransactionRequest, H160}, }; -use fendermint_materializer::{manifest::Rootnet, materials::DefaultAccount, HasEthApi}; +use fendermint_materializer::{concurrency, manifest::Rootnet, materials::DefaultAccount, HasEthApi}; use futures::FutureExt; - -use crate::with_testnet; +use crate::{with_testnet}; const MANIFEST: &str = "standalone.yaml"; @@ -46,30 +45,31 @@ where async fn test_sent_tx_found_in_mempool() { with_testnet( MANIFEST, + Some(concurrency::Config::with_parallelism_level(3)), |manifest| { // Slow down consensus to where we can see the effect of the transaction not being found by Ethereum hash. if let Rootnet::New { ref mut env, .. } = manifest.rootnet { env.insert("CMT_CONSENSUS_TIMEOUT_COMMIT".into(), "10s".into()); }; }, - |_, _, testnet| { - let test = async { - let bob = testnet.account("bob")?; - let charlie = testnet.account("charlie")?; - + |_, _, testnet, test_id| { + let sender = testnet.account_mod_nth(test_id); + let recipient = testnet.account_mod_nth(test_id + 1); + let test = async move { + println!("running (test_id={})", test_id); let pangea = testnet.node(&testnet.root().node("pangea"))?; let provider = pangea .ethapi_http_provider()? .expect("ethapi should be enabled"); - let middleware = make_middleware(provider, bob) + let middleware = make_middleware(provider, sender) .await .context("failed to set up middleware")?; - eprintln!("middleware ready, pending tests"); + println!("middleware ready, pending tests (test_id={})", test_id); // Create the simplest transaction possible: send tokens between accounts. - let to: H160 = charlie.eth_addr().into(); + let to: H160 = recipient.eth_addr().into(); let transfer = Eip1559TransactionRequest::new().to(to).value(1); let pending: PendingTransaction<_> = middleware @@ -79,7 +79,7 @@ async fn test_sent_tx_found_in_mempool() { let tx_hash = pending.tx_hash(); - eprintln!("sent pending txn {:?}", tx_hash); + println!("sent pending txn {:?} (test_id={})", tx_hash, test_id); // We expect that the transaction is pending, however it should not return an error. match middleware.get_transaction(tx_hash).await { @@ -111,8 +111,9 @@ async fn test_out_of_order_mempool() { with_testnet( MANIFEST, + None, |_| {}, - |_, _, testnet| { + |_, _, testnet, _| { let test = async { let bob = testnet.account("bob")?; let charlie = testnet.account("charlie")?; From 9c4d665d3ac054041919f1b7afe28b1fb589bd00 Mon Sep 17 00:00:00 2001 From: LePremierHomme <57456510+LePremierHomme@users.noreply.github.com> Date: Mon, 30 Dec 2024 23:15:44 +0200 Subject: [PATCH 02/14] implement concurrency::execute --- .../testing/materializer/src/concurrency.rs | 95 ++++++- .../testing/materializer/src/docker/mod.rs | 1 + .../testing/materializer/tests/docker.rs | 86 ++++--- .../materializer/tests/docker_tests/layer2.rs | 4 +- .../tests/docker_tests/root_only.rs | 4 +- .../tests/docker_tests/standalone.rs | 232 ++++++++++-------- .../tests/manifests/standalone.yaml | 111 ++++++++- 7 files changed, 377 insertions(+), 156 deletions(-) diff --git a/fendermint/testing/materializer/src/concurrency.rs b/fendermint/testing/materializer/src/concurrency.rs index a8b497fa4..6ac531c69 100644 --- a/fendermint/testing/materializer/src/concurrency.rs +++ b/fendermint/testing/materializer/src/concurrency.rs @@ -1,13 +1,100 @@ +use futures::{FutureExt}; +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::Semaphore; + +pub async fn execute(cfg: Config, test: F) -> (ExecutionSummary, Vec) +where + F: Fn(usize) -> Pin> + Send>>, +{ + let semaphore = Arc::new(Semaphore::new(cfg.max_concurrency)); + let mut test_id = 0; + let mut handles = Vec::new(); + let results = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let execution_start = Instant::now(); + loop { + if execution_start.elapsed() > cfg.duration { + break; + } + let permit = semaphore.clone().acquire_owned().await.unwrap(); + let task = test(test_id).boxed(); + let results = results.clone(); + let handle = tokio::spawn(async move { + let start = Instant::now(); + let result = task.await; + let duration = start.elapsed(); + + results.lock().await.push(ExecutionResult { + test_id, + duration, + err: result.err(), + }); + drop(permit); + }); + handles.push(handle); + test_id = test_id + 1; + } + + // Exhaust unfinished handles. + for handle in handles { + handle.await.unwrap(); + } + + let results = Arc::try_unwrap(results).unwrap().into_inner(); + let summary = ExecutionSummary::new(cfg, &results); + (summary, results) +} + +#[derive(Debug)] pub struct Config { - pub parallelism_level: usize + pub max_concurrency: usize, + pub duration: Duration, } impl Config { - pub fn with_parallelism_level(v: usize) -> Self { - Self { parallelism_level: v } + pub fn new() -> Self { + Self { + max_concurrency: 1, + duration: Duration::from_secs(1), + } } -} + pub fn with_max_concurrency(mut self, max_concurrency: usize) -> Self { + self.max_concurrency = max_concurrency; + self + } + + pub fn with_duration(mut self, duration: Duration) -> Self { + self.duration = duration; + self + } +} +#[derive(Debug)] +pub struct ExecutionResult { + pub test_id: usize, + pub duration: Duration, + pub err: Option, +} +#[derive(Debug)] +pub struct ExecutionSummary { + pub cfg: Config, + pub avg_duration: Duration, + pub num_failures: usize, +} +impl ExecutionSummary { + fn new(cfg: Config, results: &Vec) -> Self { + let total_duration: Duration = results.iter().map(|res| res.duration).sum(); + let avg_duration = total_duration / results.len() as u32; + let num_failures = results.iter().filter(|res| res.err.is_some()).count(); + Self { + cfg, + avg_duration, + num_failures, + } + } +} diff --git a/fendermint/testing/materializer/src/docker/mod.rs b/fendermint/testing/materializer/src/docker/mod.rs index e85e220e3..551bee223 100644 --- a/fendermint/testing/materializer/src/docker/mod.rs +++ b/fendermint/testing/materializer/src/docker/mod.rs @@ -160,6 +160,7 @@ pub struct DockerMaterializerState { port_ranges: BTreeMap, } +#[derive(Debug)] pub struct DockerMaterializer { dir: PathBuf, rng: StdRng, diff --git a/fendermint/testing/materializer/tests/docker.rs b/fendermint/testing/materializer/tests/docker.rs index 32ffc4bd4..0f0fb9197 100644 --- a/fendermint/testing/materializer/tests/docker.rs +++ b/fendermint/testing/materializer/tests/docker.rs @@ -6,6 +6,18 @@ //! //! `cargo test -p fendermint_materializer --test docker -- --nocapture` +use anyhow::{anyhow, Context}; +use ethers::providers::Middleware; +use fendermint_materializer::{ + concurrency, + docker::{DockerMaterializer, DockerMaterials}, + manifest::Manifest, + testnet::Testnet, + HasCometBftApi, HasEthApi, TestnetName, +}; +use futures::{Future, FutureExt}; +use lazy_static::lazy_static; +use std::sync::Arc; use std::{ collections::BTreeSet, env::current_dir, @@ -13,12 +25,6 @@ use std::{ pin::Pin, time::{Duration, Instant}, }; - -use anyhow::{anyhow, Context}; -use ethers::providers::Middleware; -use fendermint_materializer::{concurrency, docker::{DockerMaterializer, DockerMaterials}, manifest::Manifest, testnet::Testnet, HasCometBftApi, HasEthApi, TestnetName}; -use futures::{future, Future}; -use lazy_static::lazy_static; use tendermint_rpc::Client; pub type DockerTestnet = Testnet; @@ -54,15 +60,22 @@ fn read_manifest(file_name: &str) -> anyhow::Result { /// Parse a manifest file in the `manifests` directory, clean up any corresponding /// testnet resources, then materialize a testnet and run some tests. -pub async fn with_testnet(manifest_file_name: &str, concurrency: Option, alter: G, test: F) -> anyhow::Result<()> +pub async fn with_testnet( + manifest_file_name: &str, + concurrency: Option, + alter: G, + test: F, +) -> anyhow::Result<()> where -// https://users.rust-lang.org/t/function-that-takes-a-closure-with-mutable-reference-that-returns-a-future/54324 F: for<'a> Fn( - &Manifest, - &DockerMaterializer, - &'a DockerTestnet, - usize - ) -> Pin> + 'a>>, + Arc, + Arc, + Arc, + usize, + ) -> Pin> + Send>> + + Copy + + Send + + 'static, G: FnOnce(&mut Manifest), { let testnet_name = TestnetName::new( @@ -100,31 +113,42 @@ where let started = wait_for_startup(&testnet).await?; + let testnet = Arc::new(testnet); + let materializer = Arc::new(materializer); + let manifest = Arc::new(manifest); let res = if started { match concurrency { - None => test(&manifest, &materializer, &testnet, 0).await, + None => test(manifest.clone(), materializer.clone(), testnet.clone(), 0).await, Some(cfg) => { - let mut futures = Vec::new(); - let mut test_ids = Vec::new(); - for i in 0..cfg.parallelism_level { - let test_id = i; - let task = test(&manifest, &materializer, &testnet, test_id); - futures.push(task); - test_ids.push(test_id); - } + let test_generator = { + let testnet = testnet.clone(); + let materializer = materializer.clone(); + move |test_id| { + let manifest = manifest.clone(); + let materializer = materializer.clone(); + let testnet = testnet.clone(); + async move { test(manifest, materializer, testnet, test_id).await }.boxed() + } + }; - let results: Vec> = future::join_all(futures).await; + let (summary, results) = concurrency::execute(cfg, test_generator).await; let mut err = None; - for (i, result) in results.into_iter().enumerate() { - let test_id = test_ids[i]; - match result { - Ok(_) => println!("test completed successfully (test_id={})", test_id), - Err(e) => { - println!("test failed: {} (test_id={})", e, test_id); + for res in results.into_iter() { + match res.err { + None => println!( + "test completed successfully (test_id={}, duration={:?})", + res.test_id, res.duration + ), + Some(e) => { + println!( + "test failed (test_id={}, duration={:?})", + res.test_id, res.duration + ); err = Some(e); - }, + } } } + println!("{:?}", summary); err.map_or(Ok(()), Err) } } @@ -158,7 +182,7 @@ where // otherwise the system shuts down too quick, but // at least we can inspect the containers. // If they don't all get dropped, `docker system prune` helps. - let drop_handle = materializer.take_dropper(); + let drop_handle = Arc::try_unwrap(materializer).unwrap().take_dropper(); let _ = tokio::time::timeout(*TEARDOWN_TIMEOUT, drop_handle).await; res diff --git a/fendermint/testing/materializer/tests/docker_tests/layer2.rs b/fendermint/testing/materializer/tests/docker_tests/layer2.rs index 80bd44015..e711bb38d 100644 --- a/fendermint/testing/materializer/tests/docker_tests/layer2.rs +++ b/fendermint/testing/materializer/tests/docker_tests/layer2.rs @@ -38,7 +38,7 @@ async fn test_topdown_and_bottomup() { subnet.bottom_up_checkpoint.period = CHECKPOINT_PERIOD; }, |_, _, testnet, _| { - let test = async { + let test = async move { let brussels = testnet.node(&testnet.root().node("brussels"))?; let london = testnet.node(&testnet.root().subnet("england").node("london"))?; let england = testnet.subnet(&testnet.root().subnet("england"))?; @@ -142,7 +142,7 @@ async fn test_topdown_and_bottomup() { Ok(()) }; - test.boxed_local() + test.boxed() }, ) .await diff --git a/fendermint/testing/materializer/tests/docker_tests/root_only.rs b/fendermint/testing/materializer/tests/docker_tests/root_only.rs index 1c70d8896..16e22cee7 100644 --- a/fendermint/testing/materializer/tests/docker_tests/root_only.rs +++ b/fendermint/testing/materializer/tests/docker_tests/root_only.rs @@ -19,7 +19,7 @@ async fn test_full_node_sync() { None, |_| {}, |_, _, testnet, _| { - let test = async { + let test = async move { // Allow a little bit of time for node-2 to catch up with node-1. tokio::time::sleep(Duration::from_secs(5)).await; // Check that node2 is following node1. @@ -39,7 +39,7 @@ async fn test_full_node_sync() { Ok(()) }; - test.boxed_local() + test.boxed() }, ) .await diff --git a/fendermint/testing/materializer/tests/docker_tests/standalone.rs b/fendermint/testing/materializer/tests/docker_tests/standalone.rs index f9724cb8e..fbe64da73 100644 --- a/fendermint/testing/materializer/tests/docker_tests/standalone.rs +++ b/fendermint/testing/materializer/tests/docker_tests/standalone.rs @@ -1,19 +1,21 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT -use std::time::{Duration, Instant}; +use std::time::{Duration}; +use crate::with_testnet; use anyhow::{bail, Context}; use ethers::{ core::k256::ecdsa::SigningKey, middleware::SignerMiddleware, providers::{JsonRpcClient, Middleware, PendingTransaction, Provider}, signers::{Signer, Wallet}, - types::{transaction::eip2718::TypedTransaction, Eip1559TransactionRequest, H160}, + types::{Eip1559TransactionRequest, H160}, +}; +use fendermint_materializer::{ + concurrency, manifest::Rootnet, materials::DefaultAccount, HasEthApi, }; -use fendermint_materializer::{concurrency, manifest::Rootnet, materials::DefaultAccount, HasEthApi}; use futures::FutureExt; -use crate::{with_testnet}; const MANIFEST: &str = "standalone.yaml"; @@ -45,7 +47,11 @@ where async fn test_sent_tx_found_in_mempool() { with_testnet( MANIFEST, - Some(concurrency::Config::with_parallelism_level(3)), + Some( + concurrency::Config::new() + .with_max_concurrency(50) + .with_duration(Duration::from_secs(10)), + ), |manifest| { // Slow down consensus to where we can see the effect of the transaction not being found by Ethereum hash. if let Rootnet::New { ref mut env, .. } = manifest.rootnet { @@ -53,9 +59,9 @@ async fn test_sent_tx_found_in_mempool() { }; }, |_, _, testnet, test_id| { - let sender = testnet.account_mod_nth(test_id); - let recipient = testnet.account_mod_nth(test_id + 1); let test = async move { + let sender = testnet.account_mod_nth(test_id); + let recipient = testnet.account_mod_nth(test_id + 1); println!("running (test_id={})", test_id); let pangea = testnet.node(&testnet.root().node("pangea"))?; let provider = pangea @@ -68,9 +74,19 @@ async fn test_sent_tx_found_in_mempool() { println!("middleware ready, pending tests (test_id={})", test_id); + let sender: H160 = sender.eth_addr().into(); + let current_nonce = middleware + .get_transaction_count(sender, None) + .await + .context("failed to fetch nonce")?; + let nonce = current_nonce + 1; + // Create the simplest transaction possible: send tokens between accounts. let to: H160 = recipient.eth_addr().into(); - let transfer = Eip1559TransactionRequest::new().to(to).value(1); + let transfer = Eip1559TransactionRequest::new() + .to(to) + .value(1) + .nonce(nonce); let pending: PendingTransaction<_> = middleware .send_transaction(transfer, None) @@ -93,109 +109,109 @@ async fn test_sent_tx_found_in_mempool() { Ok(()) }; - test.boxed_local() + test.boxed() }, ) .await .unwrap() } -/// Test that transactions sent out-of-order with regards to the nonce are not rejected, -/// but rather get included in block eventually, their submission managed by the ethereum -/// API facade. -#[serial_test::serial] -#[tokio::test] -async fn test_out_of_order_mempool() { - const MAX_WAIT_TIME: Duration = Duration::from_secs(10); - const SLEEP_TIME: Duration = Duration::from_secs(1); - - with_testnet( - MANIFEST, - None, - |_| {}, - |_, _, testnet, _| { - let test = async { - let bob = testnet.account("bob")?; - let charlie = testnet.account("charlie")?; - - let pangea = testnet.node(&testnet.root().node("pangea"))?; - let provider = pangea - .ethapi_http_provider()? - .expect("ethapi should be enabled"); - - let middleware = make_middleware(provider, bob) - .await - .context("failed to set up middleware")?; - - // Create the simplest transaction possible: send tokens between accounts. - let to: H160 = charlie.eth_addr().into(); - let tx = Eip1559TransactionRequest::new().to(to).value(1); - let mut tx: TypedTransaction = tx.into(); - - // Fill out the nonce, gas, etc. - middleware - .fill_transaction(&mut tx, None) - .await - .context("failed to fill tx")?; - - // Create a few more transactions to be sent out-of-order. - let mut txs = vec![tx]; - - for i in 1..5 { - let mut tx = txs[0].clone(); - let nonce = tx.nonce().expect("fill_transaction filled the nonce"); - tx.set_nonce(nonce.saturating_add(i.into())); - txs.push(tx) - } - - let mut pending_txs = Vec::new(); - - // Submit transactions in opposite order. - for (i, tx) in txs.iter().enumerate().rev() { - let sig = middleware - .signer() - .sign_transaction(tx) - .await - .context("failed to sign tx")?; - - let rlp = tx.rlp_signed(&sig); - - let pending_tx: PendingTransaction<_> = middleware - .send_raw_transaction(rlp) - .await - .with_context(|| format!("failed to send tx {i}"))?; - - pending_txs.push(pending_tx) - } - - // Check that they eventually get included. - let start = Instant::now(); - 'pending: loop { - for tx in pending_txs.iter() { - let receipt = middleware - .get_transaction_receipt(tx.tx_hash()) - .await - .context("failed to get receipt")?; - - if receipt.is_none() { - if start.elapsed() > MAX_WAIT_TIME { - bail!("some transactions are still not executed") - } else { - tokio::time::sleep(SLEEP_TIME).await; - continue 'pending; - } - } - } - // All of them have receipt. - break 'pending; - } - - Ok(()) - }; - - test.boxed_local() - }, - ) - .await - .unwrap() -} +// /// Test that transactions sent out-of-order with regards to the nonce are not rejected, +// /// but rather get included in block eventually, their submission managed by the ethereum +// /// API facade. +// #[serial_test::serial] +// #[tokio::test] +// async fn test_out_of_order_mempool() { +// const MAX_WAIT_TIME: Duration = Duration::from_secs(10); +// const SLEEP_TIME: Duration = Duration::from_secs(1); +// +// with_testnet( +// MANIFEST, +// None, +// |_| {}, +// |_, _, testnet, _| { +// let test = async move { +// let bob = testnet.account("bob")?; +// let charlie = testnet.account("charlie")?; +// +// let pangea = testnet.node(&testnet.root().node("pangea"))?; +// let provider = pangea +// .ethapi_http_provider()? +// .expect("ethapi should be enabled"); +// +// let middleware = make_middleware(provider, bob) +// .await +// .context("failed to set up middleware")?; +// +// // Create the simplest transaction possible: send tokens between accounts. +// let to: H160 = charlie.eth_addr().into(); +// let tx = Eip1559TransactionRequest::new().to(to).value(1); +// let mut tx: TypedTransaction = tx.into(); +// +// // Fill out the nonce, gas, etc. +// middleware +// .fill_transaction(&mut tx, None) +// .await +// .context("failed to fill tx")?; +// +// // Create a few more transactions to be sent out-of-order. +// let mut txs = vec![tx]; +// +// for i in 1..5 { +// let mut tx = txs[0].clone(); +// let nonce = tx.nonce().expect("fill_transaction filled the nonce"); +// tx.set_nonce(nonce.saturating_add(i.into())); +// txs.push(tx) +// } +// +// let mut pending_txs = Vec::new(); +// +// // Submit transactions in opposite order. +// for (i, tx) in txs.iter().enumerate().rev() { +// let sig = middleware +// .signer() +// .sign_transaction(tx) +// .await +// .context("failed to sign tx")?; +// +// let rlp = tx.rlp_signed(&sig); +// +// let pending_tx: PendingTransaction<_> = middleware +// .send_raw_transaction(rlp) +// .await +// .with_context(|| format!("failed to send tx {i}"))?; +// +// pending_txs.push(pending_tx) +// } +// +// // Check that they eventually get included. +// let start = Instant::now(); +// 'pending: loop { +// for tx in pending_txs.iter() { +// let receipt = middleware +// .get_transaction_receipt(tx.tx_hash()) +// .await +// .context("failed to get receipt")?; +// +// if receipt.is_none() { +// if start.elapsed() > MAX_WAIT_TIME { +// bail!("some transactions are still not executed") +// } else { +// tokio::time::sleep(SLEEP_TIME).await; +// continue 'pending; +// } +// } +// } +// // All of them have receipt. +// break 'pending; +// } +// +// Ok(()) +// }; +// +// test.boxed() +// }, +// ) +// .await +// .unwrap() +// } diff --git a/fendermint/testing/materializer/tests/manifests/standalone.yaml b/fendermint/testing/materializer/tests/manifests/standalone.yaml index 2072c48c3..39c77b60d 100644 --- a/fendermint/testing/materializer/tests/manifests/standalone.yaml +++ b/fendermint/testing/materializer/tests/manifests/standalone.yaml @@ -1,18 +1,111 @@ accounts: - alice: {} - bob: {} - charlie: {} + "1": {} + "2": {} + "3": {} + "4": {} + "5": {} + "6": {} + "7": {} + "8": {} + "9": {} + "10": {} + "11": {} + "12": {} + "13": {} + "14": {} + "15": {} + "16": {} + "17": {} + "18": {} + "19": {} + "20": {} + "21": {} + "22": {} + "23": {} + "24": {} + "25": {} + "26": {} + "27": {} + "28": {} + "29": {} + "30": {} + "31": {} + "32": {} + "33": {} + "34": {} + "35": {} + "36": {} + "37": {} + "38": {} + "39": {} + "40": {} + "41": {} + "42": {} + "43": {} + "44": {} + "45": {} + "46": {} + "47": {} + "48": {} + "49": {} + "50": {} rootnet: type: New # Balances and collateral are in atto validators: - alice: '100' + "1": '100' balances: - # 100FIL is 100_000_000_000_000_000_000 - alice: '100000000000000000000' - bob: '200000000000000000000' - charlie: '300000000000000000000' + "1": '100000000000000000000' + "2": '200000000000000000000' + "3": '300000000000000000000' + "4": '100000000000000000000' + "5": '200000000000000000000' + "6": '300000000000000000000' + "7": '100000000000000000000' + "8": '200000000000000000000' + "9": '300000000000000000000' + "10": '100000000000000000000' + "11": '200000000000000000000' + "12": '300000000000000000000' + "13": '100000000000000000000' + "14": '200000000000000000000' + "15": '300000000000000000000' + "16": '100000000000000000000' + "17": '200000000000000000000' + "18": '300000000000000000000' + "19": '100000000000000000000' + "20": '200000000000000000000' + "21": '300000000000000000000' + "22": '100000000000000000000' + "23": '200000000000000000000' + "24": '300000000000000000000' + "25": '100000000000000000000' + "26": '200000000000000000000' + "27": '300000000000000000000' + "28": '100000000000000000000' + "29": '200000000000000000000' + "30": '300000000000000000000' + "31": '100000000000000000000' + "32": '200000000000000000000' + "33": '300000000000000000000' + "34": '100000000000000000000' + "35": '200000000000000000000' + "36": '300000000000000000000' + "37": '100000000000000000000' + "38": '200000000000000000000' + "39": '300000000000000000000' + "40": '100000000000000000000' + "41": '200000000000000000000' + "42": '300000000000000000000' + "43": '100000000000000000000' + "44": '200000000000000000000' + "45": '300000000000000000000' + "46": '100000000000000000000' + "47": '200000000000000000000' + "48": '300000000000000000000' + "49": '100000000000000000000' + "50": '200000000000000000000' env: CMT_CONSENSUS_TIMEOUT_COMMIT: 1s FM_LOG_LEVEL: info,fendermint=debug @@ -22,6 +115,6 @@ rootnet: pangea: mode: type: Validator - validator: alice + validator: "1" seed_nodes: [] ethapi: true From c06cc019e41396c0800e5a4a6c33aafd845f4f8e Mon Sep 17 00:00:00 2001 From: LePremierHomme <57456510+LePremierHomme@users.noreply.github.com> Date: Tue, 31 Dec 2024 10:55:02 +0200 Subject: [PATCH 03/14] add `Bencher`, wait for block inclusion --- .../testing/materializer/src/bencher.rs | 34 +++++++ .../testing/materializer/src/concurrency.rs | 41 ++++++--- fendermint/testing/materializer/src/lib.rs | 1 + .../testing/materializer/tests/docker.rs | 16 ++-- .../materializer/tests/docker_tests/layer2.rs | 2 +- .../tests/docker_tests/root_only.rs | 2 +- .../tests/docker_tests/standalone.rs | 89 +++++++++++++++++-- 7 files changed, 156 insertions(+), 29 deletions(-) create mode 100644 fendermint/testing/materializer/src/bencher.rs diff --git a/fendermint/testing/materializer/src/bencher.rs b/fendermint/testing/materializer/src/bencher.rs new file mode 100644 index 000000000..3ddc61f27 --- /dev/null +++ b/fendermint/testing/materializer/src/bencher.rs @@ -0,0 +1,34 @@ +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::Mutex; + +// TODO: remove the Arc> wrappers. +// they are not needed, as a Bencher instance shouldn't be shared among tests. +// it's just a workaround to issues involving mutable references lifetime in futures. +#[derive(Debug, Clone)] +pub struct Bencher { + pub start_time: Arc>>, + pub records: Arc>>, +} + +impl Bencher { + pub fn new() -> Self { + Self { + start_time: Arc::new(Mutex::new(None)), + records: Arc::new(Mutex::new(HashMap::new())), + } + } + + pub async fn start(&self) { + let mut start_time = self.start_time.lock().await; + *start_time = Some(Instant::now()); + } + + pub async fn record(&self, label: String) { + let start_time = self.start_time.lock().await; + let duration = start_time.unwrap().elapsed(); + let mut records = self.records.lock().await; + records.insert(label, duration); + } +} diff --git a/fendermint/testing/materializer/src/concurrency.rs b/fendermint/testing/materializer/src/concurrency.rs index 6ac531c69..b5de900b8 100644 --- a/fendermint/testing/materializer/src/concurrency.rs +++ b/fendermint/testing/materializer/src/concurrency.rs @@ -1,13 +1,15 @@ +use std::collections::HashMap; use futures::{FutureExt}; use std::future::Future; use std::pin::Pin; use std::sync::Arc; use std::time::{Duration, Instant}; -use tokio::sync::Semaphore; +use tokio::sync::{Semaphore}; +use crate::bencher::Bencher; pub async fn execute(cfg: Config, test: F) -> (ExecutionSummary, Vec) where - F: Fn(usize) -> Pin> + Send>>, + F: Fn(usize, Arc) -> Pin> + Send>>, { let semaphore = Arc::new(Semaphore::new(cfg.max_concurrency)); let mut test_id = 0; @@ -19,16 +21,15 @@ where break; } let permit = semaphore.clone().acquire_owned().await.unwrap(); - let task = test(test_id).boxed(); + let bencher = Arc::new(Bencher::new()); + let task = test(test_id, bencher.clone()).boxed(); let results = results.clone(); let handle = tokio::spawn(async move { - let start = Instant::now(); let result = task.await; - let duration = start.elapsed(); - + let records = bencher.records.lock().await.clone(); results.lock().await.push(ExecutionResult { test_id, - duration, + records, err: result.err(), }); drop(permit); @@ -74,27 +75,43 @@ impl Config { #[derive(Debug)] pub struct ExecutionResult { pub test_id: usize, - pub duration: Duration, + pub records: HashMap, pub err: Option, } #[derive(Debug)] pub struct ExecutionSummary { pub cfg: Config, - pub avg_duration: Duration, + pub avg_latencies: HashMap, pub num_failures: usize, } impl ExecutionSummary { fn new(cfg: Config, results: &Vec) -> Self { - let total_duration: Duration = results.iter().map(|res| res.duration).sum(); - let avg_duration = total_duration / results.len() as u32; let num_failures = results.iter().filter(|res| res.err.is_some()).count(); + let mut total_durations: HashMap = HashMap::new(); + let mut counts: HashMap = HashMap::new(); + for res in results { + for (key, duration) in res.records.clone() { + *total_durations.entry(key.clone()).or_insert(Duration::ZERO) += duration; + *counts.entry(key).or_insert(0) += 1; + } + } + + let avg_latencies = total_durations + .into_iter() + .map(|(key, total)| { + let count = counts[&key]; + (key, total / count as u32) + }) + .collect(); + Self { cfg, - avg_duration, + avg_latencies, num_failures, } } } + diff --git a/fendermint/testing/materializer/src/lib.rs b/fendermint/testing/materializer/src/lib.rs index f427a47d1..a6be40fa6 100644 --- a/fendermint/testing/materializer/src/lib.rs +++ b/fendermint/testing/materializer/src/lib.rs @@ -21,6 +21,7 @@ pub mod validation; #[cfg(feature = "arb")] mod arb; pub mod concurrency; +pub mod bencher; /// An ID identifying a resource within its parent. #[derive(Clone, Serialize, PartialEq, Eq, PartialOrd, Ord)] diff --git a/fendermint/testing/materializer/tests/docker.rs b/fendermint/testing/materializer/tests/docker.rs index 0f0fb9197..fa56a00e7 100644 --- a/fendermint/testing/materializer/tests/docker.rs +++ b/fendermint/testing/materializer/tests/docker.rs @@ -26,6 +26,7 @@ use std::{ time::{Duration, Instant}, }; use tendermint_rpc::Client; +use fendermint_materializer::bencher::Bencher; pub type DockerTestnet = Testnet; @@ -72,6 +73,7 @@ where Arc, Arc, usize, + Arc ) -> Pin> + Send>> + Copy + Send @@ -118,16 +120,16 @@ where let manifest = Arc::new(manifest); let res = if started { match concurrency { - None => test(manifest.clone(), materializer.clone(), testnet.clone(), 0).await, + None => test(manifest.clone(), materializer.clone(), testnet.clone(), 0, Arc::new(Bencher::new())).await, Some(cfg) => { let test_generator = { let testnet = testnet.clone(); let materializer = materializer.clone(); - move |test_id| { + move |test_id: usize, bencher: Arc| { let manifest = manifest.clone(); let materializer = materializer.clone(); let testnet = testnet.clone(); - async move { test(manifest, materializer, testnet, test_id).await }.boxed() + async move { test(manifest, materializer, testnet, test_id, bencher).await }.boxed() } }; @@ -136,13 +138,13 @@ where for res in results.into_iter() { match res.err { None => println!( - "test completed successfully (test_id={}, duration={:?})", - res.test_id, res.duration + "test completed successfully (test_id={}, records={:?})", + res.test_id, res.records ), Some(e) => { println!( - "test failed (test_id={}, duration={:?})", - res.test_id, res.duration + "test failed (test_id={}, records={:?})", + res.test_id, res.records ); err = Some(e); } diff --git a/fendermint/testing/materializer/tests/docker_tests/layer2.rs b/fendermint/testing/materializer/tests/docker_tests/layer2.rs index e711bb38d..70d2064b5 100644 --- a/fendermint/testing/materializer/tests/docker_tests/layer2.rs +++ b/fendermint/testing/materializer/tests/docker_tests/layer2.rs @@ -37,7 +37,7 @@ async fn test_topdown_and_bottomup() { subnet.bottom_up_checkpoint.period = CHECKPOINT_PERIOD; }, - |_, _, testnet, _| { + |_, _, testnet, _, _| { let test = async move { let brussels = testnet.node(&testnet.root().node("brussels"))?; let london = testnet.node(&testnet.root().subnet("england").node("london"))?; diff --git a/fendermint/testing/materializer/tests/docker_tests/root_only.rs b/fendermint/testing/materializer/tests/docker_tests/root_only.rs index 16e22cee7..39f164090 100644 --- a/fendermint/testing/materializer/tests/docker_tests/root_only.rs +++ b/fendermint/testing/materializer/tests/docker_tests/root_only.rs @@ -18,7 +18,7 @@ async fn test_full_node_sync() { MANIFEST, None, |_| {}, - |_, _, testnet, _| { + |_, _, testnet, _, _| { let test = async move { // Allow a little bit of time for node-2 to catch up with node-1. tokio::time::sleep(Duration::from_secs(5)).await; diff --git a/fendermint/testing/materializer/tests/docker_tests/standalone.rs b/fendermint/testing/materializer/tests/docker_tests/standalone.rs index fbe64da73..256273aa8 100644 --- a/fendermint/testing/materializer/tests/docker_tests/standalone.rs +++ b/fendermint/testing/materializer/tests/docker_tests/standalone.rs @@ -16,6 +16,7 @@ use fendermint_materializer::{ concurrency, manifest::Rootnet, materials::DefaultAccount, HasEthApi, }; use futures::FutureExt; +use tokio::time::sleep; const MANIFEST: &str = "standalone.yaml"; @@ -47,18 +48,78 @@ where async fn test_sent_tx_found_in_mempool() { with_testnet( MANIFEST, - Some( - concurrency::Config::new() - .with_max_concurrency(50) - .with_duration(Duration::from_secs(10)), - ), + None, |manifest| { // Slow down consensus to where we can see the effect of the transaction not being found by Ethereum hash. if let Rootnet::New { ref mut env, .. } = manifest.rootnet { env.insert("CMT_CONSENSUS_TIMEOUT_COMMIT".into(), "10s".into()); }; }, - |_, _, testnet, test_id| { + |_, _, testnet, test_id, _| { + let test = async move { + let sender = testnet.account_mod_nth(test_id); + let recipient = testnet.account_mod_nth(test_id + 1); + let pangea = testnet.node(&testnet.root().node("pangea"))?; + let provider = pangea + .ethapi_http_provider()? + .expect("ethapi should be enabled"); + + let middleware = make_middleware(provider, sender) + .await + .context("failed to set up middleware")?; + + eprintln!("middleware ready, pending tests"); + + // Create the simplest transaction possible: send tokens between accounts. + let to: H160 = recipient.eth_addr().into(); + let transfer = Eip1559TransactionRequest::new().to(to).value(1); + + let pending: PendingTransaction<_> = middleware + .send_transaction(transfer, None) + .await + .context("failed to send txn")?; + + let tx_hash = pending.tx_hash(); + + eprintln!("sent pending txn {:?}", tx_hash); + + // We expect that the transaction is pending, however it should not return an error. + match middleware.get_transaction(tx_hash).await { + Ok(Some(_)) => {} + Ok(None) => bail!("pending transaction not found by eth hash"), + Err(e) => { + bail!("failed to get pending transaction: {e}") + } + } + + Ok(()) + }; + + test.boxed() + }, + ) + .await + .unwrap() +} + + +#[serial_test::serial] +#[tokio::test] +async fn test_sent_tx_included_in_block() { + with_testnet( + MANIFEST, + Some( + concurrency::Config::new() + .with_max_concurrency(50) + .with_duration(Duration::from_secs(30)), + ), + |_| { + // Slow down consensus to where we can see the effect of the transaction not being found by Ethereum hash. + // if let Rootnet::New { ref mut env, .. } = manifest.rootnet { + // env.insert("CMT_CONSENSUS_TIMEOUT_COMMIT".into(), "10s".into()); + // }; + }, + |_, _, testnet, test_id, bencher| { let test = async move { let sender = testnet.account_mod_nth(test_id); let recipient = testnet.account_mod_nth(test_id + 1); @@ -88,6 +149,7 @@ async fn test_sent_tx_found_in_mempool() { .value(1) .nonce(nonce); + bencher.start().await; let pending: PendingTransaction<_> = middleware .send_transaction(transfer, None) .await @@ -105,6 +167,17 @@ async fn test_sent_tx_found_in_mempool() { bail!("failed to get pending transaction: {e}") } } + bencher.record("mempool".to_string()).await; + + // TODO: improve the polling or subscribe to some stream + loop { + if let Ok(Some(tx)) = middleware.get_transaction_receipt(tx_hash).await { + println!("tx included in block {:?} (test_id={})", tx.block_number, test_id); + break; + } + sleep(Duration::from_millis(100)).await; + } + bencher.record("block_inclusion".to_string()).await; Ok(()) }; @@ -112,8 +185,8 @@ async fn test_sent_tx_found_in_mempool() { test.boxed() }, ) - .await - .unwrap() + .await + .unwrap() } // /// Test that transactions sent out-of-order with regards to the nonce are not rejected, From 3cbde9d81b2d5a3ca3231278d9ac98f8c09b7e3d Mon Sep 17 00:00:00 2001 From: LePremierHomme <57456510+LePremierHomme@users.noreply.github.com> Date: Mon, 6 Jan 2025 16:39:31 +0200 Subject: [PATCH 04/14] add `NonceManager` --- .../testing/materializer/src/concurrency.rs | 117 ------------------ .../materializer/src/concurrency/config.rs | 28 +++++ .../materializer/src/concurrency/mod.rs | 61 +++++++++ .../src/concurrency/nonce_manager.rs | 32 +++++ .../materializer/src/concurrency/reporting.rs | 83 +++++++++++++ .../testing/materializer/tests/docker.rs | 58 +++++---- .../materializer/tests/docker_tests/layer2.rs | 3 +- .../tests/docker_tests/root_only.rs | 3 +- .../tests/docker_tests/standalone.rs | 42 ++++--- 9 files changed, 267 insertions(+), 160 deletions(-) delete mode 100644 fendermint/testing/materializer/src/concurrency.rs create mode 100644 fendermint/testing/materializer/src/concurrency/config.rs create mode 100644 fendermint/testing/materializer/src/concurrency/mod.rs create mode 100644 fendermint/testing/materializer/src/concurrency/nonce_manager.rs create mode 100644 fendermint/testing/materializer/src/concurrency/reporting.rs diff --git a/fendermint/testing/materializer/src/concurrency.rs b/fendermint/testing/materializer/src/concurrency.rs deleted file mode 100644 index b5de900b8..000000000 --- a/fendermint/testing/materializer/src/concurrency.rs +++ /dev/null @@ -1,117 +0,0 @@ -use std::collections::HashMap; -use futures::{FutureExt}; -use std::future::Future; -use std::pin::Pin; -use std::sync::Arc; -use std::time::{Duration, Instant}; -use tokio::sync::{Semaphore}; -use crate::bencher::Bencher; - -pub async fn execute(cfg: Config, test: F) -> (ExecutionSummary, Vec) -where - F: Fn(usize, Arc) -> Pin> + Send>>, -{ - let semaphore = Arc::new(Semaphore::new(cfg.max_concurrency)); - let mut test_id = 0; - let mut handles = Vec::new(); - let results = Arc::new(tokio::sync::Mutex::new(Vec::new())); - let execution_start = Instant::now(); - loop { - if execution_start.elapsed() > cfg.duration { - break; - } - let permit = semaphore.clone().acquire_owned().await.unwrap(); - let bencher = Arc::new(Bencher::new()); - let task = test(test_id, bencher.clone()).boxed(); - let results = results.clone(); - let handle = tokio::spawn(async move { - let result = task.await; - let records = bencher.records.lock().await.clone(); - results.lock().await.push(ExecutionResult { - test_id, - records, - err: result.err(), - }); - drop(permit); - }); - handles.push(handle); - test_id = test_id + 1; - } - - // Exhaust unfinished handles. - for handle in handles { - handle.await.unwrap(); - } - - let results = Arc::try_unwrap(results).unwrap().into_inner(); - let summary = ExecutionSummary::new(cfg, &results); - (summary, results) -} - -#[derive(Debug)] -pub struct Config { - pub max_concurrency: usize, - pub duration: Duration, -} - -impl Config { - pub fn new() -> Self { - Self { - max_concurrency: 1, - duration: Duration::from_secs(1), - } - } - - pub fn with_max_concurrency(mut self, max_concurrency: usize) -> Self { - self.max_concurrency = max_concurrency; - self - } - - pub fn with_duration(mut self, duration: Duration) -> Self { - self.duration = duration; - self - } -} -#[derive(Debug)] -pub struct ExecutionResult { - pub test_id: usize, - pub records: HashMap, - pub err: Option, -} - -#[derive(Debug)] -pub struct ExecutionSummary { - pub cfg: Config, - pub avg_latencies: HashMap, - pub num_failures: usize, -} - -impl ExecutionSummary { - fn new(cfg: Config, results: &Vec) -> Self { - let num_failures = results.iter().filter(|res| res.err.is_some()).count(); - - let mut total_durations: HashMap = HashMap::new(); - let mut counts: HashMap = HashMap::new(); - for res in results { - for (key, duration) in res.records.clone() { - *total_durations.entry(key.clone()).or_insert(Duration::ZERO) += duration; - *counts.entry(key).or_insert(0) += 1; - } - } - - let avg_latencies = total_durations - .into_iter() - .map(|(key, total)| { - let count = counts[&key]; - (key, total / count as u32) - }) - .collect(); - - Self { - cfg, - avg_latencies, - num_failures, - } - } -} - diff --git a/fendermint/testing/materializer/src/concurrency/config.rs b/fendermint/testing/materializer/src/concurrency/config.rs new file mode 100644 index 000000000..9aa8768a0 --- /dev/null +++ b/fendermint/testing/materializer/src/concurrency/config.rs @@ -0,0 +1,28 @@ +use std::time::Duration; + +#[derive(Debug, Clone)] +pub struct Execution { + pub steps: Vec, +} + +impl Execution { + pub fn new() -> Self { + Self { + steps: Vec::new(), + } + } + + pub fn add_step(mut self, max_concurrency: usize, secs: u64) -> Self { + self.steps.push(ExecutionStep { + max_concurrency, + duration: Duration::from_secs(secs) + }); + self + } +} + +#[derive(Debug, Clone)] +pub struct ExecutionStep { + pub max_concurrency: usize, + pub duration: Duration, +} diff --git a/fendermint/testing/materializer/src/concurrency/mod.rs b/fendermint/testing/materializer/src/concurrency/mod.rs new file mode 100644 index 000000000..d7e04ad2c --- /dev/null +++ b/fendermint/testing/materializer/src/concurrency/mod.rs @@ -0,0 +1,61 @@ +pub mod config; +pub mod reporting; +pub mod nonce_manager; + +pub use reporting::*; + +use futures::{FutureExt}; +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use ethers::types::spoof::nonce; +use tokio::sync::{Semaphore}; +use crate::bencher::Bencher; +use crate::concurrency::nonce_manager::NonceManager; + +pub async fn execute(cfg: config::Execution, test: F) -> Vec> +where + F: Fn(usize, Arc) -> Pin> + Send>>, +{ + let mut test_id = 0; + let mut results = Vec::new(); + for (step_id, step) in cfg.steps.iter().enumerate() { + let semaphore = Arc::new(Semaphore::new(step.max_concurrency)); + let mut handles = Vec::new(); + let step_results = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let execution_start = Instant::now(); + loop { + if execution_start.elapsed() > step.duration { + break; + } + let permit = semaphore.clone().acquire_owned().await.unwrap(); + let bencher = Arc::new(Bencher::new()); + let task = test(test_id, bencher.clone()).boxed(); + let step_results = step_results.clone(); + let handle = tokio::spawn(async move { + let result = task.await; + let records = bencher.records.lock().await.clone(); + step_results.lock().await.push(TestResult { + test_id, + step_id, + records, + err: result.err(), + }); + drop(permit); + }); + handles.push(handle); + test_id = test_id + 1; + } + + // Exhaust unfinished handles. + for handle in handles { + handle.await.unwrap(); + } + + let step_results = Arc::try_unwrap(step_results).unwrap().into_inner(); + results.push(step_results) + } + results +} + diff --git a/fendermint/testing/materializer/src/concurrency/nonce_manager.rs b/fendermint/testing/materializer/src/concurrency/nonce_manager.rs new file mode 100644 index 000000000..f7c70e5da --- /dev/null +++ b/fendermint/testing/materializer/src/concurrency/nonce_manager.rs @@ -0,0 +1,32 @@ +use std::collections::HashMap; +use std::sync::Arc; +use ethers::prelude::H160; +use ethers::types::U256; +use tokio::sync::Mutex; + +pub struct NonceManager { + nonces: Arc>> +} + +impl NonceManager { + pub fn new() -> Self { + NonceManager { + nonces: Arc::new(Mutex::new(HashMap::new())) + } + } + + pub async fn set(&self, addr: H160, amount: U256) { + let mut nonces = self.nonces.lock().await; + nonces.insert(addr, amount); + println!("insert {:?} {:?}", addr, amount); + } + + pub async fn get_and_increment(&self, addr: H160) -> U256 { + let mut nonces = self.nonces.lock().await; + let next_nonce = nonces.entry(addr).or_insert_with(U256::zero); + let current_nonce = (*next_nonce).clone(); + *next_nonce += U256::one(); + println!("get_and_increment {:?} {:?}", addr, current_nonce); + current_nonce + } +} \ No newline at end of file diff --git a/fendermint/testing/materializer/src/concurrency/reporting.rs b/fendermint/testing/materializer/src/concurrency/reporting.rs new file mode 100644 index 000000000..87d594bd2 --- /dev/null +++ b/fendermint/testing/materializer/src/concurrency/reporting.rs @@ -0,0 +1,83 @@ +use std::collections::HashMap; +use std::time::Duration; +use crate::concurrency::config; +use crate::concurrency::config::ExecutionStep; + +#[derive(Debug)] +pub struct TestResult { + pub test_id: usize, + pub step_id: usize, + pub records: HashMap, + pub err: Option, +} + +#[derive(Debug)] +pub struct StepSummary { + pub cfg: config::ExecutionStep, + pub avg_latencies: HashMap, + pub errs: Vec +} + +impl StepSummary { + fn new(cfg: ExecutionStep, results: Vec) -> Self { + let mut total_durations: HashMap = HashMap::new(); + let mut counts: HashMap = HashMap::new(); + let mut errs = Vec::new(); + for res in results { + for (key, duration) in res.records.clone() { + *total_durations.entry(key.clone()).or_insert(Duration::ZERO) += duration; + *counts.entry(key).or_insert(0) += 1; + } + if let Some(err) = res.err { + errs.push(err); + } + } + + let avg_latencies = total_durations + .into_iter() + .map(|(key, total)| { + let count = counts[&key]; + (key, total / count as u32) + }) + .collect(); + + Self { + cfg, + avg_latencies, + errs + } + } +} + +#[derive(Debug)] +pub struct ExecutionSummary { + pub summaries: Vec +} + +impl ExecutionSummary { + pub fn new(cfg: config::Execution, results: Vec>) -> Self { + let mut summaries = Vec::new(); + for (i, step_results) in results.into_iter().enumerate() { + let cfg = cfg.steps[i].clone(); + summaries.push(StepSummary::new(cfg, step_results)); + } + + Self { + summaries + } + } + + pub fn errs(&self) -> Vec { + let mut errs = Vec::new(); + for summary in self.summaries.iter() { + let cloned_errs: Vec = summary.errs + .iter() + .map(|e| + format!("{:?}", e)) + // e.chain().map(|cause|cause.to_string()).collect::>().join(" -> ")) + .collect(); + errs.extend(cloned_errs); + } + errs + } +} \ No newline at end of file diff --git a/fendermint/testing/materializer/tests/docker.rs b/fendermint/testing/materializer/tests/docker.rs index fa56a00e7..24dda5bc7 100644 --- a/fendermint/testing/materializer/tests/docker.rs +++ b/fendermint/testing/materializer/tests/docker.rs @@ -27,6 +27,7 @@ use std::{ }; use tendermint_rpc::Client; use fendermint_materializer::bencher::Bencher; +use fendermint_materializer::concurrency::nonce_manager::NonceManager; pub type DockerTestnet = Testnet; @@ -61,24 +62,30 @@ fn read_manifest(file_name: &str) -> anyhow::Result { /// Parse a manifest file in the `manifests` directory, clean up any corresponding /// testnet resources, then materialize a testnet and run some tests. -pub async fn with_testnet( +pub async fn with_testnet( manifest_file_name: &str, - concurrency: Option, + concurrency: Option, alter: G, + init: I, test: F, ) -> anyhow::Result<()> where + G: FnOnce(&mut Manifest), + I: FnOnce( + Arc, + Arc + ) -> Pin>>, F: for<'a> Fn( Arc, Arc, Arc, usize, - Arc + Arc, + Arc ) -> Pin> + Send>> + Copy + Send + 'static, - G: FnOnce(&mut Manifest), { let testnet_name = TestnetName::new( PathBuf::from(manifest_file_name) @@ -118,9 +125,21 @@ where let testnet = Arc::new(testnet); let materializer = Arc::new(materializer); let manifest = Arc::new(manifest); + let nonce_manager = Arc::new(NonceManager::new()); let res = if started { + init( + testnet.clone(), + nonce_manager.clone() + ).await; match concurrency { - None => test(manifest.clone(), materializer.clone(), testnet.clone(), 0, Arc::new(Bencher::new())).await, + None => test( + manifest.clone(), + materializer.clone(), + testnet.clone(), + 0, + Arc::new(Bencher::new()), + nonce_manager.clone() + ).await, Some(cfg) => { let test_generator = { let testnet = testnet.clone(); @@ -129,29 +148,20 @@ where let manifest = manifest.clone(); let materializer = materializer.clone(); let testnet = testnet.clone(); - async move { test(manifest, materializer, testnet, test_id, bencher).await }.boxed() + let nonce_manager = nonce_manager.clone(); + async move { test(manifest, materializer, testnet, test_id, bencher, nonce_manager).await }.boxed() } }; - let (summary, results) = concurrency::execute(cfg, test_generator).await; - let mut err = None; - for res in results.into_iter() { - match res.err { - None => println!( - "test completed successfully (test_id={}, records={:?})", - res.test_id, res.records - ), - Some(e) => { - println!( - "test failed (test_id={}, records={:?})", - res.test_id, res.records - ); - err = Some(e); - } - } - } + let results = concurrency::execute(cfg.clone(), test_generator).await; + let summary = concurrency::ExecutionSummary::new(cfg.clone(), results); println!("{:?}", summary); - err.map_or(Ok(()), Err) + + let errs = summary.errs(); + match errs.is_empty() { + true => Ok(()), + false => Err(anyhow::anyhow!(errs.join("\n"))), + } } } } else { diff --git a/fendermint/testing/materializer/tests/docker_tests/layer2.rs b/fendermint/testing/materializer/tests/docker_tests/layer2.rs index 70d2064b5..b6064abb7 100644 --- a/fendermint/testing/materializer/tests/docker_tests/layer2.rs +++ b/fendermint/testing/materializer/tests/docker_tests/layer2.rs @@ -37,7 +37,8 @@ async fn test_topdown_and_bottomup() { subnet.bottom_up_checkpoint.period = CHECKPOINT_PERIOD; }, - |_, _, testnet, _, _| { + |_, _| Box::pin(async { () }), + |_, _, testnet, _, _, _| { let test = async move { let brussels = testnet.node(&testnet.root().node("brussels"))?; let london = testnet.node(&testnet.root().subnet("england").node("london"))?; diff --git a/fendermint/testing/materializer/tests/docker_tests/root_only.rs b/fendermint/testing/materializer/tests/docker_tests/root_only.rs index 39f164090..a5fb380d6 100644 --- a/fendermint/testing/materializer/tests/docker_tests/root_only.rs +++ b/fendermint/testing/materializer/tests/docker_tests/root_only.rs @@ -18,7 +18,8 @@ async fn test_full_node_sync() { MANIFEST, None, |_| {}, - |_, _, testnet, _, _| { + |_,_| { Box::pin(async { })}, + |_, _, testnet, _, _, _| { let test = async move { // Allow a little bit of time for node-2 to catch up with node-1. tokio::time::sleep(Duration::from_secs(5)).await; diff --git a/fendermint/testing/materializer/tests/docker_tests/standalone.rs b/fendermint/testing/materializer/tests/docker_tests/standalone.rs index 256273aa8..38bc35ff6 100644 --- a/fendermint/testing/materializer/tests/docker_tests/standalone.rs +++ b/fendermint/testing/materializer/tests/docker_tests/standalone.rs @@ -1,6 +1,7 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT +use std::ops::Add; use std::time::{Duration}; use crate::with_testnet; @@ -12,8 +13,9 @@ use ethers::{ signers::{Signer, Wallet}, types::{Eip1559TransactionRequest, H160}, }; +use ethers::types::{BlockId, U256}; use fendermint_materializer::{ - concurrency, manifest::Rootnet, materials::DefaultAccount, HasEthApi, + concurrency::{self, config::Execution}, manifest::Rootnet, materials::DefaultAccount, HasEthApi, }; use futures::FutureExt; use tokio::time::sleep; @@ -55,7 +57,8 @@ async fn test_sent_tx_found_in_mempool() { env.insert("CMT_CONSENSUS_TIMEOUT_COMMIT".into(), "10s".into()); }; }, - |_, _, testnet, test_id, _| { + |_,_| { Box::pin(async { })}, + |_, _, testnet, test_id, _, _| { let test = async move { let sender = testnet.account_mod_nth(test_id); let recipient = testnet.account_mod_nth(test_id + 1); @@ -102,28 +105,31 @@ async fn test_sent_tx_found_in_mempool() { .unwrap() } - #[serial_test::serial] #[tokio::test] async fn test_sent_tx_included_in_block() { with_testnet( MANIFEST, - Some( - concurrency::Config::new() - .with_max_concurrency(50) - .with_duration(Duration::from_secs(30)), + Some(Execution::new() + .add_step(10, 5) + .add_step(100, 5) ), - |_| { + |manifest| { // Slow down consensus to where we can see the effect of the transaction not being found by Ethereum hash. - // if let Rootnet::New { ref mut env, .. } = manifest.rootnet { - // env.insert("CMT_CONSENSUS_TIMEOUT_COMMIT".into(), "10s".into()); - // }; + if let Rootnet::New { ref mut env, .. } = manifest.rootnet { + env.insert("CMT_CONSENSUS_TIMEOUT_COMMIT".into(), "1s".into()); + }; + }, + |testnet,nonce_manager| { + Box::pin(async move { + }) }, - |_, _, testnet, test_id, bencher| { + |_, _, testnet, test_id, bencher, nonce_manager| { let test = async move { let sender = testnet.account_mod_nth(test_id); let recipient = testnet.account_mod_nth(test_id + 1); println!("running (test_id={})", test_id); + let pangea = testnet.node(&testnet.root().node("pangea"))?; let provider = pangea .ethapi_http_provider()? @@ -136,11 +142,11 @@ async fn test_sent_tx_included_in_block() { println!("middleware ready, pending tests (test_id={})", test_id); let sender: H160 = sender.eth_addr().into(); - let current_nonce = middleware - .get_transaction_count(sender, None) - .await - .context("failed to fetch nonce")?; - let nonce = current_nonce + 1; + // let current_nonce = middleware + // .get_transaction_count(sender, None) + // .await + // .context("failed to fetch nonce")?; + let nonce = nonce_manager.get_and_increment(sender).await; // Create the simplest transaction possible: send tokens between accounts. let to: H160 = recipient.eth_addr().into(); @@ -149,6 +155,7 @@ async fn test_sent_tx_included_in_block() { .value(1) .nonce(nonce); + bencher.start().await; let pending: PendingTransaction<_> = middleware .send_transaction(transfer, None) @@ -173,6 +180,7 @@ async fn test_sent_tx_included_in_block() { loop { if let Ok(Some(tx)) = middleware.get_transaction_receipt(tx_hash).await { println!("tx included in block {:?} (test_id={})", tx.block_number, test_id); + break; } sleep(Duration::from_millis(100)).await; From 2de3e9af6cca053c6f8875b9f3722b97783b77b8 Mon Sep 17 00:00:00 2001 From: LePremierHomme <57456510+LePremierHomme@users.noreply.github.com> Date: Tue, 7 Jan 2025 13:46:07 +0200 Subject: [PATCH 05/14] `with_testnet` -> `make_testnet` --- .../testing/materializer/src/bencher.rs | 26 +- .../materializer/src/concurrency/config.rs | 6 +- .../materializer/src/concurrency/mod.rs | 32 +-- .../src/concurrency/nonce_manager.rs | 14 +- .../materializer/src/concurrency/reporting.rs | 40 +-- .../testing/materializer/src/docker/mod.rs | 1 + fendermint/testing/materializer/src/lib.rs | 2 +- .../testing/materializer/tests/docker.rs | 138 +++-------- .../tests/docker_tests/benches.rs | 138 +++++++++++ .../materializer/tests/docker_tests/layer2.rs | 233 +++++++++--------- .../materializer/tests/docker_tests/mod.rs | 1 + .../tests/docker_tests/root_only.rs | 64 ++--- .../tests/docker_tests/standalone.rs | 211 ++++------------ .../materializer/tests/manifests/benches.yaml | 120 +++++++++ .../tests/manifests/standalone.yaml | 111 +-------- 15 files changed, 560 insertions(+), 577 deletions(-) create mode 100644 fendermint/testing/materializer/tests/docker_tests/benches.rs create mode 100644 fendermint/testing/materializer/tests/manifests/benches.yaml diff --git a/fendermint/testing/materializer/src/bencher.rs b/fendermint/testing/materializer/src/bencher.rs index 3ddc61f27..bfb7af821 100644 --- a/fendermint/testing/materializer/src/bencher.rs +++ b/fendermint/testing/materializer/src/bencher.rs @@ -1,34 +1,26 @@ use std::collections::HashMap; -use std::sync::Arc; use std::time::{Duration, Instant}; -use tokio::sync::Mutex; -// TODO: remove the Arc> wrappers. -// they are not needed, as a Bencher instance shouldn't be shared among tests. -// it's just a workaround to issues involving mutable references lifetime in futures. #[derive(Debug, Clone)] pub struct Bencher { - pub start_time: Arc>>, - pub records: Arc>>, + pub start_time: Option, + pub records: HashMap, } impl Bencher { pub fn new() -> Self { Self { - start_time: Arc::new(Mutex::new(None)), - records: Arc::new(Mutex::new(HashMap::new())), + start_time: None, + records: HashMap::new(), } } - pub async fn start(&self) { - let mut start_time = self.start_time.lock().await; - *start_time = Some(Instant::now()); + pub async fn start(&mut self) { + self.start_time = Some(Instant::now()); } - pub async fn record(&self, label: String) { - let start_time = self.start_time.lock().await; - let duration = start_time.unwrap().elapsed(); - let mut records = self.records.lock().await; - records.insert(label, duration); + pub async fn record(&mut self, label: String) { + let duration = self.start_time.unwrap().elapsed(); + self.records.insert(label, duration); } } diff --git a/fendermint/testing/materializer/src/concurrency/config.rs b/fendermint/testing/materializer/src/concurrency/config.rs index 9aa8768a0..80a334c0f 100644 --- a/fendermint/testing/materializer/src/concurrency/config.rs +++ b/fendermint/testing/materializer/src/concurrency/config.rs @@ -7,15 +7,13 @@ pub struct Execution { impl Execution { pub fn new() -> Self { - Self { - steps: Vec::new(), - } + Self { steps: Vec::new() } } pub fn add_step(mut self, max_concurrency: usize, secs: u64) -> Self { self.steps.push(ExecutionStep { max_concurrency, - duration: Duration::from_secs(secs) + duration: Duration::from_secs(secs), }); self } diff --git a/fendermint/testing/materializer/src/concurrency/mod.rs b/fendermint/testing/materializer/src/concurrency/mod.rs index d7e04ad2c..a87d1f265 100644 --- a/fendermint/testing/materializer/src/concurrency/mod.rs +++ b/fendermint/testing/materializer/src/concurrency/mod.rs @@ -1,22 +1,20 @@ pub mod config; -pub mod reporting; pub mod nonce_manager; +pub mod reporting; pub use reporting::*; -use futures::{FutureExt}; +use crate::bencher::Bencher; +use futures::FutureExt; use std::future::Future; use std::pin::Pin; use std::sync::Arc; -use std::time::{Duration, Instant}; -use ethers::types::spoof::nonce; -use tokio::sync::{Semaphore}; -use crate::bencher::Bencher; -use crate::concurrency::nonce_manager::NonceManager; +use std::time::Instant; +use tokio::sync::Semaphore; -pub async fn execute(cfg: config::Execution, test: F) -> Vec> +pub async fn execute(cfg: config::Execution, test_factory: F) -> Vec> where - F: Fn(usize, Arc) -> Pin> + Send>>, + F: Fn(usize, Bencher) -> Pin> + Send>>, { let mut test_id = 0; let mut results = Vec::new(); @@ -30,17 +28,20 @@ where break; } let permit = semaphore.clone().acquire_owned().await.unwrap(); - let bencher = Arc::new(Bencher::new()); - let task = test(test_id, bencher.clone()).boxed(); + let bencher = Bencher::new(); + let task = test_factory(test_id, bencher).boxed(); let step_results = step_results.clone(); let handle = tokio::spawn(async move { - let result = task.await; - let records = bencher.records.lock().await.clone(); + let res = task.await; + let (bencher, err) = match res { + Ok(bencher) => (Some(bencher), None), + Err(err) => (None, Some(err)), + }; step_results.lock().await.push(TestResult { test_id, step_id, - records, - err: result.err(), + bencher, + err, }); drop(permit); }); @@ -58,4 +59,3 @@ where } results } - diff --git a/fendermint/testing/materializer/src/concurrency/nonce_manager.rs b/fendermint/testing/materializer/src/concurrency/nonce_manager.rs index f7c70e5da..446594c21 100644 --- a/fendermint/testing/materializer/src/concurrency/nonce_manager.rs +++ b/fendermint/testing/materializer/src/concurrency/nonce_manager.rs @@ -1,32 +1,30 @@ -use std::collections::HashMap; -use std::sync::Arc; use ethers::prelude::H160; use ethers::types::U256; +use std::collections::HashMap; +use std::sync::Arc; use tokio::sync::Mutex; pub struct NonceManager { - nonces: Arc>> + nonces: Arc>>, } impl NonceManager { pub fn new() -> Self { NonceManager { - nonces: Arc::new(Mutex::new(HashMap::new())) + nonces: Arc::new(Mutex::new(HashMap::new())), } } pub async fn set(&self, addr: H160, amount: U256) { let mut nonces = self.nonces.lock().await; nonces.insert(addr, amount); - println!("insert {:?} {:?}", addr, amount); } pub async fn get_and_increment(&self, addr: H160) -> U256 { let mut nonces = self.nonces.lock().await; let next_nonce = nonces.entry(addr).or_insert_with(U256::zero); - let current_nonce = (*next_nonce).clone(); + let current_nonce = *next_nonce; *next_nonce += U256::one(); - println!("get_and_increment {:?} {:?}", addr, current_nonce); current_nonce } -} \ No newline at end of file +} diff --git a/fendermint/testing/materializer/src/concurrency/reporting.rs b/fendermint/testing/materializer/src/concurrency/reporting.rs index 87d594bd2..28731ede3 100644 --- a/fendermint/testing/materializer/src/concurrency/reporting.rs +++ b/fendermint/testing/materializer/src/concurrency/reporting.rs @@ -1,21 +1,23 @@ -use std::collections::HashMap; -use std::time::Duration; +use crate::bencher::Bencher; use crate::concurrency::config; use crate::concurrency::config::ExecutionStep; +use anyhow::anyhow; +use std::collections::HashMap; +use std::time::Duration; #[derive(Debug)] pub struct TestResult { pub test_id: usize, pub step_id: usize, - pub records: HashMap, + pub bencher: Option, pub err: Option, } #[derive(Debug)] pub struct StepSummary { - pub cfg: config::ExecutionStep, + pub cfg: ExecutionStep, pub avg_latencies: HashMap, - pub errs: Vec + pub errs: Vec, } impl StepSummary { @@ -24,7 +26,8 @@ impl StepSummary { let mut counts: HashMap = HashMap::new(); let mut errs = Vec::new(); for res in results { - for (key, duration) in res.records.clone() { + let Some(bencher) = res.bencher else { continue }; + for (key, duration) in bencher.records.clone() { *total_durations.entry(key.clone()).or_insert(Duration::ZERO) += duration; *counts.entry(key).or_insert(0) += 1; } @@ -44,14 +47,14 @@ impl StepSummary { Self { cfg, avg_latencies, - errs + errs, } } } #[derive(Debug)] pub struct ExecutionSummary { - pub summaries: Vec + pub summaries: Vec, } impl ExecutionSummary { @@ -62,22 +65,29 @@ impl ExecutionSummary { summaries.push(StepSummary::new(cfg, step_results)); } - Self { - summaries + Self { summaries } + } + + pub fn to_result(&self) -> anyhow::Result<()> { + let errs = self.errs(); + if errs.is_empty() { + Ok(()) + } else { + Err(anyhow!(errs.join("\n"))) } } pub fn errs(&self) -> Vec { let mut errs = Vec::new(); for summary in self.summaries.iter() { - let cloned_errs: Vec = summary.errs + let cloned_errs: Vec = summary + .errs .iter() - .map(|e| - format!("{:?}", e)) - // e.chain().map(|cause|cause.to_string()).collect::>().join(" -> ")) + .map(|e| format!("{:?}", e)) + // e.chain().map(|cause|cause.to_string()).collect::>().join(" -> ")) .collect(); errs.extend(cloned_errs); } errs } -} \ No newline at end of file +} diff --git a/fendermint/testing/materializer/src/docker/mod.rs b/fendermint/testing/materializer/src/docker/mod.rs index 551bee223..7e8fdcc9a 100644 --- a/fendermint/testing/materializer/src/docker/mod.rs +++ b/fendermint/testing/materializer/src/docker/mod.rs @@ -90,6 +90,7 @@ macro_rules! env_vars { }; } +#[derive(Debug)] pub struct DockerMaterials; impl Materials for DockerMaterials { diff --git a/fendermint/testing/materializer/src/lib.rs b/fendermint/testing/materializer/src/lib.rs index a6be40fa6..0ea4dd490 100644 --- a/fendermint/testing/materializer/src/lib.rs +++ b/fendermint/testing/materializer/src/lib.rs @@ -20,8 +20,8 @@ pub mod validation; #[cfg(feature = "arb")] mod arb; -pub mod concurrency; pub mod bencher; +pub mod concurrency; /// An ID identifying a resource within its parent. #[derive(Clone, Serialize, PartialEq, Eq, PartialOrd, Ord)] diff --git a/fendermint/testing/materializer/tests/docker.rs b/fendermint/testing/materializer/tests/docker.rs index 24dda5bc7..af492b4fd 100644 --- a/fendermint/testing/materializer/tests/docker.rs +++ b/fendermint/testing/materializer/tests/docker.rs @@ -9,7 +9,6 @@ use anyhow::{anyhow, Context}; use ethers::providers::Middleware; use fendermint_materializer::{ - concurrency, docker::{DockerMaterializer, DockerMaterials}, manifest::Manifest, testnet::Testnet, @@ -17,7 +16,6 @@ use fendermint_materializer::{ }; use futures::{Future, FutureExt}; use lazy_static::lazy_static; -use std::sync::Arc; use std::{ collections::BTreeSet, env::current_dir, @@ -26,8 +24,6 @@ use std::{ time::{Duration, Instant}, }; use tendermint_rpc::Client; -use fendermint_materializer::bencher::Bencher; -use fendermint_materializer::concurrency::nonce_manager::NonceManager; pub type DockerTestnet = Testnet; @@ -61,31 +57,16 @@ fn read_manifest(file_name: &str) -> anyhow::Result { } /// Parse a manifest file in the `manifests` directory, clean up any corresponding -/// testnet resources, then materialize a testnet and run some tests. -pub async fn with_testnet( +/// testnet resources, then materialize a testnet and provide a cleanup function. +pub async fn make_testnet( manifest_file_name: &str, - concurrency: Option, - alter: G, - init: I, - test: F, -) -> anyhow::Result<()> + alter: F, +) -> anyhow::Result<( + DockerTestnet, + Box Pin + Send>> + Send>, +)> where - G: FnOnce(&mut Manifest), - I: FnOnce( - Arc, - Arc - ) -> Pin>>, - F: for<'a> Fn( - Arc, - Arc, - Arc, - usize, - Arc, - Arc - ) -> Pin> + Send>> - + Copy - + Send - + 'static, + F: FnOnce(&mut Manifest), { let testnet_name = TestnetName::new( PathBuf::from(manifest_file_name) @@ -121,83 +102,44 @@ where .context("failed to set up testnet")?; let started = wait_for_startup(&testnet).await?; + if !started { + return Err(anyhow!("the startup sequence timed out")); + } - let testnet = Arc::new(testnet); - let materializer = Arc::new(materializer); - let manifest = Arc::new(manifest); - let nonce_manager = Arc::new(NonceManager::new()); - let res = if started { - init( - testnet.clone(), - nonce_manager.clone() - ).await; - match concurrency { - None => test( - manifest.clone(), - materializer.clone(), - testnet.clone(), - 0, - Arc::new(Bencher::new()), - nonce_manager.clone() - ).await, - Some(cfg) => { - let test_generator = { - let testnet = testnet.clone(); - let materializer = materializer.clone(); - move |test_id: usize, bencher: Arc| { - let manifest = manifest.clone(); - let materializer = materializer.clone(); - let testnet = testnet.clone(); - let nonce_manager = nonce_manager.clone(); - async move { test(manifest, materializer, testnet, test_id, bencher, nonce_manager).await }.boxed() + let cleanup = Box::new(|failure: bool, testnet: DockerTestnet| { + async move { + println!("Cleaning up..."); + + // Print all logs on failure. + // Some might be available in logs in the files which are left behind, + // e.g. for `fendermint` we have logs, but maybe not for `cometbft`. + if failure && *PRINT_LOGS_ON_ERROR { + for (name, node) in testnet.nodes() { + let name = name.path_string(); + for log in node.fendermint_logs().await { + eprintln!("{name}/fendermint: {log}"); + } + for log in node.cometbft_logs().await { + eprintln!("{name}/cometbft: {log}"); + } + for log in node.ethapi_logs().await { + eprintln!("{name}/ethapi: {log}"); } - }; - - let results = concurrency::execute(cfg.clone(), test_generator).await; - let summary = concurrency::ExecutionSummary::new(cfg.clone(), results); - println!("{:?}", summary); - - let errs = summary.errs(); - match errs.is_empty() { - true => Ok(()), - false => Err(anyhow::anyhow!(errs.join("\n"))), } } - } - } else { - Err(anyhow!("the startup sequence timed out")) - }; - - // Print all logs on failure. - // Some might be available in logs in the files which are left behind, - // e.g. for `fendermint` we have logs, but maybe not for `cometbft`. - if res.is_err() && *PRINT_LOGS_ON_ERROR { - for (name, node) in testnet.nodes() { - let name = name.path_string(); - for log in node.fendermint_logs().await { - eprintln!("{name}/fendermint: {log}"); - } - for log in node.cometbft_logs().await { - eprintln!("{name}/cometbft: {log}"); - } - for log in node.ethapi_logs().await { - eprintln!("{name}/ethapi: {log}"); - } - } - } - - // Tear down the testnet. - drop(testnet); - // Allow some time for containers to be dropped. - // This only happens if the testnet setup succeeded, - // otherwise the system shuts down too quick, but - // at least we can inspect the containers. - // If they don't all get dropped, `docker system prune` helps. - let drop_handle = Arc::try_unwrap(materializer).unwrap().take_dropper(); - let _ = tokio::time::timeout(*TEARDOWN_TIMEOUT, drop_handle).await; + // Allow some time for containers to be dropped. + // This only happens if the testnet setup succeeded, + // otherwise the system shuts down too quick, but + // at least we can inspect the containers. + // If they don't all get dropped, `docker system prune` helps. + let drop_handle = materializer.take_dropper(); + let _ = tokio::time::timeout(*TEARDOWN_TIMEOUT, drop_handle).await; + } + .boxed() + }); - res + Ok((testnet, cleanup)) } /// Allow time for things to consolidate and APIs to start. diff --git a/fendermint/testing/materializer/tests/docker_tests/benches.rs b/fendermint/testing/materializer/tests/docker_tests/benches.rs new file mode 100644 index 000000000..088c5bddc --- /dev/null +++ b/fendermint/testing/materializer/tests/docker_tests/benches.rs @@ -0,0 +1,138 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +use std::sync::Arc; +use std::time::Duration; + +use crate::make_testnet; +use anyhow::{bail, Context}; +use ethers::{ + core::k256::ecdsa::SigningKey, + middleware::SignerMiddleware, + providers::{JsonRpcClient, Middleware, PendingTransaction, Provider}, + signers::{Signer, Wallet}, + types::{Eip1559TransactionRequest, H160}, +}; +use fendermint_materializer::bencher::Bencher; +use fendermint_materializer::concurrency::nonce_manager::NonceManager; +use fendermint_materializer::{ + concurrency::{self, config::Execution}, + materials::DefaultAccount, + HasEthApi, +}; +use futures::FutureExt; +use tokio::time::sleep; + +const MANIFEST: &str = "benches.yaml"; + +pub type TestMiddleware = SignerMiddleware, Wallet>; + +/// Create a middleware that will assign nonces and sign the message. +async fn make_middleware( + provider: Provider, + sender: &DefaultAccount, +) -> anyhow::Result> +where + C: JsonRpcClient, +{ + let chain_id = provider + .get_chainid() + .await + .context("failed to get chain ID")?; + + let wallet: Wallet = Wallet::from_bytes(sender.secret_key().serialize().as_ref())? + .with_chain_id(chain_id.as_u64()); + + Ok(SignerMiddleware::new(provider, wallet)) +} + +#[serial_test::serial] +#[tokio::test] +async fn test_concurrent_transfer() -> Result<(), anyhow::Error> { + let (testnet, cleanup) = make_testnet(MANIFEST, |_| {}).await?; + + // Drive concurrency. + let cfg = Execution::new() + .add_step(10, 5) + .add_step(100, 5) + .add_step(200, 5); + let testnet = Arc::new(testnet); + let testnet_clone = testnet.clone(); + let nonce_manager = Arc::new(NonceManager::new()); + let results = concurrency::execute(cfg.clone(), move |test_id: usize, mut bencher: Bencher| { + let testnet = testnet_clone.clone(); + let nonce_manager = nonce_manager.clone(); + + let test = async move { + let sender = testnet.account_mod_nth(test_id); + let recipient = testnet.account_mod_nth(test_id + 1); + println!("running (test_id={})", test_id); + + let pangea = testnet.node(&testnet.root().node("pangea"))?; + let provider = pangea + .ethapi_http_provider()? + .expect("ethapi should be enabled"); + + let middleware = make_middleware(provider, sender) + .await + .context("failed to set up middleware")?; + + println!("middleware ready, pending tests (test_id={})", test_id); + + let sender: H160 = sender.eth_addr().into(); + let nonce = nonce_manager.get_and_increment(sender).await; + + // Create the simplest transaction possible: send tokens between accounts. + let to: H160 = recipient.eth_addr().into(); + let transfer = Eip1559TransactionRequest::new() + .to(to) + .value(1) + .nonce(nonce); + + bencher.start().await; + + let pending: PendingTransaction<_> = middleware + .send_transaction(transfer, None) + .await + .context("failed to send txn")?; + let tx_hash = pending.tx_hash(); + println!("sent pending txn {:?} (test_id={})", tx_hash, test_id); + + // We expect that the transaction is pending, however it should not return an error. + match middleware.get_transaction(tx_hash).await { + Ok(Some(_)) => {} + Ok(None) => bail!("pending transaction not found by eth hash"), + Err(e) => { + bail!("failed to get pending transaction: {e}") + } + } + bencher.record("mempool".to_string()).await; + + loop { + if let Ok(Some(tx)) = middleware.get_transaction_receipt(tx_hash).await { + println!( + "tx included in block {:?} (test_id={})", + tx.block_number, test_id + ); + break; + } + sleep(Duration::from_millis(100)).await; + } + bencher.record("block_inclusion".to_string()).await; + + Ok(bencher) + }; + test.boxed() + }) + .await; + + let summary = concurrency::ExecutionSummary::new(cfg.clone(), results); + println!("{:?}", summary); + + let Ok(testnet) = Arc::try_unwrap(testnet) else { + bail!("Arc::try_unwrap(testnet)"); + }; + let res = summary.to_result(); + cleanup(res.is_err(), testnet).await; + res +} diff --git a/fendermint/testing/materializer/tests/docker_tests/layer2.rs b/fendermint/testing/materializer/tests/docker_tests/layer2.rs index b6064abb7..11cf03c1c 100644 --- a/fendermint/testing/materializer/tests/docker_tests/layer2.rs +++ b/fendermint/testing/materializer/tests/docker_tests/layer2.rs @@ -3,7 +3,6 @@ use anyhow::{anyhow, bail, Context}; use ethers::core::types as et; use ethers::providers::Middleware; -use futures::FutureExt; use std::sync::Arc; use std::time::Duration; @@ -14,7 +13,7 @@ use fendermint_vm_message::conv::from_fvm::to_eth_address; use ipc_actors_abis::gateway_getter_facet::{GatewayGetterFacet, ParentFinality}; use ipc_actors_abis::subnet_actor_getter_facet::SubnetActorGetterFacet; -use crate::with_testnet; +use crate::make_testnet; const MANIFEST: &str = "layer2.yaml"; const CHECKPOINT_PERIOD: u64 = 10; @@ -24,128 +23,122 @@ const MAX_RETRIES: u32 = 5; /// Test that top-down syncing and bottom-up checkpoint submission work. #[serial_test::serial] #[tokio::test] -async fn test_topdown_and_bottomup() { - with_testnet( - MANIFEST, - None, - |manifest| { - // Try to make sure the bottom-up checkpoint period is quick enough for reasonable test runtime. - let subnet = manifest - .subnets - .get_mut(&ResourceId::from("england")) - .expect("subnet not found"); - - subnet.bottom_up_checkpoint.period = CHECKPOINT_PERIOD; - }, - |_, _| Box::pin(async { () }), - |_, _, testnet, _, _, _| { - let test = async move { - let brussels = testnet.node(&testnet.root().node("brussels"))?; - let london = testnet.node(&testnet.root().subnet("england").node("london"))?; - let england = testnet.subnet(&testnet.root().subnet("england"))?; - - let london_provider = Arc::new( - london - .ethapi_http_provider()? - .ok_or_else(|| anyhow!("ethapi should be enabled"))?, - ); - - let brussels_provider = Arc::new( - brussels - .ethapi_http_provider()? - .ok_or_else(|| anyhow!("ethapi should be enabled"))?, - ); - - // Gateway actor on the child - let england_gateway = GatewayGetterFacet::new( - builtin_actor_eth_addr(ipc::GATEWAY_ACTOR_ID), - london_provider.clone(), - ); - - // Subnet actor on the parent - let england_subnet = SubnetActorGetterFacet::new( - to_eth_address(&england.subnet_id.subnet_actor()) - .and_then(|a| a.ok_or_else(|| anyhow!("not an eth address")))?, - brussels_provider.clone(), - ); - - // Query the latest committed parent finality and compare to the parent. - { - let mut retry = 0; - loop { - let finality: ParentFinality = england_gateway - .get_latest_parent_finality() - .call() - .await - .context("failed to get parent finality")?; - - // If the latest finality is not zero it means the syncer is working, - if finality.height.is_zero() { - if retry < MAX_RETRIES { - eprintln!("waiting for syncing with the parent..."); - tokio::time::sleep(Duration::from_secs(SLEEP_SECS)).await; - retry += 1; - continue; - } - bail!("the parent finality is still zero"); - } - - // Check that the block hash of the parent is actually the same at that height. - let parent_block: Option> = brussels_provider - .get_block(finality.height.as_u64()) - .await - .context("failed to get parent block")?; - - let Some(parent_block_hash) = parent_block.and_then(|b| b.hash) else { - bail!("cannot find parent block at final height"); - }; - - if parent_block_hash.0 != finality.block_hash { - bail!("the finality block hash is different from the API"); - } - break; +async fn test_topdown_and_bottomup() -> Result<(), anyhow::Error> { + let (testnet, cleanup) = make_testnet(MANIFEST, |manifest| { + // Try to make sure the bottom-up checkpoint period is quick enough for reasonable test runtime. + let subnet = manifest + .subnets + .get_mut(&ResourceId::from("england")) + .expect("subnet not found"); + + subnet.bottom_up_checkpoint.period = CHECKPOINT_PERIOD; + }) + .await?; + + let res = { + let brussels = testnet.node(&testnet.root().node("brussels"))?; + let london = testnet.node(&testnet.root().subnet("england").node("london"))?; + let england = testnet.subnet(&testnet.root().subnet("england"))?; + + let london_provider = Arc::new( + london + .ethapi_http_provider()? + .ok_or_else(|| anyhow!("ethapi should be enabled"))?, + ); + + let brussels_provider = Arc::new( + brussels + .ethapi_http_provider()? + .ok_or_else(|| anyhow!("ethapi should be enabled"))?, + ); + + // Gateway actor on the child + let england_gateway = GatewayGetterFacet::new( + builtin_actor_eth_addr(ipc::GATEWAY_ACTOR_ID), + london_provider.clone(), + ); + + // Subnet actor on the parent + let england_subnet = SubnetActorGetterFacet::new( + to_eth_address(&england.subnet_id.subnet_actor()) + .and_then(|a| a.ok_or_else(|| anyhow!("not an eth address")))?, + brussels_provider.clone(), + ); + + // Query the latest committed parent finality and compare to the parent. + { + let mut retry = 0; + loop { + let finality: ParentFinality = england_gateway + .get_latest_parent_finality() + .call() + .await + .context("failed to get parent finality")?; + + // If the latest finality is not zero it means the syncer is working, + if finality.height.is_zero() { + if retry < MAX_RETRIES { + eprintln!("waiting for syncing with the parent..."); + tokio::time::sleep(Duration::from_secs(SLEEP_SECS)).await; + retry += 1; + continue; } + bail!("the parent finality is still zero"); } - // Check that the parent knows about a checkpoint submitted from the child. - { - let mut retry = 0; - loop { - // NOTE: The implementation of the following method seems like a nonsense; - // I don't know if there is a way to ask the gateway what the latest - // checkpoint is, so we'll just have to go to the parent directly. - // let (has_checkpoint, epoch, _): (bool, et::U256, _) = england_gateway - // .get_current_bottom_up_checkpoint() - // .call() - // .await - // .context("failed to get current bottomup checkpoint")?; - let ckpt_height: et::U256 = england_subnet - .last_bottom_up_checkpoint_height() - .call() - .await - .context("failed to query last checkpoint height")?; - - if !ckpt_height.is_zero() { - break; - } - - if retry < MAX_RETRIES { - eprintln!("waiting for a checkpoint to be submitted..."); - tokio::time::sleep(Duration::from_secs(SLEEP_SECS)).await; - retry += 1; - continue; - } - - bail!("hasn't submitted a bottom-up checkpoint"); - } + // Check that the block hash of the parent is actually the same at that height. + let parent_block: Option> = brussels_provider + .get_block(finality.height.as_u64()) + .await + .context("failed to get parent block")?; + + let Some(parent_block_hash) = parent_block.and_then(|b| b.hash) else { + bail!("cannot find parent block at final height"); + }; + + if parent_block_hash.0 != finality.block_hash { + bail!("the finality block hash is different from the API"); + } + break; + } + } + + // Check that the parent knows about a checkpoint submitted from the child. + { + let mut retry = 0; + loop { + // NOTE: The implementation of the following method seems like a nonsense; + // I don't know if there is a way to ask the gateway what the latest + // checkpoint is, so we'll just have to go to the parent directly. + // let (has_checkpoint, epoch, _): (bool, et::U256, _) = england_gateway + // .get_current_bottom_up_checkpoint() + // .call() + // .await + // .context("failed to get current bottomup checkpoint")?; + let ckpt_height: et::U256 = england_subnet + .last_bottom_up_checkpoint_height() + .call() + .await + .context("failed to query last checkpoint height")?; + + if !ckpt_height.is_zero() { + break; } - Ok(()) - }; + if retry < MAX_RETRIES { + eprintln!("waiting for a checkpoint to be submitted..."); + tokio::time::sleep(Duration::from_secs(SLEEP_SECS)).await; + retry += 1; + continue; + } + + bail!("hasn't submitted a bottom-up checkpoint"); + } + } + + Ok(()) + }; - test.boxed() - }, - ) - .await - .unwrap() + cleanup(res.is_err(), testnet).await; + res } diff --git a/fendermint/testing/materializer/tests/docker_tests/mod.rs b/fendermint/testing/materializer/tests/docker_tests/mod.rs index a74d346dd..b3bf9de6d 100644 --- a/fendermint/testing/materializer/tests/docker_tests/mod.rs +++ b/fendermint/testing/materializer/tests/docker_tests/mod.rs @@ -5,6 +5,7 @@ //! sharing their materializer state. // Tests using the manifest bearing their name. +mod benches; pub mod layer2; pub mod root_only; pub mod standalone; diff --git a/fendermint/testing/materializer/tests/docker_tests/root_only.rs b/fendermint/testing/materializer/tests/docker_tests/root_only.rs index a5fb380d6..d1f5fa65e 100644 --- a/fendermint/testing/materializer/tests/docker_tests/root_only.rs +++ b/fendermint/testing/materializer/tests/docker_tests/root_only.rs @@ -1,48 +1,38 @@ -use std::time::Duration; - // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT +use crate::make_testnet; use anyhow::{anyhow, bail}; use ethers::{providers::Middleware, types::U64}; use fendermint_materializer::HasEthApi; -use futures::FutureExt; - -use crate::with_testnet; +use std::time::Duration; const MANIFEST: &str = "root-only.yaml"; #[serial_test::serial] #[tokio::test] -async fn test_full_node_sync() { - with_testnet( - MANIFEST, - None, - |_| {}, - |_,_| { Box::pin(async { })}, - |_, _, testnet, _, _, _| { - let test = async move { - // Allow a little bit of time for node-2 to catch up with node-1. - tokio::time::sleep(Duration::from_secs(5)).await; - // Check that node2 is following node1. - let node2 = testnet.root().node("node-2"); - let dnode2 = testnet.node(&node2)?; - - let provider = dnode2 - .ethapi_http_provider()? - .ok_or_else(|| anyhow!("node-2 has ethapi enabled"))?; - - let bn = provider.get_block_number().await?; - - if bn <= U64::one() { - bail!("expected a block beyond genesis"); - } - - Ok(()) - }; - - test.boxed() - }, - ) - .await - .unwrap() +async fn test_full_node_sync() -> Result<(), anyhow::Error> { + let (testnet, cleanup) = make_testnet(MANIFEST, |_| {}).await?; + + let res = { + // Allow a little bit of time for node-2 to catch up with node-1. + tokio::time::sleep(Duration::from_secs(5)).await; + // Check that node2 is following node1. + let node2 = testnet.root().node("node-2"); + let dnode2 = testnet.node(&node2)?; + + let provider = dnode2 + .ethapi_http_provider()? + .ok_or_else(|| anyhow!("node-2 has ethapi enabled"))?; + + let bn = provider.get_block_number().await?; + + if bn <= U64::one() { + bail!("expected a block beyond genesis"); + } + + Ok(()) + }; + + cleanup(res.is_err(), testnet).await; + res } diff --git a/fendermint/testing/materializer/tests/docker_tests/standalone.rs b/fendermint/testing/materializer/tests/docker_tests/standalone.rs index 38bc35ff6..abdad4cfe 100644 --- a/fendermint/testing/materializer/tests/docker_tests/standalone.rs +++ b/fendermint/testing/materializer/tests/docker_tests/standalone.rs @@ -1,10 +1,6 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT - -use std::ops::Add; -use std::time::{Duration}; - -use crate::with_testnet; +use crate::make_testnet; use anyhow::{bail, Context}; use ethers::{ core::k256::ecdsa::SigningKey, @@ -13,12 +9,7 @@ use ethers::{ signers::{Signer, Wallet}, types::{Eip1559TransactionRequest, H160}, }; -use ethers::types::{BlockId, U256}; -use fendermint_materializer::{ - concurrency::{self, config::Execution}, manifest::Rootnet, materials::DefaultAccount, HasEthApi, -}; -use futures::FutureExt; -use tokio::time::sleep; +use fendermint_materializer::{manifest::Rootnet, materials::DefaultAccount, HasEthApi}; const MANIFEST: &str = "standalone.yaml"; @@ -47,154 +38,56 @@ where /// from the ethereum API instance it was sent to even before it is included in the block. #[serial_test::serial] #[tokio::test] -async fn test_sent_tx_found_in_mempool() { - with_testnet( - MANIFEST, - None, - |manifest| { - // Slow down consensus to where we can see the effect of the transaction not being found by Ethereum hash. - if let Rootnet::New { ref mut env, .. } = manifest.rootnet { - env.insert("CMT_CONSENSUS_TIMEOUT_COMMIT".into(), "10s".into()); - }; - }, - |_,_| { Box::pin(async { })}, - |_, _, testnet, test_id, _, _| { - let test = async move { - let sender = testnet.account_mod_nth(test_id); - let recipient = testnet.account_mod_nth(test_id + 1); - let pangea = testnet.node(&testnet.root().node("pangea"))?; - let provider = pangea - .ethapi_http_provider()? - .expect("ethapi should be enabled"); - - let middleware = make_middleware(provider, sender) - .await - .context("failed to set up middleware")?; - - eprintln!("middleware ready, pending tests"); - - // Create the simplest transaction possible: send tokens between accounts. - let to: H160 = recipient.eth_addr().into(); - let transfer = Eip1559TransactionRequest::new().to(to).value(1); - - let pending: PendingTransaction<_> = middleware - .send_transaction(transfer, None) - .await - .context("failed to send txn")?; - - let tx_hash = pending.tx_hash(); - - eprintln!("sent pending txn {:?}", tx_hash); - - // We expect that the transaction is pending, however it should not return an error. - match middleware.get_transaction(tx_hash).await { - Ok(Some(_)) => {} - Ok(None) => bail!("pending transaction not found by eth hash"), - Err(e) => { - bail!("failed to get pending transaction: {e}") - } - } - - Ok(()) - }; - - test.boxed() - }, - ) - .await - .unwrap() -} - -#[serial_test::serial] -#[tokio::test] -async fn test_sent_tx_included_in_block() { - with_testnet( - MANIFEST, - Some(Execution::new() - .add_step(10, 5) - .add_step(100, 5) - ), - |manifest| { - // Slow down consensus to where we can see the effect of the transaction not being found by Ethereum hash. - if let Rootnet::New { ref mut env, .. } = manifest.rootnet { - env.insert("CMT_CONSENSUS_TIMEOUT_COMMIT".into(), "1s".into()); - }; - }, - |testnet,nonce_manager| { - Box::pin(async move { - }) - }, - |_, _, testnet, test_id, bencher, nonce_manager| { - let test = async move { - let sender = testnet.account_mod_nth(test_id); - let recipient = testnet.account_mod_nth(test_id + 1); - println!("running (test_id={})", test_id); - - let pangea = testnet.node(&testnet.root().node("pangea"))?; - let provider = pangea - .ethapi_http_provider()? - .expect("ethapi should be enabled"); - - let middleware = make_middleware(provider, sender) - .await - .context("failed to set up middleware")?; - - println!("middleware ready, pending tests (test_id={})", test_id); - - let sender: H160 = sender.eth_addr().into(); - // let current_nonce = middleware - // .get_transaction_count(sender, None) - // .await - // .context("failed to fetch nonce")?; - let nonce = nonce_manager.get_and_increment(sender).await; - - // Create the simplest transaction possible: send tokens between accounts. - let to: H160 = recipient.eth_addr().into(); - let transfer = Eip1559TransactionRequest::new() - .to(to) - .value(1) - .nonce(nonce); - - - bencher.start().await; - let pending: PendingTransaction<_> = middleware - .send_transaction(transfer, None) - .await - .context("failed to send txn")?; - - let tx_hash = pending.tx_hash(); - - println!("sent pending txn {:?} (test_id={})", tx_hash, test_id); - - // We expect that the transaction is pending, however it should not return an error. - match middleware.get_transaction(tx_hash).await { - Ok(Some(_)) => {} - Ok(None) => bail!("pending transaction not found by eth hash"), - Err(e) => { - bail!("failed to get pending transaction: {e}") - } - } - bencher.record("mempool".to_string()).await; - - // TODO: improve the polling or subscribe to some stream - loop { - if let Ok(Some(tx)) = middleware.get_transaction_receipt(tx_hash).await { - println!("tx included in block {:?} (test_id={})", tx.block_number, test_id); - - break; - } - sleep(Duration::from_millis(100)).await; - } - bencher.record("block_inclusion".to_string()).await; - - Ok(()) - }; - - test.boxed() - }, - ) - .await - .unwrap() +async fn test_sent_tx_found_in_mempool() -> Result<(), anyhow::Error> { + let (testnet, cleanup) = make_testnet(MANIFEST, |manifest| { + // Slow down consensus to where we can see the effect of the transaction not being found by Ethereum hash. + if let Rootnet::New { ref mut env, .. } = manifest.rootnet { + env.insert("CMT_CONSENSUS_TIMEOUT_COMMIT".into(), "10s".into()); + }; + }) + .await?; + + let res = { + let bob = testnet.account("bob")?; + let charlie = testnet.account("charlie")?; + + let pangea = testnet.node(&testnet.root().node("pangea"))?; + let provider = pangea + .ethapi_http_provider()? + .expect("ethapi should be enabled"); + + let middleware = make_middleware(provider, bob) + .await + .context("failed to set up middleware")?; + + eprintln!("middleware ready, pending tests"); + + // Create the simplest transaction possible: send tokens between accounts. + let to: H160 = charlie.eth_addr().into(); + let transfer = Eip1559TransactionRequest::new().to(to).value(1); + + let pending: PendingTransaction<_> = middleware + .send_transaction(transfer, None) + .await + .context("failed to send txn")?; + + let tx_hash = pending.tx_hash(); + + eprintln!("sent pending txn {:?}", tx_hash); + + // We expect that the transaction is pending, however it should not return an error. + match middleware.get_transaction(tx_hash).await { + Ok(Some(_)) => {} + Ok(None) => bail!("pending transaction not found by eth hash"), + Err(e) => { + bail!("failed to get pending transaction: {e}") + } + } + Ok(()) + }; + + cleanup(res.is_err(), testnet).await; + res } // /// Test that transactions sent out-of-order with regards to the nonce are not rejected, diff --git a/fendermint/testing/materializer/tests/manifests/benches.yaml b/fendermint/testing/materializer/tests/manifests/benches.yaml new file mode 100644 index 000000000..39c77b60d --- /dev/null +++ b/fendermint/testing/materializer/tests/manifests/benches.yaml @@ -0,0 +1,120 @@ +accounts: + "1": {} + "2": {} + "3": {} + "4": {} + "5": {} + "6": {} + "7": {} + "8": {} + "9": {} + "10": {} + "11": {} + "12": {} + "13": {} + "14": {} + "15": {} + "16": {} + "17": {} + "18": {} + "19": {} + "20": {} + "21": {} + "22": {} + "23": {} + "24": {} + "25": {} + "26": {} + "27": {} + "28": {} + "29": {} + "30": {} + "31": {} + "32": {} + "33": {} + "34": {} + "35": {} + "36": {} + "37": {} + "38": {} + "39": {} + "40": {} + "41": {} + "42": {} + "43": {} + "44": {} + "45": {} + "46": {} + "47": {} + "48": {} + "49": {} + "50": {} + +rootnet: + type: New + # Balances and collateral are in atto + validators: + "1": '100' + balances: + "1": '100000000000000000000' + "2": '200000000000000000000' + "3": '300000000000000000000' + "4": '100000000000000000000' + "5": '200000000000000000000' + "6": '300000000000000000000' + "7": '100000000000000000000' + "8": '200000000000000000000' + "9": '300000000000000000000' + "10": '100000000000000000000' + "11": '200000000000000000000' + "12": '300000000000000000000' + "13": '100000000000000000000' + "14": '200000000000000000000' + "15": '300000000000000000000' + "16": '100000000000000000000' + "17": '200000000000000000000' + "18": '300000000000000000000' + "19": '100000000000000000000' + "20": '200000000000000000000' + "21": '300000000000000000000' + "22": '100000000000000000000' + "23": '200000000000000000000' + "24": '300000000000000000000' + "25": '100000000000000000000' + "26": '200000000000000000000' + "27": '300000000000000000000' + "28": '100000000000000000000' + "29": '200000000000000000000' + "30": '300000000000000000000' + "31": '100000000000000000000' + "32": '200000000000000000000' + "33": '300000000000000000000' + "34": '100000000000000000000' + "35": '200000000000000000000' + "36": '300000000000000000000' + "37": '100000000000000000000' + "38": '200000000000000000000' + "39": '300000000000000000000' + "40": '100000000000000000000' + "41": '200000000000000000000' + "42": '300000000000000000000' + "43": '100000000000000000000' + "44": '200000000000000000000' + "45": '300000000000000000000' + "46": '100000000000000000000' + "47": '200000000000000000000' + "48": '300000000000000000000' + "49": '100000000000000000000' + "50": '200000000000000000000' + env: + CMT_CONSENSUS_TIMEOUT_COMMIT: 1s + FM_LOG_LEVEL: info,fendermint=debug + + nodes: + # A singleton node. + pangea: + mode: + type: Validator + validator: "1" + seed_nodes: [] + ethapi: true diff --git a/fendermint/testing/materializer/tests/manifests/standalone.yaml b/fendermint/testing/materializer/tests/manifests/standalone.yaml index 39c77b60d..2072c48c3 100644 --- a/fendermint/testing/materializer/tests/manifests/standalone.yaml +++ b/fendermint/testing/materializer/tests/manifests/standalone.yaml @@ -1,111 +1,18 @@ accounts: - "1": {} - "2": {} - "3": {} - "4": {} - "5": {} - "6": {} - "7": {} - "8": {} - "9": {} - "10": {} - "11": {} - "12": {} - "13": {} - "14": {} - "15": {} - "16": {} - "17": {} - "18": {} - "19": {} - "20": {} - "21": {} - "22": {} - "23": {} - "24": {} - "25": {} - "26": {} - "27": {} - "28": {} - "29": {} - "30": {} - "31": {} - "32": {} - "33": {} - "34": {} - "35": {} - "36": {} - "37": {} - "38": {} - "39": {} - "40": {} - "41": {} - "42": {} - "43": {} - "44": {} - "45": {} - "46": {} - "47": {} - "48": {} - "49": {} - "50": {} + alice: {} + bob: {} + charlie: {} rootnet: type: New # Balances and collateral are in atto validators: - "1": '100' + alice: '100' balances: - "1": '100000000000000000000' - "2": '200000000000000000000' - "3": '300000000000000000000' - "4": '100000000000000000000' - "5": '200000000000000000000' - "6": '300000000000000000000' - "7": '100000000000000000000' - "8": '200000000000000000000' - "9": '300000000000000000000' - "10": '100000000000000000000' - "11": '200000000000000000000' - "12": '300000000000000000000' - "13": '100000000000000000000' - "14": '200000000000000000000' - "15": '300000000000000000000' - "16": '100000000000000000000' - "17": '200000000000000000000' - "18": '300000000000000000000' - "19": '100000000000000000000' - "20": '200000000000000000000' - "21": '300000000000000000000' - "22": '100000000000000000000' - "23": '200000000000000000000' - "24": '300000000000000000000' - "25": '100000000000000000000' - "26": '200000000000000000000' - "27": '300000000000000000000' - "28": '100000000000000000000' - "29": '200000000000000000000' - "30": '300000000000000000000' - "31": '100000000000000000000' - "32": '200000000000000000000' - "33": '300000000000000000000' - "34": '100000000000000000000' - "35": '200000000000000000000' - "36": '300000000000000000000' - "37": '100000000000000000000' - "38": '200000000000000000000' - "39": '300000000000000000000' - "40": '100000000000000000000' - "41": '200000000000000000000' - "42": '300000000000000000000' - "43": '100000000000000000000' - "44": '200000000000000000000' - "45": '300000000000000000000' - "46": '100000000000000000000' - "47": '200000000000000000000' - "48": '300000000000000000000' - "49": '100000000000000000000' - "50": '200000000000000000000' + # 100FIL is 100_000_000_000_000_000_000 + alice: '100000000000000000000' + bob: '200000000000000000000' + charlie: '300000000000000000000' env: CMT_CONSENSUS_TIMEOUT_COMMIT: 1s FM_LOG_LEVEL: info,fendermint=debug @@ -115,6 +22,6 @@ rootnet: pangea: mode: type: Validator - validator: "1" + validator: alice seed_nodes: [] ethapi: true From 82ed89b6b11f808a6bb278ffc39c600b27359393 Mon Sep 17 00:00:00 2001 From: LePremierHomme <57456510+LePremierHomme@users.noreply.github.com> Date: Tue, 7 Jan 2025 14:22:01 +0200 Subject: [PATCH 06/14] add license headers --- .../testing/materializer/src/concurrency/config.rs | 3 +++ .../testing/materializer/src/concurrency/mod.rs | 3 +++ .../materializer/src/concurrency/nonce_manager.rs | 3 +++ .../testing/materializer/src/concurrency/reporting.rs | 11 +++++------ .../materializer/tests/docker_tests/benches.rs | 1 - 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/fendermint/testing/materializer/src/concurrency/config.rs b/fendermint/testing/materializer/src/concurrency/config.rs index 80a334c0f..cdbcbc9ae 100644 --- a/fendermint/testing/materializer/src/concurrency/config.rs +++ b/fendermint/testing/materializer/src/concurrency/config.rs @@ -1,3 +1,6 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + use std::time::Duration; #[derive(Debug, Clone)] diff --git a/fendermint/testing/materializer/src/concurrency/mod.rs b/fendermint/testing/materializer/src/concurrency/mod.rs index a87d1f265..351fd5ead 100644 --- a/fendermint/testing/materializer/src/concurrency/mod.rs +++ b/fendermint/testing/materializer/src/concurrency/mod.rs @@ -1,3 +1,6 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + pub mod config; pub mod nonce_manager; pub mod reporting; diff --git a/fendermint/testing/materializer/src/concurrency/nonce_manager.rs b/fendermint/testing/materializer/src/concurrency/nonce_manager.rs index 446594c21..a63653833 100644 --- a/fendermint/testing/materializer/src/concurrency/nonce_manager.rs +++ b/fendermint/testing/materializer/src/concurrency/nonce_manager.rs @@ -1,3 +1,6 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + use ethers::prelude::H160; use ethers::types::U256; use std::collections::HashMap; diff --git a/fendermint/testing/materializer/src/concurrency/reporting.rs b/fendermint/testing/materializer/src/concurrency/reporting.rs index 28731ede3..1a2f44cde 100644 --- a/fendermint/testing/materializer/src/concurrency/reporting.rs +++ b/fendermint/testing/materializer/src/concurrency/reporting.rs @@ -1,3 +1,6 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + use crate::bencher::Bencher; use crate::concurrency::config; use crate::concurrency::config::ExecutionStep; @@ -80,12 +83,8 @@ impl ExecutionSummary { pub fn errs(&self) -> Vec { let mut errs = Vec::new(); for summary in self.summaries.iter() { - let cloned_errs: Vec = summary - .errs - .iter() - .map(|e| format!("{:?}", e)) - // e.chain().map(|cause|cause.to_string()).collect::>().join(" -> ")) - .collect(); + let cloned_errs: Vec = + summary.errs.iter().map(|e| format!("{:?}", e)).collect(); errs.extend(cloned_errs); } errs diff --git a/fendermint/testing/materializer/tests/docker_tests/benches.rs b/fendermint/testing/materializer/tests/docker_tests/benches.rs index 088c5bddc..3faa7b57f 100644 --- a/fendermint/testing/materializer/tests/docker_tests/benches.rs +++ b/fendermint/testing/materializer/tests/docker_tests/benches.rs @@ -76,7 +76,6 @@ async fn test_concurrent_transfer() -> Result<(), anyhow::Error> { let middleware = make_middleware(provider, sender) .await .context("failed to set up middleware")?; - println!("middleware ready, pending tests (test_id={})", test_id); let sender: H160 = sender.eth_addr().into(); From 8d9041ded0fd4597f5a7b773c44429881f31b71c Mon Sep 17 00:00:00 2001 From: LePremierHomme <57456510+LePremierHomme@users.noreply.github.com> Date: Tue, 7 Jan 2025 14:36:40 +0200 Subject: [PATCH 07/14] add license headers --- fendermint/testing/materializer/src/bencher.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fendermint/testing/materializer/src/bencher.rs b/fendermint/testing/materializer/src/bencher.rs index bfb7af821..c7ee20ca7 100644 --- a/fendermint/testing/materializer/src/bencher.rs +++ b/fendermint/testing/materializer/src/bencher.rs @@ -1,3 +1,6 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + use std::collections::HashMap; use std::time::{Duration, Instant}; From 4bdfad7057cdc9076b8659be283737728ff48f4b Mon Sep 17 00:00:00 2001 From: LePremierHomme <57456510+LePremierHomme@users.noreply.github.com> Date: Tue, 7 Jan 2025 14:50:09 +0200 Subject: [PATCH 08/14] clippy --- fendermint/testing/materializer/src/bencher.rs | 2 +- fendermint/testing/materializer/src/concurrency/config.rs | 2 +- fendermint/testing/materializer/src/concurrency/mod.rs | 2 +- .../testing/materializer/src/concurrency/nonce_manager.rs | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/fendermint/testing/materializer/src/bencher.rs b/fendermint/testing/materializer/src/bencher.rs index c7ee20ca7..57d4900e4 100644 --- a/fendermint/testing/materializer/src/bencher.rs +++ b/fendermint/testing/materializer/src/bencher.rs @@ -4,7 +4,7 @@ use std::collections::HashMap; use std::time::{Duration, Instant}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct Bencher { pub start_time: Option, pub records: HashMap, diff --git a/fendermint/testing/materializer/src/concurrency/config.rs b/fendermint/testing/materializer/src/concurrency/config.rs index cdbcbc9ae..ceed20a21 100644 --- a/fendermint/testing/materializer/src/concurrency/config.rs +++ b/fendermint/testing/materializer/src/concurrency/config.rs @@ -3,7 +3,7 @@ use std::time::Duration; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct Execution { pub steps: Vec, } diff --git a/fendermint/testing/materializer/src/concurrency/mod.rs b/fendermint/testing/materializer/src/concurrency/mod.rs index 351fd5ead..938df4870 100644 --- a/fendermint/testing/materializer/src/concurrency/mod.rs +++ b/fendermint/testing/materializer/src/concurrency/mod.rs @@ -49,7 +49,7 @@ where drop(permit); }); handles.push(handle); - test_id = test_id + 1; + test_id += 1; } // Exhaust unfinished handles. diff --git a/fendermint/testing/materializer/src/concurrency/nonce_manager.rs b/fendermint/testing/materializer/src/concurrency/nonce_manager.rs index a63653833..b397e0dd6 100644 --- a/fendermint/testing/materializer/src/concurrency/nonce_manager.rs +++ b/fendermint/testing/materializer/src/concurrency/nonce_manager.rs @@ -7,6 +7,7 @@ use std::collections::HashMap; use std::sync::Arc; use tokio::sync::Mutex; +#[derive(Default)] pub struct NonceManager { nonces: Arc>>, } From 2cae29d8ba38c4d1e223499a60e9c2eff55003e3 Mon Sep 17 00:00:00 2001 From: LePremierHomme <57456510+LePremierHomme@users.noreply.github.com> Date: Wed, 8 Jan 2025 13:29:17 +0200 Subject: [PATCH 09/14] extract provider and chain_id, add reporting table --- Cargo.lock | 7 ++++ Cargo.toml | 1 + fendermint/testing/materializer/Cargo.toml | 1 + .../materializer/src/concurrency/reporting.rs | 37 ++++++++++++++++++- .../tests/docker_tests/benches.rs | 32 ++++++++-------- 5 files changed, 62 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f584133e4..766d76692 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3151,6 +3151,7 @@ dependencies = [ "serial_test", "tempfile", "tendermint-rpc", + "text-tables", "tokio", "toml 0.8.19", "tracing", @@ -9554,6 +9555,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "text-tables" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dc41925991e82af3c3e21e25a9aad92e72930af57fbcc4b07867a18d1cd0459" + [[package]] name = "textwrap" version = "0.11.0" diff --git a/Cargo.toml b/Cargo.toml index 30c5cc7d1..bf6855ffe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -172,6 +172,7 @@ tracing-subscriber = { version = "0.3", features = [ "registry", ] } tracing-appender = "0.2.3" +text-tables = "0.3.1" url = { version = "2.4.1", features = ["serde"] } zeroize = "1.6" diff --git a/fendermint/testing/materializer/Cargo.toml b/fendermint/testing/materializer/Cargo.toml index fabce48d2..dabf71418 100644 --- a/fendermint/testing/materializer/Cargo.toml +++ b/fendermint/testing/materializer/Cargo.toml @@ -29,6 +29,7 @@ tendermint-rpc = { workspace = true } tokio = { workspace = true } toml = { workspace = true } tracing = { workspace = true } +text-tables = { workspace = true } url = { workspace = true } arbitrary = { workspace = true, optional = true } diff --git a/fendermint/testing/materializer/src/concurrency/reporting.rs b/fendermint/testing/materializer/src/concurrency/reporting.rs index 1a2f44cde..242a54082 100644 --- a/fendermint/testing/materializer/src/concurrency/reporting.rs +++ b/fendermint/testing/materializer/src/concurrency/reporting.rs @@ -5,7 +5,8 @@ use crate::bencher::Bencher; use crate::concurrency::config; use crate::concurrency::config::ExecutionStep; use anyhow::anyhow; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; +use std::io; use std::time::Duration; #[derive(Debug)] @@ -89,4 +90,38 @@ impl ExecutionSummary { } errs } + + pub fn print(&self) { + let mut data = vec![]; + + let latencies: HashSet = self + .summaries + .iter() + .flat_map(|summary| summary.avg_latencies.keys().cloned()) + .collect(); + + let mut header = vec!["max_concurrency".to_string(), "duration".to_string()]; + header.extend(latencies.iter().map(|key| format!("{} latency (ms)", key))); + data.push(header); + + for summary in self.summaries.iter() { + let mut row = vec![]; + row.push(summary.cfg.max_concurrency.to_string()); + row.push(summary.cfg.duration.as_secs().to_string()); + + for key in &latencies { + let latency = summary + .avg_latencies + .get(key) + .map_or(String::from("-"), |duration| { + duration.as_millis().to_string() + }); + row.push(latency); + } + + data.push(row); + } + + text_tables::render(&mut io::stdout(), data).unwrap(); + } } diff --git a/fendermint/testing/materializer/tests/docker_tests/benches.rs b/fendermint/testing/materializer/tests/docker_tests/benches.rs index 3faa7b57f..1a143c9f5 100644 --- a/fendermint/testing/materializer/tests/docker_tests/benches.rs +++ b/fendermint/testing/materializer/tests/docker_tests/benches.rs @@ -6,6 +6,7 @@ use std::time::Duration; use crate::make_testnet; use anyhow::{bail, Context}; +use ethers::types::U256; use ethers::{ core::k256::ecdsa::SigningKey, middleware::SignerMiddleware, @@ -31,15 +32,11 @@ pub type TestMiddleware = SignerMiddleware, Wallet>; async fn make_middleware( provider: Provider, sender: &DefaultAccount, + chain_id: U256, ) -> anyhow::Result> where C: JsonRpcClient, { - let chain_id = provider - .get_chainid() - .await - .context("failed to get chain ID")?; - let wallet: Wallet = Wallet::from_bytes(sender.secret_key().serialize().as_ref())? .with_chain_id(chain_id.as_u64()); @@ -51,32 +48,37 @@ where async fn test_concurrent_transfer() -> Result<(), anyhow::Error> { let (testnet, cleanup) = make_testnet(MANIFEST, |_| {}).await?; + let pangea = testnet.node(&testnet.root().node("pangea"))?; + let provider = pangea + .ethapi_http_provider()? + .expect("ethapi should be enabled"); + let chain_id = provider + .get_chainid() + .await + .context("failed to get chain ID")?; + // Drive concurrency. let cfg = Execution::new() .add_step(10, 5) .add_step(100, 5) - .add_step(200, 5); + .add_step(150, 5); let testnet = Arc::new(testnet); let testnet_clone = testnet.clone(); let nonce_manager = Arc::new(NonceManager::new()); + let results = concurrency::execute(cfg.clone(), move |test_id: usize, mut bencher: Bencher| { let testnet = testnet_clone.clone(); let nonce_manager = nonce_manager.clone(); + let provider = provider.clone(); let test = async move { let sender = testnet.account_mod_nth(test_id); let recipient = testnet.account_mod_nth(test_id + 1); println!("running (test_id={})", test_id); - let pangea = testnet.node(&testnet.root().node("pangea"))?; - let provider = pangea - .ethapi_http_provider()? - .expect("ethapi should be enabled"); - - let middleware = make_middleware(provider, sender) + let middleware = make_middleware(provider, sender, chain_id) .await .context("failed to set up middleware")?; - println!("middleware ready, pending tests (test_id={})", test_id); let sender: H160 = sender.eth_addr().into(); let nonce = nonce_manager.get_and_increment(sender).await; @@ -126,12 +128,12 @@ async fn test_concurrent_transfer() -> Result<(), anyhow::Error> { .await; let summary = concurrency::ExecutionSummary::new(cfg.clone(), results); - println!("{:?}", summary); + summary.print(); + let res = summary.to_result(); let Ok(testnet) = Arc::try_unwrap(testnet) else { bail!("Arc::try_unwrap(testnet)"); }; - let res = summary.to_result(); cleanup(res.is_err(), testnet).await; res } From 886a0e2ebbc825431c093d81051caf8e500258f5 Mon Sep 17 00:00:00 2001 From: LePremierHomme <57456510+LePremierHomme@users.noreply.github.com> Date: Fri, 10 Jan 2025 10:45:19 +0200 Subject: [PATCH 10/14] add basic TPS analysis --- .../testing/materializer/src/bencher.rs | 21 ++++++-- .../materializer/src/concurrency/collect.rs | 37 +++++++++++++ .../materializer/src/concurrency/mod.rs | 2 + .../materializer/src/concurrency/reporting.rs | 50 +++++++++++++---- .../materializer/src/concurrency/signal.rs | 24 +++++++++ .../tests/docker_tests/benches.rs | 53 +++++++++++++++---- 6 files changed, 164 insertions(+), 23 deletions(-) create mode 100644 fendermint/testing/materializer/src/concurrency/collect.rs create mode 100644 fendermint/testing/materializer/src/concurrency/signal.rs diff --git a/fendermint/testing/materializer/src/bencher.rs b/fendermint/testing/materializer/src/bencher.rs index 57d4900e4..6a204f61a 100644 --- a/fendermint/testing/materializer/src/bencher.rs +++ b/fendermint/testing/materializer/src/bencher.rs @@ -7,23 +7,34 @@ use std::time::{Duration, Instant}; #[derive(Debug, Clone, Default)] pub struct Bencher { pub start_time: Option, - pub records: HashMap, + pub latencies: HashMap, + pub block_inclusion: Option, } impl Bencher { pub fn new() -> Self { Self { start_time: None, - records: HashMap::new(), + latencies: HashMap::new(), + block_inclusion: None, } } - pub async fn start(&mut self) { + pub fn start(&mut self) { self.start_time = Some(Instant::now()); } - pub async fn record(&mut self, label: String) { + pub fn mempool(&mut self) { + self.set_latency("mempool".to_string()); + } + + pub fn block_inclusion(&mut self, block_number: u64) { + self.set_latency("block_inclusion".to_string()); + self.block_inclusion = Some(block_number); + } + + fn set_latency(&mut self, label: String) { let duration = self.start_time.unwrap().elapsed(); - self.records.insert(label, duration); + self.latencies.insert(label, duration); } } diff --git a/fendermint/testing/materializer/src/concurrency/collect.rs b/fendermint/testing/materializer/src/concurrency/collect.rs new file mode 100644 index 000000000..7bec16769 --- /dev/null +++ b/fendermint/testing/materializer/src/concurrency/collect.rs @@ -0,0 +1,37 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +use crate::concurrency::signal::Signal; +use ethers::prelude::H256; +use ethers::providers::Http; +use ethers::providers::{Middleware, Provider}; +use ethers::types::Block; +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Duration; +use tokio::time::sleep; + +pub async fn collect_blocks( + cancel: Arc, + provider: Provider, + assert: F, +) -> anyhow::Result>> +where + F: Fn(&Block), +{ + let mut blocks = HashMap::new(); + loop { + if cancel.received() { + break; + } + + // TODO: improve: use less calls, make sure blocks cannot be missed + let block_number = provider.get_block_number().await?; + let block = provider.get_block(block_number).await?.unwrap(); + assert(&block); + blocks.insert(block_number.as_u64(), block); + + sleep(Duration::from_millis(100)).await; + } + Ok(blocks) +} diff --git a/fendermint/testing/materializer/src/concurrency/mod.rs b/fendermint/testing/materializer/src/concurrency/mod.rs index 938df4870..6d497056c 100644 --- a/fendermint/testing/materializer/src/concurrency/mod.rs +++ b/fendermint/testing/materializer/src/concurrency/mod.rs @@ -1,8 +1,10 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT +pub mod collect; pub mod config; pub mod nonce_manager; +pub mod signal; pub mod reporting; pub use reporting::*; diff --git a/fendermint/testing/materializer/src/concurrency/reporting.rs b/fendermint/testing/materializer/src/concurrency/reporting.rs index 242a54082..c526e8907 100644 --- a/fendermint/testing/materializer/src/concurrency/reporting.rs +++ b/fendermint/testing/materializer/src/concurrency/reporting.rs @@ -5,6 +5,7 @@ use crate::bencher::Bencher; use crate::concurrency::config; use crate::concurrency::config::ExecutionStep; use anyhow::anyhow; +use ethers::prelude::{Block, H256}; use std::collections::{HashMap, HashSet}; use std::io; use std::time::Duration; @@ -21,29 +22,50 @@ pub struct TestResult { pub struct StepSummary { pub cfg: ExecutionStep, pub avg_latencies: HashMap, + pub avg_tps: u64, pub errs: Vec, } impl StepSummary { fn new(cfg: ExecutionStep, results: Vec) -> Self { - let mut total_durations: HashMap = HashMap::new(); - let mut counts: HashMap = HashMap::new(); + let mut sum_latencies: HashMap = HashMap::new(); + let mut count_latencies: HashMap = HashMap::new(); + let mut block_inclusions: HashMap = HashMap::new(); let mut errs = Vec::new(); + for res in results { let Some(bencher) = res.bencher else { continue }; - for (key, duration) in bencher.records.clone() { - *total_durations.entry(key.clone()).or_insert(Duration::ZERO) += duration; - *counts.entry(key).or_insert(0) += 1; + + for (key, duration) in bencher.latencies.clone() { + *sum_latencies.entry(key.clone()).or_insert(Duration::ZERO) += duration; + *count_latencies.entry(key).or_insert(0) += 1; } + + if let Some(block) = bencher.block_inclusion { + *block_inclusions.entry(block).or_insert(0) += 1; + } + if let Some(err) = res.err { errs.push(err); } } - let avg_latencies = total_durations + // TODO: improve: + // 1. don't assume block time is 1s. + // 2. don't scope the count to execution step, + // because blocks might be shared with prev/next step, + // which skews the results. + // 3. don't use naive avg. + let avg_tps = { + let sum: u64 = block_inclusions.values().sum(); + let count = block_inclusions.len(); + sum / count as u64 + }; + + let avg_latencies = sum_latencies .into_iter() .map(|(key, total)| { - let count = counts[&key]; + let count = count_latencies[&key]; (key, total / count as u32) }) .collect(); @@ -51,6 +73,7 @@ impl StepSummary { Self { cfg, avg_latencies, + avg_tps, errs, } } @@ -62,7 +85,11 @@ pub struct ExecutionSummary { } impl ExecutionSummary { - pub fn new(cfg: config::Execution, results: Vec>) -> Self { + pub fn new( + cfg: config::Execution, + _blocks: HashMap>, + results: Vec>, + ) -> Self { let mut summaries = Vec::new(); for (i, step_results) in results.into_iter().enumerate() { let cfg = cfg.steps[i].clone(); @@ -100,7 +127,11 @@ impl ExecutionSummary { .flat_map(|summary| summary.avg_latencies.keys().cloned()) .collect(); - let mut header = vec!["max_concurrency".to_string(), "duration".to_string()]; + let mut header = vec![ + "max_concurrency".to_string(), + "duration (s)".to_string(), + "TPS".to_string(), + ]; header.extend(latencies.iter().map(|key| format!("{} latency (ms)", key))); data.push(header); @@ -108,6 +139,7 @@ impl ExecutionSummary { let mut row = vec![]; row.push(summary.cfg.max_concurrency.to_string()); row.push(summary.cfg.duration.as_secs().to_string()); + row.push(summary.avg_tps.to_string()); for key in &latencies { let latency = summary diff --git a/fendermint/testing/materializer/src/concurrency/signal.rs b/fendermint/testing/materializer/src/concurrency/signal.rs new file mode 100644 index 000000000..0067583cf --- /dev/null +++ b/fendermint/testing/materializer/src/concurrency/signal.rs @@ -0,0 +1,24 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +pub struct Signal(tokio::sync::Semaphore); + +impl Signal { + pub fn new() -> Self { + Self(tokio::sync::Semaphore::new(0)) + } + + pub fn send(&self) { + self.0.close(); + } + + pub fn received(&self) -> bool { + self.0.is_closed() + } +} + +impl Default for Signal { + fn default() -> Self { + Self::new() + } +} diff --git a/fendermint/testing/materializer/tests/docker_tests/benches.rs b/fendermint/testing/materializer/tests/docker_tests/benches.rs index 1a143c9f5..2a0efe554 100644 --- a/fendermint/testing/materializer/tests/docker_tests/benches.rs +++ b/fendermint/testing/materializer/tests/docker_tests/benches.rs @@ -6,6 +6,7 @@ use std::time::Duration; use crate::make_testnet; use anyhow::{bail, Context}; +use ethers::prelude::{Block, H256}; use ethers::types::U256; use ethers::{ core::k256::ecdsa::SigningKey, @@ -15,7 +16,9 @@ use ethers::{ types::{Eip1559TransactionRequest, H160}, }; use fendermint_materializer::bencher::Bencher; +use fendermint_materializer::concurrency::collect::collect_blocks; use fendermint_materializer::concurrency::nonce_manager::NonceManager; +use fendermint_materializer::concurrency::signal::Signal; use fendermint_materializer::{ concurrency::{self, config::Execution}, materials::DefaultAccount, @@ -43,11 +46,17 @@ where Ok(SignerMiddleware::new(provider, wallet)) } +const BLOCK_GAS_LIMIT: u64 = 10_000_000_000; +const MAX_TX_GAS_LIMIT: u64 = 3_000_000; + #[serial_test::serial] #[tokio::test] async fn test_concurrent_transfer() -> Result<(), anyhow::Error> { let (testnet, cleanup) = make_testnet(MANIFEST, |_| {}).await?; + let block_gas_limit = U256::from(BLOCK_GAS_LIMIT); + let max_tx_gas_limit = U256::from(MAX_TX_GAS_LIMIT); + let pangea = testnet.node(&testnet.root().node("pangea"))?; let provider = pangea .ethapi_http_provider()? @@ -57,11 +66,27 @@ async fn test_concurrent_transfer() -> Result<(), anyhow::Error> { .await .context("failed to get chain ID")?; + let cancel = Arc::new(Signal::new()); + + // Set up background blocks collector. + let blocks_collector = { + let cancel = cancel.clone(); + let provider = provider.clone(); + let assert = move |block: &Block| { + // Make sure block gas limit isn't the bottleneck. + let unused_gas_limit = block_gas_limit - block.gas_limit; + assert!(unused_gas_limit >= max_tx_gas_limit); + }; + tokio::spawn(collect_blocks(cancel, provider, assert)) + }; + // Drive concurrency. let cfg = Execution::new() + .add_step(1, 5) .add_step(10, 5) .add_step(100, 5) - .add_step(150, 5); + .add_step(150, 5) + .add_step(200, 5); let testnet = Arc::new(testnet); let testnet_clone = testnet.clone(); let nonce_manager = Arc::new(NonceManager::new()); @@ -78,26 +103,33 @@ async fn test_concurrent_transfer() -> Result<(), anyhow::Error> { let middleware = make_middleware(provider, sender, chain_id) .await - .context("failed to set up middleware")?; + .context("make_middleware")?; let sender: H160 = sender.eth_addr().into(); let nonce = nonce_manager.get_and_increment(sender).await; // Create the simplest transaction possible: send tokens between accounts. let to: H160 = recipient.eth_addr().into(); - let transfer = Eip1559TransactionRequest::new() + let mut tx = Eip1559TransactionRequest::new() .to(to) .value(1) .nonce(nonce); - bencher.start().await; + let gas_estimation = middleware + .estimate_gas(&tx.clone().into(), None) + .await + .unwrap(); + tx = tx.gas(gas_estimation); + assert!(gas_estimation <= max_tx_gas_limit); + + bencher.start(); let pending: PendingTransaction<_> = middleware - .send_transaction(transfer, None) + .send_transaction(tx, None) .await .context("failed to send txn")?; let tx_hash = pending.tx_hash(); - println!("sent pending txn {:?} (test_id={})", tx_hash, test_id); + println!("sent tx {:?} (test_id={})", tx_hash, test_id); // We expect that the transaction is pending, however it should not return an error. match middleware.get_transaction(tx_hash).await { @@ -107,7 +139,7 @@ async fn test_concurrent_transfer() -> Result<(), anyhow::Error> { bail!("failed to get pending transaction: {e}") } } - bencher.record("mempool".to_string()).await; + bencher.mempool(); loop { if let Ok(Some(tx)) = middleware.get_transaction_receipt(tx_hash).await { @@ -115,11 +147,12 @@ async fn test_concurrent_transfer() -> Result<(), anyhow::Error> { "tx included in block {:?} (test_id={})", tx.block_number, test_id ); + let block_number = tx.block_number.unwrap().as_u64(); + bencher.block_inclusion(block_number); break; } sleep(Duration::from_millis(100)).await; } - bencher.record("block_inclusion".to_string()).await; Ok(bencher) }; @@ -127,7 +160,9 @@ async fn test_concurrent_transfer() -> Result<(), anyhow::Error> { }) .await; - let summary = concurrency::ExecutionSummary::new(cfg.clone(), results); + cancel.send(); + let blocks = blocks_collector.await??; + let summary = concurrency::ExecutionSummary::new(cfg.clone(), blocks, results); summary.print(); let res = summary.to_result(); From d841316773b56718157620e968951fdd975ccba3 Mon Sep 17 00:00:00 2001 From: LePremierHomme <57456510+LePremierHomme@users.noreply.github.com> Date: Fri, 10 Jan 2025 10:50:04 +0200 Subject: [PATCH 11/14] cargo fmt --- fendermint/testing/materializer/src/concurrency/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fendermint/testing/materializer/src/concurrency/mod.rs b/fendermint/testing/materializer/src/concurrency/mod.rs index 6d497056c..8d4f0d2d4 100644 --- a/fendermint/testing/materializer/src/concurrency/mod.rs +++ b/fendermint/testing/materializer/src/concurrency/mod.rs @@ -4,8 +4,8 @@ pub mod collect; pub mod config; pub mod nonce_manager; -pub mod signal; pub mod reporting; +pub mod signal; pub use reporting::*; From 034c799af4441b946d861c401184a96dec4dcb66 Mon Sep 17 00:00:00 2001 From: LePremierHomme <57456510+LePremierHomme@users.noreply.github.com> Date: Mon, 13 Jan 2025 19:40:06 +0200 Subject: [PATCH 12/14] fix `test_out_of_order_mempool` --- .../tests/docker_tests/standalone.rs | 195 +++++++++--------- 1 file changed, 96 insertions(+), 99 deletions(-) diff --git a/fendermint/testing/materializer/tests/docker_tests/standalone.rs b/fendermint/testing/materializer/tests/docker_tests/standalone.rs index abdad4cfe..346cc071c 100644 --- a/fendermint/testing/materializer/tests/docker_tests/standalone.rs +++ b/fendermint/testing/materializer/tests/docker_tests/standalone.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0, MIT use crate::make_testnet; use anyhow::{bail, Context}; +use ethers::prelude::transaction::eip2718::TypedTransaction; use ethers::{ core::k256::ecdsa::SigningKey, middleware::SignerMiddleware, @@ -10,6 +11,7 @@ use ethers::{ types::{Eip1559TransactionRequest, H160}, }; use fendermint_materializer::{manifest::Rootnet, materials::DefaultAccount, HasEthApi}; +use std::time::{Duration, Instant}; const MANIFEST: &str = "standalone.yaml"; @@ -90,102 +92,97 @@ async fn test_sent_tx_found_in_mempool() -> Result<(), anyhow::Error> { res } -// /// Test that transactions sent out-of-order with regards to the nonce are not rejected, -// /// but rather get included in block eventually, their submission managed by the ethereum -// /// API facade. -// #[serial_test::serial] -// #[tokio::test] -// async fn test_out_of_order_mempool() { -// const MAX_WAIT_TIME: Duration = Duration::from_secs(10); -// const SLEEP_TIME: Duration = Duration::from_secs(1); -// -// with_testnet( -// MANIFEST, -// None, -// |_| {}, -// |_, _, testnet, _| { -// let test = async move { -// let bob = testnet.account("bob")?; -// let charlie = testnet.account("charlie")?; -// -// let pangea = testnet.node(&testnet.root().node("pangea"))?; -// let provider = pangea -// .ethapi_http_provider()? -// .expect("ethapi should be enabled"); -// -// let middleware = make_middleware(provider, bob) -// .await -// .context("failed to set up middleware")?; -// -// // Create the simplest transaction possible: send tokens between accounts. -// let to: H160 = charlie.eth_addr().into(); -// let tx = Eip1559TransactionRequest::new().to(to).value(1); -// let mut tx: TypedTransaction = tx.into(); -// -// // Fill out the nonce, gas, etc. -// middleware -// .fill_transaction(&mut tx, None) -// .await -// .context("failed to fill tx")?; -// -// // Create a few more transactions to be sent out-of-order. -// let mut txs = vec![tx]; -// -// for i in 1..5 { -// let mut tx = txs[0].clone(); -// let nonce = tx.nonce().expect("fill_transaction filled the nonce"); -// tx.set_nonce(nonce.saturating_add(i.into())); -// txs.push(tx) -// } -// -// let mut pending_txs = Vec::new(); -// -// // Submit transactions in opposite order. -// for (i, tx) in txs.iter().enumerate().rev() { -// let sig = middleware -// .signer() -// .sign_transaction(tx) -// .await -// .context("failed to sign tx")?; -// -// let rlp = tx.rlp_signed(&sig); -// -// let pending_tx: PendingTransaction<_> = middleware -// .send_raw_transaction(rlp) -// .await -// .with_context(|| format!("failed to send tx {i}"))?; -// -// pending_txs.push(pending_tx) -// } -// -// // Check that they eventually get included. -// let start = Instant::now(); -// 'pending: loop { -// for tx in pending_txs.iter() { -// let receipt = middleware -// .get_transaction_receipt(tx.tx_hash()) -// .await -// .context("failed to get receipt")?; -// -// if receipt.is_none() { -// if start.elapsed() > MAX_WAIT_TIME { -// bail!("some transactions are still not executed") -// } else { -// tokio::time::sleep(SLEEP_TIME).await; -// continue 'pending; -// } -// } -// } -// // All of them have receipt. -// break 'pending; -// } -// -// Ok(()) -// }; -// -// test.boxed() -// }, -// ) -// .await -// .unwrap() -// } +/// Test that transactions sent out-of-order with regards to the nonce are not rejected, +/// but rather get included in block eventually, their submission managed by the ethereum +/// API facade. +#[serial_test::serial] +#[tokio::test] +async fn test_out_of_order_mempool() { + const MAX_WAIT_TIME: Duration = Duration::from_secs(10); + const SLEEP_TIME: Duration = Duration::from_secs(1); + + let (testnet, cleanup) = make_testnet(MANIFEST, |_| {}).await.unwrap(); + + let res = async { + let bob = testnet.account("bob")?; + let charlie = testnet.account("charlie")?; + + let pangea = testnet.node(&testnet.root().node("pangea"))?; + let provider = pangea + .ethapi_http_provider()? + .expect("ethapi should be enabled"); + + let middleware = make_middleware(provider, bob) + .await + .context("failed to set up middleware")?; + + // Create the simplest transaction possible: send tokens between accounts. + let to: H160 = charlie.eth_addr().into(); + let tx = Eip1559TransactionRequest::new().to(to).value(1); + let mut tx: TypedTransaction = tx.into(); + + // Fill out the nonce, gas, etc. + middleware + .fill_transaction(&mut tx, None) + .await + .context("failed to fill tx")?; + + // Create a few more transactions to be sent out-of-order. + let mut txs = vec![tx]; + + for i in 1..5 { + let mut tx = txs[0].clone(); + let nonce = tx.nonce().expect("fill_transaction filled the nonce"); + tx.set_nonce(nonce.saturating_add(i.into())); + txs.push(tx) + } + + let mut pending_txs = Vec::new(); + + // Submit transactions in opposite order. + for (i, tx) in txs.iter().enumerate().rev() { + let sig = middleware + .signer() + .sign_transaction(tx) + .await + .context("failed to sign tx")?; + + let rlp = tx.rlp_signed(&sig); + + let pending_tx: PendingTransaction<_> = middleware + .send_raw_transaction(rlp) + .await + .with_context(|| format!("failed to send tx {i}"))?; + + pending_txs.push(pending_tx) + } + + // Check that they eventually get included. + let start = Instant::now(); + 'pending: loop { + for tx in pending_txs.iter() { + let receipt = middleware + .get_transaction_receipt(tx.tx_hash()) + .await + .context("failed to get receipt")?; + + if receipt.is_none() { + if start.elapsed() > MAX_WAIT_TIME { + bail!("some transactions are still not executed") + } else { + tokio::time::sleep(SLEEP_TIME).await; + continue 'pending; + } + } + } + // All of them have receipt. + break 'pending; + } + + Ok(()) + }; + + let res = res.await; + cleanup(res.is_err(), testnet).await; + res.unwrap() +} From d395d1bca658cef4e133c216043b0f8c75d6ae77 Mon Sep 17 00:00:00 2001 From: LePremierHomme <57456510+LePremierHomme@users.noreply.github.com> Date: Thu, 16 Jan 2025 14:23:01 +0200 Subject: [PATCH 13/14] add tps, dataset metrics --- .../materializer/src/concurrency/mod.rs | 30 ++- .../materializer/src/concurrency/reporting.rs | 159 --------------- .../src/concurrency/reporting/dataset.rs | 87 ++++++++ .../src/concurrency/reporting/mod.rs | 21 ++ .../src/concurrency/reporting/summary.rs | 185 ++++++++++++++++++ .../src/concurrency/reporting/tps.rs | 85 ++++++++ .../tests/docker_tests/benches.rs | 31 +-- 7 files changed, 417 insertions(+), 181 deletions(-) delete mode 100644 fendermint/testing/materializer/src/concurrency/reporting.rs create mode 100644 fendermint/testing/materializer/src/concurrency/reporting/dataset.rs create mode 100644 fendermint/testing/materializer/src/concurrency/reporting/mod.rs create mode 100644 fendermint/testing/materializer/src/concurrency/reporting/summary.rs create mode 100644 fendermint/testing/materializer/src/concurrency/reporting/tps.rs diff --git a/fendermint/testing/materializer/src/concurrency/mod.rs b/fendermint/testing/materializer/src/concurrency/mod.rs index 8d4f0d2d4..5eaf8a6e2 100644 --- a/fendermint/testing/materializer/src/concurrency/mod.rs +++ b/fendermint/testing/materializer/src/concurrency/mod.rs @@ -7,9 +7,9 @@ pub mod nonce_manager; pub mod reporting; pub mod signal; -pub use reporting::*; - use crate::bencher::Bencher; +use crate::concurrency::reporting::TestResult; +use ethers::types::H256; use futures::FutureExt; use std::future::Future; use std::pin::Pin; @@ -17,9 +17,21 @@ use std::sync::Arc; use std::time::Instant; use tokio::sync::Semaphore; +#[derive(Debug)] +pub struct TestInput { + pub test_id: usize, + pub bencher: Bencher, +} + +#[derive(Debug)] +pub struct TestOutput { + pub bencher: Bencher, + pub tx_hash: H256, +} + pub async fn execute(cfg: config::Execution, test_factory: F) -> Vec> where - F: Fn(usize, Bencher) -> Pin> + Send>>, + F: Fn(TestInput) -> Pin> + Send>>, { let mut test_id = 0; let mut results = Vec::new(); @@ -34,18 +46,20 @@ where } let permit = semaphore.clone().acquire_owned().await.unwrap(); let bencher = Bencher::new(); - let task = test_factory(test_id, bencher).boxed(); + let test_input = TestInput { test_id, bencher }; + let task = test_factory(test_input).boxed(); let step_results = step_results.clone(); let handle = tokio::spawn(async move { - let res = task.await; - let (bencher, err) = match res { - Ok(bencher) => (Some(bencher), None), - Err(err) => (None, Some(err)), + let test_output = task.await; + let (bencher, tx_hash, err) = match test_output { + Ok(test_output) => (Some(test_output.bencher), Some(test_output.tx_hash), None), + Err(err) => (None, None, Some(err)), }; step_results.lock().await.push(TestResult { test_id, step_id, bencher, + tx_hash, err, }); drop(permit); diff --git a/fendermint/testing/materializer/src/concurrency/reporting.rs b/fendermint/testing/materializer/src/concurrency/reporting.rs deleted file mode 100644 index c526e8907..000000000 --- a/fendermint/testing/materializer/src/concurrency/reporting.rs +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright 2022-2024 Protocol Labs -// SPDX-License-Identifier: Apache-2.0, MIT - -use crate::bencher::Bencher; -use crate::concurrency::config; -use crate::concurrency::config::ExecutionStep; -use anyhow::anyhow; -use ethers::prelude::{Block, H256}; -use std::collections::{HashMap, HashSet}; -use std::io; -use std::time::Duration; - -#[derive(Debug)] -pub struct TestResult { - pub test_id: usize, - pub step_id: usize, - pub bencher: Option, - pub err: Option, -} - -#[derive(Debug)] -pub struct StepSummary { - pub cfg: ExecutionStep, - pub avg_latencies: HashMap, - pub avg_tps: u64, - pub errs: Vec, -} - -impl StepSummary { - fn new(cfg: ExecutionStep, results: Vec) -> Self { - let mut sum_latencies: HashMap = HashMap::new(); - let mut count_latencies: HashMap = HashMap::new(); - let mut block_inclusions: HashMap = HashMap::new(); - let mut errs = Vec::new(); - - for res in results { - let Some(bencher) = res.bencher else { continue }; - - for (key, duration) in bencher.latencies.clone() { - *sum_latencies.entry(key.clone()).or_insert(Duration::ZERO) += duration; - *count_latencies.entry(key).or_insert(0) += 1; - } - - if let Some(block) = bencher.block_inclusion { - *block_inclusions.entry(block).or_insert(0) += 1; - } - - if let Some(err) = res.err { - errs.push(err); - } - } - - // TODO: improve: - // 1. don't assume block time is 1s. - // 2. don't scope the count to execution step, - // because blocks might be shared with prev/next step, - // which skews the results. - // 3. don't use naive avg. - let avg_tps = { - let sum: u64 = block_inclusions.values().sum(); - let count = block_inclusions.len(); - sum / count as u64 - }; - - let avg_latencies = sum_latencies - .into_iter() - .map(|(key, total)| { - let count = count_latencies[&key]; - (key, total / count as u32) - }) - .collect(); - - Self { - cfg, - avg_latencies, - avg_tps, - errs, - } - } -} - -#[derive(Debug)] -pub struct ExecutionSummary { - pub summaries: Vec, -} - -impl ExecutionSummary { - pub fn new( - cfg: config::Execution, - _blocks: HashMap>, - results: Vec>, - ) -> Self { - let mut summaries = Vec::new(); - for (i, step_results) in results.into_iter().enumerate() { - let cfg = cfg.steps[i].clone(); - summaries.push(StepSummary::new(cfg, step_results)); - } - - Self { summaries } - } - - pub fn to_result(&self) -> anyhow::Result<()> { - let errs = self.errs(); - if errs.is_empty() { - Ok(()) - } else { - Err(anyhow!(errs.join("\n"))) - } - } - - pub fn errs(&self) -> Vec { - let mut errs = Vec::new(); - for summary in self.summaries.iter() { - let cloned_errs: Vec = - summary.errs.iter().map(|e| format!("{:?}", e)).collect(); - errs.extend(cloned_errs); - } - errs - } - - pub fn print(&self) { - let mut data = vec![]; - - let latencies: HashSet = self - .summaries - .iter() - .flat_map(|summary| summary.avg_latencies.keys().cloned()) - .collect(); - - let mut header = vec![ - "max_concurrency".to_string(), - "duration (s)".to_string(), - "TPS".to_string(), - ]; - header.extend(latencies.iter().map(|key| format!("{} latency (ms)", key))); - data.push(header); - - for summary in self.summaries.iter() { - let mut row = vec![]; - row.push(summary.cfg.max_concurrency.to_string()); - row.push(summary.cfg.duration.as_secs().to_string()); - row.push(summary.avg_tps.to_string()); - - for key in &latencies { - let latency = summary - .avg_latencies - .get(key) - .map_or(String::from("-"), |duration| { - duration.as_millis().to_string() - }); - row.push(latency); - } - - data.push(row); - } - - text_tables::render(&mut io::stdout(), data).unwrap(); - } -} diff --git a/fendermint/testing/materializer/src/concurrency/reporting/dataset.rs b/fendermint/testing/materializer/src/concurrency/reporting/dataset.rs new file mode 100644 index 000000000..cbeb293d8 --- /dev/null +++ b/fendermint/testing/materializer/src/concurrency/reporting/dataset.rs @@ -0,0 +1,87 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +use anyhow::anyhow; +use std::fmt::{Display, Formatter}; + +#[derive(Debug)] +pub struct Metrics { + pub mean: f64, + pub median: f64, + pub max: f64, + pub min: f64, + pub percentile_90: f64, +} + +impl Display for Metrics { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "mean: {:.2}, median: {:.2}, max: {:.2}, min: {:.2}, 90th: {:.2}", + self.mean, self.median, self.max, self.min, self.percentile_90 + ) + } +} + +impl Metrics { + pub fn format_median(&self) -> String { + format!("median: {:.2}", self.median) + } +} + +pub fn calc_metrics(data: Vec) -> anyhow::Result { + if data.is_empty() { + return Err(anyhow!("empty data")); + } + + let mut sorted_data = data.clone(); + sorted_data.sort_by(|a, b| a.partial_cmp(b).unwrap()); + + let count = sorted_data.len(); + let mean: f64 = sorted_data.iter().sum::() / count as f64; + + let median = if count % 2 == 0 { + (sorted_data[count / 2 - 1] + sorted_data[count / 2]) / 2.0 + } else { + sorted_data[count / 2] + }; + + let max = *sorted_data.last().unwrap(); + let min = *sorted_data.first().unwrap(); + + let percentile_90_index = ((count as f64) * 0.9).ceil() as usize - 1; + let percentile_90 = sorted_data[percentile_90_index]; + + Ok(Metrics { + mean, + median, + max, + min, + percentile_90, + }) +} + +#[cfg(test)] +mod tests { + use super::super::FLOAT_TOLERANCE; + use super::*; + + #[test] + fn test_calc_dataset_metrics() { + let data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0]; + + let expected_mean = 5.5; + let expected_median = 5.5; + let expected_max = 10.0; + let expected_min = 1.0; + let expected_percentile_90 = 9.0; + + let metrics = calc_metrics(data).unwrap(); + + assert!((metrics.mean - expected_mean).abs() < FLOAT_TOLERANCE); + assert!((metrics.median - expected_median).abs() < FLOAT_TOLERANCE); + assert!((metrics.max - expected_max).abs() < FLOAT_TOLERANCE); + assert!((metrics.min - expected_min).abs() < FLOAT_TOLERANCE); + assert!((metrics.percentile_90 - expected_percentile_90).abs() < FLOAT_TOLERANCE); + } +} diff --git a/fendermint/testing/materializer/src/concurrency/reporting/mod.rs b/fendermint/testing/materializer/src/concurrency/reporting/mod.rs new file mode 100644 index 000000000..2ae3337f8 --- /dev/null +++ b/fendermint/testing/materializer/src/concurrency/reporting/mod.rs @@ -0,0 +1,21 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +pub mod dataset; +pub mod summary; +pub mod tps; + +use crate::bencher::Bencher; +use ethers::prelude::H256; + +#[cfg(test)] +const FLOAT_TOLERANCE: f64 = 1e-6; + +#[derive(Debug)] +pub struct TestResult { + pub test_id: usize, + pub step_id: usize, + pub tx_hash: Option, + pub bencher: Option, + pub err: Option, +} diff --git a/fendermint/testing/materializer/src/concurrency/reporting/summary.rs b/fendermint/testing/materializer/src/concurrency/reporting/summary.rs new file mode 100644 index 000000000..a5893523d --- /dev/null +++ b/fendermint/testing/materializer/src/concurrency/reporting/summary.rs @@ -0,0 +1,185 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +use super::{ + dataset::{calc_metrics, Metrics}, + tps::calc_tps, + TestResult, +}; +use crate::concurrency::config; +use crate::concurrency::config::ExecutionStep; +use anyhow::anyhow; +use ethers::prelude::{Block, H256}; +use std::collections::{HashMap, HashSet}; +use std::io; + +#[derive(Debug)] +pub struct ExecutionSummary { + pub summaries: Vec, +} + +impl ExecutionSummary { + pub fn new( + cfg: config::Execution, + blocks: HashMap>, + results: Vec>, + ) -> Self { + let step_txs = Self::map_results_to_txs(&results); + let step_blocks = Self::map_blocks_to_steps(blocks, step_txs); + + let mut summaries = Vec::new(); + for (i, results) in results.into_iter().enumerate() { + let cfg = cfg.steps[i].clone(); + let blocks = step_blocks[i].clone(); + summaries.push(StepSummary::new(cfg, results, blocks)); + } + + Self { summaries } + } + + pub fn to_result(&self) -> anyhow::Result<()> { + let errs = self.errs(); + if errs.is_empty() { + Ok(()) + } else { + Err(anyhow!(errs.join("\n"))) + } + } + + pub fn errs(&self) -> Vec { + let mut errs = Vec::new(); + for summary in self.summaries.iter() { + let cloned_errs: Vec = + summary.errs.iter().map(|e| format!("{:?}", e)).collect(); + errs.extend(cloned_errs); + } + errs + } + + pub fn print(&self) { + let mut data = vec![]; + + let latencies: HashSet = self + .summaries + .iter() + .flat_map(|summary| summary.latencies.keys().cloned()) + .collect(); + + let mut header = vec![ + "max_concurrency".to_string(), + "duration".to_string(), + "TPS".to_string(), + ]; + header.extend(latencies.iter().map(|key| format!("latency ({}) ", key))); + data.push(header); + + for summary in self.summaries.iter() { + let mut row = vec![]; + row.push(summary.cfg.max_concurrency.to_string()); + row.push(summary.cfg.duration.as_secs().to_string()); + row.push(summary.tps.format_median()); + + for key in &latencies { + let latency = summary + .latencies + .get(key) + .map_or(String::from("-"), |metrics| metrics.format_median() + "s"); + row.push(latency); + } + + data.push(row); + } + + text_tables::render(&mut io::stdout(), data).unwrap(); + } + + fn map_results_to_txs(results: &[Vec]) -> Vec> { + results + .iter() + .map(|step_results| { + step_results + .iter() + .filter_map(|result| result.tx_hash) + .collect() + }) + .collect() + } + + pub fn map_blocks_to_steps( + blocks: HashMap>, + step_txs: Vec>, + ) -> Vec>> { + let mut sorted_blocks: Vec<_> = blocks.into_iter().collect(); + sorted_blocks.sort_by_key(|(block_number, _)| *block_number); + + let mut step_mapped_blocks: Vec>> = vec![Vec::new(); step_txs.len()]; + + for (_, block) in sorted_blocks { + // Determine the max step_id based on the transactions in the block + let latest_step_id = block + .transactions + .iter() + .filter_map(|tx_hash| { + step_txs.iter().enumerate().find_map(|(step_id, txs)| { + if txs.contains(tx_hash) { + Some(step_id) + } else { + None + } + }) + }) + .max(); + + if let Some(step_id) = latest_step_id { + // Add the block to the corresponding step_id. + step_mapped_blocks[step_id].push(block); + } + } + + step_mapped_blocks + } +} + +#[derive(Debug)] +pub struct StepSummary { + pub cfg: ExecutionStep, + pub latencies: HashMap, + pub tps: Metrics, + pub errs: Vec, +} + +impl StepSummary { + fn new(cfg: ExecutionStep, results: Vec, blocks: Vec>) -> Self { + let mut latencies: HashMap> = HashMap::new(); + let mut errs = Vec::new(); + + for res in results { + let Some(bencher) = res.bencher else { continue }; + + for (key, duration) in bencher.latencies.clone() { + latencies + .entry(key.clone()) + .or_default() + .push(duration.as_secs_f64()); + } + + if let Some(err) = res.err { + errs.push(err); + } + } + + let latencies = latencies + .into_iter() + .map(|(key, dataset)| (key, calc_metrics(dataset).unwrap())) + .collect(); + + let tps = calc_metrics(calc_tps(blocks)).unwrap(); + + Self { + cfg, + latencies, + tps, + errs, + } + } +} diff --git a/fendermint/testing/materializer/src/concurrency/reporting/tps.rs b/fendermint/testing/materializer/src/concurrency/reporting/tps.rs new file mode 100644 index 000000000..93d0b74e4 --- /dev/null +++ b/fendermint/testing/materializer/src/concurrency/reporting/tps.rs @@ -0,0 +1,85 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +use ethers::prelude::{Block, H256, U256}; + +pub fn calc_tps(blocks: Vec>) -> Vec { + let mut tps = Vec::new(); + + let offset = 1; // The first block is skipped for lacking a block interval. + for i in offset..blocks.len() { + let prev_block = &blocks[i - 1]; + let curr_block = &blocks[i]; + + let interval = curr_block.timestamp.saturating_sub(prev_block.timestamp); + + if interval.le(&U256::zero()) { + continue; + } + + let interval = interval.as_u64() as f64; + let tx_count = curr_block.transactions.len() as f64; + let block_tps = tx_count / interval; + tps.push(block_tps); + } + tps +} + +#[cfg(test)] +mod tests { + use super::super::FLOAT_TOLERANCE; + use super::*; + + #[test] + fn test_calc_tps() { + let tx = H256::random(); + let blocks = vec![ + Block { + timestamp: U256::from(100), + transactions: vec![tx; 500], + ..Default::default() + }, + Block { + timestamp: U256::from(110), + transactions: vec![tx; 600], + ..Default::default() + }, + Block { + timestamp: U256::from(130), + transactions: vec![tx; 1400], + ..Default::default() + }, + Block { + timestamp: U256::from(160), + transactions: vec![tx; 2400], + ..Default::default() + }, + Block { + timestamp: U256::from(200), + transactions: vec![tx; 4000], + ..Default::default() + }, + ] + .into_iter() + .collect(); + + let tps = calc_tps(blocks); + assert_eq!(tps.len(), 4); // Block 1 is skipped. + let expected_tps = [ + 600.0 / 10.0, // Block 2: 600 transactions in 10 seconds = 60 TPS + 1400.0 / 20.0, // Block 3: 1400 transactions in 20 seconds = 70 TPS + 2400.0 / 30.0, // Block 4: 2400 transactions in 30 seconds = 80 TPS + 4000.0 / 40.0, // Block 5: 4000 transactions in 40 seconds = 100 TPS + ]; + + for (i, &expected) in expected_tps.iter().enumerate() { + assert!( + (tps[i] - expected).abs() < FLOAT_TOLERANCE, + "mismatch at index {}: got {}, expected {}", + i, + tps[i], + expected + ); + } + } +} diff --git a/fendermint/testing/materializer/tests/docker_tests/benches.rs b/fendermint/testing/materializer/tests/docker_tests/benches.rs index 2a0efe554..dc09d09a1 100644 --- a/fendermint/testing/materializer/tests/docker_tests/benches.rs +++ b/fendermint/testing/materializer/tests/docker_tests/benches.rs @@ -15,10 +15,11 @@ use ethers::{ signers::{Signer, Wallet}, types::{Eip1559TransactionRequest, H160}, }; -use fendermint_materializer::bencher::Bencher; use fendermint_materializer::concurrency::collect::collect_blocks; use fendermint_materializer::concurrency::nonce_manager::NonceManager; +use fendermint_materializer::concurrency::reporting::summary::ExecutionSummary; use fendermint_materializer::concurrency::signal::Signal; +use fendermint_materializer::concurrency::TestOutput; use fendermint_materializer::{ concurrency::{self, config::Execution}, materials::DefaultAccount, @@ -85,21 +86,20 @@ async fn test_concurrent_transfer() -> Result<(), anyhow::Error> { .add_step(1, 5) .add_step(10, 5) .add_step(100, 5) - .add_step(150, 5) - .add_step(200, 5); + .add_step(150, 5); let testnet = Arc::new(testnet); let testnet_clone = testnet.clone(); let nonce_manager = Arc::new(NonceManager::new()); - let results = concurrency::execute(cfg.clone(), move |test_id: usize, mut bencher: Bencher| { + let results = concurrency::execute(cfg.clone(), move |mut input| { let testnet = testnet_clone.clone(); let nonce_manager = nonce_manager.clone(); let provider = provider.clone(); let test = async move { - let sender = testnet.account_mod_nth(test_id); - let recipient = testnet.account_mod_nth(test_id + 1); - println!("running (test_id={})", test_id); + let sender = testnet.account_mod_nth(input.test_id); + let recipient = testnet.account_mod_nth(input.test_id + 1); + println!("running (test_id={})", input.test_id); let middleware = make_middleware(provider, sender, chain_id) .await @@ -122,14 +122,14 @@ async fn test_concurrent_transfer() -> Result<(), anyhow::Error> { tx = tx.gas(gas_estimation); assert!(gas_estimation <= max_tx_gas_limit); - bencher.start(); + input.bencher.start(); let pending: PendingTransaction<_> = middleware .send_transaction(tx, None) .await .context("failed to send txn")?; let tx_hash = pending.tx_hash(); - println!("sent tx {:?} (test_id={})", tx_hash, test_id); + println!("sent tx {:?} (test_id={})", tx_hash, input.test_id); // We expect that the transaction is pending, however it should not return an error. match middleware.get_transaction(tx_hash).await { @@ -139,22 +139,25 @@ async fn test_concurrent_transfer() -> Result<(), anyhow::Error> { bail!("failed to get pending transaction: {e}") } } - bencher.mempool(); + input.bencher.mempool(); loop { if let Ok(Some(tx)) = middleware.get_transaction_receipt(tx_hash).await { println!( "tx included in block {:?} (test_id={})", - tx.block_number, test_id + tx.block_number, input.test_id ); let block_number = tx.block_number.unwrap().as_u64(); - bencher.block_inclusion(block_number); + input.bencher.block_inclusion(block_number); break; } sleep(Duration::from_millis(100)).await; } - Ok(bencher) + Ok(TestOutput { + bencher: input.bencher, + tx_hash, + }) }; test.boxed() }) @@ -162,7 +165,7 @@ async fn test_concurrent_transfer() -> Result<(), anyhow::Error> { cancel.send(); let blocks = blocks_collector.await??; - let summary = concurrency::ExecutionSummary::new(cfg.clone(), blocks, results); + let summary = ExecutionSummary::new(cfg.clone(), blocks, results); summary.print(); let res = summary.to_result(); From 14f948cd7642b11a98ce94774794e4051f51c4d3 Mon Sep 17 00:00:00 2001 From: LePremierHomme <57456510+LePremierHomme@users.noreply.github.com> Date: Thu, 16 Jan 2025 14:37:44 +0200 Subject: [PATCH 14/14] rename test func --- fendermint/testing/materializer/tests/docker_tests/benches.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fendermint/testing/materializer/tests/docker_tests/benches.rs b/fendermint/testing/materializer/tests/docker_tests/benches.rs index dc09d09a1..041746bc5 100644 --- a/fendermint/testing/materializer/tests/docker_tests/benches.rs +++ b/fendermint/testing/materializer/tests/docker_tests/benches.rs @@ -52,7 +52,7 @@ const MAX_TX_GAS_LIMIT: u64 = 3_000_000; #[serial_test::serial] #[tokio::test] -async fn test_concurrent_transfer() -> Result<(), anyhow::Error> { +async fn test_native_coin_transfer() -> Result<(), anyhow::Error> { let (testnet, cleanup) = make_testnet(MANIFEST, |_| {}).await?; let block_gas_limit = U256::from(BLOCK_GAS_LIMIT);