add tps, dataset metrics

fendermint/testing/materializer/src/concurrency/mod.rs

@@ -7,19 +7,31 @@ pub mod nonce_manager;
 pub mod reporting;
 pub mod signal;
 
-pub use reporting::*;
-
 use crate::bencher::Bencher;
+use crate::concurrency::reporting::TestResult;
+use ethers::types::H256;
 use futures::FutureExt;
 use std::future::Future;
 use std::pin::Pin;
 use std::sync::Arc;
 use std::time::Instant;
 use tokio::sync::Semaphore;
 
+#[derive(Debug)]
+pub struct TestInput {
+    pub test_id: usize,
+    pub bencher: Bencher,
+}
+
+#[derive(Debug)]
+pub struct TestOutput {
+    pub bencher: Bencher,
+    pub tx_hash: H256,
+}
+
 pub async fn execute<F>(cfg: config::Execution, test_factory: F) -> Vec<Vec<TestResult>>
 where
-    F: Fn(usize, Bencher) -> Pin<Box<dyn Future<Output = anyhow::Result<Bencher>> + Send>>,
+    F: Fn(TestInput) -> Pin<Box<dyn Future<Output = anyhow::Result<TestOutput>> + Send>>,
 {
     let mut test_id = 0;
     let mut results = Vec::new();
@@ -34,18 +46,20 @@ where
             }
             let permit = semaphore.clone().acquire_owned().await.unwrap();
             let bencher = Bencher::new();
-            let task = test_factory(test_id, bencher).boxed();
+            let test_input = TestInput { test_id, bencher };
+            let task = test_factory(test_input).boxed();
             let step_results = step_results.clone();
             let handle = tokio::spawn(async move {
-                let res = task.await;
-                let (bencher, err) = match res {
-                    Ok(bencher) => (Some(bencher), None),
-                    Err(err) => (None, Some(err)),
+                let test_output = task.await;
+                let (bencher, tx_hash, err) = match test_output {
+                    Ok(test_output) => (Some(test_output.bencher), Some(test_output.tx_hash), None),
+                    Err(err) => (None, None, Some(err)),
                 };
                 step_results.lock().await.push(TestResult {
                     test_id,
                     step_id,
                     bencher,
+                    tx_hash,
                     err,
                 });
                 drop(permit);

fendermint/testing/materializer/src/concurrency/reporting/dataset.rs

@@ -0,0 +1,87 @@
+// Copyright 2022-2024 Protocol Labs
+// SPDX-License-Identifier: Apache-2.0, MIT
+
+use anyhow::anyhow;
+use std::fmt::{Display, Formatter};
+
+#[derive(Debug)]
+pub struct Metrics {
+    pub mean: f64,
+    pub median: f64,
+    pub max: f64,
+    pub min: f64,
+    pub percentile_90: f64,
+}
+
+impl Display for Metrics {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "mean: {:.2}, median: {:.2}, max: {:.2}, min: {:.2}, 90th: {:.2}",
+            self.mean, self.median, self.max, self.min, self.percentile_90
+        )
+    }
+}
+
+impl Metrics {
+    pub fn format_median(&self) -> String {
+        format!("median: {:.2}", self.median)
+    }
+}
+
+pub fn calc_metrics(data: Vec<f64>) -> anyhow::Result<Metrics> {
+    if data.is_empty() {
+        return Err(anyhow!("empty data"));
+    }
+
+    let mut sorted_data = data.clone();
+    sorted_data.sort_by(|a, b| a.partial_cmp(b).unwrap());
+
+    let count = sorted_data.len();
+    let mean: f64 = sorted_data.iter().sum::<f64>() / count as f64;
+
+    let median = if count % 2 == 0 {
+        (sorted_data[count / 2 - 1] + sorted_data[count / 2]) / 2.0
+    } else {
+        sorted_data[count / 2]
+    };
+
+    let max = *sorted_data.last().unwrap();
+    let min = *sorted_data.first().unwrap();
+
+    let percentile_90_index = ((count as f64) * 0.9).ceil() as usize - 1;
+    let percentile_90 = sorted_data[percentile_90_index];
+
+    Ok(Metrics {
+        mean,
+        median,
+        max,
+        min,
+        percentile_90,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::super::FLOAT_TOLERANCE;
+    use super::*;
+
+    #[test]
+    fn test_calc_dataset_metrics() {
+        let data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
+
+        let expected_mean = 5.5;
+        let expected_median = 5.5;
+        let expected_max = 10.0;
+        let expected_min = 1.0;
+        let expected_percentile_90 = 9.0;
+
+        let metrics = calc_metrics(data).unwrap();
+
+        assert!((metrics.mean - expected_mean).abs() < FLOAT_TOLERANCE);
+        assert!((metrics.median - expected_median).abs() < FLOAT_TOLERANCE);
+        assert!((metrics.max - expected_max).abs() < FLOAT_TOLERANCE);
+        assert!((metrics.min - expected_min).abs() < FLOAT_TOLERANCE);
+        assert!((metrics.percentile_90 - expected_percentile_90).abs() < FLOAT_TOLERANCE);
+    }
+}

fendermint/testing/materializer/src/concurrency/reporting/mod.rs

@@ -0,0 +1,21 @@
+// Copyright 2022-2024 Protocol Labs
+// SPDX-License-Identifier: Apache-2.0, MIT
+
+pub mod dataset;
+pub mod summary;
+pub mod tps;
+
+use crate::bencher::Bencher;
+use ethers::prelude::H256;
+
+#[cfg(test)]
+const FLOAT_TOLERANCE: f64 = 1e-6;
+
+#[derive(Debug)]
+pub struct TestResult {
+    pub test_id: usize,
+    pub step_id: usize,
+    pub tx_hash: Option<H256>,
+    pub bencher: Option<Bencher>,
+    pub err: Option<anyhow::Error>,
+}