diff --git a/aptos-move/e2e-benchmark/data/calibration_values.tsv b/aptos-move/e2e-benchmark/data/calibration_values.tsv index 87dcf3bb84ab4..bdd6d9d7a895f 100644 --- a/aptos-move/e2e-benchmark/data/calibration_values.tsv +++ b/aptos-move/e2e-benchmark/data/calibration_values.tsv @@ -29,5 +29,7 @@ VectorTrimAppend { vec_len: 3000, element_len: 1, index: 100, repeats: 1000 } 11 VectorTrimAppend { vec_len: 3000, element_len: 1, index: 2990, repeats: 1000 } 119 0.947 1.093 18431.4 VectorRemoveInsert { vec_len: 3000, element_len: 1, index: 100, repeats: 1000 } 119 0.943 1.107 29034.8 VectorRemoveInsert { vec_len: 3000, element_len: 1, index: 2998, repeats: 1000 } 119 0.954 1.149 20047.3 +VectorRangeMove { vec_len: 3000, element_len: 1, index: 1000, move_len: 500, repeats: 1000 } 6 0.925 1.001 32535.4 VectorTrimAppend { vec_len: 100, element_len: 100, index: 0, repeats: 0 } 119 0.909 1.201 293.5 VectorTrimAppend { vec_len: 100, element_len: 100, index: 10, repeats: 1000 } 119 0.951 1.143 12571.2 +VectorRangeMove { vec_len: 100, element_len: 100, index: 50, move_len: 10, repeats: 1000 } 6 0.925 1.001 5316.2 diff --git a/aptos-move/e2e-benchmark/src/main.rs b/aptos-move/e2e-benchmark/src/main.rs index 8dc111a1aa31c..a4d4d1aadc40e 100644 --- a/aptos-move/e2e-benchmark/src/main.rs +++ b/aptos-move/e2e-benchmark/src/main.rs @@ -86,6 +86,8 @@ const ABSOLUTE_BUFFER_US: f64 = 2.0; struct CalibrationInfo { // count: usize, expected_time_micros: f64, + min_ratio: f64, + max_ratio: f64, } fn get_parsed_calibration_values() -> HashMap { @@ -100,6 +102,8 @@ fn get_parsed_calibration_values() -> HashMap { (parts[0].to_string(), CalibrationInfo { // count: parts[1].parse().unwrap(), expected_time_micros: parts[parts.len() - 1].parse().unwrap(), + min_ratio: parts[2].parse().unwrap(), + max_ratio: parts[3].parse().unwrap(), }) }) .collect() @@ -206,13 +210,13 @@ fn main() { index: 2998, repeats: 1000, }, - // EntryPoints::VectorRangeMove { - // vec_len: 3000, - // element_len: 1, - // index: 1000, - // move_len: 500, - // repeats: 1000, - // }, + EntryPoints::VectorRangeMove { + vec_len: 3000, + element_len: 1, + index: 1000, + move_len: 500, + repeats: 1000, + }, // vectors with large elements EntryPoints::VectorTrimAppend { // baseline, only vector creation @@ -227,13 +231,13 @@ fn main() { index: 10, repeats: 1000, }, - // EntryPoints::VectorRangeMove { - // vec_len: 100, - // element_len: 100, - // index: 50, - // move_len: 10, - // repeats: 1000, - // }, + EntryPoints::VectorRangeMove { + vec_len: 100, + element_len: 100, + index: 50, + move_len: 10, + repeats: 1000, + }, ]; let mut failures = Vec::new(); @@ -246,10 +250,10 @@ fn main() { for (index, entry_point) in entry_points.into_iter().enumerate() { let entry_point_name = format!("{:?}", entry_point); - let expected_time_micros = calibration_values + let cur_calibration = calibration_values .get(&entry_point_name) - .expect(&entry_point_name) - .expected_time_micros; + .expect(&entry_point_name); + let expected_time_micros = cur_calibration.expected_time_micros; let publisher = executor.new_account_at(AccountAddress::random()); let mut package_handler = @@ -317,17 +321,23 @@ fn main() { "test_index": index, })); - if elapsed_micros > expected_time_micros * (1.0 + ALLOWED_REGRESSION) + ABSOLUTE_BUFFER_US { + let max_regression = f64::max( + expected_time_micros * (1.0 + ALLOWED_REGRESSION) + ABSOLUTE_BUFFER_US, + expected_time_micros * cur_calibration.max_ratio, + ); + let max_improvement = f64::min( + expected_time_micros * (1.0 - ALLOWED_IMPROVEMENT) - ABSOLUTE_BUFFER_US, + expected_time_micros * cur_calibration.min_ratio, + ); + if elapsed_micros > max_regression { failures.push(format!( - "Performance regression detected: {:.1}us, expected: {:.1}us, diff: {}%, for {:?}", - elapsed_micros, expected_time_micros, diff, entry_point + "Performance regression detected: {:.1}us, expected: {:.1}us, limit: {:.1}us, diff: {}%, for {:?}", + elapsed_micros, expected_time_micros, max_regression, diff, entry_point )); - } else if elapsed_micros + ABSOLUTE_BUFFER_US - < expected_time_micros * (1.0 - ALLOWED_IMPROVEMENT) - { + } else if elapsed_micros < max_improvement { failures.push(format!( - "Performance improvement detected: {:.1}us, expected {:.1}us, diff: {}%, for {:?}. You need to adjust expected time!", - elapsed_micros, expected_time_micros, diff, entry_point + "Performance improvement detected: {:.1}us, expected {:.1}us, limit {:.1}us, diff: {}%, for {:?}. You need to adjust expected time!", + elapsed_micros, expected_time_micros, max_improvement, diff, entry_point )); } } diff --git a/crates/transaction-generator-lib/src/publishing/prebuild_packages.rs b/crates/transaction-generator-lib/src/publishing/prebuild_packages.rs index 72585677e0683..654f013a9ef28 100644 --- a/crates/transaction-generator-lib/src/publishing/prebuild_packages.rs +++ b/crates/transaction-generator-lib/src/publishing/prebuild_packages.rs @@ -50,9 +50,9 @@ pub fn create_prebuilt_packages_rs_file( writeln!( string_buffer, " +use aptos_transaction_generator_lib::entry_point_trait::PreBuiltPackages; use once_cell::sync::Lazy; -use std::collections::HashMap; -use aptos_transaction_generator_lib::entry_point_trait::PreBuiltPackages;", +use std::collections::HashMap;", ) .expect("Use directive failed"); writeln!(string_buffer).expect("Empty line failed"); diff --git a/crates/transaction-workloads-lib/src/raw_module_data.rs b/crates/transaction-workloads-lib/src/raw_module_data.rs index 026240b04eb43..4dff3d5b78653 100644 --- a/crates/transaction-workloads-lib/src/raw_module_data.rs +++ b/crates/transaction-workloads-lib/src/raw_module_data.rs @@ -12,9 +12,9 @@ // This crate should also provide a Rust file that allows proper manipulation of each // module defined below. +use aptos_transaction_generator_lib::entry_point_trait::PreBuiltPackages; use once_cell::sync::Lazy; use std::collections::HashMap; -use aptos_transaction_generator_lib::entry_point_trait::PreBuiltPackages; #[rustfmt::skip] pub static PACKAGE_SIMPLE_METADATA: Lazy> = Lazy::new(|| {