diff --git a/apps/fortuna/Cargo.lock b/apps/fortuna/Cargo.lock index 78a359166..da418bd62 100644 --- a/apps/fortuna/Cargo.lock +++ b/apps/fortuna/Cargo.lock @@ -1503,7 +1503,7 @@ dependencies = [ [[package]] name = "fortuna" -version = "7.2.0" +version = "7.3.0" dependencies = [ "anyhow", "axum", diff --git a/apps/fortuna/Cargo.toml b/apps/fortuna/Cargo.toml index 959fa1c03..923ccafd0 100644 --- a/apps/fortuna/Cargo.toml +++ b/apps/fortuna/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fortuna" -version = "7.2.0" +version = "7.3.0" edition = "2021" [dependencies] diff --git a/apps/fortuna/src/keeper.rs b/apps/fortuna/src/keeper.rs index a836adea4..c40ce770d 100644 --- a/apps/fortuna/src/keeper.rs +++ b/apps/fortuna/src/keeper.rs @@ -69,6 +69,7 @@ pub struct KeeperMetrics { pub balance: Family>, pub collected_fee: Family>, pub current_fee: Family>, + pub target_provider_fee: Family>, pub total_gas_spent: Family>, pub total_gas_fee_spent: Family>, pub requests: Family, @@ -78,6 +79,10 @@ pub struct KeeperMetrics { pub requests_reprocessed: Family, pub reveals: Family, pub request_duration_ms: Family, + pub retry_count: Family, + pub final_gas_multiplier: Family, + pub final_fee_multiplier: Family, + pub gas_price_estimate: Family>, } impl Default for KeeperMetrics { @@ -88,6 +93,7 @@ impl Default for KeeperMetrics { balance: Family::default(), collected_fee: Family::default(), current_fee: Family::default(), + target_provider_fee: Family::default(), total_gas_spent: Family::default(), total_gas_fee_spent: Family::default(), requests: Family::default(), @@ -105,6 +111,18 @@ impl Default for KeeperMetrics { .into_iter(), ) }), + retry_count: Family::new_with_constructor(|| { + Histogram::new(vec![0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 10.0, 15.0, 20.0].into_iter()) + }), + final_gas_multiplier: Family::new_with_constructor(|| { + Histogram::new( + vec![100.0, 125.0, 150.0, 200.0, 300.0, 400.0, 500.0, 600.0].into_iter(), + ) + }), + final_fee_multiplier: Family::new_with_constructor(|| { + Histogram::new(vec![100.0, 110.0, 120.0, 140.0, 160.0, 180.0, 200.0].into_iter()) + }), + gas_price_estimate: Family::default(), } } } @@ -174,6 +192,12 @@ impl KeeperMetrics { keeper_metrics.current_fee.clone(), ); + writable_registry.register( + "target_provider_fee", + "Target fee in ETH -- differs from current_fee in that this is the goal, and current_fee is the on-chain value.", + keeper_metrics.target_provider_fee.clone(), + ); + writable_registry.register( "total_gas_spent", "Total gas spent revealing requests", @@ -198,6 +222,30 @@ impl KeeperMetrics { keeper_metrics.request_duration_ms.clone(), ); + writable_registry.register( + "retry_count", + "Number of retries for successful transactions", + keeper_metrics.retry_count.clone(), + ); + + writable_registry.register( + "final_gas_multiplier", + "Final gas multiplier percentage for successful transactions", + keeper_metrics.final_gas_multiplier.clone(), + ); + + writable_registry.register( + "final_fee_multiplier", + "Final fee multiplier percentage for successful transactions", + keeper_metrics.final_fee_multiplier.clone(), + ); + + writable_registry.register( + "gas_price_estimate", + "Gas price estimate for the blockchain (in gwei)", + keeper_metrics.gas_price_estimate.clone(), + ); + keeper_metrics } } @@ -327,6 +375,7 @@ pub async fn run_keeper_threads( spawn( adjust_fee_wrapper( contract.clone(), + chain_state.clone(), chain_state.provider_address, ADJUST_FEE_INTERVAL, chain_eth_config.legacy_tx, @@ -346,6 +395,7 @@ pub async fn run_keeper_threads( u64::try_from(100 + chain_eth_config.max_profit_pct) .expect("max_profit_pct must be >= -100"), chain_eth_config.fee, + metrics.clone(), ) .in_current_span(), ); @@ -479,6 +529,25 @@ pub async fn process_event_with_backoff( .request_duration_ms .get_or_create(&account_label) .observe(duration.as_millis() as f64); + + // Track retry count, gas multiplier, and fee multiplier for successful transactions + let num_retries = num_retries.load(std::sync::atomic::Ordering::Relaxed); + metrics + .retry_count + .get_or_create(&account_label) + .observe(num_retries as f64); + + let gas_multiplier = escalation_policy.get_gas_multiplier_pct(num_retries); + metrics + .final_gas_multiplier + .get_or_create(&account_label) + .observe(gas_multiplier as f64); + + let fee_multiplier = escalation_policy.get_fee_multiplier_pct(num_retries); + metrics + .final_fee_multiplier + .get_or_create(&account_label) + .observe(fee_multiplier as f64); } Err(e) => { // In case the callback did not succeed, we double-check that the request is still on-chain. @@ -1133,6 +1202,7 @@ pub async fn send_and_confirm(contract_call: PythContractCall) -> Result<()> { #[allow(clippy::too_many_arguments)] pub async fn adjust_fee_wrapper( contract: Arc, + chain_state: BlockchainState, provider_address: Address, poll_interval: Duration, legacy_tx: bool, @@ -1141,6 +1211,7 @@ pub async fn adjust_fee_wrapper( target_profit_pct: u64, max_profit_pct: u64, min_fee_wei: u128, + metrics: Arc, ) { // The maximum balance of accrued fees + provider wallet balance. None if we haven't observed a value yet. let mut high_water_pnl: Option = None; @@ -1149,6 +1220,7 @@ pub async fn adjust_fee_wrapper( loop { if let Err(e) = adjust_fee_if_necessary( contract.clone(), + chain_state.id.clone(), provider_address, legacy_tx, gas_limit, @@ -1158,6 +1230,7 @@ pub async fn adjust_fee_wrapper( min_fee_wei, &mut high_water_pnl, &mut sequence_number_of_last_fee_update, + metrics.clone(), ) .in_current_span() .await @@ -1232,6 +1305,7 @@ pub async fn update_commitments_if_necessary( #[allow(clippy::too_many_arguments)] pub async fn adjust_fee_if_necessary( contract: Arc, + chain_id: ChainId, provider_address: Address, legacy_tx: bool, gas_limit: u64, @@ -1241,6 +1315,7 @@ pub async fn adjust_fee_if_necessary( min_fee_wei: u128, high_water_pnl: &mut Option, sequence_number_of_last_fee_update: &mut Option, + metrics: Arc, ) -> Result<()> { let provider_info = contract .get_provider_info(provider_address) @@ -1256,6 +1331,17 @@ pub async fn adjust_fee_if_necessary( let max_callback_cost: u128 = estimate_tx_cost(contract.clone(), legacy_tx, gas_limit.into()) .await .map_err(|e| anyhow!("Could not estimate transaction cost. error {:?}", e))?; + + let account_label = AccountLabel { + chain_id: chain_id.clone(), + address: provider_address.to_string(), + }; + + metrics + .gas_price_estimate + .get_or_create(&account_label) + .set((max_callback_cost / u128::from(gas_limit)) as f64 / 1e9); + let target_fee_min = std::cmp::max( (max_callback_cost * u128::from(min_profit_pct)) / 100, min_fee_wei, @@ -1264,6 +1350,11 @@ pub async fn adjust_fee_if_necessary( (max_callback_cost * u128::from(target_profit_pct)) / 100, min_fee_wei, ); + metrics + .target_provider_fee + .get_or_create(&account_label) + .set(((max_callback_cost * u128::from(target_profit_pct)) / 100) as f64 / 1e18); + let target_fee_max = std::cmp::max( (max_callback_cost * u128::from(max_profit_pct)) / 100, min_fee_wei,