diff --git a/crates/astria-conductor/src/conductor/inner.rs b/crates/astria-conductor/src/conductor/inner.rs index 67a919466..6f99514b8 100644 --- a/crates/astria-conductor/src/conductor/inner.rs +++ b/crates/astria-conductor/src/conductor/inner.rs @@ -250,12 +250,14 @@ impl ConductorInner { name, error, } => { - if check_for_restart(name, error) { + if check_err_for_restart(name, error) { restart_or_shutdown = RestartOrShutdown::Restart; } } - ExitReason::StopHeightExceded(_) => { - restart_or_shutdown = RestartOrShutdown::Restart; + ExitReason::StopHeightExceded(stop_height_exceded) => { + if !stop_height_exceded.halt() { + restart_or_shutdown = RestartOrShutdown::Restart; + } } ExitReason::ChannelsClosed => { info!("firm and soft block channels are both closed, shutting down"); @@ -272,7 +274,7 @@ impl ConductorInner { info!(name, message, %exit_reason); } Err(error) => { - if check_for_restart(name, &error) + if check_err_for_restart(name, &error) && !matches!(exit_reason, ExitReason::ShutdownSignal) { restart_or_shutdown = RestartOrShutdown::Restart; @@ -329,7 +331,7 @@ fn report_exit(exit_reason: &ExitReason, message: &str) { } #[instrument(skip_all)] -fn check_for_restart(name: &str, err: &eyre::Report) -> bool { +fn check_err_for_restart(name: &str, err: &eyre::Report) -> bool { if name != ConductorInner::EXECUTOR { return false; } @@ -350,12 +352,12 @@ mod tests { use astria_eyre::eyre::WrapErr as _; #[test] - fn check_for_restart_ok() { + fn check_err_for_restart_ok() { let tonic_error: Result<&str, tonic::Status> = Err(tonic::Status::new(tonic::Code::PermissionDenied, "error")); let err = tonic_error.wrap_err("wrapper_1"); let err = err.wrap_err("wrapper_2"); let err = err.wrap_err("wrapper_3"); - assert!(super::check_for_restart("executor", &err.unwrap_err())); + assert!(super::check_err_for_restart("executor", &err.unwrap_err())); } } diff --git a/crates/astria-conductor/src/executor/mod.rs b/crates/astria-conductor/src/executor/mod.rs index d6d38223d..5c12bb04a 100644 --- a/crates/astria-conductor/src/executor/mod.rs +++ b/crates/astria-conductor/src/executor/mod.rs @@ -70,22 +70,40 @@ pub(crate) struct StateIsInit; #[derive(Debug)] pub(crate) enum StopHeightExceded { - Celestia { firm_height: u64, stop_height: u64 }, - Sequencer { soft_height: u64, stop_height: u64 }, + Celestia { + firm_height: u64, + stop_height: u64, + halt: bool, + }, + Sequencer { + soft_height: u64, + stop_height: u64, + halt: bool, + }, } impl StopHeightExceded { - fn celestia(firm_height: u64, stop_height: u64) -> Self { + fn celestia(firm_height: u64, stop_height: u64, halt: bool) -> Self { StopHeightExceded::Celestia { firm_height, stop_height, + halt, } } - fn sequencer(soft_height: u64, stop_height: u64) -> Self { + fn sequencer(soft_height: u64, stop_height: u64, halt: bool) -> Self { StopHeightExceded::Sequencer { soft_height, stop_height, + halt, + } + } + + /// Returns whether the conductor should halt at the stop height. + pub(crate) fn halt(&self) -> bool { + match self { + StopHeightExceded::Celestia { halt, .. } + | StopHeightExceded::Sequencer { halt, .. } => *halt, } } } @@ -96,17 +114,27 @@ impl std::fmt::Display for StopHeightExceded { StopHeightExceded::Celestia { firm_height, stop_height, - } => write!( - f, - "firm block height {firm_height} exceded stop height {stop_height}", - ), + halt, + } => { + let halt_msg = if *halt { "halting" } else { "restarting" }; + write!( + f, + "firm block height {firm_height} exceded stop height {stop_height}, \ + {halt_msg}...", + ) + } StopHeightExceded::Sequencer { soft_height, stop_height, - } => write!( - f, - "soft block height {soft_height} exceded stop height {stop_height}", - ), + halt, + } => { + let halt_msg = if *halt { "halting" } else { "restarting" }; + write!( + f, + "soft block height {soft_height} exceded stop height {stop_height}, \ + {halt_msg}...", + ) + } } } } @@ -483,6 +511,7 @@ impl Executor { return Ok(Some(StopHeightExceded::sequencer( executable_block.height.value(), self.state.sequencer_stop_block_height(), + self.state.genesis_info().halt_at_stop_height(), ))); } @@ -557,6 +586,7 @@ impl Executor { return Ok(Some(StopHeightExceded::celestia( block.header.height().value(), self.state.sequencer_stop_block_height(), + self.state.genesis_info().halt_at_stop_height(), ))); } diff --git a/crates/astria-conductor/tests/blackbox/soft_and_firm.rs b/crates/astria-conductor/tests/blackbox/soft_and_firm.rs index f5c55b558..11e7e6760 100644 --- a/crates/astria-conductor/tests/blackbox/soft_and_firm.rs +++ b/crates/astria-conductor/tests/blackbox/soft_and_firm.rs @@ -820,114 +820,113 @@ async fn conductor_restarts_after_reaching_stop_height() { .expect("conductor should have updated the firm commitment state within 1000ms"); } -// ** Disabled until logic to handle this is implemented ** -// /// Tests if the conductor correctly stops and does not restart after reaching the sequencer stop -// /// height if genesis info's `halt_at_stop_height` is `true`. -// /// -// /// This test consists of the following steps: -// /// 1. Mount commitment state and genesis info with a sequencer stop height of 3, expecting only -// 1 /// response. -// /// 2. Mount Celestia network head and sequencer genesis. -// /// 3. Mount ABCI info and sequencer (soft blocks) for height 3. -// /// 4. Mount firm blocks at height 3, with corresponding `update_commitment_state` mount. -// /// 5. Mount `execute_block` and `update_commitment_state` for firm block at height -// /// 3. The soft block should not be executed since it is at the stop height, but the firm -// should /// be. -// /// 6. Await satisfaction of the `execute_block` and `update_commitment_state` for the firm block -// at /// height 3 with a timeout of 1000ms. -// /// 7. Allow ample time for the conductor to potentially restart erroneously. -// #[tokio::test(flavor = "multi_thread", worker_threads = 1)] -// async fn conductor_stops_at_stop_height() { -// let test_conductor = spawn_conductor(CommitLevel::SoftAndFirm).await; - -// mount_get_genesis_info!( -// test_conductor, -// sequencer_start_block_height: 1, -// sequencer_stop_block_height: 3, -// celestia_block_variance: 10, -// rollup_start_block_height: 0, -// up_to_n_times: 2, // allow for calls after an potential erroneous restart -// halt_at_stop_height: true, -// expected_calls: 1, -// ); - -// mount_get_commitment_state!( -// test_conductor, -// firm: ( -// number: 1, -// hash: [1; 64], -// parent: [0; 64], -// ), -// soft: ( -// number: 1, -// hash: [1; 64], -// parent: [0; 64], -// ), -// base_celestia_height: 1, -// ); - -// mount_sequencer_genesis!(test_conductor); -// mount_celestia_header_network_head!( -// test_conductor, -// height: 1u32, -// ); -// mount_abci_info!( -// test_conductor, -// latest_sequencer_height: 3, -// ); - -// // Mount soft blocks for height 3 -// mount_get_filtered_sequencer_block!( -// test_conductor, -// sequencer_height: 3, -// ); - -// // Mount firm blocks for height 3 -// mount_celestia_blobs!( -// test_conductor, -// celestia_height: 1, -// sequencer_heights: [3], -// ); -// mount_sequencer_commit!( -// test_conductor, -// height: 3u32, -// ); -// mount_sequencer_validator_set!(test_conductor, height: 2u32); - -// let execute_block_1 = mount_executed_block!( -// test_conductor, -// mock_name: "execute_block_1", -// number: 2, -// hash: [2; 64], -// parent: [1; 64], -// ); - -// let update_commitment_state_firm_1 = mount_update_commitment_state!( -// test_conductor, -// mock_name: "update_commitment_state_firm_1", -// firm: ( -// number: 2, -// hash: [2; 64], -// parent: [1; 64], -// ), -// soft: ( -// number: 2, -// hash: [2; 64], -// parent: [1; 64], -// ), -// base_celestia_height: 1, -// ); - -// timeout( -// Duration::from_millis(1000), -// join( -// execute_block_1.wait_until_satisfied(), -// update_commitment_state_firm_1.wait_until_satisfied(), -// ), -// ) -// .await -// .expect("conductor should have updated the firm commitment state within 1000ms"); - -// // Allow time for a potential erroneous restart -// sleep(Duration::from_millis(1000)).await; -// } +/// Tests if the conductor correctly stops and does not restart after reaching the sequencer stop +/// height if genesis info's `halt_at_stop_height` is `true`. +/// +/// This test consists of the following steps: +/// 1. Mount commitment state and genesis info with a sequencer stop height of 3, expecting only 1 +/// response. +/// 2. Mount Celestia network head and sequencer genesis. +/// 3. Mount ABCI info and sequencer (soft blocks) for height 3. +/// 4. Mount firm blocks at height 3, with corresponding `update_commitment_state` mount. +/// 5. Mount `execute_block` and `update_commitment_state` for firm block at height +/// 3. The soft block should not be executed since it is at the stop height, but the firmshould +/// be. +/// 6. Await satisfaction of the `execute_block` and `update_commitment_state` for the firm block +/// height 3 with a timeout of 1000ms. +/// 7. Allow ample time for the conductor to potentially restart erroneously. +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn conductor_stops_at_stop_height() { + let test_conductor = spawn_conductor(CommitLevel::SoftAndFirm).await; + + mount_get_genesis_info!( + test_conductor, + sequencer_start_block_height: 1, + sequencer_stop_block_height: 3, + celestia_block_variance: 10, + rollup_start_block_height: 0, + up_to_n_times: 2, // allow for calls after an potential erroneous restart + halt_at_stop_height: true, + expected_calls: 1, + ); + + mount_get_commitment_state!( + test_conductor, + firm: ( + number: 1, + hash: [1; 64], + parent: [0; 64], + ), + soft: ( + number: 1, + hash: [1; 64], + parent: [0; 64], + ), + base_celestia_height: 1, + ); + + mount_sequencer_genesis!(test_conductor); + mount_celestia_header_network_head!( + test_conductor, + height: 1u32, + ); + mount_abci_info!( + test_conductor, + latest_sequencer_height: 3, + ); + + // Mount soft blocks for height 3 + mount_get_filtered_sequencer_block!( + test_conductor, + sequencer_height: 3, + ); + + // Mount firm blocks for height 3 + mount_celestia_blobs!( + test_conductor, + celestia_height: 1, + sequencer_heights: [3], + ); + mount_sequencer_commit!( + test_conductor, + height: 3u32, + ); + mount_sequencer_validator_set!(test_conductor, height: 2u32); + + let execute_block_1 = mount_executed_block!( + test_conductor, + mock_name: "execute_block_1", + number: 2, + hash: [2; 64], + parent: [1; 64], + ); + + let update_commitment_state_firm_1 = mount_update_commitment_state!( + test_conductor, + mock_name: "update_commitment_state_firm_1", + firm: ( + number: 2, + hash: [2; 64], + parent: [1; 64], + ), + soft: ( + number: 2, + hash: [2; 64], + parent: [1; 64], + ), + base_celestia_height: 1, + ); + + timeout( + Duration::from_millis(1000), + join( + execute_block_1.wait_until_satisfied(), + update_commitment_state_firm_1.wait_until_satisfied(), + ), + ) + .await + .expect("conductor should have updated the firm commitment state within 1000ms"); + + // Allow time for a potential erroneous restart + sleep(Duration::from_millis(1000)).await; +} diff --git a/crates/astria-core/src/execution/v1/mod.rs b/crates/astria-core/src/execution/v1/mod.rs index 80ebe83c3..ac8dc15a7 100644 --- a/crates/astria-core/src/execution/v1/mod.rs +++ b/crates/astria-core/src/execution/v1/mod.rs @@ -111,6 +111,11 @@ impl GenesisInfo { pub fn rollup_start_block_height(&self) -> u64 { self.rollup_start_block_height } + + #[must_use] + pub fn halt_at_stop_height(&self) -> bool { + self.halt_at_stop_height + } } impl From for raw::GenesisInfo {