Skip to content

Commit

Permalink
add halt logic
Browse files Browse the repository at this point in the history
  • Loading branch information
@ethanoroshiba
ethanoroshiba committed Jan 6, 2025
1 parent 00b10a4 commit 02d04d7
Show file tree
Hide file tree
Showing 4 changed files with 166 additions and 130 deletions.
16 changes: 9 additions & 7 deletions crates/astria-conductor/src/conductor/inner.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,12 +250,14 @@ impl ConductorInner {
name,
error,
} => {
if check_for_restart(name, error) {
if check_err_for_restart(name, error) {
restart_or_shutdown = RestartOrShutdown::Restart;
}
}
ExitReason::StopHeightExceded(_) => {
restart_or_shutdown = RestartOrShutdown::Restart;
ExitReason::StopHeightExceded(stop_height_exceded) => {
if !stop_height_exceded.halt() {
restart_or_shutdown = RestartOrShutdown::Restart;
}
}
ExitReason::ChannelsClosed => {
info!("firm and soft block channels are both closed, shutting down");
Expand All @@ -272,7 +274,7 @@ impl ConductorInner {
info!(name, message, %exit_reason);
}
Err(error) => {
if check_for_restart(name, &error)
if check_err_for_restart(name, &error)
&& !matches!(exit_reason, ExitReason::ShutdownSignal)
{
restart_or_shutdown = RestartOrShutdown::Restart;
Expand Down Expand Up @@ -329,7 +331,7 @@ fn report_exit(exit_reason: &ExitReason, message: &str) {
}

#[instrument(skip_all)]
fn check_for_restart(name: &str, err: &eyre::Report) -> bool {
fn check_err_for_restart(name: &str, err: &eyre::Report) -> bool {
if name != ConductorInner::EXECUTOR {
return false;
}
Expand All @@ -350,12 +352,12 @@ mod tests {
use astria_eyre::eyre::WrapErr as _;

#[test]
fn check_for_restart_ok() {
fn check_err_for_restart_ok() {
let tonic_error: Result<&str, tonic::Status> =
Err(tonic::Status::new(tonic::Code::PermissionDenied, "error"));
let err = tonic_error.wrap_err("wrapper_1");
let err = err.wrap_err("wrapper_2");
let err = err.wrap_err("wrapper_3");
assert!(super::check_for_restart("executor", &err.unwrap_err()));
assert!(super::check_err_for_restart("executor", &err.unwrap_err()));
}
}
54 changes: 42 additions & 12 deletions crates/astria-conductor/src/executor/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,22 +70,40 @@ pub(crate) struct StateIsInit;

#[derive(Debug)]
pub(crate) enum StopHeightExceded {
Celestia { firm_height: u64, stop_height: u64 },
Sequencer { soft_height: u64, stop_height: u64 },
Celestia {
firm_height: u64,
stop_height: u64,
halt: bool,
},
Sequencer {
soft_height: u64,
stop_height: u64,
halt: bool,
},
}

impl StopHeightExceded {
fn celestia(firm_height: u64, stop_height: u64) -> Self {
fn celestia(firm_height: u64, stop_height: u64, halt: bool) -> Self {
StopHeightExceded::Celestia {
firm_height,
stop_height,
halt,
}
}

fn sequencer(soft_height: u64, stop_height: u64) -> Self {
fn sequencer(soft_height: u64, stop_height: u64, halt: bool) -> Self {
StopHeightExceded::Sequencer {
soft_height,
stop_height,
halt,
}
}

/// Returns whether the conductor should halt at the stop height.
pub(crate) fn halt(&self) -> bool {
match self {
StopHeightExceded::Celestia { halt, .. }
| StopHeightExceded::Sequencer { halt, .. } => *halt,
}
}
}
Expand All @@ -96,17 +114,27 @@ impl std::fmt::Display for StopHeightExceded {
StopHeightExceded::Celestia {
firm_height,
stop_height,
} => write!(
f,
"firm block height {firm_height} exceded stop height {stop_height}",
),
halt,
} => {
let halt_msg = if *halt { "halting" } else { "restarting" };
write!(
f,
"firm block height {firm_height} exceded stop height {stop_height}, \
{halt_msg}...",
)
}
StopHeightExceded::Sequencer {
soft_height,
stop_height,
} => write!(
f,
"soft block height {soft_height} exceded stop height {stop_height}",
),
halt,
} => {
let halt_msg = if *halt { "halting" } else { "restarting" };
write!(
f,
"soft block height {soft_height} exceded stop height {stop_height}, \
{halt_msg}...",
)
}
}
}
}
Expand Down Expand Up @@ -483,6 +511,7 @@ impl Executor {
return Ok(Some(StopHeightExceded::sequencer(
executable_block.height.value(),
self.state.sequencer_stop_block_height(),
self.state.genesis_info().halt_at_stop_height(),
)));
}

Expand Down Expand Up @@ -557,6 +586,7 @@ impl Executor {
return Ok(Some(StopHeightExceded::celestia(
block.header.height().value(),
self.state.sequencer_stop_block_height(),
self.state.genesis_info().halt_at_stop_height(),
)));
}

Expand Down
221 changes: 110 additions & 111 deletions crates/astria-conductor/tests/blackbox/soft_and_firm.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -820,114 +820,113 @@ async fn conductor_restarts_after_reaching_stop_height() {
.expect("conductor should have updated the firm commitment state within 1000ms");
}

// ** Disabled until logic to handle this is implemented **
// /// Tests if the conductor correctly stops and does not restart after reaching the sequencer stop
// /// height if genesis info's `halt_at_stop_height` is `true`.
// ///
// /// This test consists of the following steps:
// /// 1. Mount commitment state and genesis info with a sequencer stop height of 3, expecting only
// 1 /// response.
// /// 2. Mount Celestia network head and sequencer genesis.
// /// 3. Mount ABCI info and sequencer (soft blocks) for height 3.
// /// 4. Mount firm blocks at height 3, with corresponding `update_commitment_state` mount.
// /// 5. Mount `execute_block` and `update_commitment_state` for firm block at height
// /// 3. The soft block should not be executed since it is at the stop height, but the firm
// should /// be.
// /// 6. Await satisfaction of the `execute_block` and `update_commitment_state` for the firm block
// at /// height 3 with a timeout of 1000ms.
// /// 7. Allow ample time for the conductor to potentially restart erroneously.
// #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
// async fn conductor_stops_at_stop_height() {
// let test_conductor = spawn_conductor(CommitLevel::SoftAndFirm).await;

// mount_get_genesis_info!(
// test_conductor,
// sequencer_start_block_height: 1,
// sequencer_stop_block_height: 3,
// celestia_block_variance: 10,
// rollup_start_block_height: 0,
// up_to_n_times: 2, // allow for calls after an potential erroneous restart
// halt_at_stop_height: true,
// expected_calls: 1,
// );

// mount_get_commitment_state!(
// test_conductor,
// firm: (
// number: 1,
// hash: [1; 64],
// parent: [0; 64],
// ),
// soft: (
// number: 1,
// hash: [1; 64],
// parent: [0; 64],
// ),
// base_celestia_height: 1,
// );

// mount_sequencer_genesis!(test_conductor);
// mount_celestia_header_network_head!(
// test_conductor,
// height: 1u32,
// );
// mount_abci_info!(
// test_conductor,
// latest_sequencer_height: 3,
// );

// // Mount soft blocks for height 3
// mount_get_filtered_sequencer_block!(
// test_conductor,
// sequencer_height: 3,
// );

// // Mount firm blocks for height 3
// mount_celestia_blobs!(
// test_conductor,
// celestia_height: 1,
// sequencer_heights: [3],
// );
// mount_sequencer_commit!(
// test_conductor,
// height: 3u32,
// );
// mount_sequencer_validator_set!(test_conductor, height: 2u32);

// let execute_block_1 = mount_executed_block!(
// test_conductor,
// mock_name: "execute_block_1",
// number: 2,
// hash: [2; 64],
// parent: [1; 64],
// );

// let update_commitment_state_firm_1 = mount_update_commitment_state!(
// test_conductor,
// mock_name: "update_commitment_state_firm_1",
// firm: (
// number: 2,
// hash: [2; 64],
// parent: [1; 64],
// ),
// soft: (
// number: 2,
// hash: [2; 64],
// parent: [1; 64],
// ),
// base_celestia_height: 1,
// );

// timeout(
// Duration::from_millis(1000),
// join(
// execute_block_1.wait_until_satisfied(),
// update_commitment_state_firm_1.wait_until_satisfied(),
// ),
// )
// .await
// .expect("conductor should have updated the firm commitment state within 1000ms");

// // Allow time for a potential erroneous restart
// sleep(Duration::from_millis(1000)).await;
// }
/// Tests if the conductor correctly stops and does not restart after reaching the sequencer stop
/// height if genesis info's `halt_at_stop_height` is `true`.
///
/// This test consists of the following steps:
/// 1. Mount commitment state and genesis info with a sequencer stop height of 3, expecting only 1
/// response.
/// 2. Mount Celestia network head and sequencer genesis.
/// 3. Mount ABCI info and sequencer (soft blocks) for height 3.
/// 4. Mount firm blocks at height 3, with corresponding `update_commitment_state` mount.
/// 5. Mount `execute_block` and `update_commitment_state` for firm block at height
/// 3. The soft block should not be executed since it is at the stop height, but the firmshould
/// be.
/// 6. Await satisfaction of the `execute_block` and `update_commitment_state` for the firm block
/// height 3 with a timeout of 1000ms.
/// 7. Allow ample time for the conductor to potentially restart erroneously.
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
async fn conductor_stops_at_stop_height() {
let test_conductor = spawn_conductor(CommitLevel::SoftAndFirm).await;

mount_get_genesis_info!(
test_conductor,
sequencer_start_block_height: 1,
sequencer_stop_block_height: 3,
celestia_block_variance: 10,
rollup_start_block_height: 0,
up_to_n_times: 2, // allow for calls after an potential erroneous restart
halt_at_stop_height: true,
expected_calls: 1,
);

mount_get_commitment_state!(
test_conductor,
firm: (
number: 1,
hash: [1; 64],
parent: [0; 64],
),
soft: (
number: 1,
hash: [1; 64],
parent: [0; 64],
),
base_celestia_height: 1,
);

mount_sequencer_genesis!(test_conductor);
mount_celestia_header_network_head!(
test_conductor,
height: 1u32,
);
mount_abci_info!(
test_conductor,
latest_sequencer_height: 3,
);

// Mount soft blocks for height 3
mount_get_filtered_sequencer_block!(
test_conductor,
sequencer_height: 3,
);

// Mount firm blocks for height 3
mount_celestia_blobs!(
test_conductor,
celestia_height: 1,
sequencer_heights: [3],
);
mount_sequencer_commit!(
test_conductor,
height: 3u32,
);
mount_sequencer_validator_set!(test_conductor, height: 2u32);

let execute_block_1 = mount_executed_block!(
test_conductor,
mock_name: "execute_block_1",
number: 2,
hash: [2; 64],
parent: [1; 64],
);

let update_commitment_state_firm_1 = mount_update_commitment_state!(
test_conductor,
mock_name: "update_commitment_state_firm_1",
firm: (
number: 2,
hash: [2; 64],
parent: [1; 64],
),
soft: (
number: 2,
hash: [2; 64],
parent: [1; 64],
),
base_celestia_height: 1,
);

timeout(
Duration::from_millis(1000),
join(
execute_block_1.wait_until_satisfied(),
update_commitment_state_firm_1.wait_until_satisfied(),
),
)
.await
.expect("conductor should have updated the firm commitment state within 1000ms");

// Allow time for a potential erroneous restart
sleep(Duration::from_millis(1000)).await;
}
5 changes: 5 additions & 0 deletions crates/astria-core/src/execution/v1/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,11 @@ impl GenesisInfo {
pub fn rollup_start_block_height(&self) -> u64 {
self.rollup_start_block_height
}

#[must_use]
pub fn halt_at_stop_height(&self) -> bool {
self.halt_at_stop_height
}
}

impl From<GenesisInfo> for raw::GenesisInfo {
Expand Down

0 comments on commit 02d04d7

Please sign in to comment.