Validator health status

primitives/account/src/account/staking_contract/validator.rs

@@ -46,6 +46,12 @@ use crate::{
 ///      in the first place.
 /// (**) The validator may be set to automatically reactivate itself upon inactivation.
 ///      If this setting is not enabled the state change can only be triggered manually.
+///      However, there is a validator health status with the following states:
+///      -> Green: Everything is working as expected, if the validator is deactivated its status changes to Yellow
+///      -> Yellow: If the validator is deactivated again, its status is changed to Red
+///      -> Red: If the validator is deactivated again, the automatic reactivate (if enabled) has no effect
+///              Human intervention is required at this point
+///      To go from Red to Yellow or Yellow to Green, the validator needs to be active for at least a quarter of an epoch
 ///
 /// Create, Update, Deactivate, Retire and Re-activate are incoming transactions to the staking contract.
 /// Delete is an outgoing transaction from the staking contract.

test-utils/src/validator.rs

@@ -113,7 +113,7 @@ where
         let (v, c) = build_validator(
             peer_ids[i],
             Address::from(&validator_keys[i]),
-            false,
+            true,
             signing_keys[i].clone(),
             voting_keys[i].clone(),
             fee_keys[i].clone(),

validator/src/micro.rs

@@ -10,6 +10,7 @@ use futures::{future::BoxFuture, ready, FutureExt, Stream};
 use nimiq_block::{Block, EquivocationProof, MicroBlock, SkipBlockInfo};
 use nimiq_blockchain::{BlockProducer, BlockProducerError, Blockchain};
 use nimiq_blockchain_interface::AbstractBlockchain;
+use nimiq_keys::Address;
 use nimiq_mempool::mempool::Mempool;
 use nimiq_primitives::policy::Policy;
 use nimiq_time::sleep;
@@ -36,6 +37,8 @@ struct NextProduceMicroBlockEvent<TValidatorNetwork> {
     block_number: u32,
     producer_timeout: Duration,
     block_separation_time: Duration,
+    validator_address: Address,
+    publish_block: bool,
 }
 
 impl<TValidatorNetwork: ValidatorNetwork + 'static> NextProduceMicroBlockEvent<TValidatorNetwork> {
@@ -53,6 +56,8 @@ impl<TValidatorNetwork: ValidatorNetwork + 'static> NextProduceMicroBlockEvent<T
         block_number: u32,
         producer_timeout: Duration,
         block_separation_time: Duration,
+        validator_address: Address,
+        publish_block: bool,
     ) -> Self {
         Self {
             blockchain,
@@ -65,6 +70,8 @@ impl<TValidatorNetwork: ValidatorNetwork + 'static> NextProduceMicroBlockEvent<T
             block_number,
             producer_timeout,
             block_separation_time,
+            validator_address,
+            publish_block,
         }
     }
 
@@ -117,6 +124,7 @@ impl<TValidatorNetwork: ValidatorNetwork + 'static> NextProduceMicroBlockEvent<T
             info!(
                 block_number = self.block_number,
                 slot_band = self.validator_slot_band,
+                address = %self.validator_address,
                 "Our turn, producing micro block #{}",
                 self.block_number,
             );
@@ -153,6 +161,12 @@ impl<TValidatorNetwork: ValidatorNetwork + 'static> NextProduceMicroBlockEvent<T
                 num_transactions
             );
 
+            if !self.publish_block {
+                log::warn!("Not publishing block {} ", block.block_number());
+                let event = ProduceMicroBlockEvent::MicroBlock;
+                break Some(Some(event));
+            }
+
             // Publish the block. It is valid as we have just created it.
             Validator::publish_block(Arc::clone(&self.network), block.clone());
 
@@ -194,6 +208,7 @@ impl<TValidatorNetwork: ValidatorNetwork + 'static> NextProduceMicroBlockEvent<T
         debug!(
             block_number = self.block_number,
             slot_band = self.validator_slot_band,
+            address = %self.validator_address,
             "Not our turn, waiting for micro block #{}",
             self.block_number,
         );
@@ -404,6 +419,8 @@ impl<TValidatorNetwork: ValidatorNetwork + 'static> ProduceMicroBlock<TValidator
         block_number: u32,
         producer_timeout: Duration,
         block_separation_time: Duration,
+        validator_address: Address,
+        publish_block: bool,
     ) -> Self {
         let next_event = NextProduceMicroBlockEvent::new(
             blockchain,
@@ -416,6 +433,8 @@ impl<TValidatorNetwork: ValidatorNetwork + 'static> ProduceMicroBlock<TValidator
             block_number,
             producer_timeout,
             block_separation_time,
+            validator_address,
+            publish_block,
         )
         .next()
         .boxed();

validator/src/validator.rs

@@ -62,6 +62,28 @@ pub struct ConsensusState {
     equivocation_proofs: EquivocationProofPool,
 }
 
+/// Struct that represents the overall health of a validator
+/// Green means the Validator is working as expected,
+/// If the validator is deactivated, we change its health to Yellow
+/// If the validator is Yellow and it is deactivated again, we change its health to Red
+/// While in the Red state, the automatic reactivate has no effect and human intervention is required
+/// If the validator is Yellow and is not deactivated in a quarter of an epoch, we change its status to Green.
+/// If the validator is Red and is not deactivated in one epoch, we change its status to Yellow.
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub enum ValidatorHealth {
+    Green,
+    Yellow(u32),
+    Red(u32),
+}
+
+/// Struct that represents the overall Validator Health
+pub struct HealthState {
+    /// The current validator health
+    pub health: ValidatorHealth,
+    /// For testing/debug purposes control wether produced blocks are published by the validator
+    pub publish: bool,
+}
+
 /// Validator inactivity
 struct InactivityState {
     inactive_tx_hash: Blake2bHash,
@@ -76,6 +98,7 @@ pub struct ValidatorProxy {
     pub automatic_reactivate: Arc<AtomicBool>,
     pub slot_band: Arc<RwLock<Option<u16>>>,
     pub consensus_state: Arc<RwLock<ConsensusState>>,
+    pub validator_health: Arc<RwLock<HealthState>>,
 }
 
 impl Clone for ValidatorProxy {
@@ -88,6 +111,7 @@ impl Clone for ValidatorProxy {
             automatic_reactivate: Arc::clone(&self.automatic_reactivate),
             slot_band: Arc::clone(&self.slot_band),
             consensus_state: Arc::clone(&self.consensus_state),
+            validator_health: Arc::clone(&self.validator_health),
         }
     }
 }
@@ -119,6 +143,8 @@ where
     slot_band: Arc<RwLock<Option<u16>>>,
     consensus_state: Arc<RwLock<ConsensusState>>,
     validator_state: Option<InactivityState>,
+    health_state: Arc<RwLock<HealthState>>,
+
     automatic_reactivate: Arc<AtomicBool>,
 
     macro_producer: Option<ProduceMacroBlock<TValidatorNetwork>>,
@@ -200,6 +226,11 @@ where
                 .await
         });
 
+        let health_state = HealthState {
+            health: ValidatorHealth::Green,
+            publish: true,
+        };
+
         Self {
             consensus: consensus.proxy(),
             blockchain,
@@ -222,6 +253,8 @@ where
             slot_band: Arc::new(RwLock::new(None)),
             consensus_state: Arc::new(RwLock::new(blockchain_state)),
             validator_state: None,
+            health_state: Arc::new(RwLock::new(health_state)),
+
             automatic_reactivate,
 
             macro_producer: None,
@@ -448,6 +481,8 @@ where
                     next_block_number,
                     Self::compute_micro_block_producer_timeout(head, &blockchain),
                     Self::BLOCK_SEPARATION_TIME,
+                    self.validator_address.read().clone(),
+                    self.health_state.read().publish,
                 ));
             }
         }
@@ -737,6 +772,7 @@ where
             automatic_reactivate: Arc::clone(&self.automatic_reactivate),
             slot_band: Arc::clone(&self.slot_band),
             consensus_state: Arc::clone(&self.consensus_state),
+            validator_health: Arc::clone(&self.health_state),
         }
     }
 
@@ -833,13 +869,44 @@ where
         // Once the validator can be active is established, check the validator staking state.
         if self.is_synced() {
             let blockchain = self.blockchain.read();
+            let block_number = blockchain.block_number();
             match self.get_staking_state(&blockchain) {
                 ValidatorStakingState::Active => {
                     drop(blockchain);
                     if self.validator_state.is_some() {
                         self.validator_state = None;
                         info!("Automatically reactivated.");
                     }
+
+                    let validator_health = self.health_state.read().health;
+                    match validator_health {
+                        ValidatorHealth::Green => {}
+                        ValidatorHealth::Yellow(yellow_block_number) => {
+                            let blocks_diff = block_number - yellow_block_number;
+                            debug!(
+                                "Current validator health {} is yellow, blocks diff: {} ",
+                                self.validator_address.read(),
+                                blocks_diff
+                            );
+                            if blocks_diff >= Policy::blocks_per_epoch() / 4 {
+                                log::info!("Changing the validator health back to green");
+                                self.health_state.write().health = ValidatorHealth::Green;
+                            }
+                        }
+                        ValidatorHealth::Red(red_block_number) => {
+                            let blocks_diff = block_number - red_block_number;
+                            debug!(
+                                "Current validator health {} is red, blocks diff: {} ",
+                                self.validator_address.read(),
+                                blocks_diff
+                            );
+                            if blocks_diff >= Policy::blocks_per_epoch() / 4 {
+                                log::info!("Changing the validator health back to yellow");
+                                self.health_state.write().health =
+                                    ValidatorHealth::Yellow(block_number);
+                            }
+                        }
+                    }
                 }
                 ValidatorStakingState::Inactive(jailed_from) => {
                     if self.validator_state.is_none()
@@ -850,9 +917,36 @@ where
                             .unwrap_or(true)
                         && self.automatic_reactivate.load(Ordering::Acquire)
                     {
-                        let inactivity_state = self.reactivate(&blockchain);
-                        drop(blockchain);
-                        self.validator_state = Some(inactivity_state);
+                        let validator_health = self.health_state.read().health;
+                        match validator_health {
+                            ValidatorHealth::Green => {
+                                log::warn!(
+                                    "The validator {} was inactivated, changing its health to Yellow",
+                                    self.validator_address.read()
+                                );
+                                let inactivity_state = self.reactivate(&blockchain);
+                                drop(blockchain);
+                                self.validator_state = Some(inactivity_state);
+                                self.health_state.write().health =
+                                    ValidatorHealth::Yellow(block_number);
+                            }
+                            ValidatorHealth::Yellow(_) => {
+                                log::warn!(
+                                    "The validator {} was inactivated again, changing its health to Red",
+                                    self.validator_address.read()
+                                );
+                                let inactivity_state = self.reactivate(&blockchain);
+                                drop(blockchain);
+                                self.validator_state = Some(inactivity_state);
+                                self.health_state.write().health =
+                                    ValidatorHealth::Red(block_number);
+                            }
+                            ValidatorHealth::Red(_) => {
+                                log::warn!(
+                                    "The validator needs human intervention, no automatic reactivate"
+                                );
+                            }
+                        }
                     }
                 }
                 ValidatorStakingState::UnknownOrNoStake => {}