From b9a0c303ddf0e957909dfcbc1f986c85573cb85e Mon Sep 17 00:00:00 2001
From: viquezclaudio <viquezclaudio@gmail.com>
Date: Thu, 28 Nov 2024 10:14:56 -0600
Subject: [PATCH 1/3] Validator health status

---
 .../src/account/staking_contract/validator.rs |   6 +
 test-utils/src/validator.rs                   |   2 +-
 validator/src/micro.rs                        |  19 ++
 validator/src/validator.rs                    | 100 ++++++++-
 validator/tests/mock.rs                       | 209 +++++++++++++++++-
 5 files changed, 329 insertions(+), 7 deletions(-)

diff --git a/primitives/account/src/account/staking_contract/validator.rs b/primitives/account/src/account/staking_contract/validator.rs
index eab42ad87c..93ce458dca 100644
--- a/primitives/account/src/account/staking_contract/validator.rs
+++ b/primitives/account/src/account/staking_contract/validator.rs
@@ -46,6 +46,12 @@ use crate::{
 ///      in the first place.
 /// (**) The validator may be set to automatically reactivate itself upon inactivation.
 ///      If this setting is not enabled the state change can only be triggered manually.
+///      However, there is a validator health status with the following states:
+///      -> Green: Everything is working as expected, if the validator is deactivated its status changes to Yellow
+///      -> Yellow: If the validator is deactivated again, its status is changed to Red
+///      -> Red: If the validator is deactivated again, the automatic reactivate (if enabled) has no effect
+///              Human intervention is required at this point
+///      To go from Red to Yellow or Yellow to Green, the validator needs to be active for at least a quarter of an epoch
 ///
 /// Create, Update, Deactivate, Retire and Re-activate are incoming transactions to the staking contract.
 /// Delete is an outgoing transaction from the staking contract.
diff --git a/test-utils/src/validator.rs b/test-utils/src/validator.rs
index 7be7081fb0..f7969ee95f 100644
--- a/test-utils/src/validator.rs
+++ b/test-utils/src/validator.rs
@@ -113,7 +113,7 @@ where
         let (v, c) = build_validator(
             peer_ids[i],
             Address::from(&validator_keys[i]),
-            false,
+            true,
             signing_keys[i].clone(),
             voting_keys[i].clone(),
             fee_keys[i].clone(),
diff --git a/validator/src/micro.rs b/validator/src/micro.rs
index 4d3e60494c..61dce72177 100644
--- a/validator/src/micro.rs
+++ b/validator/src/micro.rs
@@ -10,6 +10,7 @@ use futures::{future::BoxFuture, ready, FutureExt, Stream};
 use nimiq_block::{Block, EquivocationProof, MicroBlock, SkipBlockInfo};
 use nimiq_blockchain::{BlockProducer, BlockProducerError, Blockchain};
 use nimiq_blockchain_interface::AbstractBlockchain;
+use nimiq_keys::Address;
 use nimiq_mempool::mempool::Mempool;
 use nimiq_primitives::policy::Policy;
 use nimiq_time::sleep;
@@ -36,6 +37,8 @@ struct NextProduceMicroBlockEvent<TValidatorNetwork> {
     block_number: u32,
     producer_timeout: Duration,
     block_separation_time: Duration,
+    validator_address: Address,
+    publish_block: bool,
 }
 
 impl<TValidatorNetwork: ValidatorNetwork + 'static> NextProduceMicroBlockEvent<TValidatorNetwork> {
@@ -53,6 +56,8 @@ impl<TValidatorNetwork: ValidatorNetwork + 'static> NextProduceMicroBlockEvent<T
         block_number: u32,
         producer_timeout: Duration,
         block_separation_time: Duration,
+        validator_address: Address,
+        publish_block: bool,
     ) -> Self {
         Self {
             blockchain,
@@ -65,6 +70,8 @@ impl<TValidatorNetwork: ValidatorNetwork + 'static> NextProduceMicroBlockEvent<T
             block_number,
             producer_timeout,
             block_separation_time,
+            validator_address,
+            publish_block,
         }
     }
 
@@ -117,6 +124,7 @@ impl<TValidatorNetwork: ValidatorNetwork + 'static> NextProduceMicroBlockEvent<T
             info!(
                 block_number = self.block_number,
                 slot_band = self.validator_slot_band,
+                address = %self.validator_address,
                 "Our turn, producing micro block #{}",
                 self.block_number,
             );
@@ -153,6 +161,12 @@ impl<TValidatorNetwork: ValidatorNetwork + 'static> NextProduceMicroBlockEvent<T
                 num_transactions
             );
 
+            if !self.publish_block {
+                log::warn!("Not publishing block {} ", block.block_number());
+                let event = ProduceMicroBlockEvent::MicroBlock;
+                break Some(Some(event));
+            }
+
             // Publish the block. It is valid as we have just created it.
             Validator::publish_block(Arc::clone(&self.network), block.clone());
 
@@ -194,6 +208,7 @@ impl<TValidatorNetwork: ValidatorNetwork + 'static> NextProduceMicroBlockEvent<T
         debug!(
             block_number = self.block_number,
             slot_band = self.validator_slot_band,
+            address = %self.validator_address,
             "Not our turn, waiting for micro block #{}",
             self.block_number,
         );
@@ -404,6 +419,8 @@ impl<TValidatorNetwork: ValidatorNetwork + 'static> ProduceMicroBlock<TValidator
         block_number: u32,
         producer_timeout: Duration,
         block_separation_time: Duration,
+        validator_address: Address,
+        publish_block: bool,
     ) -> Self {
         let next_event = NextProduceMicroBlockEvent::new(
             blockchain,
@@ -416,6 +433,8 @@ impl<TValidatorNetwork: ValidatorNetwork + 'static> ProduceMicroBlock<TValidator
             block_number,
             producer_timeout,
             block_separation_time,
+            validator_address,
+            publish_block,
         )
         .next()
         .boxed();
diff --git a/validator/src/validator.rs b/validator/src/validator.rs
index 4c007fd451..a5e39237d0 100644
--- a/validator/src/validator.rs
+++ b/validator/src/validator.rs
@@ -62,6 +62,28 @@ pub struct ConsensusState {
     equivocation_proofs: EquivocationProofPool,
 }
 
+/// Struct that represents the overall health of a validator
+/// Green means the Validator is working as expected,
+/// If the validator is deactivated, we change its health to Yellow
+/// If the validator is Yellow and it is deactivated again, we change its health to Red
+/// While in the Red state, the automatic reactivate has no effect and human intervention is required
+/// If the validator is Yellow and is not deactivated in a quarter of an epoch, we change its status to Green.
+/// If the validator is Red and is not deactivated in one epoch, we change its status to Yellow.
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub enum ValidatorHealth {
+    Green,
+    Yellow(u32),
+    Red(u32),
+}
+
+/// Struct that represents the overall Validator Health
+pub struct HealthState {
+    /// The current validator health
+    pub health: ValidatorHealth,
+    /// For testing/debug purposes control wether produced blocks are published by the validator
+    pub publish: bool,
+}
+
 /// Validator inactivity
 struct InactivityState {
     inactive_tx_hash: Blake2bHash,
@@ -76,6 +98,7 @@ pub struct ValidatorProxy {
     pub automatic_reactivate: Arc<AtomicBool>,
     pub slot_band: Arc<RwLock<Option<u16>>>,
     pub consensus_state: Arc<RwLock<ConsensusState>>,
+    pub validator_health: Arc<RwLock<HealthState>>,
 }
 
 impl Clone for ValidatorProxy {
@@ -88,6 +111,7 @@ impl Clone for ValidatorProxy {
             automatic_reactivate: Arc::clone(&self.automatic_reactivate),
             slot_band: Arc::clone(&self.slot_band),
             consensus_state: Arc::clone(&self.consensus_state),
+            validator_health: Arc::clone(&self.validator_health),
         }
     }
 }
@@ -119,6 +143,8 @@ where
     slot_band: Arc<RwLock<Option<u16>>>,
     consensus_state: Arc<RwLock<ConsensusState>>,
     validator_state: Option<InactivityState>,
+    health_state: Arc<RwLock<HealthState>>,
+
     automatic_reactivate: Arc<AtomicBool>,
 
     macro_producer: Option<ProduceMacroBlock<TValidatorNetwork>>,
@@ -200,6 +226,11 @@ where
                 .await
         });
 
+        let health_state = HealthState {
+            health: ValidatorHealth::Green,
+            publish: true,
+        };
+
         Self {
             consensus: consensus.proxy(),
             blockchain,
@@ -222,6 +253,8 @@ where
             slot_band: Arc::new(RwLock::new(None)),
             consensus_state: Arc::new(RwLock::new(blockchain_state)),
             validator_state: None,
+            health_state: Arc::new(RwLock::new(health_state)),
+
             automatic_reactivate,
 
             macro_producer: None,
@@ -448,6 +481,8 @@ where
                     next_block_number,
                     Self::compute_micro_block_producer_timeout(head, &blockchain),
                     Self::BLOCK_SEPARATION_TIME,
+                    self.validator_address.read().clone(),
+                    self.health_state.read().publish,
                 ));
             }
         }
@@ -737,6 +772,7 @@ where
             automatic_reactivate: Arc::clone(&self.automatic_reactivate),
             slot_band: Arc::clone(&self.slot_band),
             consensus_state: Arc::clone(&self.consensus_state),
+            validator_health: Arc::clone(&self.health_state),
         }
     }
 
@@ -833,6 +869,7 @@ where
         // Once the validator can be active is established, check the validator staking state.
         if self.is_synced() {
             let blockchain = self.blockchain.read();
+            let block_number = blockchain.block_number();
             match self.get_staking_state(&blockchain) {
                 ValidatorStakingState::Active => {
                     drop(blockchain);
@@ -840,6 +877,36 @@ where
                         self.validator_state = None;
                         info!("Automatically reactivated.");
                     }
+
+                    let validator_health = self.health_state.read().health;
+                    match validator_health {
+                        ValidatorHealth::Green => {}
+                        ValidatorHealth::Yellow(yellow_block_number) => {
+                            let blocks_diff = block_number - yellow_block_number;
+                            debug!(
+                                "Current validator health {} is yellow, blocks diff: {} ",
+                                self.validator_address.read(),
+                                blocks_diff
+                            );
+                            if blocks_diff >= Policy::blocks_per_epoch() / 4 {
+                                log::info!("Changing the validator health back to green");
+                                self.health_state.write().health = ValidatorHealth::Green;
+                            }
+                        }
+                        ValidatorHealth::Red(red_block_number) => {
+                            let blocks_diff = block_number - red_block_number;
+                            debug!(
+                                "Current validator health {} is red, blocks diff: {} ",
+                                self.validator_address.read(),
+                                blocks_diff
+                            );
+                            if blocks_diff >= Policy::blocks_per_epoch() / 4 {
+                                log::info!("Changing the validator health back to yellow");
+                                self.health_state.write().health =
+                                    ValidatorHealth::Yellow(block_number);
+                            }
+                        }
+                    }
                 }
                 ValidatorStakingState::Inactive(jailed_from) => {
                     if self.validator_state.is_none()
@@ -850,9 +917,36 @@ where
                             .unwrap_or(true)
                         && self.automatic_reactivate.load(Ordering::Acquire)
                     {
-                        let inactivity_state = self.reactivate(&blockchain);
-                        drop(blockchain);
-                        self.validator_state = Some(inactivity_state);
+                        let validator_health = self.health_state.read().health;
+                        match validator_health {
+                            ValidatorHealth::Green => {
+                                log::warn!(
+                                    "The validator {} was inactivated, changing its health to Yellow",
+                                    self.validator_address.read()
+                                );
+                                let inactivity_state = self.reactivate(&blockchain);
+                                drop(blockchain);
+                                self.validator_state = Some(inactivity_state);
+                                self.health_state.write().health =
+                                    ValidatorHealth::Yellow(block_number);
+                            }
+                            ValidatorHealth::Yellow(_) => {
+                                log::warn!(
+                                    "The validator {} was inactivated again, changing its health to Red",
+                                    self.validator_address.read()
+                                );
+                                let inactivity_state = self.reactivate(&blockchain);
+                                drop(blockchain);
+                                self.validator_state = Some(inactivity_state);
+                                self.health_state.write().health =
+                                    ValidatorHealth::Red(block_number);
+                            }
+                            ValidatorHealth::Red(_) => {
+                                log::warn!(
+                                    "The validator needs human intervention, no automatic reactivate"
+                                );
+                            }
+                        }
                     }
                 }
                 ValidatorStakingState::UnknownOrNoStake => {}
diff --git a/validator/tests/mock.rs b/validator/tests/mock.rs
index a16dfd1141..0acc98056c 100644
--- a/validator/tests/mock.rs
+++ b/validator/tests/mock.rs
@@ -15,7 +15,7 @@ use nimiq_network_interface::{
 };
 use nimiq_network_libp2p::Network;
 use nimiq_network_mock::{MockHub, MockNetwork};
-use nimiq_primitives::{networks::NetworkId, policy::Policy};
+use nimiq_primitives::{coin::Coin, networks::NetworkId, policy::Policy};
 use nimiq_test_log::test;
 use nimiq_test_utils::{
     test_network::TestNetwork,
@@ -24,9 +24,11 @@ use nimiq_test_utils::{
     },
 };
 use nimiq_time::{sleep, timeout};
+use nimiq_transaction_builder::TransactionBuilder;
 use nimiq_utils::spawn;
-use nimiq_validator::aggregation::{
-    skip_block::SignedSkipBlockMessage, update::SerializableLevelUpdate,
+use nimiq_validator::{
+    aggregation::{skip_block::SignedSkipBlockMessage, update::SerializableLevelUpdate},
+    validator::ValidatorHealth,
 };
 use serde::{Deserialize, Serialize};
 
@@ -176,6 +178,207 @@ async fn validators_can_do_skip_block() {
     assert!(block.block_number() > Policy::genesis_block_number());
 }
 
+#[test(tokio::test)]
+async fn validator_can_recover_from_yellow_health() {
+    let env =
+        MdbxDatabase::new_volatile(Default::default()).expect("Could not open a volatile database");
+
+    let validators =
+        build_validators::<Network>(env, &(5u64..=10u64).collect::<Vec<_>>(), &mut None, false)
+            .await;
+
+    // Listen for blockchain events from the new block producer (after a skip block).
+    let validator = validators.first().unwrap();
+    let validator_proxy = validator.proxy();
+
+    let validator = validators.last().unwrap();
+    let blockchain = Arc::clone(&validator.blockchain);
+    let events = blockchain.read().notifier_as_stream();
+
+    for validator in validators {
+        log::info!("Spawning validator: {}", validator.validator_address());
+        spawn(validator);
+    }
+
+    validator_proxy.validator_health.write().publish = false;
+
+    events.take(10).for_each(|_| future::ready(())).await;
+
+    let current_validator_health = validator_proxy.validator_health.read().health;
+
+    match current_validator_health {
+        ValidatorHealth::Yellow(block_number) => {
+            log::info!(
+                "Current validator health is yellow, as expected, inactivated block {}",
+                block_number
+            )
+        }
+        _ => panic!("Validator Health different than expected"),
+    };
+
+    // The validator should no longer be skip blocked:
+    validator_proxy.validator_health.write().publish = true;
+
+    let events = blockchain.read().notifier_as_stream();
+    events.take(30).for_each(|_| future::ready(())).await;
+
+    assert_eq!(
+        validator_proxy.validator_health.read().health,
+        ValidatorHealth::Green
+    );
+}
+
+#[test(tokio::test)]
+async fn validator_health_to_red() {
+    let env =
+        MdbxDatabase::new_volatile(Default::default()).expect("Could not open a volatile database");
+
+    let validators =
+        build_validators::<Network>(env, &(5u64..=10u64).collect::<Vec<_>>(), &mut None, false)
+            .await;
+
+    // Listen for blockchain events from the new block producer (after a skip block).
+    let validator = validators.first().unwrap();
+    let validator_proxy = validator.proxy();
+
+    let validator = validators.last().unwrap();
+    let blockchain = Arc::clone(&validator.blockchain);
+    let events = blockchain.read().notifier_as_stream();
+
+    for validator in validators {
+        log::info!("Spawning validator: {}", validator.validator_address());
+        spawn(validator);
+    }
+
+    validator_proxy.validator_health.write().publish = false;
+
+    events.take(10).for_each(|_| future::ready(())).await;
+
+    let current_validator_health = validator_proxy.validator_health.read().health;
+
+    match current_validator_health {
+        ValidatorHealth::Yellow(block_number) => {
+            log::info!(
+                "Current validator health is yellow, as expected, inactivated block {}",
+                block_number
+            )
+        }
+        _ => panic!("Validator Health different than expected"),
+    };
+
+    let events = blockchain.read().notifier_as_stream();
+
+    // Now we produce more blocks, and the validator should be inactivated again
+    events.take(20).for_each(|_| future::ready(())).await;
+
+    let current_validator_health = validator_proxy.validator_health.read().health;
+
+    match current_validator_health {
+        ValidatorHealth::Red(block_number) => {
+            log::info!(
+                "Current validator health is red, as expected, inactivated block {}",
+                block_number
+            )
+        }
+        _ => panic!("Validator Health different than expected"),
+    };
+}
+
+#[test(tokio::test)]
+async fn validator_health_fully_recover() {
+    let env =
+        MdbxDatabase::new_volatile(Default::default()).expect("Could not open a volatile database");
+
+    let validators =
+        build_validators::<Network>(env, &(5u64..=10u64).collect::<Vec<_>>(), &mut None, false)
+            .await;
+
+    // Listen for blockchain events from the new block producer (after a skip block).
+    let validator = validators.first().unwrap();
+    let consensus = validator.consensus.clone();
+    let validator_proxy = validator.proxy();
+    let validator_address = validator.validator_address();
+
+    log::info!(
+        "Listening to blockchain events from validator {} ",
+        validator_address,
+    );
+
+    let validator = validators.last().unwrap();
+    let blockchain = Arc::clone(&validator.blockchain);
+
+    let events = blockchain.read().notifier_as_stream();
+
+    for validator in validators {
+        log::info!("Spawning validator: {}", validator.validator_address());
+        spawn(validator);
+    }
+
+    validator_proxy.validator_health.write().publish = false;
+
+    events.take(10).for_each(|_| future::ready(())).await;
+
+    let current_validator_health = validator_proxy.validator_health.read().health;
+
+    match current_validator_health {
+        ValidatorHealth::Yellow(block_number) => {
+            log::info!(
+                "Current validator health is yellow, as expected, inactivated block {}",
+                block_number
+            )
+        }
+        _ => panic!("Validator Health different than expected"),
+    };
+
+    let events = blockchain.read().notifier_as_stream();
+
+    // Now we produce more blocks, and the validator should be inactivated again
+    events.take(20).for_each(|_| future::ready(())).await;
+
+    let current_validator_health = validator_proxy.validator_health.read().health;
+
+    match current_validator_health {
+        ValidatorHealth::Red(block_number) => {
+            log::info!(
+                "Current validator health is red, as expected, inactivated block {}",
+                block_number
+            )
+        }
+        _ => panic!("Validator Health different than expected"),
+    };
+
+    // Since the validator needs manual intervention, we are going to send the reactivate transaction
+
+    let reactivate_transaction = TransactionBuilder::new_reactivate_validator(
+        &validator_proxy.fee_key.read(),
+        validator_address,
+        &validator_proxy.signing_key.read(),
+        Coin::ZERO,
+        Policy::genesis_block_number(),
+        NetworkId::UnitAlbatross,
+    );
+
+    spawn(async move {
+        log::info!("Sending reactivate transaction to the network");
+        if consensus
+            .send_transaction(reactivate_transaction.clone())
+            .await
+            .is_err()
+        {
+            log::error!("Failed to send reactivate transaction");
+        }
+    });
+
+    validator_proxy.validator_health.write().publish = true;
+
+    let events = blockchain.read().notifier_as_stream();
+    events.take(70).for_each(|_| future::ready(())).await;
+
+    let current_validator_health = validator_proxy.validator_health.read().health;
+
+    assert_eq!(current_validator_health, ValidatorHealth::Green);
+}
+
 fn create_skip_block_update(
     skip_block_info: SkipBlockInfo,
     key_pair: BlsKeyPair,

From f6c979e514d6b54372198fe4489beaf4aab4ac74 Mon Sep 17 00:00:00 2001
From: viquezclaudio <viquezclaudio@gmail.com>
Date: Mon, 23 Dec 2024 14:19:24 -0600
Subject: [PATCH 2/3] Track blocks that are produced by validator

Track the number of blocks that are succesfully produced when changing the validator health
---
 validator/src/micro.rs     | 21 ++++++++++++--------
 validator/src/validator.rs | 39 +++++++++++++++++++++++---------------
 validator/tests/mock.rs    |  4 ++--
 3 files changed, 39 insertions(+), 25 deletions(-)

diff --git a/validator/src/micro.rs b/validator/src/micro.rs
index 61dce72177..e81f55aebf 100644
--- a/validator/src/micro.rs
+++ b/validator/src/micro.rs
@@ -19,7 +19,10 @@ use nimiq_validator_network::ValidatorNetwork;
 use nimiq_vrf::VrfSeed;
 use parking_lot::RwLock;
 
-use crate::{aggregation::skip_block::SkipBlockAggregation, validator::Validator};
+use crate::{
+    aggregation::skip_block::SkipBlockAggregation,
+    validator::{HealthState, Validator},
+};
 
 pub(crate) enum ProduceMicroBlockEvent {
     MicroBlock,
@@ -38,7 +41,7 @@ struct NextProduceMicroBlockEvent<TValidatorNetwork> {
     producer_timeout: Duration,
     block_separation_time: Duration,
     validator_address: Address,
-    publish_block: bool,
+    health_state: Arc<RwLock<HealthState>>,
 }
 
 impl<TValidatorNetwork: ValidatorNetwork + 'static> NextProduceMicroBlockEvent<TValidatorNetwork> {
@@ -57,7 +60,7 @@ impl<TValidatorNetwork: ValidatorNetwork + 'static> NextProduceMicroBlockEvent<T
         producer_timeout: Duration,
         block_separation_time: Duration,
         validator_address: Address,
-        publish_block: bool,
+        health_state: Arc<RwLock<HealthState>>,
     ) -> Self {
         Self {
             blockchain,
@@ -71,7 +74,7 @@ impl<TValidatorNetwork: ValidatorNetwork + 'static> NextProduceMicroBlockEvent<T
             producer_timeout,
             block_separation_time,
             validator_address,
-            publish_block,
+            health_state,
         }
     }
 
@@ -161,8 +164,8 @@ impl<TValidatorNetwork: ValidatorNetwork + 'static> NextProduceMicroBlockEvent<T
                 num_transactions
             );
 
-            if !self.publish_block {
-                log::warn!("Not publishing block {} ", block.block_number());
+            if !self.health_state.read().publish {
+                log::warn!(block = block.block_number(), "Not publishing block");
                 let event = ProduceMicroBlockEvent::MicroBlock;
                 break Some(Some(event));
             }
@@ -195,6 +198,8 @@ impl<TValidatorNetwork: ValidatorNetwork + 'static> NextProduceMicroBlockEvent<T
                 continue;
             }
 
+            self.health_state.write().blk_cnt += 1;
+
             let event = result
                 .map(move |_result| ProduceMicroBlockEvent::MicroBlock)
                 .ok();
@@ -420,7 +425,7 @@ impl<TValidatorNetwork: ValidatorNetwork + 'static> ProduceMicroBlock<TValidator
         producer_timeout: Duration,
         block_separation_time: Duration,
         validator_address: Address,
-        publish_block: bool,
+        health_state: Arc<RwLock<HealthState>>,
     ) -> Self {
         let next_event = NextProduceMicroBlockEvent::new(
             blockchain,
@@ -434,7 +439,7 @@ impl<TValidatorNetwork: ValidatorNetwork + 'static> ProduceMicroBlock<TValidator
             producer_timeout,
             block_separation_time,
             validator_address,
-            publish_block,
+            health_state,
         )
         .next()
         .boxed();
diff --git a/validator/src/validator.rs b/validator/src/validator.rs
index a5e39237d0..dfb217e639 100644
--- a/validator/src/validator.rs
+++ b/validator/src/validator.rs
@@ -51,6 +51,9 @@ use crate::{
     r#macro::{MappedReturn, ProduceMacroBlock, ProposalTopic},
 };
 
+/// The number of blocks a validator needs to produce in time to improve its health
+const VALIDATOR_HEALTH_THRESHOLD: u32 = 5;
+
 #[derive(PartialEq)]
 enum ValidatorStakingState {
     Active,
@@ -80,6 +83,8 @@ pub enum ValidatorHealth {
 pub struct HealthState {
     /// The current validator health
     pub health: ValidatorHealth,
+    /// Number of blocks that we have produced in time(without being inactivated)
+    pub blk_cnt: u32,
     /// For testing/debug purposes control wether produced blocks are published by the validator
     pub publish: bool,
 }
@@ -229,6 +234,7 @@ where
         let health_state = HealthState {
             health: ValidatorHealth::Green,
             publish: true,
+            blk_cnt: 0,
         };
 
         Self {
@@ -482,7 +488,7 @@ where
                     Self::compute_micro_block_producer_timeout(head, &blockchain),
                     Self::BLOCK_SEPARATION_TIME,
                     self.validator_address.read().clone(),
-                    self.health_state.read().publish,
+                    Arc::clone(&self.health_state),
                 ));
             }
         }
@@ -882,28 +888,29 @@ where
                     match validator_health {
                         ValidatorHealth::Green => {}
                         ValidatorHealth::Yellow(yellow_block_number) => {
-                            let blocks_diff = block_number - yellow_block_number;
                             debug!(
-                                "Current validator health {} is yellow, blocks diff: {} ",
-                                self.validator_address.read(),
-                                blocks_diff
+                                address = %self.validator_address.read(),
+                                inactivated = yellow_block_number,
+                                good_blocks = %self.health_state.read().blk_cnt,
+                                "Current validator health is yellow",
                             );
-                            if blocks_diff >= Policy::blocks_per_epoch() / 4 {
+                            if self.health_state.read().blk_cnt >= VALIDATOR_HEALTH_THRESHOLD {
                                 log::info!("Changing the validator health back to green");
                                 self.health_state.write().health = ValidatorHealth::Green;
+                                self.health_state.write().blk_cnt = 0;
                             }
                         }
                         ValidatorHealth::Red(red_block_number) => {
-                            let blocks_diff = block_number - red_block_number;
                             debug!(
-                                "Current validator health {} is red, blocks diff: {} ",
-                                self.validator_address.read(),
-                                blocks_diff
+                                address = %self.validator_address.read(),
+                                inactivated = red_block_number,
+                                "Current validator health is red",
                             );
-                            if blocks_diff >= Policy::blocks_per_epoch() / 4 {
+                            if self.health_state.read().blk_cnt >= VALIDATOR_HEALTH_THRESHOLD {
                                 log::info!("Changing the validator health back to yellow");
                                 self.health_state.write().health =
                                     ValidatorHealth::Yellow(block_number);
+                                self.health_state.write().blk_cnt = 0;
                             }
                         }
                     }
@@ -921,25 +928,27 @@ where
                         match validator_health {
                             ValidatorHealth::Green => {
                                 log::warn!(
-                                    "The validator {} was inactivated, changing its health to Yellow",
-                                    self.validator_address.read()
+                                    address=%self.validator_address.read(),
+                                    "The validator was inactivated, changing its health to Yellow",
                                 );
                                 let inactivity_state = self.reactivate(&blockchain);
                                 drop(blockchain);
                                 self.validator_state = Some(inactivity_state);
                                 self.health_state.write().health =
                                     ValidatorHealth::Yellow(block_number);
+                                self.health_state.write().blk_cnt = 0;
                             }
                             ValidatorHealth::Yellow(_) => {
                                 log::warn!(
-                                    "The validator {} was inactivated again, changing its health to Red",
-                                    self.validator_address.read()
+                                    address=%self.validator_address.read(),
+                                    "The validator was inactivated again, changing its health to Red",
                                 );
                                 let inactivity_state = self.reactivate(&blockchain);
                                 drop(blockchain);
                                 self.validator_state = Some(inactivity_state);
                                 self.health_state.write().health =
                                     ValidatorHealth::Red(block_number);
+                                self.health_state.write().blk_cnt = 0;
                             }
                             ValidatorHealth::Red(_) => {
                                 log::warn!(
diff --git a/validator/tests/mock.rs b/validator/tests/mock.rs
index 0acc98056c..100229f9fd 100644
--- a/validator/tests/mock.rs
+++ b/validator/tests/mock.rs
@@ -220,7 +220,7 @@ async fn validator_can_recover_from_yellow_health() {
     validator_proxy.validator_health.write().publish = true;
 
     let events = blockchain.read().notifier_as_stream();
-    events.take(30).for_each(|_| future::ready(())).await;
+    events.take(40).for_each(|_| future::ready(())).await;
 
     assert_eq!(
         validator_proxy.validator_health.read().health,
@@ -372,7 +372,7 @@ async fn validator_health_fully_recover() {
     validator_proxy.validator_health.write().publish = true;
 
     let events = blockchain.read().notifier_as_stream();
-    events.take(70).for_each(|_| future::ready(())).await;
+    events.take(100).for_each(|_| future::ready(())).await;
 
     let current_validator_health = validator_proxy.validator_health.read().health;
 

From cccd5e5b50ae9c4e12fbb2a9eaead29ca359112a Mon Sep 17 00:00:00 2001
From: viquezclaudio <viquezclaudio@gmail.com>
Date: Mon, 13 Jan 2025 15:12:50 -0600
Subject: [PATCH 3/3] Use a cuadratic backup curve for reactivate transactions

When a validator is deactivated, the reactivate transaction will have a delay
based on the number of consecutive deactivations the validator has experienced
in the current epoch.
The amount of delayed blocks is of quadratic nature
---
 test-utils/src/validator.rs    |   3 +-
 validator/src/micro.rs         |   7 +-
 validator/src/validator.rs     | 156 ++++++++++++++++++++-------------
 validator/tests/integration.rs |  11 ++-
 validator/tests/mock.rs        | 117 +++++++++++--------------
 5 files changed, 162 insertions(+), 132 deletions(-)

diff --git a/test-utils/src/validator.rs b/test-utils/src/validator.rs
index f7969ee95f..3fe39dde7b 100644
--- a/test-utils/src/validator.rs
+++ b/test-utils/src/validator.rs
@@ -66,6 +66,7 @@ pub async fn build_validators<N: TestNetwork + NetworkInterface>(
     peer_ids: &[u64],
     hub: &mut Option<MockHub>,
     is_prover_active: bool,
+    automatic_reactivate: bool,
 ) -> Vec<Validator<ValidatorNetworkImpl<N>>>
 where
     N::Error: Send + Sync,
@@ -113,7 +114,7 @@ where
         let (v, c) = build_validator(
             peer_ids[i],
             Address::from(&validator_keys[i]),
-            true,
+            automatic_reactivate,
             signing_keys[i].clone(),
             voting_keys[i].clone(),
             fee_keys[i].clone(),
diff --git a/validator/src/micro.rs b/validator/src/micro.rs
index e81f55aebf..ea764a82fd 100644
--- a/validator/src/micro.rs
+++ b/validator/src/micro.rs
@@ -166,8 +166,7 @@ impl<TValidatorNetwork: ValidatorNetwork + 'static> NextProduceMicroBlockEvent<T
 
             if !self.health_state.read().publish {
                 log::warn!(block = block.block_number(), "Not publishing block");
-                let event = ProduceMicroBlockEvent::MicroBlock;
-                break Some(Some(event));
+                break Some(Some(ProduceMicroBlockEvent::MicroBlock));
             }
 
             // Publish the block. It is valid as we have just created it.
@@ -198,7 +197,9 @@ impl<TValidatorNetwork: ValidatorNetwork + 'static> NextProduceMicroBlockEvent<T
                 continue;
             }
 
-            self.health_state.write().blk_cnt += 1;
+            // Each successfull block will decrease the number of inactivations
+            let current_inactivations = self.health_state.read().inactivations;
+            self.health_state.write().inactivations = current_inactivations.saturating_sub(1);
 
             let event = result
                 .map(move |_result| ProduceMicroBlockEvent::MicroBlock)
diff --git a/validator/src/validator.rs b/validator/src/validator.rs
index dfb217e639..43d137c4e6 100644
--- a/validator/src/validator.rs
+++ b/validator/src/validator.rs
@@ -1,4 +1,5 @@
 use std::{
+    cmp,
     error::Error,
     future::Future,
     pin::Pin,
@@ -51,8 +52,12 @@ use crate::{
     r#macro::{MappedReturn, ProduceMacroBlock, ProposalTopic},
 };
 
-/// The number of blocks a validator needs to produce in time to improve its health
-const VALIDATOR_HEALTH_THRESHOLD: u32 = 5;
+/// The number of consecutive inactivations from which a validator is considered with yellow health
+const VALIDATOR_YELLOW_HEALTH_INACTIVATIONS: u32 = 2;
+/// The number of consecutive inactivations from which a validator is considered with yellow health
+const VALIDATOR_RED_HEALTH_INACTIVATIONS: u32 = 4;
+// The maximum number of blocks the reactivate transaction can be delayed
+const MAX_REACTIVATE_DELAY: u32 = 10_000;
 
 #[derive(PartialEq)]
 enum ValidatorStakingState {
@@ -75,18 +80,22 @@ pub struct ConsensusState {
 #[derive(Clone, Copy, Debug, PartialEq)]
 pub enum ValidatorHealth {
     Green,
-    Yellow(u32),
-    Red(u32),
+    Yellow,
+    Red,
 }
 
 /// Struct that represents the overall Validator Health
 pub struct HealthState {
     /// The current validator health
     pub health: ValidatorHealth,
-    /// Number of blocks that we have produced in time(without being inactivated)
-    pub blk_cnt: u32,
-    /// For testing/debug purposes control wether produced blocks are published by the validator
+    /// For testing/debug purposes control whether produced blocks are published by the validator
     pub publish: bool,
+    /// Number of inactivations that have ocurred in the current epoch
+    pub inactivations: u32,
+    /// Next block number where the re-activate txns should be sent
+    pub reactivate_bn: u32,
+    /// Flag that indicates an ongoing reactivation
+    pub pending_reactivate: bool,
 }
 
 /// Validator inactivity
@@ -234,7 +243,9 @@ where
         let health_state = HealthState {
             health: ValidatorHealth::Green,
             publish: true,
-            blk_cnt: 0,
+            inactivations: 0,
+            reactivate_bn: 0,
+            pending_reactivate: false,
         };
 
         Self {
@@ -508,6 +519,10 @@ where
                 self.on_blockchain_extended(hash);
             }
             BlockchainEvent::EpochFinalized(ref hash) => {
+                // Reset the inactivations counter
+                self.health_state.write().inactivations = 0;
+                // Reset the validator health every epoch
+                self.health_state.write().health = ValidatorHealth::Green;
                 self.init_epoch();
                 // The on_blockchain_extended is necessary for the order of events to not matter.
                 self.on_blockchain_extended(hash);
@@ -537,6 +552,14 @@ where
 
         self.check_reactivate(block.block_number());
         self.init_block_producer(Some(hash));
+
+        let block_number = block.block_number();
+        let blockchain = self.blockchain.read();
+
+        if block_number == self.health_state.read().reactivate_bn {
+            let inactivity_state = self.reactivate(&blockchain);
+            self.validator_state = Some(inactivity_state);
+        }
     }
 
     fn on_blockchain_rebranched(
@@ -718,6 +741,11 @@ where
         )
     }
 
+    // Computes the next block number where we should send the next reactivate transaction
+    fn get_reactivate_delay(&self, inactivations: u32) -> u32 {
+        cmp::min(inactivations.pow(2), MAX_REACTIVATE_DELAY)
+    }
+
     fn reactivate(&self, blockchain: &Blockchain) -> InactivityState {
         let validity_start_height = blockchain.block_number();
 
@@ -733,7 +761,7 @@ where
 
         let cn = self.consensus.clone();
         spawn(async move {
-            debug!("Sending reactivate transaction to the network");
+            info!("Sending reactivate transaction to the network");
             if cn
                 .send_transaction(reactivate_transaction.clone())
                 .await
@@ -878,39 +906,34 @@ where
             let block_number = blockchain.block_number();
             match self.get_staking_state(&blockchain) {
                 ValidatorStakingState::Active => {
-                    drop(blockchain);
                     if self.validator_state.is_some() {
+                        drop(blockchain);
                         self.validator_state = None;
+                        self.health_state.write().pending_reactivate = false;
                         info!("Automatically reactivated.");
                     }
+                    let inactivations = self.health_state.read().inactivations;
+
+                    log::warn!(
+                        address=%self.validator_address.read(),
+                        inactivations,
+                        "Inactivations counter",
+                    );
 
                     let validator_health = self.health_state.read().health;
+
                     match validator_health {
                         ValidatorHealth::Green => {}
-                        ValidatorHealth::Yellow(yellow_block_number) => {
-                            debug!(
-                                address = %self.validator_address.read(),
-                                inactivated = yellow_block_number,
-                                good_blocks = %self.health_state.read().blk_cnt,
-                                "Current validator health is yellow",
-                            );
-                            if self.health_state.read().blk_cnt >= VALIDATOR_HEALTH_THRESHOLD {
-                                log::info!("Changing the validator health back to green");
+                        ValidatorHealth::Yellow => {
+                            if inactivations < VALIDATOR_YELLOW_HEALTH_INACTIVATIONS {
+                                log::info!(inactivations, "Changed validator health to green");
                                 self.health_state.write().health = ValidatorHealth::Green;
-                                self.health_state.write().blk_cnt = 0;
                             }
                         }
-                        ValidatorHealth::Red(red_block_number) => {
-                            debug!(
-                                address = %self.validator_address.read(),
-                                inactivated = red_block_number,
-                                "Current validator health is red",
-                            );
-                            if self.health_state.read().blk_cnt >= VALIDATOR_HEALTH_THRESHOLD {
-                                log::info!("Changing the validator health back to yellow");
-                                self.health_state.write().health =
-                                    ValidatorHealth::Yellow(block_number);
-                                self.health_state.write().blk_cnt = 0;
+                        ValidatorHealth::Red => {
+                            if inactivations < VALIDATOR_RED_HEALTH_INACTIVATIONS {
+                                log::info!(inactivations, "Changed validator health to yellow");
+                                self.health_state.write().health = ValidatorHealth::Yellow;
                             }
                         }
                     }
@@ -924,37 +947,48 @@ where
                             .unwrap_or(true)
                         && self.automatic_reactivate.load(Ordering::Acquire)
                     {
-                        let validator_health = self.health_state.read().health;
-                        match validator_health {
-                            ValidatorHealth::Green => {
-                                log::warn!(
-                                    address=%self.validator_address.read(),
-                                    "The validator was inactivated, changing its health to Yellow",
-                                );
-                                let inactivity_state = self.reactivate(&blockchain);
-                                drop(blockchain);
-                                self.validator_state = Some(inactivity_state);
-                                self.health_state.write().health =
-                                    ValidatorHealth::Yellow(block_number);
-                                self.health_state.write().blk_cnt = 0;
-                            }
-                            ValidatorHealth::Yellow(_) => {
-                                log::warn!(
-                                    address=%self.validator_address.read(),
-                                    "The validator was inactivated again, changing its health to Red",
-                                );
-                                let inactivity_state = self.reactivate(&blockchain);
-                                drop(blockchain);
-                                self.validator_state = Some(inactivity_state);
-                                self.health_state.write().health =
-                                    ValidatorHealth::Red(block_number);
-                                self.health_state.write().blk_cnt = 0;
-                            }
-                            ValidatorHealth::Red(_) => {
-                                log::warn!(
-                                    "The validator needs human intervention, no automatic reactivate"
-                                );
+                        // Keep track of how many times we have been deactivated in the current epoch.
+                        if !self.health_state.read().pending_reactivate {
+                            let mut health_state = self.health_state.write();
+                            health_state.inactivations += 1;
+                            health_state.reactivate_bn = block_number
+                                + self.get_reactivate_delay(health_state.inactivations);
+                            health_state.pending_reactivate = true;
+                            drop(health_state);
+
+                            let inactivations = self.health_state.read().inactivations;
+
+                            let validator_health = self.health_state.read().health;
+                            match validator_health {
+                                ValidatorHealth::Green => {
+                                    if inactivations >= VALIDATOR_YELLOW_HEALTH_INACTIVATIONS {
+                                        log::warn!(
+                                            inactivations,
+                                            "Changed validator health to yellow"
+                                        );
+                                        self.health_state.write().health = ValidatorHealth::Yellow;
+                                    }
+                                }
+                                ValidatorHealth::Yellow => {
+                                    if inactivations >= VALIDATOR_RED_HEALTH_INACTIVATIONS {
+                                        log::warn!(
+                                            inactivations,
+                                            "Changed validator health to red"
+                                        );
+                                        self.health_state.write().health = ValidatorHealth::Red;
+                                    }
+                                }
+                                ValidatorHealth::Red => {
+                                    log::warn!("Validator health is still red")
+                                }
                             }
+
+                            log::warn!(
+                                "Current inactivations counter: {}, next reactivate bn {}, current bn {} ",
+                                inactivations,
+                                self.health_state.read().reactivate_bn,
+                                block_number
+                            );
                         }
                     }
                 }
diff --git a/validator/tests/integration.rs b/validator/tests/integration.rs
index 371127bbdd..d6f83efb5a 100644
--- a/validator/tests/integration.rs
+++ b/validator/tests/integration.rs
@@ -86,9 +86,14 @@ async fn four_validators_can_create_an_epoch() {
     let env =
         MdbxDatabase::new_volatile(Default::default()).expect("Could not open a volatile database");
 
-    let validators =
-        build_validators::<Network>(env, &(1u64..=4u64).collect::<Vec<_>>(), &mut None, false)
-            .await;
+    let validators = build_validators::<Network>(
+        env,
+        &(1u64..=4u64).collect::<Vec<_>>(),
+        &mut None,
+        false,
+        false,
+    )
+    .await;
 
     let blockchain = Arc::clone(&validators.first().unwrap().blockchain);
 
diff --git a/validator/tests/mock.rs b/validator/tests/mock.rs
index 100229f9fd..65c13866b0 100644
--- a/validator/tests/mock.rs
+++ b/validator/tests/mock.rs
@@ -107,6 +107,7 @@ async fn four_validators_can_create_micro_blocks() {
         &(1u64..=4u64).collect::<Vec<_>>(),
         &mut Some(hub),
         false,
+        false,
     )
     .await;
 
@@ -145,9 +146,14 @@ async fn validators_can_do_skip_block() {
     let env =
         MdbxDatabase::new_volatile(Default::default()).expect("Could not open a volatile database");
 
-    let mut validators =
-        build_validators::<Network>(env, &(5u64..=10u64).collect::<Vec<_>>(), &mut None, false)
-            .await;
+    let mut validators = build_validators::<Network>(
+        env,
+        &(5u64..=10u64).collect::<Vec<_>>(),
+        &mut None,
+        false,
+        false,
+    )
+    .await;
 
     // Disconnect the next block producer.
     let _validator = pop_validator_for_slot(
@@ -183,9 +189,14 @@ async fn validator_can_recover_from_yellow_health() {
     let env =
         MdbxDatabase::new_volatile(Default::default()).expect("Could not open a volatile database");
 
-    let validators =
-        build_validators::<Network>(env, &(5u64..=10u64).collect::<Vec<_>>(), &mut None, false)
-            .await;
+    let validators = build_validators::<Network>(
+        env,
+        &(5u64..=10u64).collect::<Vec<_>>(),
+        &mut None,
+        false,
+        true,
+    )
+    .await;
 
     // Listen for blockchain events from the new block producer (after a skip block).
     let validator = validators.first().unwrap();
@@ -202,16 +213,18 @@ async fn validator_can_recover_from_yellow_health() {
 
     validator_proxy.validator_health.write().publish = false;
 
-    events.take(10).for_each(|_| future::ready(())).await;
+    log::info!(
+        "Validator proxy address {}",
+        validator_proxy.validator_address.read()
+    );
+
+    events.take(30).for_each(|_| future::ready(())).await;
 
     let current_validator_health = validator_proxy.validator_health.read().health;
 
     match current_validator_health {
-        ValidatorHealth::Yellow(block_number) => {
-            log::info!(
-                "Current validator health is yellow, as expected, inactivated block {}",
-                block_number
-            )
+        ValidatorHealth::Yellow => {
+            log::info!("Current validator health is yellow, as expecteds",)
         }
         _ => panic!("Validator Health different than expected"),
     };
@@ -220,7 +233,7 @@ async fn validator_can_recover_from_yellow_health() {
     validator_proxy.validator_health.write().publish = true;
 
     let events = blockchain.read().notifier_as_stream();
-    events.take(40).for_each(|_| future::ready(())).await;
+    events.take(30).for_each(|_| future::ready(())).await;
 
     assert_eq!(
         validator_proxy.validator_health.read().health,
@@ -233,9 +246,14 @@ async fn validator_health_to_red() {
     let env =
         MdbxDatabase::new_volatile(Default::default()).expect("Could not open a volatile database");
 
-    let validators =
-        build_validators::<Network>(env, &(5u64..=10u64).collect::<Vec<_>>(), &mut None, false)
-            .await;
+    let validators = build_validators::<Network>(
+        env,
+        &(5u64..=10u64).collect::<Vec<_>>(),
+        &mut None,
+        false,
+        true,
+    )
+    .await;
 
     // Listen for blockchain events from the new block producer (after a skip block).
     let validator = validators.first().unwrap();
@@ -252,16 +270,13 @@ async fn validator_health_to_red() {
 
     validator_proxy.validator_health.write().publish = false;
 
-    events.take(10).for_each(|_| future::ready(())).await;
+    events.take(30).for_each(|_| future::ready(())).await;
 
     let current_validator_health = validator_proxy.validator_health.read().health;
 
     match current_validator_health {
-        ValidatorHealth::Yellow(block_number) => {
-            log::info!(
-                "Current validator health is yellow, as expected, inactivated block {}",
-                block_number
-            )
+        ValidatorHealth::Yellow => {
+            log::info!("Current validator health is yellow, as expected",)
         }
         _ => panic!("Validator Health different than expected"),
     };
@@ -274,11 +289,8 @@ async fn validator_health_to_red() {
     let current_validator_health = validator_proxy.validator_health.read().health;
 
     match current_validator_health {
-        ValidatorHealth::Red(block_number) => {
-            log::info!(
-                "Current validator health is red, as expected, inactivated block {}",
-                block_number
-            )
+        ValidatorHealth::Red => {
+            log::info!("Current validator health is red, as expected",)
         }
         _ => panic!("Validator Health different than expected"),
     };
@@ -289,13 +301,17 @@ async fn validator_health_fully_recover() {
     let env =
         MdbxDatabase::new_volatile(Default::default()).expect("Could not open a volatile database");
 
-    let validators =
-        build_validators::<Network>(env, &(5u64..=10u64).collect::<Vec<_>>(), &mut None, false)
-            .await;
+    let validators = build_validators::<Network>(
+        env,
+        &(5u64..=10u64).collect::<Vec<_>>(),
+        &mut None,
+        false,
+        true,
+    )
+    .await;
 
     // Listen for blockchain events from the new block producer (after a skip block).
     let validator = validators.first().unwrap();
-    let consensus = validator.consensus.clone();
     let validator_proxy = validator.proxy();
     let validator_address = validator.validator_address();
 
@@ -316,16 +332,13 @@ async fn validator_health_fully_recover() {
 
     validator_proxy.validator_health.write().publish = false;
 
-    events.take(10).for_each(|_| future::ready(())).await;
+    events.take(30).for_each(|_| future::ready(())).await;
 
     let current_validator_health = validator_proxy.validator_health.read().health;
 
     match current_validator_health {
-        ValidatorHealth::Yellow(block_number) => {
-            log::info!(
-                "Current validator health is yellow, as expected, inactivated block {}",
-                block_number
-            )
+        ValidatorHealth::Yellow => {
+            log::info!("Current validator health is yellow, as expected",)
         }
         _ => panic!("Validator Health different than expected"),
     };
@@ -338,37 +351,12 @@ async fn validator_health_fully_recover() {
     let current_validator_health = validator_proxy.validator_health.read().health;
 
     match current_validator_health {
-        ValidatorHealth::Red(block_number) => {
-            log::info!(
-                "Current validator health is red, as expected, inactivated block {}",
-                block_number
-            )
+        ValidatorHealth::Red => {
+            log::info!("Current validator health is red, as expected")
         }
         _ => panic!("Validator Health different than expected"),
     };
 
-    // Since the validator needs manual intervention, we are going to send the reactivate transaction
-
-    let reactivate_transaction = TransactionBuilder::new_reactivate_validator(
-        &validator_proxy.fee_key.read(),
-        validator_address,
-        &validator_proxy.signing_key.read(),
-        Coin::ZERO,
-        Policy::genesis_block_number(),
-        NetworkId::UnitAlbatross,
-    );
-
-    spawn(async move {
-        log::info!("Sending reactivate transaction to the network");
-        if consensus
-            .send_transaction(reactivate_transaction.clone())
-            .await
-            .is_err()
-        {
-            log::error!("Failed to send reactivate transaction");
-        }
-    });
-
     validator_proxy.validator_health.write().publish = true;
 
     let events = blockchain.read().notifier_as_stream();
@@ -430,6 +418,7 @@ async fn validator_can_catch_up() {
         &(9u64..=16u64).collect::<Vec<_>>(),
         &mut Some(hub),
         false,
+        false,
     )
     .await;
     // Maintain a collection of the corresponding networks.