Merge branch 'main' into 1541-escape-hatch-purge-elements-from-stateu…

…pdates-and-hotshotcommitments-arrays-once-it-has-over-10-days-of-worth-of-data

Cargo.toml

@@ -107,7 +107,6 @@ jf-relation = { git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4
 jf-utils = { git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4.5" }
 libp2p = { version = "0.53", default-features = false }
 log-panics = { version = "2.0", features = ["with-backtrace"] }
-snafu = "0.8"
 strum = { version = "0.26", features = ["derive"] }
 surf-disco = "0.9"
 tagged-base64 = "0.4"

builder/Cargo.toml

@@ -40,7 +40,6 @@ rand = "0.8.5"
 sequencer = { path = "../sequencer", features = ["testing"] }
 sequencer-utils = { path = "../utils" }
 serde = { workspace = true }
-snafu = { workspace = true }
 surf = "2.3.1"
 surf-disco = { workspace = true }
 tide-disco = { workspace = true }

builder/src/bin/permissioned-builder.rs

@@ -5,7 +5,7 @@ use std::{
 use anyhow::{bail, Context};
 use builder::permissioned::init_node;
 use clap::Parser;
-use espresso_types::eth_signature_key::EthKeyPair;
+use espresso_types::{eth_signature_key::EthKeyPair, parse_duration};
 use ethers::types::Address;
 use hotshot_types::{
     data::ViewNumber,
@@ -14,9 +14,7 @@ use hotshot_types::{
     traits::{metrics::NoMetrics, node_implementation::ConsensusTime},
 };
 use libp2p::Multiaddr;
-use sequencer::{
-    options::parse_duration, persistence::no_storage::NoStorage, Genesis, L1Params, NetworkParams,
-};
+use sequencer::{persistence::no_storage::NoStorage, Genesis, L1Params, NetworkParams};
 use sequencer_utils::logging;
 use url::Url;
 use vbs::version::{StaticVersion, StaticVersionType};

builder/src/bin/permissionless-builder.rs

@@ -1,14 +1,12 @@
-use std::{num::NonZeroUsize, path::PathBuf, str::FromStr, time::Duration};
+use std::{num::NonZeroUsize, path::PathBuf, time::Duration};
 
 use builder::non_permissioned::{build_instance_state, BuilderConfig};
 use clap::Parser;
-use cld::ClDuration;
-use espresso_types::eth_signature_key::EthKeyPair;
+use espresso_types::{eth_signature_key::EthKeyPair, parse_duration};
 use hotshot::traits::ValidatedState;
 use hotshot_types::{data::ViewNumber, traits::node_implementation::ConsensusTime};
 use sequencer::{Genesis, L1Params};
 use sequencer_utils::logging;
-use snafu::Snafu;
 use url::Url;
 use vbs::version::{StaticVersion, StaticVersionType};
 
@@ -88,19 +86,6 @@ struct NonPermissionedBuilderOptions {
     logging: logging::Config,
 }
 
-#[derive(Clone, Debug, Snafu)]
-struct ParseDurationError {
-    reason: String,
-}
-
-fn parse_duration(s: &str) -> Result<Duration, ParseDurationError> {
-    ClDuration::from_str(s)
-        .map(Duration::from)
-        .map_err(|err| ParseDurationError {
-            reason: err.to_string(),
-        })
-}
-
 #[async_std::main]
 async fn main() -> anyhow::Result<()> {
     let opt = NonPermissionedBuilderOptions::parse();

builder/src/lib.rs

@@ -165,7 +165,6 @@ pub mod testing {
     use portpicker::pick_unused_port;
     use sequencer::state_signature::StateSignatureMemStorage;
     use serde::{Deserialize, Serialize};
-    use snafu::{guide::feature_flags, *};
     use vbs::version::StaticVersion;
 
     use super::*;

hotshot-state-prover/Cargo.toml

@@ -36,7 +36,6 @@ jf-utils = { workspace = true }
 reqwest = { workspace = true }
 sequencer-utils = { path = "../utils" }
 serde = { workspace = true }
-snafu = { workspace = true }
 surf-disco = { workspace = true }
 tide-disco = { workspace = true }
 time = { workspace = true }

hotshot-state-prover/src/bin/state-prover.rs

@@ -1,8 +1,7 @@
-use std::{str::FromStr as _, time::Duration};
+use std::time::Duration;
 
 use clap::Parser;
-use cld::ClDuration;
-use espresso_types::SeqTypes;
+use espresso_types::{parse_duration, SeqTypes};
 use ethers::{
     providers::{Http, Middleware, Provider},
     signers::{coins_bip39::English, MnemonicBuilder, Signer},
@@ -12,7 +11,6 @@ use hotshot_stake_table::config::STAKE_TABLE_CAPACITY;
 use hotshot_state_prover::service::{run_prover_once, run_prover_service, StateProverConfig};
 use hotshot_types::traits::node_implementation::NodeType;
 use sequencer_utils::logging;
-use snafu::Snafu;
 use url::Url;
 use vbs::version::StaticVersionType;
 
@@ -85,19 +83,6 @@ struct Args {
     logging: logging::Config,
 }
 
-#[derive(Clone, Debug, Snafu)]
-pub struct ParseDurationError {
-    reason: String,
-}
-
-fn parse_duration(s: &str) -> Result<Duration, ParseDurationError> {
-    ClDuration::from_str(s)
-        .map(Duration::from)
-        .map_err(|err| ParseDurationError {
-            reason: err.to_string(),
-        })
-}
-
 #[async_std::main]
 async fn main() {
     let args = Args::parse();

marketplace-builder/Cargo.toml

@@ -40,7 +40,6 @@ portpicker = { workspace = true }
 rand = "0.8.5"
 sequencer = { path = "../sequencer", features = ["testing"] }
 serde = { workspace = true }
-snafu = { workspace = true }
 surf = "2.3.1"
 surf-disco = { workspace = true }
 tagged-base64 = { workspace = true }

marketplace-builder/src/bin/marketplace-builder.rs

@@ -1,9 +1,10 @@
-use std::{num::NonZeroUsize, path::PathBuf, str::FromStr, time::Duration};
+use std::{num::NonZeroUsize, path::PathBuf, time::Duration};
 
 use async_compatibility_layer::logging::{setup_backtrace, setup_logging};
 use clap::Parser;
-use cld::ClDuration;
-use espresso_types::{eth_signature_key::EthKeyPair, FeeAmount, NamespaceId, SeqTypes};
+use espresso_types::{
+    eth_signature_key::EthKeyPair, parse_duration, FeeAmount, NamespaceId, SeqTypes,
+};
 use hotshot::traits::ValidatedState;
 use hotshot_types::{data::ViewNumber, traits::node_implementation::ConsensusTime};
 use marketplace_builder::{
@@ -12,7 +13,6 @@ use marketplace_builder::{
 };
 use marketplace_builder_core::testing::basic_test::NodeType;
 use sequencer::{Genesis, L1Params};
-use snafu::Snafu;
 use url::Url;
 use vbs::version::StaticVersionType;
 
@@ -118,19 +118,6 @@ struct NonPermissionedBuilderOptions {
     bid_amount: FeeAmount,
 }
 
-#[derive(Clone, Debug, Snafu)]
-struct ParseDurationError {
-    reason: String,
-}
-
-fn parse_duration(s: &str) -> Result<Duration, ParseDurationError> {
-    ClDuration::from_str(s)
-        .map(Duration::from)
-        .map_err(|err| ParseDurationError {
-            reason: err.to_string(),
-        })
-}
-
 #[async_std::main]
 async fn main() -> anyhow::Result<()> {
     setup_logging();

marketplace-solver/src/options.rs

@@ -1,7 +1,7 @@
-use std::{str::FromStr, time::Duration};
+use std::time::Duration;
 
 use clap::Parser;
-use thiserror::Error;
+use espresso_types::parse_duration;
 use tide_disco::Url;
 
 use crate::database::PostgresClient;
@@ -68,13 +68,3 @@ impl DatabaseOptions {
         PostgresClient::connect(self).await
     }
 }
-
-#[derive(Clone, Debug, Error)]
-#[error("failed to parse `{0}`")]
-pub struct ParseDurationError(String);
-
-pub fn parse_duration(s: &str) -> Result<Duration, ParseDurationError> {
-    cld::ClDuration::from_str(s)
-        .map(Duration::from)
-        .map_err(|err| ParseDurationError(err.to_string()))
-}

node-metrics/Cargo.toml

@@ -21,6 +21,9 @@ futures = { workspace = true }
 hotshot = { workspace = true }
 hotshot-query-service = { workspace = true }
 hotshot-stake-table = { workspace = true }
+
+# Dependencies for feature `testing`
+hotshot-testing = { workspace = true, optional = true }
 hotshot-types = { workspace = true }
 prometheus-parse = { version = "^0.2.5" }
 reqwest = { workspace = true }
@@ -34,6 +37,3 @@ toml = { workspace = true }
 tracing = { workspace = true }
 url = { workspace = true }
 vbs = { workspace = true }
-
-# Dependencies for feature `testing`
-hotshot-testing = { workspace = true, optional = true }

sequencer/Cargo.toml

@@ -88,7 +88,7 @@ sequencer-utils = { path = "../utils" }
 serde = { workspace = true }
 serde_json = { workspace = true }
 sha2 = "0.10"                                                      # TODO temporary, used only for VID, should be set in hotshot
-snafu = { workspace = true }
+snafu = "0.8"
 static_assertions = "1"
 strum = { workspace = true }
 surf-disco = { workspace = true }
@@ -101,7 +101,6 @@ tracing-subscriber = "0.3.18"
 url = { workspace = true }
 vbs = { workspace = true }
 vec1 = { workspace = true }
-
 [package.metadata.cargo-udeps.ignore]
 normal = ["hotshot-testing"]
 

sequencer/src/api.rs

@@ -1045,7 +1045,8 @@ mod test {
     use espresso_types::{
         mock::MockStateCatchup,
         v0_1::{UpgradeMode, ViewBasedUpgrade},
-        FeeAccount, FeeAmount, Header, Upgrade, UpgradeType, ValidatedState,
+        FeeAccount, FeeAmount, Header, TimeBasedUpgrade, Timestamp, Upgrade, UpgradeType,
+        ValidatedState,
     };
     use ethers::utils::Anvil;
     use futures::{
@@ -1067,13 +1068,14 @@ mod test {
     };
     use jf_merkle_tree::prelude::{MerkleProof, Sha3Node};
     use portpicker::pick_unused_port;
-    use sequencer_utils::test_utils::setup_test;
+    use sequencer_utils::{ser::FromStringOrInteger, test_utils::setup_test};
     use surf_disco::Client;
     use test_helpers::{
         catchup_test_helper, state_signature_test_helper, status_test_helper, submit_test_helper,
         TestNetwork, TestNetworkConfigBuilder,
     };
     use tide_disco::{app::AppHealth, error::ServerError, healthcheck::HealthStatus};
+    use time::OffsetDateTime;
 
     use self::{
         data_source::{testing::TestableSequencerDataSource, PublicHotShotConfig},
@@ -1429,9 +1431,33 @@ mod test {
     }
 
     #[async_std::test]
-    async fn test_chain_config_upgrade() {
+    async fn test_chain_config_upgrade_view_based() {
+        setup_test();
+
+        test_chain_config_upgrade_helper(UpgradeMode::View(ViewBasedUpgrade {
+            start_voting_view: None,
+            stop_voting_view: None,
+            start_proposing_view: 1,
+            stop_proposing_view: 10,
+        }))
+        .await;
+    }
+
+    #[async_std::test]
+    async fn test_chain_config_upgrade_time_based() {
         setup_test();
 
+        let now = OffsetDateTime::now_utc().unix_timestamp() as u64;
+        test_chain_config_upgrade_helper(UpgradeMode::Time(TimeBasedUpgrade {
+            start_proposing_time: Timestamp::from_integer(now).unwrap(),
+            stop_proposing_time: Timestamp::from_integer(now + 500).unwrap(),
+            start_voting_time: None,
+            stop_voting_time: None,
+        }))
+        .await;
+    }
+
+    async fn test_chain_config_upgrade_helper(mode: UpgradeMode) {
         let port = pick_unused_port().expect("No ports free");
         let anvil = Anvil::new().spawn();
         let l1 = anvil.endpoint().parse().unwrap();
@@ -1446,12 +1472,7 @@ mod test {
         upgrades.insert(
             <SeqTypes as NodeType>::Upgrade::VERSION,
             Upgrade {
-                mode: UpgradeMode::View(ViewBasedUpgrade {
-                    start_voting_view: None,
-                    stop_voting_view: None,
-                    start_proposing_view: 1,
-                    stop_proposing_view: 10,
-                }),
+                mode,
                 upgrade_type: UpgradeType::ChainConfig {
                     chain_config: chain_config_upgrade,
                 },

sequencer/src/api/endpoints.rs

@@ -51,6 +51,9 @@ pub(super) type AvailState<N, P, D, Ver> = Arc<RwLock<StorageState<N, P, D, Ver>
 
 type AvailabilityApi<N, P, D, Ver> = Api<AvailState<N, P, D, Ver>, availability::Error, Ver>;
 
+// TODO (abdul): replace snafu with `this_error` in  hotshot query service
+// Snafu has been replaced by `this_error` everywhere.
+// However, the query service still uses snafu
 pub(super) fn availability<N, P, D, Ver: StaticVersionType + 'static>(
     bind_version: Ver,
 ) -> Result<AvailabilityApi<N, P, D, Ver>>

sequencer/src/bin/cdn-broker.rs

@@ -3,12 +3,9 @@
 use anyhow::{Context, Result};
 use cdn_broker::{reexports::crypto::signature::KeyPair, Broker, Config};
 use clap::Parser;
-use espresso_types::SeqTypes;
+use espresso_types::{parse_size, SeqTypes};
 use hotshot_types::traits::{node_implementation::NodeType, signature_key::SignatureKey};
-use sequencer::{
-    network::cdn::{ProductionDef, WrappedSignatureKey},
-    options::parse_size,
-};
+use sequencer::network::cdn::{ProductionDef, WrappedSignatureKey};
 use sha2::Digest;
 use tracing_subscriber::EnvFilter;
 

sequencer/src/bin/cdn-marshal.rs

@@ -4,8 +4,8 @@
 use anyhow::{Context, Result};
 use cdn_marshal::{Config, Marshal};
 use clap::Parser;
-use espresso_types::SeqTypes;
-use sequencer::{network::cdn::ProductionDef, options::parse_size};
+use espresso_types::{parse_size, SeqTypes};
+use sequencer::network::cdn::ProductionDef;
 use tracing_subscriber::EnvFilter;
 
 #[derive(Parser, Debug)]

sequencer/src/bin/commitment-task.rs

@@ -2,14 +2,11 @@ use std::{io, time::Duration};
 
 use async_std::task::spawn;
 use clap::Parser;
-use espresso_types::SeqTypes;
+use espresso_types::{parse_duration, SeqTypes};
 use ethers::prelude::*;
 use futures::FutureExt;
 use hotshot_types::traits::node_implementation::NodeType;
-use sequencer::{
-    hotshot_commitment::{run_hotshot_commitment_task, CommitmentTaskOptions},
-    options::parse_duration,
-};
+use sequencer::hotshot_commitment::{run_hotshot_commitment_task, CommitmentTaskOptions};
 use sequencer_utils::logging;
 use tide_disco::{error::ServerError, Api};
 use url::Url;