feat: enable configuration hot-reloading (#231)

api/signer-api.yml

@@ -5,6 +5,7 @@ info:
   description: API that allows commit modules to request generic signatures from validators
 tags:
   - name: Signer
+  - name: Management
 paths:
   /signer/v1/get_pubkeys:
     get:
@@ -254,6 +255,20 @@ paths:
                     type: string
                     example: "Internal error"
 
+  /status:
+    get:
+      summary: Get the status of the Signer API module
+      tags:
+        - Management
+      responses:
+        "200":
+          description: Success
+          content:
+            text/plain:
+              schema:
+                type: string
+                example: "OK"
+
 components:
   securitySchemes:
     BearerAuth:

bin/src/lib.rs

@@ -20,7 +20,7 @@ pub mod prelude {
     pub use cb_metrics::provider::MetricsProvider;
     pub use cb_pbs::{
         get_header, get_status, register_validator, submit_block, BuilderApi, BuilderApiState,
-        DefaultBuilderApi, PbsService, PbsState,
+        DefaultBuilderApi, PbsService, PbsState, PbsStateGuard,
     };
     // The TreeHash derive macro requires tree_hash as import
     pub mod tree_hash {

crates/common/src/commit/constants.rs

@@ -2,3 +2,4 @@ pub const GET_PUBKEYS_PATH: &str = "/signer/v1/get_pubkeys";
 pub const REQUEST_SIGNATURE_PATH: &str = "/signer/v1/request_signature";
 pub const GENERATE_PROXY_KEY_PATH: &str = "/signer/v1/generate_proxy_key";
 pub const STATUS_PATH: &str = "/status";
+pub const RELOAD_PATH: &str = "/reload";

crates/common/src/pbs/constants.rs

@@ -6,6 +6,7 @@ pub const GET_HEADER_PATH: &str = "/header/{slot}/{parent_hash}/{pubkey}";
 pub const GET_STATUS_PATH: &str = "/status";
 pub const REGISTER_VALIDATOR_PATH: &str = "/validators";
 pub const SUBMIT_BLOCK_PATH: &str = "/blinded_blocks";
+pub const RELOAD_PATH: &str = "/reload";
 
 // https://ethereum.github.io/builder-specs/#/Builder
 

crates/common/src/pbs/event.rs

@@ -37,6 +37,8 @@ pub enum BuilderEvent {
     },
     RegisterValidatorRequest(Vec<ValidatorRegistration>),
     RegisterValidatorResponse,
+    ReloadEvent,
+    ReloadResponse,
 }
 
 #[derive(Debug, Clone)]

crates/pbs/src/api.rs

@@ -7,13 +7,13 @@ use cb_common::pbs::{
 
 use crate::{
     mev_boost,
-    state::{BuilderApiState, PbsState},
+    state::{BuilderApiState, PbsState, PbsStateGuard},
 };
 
 #[async_trait]
 pub trait BuilderApi<S: BuilderApiState>: 'static {
     /// Use to extend the BuilderApi
-    fn extra_routes() -> Option<Router<PbsState<S>>> {
+    fn extra_routes() -> Option<Router<PbsStateGuard<S>>> {
         None
     }
 
@@ -48,6 +48,10 @@ pub trait BuilderApi<S: BuilderApiState>: 'static {
     ) -> eyre::Result<()> {
         mev_boost::register_validator(registrations, req_headers, state).await
     }
+
+    async fn reload(state: PbsState<S>) -> eyre::Result<PbsState<S>> {
+        mev_boost::reload(state).await
+    }
 }
 
 pub struct DefaultBuilderApi;

crates/pbs/src/constants.rs

@@ -2,6 +2,7 @@ pub const STATUS_ENDPOINT_TAG: &str = "status";
 pub const REGISTER_VALIDATOR_ENDPOINT_TAG: &str = "register_validator";
 pub const SUBMIT_BLINDED_BLOCK_ENDPOINT_TAG: &str = "submit_blinded_block";
 pub const GET_HEADER_ENDPOINT_TAG: &str = "get_header";
+pub const RELOAD_ENDPOINT_TAG: &str = "reload";
 
 /// For metrics recorded when a request times out
 pub const TIMEOUT_ERROR_CODE: u16 = 555;

crates/pbs/src/error.rs

@@ -5,22 +5,25 @@ use axum::{http::StatusCode, response::IntoResponse};
 pub enum PbsClientError {
     NoResponse,
     NoPayload,
+    Internal,
 }
 
 impl PbsClientError {
     pub fn status_code(&self) -> StatusCode {
         match self {
             PbsClientError::NoResponse => StatusCode::BAD_GATEWAY,
             PbsClientError::NoPayload => StatusCode::BAD_GATEWAY,
+            PbsClientError::Internal => StatusCode::INTERNAL_SERVER_ERROR,
         }
     }
 }
 
 impl IntoResponse for PbsClientError {
     fn into_response(self) -> axum::response::Response {
-        let msg = match self {
-            PbsClientError::NoResponse => "no response from relays",
-            PbsClientError::NoPayload => "no payload from relays",
+        let msg = match &self {
+            PbsClientError::NoResponse => "no response from relays".to_string(),
+            PbsClientError::NoPayload => "no payload from relays".to_string(),
+            PbsClientError::Internal => "internal server error".to_string(),
         };
 
         (self.status_code(), msg).into_response()

crates/pbs/src/lib.rs

@@ -12,4 +12,4 @@ pub use api::*;
 pub use constants::*;
 pub use mev_boost::*;
 pub use service::PbsService;
-pub use state::{BuilderApiState, PbsState};
+pub use state::{BuilderApiState, PbsState, PbsStateGuard};

crates/pbs/src/mev_boost/mod.rs

@@ -1,9 +1,11 @@
 mod get_header;
 mod register_validator;
+mod reload;
 mod status;
 mod submit_block;
 
 pub use get_header::get_header;
 pub use register_validator::register_validator;
+pub use reload::reload;
 pub use status::get_status;
 pub use submit_block::submit_block;

crates/pbs/src/mev_boost/reload.rs

@@ -0,0 +1,27 @@
+use cb_common::config::load_pbs_config;
+use tracing::warn;
+
+use crate::{BuilderApiState, PbsState};
+
+/// Reload the PBS state with the latest configuration in the config file
+/// Returns 200 if successful or 500 if failed
+pub async fn reload<S: BuilderApiState>(state: PbsState<S>) -> eyre::Result<PbsState<S>> {
+    let pbs_config = load_pbs_config().await?;
+    let new_state = PbsState::new(pbs_config).with_data(state.data);
+
+    if state.config.pbs_config.host != new_state.config.pbs_config.host {
+        warn!(
+            "Host change for PBS module require a full restart. Old: {}, New: {}",
+            state.config.pbs_config.host, new_state.config.pbs_config.host
+        );
+    }
+
+    if state.config.pbs_config.port != new_state.config.pbs_config.port {
+        warn!(
+            "Port change for PBS module require a full restart. Old: {}, New: {}",
+            state.config.pbs_config.port, new_state.config.pbs_config.port
+        );
+    }
+
+    Ok(new_state)
+}

crates/pbs/src/routes/get_header.rs

@@ -17,15 +17,17 @@ use crate::{
     constants::GET_HEADER_ENDPOINT_TAG,
     error::PbsClientError,
     metrics::BEACON_NODE_STATUS,
-    state::{BuilderApiState, PbsState},
+    state::{BuilderApiState, PbsStateGuard},
 };
 
 #[tracing::instrument(skip_all, name = "get_header", fields(req_id = %Uuid::new_v4(), slot = params.slot))]
 pub async fn handle_get_header<S: BuilderApiState, A: BuilderApi<S>>(
-    State(state): State<PbsState<S>>,
+    State(state): State<PbsStateGuard<S>>,
     req_headers: HeaderMap,
     Path(params): Path<GetHeaderParams>,
 ) -> Result<impl IntoResponse, PbsClientError> {
+    let state = state.read().clone();
+
     state.publish_event(BuilderEvent::GetHeaderRequest(params));
 
     let ua = get_user_agent(&req_headers);

crates/pbs/src/routes/mod.rs

@@ -1,5 +1,6 @@
 mod get_header;
 mod register_validator;
+mod reload;
 mod router;
 mod status;
 mod submit_block;

crates/pbs/src/routes/register_validator.rs

@@ -16,15 +16,17 @@ use crate::{
     constants::REGISTER_VALIDATOR_ENDPOINT_TAG,
     error::PbsClientError,
     metrics::BEACON_NODE_STATUS,
-    state::{BuilderApiState, PbsState},
+    state::{BuilderApiState, PbsStateGuard},
 };
 
 #[tracing::instrument(skip_all, name = "register_validators", fields(req_id = %Uuid::new_v4()))]
 pub async fn handle_register_validator<S: BuilderApiState, A: BuilderApi<S>>(
-    State(state): State<PbsState<S>>,
+    State(state): State<PbsStateGuard<S>>,
     req_headers: HeaderMap,
     Json(registrations): Json<Vec<ValidatorRegistration>>,
 ) -> Result<impl IntoResponse, PbsClientError> {
+    let state = state.read().clone();
+
     trace!(?registrations);
     state.publish_event(BuilderEvent::RegisterValidatorRequest(registrations.clone()));
 

crates/pbs/src/routes/reload.rs

@@ -0,0 +1,47 @@
+use axum::{extract::State, http::HeaderMap, response::IntoResponse};
+use cb_common::{pbs::BuilderEvent, utils::get_user_agent};
+use reqwest::StatusCode;
+use tracing::{error, info};
+use uuid::Uuid;
+
+use crate::{
+    error::PbsClientError,
+    metrics::BEACON_NODE_STATUS,
+    state::{BuilderApiState, PbsStateGuard},
+    BuilderApi, RELOAD_ENDPOINT_TAG,
+};
+
+#[tracing::instrument(skip_all, name = "reload", fields(req_id = %Uuid::new_v4()))]
+pub async fn handle_reload<S: BuilderApiState, A: BuilderApi<S>>(
+    req_headers: HeaderMap,
+    State(state): State<PbsStateGuard<S>>,
+) -> Result<impl IntoResponse, PbsClientError> {
+    let prev_state = state.read().clone();
+
+    prev_state.publish_event(BuilderEvent::ReloadEvent);
+
+    let ua = get_user_agent(&req_headers);
+
+    info!(ua, relay_check = prev_state.config.pbs_config.relay_check);
+
+    match A::reload(prev_state.clone()).await {
+        Ok(new_state) => {
+            prev_state.publish_event(BuilderEvent::ReloadResponse);
+            info!("config reload successful");
+
+            *state.write() = new_state;
+
+            BEACON_NODE_STATUS.with_label_values(&["200", RELOAD_ENDPOINT_TAG]).inc();
+            Ok((StatusCode::OK, "OK"))
+        }
+        Err(err) => {
+            error!(%err, "config reload failed");
+
+            let err = PbsClientError::Internal;
+            BEACON_NODE_STATUS
+                .with_label_values(&[err.status_code().as_str(), RELOAD_ENDPOINT_TAG])
+                .inc();
+            Err(err)
+        }
+    }
+}

crates/pbs/src/routes/router.rs

@@ -3,23 +3,28 @@ use axum::{
     Router,
 };
 use cb_common::pbs::{
-    BUILDER_API_PATH, GET_HEADER_PATH, GET_STATUS_PATH, REGISTER_VALIDATOR_PATH, SUBMIT_BLOCK_PATH,
+    BUILDER_API_PATH, GET_HEADER_PATH, GET_STATUS_PATH, REGISTER_VALIDATOR_PATH, RELOAD_PATH,
+    SUBMIT_BLOCK_PATH,
 };
 
-use super::{handle_get_header, handle_get_status, handle_register_validator, handle_submit_block};
+use super::{
+    handle_get_header, handle_get_status, handle_register_validator, handle_submit_block,
+    reload::handle_reload,
+};
 use crate::{
     api::BuilderApi,
-    state::{BuilderApiState, PbsState},
+    state::{BuilderApiState, PbsStateGuard},
 };
 
-pub fn create_app_router<S: BuilderApiState, A: BuilderApi<S>>(state: PbsState<S>) -> Router {
+pub fn create_app_router<S: BuilderApiState, A: BuilderApi<S>>(state: PbsStateGuard<S>) -> Router {
     let builder_routes = Router::new()
         .route(GET_HEADER_PATH, get(handle_get_header::<S, A>))
         .route(GET_STATUS_PATH, get(handle_get_status::<S, A>))
         .route(REGISTER_VALIDATOR_PATH, post(handle_register_validator::<S, A>))
         .route(SUBMIT_BLOCK_PATH, post(handle_submit_block::<S, A>));
+    let reload_router = Router::new().route(RELOAD_PATH, post(handle_reload::<S, A>));
 
-    let builder_api = Router::new().nest(BUILDER_API_PATH, builder_routes);
+    let builder_api = Router::new().nest(BUILDER_API_PATH, builder_routes).merge(reload_router);
 
     let app = if let Some(extra_routes) = A::extra_routes() {
         builder_api.merge(extra_routes)

crates/pbs/src/routes/status.rs

@@ -9,14 +9,16 @@ use crate::{
     constants::STATUS_ENDPOINT_TAG,
     error::PbsClientError,
     metrics::BEACON_NODE_STATUS,
-    state::{BuilderApiState, PbsState},
+    state::{BuilderApiState, PbsStateGuard},
 };
 
 #[tracing::instrument(skip_all, name = "status", fields(req_id = %Uuid::new_v4()))]
 pub async fn handle_get_status<S: BuilderApiState, A: BuilderApi<S>>(
     req_headers: HeaderMap,
-    State(state): State<PbsState<S>>,
+    State(state): State<PbsStateGuard<S>>,
 ) -> Result<impl IntoResponse, PbsClientError> {
+    let state = state.read().clone();
+
     state.publish_event(BuilderEvent::GetStatusEvent);
 
     let ua = get_user_agent(&req_headers);

crates/pbs/src/routes/submit_block.rs

@@ -12,15 +12,17 @@ use crate::{
     constants::SUBMIT_BLINDED_BLOCK_ENDPOINT_TAG,
     error::PbsClientError,
     metrics::BEACON_NODE_STATUS,
-    state::{BuilderApiState, PbsState},
+    state::{BuilderApiState, PbsStateGuard},
 };
 
 #[tracing::instrument(skip_all, name = "submit_blinded_block", fields(req_id = %Uuid::new_v4(), slot = signed_blinded_block.message.slot))]
 pub async fn handle_submit_block<S: BuilderApiState, A: BuilderApi<S>>(
-    State(state): State<PbsState<S>>,
+    State(state): State<PbsStateGuard<S>>,
     req_headers: HeaderMap,
     Json(signed_blinded_block): Json<SignedBlindedBeaconBlock>,
 ) -> Result<impl IntoResponse, PbsClientError> {
+    let state = state.read().clone();
+
     trace!(?signed_blinded_block);
     state.publish_event(BuilderEvent::SubmitBlockRequest(Box::new(signed_blinded_block.clone())));
 

crates/pbs/src/service.rs

@@ -7,6 +7,7 @@ use cb_common::{
 };
 use cb_metrics::provider::MetricsProvider;
 use eyre::{bail, Context, Result};
+use parking_lot::RwLock;
 use prometheus::core::Collector;
 use tokio::net::TcpListener;
 use tracing::info;
@@ -28,7 +29,7 @@ impl PbsService {
             state.config.event_publisher.as_ref().map(|e| e.n_subscribers()).unwrap_or_default();
         info!(version = COMMIT_BOOST_VERSION, commit = COMMIT_BOOST_COMMIT, ?addr, events_subs, chain =? state.config.chain, "starting PBS service");
 
-        let app = create_app_router::<S, A>(state);
+        let app = create_app_router::<S, A>(RwLock::new(state).into());
         let listener = TcpListener::bind(addr).await?;
 
         let task =

crates/pbs/src/state.rs

@@ -1,12 +1,17 @@
+use std::sync::Arc;
+
 use alloy::rpc::types::beacon::BlsPublicKey;
 use cb_common::{
     config::{PbsConfig, PbsModuleConfig},
     pbs::{BuilderEvent, RelayClient},
 };
+use parking_lot::RwLock;
 
 pub trait BuilderApiState: Clone + Sync + Send + 'static {}
 impl BuilderApiState for () {}
 
+pub type PbsStateGuard<S> = Arc<RwLock<PbsState<S>>>;
+
 /// Config for the Pbs module. It can be extended by adding extra data to the
 /// state for modules that need it
 // TODO: consider remove state from the PBS module altogether