From 0cd76dfaf087c7df52b15cb6befe2b2d3a8c5d4b Mon Sep 17 00:00:00 2001 From: Reisen Date: Wed, 29 May 2024 13:42:46 +0100 Subject: [PATCH] refactor(agent): remove dashboard (#125) --- config/config.toml | 2 +- src/agent.rs | 1 - src/agent/dashboard.rs | 283 ----------------------------------------- src/agent/metrics.rs | 56 +++----- 4 files changed, 16 insertions(+), 326 deletions(-) delete mode 100644 src/agent/dashboard.rs diff --git a/config/config.toml b/config/config.toml index 6767b91..8f35ade 100644 --- a/config/config.toml +++ b/config/config.toml @@ -108,7 +108,7 @@ key_store.mapping_key = "RelevantOracleMappingAddress" # [metrics_server] # -# Where to serve the quick-access dashboard and metrics. Metrics live under "/metrics" +# Where to serve metrics. Metrics live under "/metrics" # NOTE: non-loopback addresses must be used carefully, making sure the # connection is not exposed for unauthorized access. # bind_address = "127.0.0.1:8888" diff --git a/src/agent.rs b/src/agent.rs index f6bd263..c309075 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -62,7 +62,6 @@ Note that there is an Oracle and Exporter for each network, but only one Local S ################################################################################################################################## */ -pub mod dashboard; pub mod legacy_schedule; pub mod market_schedule; pub mod metrics; diff --git a/src/agent/dashboard.rs b/src/agent/dashboard.rs deleted file mode 100644 index b6ca7c9..0000000 --- a/src/agent/dashboard.rs +++ /dev/null @@ -1,283 +0,0 @@ -use { - super::{ - solana::{ - network::Network, - oracle::PriceEntry, - }, - state::{ - global::GlobalStore, - local::{ - LocalStore, - PriceInfo, - }, - }, - }, - crate::agent::{ - metrics::MetricsServer, - state::global::{ - AllAccountsData, - AllAccountsMetadata, - PriceAccountMetadata, - }, - }, - chrono::DateTime, - pyth_sdk::{ - Identifier, - PriceIdentifier, - }, - slog::Logger, - solana_sdk::pubkey::Pubkey, - std::{ - collections::{ - BTreeMap, - BTreeSet, - HashMap, - HashSet, - }, - time::Duration, - }, - typed_html::{ - dom::DOMTree, - html, - text, - }, -}; - -impl MetricsServer { - /// Create an HTML view of store data - pub async fn render_dashboard(&self) -> Result> { - // Request price data from local and global store - let local_data = LocalStore::get_all_price_infos(&*self.adapter).await; - let global_data = GlobalStore::accounts_data(&*self.adapter, Network::Primary).await?; - let global_metadata = GlobalStore::accounts_metadata(&*self.adapter).await?; - - let symbol_view = - build_dashboard_data(local_data, global_data, global_metadata, &self.logger); - - // Note the uptime and adjust to whole seconds for cleaner output - let uptime = Duration::from_secs(self.start_time.elapsed().as_secs()); - - // Build and collect table rows - let mut rows = vec![]; - - for (symbol, data) in symbol_view { - for (price_pubkey, price_data) in data.prices { - let price_string = if let Some(global_data) = price_data.global_data { - let expo = global_data.expo; - let price_with_expo: f64 = global_data.agg.price as f64 * 10f64.powi(expo); - format!("{:.2}", price_with_expo) - } else { - "no data".to_string() - }; - - let last_publish_string = if let Some(global_data) = price_data.global_data { - if let Some(datetime) = DateTime::from_timestamp(global_data.timestamp, 0) { - datetime.format("%Y-%m-%d %H:%M:%S").to_string() - } else { - format!("Invalid timestamp {}", global_data.timestamp) - } - } else { - "no data".to_string() - }; - - let last_local_update_string = if let Some(local_data) = price_data.local_data { - local_data.timestamp.format("%Y-%m-%d %H:%M:%S").to_string() - } else { - "no data".to_string() - }; - - let row_snippet = html! { - - {text!(symbol.clone())} - {text!(data.product.to_string())} - {text!(price_pubkey.to_string())} - {text!(price_string)} - {text!(last_publish_string)} - {text!(last_local_update_string)} - - }; - rows.push(row_snippet); - } - } - - let title_string = concat!("Pyth Agent Dashboard - ", env!("CARGO_PKG_VERSION")); - let res_html: DOMTree = html! { - - - {text!(title_string)} - - - -

{text!(title_string)}

- {text!("Uptime: {}", humantime::format_duration(uptime))} -

"State Overview"

- - - - - - - - - - { rows } -
"Symbol""Product ID""Price ID""Last Published Price""Last Publish Time""Last Local Update Time"
- - - }; - Ok(res_html.to_string()) - } -} - -#[derive(Debug)] -pub struct DashboardSymbolView { - product: Pubkey, - prices: BTreeMap, -} - -#[derive(Debug)] -pub struct DashboardPriceView { - local_data: Option, - global_data: Option, - _global_metadata: Option, -} - -/// Turn global/local store state into a single per-symbol view. -/// -/// The dashboard data comes from three sources - the global store -/// (observed on-chain state) data, global store metadata and local -/// store data (local state possibly not yet committed to the oracle -/// contract). -/// -/// The view is indexed by human-readable symbol name or a stringified -/// public key if symbol name can't be found. -pub fn build_dashboard_data( - mut local_data: HashMap, - mut global_data: AllAccountsData, - mut global_metadata: AllAccountsMetadata, - logger: &Logger, -) -> BTreeMap { - let mut ret = BTreeMap::new(); - - debug!(logger, "Building dashboard data"; - "local_data_len" => local_data.len(), - "global_data_products_len" => global_data.product_accounts.len(), - "global_data_prices_len" => global_data.price_accounts.len(), - "global_metadata_products_len" => global_metadata.product_accounts_metadata.len(), - "global_metadata_prices_len" => global_metadata.price_accounts_metadata.len(), - ); - - // Learn all the product/price keys in the system, - let all_product_keys_iter = global_metadata.product_accounts_metadata.keys().cloned(); - - let all_product_keys_dedup = all_product_keys_iter.collect::>(); - - let all_price_keys_iter = global_data - .price_accounts - .keys() - .chain(global_metadata.price_accounts_metadata.keys()) - .cloned() - .chain(local_data.keys().map(|identifier| { - let bytes = identifier.to_bytes(); - Pubkey::new_from_array(bytes) - })); - - let mut all_price_keys_dedup = all_price_keys_iter.collect::>(); - - // query all the keys and assemvle them into the view - - let mut remaining_product_keys = all_product_keys_dedup.clone(); - - for product_key in all_product_keys_dedup { - let _product_data = global_data.product_accounts.remove(&product_key); - - if let Some(mut product_metadata) = global_metadata - .product_accounts_metadata - .remove(&product_key) - { - let mut symbol_name = product_metadata - .attr_dict - .get("symbol") - .cloned() - // Use product key for unnamed products - .unwrap_or(format!("unnamed product {}", product_key)); - - // Sort and deduplicate prices - let this_product_price_keys_dedup = product_metadata - .price_accounts - .drain(0..) - .collect::>(); - - let mut prices = BTreeMap::new(); - - // Extract information about each price - for price_key in this_product_price_keys_dedup { - let price_global_data = global_data.price_accounts.remove(&price_key); - let price_global_metadata = - global_metadata.price_accounts_metadata.remove(&price_key); - - let price_identifier = Identifier::new(price_key.to_bytes()); - let price_local_data = local_data.remove(&price_identifier); - - prices.insert( - price_key, - DashboardPriceView { - local_data: price_local_data, - global_data: price_global_data, - _global_metadata: price_global_metadata, - }, - ); - // Mark this price as done - all_price_keys_dedup.remove(&price_key); - } - - // Mark this product as done - remaining_product_keys.remove(&product_key); - - let symbol_view = DashboardSymbolView { - product: product_key, - prices, - }; - - if ret.contains_key(&symbol_name) { - let new_symbol_name = format!("{} (duplicate)", symbol_name); - - warn!(logger, "Dashboard: duplicate symbol name detected, renaming"; - "symbol_name" => &symbol_name, - "symbol_renamed_to" => &new_symbol_name, - "conflicting_symbol_data" => format!("{:?}", symbol_view), - ); - - symbol_name = new_symbol_name; - } - - ret.insert(symbol_name, symbol_view); - } else { - // This logging handles only missing products that we - // should have found. Missing prices are okay, appearing - // in cases where no on-chain queries or publishing took - // place yet. - warn!(logger, "Dashboard: Failed to look up product metadata"; "product_id" => product_key.to_string()); - } - } - - if !(all_price_keys_dedup.is_empty() && remaining_product_keys.is_empty()) { - let remaining_products: Vec<_> = remaining_product_keys.drain().collect(); - let remaining_prices: Vec<_> = all_price_keys_dedup.drain().collect(); - warn!(logger, "Dashboard: Orphaned product/price IDs detected"; - "remaining_product_ids" => format!("{:?}", remaining_products), - "remaining_price_ids" => format!("{:?}", remaining_prices)); - } - - ret -} diff --git a/src/agent/metrics.rs b/src/agent/metrics.rs index 0beb932..b806840 100644 --- a/src/agent/metrics.rs +++ b/src/agent/metrics.rs @@ -65,7 +65,7 @@ lazy_static! { } /// Internal metrics server state, holds state needed for serving -/// dashboard and metrics. +/// metrics. pub struct MetricsServer { pub start_time: Instant, pub logger: Logger, @@ -73,7 +73,7 @@ pub struct MetricsServer { } impl MetricsServer { - /// Instantiate a metrics API with a dashboard + /// Instantiate a metrics API. pub async fn spawn(addr: impl Into + 'static, logger: Logger, adapter: Arc) { let server = MetricsServer { start_time: Instant::now(), @@ -82,56 +82,30 @@ impl MetricsServer { }; let shared_state = Arc::new(Mutex::new(server)); - - let shared_state4dashboard = shared_state.clone(); - let dashboard_route = warp::path("dashboard") - .or(warp::path::end()) - .and_then(move |_| { - let shared_state = shared_state4dashboard.clone(); - async move { - let locked_state = shared_state.lock().await; - let response = locked_state - .render_dashboard() // Defined in a separate impl block near dashboard-specific code - .await - .unwrap_or_else(|e| { - // Add logging here - error!(locked_state.logger,"Dashboard: Rendering failed"; "error" => e.to_string()); - - // Withhold failure details from client - "Could not render dashboard! See the logs for details".to_owned() - }); - Result::, Rejection>::Ok(Box::new(reply::with_status( - reply::html(response), - StatusCode::OK, - ))) - } - }); - let shared_state4metrics = shared_state.clone(); let metrics_route = warp::path("metrics") .and(warp::path::end()) .and_then(move || { let shared_state = shared_state4metrics.clone(); async move { - let locked_state = shared_state.lock().await; + let locked_state = shared_state.lock().await; let mut buf = String::new(); - let response = encode(&mut buf, &&PROMETHEUS_REGISTRY.lock().await).map_err(|e| -> Box {e.into() - }).and_then(|_| -> Result<_, Box> { - - Ok(Box::new(reply::with_status(buf, StatusCode::OK))) - }).unwrap_or_else(|e| { - error!(locked_state.logger, "Metrics: Could not gather metrics from registry"; "error" => e.to_string()); - - Box::new(reply::with_status("Could not gather metrics. See logs for details".to_string(), StatusCode::INTERNAL_SERVER_ERROR)) - }); + let response = encode(&mut buf, &&PROMETHEUS_REGISTRY.lock().await) + .map_err(|e| -> Box { + e.into() + }) + .and_then(|_| -> Result<_, Box> { + Ok(Box::new(reply::with_status(buf, StatusCode::OK))) + }).unwrap_or_else(|e| { + error!(locked_state.logger, "Metrics: Could not gather metrics from registry"; "error" => e.to_string()); + Box::new(reply::with_status("Could not gather metrics. See logs for details".to_string(), StatusCode::INTERNAL_SERVER_ERROR)) + }); - Result::, Rejection>::Ok(response) + Result::, Rejection>::Ok(response) } }); - warp::serve(dashboard_route.or(metrics_route)) - .bind(addr) - .await; + warp::serve(metrics_route).bind(addr).await; } }