From 1d08d8b4f826130129172fd10eaf9ac27d39d0ac Mon Sep 17 00:00:00 2001
From: Reisen <reisen@morphism.org>
Date: Wed, 29 May 2024 09:13:46 +0000
Subject: [PATCH] refactor(agent): remove dashboard
---
config/config.toml | 2 +-
src/agent.rs | 1 -
src/agent/dashboard.rs | 283 -----------------------------------------
src/agent/metrics.rs | 56 +++-----
4 files changed, 16 insertions(+), 326 deletions(-)
delete mode 100644 src/agent/dashboard.rs
diff --git a/config/config.toml b/config/config.toml
index 6767b91..8f35ade 100644
--- a/config/config.toml
+++ b/config/config.toml
@@ -108,7 +108,7 @@ key_store.mapping_key = "RelevantOracleMappingAddress"
# [metrics_server]
#
-# Where to serve the quick-access dashboard and metrics. Metrics live under "/metrics"
+# Where to serve metrics. Metrics live under "/metrics"
# NOTE: non-loopback addresses must be used carefully, making sure the
# connection is not exposed for unauthorized access.
# bind_address = "127.0.0.1:8888"
diff --git a/src/agent.rs b/src/agent.rs
index f6bd263..c309075 100644
--- a/src/agent.rs
+++ b/src/agent.rs
@@ -62,7 +62,6 @@ Note that there is an Oracle and Exporter for each network, but only one Local S
################################################################################################################################## */
-pub mod dashboard;
pub mod legacy_schedule;
pub mod market_schedule;
pub mod metrics;
diff --git a/src/agent/dashboard.rs b/src/agent/dashboard.rs
deleted file mode 100644
index b6ca7c9..0000000
--- a/src/agent/dashboard.rs
+++ /dev/null
@@ -1,283 +0,0 @@
-use {
- super::{
- solana::{
- network::Network,
- oracle::PriceEntry,
- },
- state::{
- global::GlobalStore,
- local::{
- LocalStore,
- PriceInfo,
- },
- },
- },
- crate::agent::{
- metrics::MetricsServer,
- state::global::{
- AllAccountsData,
- AllAccountsMetadata,
- PriceAccountMetadata,
- },
- },
- chrono::DateTime,
- pyth_sdk::{
- Identifier,
- PriceIdentifier,
- },
- slog::Logger,
- solana_sdk::pubkey::Pubkey,
- std::{
- collections::{
- BTreeMap,
- BTreeSet,
- HashMap,
- HashSet,
- },
- time::Duration,
- },
- typed_html::{
- dom::DOMTree,
- html,
- text,
- },
-};
-
-impl MetricsServer {
- /// Create an HTML view of store data
- pub async fn render_dashboard(&self) -> Result<String, Box<dyn std::error::Error>> {
- // Request price data from local and global store
- let local_data = LocalStore::get_all_price_infos(&*self.adapter).await;
- let global_data = GlobalStore::accounts_data(&*self.adapter, Network::Primary).await?;
- let global_metadata = GlobalStore::accounts_metadata(&*self.adapter).await?;
-
- let symbol_view =
- build_dashboard_data(local_data, global_data, global_metadata, &self.logger);
-
- // Note the uptime and adjust to whole seconds for cleaner output
- let uptime = Duration::from_secs(self.start_time.elapsed().as_secs());
-
- // Build and collect table rows
- let mut rows = vec![];
-
- for (symbol, data) in symbol_view {
- for (price_pubkey, price_data) in data.prices {
- let price_string = if let Some(global_data) = price_data.global_data {
- let expo = global_data.expo;
- let price_with_expo: f64 = global_data.agg.price as f64 * 10f64.powi(expo);
- format!("{:.2}", price_with_expo)
- } else {
- "no data".to_string()
- };
-
- let last_publish_string = if let Some(global_data) = price_data.global_data {
- if let Some(datetime) = DateTime::from_timestamp(global_data.timestamp, 0) {
- datetime.format("%Y-%m-%d %H:%M:%S").to_string()
- } else {
- format!("Invalid timestamp {}", global_data.timestamp)
- }
- } else {
- "no data".to_string()
- };
-
- let last_local_update_string = if let Some(local_data) = price_data.local_data {
- local_data.timestamp.format("%Y-%m-%d %H:%M:%S").to_string()
- } else {
- "no data".to_string()
- };
-
- let row_snippet = html! {
- <tr>
- <td>{text!(symbol.clone())}</td>
- <td>{text!(data.product.to_string())}</td>
- <td>{text!(price_pubkey.to_string())}</td>
- <td>{text!(price_string)}</td>
- <td>{text!(last_publish_string)}</td>
- <td>{text!(last_local_update_string)}</td>
- </tr>
- };
- rows.push(row_snippet);
- }
- }
-
- let title_string = concat!("Pyth Agent Dashboard - ", env!("CARGO_PKG_VERSION"));
- let res_html: DOMTree<String> = html! {
- <html>
- <head>
- <title>{text!(title_string)}</title>
- <style>
- """
-table {
- width: 100%;
- border-collapse: collapse;
-}
-table, th, td {
- border: 1px solid;
-}
-"""
- </style>
- </head>
- <body>
- <h1>{text!(title_string)}</h1>
- {text!("Uptime: {}", humantime::format_duration(uptime))}
- <h2>"State Overview"</h2>
- <table>
- <tr>
- <th>"Symbol"</th>
- <th>"Product ID"</th>
- <th>"Price ID"</th>
- <th>"Last Published Price"</th>
- <th>"Last Publish Time"</th>
- <th>"Last Local Update Time"</th>
- </tr>
- { rows }
- </table>
- </body>
- </html>
- };
- Ok(res_html.to_string())
- }
-}
-
-#[derive(Debug)]
-pub struct DashboardSymbolView {
- product: Pubkey,
- prices: BTreeMap<Pubkey, DashboardPriceView>,
-}
-
-#[derive(Debug)]
-pub struct DashboardPriceView {
- local_data: Option<PriceInfo>,
- global_data: Option<PriceEntry>,
- _global_metadata: Option<PriceAccountMetadata>,
-}
-
-/// Turn global/local store state into a single per-symbol view.
-///
-/// The dashboard data comes from three sources - the global store
-/// (observed on-chain state) data, global store metadata and local
-/// store data (local state possibly not yet committed to the oracle
-/// contract).
-///
-/// The view is indexed by human-readable symbol name or a stringified
-/// public key if symbol name can't be found.
-pub fn build_dashboard_data(
- mut local_data: HashMap<PriceIdentifier, PriceInfo>,
- mut global_data: AllAccountsData,
- mut global_metadata: AllAccountsMetadata,
- logger: &Logger,
-) -> BTreeMap<String, DashboardSymbolView> {
- let mut ret = BTreeMap::new();
-
- debug!(logger, "Building dashboard data";
- "local_data_len" => local_data.len(),
- "global_data_products_len" => global_data.product_accounts.len(),
- "global_data_prices_len" => global_data.price_accounts.len(),
- "global_metadata_products_len" => global_metadata.product_accounts_metadata.len(),
- "global_metadata_prices_len" => global_metadata.price_accounts_metadata.len(),
- );
-
- // Learn all the product/price keys in the system,
- let all_product_keys_iter = global_metadata.product_accounts_metadata.keys().cloned();
-
- let all_product_keys_dedup = all_product_keys_iter.collect::<HashSet<Pubkey>>();
-
- let all_price_keys_iter = global_data
- .price_accounts
- .keys()
- .chain(global_metadata.price_accounts_metadata.keys())
- .cloned()
- .chain(local_data.keys().map(|identifier| {
- let bytes = identifier.to_bytes();
- Pubkey::new_from_array(bytes)
- }));
-
- let mut all_price_keys_dedup = all_price_keys_iter.collect::<HashSet<Pubkey>>();
-
- // query all the keys and assemvle them into the view
-
- let mut remaining_product_keys = all_product_keys_dedup.clone();
-
- for product_key in all_product_keys_dedup {
- let _product_data = global_data.product_accounts.remove(&product_key);
-
- if let Some(mut product_metadata) = global_metadata
- .product_accounts_metadata
- .remove(&product_key)
- {
- let mut symbol_name = product_metadata
- .attr_dict
- .get("symbol")
- .cloned()
- // Use product key for unnamed products
- .unwrap_or(format!("unnamed product {}", product_key));
-
- // Sort and deduplicate prices
- let this_product_price_keys_dedup = product_metadata
- .price_accounts
- .drain(0..)
- .collect::<BTreeSet<_>>();
-
- let mut prices = BTreeMap::new();
-
- // Extract information about each price
- for price_key in this_product_price_keys_dedup {
- let price_global_data = global_data.price_accounts.remove(&price_key);
- let price_global_metadata =
- global_metadata.price_accounts_metadata.remove(&price_key);
-
- let price_identifier = Identifier::new(price_key.to_bytes());
- let price_local_data = local_data.remove(&price_identifier);
-
- prices.insert(
- price_key,
- DashboardPriceView {
- local_data: price_local_data,
- global_data: price_global_data,
- _global_metadata: price_global_metadata,
- },
- );
- // Mark this price as done
- all_price_keys_dedup.remove(&price_key);
- }
-
- // Mark this product as done
- remaining_product_keys.remove(&product_key);
-
- let symbol_view = DashboardSymbolView {
- product: product_key,
- prices,
- };
-
- if ret.contains_key(&symbol_name) {
- let new_symbol_name = format!("{} (duplicate)", symbol_name);
-
- warn!(logger, "Dashboard: duplicate symbol name detected, renaming";
- "symbol_name" => &symbol_name,
- "symbol_renamed_to" => &new_symbol_name,
- "conflicting_symbol_data" => format!("{:?}", symbol_view),
- );
-
- symbol_name = new_symbol_name;
- }
-
- ret.insert(symbol_name, symbol_view);
- } else {
- // This logging handles only missing products that we
- // should have found. Missing prices are okay, appearing
- // in cases where no on-chain queries or publishing took
- // place yet.
- warn!(logger, "Dashboard: Failed to look up product metadata"; "product_id" => product_key.to_string());
- }
- }
-
- if !(all_price_keys_dedup.is_empty() && remaining_product_keys.is_empty()) {
- let remaining_products: Vec<_> = remaining_product_keys.drain().collect();
- let remaining_prices: Vec<_> = all_price_keys_dedup.drain().collect();
- warn!(logger, "Dashboard: Orphaned product/price IDs detected";
- "remaining_product_ids" => format!("{:?}", remaining_products),
- "remaining_price_ids" => format!("{:?}", remaining_prices));
- }
-
- ret
-}
diff --git a/src/agent/metrics.rs b/src/agent/metrics.rs
index 0beb932..b806840 100644
--- a/src/agent/metrics.rs
+++ b/src/agent/metrics.rs
@@ -65,7 +65,7 @@ lazy_static! {
}
/// Internal metrics server state, holds state needed for serving
-/// dashboard and metrics.
+/// metrics.
pub struct MetricsServer {
pub start_time: Instant,
pub logger: Logger,
@@ -73,7 +73,7 @@ pub struct MetricsServer {
}
impl MetricsServer {
- /// Instantiate a metrics API with a dashboard
+ /// Instantiate a metrics API.
pub async fn spawn(addr: impl Into<SocketAddr> + 'static, logger: Logger, adapter: Arc<State>) {
let server = MetricsServer {
start_time: Instant::now(),
@@ -82,56 +82,30 @@ impl MetricsServer {
};
let shared_state = Arc::new(Mutex::new(server));
-
- let shared_state4dashboard = shared_state.clone();
- let dashboard_route = warp::path("dashboard")
- .or(warp::path::end())
- .and_then(move |_| {
- let shared_state = shared_state4dashboard.clone();
- async move {
- let locked_state = shared_state.lock().await;
- let response = locked_state
- .render_dashboard() // Defined in a separate impl block near dashboard-specific code
- .await
- .unwrap_or_else(|e| {
- // Add logging here
- error!(locked_state.logger,"Dashboard: Rendering failed"; "error" => e.to_string());
-
- // Withhold failure details from client
- "Could not render dashboard! See the logs for details".to_owned()
- });
- Result::<Box<dyn Reply>, Rejection>::Ok(Box::new(reply::with_status(
- reply::html(response),
- StatusCode::OK,
- )))
- }
- });
-
let shared_state4metrics = shared_state.clone();
let metrics_route = warp::path("metrics")
.and(warp::path::end())
.and_then(move || {
let shared_state = shared_state4metrics.clone();
async move {
- let locked_state = shared_state.lock().await;
+ let locked_state = shared_state.lock().await;
let mut buf = String::new();
- let response = encode(&mut buf, &&PROMETHEUS_REGISTRY.lock().await).map_err(|e| -> Box<dyn std::error::Error> {e.into()
- }).and_then(|_| -> Result<_, Box<dyn std::error::Error>> {
-
- Ok(Box::new(reply::with_status(buf, StatusCode::OK)))
- }).unwrap_or_else(|e| {
- error!(locked_state.logger, "Metrics: Could not gather metrics from registry"; "error" => e.to_string());
-
- Box::new(reply::with_status("Could not gather metrics. See logs for details".to_string(), StatusCode::INTERNAL_SERVER_ERROR))
- });
+ let response = encode(&mut buf, &&PROMETHEUS_REGISTRY.lock().await)
+ .map_err(|e| -> Box<dyn std::error::Error> {
+ e.into()
+ })
+ .and_then(|_| -> Result<_, Box<dyn std::error::Error>> {
+ Ok(Box::new(reply::with_status(buf, StatusCode::OK)))
+ }).unwrap_or_else(|e| {
+ error!(locked_state.logger, "Metrics: Could not gather metrics from registry"; "error" => e.to_string());
+ Box::new(reply::with_status("Could not gather metrics. See logs for details".to_string(), StatusCode::INTERNAL_SERVER_ERROR))
+ });
- Result::<Box<dyn Reply>, Rejection>::Ok(response)
+ Result::<Box<dyn Reply>, Rejection>::Ok(response)
}
});
- warp::serve(dashboard_route.or(metrics_route))
- .bind(addr)
- .await;
+ warp::serve(metrics_route).bind(addr).await;
}
}