From f08e2f6223982996ffecd93e81a6942b5584b6db Mon Sep 17 00:00:00 2001 From: Theo Butler Date: Wed, 15 Nov 2023 12:13:56 -0500 Subject: [PATCH] refactor: UDecimal18 --- Cargo.lock | 6 +- Cargo.toml | 1 - graph-gateway/Cargo.toml | 3 +- graph-gateway/src/budgets.rs | 59 ++-- graph-gateway/src/client_query.rs | 32 +- graph-gateway/src/config.rs | 9 +- graph-gateway/src/exchange_rate.rs | 34 +- graph-gateway/src/main.rs | 16 +- graph-gateway/src/network_subgraph.rs | 17 +- graph-gateway/src/receipts.rs | 20 +- graph-gateway/src/subgraph_studio.rs | 6 +- graph-gateway/src/topology.rs | 7 +- graph-gateway/src/vouchers.rs | 31 +- indexer-selection/Cargo.toml | 2 - indexer-selection/bin/sim.rs | 23 +- indexer-selection/src/actor.rs | 10 +- indexer-selection/src/economic_security.rs | 97 +----- indexer-selection/src/fee.rs | 39 ++- indexer-selection/src/lib.rs | 22 +- indexer-selection/src/score.rs | 2 +- indexer-selection/src/simulation.rs | 6 +- indexer-selection/src/test.rs | 39 ++- indexer-selection/src/test_utils.rs | 2 +- prelude/Cargo.toml | 3 +- prelude/src/decimal.rs | 355 +++++++-------------- prelude/src/lib.rs | 30 +- 26 files changed, 318 insertions(+), 553 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ae1d9c157..8f923e563 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1722,6 +1722,7 @@ dependencies = [ "prost", "rand", "rdkafka", + "receipts", "regex", "reqwest", "secp256k1", @@ -2084,8 +2085,6 @@ dependencies = [ "prelude", "rand", "rand_distr", - "receipts", - "secp256k1", "siphasher", "tokio", "toolshed", @@ -2907,9 +2906,8 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" name = "prelude" version = "0.0.1" dependencies = [ + "alloy-primitives", "anyhow", - "primitive-types", - "serde", "siphasher", "snmalloc-rs", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 011572dcf..a814a12d6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,6 @@ reqwest = { version = "0.11", default-features = false, features = [ "default-tls", "gzip", ] } -secp256k1 = { version = "0.24", default-features = false } serde = { version = "1.0", features = ["derive"] } siphasher = "1.0.0" tokio = { version = "1.24", features = [ diff --git a/graph-gateway/Cargo.toml b/graph-gateway/Cargo.toml index 9275bdb79..84f97b1f6 100644 --- a/graph-gateway/Cargo.toml +++ b/graph-gateway/Cargo.toml @@ -32,8 +32,9 @@ prometheus = { version = "0.13", default-features = false } prost = "0.12.1" rand.workspace = true rdkafka = { version = "0.36.0", features = ["gssapi", "tracing"] } +receipts = { git = "ssh://git@github.com/edgeandnode/receipts.git", rev = "89a821c" } reqwest.workspace = true -secp256k1.workspace = true +secp256k1 = { version = "0.24", default-features = false } semver = "1.0" serde.workspace = true serde_json = { version = "1.0", features = ["raw_value"] } diff --git a/graph-gateway/src/budgets.rs b/graph-gateway/src/budgets.rs index d769161da..ac731346b 100644 --- a/graph-gateway/src/budgets.rs +++ b/graph-gateway/src/budgets.rs @@ -5,7 +5,7 @@ use indexer_selection::{ decay::{Decay, FastDecayBuffer}, impl_struct_decay, }; -use prelude::USD; +use prelude::{UDecimal18, USD}; use tokio::{ select, spawn, sync::mpsc, @@ -34,7 +34,7 @@ pub struct Feedback { impl Budgeter { pub fn new(query_fees_target: USD) -> Self { - assert!(query_fees_target.as_f64() >= MAX_DISCOUNT_USD); + assert!(f64::from(query_fees_target.0) >= MAX_DISCOUNT_USD); let (feedback_tx, feedback_rx) = mpsc::unbounded_channel(); let (budgets_tx, budgets_rx) = Eventual::new(); Actor::create(feedback_rx, budgets_tx, query_fees_target); @@ -51,7 +51,7 @@ impl Budgeter { .value_immediate() .and_then(|budgets| budgets.get(deployment).copied()) .unwrap_or(self.query_fees_target); - budget * USD::try_from(query_count).unwrap() + USD(budget.0 * UDecimal18::from(query_count as u128)) } } @@ -109,17 +109,17 @@ impl Actor { } let target = self.controller.target_query_fees; let control_variable = self.controller.control_variable(); - tracing::debug!(budget_control_variable = %control_variable); + tracing::debug!(budget_control_variable = ?control_variable); let now = Instant::now(); let budgets = self .volume_estimators .iter() .map(|(deployment, volume_estimator)| { let volume = volume_estimator.monthly_volume_estimate(now) as u64; - let mut budget = volume_discount(volume, target) * control_variable; + let mut budget = volume_discount(volume, target).0 * control_variable; // limit budget to 100x target - budget = budget.min(target * USD::try_from(100_u64).unwrap()); - (*deployment, budget) + budget = budget.min(target.0 * UDecimal18::from(100)); + (*deployment, USD(budget)) }) .collect(); @@ -131,13 +131,13 @@ fn volume_discount(monthly_volume: u64, target: USD) -> USD { // Discount the budget, based on a generalized logistic function. We apply little to no discount // between 0 and ~10e3 queries per month. And we limit the discount to 10E-6 USD. // https://www.desmos.com/calculator/whtakt50sa - let b_max = target.as_f64(); + let b_max: f64 = target.0.into(); let b_min = b_max - MAX_DISCOUNT_USD; let m: f64 = 1e6; let z: f64 = 0.45; let v = monthly_volume as f64; let budget = b_min + ((b_max - b_min) * m.powf(z)) / (v + m).powf(z); - budget.try_into().unwrap_or_default() + USD(budget.try_into().unwrap_or_default()) } /// State for the control loop targeting `recent_query_fees`. @@ -152,32 +152,33 @@ impl Controller { fn new(target_query_fees: USD) -> Self { Self { target_query_fees, - recent_fees: USD::zero(), + recent_fees: USD(UDecimal18::from(0)), recent_query_count: 0, error_history: FastDecayBuffer::default(), } } fn add_queries(&mut self, fees: USD, query_count: u64) { - self.recent_fees += fees; + self.recent_fees = USD(self.recent_fees.0 + fees.0); self.recent_query_count += query_count; } - fn control_variable(&mut self) -> USD { + fn control_variable(&mut self) -> UDecimal18 { // See the following link if you're unfamiliar with PID controllers: // https://en.wikipedia.org/wiki/Proportional%E2%80%93integral%E2%80%93derivative_controller - let process_variable = self.recent_fees.as_f64() / self.recent_query_count.max(1) as f64; + let process_variable = + f64::from(self.recent_fees.0) / self.recent_query_count.max(1) as f64; METRICS.avg_query_fees.set(process_variable); - self.recent_fees = USD::zero(); + self.recent_fees = USD(UDecimal18::from(0)); self.recent_query_count = 0; self.error_history.decay(); - let error = self.target_query_fees.as_f64() - process_variable; + let error = f64::from(self.target_query_fees.0) - process_variable; *self.error_history.current_mut() = error; let i: f64 = self.error_history.frames().iter().sum(); let k_i = 3e4; - USD::try_from(1.0).unwrap() + USD::try_from(i * k_i).unwrap_or(USD::zero()) + UDecimal18::from(1) + UDecimal18::try_from(i * k_i).unwrap_or_default() } } @@ -332,32 +333,32 @@ mod tests { process_variable_multiplier: f64, tolerance: f64, ) { - let setpoint = controller.target_query_fees.as_f64(); - let mut process_variable = USD::zero(); + let setpoint: f64 = controller.target_query_fees.0.into(); + let mut process_variable = 0.0; for i in 0..30 { - let control_variable = controller.control_variable(); - process_variable = controller.target_query_fees - * USD::try_from(process_variable_multiplier).unwrap() + let control_variable: f64 = controller.control_variable().into(); + process_variable = f64::from(controller.target_query_fees.0) + * process_variable_multiplier * control_variable; println!( "{i:02} SP={setpoint:.6}, PV={:.8}, CV={:.8}", - process_variable.as_f64(), - control_variable.as_f64(), + process_variable, control_variable, ); - controller.add_queries(process_variable, 1); + controller.add_queries(USD(UDecimal18::try_from(process_variable).unwrap()), 1); } - assert_within(process_variable.as_f64(), setpoint, tolerance); + assert_within(process_variable, setpoint, tolerance); } for setpoint in [20e-6, 40e-6] { - let mut controller = Controller::new(USD::try_from(setpoint).unwrap()); + let setpoint = USD(UDecimal18::try_from(setpoint).unwrap()); + let mut controller = Controller::new(setpoint); test_controller(&mut controller, 0.2, 1e-6); - let mut controller = Controller::new(USD::try_from(setpoint).unwrap()); + let mut controller = Controller::new(setpoint); test_controller(&mut controller, 0.6, 1e-6); - let mut controller = Controller::new(USD::try_from(setpoint).unwrap()); + let mut controller = Controller::new(setpoint); test_controller(&mut controller, 0.8, 1e-6); - let mut controller = Controller::new(USD::try_from(setpoint).unwrap()); + let mut controller = Controller::new(setpoint); test_controller(&mut controller, 0.2, 1e-6); test_controller(&mut controller, 0.6, 1e-6); test_controller(&mut controller, 0.7, 1e-6); diff --git a/graph-gateway/src/client_query.rs b/graph-gateway/src/client_query.rs index 306354dc1..5345f5432 100644 --- a/graph-gateway/src/client_query.rs +++ b/graph-gateway/src/client_query.rs @@ -24,7 +24,7 @@ use eventuals::{Eventual, Ptr}; use futures::future::join_all; use graphql::graphql_parser::query::{OperationDefinition, SelectionSet}; use lazy_static::lazy_static; -use prelude::USD; +use prelude::{UDecimal18, USD}; use prost::bytes::Buf; use rand::{rngs::SmallRng, SeedableRng as _}; use serde::Deserialize; @@ -484,19 +484,20 @@ async fn handle_client_query_inner( // This `.min` prevents the budget from being set far beyond what it would be // automatically. The reason this is important is because sometimes queries are // subsidized and we would be at-risk to allow arbitrarily high values. - budget = user_budget.min(budget * USD::try_from(10_u64).unwrap()); + budget = USD(user_budget.0.min(budget.0 * UDecimal18::from(10))); // TOOD: budget = user_budget.max(budget * USD::try_from(0.1_f64).unwrap()); } - let budget: GRT = ctx + let grt_per_usd = ctx .isa_state .latest() .network_params - .usd_to_grt(budget) + .grt_per_usd .ok_or_else(|| Error::Internal(anyhow!("Missing exchange rate")))?; + let budget = GRT(budget.0 * grt_per_usd.0); tracing::info!( target: reports::CLIENT_QUERY_TARGET, query_count = budget_query_count, - budget_grt = budget.as_f64() as f32, + budget_grt = f64::from(budget.0) as f32, ); let mut utility_params = UtilityParameters { @@ -510,7 +511,7 @@ async fn handle_client_query_inner( let mut rng = SmallRng::from_entropy(); let mut total_indexer_queries = 0; - let mut total_indexer_fees = GRT::zero(); + let mut total_indexer_fees = GRT(UDecimal18::from(0)); // Used to track how many times an indexer failed to resolve a block. This may indicate that // our latest block has been uncled. let mut latest_unresolved: u32 = 0; @@ -580,10 +581,10 @@ async fn handle_client_query_inner( // Double the budget & retry if there is any indexer requesting a higher fee. if !last_retry && isa_errors.contains_key(&IndexerSelectionError::FeeTooHigh) { - utility_params.budget = utility_params.budget * GRT::try_from(2_u64).unwrap(); + utility_params.budget = GRT(utility_params.budget.0 * UDecimal18::from(2)); tracing::info!( target: reports::CLIENT_QUERY_TARGET, - budget_grt = budget.as_f64() as f32, + budget_grt = f64::from(budget.0) as f32, "increase_budget" ); continue; @@ -595,10 +596,10 @@ async fn handle_client_query_inner( ))); } - total_indexer_fees += selections.iter().map(|s| s.fee).sum(); + total_indexer_fees = GRT(total_indexer_fees.0 + selections.iter().map(|s| s.fee.0).sum()); tracing::info!( target: reports::CLIENT_QUERY_TARGET, - indexer_fees_grt = total_indexer_fees.as_f64() as f32, + indexer_fees_grt = f64::from(total_indexer_fees.0) as f32, ); // The gateway's current strategy for predicting is optimized for keeping responses close to chain head. We've @@ -687,12 +688,7 @@ async fn handle_client_query_inner( Some(Err(_)) | None => (), Some(Ok(outcome)) => { if !ignore_budget_feedback { - let total_indexer_fees: USD = ctx - .isa_state - .latest() - .network_params - .grt_to_usd(total_indexer_fees) - .unwrap(); + let total_indexer_fees: USD = USD(total_indexer_fees.0 / grt_per_usd.0); let _ = ctx.budgeter.feedback.send(budgets::Feedback { deployment: budget_deployment, fees: total_indexer_fees, @@ -738,13 +734,13 @@ async fn handle_indexer_query( %deployment, url = %selection.url, blocks_behind = selection.blocks_behind, - fee_grt = selection.fee.as_f64() as f32, + fee_grt = f64::from(selection.fee.0) as f32, subgraph_chain = %ctx.deployment.manifest.network, ); let receipt = ctx .receipt_signer - .create_receipt(&selection.indexing, selection.fee) + .create_receipt(&selection.indexing, &selection.fee) .await .ok_or(IndexerError::NoAllocation); diff --git a/graph-gateway/src/config.rs b/graph-gateway/src/config.rs index 65ad2308e..7f1d53289 100644 --- a/graph-gateway/src/config.rs +++ b/graph-gateway/src/config.rs @@ -5,14 +5,13 @@ use std::{collections::BTreeMap, fmt, path::PathBuf}; use alloy_primitives::{Address, U256}; use graph_subscriptions::subscription_tier::{SubscriptionTier, SubscriptionTiers}; +use prelude::UDecimal18; +use secp256k1::SecretKey; use semver::Version; use serde::Deserialize; use serde_with::{serde_as, DisplayFromStr, FromInto}; use toolshed::url::Url; -use indexer_selection::SecretKey; -use prelude::USD; - use crate::chains::ethereum; use crate::poi::ProofOfIndexingInfo; @@ -110,8 +109,10 @@ impl From for ethereum::Provider { #[derive(Debug, Deserialize)] #[serde(untagged)] pub enum ExchangeRateProvider { + /// Ethereum RPC provider Rpc(#[serde_as(as = "DisplayFromStr")] Url), - Fixed(USD), + /// Fixed conversion rate of GRT/USD + Fixed(#[serde_as(as = "DisplayFromStr")] UDecimal18), } #[derive(Debug, Deserialize)] diff --git a/graph-gateway/src/exchange_rate.rs b/graph-gateway/src/exchange_rate.rs index 2e2286e97..61258de83 100644 --- a/graph-gateway/src/exchange_rate.rs +++ b/graph-gateway/src/exchange_rate.rs @@ -1,25 +1,25 @@ use std::sync::Arc; use std::time::Duration; -use anyhow::{anyhow, ensure}; +use alloy_primitives::U256; +use anyhow::ensure; use ethers::{ abi::Address, prelude::{abigen, Http}, providers::Provider, }; use eventuals::{Eventual, EventualExt, EventualWriter}; +use prelude::{UDecimal18, GRT}; use tokio::sync::Mutex; use toolshed::url::Url; -use prelude::{UDecimal, USD}; - abigen!( UniswapV3Pool, "graph-gateway/src/contract_abis/UniswapV3Pool.json", event_derives(serde::Deserialize, serde::Serialize); ); -pub async fn usd_to_grt(provider: Url) -> anyhow::Result> { +pub async fn grt_per_usd(provider: Url) -> anyhow::Result> { // https://info.uniswap.org/#/pools/0x4d1fb02fa84eda35881902e8e0fdacc3a873398b let uniswap_v3_pool: Address = "0x4d1Fb02fa84EdA35881902e8E0fdacC3a873398B" .parse() @@ -38,14 +38,14 @@ pub async fn usd_to_grt(provider: Url) -> anyhow::Result> { ensure!(pool.token_1().await.unwrap() == grt); let (writer, reader) = Eventual::new(); - let writer: &'static Mutex> = Box::leak(Box::new(Mutex::new(writer))); + let writer: &'static Mutex> = Box::leak(Box::new(Mutex::new(writer))); eventuals::timer(Duration::from_secs(60)) .pipe_async(move |_| async { - match fetch_usd_to_grt(pool).await { - Err(usd_to_grt_err) => tracing::error!(%usd_to_grt_err), - Ok(usd_to_grt) => { - tracing::info!(%usd_to_grt); - writer.lock().await.write(usd_to_grt); + match fetch_grt_per_usd(pool).await { + Err(grt_per_usd_err) => tracing::error!(%grt_per_usd_err), + Ok(grt_per_usd) => { + tracing::info!(grt_per_usd = %grt_per_usd.0); + writer.lock().await.write(grt_per_usd); } }; }) @@ -53,17 +53,17 @@ pub async fn usd_to_grt(provider: Url) -> anyhow::Result> { Ok(reader) } -async fn fetch_usd_to_grt(pool: &UniswapV3Pool>) -> anyhow::Result { - const GRT_DECIMALS: u8 = 18; - const USDC_DECIMALS: u8 = 6; +async fn fetch_grt_per_usd(pool: &UniswapV3Pool>) -> anyhow::Result { + const GRT_DECIMALS: u32 = 18; + const USDC_DECIMALS: u32 = 6; // https://docs.uniswap.org/contracts/v3/reference/core/interfaces/pool/IUniswapV3PoolDerivedState#observe // token1/token0 -> GRT/USDC let twap_seconds: u32 = 60 * 20; let (tick_cumulatives, _) = pool.observe(vec![twap_seconds, 0]).await?; ensure!(tick_cumulatives.len() == 2); let tick = (tick_cumulatives[1] - tick_cumulatives[0]) / twap_seconds as i64; - let price = UDecimal::<0>::try_from(1.0001_f64.powi(tick as i32) as u128) - .map_err(|err| anyhow!(err))? - .shift::<{ GRT_DECIMALS - USDC_DECIMALS }>(); - Ok(price.change_precision()) + let price = U256::try_from(1.0001_f64.powi(tick as i32)).unwrap(); + let shift = U256::from(10_u128.pow(18 - (GRT_DECIMALS - USDC_DECIMALS))); + let price = UDecimal18::from_raw_u256(price * shift); + Ok(GRT(price)) } diff --git a/graph-gateway/src/main.rs b/graph-gateway/src/main.rs index 1023e7b82..344a0d05c 100644 --- a/graph-gateway/src/main.rs +++ b/graph-gateway/src/main.rs @@ -46,8 +46,7 @@ use graph_gateway::{ vouchers, JsonResponse, }; use indexer_selection::{actor::Update, BlockStatus, Indexing}; -use prelude::buffer_queue::QueueWriter; -use prelude::{buffer_queue, double_buffer}; +use prelude::{buffer_queue::QueueWriter, *}; use secp256k1::SecretKey; // Moving the `exchange_rate` module to `lib.rs` makes the doctests to fail during the compilation @@ -118,14 +117,14 @@ async fn main() { .build() .unwrap(); - let usd_to_grt = match config.exchange_rate_provider { - ExchangeRateProvider::Fixed(usd_to_grt) => Eventual::from_value(usd_to_grt), - ExchangeRateProvider::Rpc(url) => exchange_rate::usd_to_grt(url).await.unwrap(), + let grt_per_usd: Eventual = match config.exchange_rate_provider { + ExchangeRateProvider::Fixed(grt_per_usd) => Eventual::from_value(GRT(grt_per_usd)), + ExchangeRateProvider::Rpc(url) => exchange_rate::grt_per_usd(url).await.unwrap(), }; update_from_eventual( - usd_to_grt.clone(), + grt_per_usd.clone(), update_writer.clone(), - Update::USDToGRTConversion, + Update::GRTPerUSD, ); let network_subgraph_client = @@ -268,11 +267,12 @@ async fn main() { let query_fees_target = config .query_fees_target .try_into() + .map(USD) .expect("invalid query_fees_target"); let budgeter: &'static Budgeter = Box::leak(Box::new(Budgeter::new(query_fees_target))); tracing::info!("Waiting for exchange rate..."); - usd_to_grt.value().await.unwrap(); + grt_per_usd.value().await.unwrap(); tracing::info!("Waiting for ISA setup..."); update_writer.flush().await.unwrap(); diff --git a/graph-gateway/src/network_subgraph.rs b/graph-gateway/src/network_subgraph.rs index 59302a68f..304f51958 100644 --- a/graph-gateway/src/network_subgraph.rs +++ b/graph-gateway/src/network_subgraph.rs @@ -1,10 +1,10 @@ use std::sync::Arc; use std::time::Duration; -use alloy_primitives::Address; +use alloy_primitives::{Address, U256}; use anyhow::anyhow; use eventuals::{self, Eventual, EventualExt as _, EventualWriter, Ptr}; -use prelude::{GRTWei, PPM}; +use prelude::*; use serde::Deserialize; use serde_json::json; use tokio::sync::Mutex; @@ -18,7 +18,7 @@ pub struct Data { } pub struct NetworkParams { - pub slashing_percentage: PPM, + pub slashing_percentage: UDecimal18, } #[derive(Debug, Deserialize)] @@ -50,7 +50,7 @@ pub struct SubgraphDeployment { #[serde(rename_all = "camelCase")] pub struct Allocation { pub id: Address, - pub allocated_tokens: GRTWei, + pub allocated_tokens: U256, pub indexer: Indexer, } @@ -59,7 +59,7 @@ pub struct Allocation { pub struct Indexer { pub id: Address, pub url: Option, - pub staked_tokens: GRTWei, + pub staked_tokens: U256, } pub struct Client { @@ -123,10 +123,9 @@ impl Client { .ok_or_else(|| anyhow!("Discarding empty update (graphNetwork)"))?; Ok(NetworkParams { - slashing_percentage: response - .slashing_percentage - .try_into() - .map_err(|_| anyhow!("Failed to parse slashingPercentage"))?, + slashing_percentage: UDecimal18::from_raw_u256( + U256::from(response.slashing_percentage) * U256::from(1_000_000_000_000_u128), + ), }) } diff --git a/graph-gateway/src/receipts.rs b/graph-gateway/src/receipts.rs index 74a4ecd15..560402bcc 100644 --- a/graph-gateway/src/receipts.rs +++ b/graph-gateway/src/receipts.rs @@ -1,5 +1,4 @@ -use std::collections::HashSet; -use std::collections::{hash_map::Entry, HashMap}; +use std::collections::{hash_map::Entry, HashMap, HashSet}; use std::sync::Arc; use std::time::SystemTime; @@ -7,16 +6,15 @@ use alloy_primitives::{Address, U256}; use alloy_sol_types::Eip712Domain; use ethers::signers::Wallet; use eventuals::{Eventual, Ptr}; +use indexer_selection::Indexing; +use prelude::GRT; use rand::RngCore; +pub use receipts::{QueryStatus as ReceiptStatus, ReceiptPool}; +use secp256k1::SecretKey; use tap_core::eip_712_signed_message::EIP712SignedMessage; use tap_core::tap_receipt::Receipt; use tokio::sync::{Mutex, RwLock}; -pub use indexer_selection::receipts::QueryStatus as ReceiptStatus; -use indexer_selection::receipts::ReceiptPool; -use indexer_selection::{Indexing, SecretKey}; -use prelude::GRT; - pub struct ReceiptSigner { signer: SecretKey, domain: Eip712Domain, @@ -72,7 +70,7 @@ impl ReceiptSigner { } } - pub async fn create_receipt(&self, indexing: &Indexing, fee: GRT) -> Option { + pub async fn create_receipt(&self, indexing: &Indexing, fee: &GRT) -> Option { if self .legacy_indexers .value_immediate() @@ -82,7 +80,9 @@ impl ReceiptSigner { let legacy_pools = self.legacy_pools.read().await; let legacy_pool = legacy_pools.get(indexing)?; let mut legacy_pool = legacy_pool.lock().await; - let receipt = legacy_pool.commit(fee.shift::<0>().as_u256()).ok()?; + let locked_fee = + primitive_types::U256::from_little_endian(&fee.0.raw_u256().as_le_bytes()); + let receipt = legacy_pool.commit(locked_fee).ok()?; return Some(ScalarReceipt::Legacy(receipt)); } @@ -99,7 +99,7 @@ impl ReceiptSigner { allocation_id: allocation.0 .0.into(), timestamp_ns, nonce, - value: fee.shift::<0>().as_u256().as_u128(), + value: fee.0.as_u128().unwrap_or(0), }; let wallet = Wallet::from_bytes(self.signer.as_ref()).expect("failed to prepare receipt wallet"); diff --git a/graph-gateway/src/subgraph_studio.rs b/graph-gateway/src/subgraph_studio.rs index c9e084132..1bcdaf858 100644 --- a/graph-gateway/src/subgraph_studio.rs +++ b/graph-gateway/src/subgraph_studio.rs @@ -2,7 +2,7 @@ use std::{collections::HashMap, error::Error, sync::Arc}; use alloy_primitives::Address; use eventuals::{self, Eventual, EventualExt as _, EventualWriter, Ptr}; -use prelude::USD; +use prelude::{UDecimal18, USD}; use serde::Deserialize; use tokio::{sync::Mutex, time::Duration}; use toolshed::thegraph::{DeploymentId, SubgraphId}; @@ -86,7 +86,9 @@ impl Client { user_id: api_key.user_id, user_address: api_key.user_address.parse().ok()?, query_status: api_key.query_status, - max_budget: api_key.max_budget.and_then(|b| USD::try_from(b).ok()), + max_budget: api_key + .max_budget + .and_then(|b| UDecimal18::try_from(b).ok().map(USD)), subgraphs: api_key .subgraphs .into_iter() diff --git a/graph-gateway/src/topology.rs b/graph-gateway/src/topology.rs index ce5ff2ed8..2cca39cb1 100644 --- a/graph-gateway/src/topology.rs +++ b/graph-gateway/src/topology.rs @@ -193,16 +193,17 @@ impl GraphNetwork { Indexer { id, url, - staked_tokens: allocation.indexer.staked_tokens.change_precision(), + staked_tokens: GRT(allocation.indexer.staked_tokens.into()), largest_allocation: allocation.id, - allocated_tokens: allocation.allocated_tokens.change_precision(), + allocated_tokens: GRT(allocation.allocated_tokens.into()), }, )) }) .into_group_map() // TODO: remove need for itertools here: https://github.com/rust-lang/rust/issues/80552 .into_iter() .filter_map(|(_, mut allocations)| { - let total_allocation: GRT = allocations.iter().map(|a| a.allocated_tokens).sum(); + let total_allocation: GRT = + GRT(allocations.iter().map(|a| a.allocated_tokens.0).sum()); // last allocation is latest: 9936786a-e286-45f3-9190-8409d8389e88 let mut indexer = allocations.pop()?; indexer.allocated_tokens = total_allocation; diff --git a/graph-gateway/src/vouchers.rs b/graph-gateway/src/vouchers.rs index 231d39de0..8e6192de6 100644 --- a/graph-gateway/src/vouchers.rs +++ b/graph-gateway/src/vouchers.rs @@ -1,18 +1,12 @@ -use std::str::FromStr; - -use alloy_primitives::{Address, FixedBytes}; +use alloy_primitives::{Address, FixedBytes, U256}; use axum::{body::Bytes, extract::State, http::StatusCode}; use lazy_static::lazy_static; -use primitive_types::U256; use secp256k1::{PublicKey, Secp256k1}; -use serde::{Deserialize, Deserializer}; +use serde::Deserialize; use serde_json::json; -use indexer_selection::{ - receipts::{self, combine_partial_vouchers, receipts_to_partial_voucher, receipts_to_voucher}, - SecretKey, -}; -use prelude::GRTWei; +use receipts::{self, combine_partial_vouchers, receipts_to_partial_voucher, receipts_to_voucher}; +use secp256k1::SecretKey; use crate::{json_response, metrics::*, JsonResponse}; @@ -50,7 +44,7 @@ fn process_oneshot_voucher(signer: &SecretKey, payload: &Bytes) -> Result U256::from(10000000000000000000000000_u128) { + if voucher.fees > primitive_types::U256::from(10000000000000000000000000_u128) { tracing::error!(excessive_voucher_fees = %voucher.fees); return Err("Voucher value too large".into()); } @@ -95,7 +89,7 @@ fn process_partial_voucher(signer: &SecretKey, payload: &Bytes) -> Result U256::from(10000000000000000000000000u128) { + if partial_voucher.voucher.fees > primitive_types::U256::from(10000000000000000000000000u128) { tracing::error!(excessive_voucher_fees = %partial_voucher.voucher.fees); return Err("Voucher value too large".into()); } @@ -139,7 +133,7 @@ fn process_voucher(signer: &SecretKey, payload: &Bytes) -> Result Result U256::from(10000000000000000000000000u128) { + if voucher.fees > primitive_types::U256::from(10000000000000000000000000u128) { tracing::error!(excessive_voucher_fees = %voucher.fees); return Err("Voucher value too large".into()); } @@ -188,7 +182,6 @@ struct VoucherRequest { #[derive(Deserialize)] struct PartialVoucher { signature: Signature, - #[serde(deserialize_with = "deserialize_u256")] fees: U256, receipt_id_min: ReceiptID, receipt_id_max: ReceiptID, @@ -196,11 +189,3 @@ struct PartialVoucher { type Signature = FixedBytes<65>; type ReceiptID = FixedBytes<15>; - -fn deserialize_u256<'de, D: Deserializer<'de>>(deserializer: D) -> Result { - let input: &str = Deserialize::deserialize(deserializer)?; - // U256::from_str is busted, so use the equivalent decimals representation - Ok(GRTWei::from_str(input) - .map_err(serde::de::Error::custom)? - .as_u256()) -} diff --git a/indexer-selection/Cargo.toml b/indexer-selection/Cargo.toml index 02df6d928..5141dc97f 100644 --- a/indexer-selection/Cargo.toml +++ b/indexer-selection/Cargo.toml @@ -19,8 +19,6 @@ permutation = "0.4" prelude = { path = "../prelude" } rand.workspace = true rand_distr = "0.4" -receipts = { git = "ssh://git@github.com/edgeandnode/receipts.git", rev = "89a821c" } -secp256k1.workspace = true siphasher.workspace = true tokio.workspace = true toolshed.workspace = true diff --git a/indexer-selection/bin/sim.rs b/indexer-selection/bin/sim.rs index 375c27c6b..945d7ff72 100644 --- a/indexer-selection/bin/sim.rs +++ b/indexer-selection/bin/sim.rs @@ -21,17 +21,17 @@ async fn main() -> Result<()> { let fields = line.split(',').collect::>(); Some(IndexerCharacteristics { address: fields[0].parse().expect("address"), - fee: fields[1].parse().expect("fee"), + fee: GRT(fields[1].parse().expect("fee")), blocks_behind: fields[2].parse::().expect("blocks_behind").round() as u64, latency_ms: fields[3].parse::().expect("latency_ms").round() as u64, success_rate: fields[4].parse().expect("success_rate"), - allocation: fields[5].parse().expect("allocation"), - stake: fields[6].parse().expect("stake"), + allocation: GRT(fields[5].parse().expect("allocation")), + stake: GRT(fields[6].parse().expect("stake")), }) }) .collect::>(); - let budget = "0.01".parse().unwrap(); + let budget = GRT(UDecimal18::try_from(0.01).unwrap()); let freshness_requirements = BlockRequirements { range: None, has_latest: true, @@ -55,14 +55,11 @@ async fn main() -> Result<()> { for selection_limit in [1, 3] { let results = simulate(&characteristics, ¶ms, 100, selection_limit).await?; - let total_cost = results - .selections - .iter() - .fold(GRT::zero(), |sum, s| sum + s.fee); + let total_cost = GRT(results.selections.iter().map(|s| s.fee.0).sum()); eprintln!( "| {} | {:.6} | {:.0} | {:.2} | {:.2} | {:.2} |", selection_limit, - total_cost, + total_cost.0, results.avg_latency, results.avg_blocks_behind, results.selections.len() as f64 / results.client_queries as f64, @@ -77,12 +74,12 @@ async fn main() -> Result<()> { .collect::>(); let detail = format!( "fee={:.4} behind={:02} latency={:04} success={:.3} alloc={:1.0e} stake={:1.0e}", - indexer.fee.as_f64(), + f64::from(indexer.fee.0), indexer.blocks_behind, indexer.latency_ms, indexer.success_rate, - indexer.allocation.as_f64(), - indexer.stake.as_f64(), + f64::from(indexer.allocation.0), + f64::from(indexer.stake.0), ); println!( "selection_limit={},{},{},{},{}", @@ -90,7 +87,7 @@ async fn main() -> Result<()> { indexer.address, detail, selections.len(), - selections.iter().fold(GRT::zero(), |sum, s| sum + s.fee), + selections.iter().map(|s| s.fee.0).sum::(), ); } } diff --git a/indexer-selection/src/actor.rs b/indexer-selection/src/actor.rs index b35098e14..31cb19cf1 100644 --- a/indexer-selection/src/actor.rs +++ b/indexer-selection/src/actor.rs @@ -15,8 +15,8 @@ use crate::{IndexerErrorObservation, Indexing, IndexingStatus, State}; #[derive(Debug)] pub enum Update { - USDToGRTConversion(GRT), - SlashingPercentage(PPM), + GRTPerUSD(GRT), + SlashingPercentage(UDecimal18), Indexings(HashMap), QueryObservation { indexing: Indexing, @@ -65,12 +65,10 @@ pub async fn process_updates( pub fn apply_state_update(state: &mut State, update: &Update) { match update { - Update::USDToGRTConversion(usd_to_grt) => { - tracing::info!(%usd_to_grt); - state.network_params.usd_to_grt_conversion = Some(*usd_to_grt); + Update::GRTPerUSD(grt_per_usd) => { + state.network_params.grt_per_usd = Some(*grt_per_usd); } Update::SlashingPercentage(slashing_percentage) => { - tracing::info!(%slashing_percentage); state.network_params.slashing_percentage = Some(*slashing_percentage); } Update::Indexings(indexings) => { diff --git a/indexer-selection/src/economic_security.rs b/indexer-selection/src/economic_security.rs index c9dd169e3..fd12d62d5 100644 --- a/indexer-selection/src/economic_security.rs +++ b/indexer-selection/src/economic_security.rs @@ -2,25 +2,16 @@ use prelude::*; #[derive(Default)] pub struct NetworkParameters { - pub slashing_percentage: Option, - pub usd_to_grt_conversion: Option, + pub slashing_percentage: Option, + pub grt_per_usd: Option, } impl NetworkParameters { - pub fn usd_to_grt(&self, usd: USD) -> Option { - let conversion_rate = self.usd_to_grt_conversion?; - Some(usd * conversion_rate) - } - - pub fn grt_to_usd(&self, grt: GRT) -> Option { - let conversion_rate = self.usd_to_grt_conversion?; - Some(grt / conversion_rate) - } - pub fn slashable_usd(&self, indexer_stake: GRT) -> Option { let slashing_percentage = self.slashing_percentage?; - let slashable_grt = indexer_stake * slashing_percentage.change_precision(); - self.grt_to_usd(slashable_grt) + let slashable_grt = indexer_stake.0 * slashing_percentage; + let slashable_usd = slashable_grt / self.grt_per_usd?.0; + Some(USD(slashable_usd)) } } @@ -30,68 +21,6 @@ mod tests { use super::*; - #[test] - fn two_usd_to_grt() { - let params = NetworkParameters { - usd_to_grt_conversion: "0.511732966311998143".parse().ok(), - slashing_percentage: 0u64.try_into().ok(), - }; - // Check conversion of 2 USD to GRT - assert_eq!( - params.usd_to_grt(2u64.try_into().unwrap()), - Some("1.023465932623996286".parse().unwrap()) - ); - } - - #[test] - fn two_grt_to_usd() { - let params = NetworkParameters { - usd_to_grt_conversion: "0.511732966311998143".parse().ok(), - slashing_percentage: 0u64.try_into().ok(), - }; - // Check conversion of 2 GRT to USD - assert_eq!( - params.grt_to_usd(2u64.try_into().unwrap()), - Some("3.908288368470326937".parse().unwrap()) - ); - } - - #[test] - fn trillion_usd_to_grt() { - let mut params = NetworkParameters { - usd_to_grt_conversion: None, - slashing_percentage: 0u64.try_into().ok(), - }; - let trillion: USD = 10u64.pow(12).try_into().unwrap(); - // Assert None is returned if eventual has no value. - assert_eq!(params.usd_to_grt(trillion), None); - // Set conversion rate - params.usd_to_grt_conversion = "0.511732966311998143".parse().ok(); - // Check conversion of 1 trillion USD to GRT - assert_eq!( - params.usd_to_grt(trillion), - Some("511732966311.998143".parse().unwrap()) - ); - } - - #[test] - fn trillion_grt_to_usd() { - let mut params = NetworkParameters { - usd_to_grt_conversion: None, - slashing_percentage: 0u64.try_into().ok(), - }; - let trillion: GRT = 10u64.pow(12).try_into().unwrap(); - // Assert None is returned if eventual has no value. - assert_eq!(params.grt_to_usd(trillion), None); - // Set conversion rate - params.usd_to_grt_conversion = "0.511732966311998143".parse().ok(); - // Check conversion of 1 trillion GRT to USD - assert_eq!( - params.grt_to_usd(trillion), - Some("1954144184235.163468761907206198".parse().unwrap()) - ); - } - #[test] fn high_stake() { // $1m dollars amounts to ~80% utility @@ -118,20 +47,20 @@ mod tests { } fn test_economic_security_utility( - usd_to_grt: u64, + grt_per_usd: u128, slashing_percentage: f64, - stake: u64, + stake: u128, u_a: f64, - expected_slashable: u64, + expected_slashable: u128, expected_utility: f64, ) { let params = NetworkParameters { - usd_to_grt_conversion: usd_to_grt.try_into().ok(), - slashing_percentage: slashing_percentage.to_string().parse().ok(), + grt_per_usd: Some(GRT(UDecimal18::from(grt_per_usd))), + slashing_percentage: UDecimal18::try_from(slashing_percentage).ok(), }; - let slashable = params.slashable_usd(stake.try_into().unwrap()).unwrap(); - let utility = ConcaveUtilityParameters::one(u_a).concave_utility(slashable.as_f64()); - assert_eq!(slashable, expected_slashable.try_into().unwrap()); + let slashable = params.slashable_usd(GRT(UDecimal18::from(stake))).unwrap(); + let utility = ConcaveUtilityParameters::one(u_a).concave_utility(slashable.0.into()); + assert_eq!(slashable.0, UDecimal18::from(expected_slashable)); assert_within(utility.utility, expected_utility, 0.01); } } diff --git a/indexer-selection/src/fee.rs b/indexer-selection/src/fee.rs index 45ef4c7b1..d7cce912e 100644 --- a/indexer-selection/src/fee.rs +++ b/indexer-selection/src/fee.rs @@ -1,9 +1,10 @@ use std::convert::TryFrom; +use alloy_primitives::U256; use cost_model::{CostError, CostModel}; use eventuals::Ptr; -use prelude::{GRTWei, GRT}; +use prelude::{UDecimal18, GRT}; use crate::{utility::UtilityFactor, Context, IndexerError, InputError, SelectionError}; @@ -77,12 +78,13 @@ const WEIGHT: f64 = 0.5; // 7a3da342-c049-4ab0-8058-91880491b442 pub fn fee_utility(fee: &GRT, budget: &GRT) -> UtilityFactor { // Any fee over budget has zero utility. - if *fee > *budget { + if fee.0 > budget.0 { return UtilityFactor::one(0.0); } - let one_wei: GRT = GRTWei::try_from(1u64).unwrap().shift(); - let scaled_fee = *fee / budget.saturating_add(one_wei); - let mut utility = (scaled_fee.as_f64() + S).recip() - S; + let one_wei = UDecimal18::from_raw_u256(U256::from(1)); + let scaled_fee: f64 = (fee.0 / budget.0.saturating_add(one_wei)).into(); + println!("{} {}, {scaled_fee}", fee.0, budget.0); + let mut utility = (scaled_fee + S).recip() - S; // Set minimum utility, since small negative utility can result from loss of precision when the // fee approaches the budget. utility = utility.max(1e-18); @@ -105,7 +107,7 @@ fn min_optimal_fee(budget: &GRT) -> GRT { let w = WEIGHT; let mut min_rate = (4.0 * S.powi(2) * w + w.powi(2) - 2.0 * w + 1.0).sqrt(); min_rate = (min_rate - 2.0 * S.powi(2) - w + 1.0) / (2.0 * S); - *budget * GRT::try_from(min_rate).unwrap() + GRT(budget.0 * UDecimal18::try_from(min_rate).unwrap()) } pub fn indexer_fee( @@ -118,8 +120,11 @@ pub fn indexer_fee( .as_ref() .map(|model| model.cost_with_context(context)) { - None => GRT::zero(), - Some(Ok(fee)) => fee.to_string().parse::().unwrap().shift(), + None => GRT(UDecimal18::from(0)), + Some(Ok(fee)) => { + let fee = U256::try_from_be_slice(&fee.to_bytes_be()).unwrap_or(U256::MAX); + GRT(UDecimal18::from_raw_u256(fee)) + } Some(Err(CostError::CostModelFail | CostError::QueryNotCosted)) => { return Err(IndexerError::QueryNotCosted.into()); } @@ -134,16 +139,16 @@ pub fn indexer_fee( }; // Any fee over budget is refused. - if fee > *budget { + if fee.0 > budget.0 { return Err(IndexerError::FeeTooHigh.into()); } - let budget = *budget / GRT::try_from(max_indexers).unwrap(); + let budget = GRT(budget.0 / UDecimal18::from(max_indexers as u128)); let min_optimal_fee = min_optimal_fee(&budget); // If their fee is less than the min optimal, lerp between them so that // indexers are rewarded for being closer. - if fee < min_optimal_fee { - fee = (min_optimal_fee + fee) * GRT::try_from(0.75).unwrap(); + if fee.0 < min_optimal_fee.0 { + fee = GRT((min_optimal_fee.0 + fee.0) * UDecimal18::try_from(0.75).unwrap()); } Ok(fee) @@ -159,16 +164,20 @@ mod test { #[test] fn test() { - let cost_model = Some(Ptr::new(default_cost_model("0.01".parse().unwrap()))); + let cost_model = Some(Ptr::new(default_cost_model(GRT(UDecimal18::try_from( + 0.01, + ) + .unwrap())))); let mut context = Context::new(BASIC_QUERY, "").unwrap(); // Expected values based on https://www.desmos.com/calculator/kxd4kpjxi5 let tests = [(0.01, 0.0), (0.02, 0.27304), (0.1, 0.50615), (1.0, 0.55769)]; for (budget, expected_utility) in tests { - let budget = budget.to_string().parse::().unwrap(); + let budget = GRT(UDecimal18::try_from(budget).unwrap()); let fee = indexer_fee(&cost_model, &mut context, &budget, 1).unwrap(); let utility = fee_utility(&fee, &budget); + println!("{fee:?} / {budget:?}, {expected_utility}, {utility:?}"); let utility = utility.utility.powf(utility.weight); - assert!(fee >= "0.01".parse::().unwrap()); + assert!(fee.0 >= UDecimal18::try_from(0.01).unwrap()); assert_within(utility, expected_utility, 0.0001); } } diff --git a/indexer-selection/src/lib.rs b/indexer-selection/src/lib.rs index 6fd8de886..ea740e719 100644 --- a/indexer-selection/src/lib.rs +++ b/indexer-selection/src/lib.rs @@ -10,15 +10,12 @@ use alloy_primitives::{Address, BlockHash, BlockNumber}; pub use cost_model::{self, CostModel}; use num_traits::Zero as _; pub use ordered_float::NotNan; +use prelude::*; use rand::{prelude::SmallRng, Rng as _}; -pub use receipts; -pub use secp256k1::SecretKey; +use score::{expected_individual_score, ExpectedValue}; use toolshed::thegraph::{BlockPointer, DeploymentId}; use toolshed::url::Url; -use prelude::*; -use score::{expected_individual_score, ExpectedValue}; - pub use crate::{ economic_security::NetworkParameters, indexing::{BlockStatus, IndexingState, IndexingStatus}, @@ -27,7 +24,6 @@ pub use crate::{ }; use crate::{ fee::indexer_fee, - receipts::BorrowFail, score::{select_indexers, SelectionFactors}, }; @@ -111,14 +107,6 @@ pub enum IndexerErrorObservation { Other, } -impl From for IndexerError { - fn from(from: BorrowFail) -> Self { - match from { - BorrowFail::NoAllocation => Self::NoAllocation, - } - } -} - #[derive(Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] pub enum UnresolvedBlock { WithHash(BlockHash), @@ -288,7 +276,7 @@ impl State { return Err(IndexerError::MissingRequiredBlock.into()); } - if state.status.stake == GRT::zero() { + if state.status.stake == GRT(UDecimal18::from(0)) { return Err(IndexerError::NoStake.into()); } @@ -306,7 +294,7 @@ impl State { let reliability = state.reliability.expected_value(); let perf_success = state.perf_success.expected_value(); - let slashable_usd = slashable.as_f64(); + let slashable_usd = slashable.0.into(); let expected_score = NotNan::new(expected_individual_score( params, @@ -338,7 +326,7 @@ impl State { /// Sybil protection fn sybil(indexer_allocation: &GRT) -> Result, IndexerError> { - let identity = indexer_allocation.as_f64(); + let identity: f64 = indexer_allocation.0.into(); // There is a GIP out there which would allow for allocations with 0 GRT stake. // For example, MIPS. We don't want for those to never be selected. Furthermore, diff --git a/indexer-selection/src/score.rs b/indexer-selection/src/score.rs index 289adcdd5..fd2180f91 100644 --- a/indexer-selection/src/score.rs +++ b/indexer-selection/src/score.rs @@ -110,7 +110,7 @@ pub fn select_indexers( impl MetaIndexer<'_> { fn fee(&self) -> GRT { - self.0.iter().map(|f| f.fee).sum() + GRT(self.0.iter().map(|f| f.fee.0).sum()) } fn score(&self, params: &UtilityParameters) -> f64 { diff --git a/indexer-selection/src/simulation.rs b/indexer-selection/src/simulation.rs index 51ef71370..e8704076d 100644 --- a/indexer-selection/src/simulation.rs +++ b/indexer-selection/src/simulation.rs @@ -8,7 +8,7 @@ use rand::{prelude::SmallRng, Rng as _, SeedableRng as _}; use rand_distr::Normal; use prelude::test_utils::{bytes_from_id, init_test_tracing}; -use prelude::GRT; +use prelude::{UDecimal18, GRT}; use toolshed::thegraph::DeploymentId; use crate::test_utils::default_cost_model; @@ -53,8 +53,8 @@ pub async fn simulate( }; let mut isa = State::default(); - isa.network_params.slashing_percentage = "0.1".parse().ok(); - isa.network_params.usd_to_grt_conversion = "0.1".parse().ok(); + isa.network_params.slashing_percentage = UDecimal18::try_from(0.1).ok(); + isa.network_params.grt_per_usd = UDecimal18::try_from(0.1).ok().map(GRT); for characteristics in characteristics { let indexing = Indexing { diff --git a/indexer-selection/src/test.rs b/indexer-selection/src/test.rs index 031e363ec..12d640b58 100644 --- a/indexer-selection/src/test.rs +++ b/indexer-selection/src/test.rs @@ -15,7 +15,7 @@ use tokio::spawn; use prelude::buffer_queue::QueueWriter; use prelude::test_utils::bytes_from_id; -use prelude::{buffer_queue, double_buffer, GRT, PPM}; +use prelude::{buffer_queue, double_buffer, UDecimal18, GRT}; use toolshed::thegraph::{BlockPointer, DeploymentId}; use crate::actor::{process_updates, Update}; @@ -36,8 +36,8 @@ struct Config { #[derive(Debug)] struct Topology { - usd_to_grt_conversion: GRT, - slashing_percentage: PPM, + grt_per_usd: GRT, + slashing_percentage: UDecimal18, blocks: Vec, deployments: HashSet, indexings: HashMap, @@ -55,8 +55,8 @@ struct Request { fn base_indexing_status() -> IndexingStatus { IndexingStatus { url: "http://localhost:8000".parse().unwrap(), - stake: "1".parse().unwrap(), - allocation: "1".parse().unwrap(), + stake: GRT(UDecimal18::from(1)), + allocation: GRT(UDecimal18::from(1)), cost_model: None, block: Some(BlockStatus { reported_number: 0, @@ -100,15 +100,15 @@ impl Topology { .filter_map(|_| Self::gen_indexing(rng, config, &blocks, &deployments)) .collect(); let state = Self { - usd_to_grt_conversion: "1.0".parse().unwrap(), - slashing_percentage: "0.1".parse().unwrap(), + grt_per_usd: GRT(UDecimal18::from(1)), + slashing_percentage: UDecimal18::try_from(0.1).unwrap(), blocks, deployments, indexings, }; update_writer - .write(Update::USDToGRTConversion(state.usd_to_grt_conversion)) + .write(Update::GRTPerUSD(state.grt_per_usd)) .unwrap(); update_writer .write(Update::SlashingPercentage(state.slashing_percentage)) @@ -149,7 +149,7 @@ impl Topology { } fn gen_grt(rng: &mut SmallRng, table: &[f64; 4]) -> GRT { - GRT::try_from(*table.choose(rng).unwrap()).unwrap() + GRT(UDecimal18::try_from(*table.choose(rng).unwrap()).unwrap()) } fn gen_request(&self, rng: &mut SmallRng) -> Option { @@ -167,7 +167,7 @@ impl Topology { .map(|indexing| indexing.indexer) .collect(), params: utiliy_params( - "1.0".parse().unwrap(), + GRT(UDecimal18::from(1)), BlockRequirements { range: required_block.map(|b| (0, b)), has_latest: required_block.is_some() && rng.gen_bool(0.5), @@ -191,11 +191,8 @@ impl Topology { let mut context = Context::new(&request.query, "").unwrap(); - let fees = selections - .iter() - .map(|s| s.fee) - .fold(GRT::zero(), |sum, fee| sum + fee); - ensure!(fees <= request.params.budget); + let fees = GRT(selections.iter().map(|s| s.fee.0).sum()); + ensure!(fees.0 <= request.params.budget.0); let indexers_dedup: BTreeSet
= request.indexers.iter().copied().collect(); ensure!(indexers_dedup.len() == request.indexers.len()); @@ -226,9 +223,9 @@ impl Topology { set_err(IndexerError::MissingRequiredBlock); } else if status.block.is_none() { set_err(IndexerError::NoStatus); - } else if status.stake == GRT::zero() { + } else if status.stake == GRT(UDecimal18::from(0)) { set_err(IndexerError::NoStake); - } else if fee > request.params.budget.as_f64() { + } else if fee > request.params.budget.0.into() { set_err(IndexerError::FeeTooHigh); } } @@ -246,7 +243,7 @@ impl Topology { #[tokio::test] async fn fuzz() { - // init_tracing(false); + // crate::init_tracing(false); let seed = env::vars() .find(|(k, _)| k == "TEST_SEED") @@ -318,8 +315,8 @@ fn favor_higher_version() { let mut state = State { network_params: NetworkParameters { - usd_to_grt_conversion: Some("1".parse().unwrap()), - slashing_percentage: Some("0.1".parse().unwrap()), + grt_per_usd: Some(GRT(UDecimal18::from(1))), + slashing_percentage: Some(UDecimal18::try_from(0.1).unwrap()), }, indexings: HashMap::from_iter([]), }; @@ -339,7 +336,7 @@ fn favor_higher_version() { ); let params = utiliy_params( - "1".parse().unwrap(), + GRT(UDecimal18::from(1)), BlockRequirements { range: None, has_latest: true, diff --git a/indexer-selection/src/test_utils.rs b/indexer-selection/src/test_utils.rs index 04e139cb9..14fcabc77 100644 --- a/indexer-selection/src/test_utils.rs +++ b/indexer-selection/src/test_utils.rs @@ -12,7 +12,7 @@ use crate::{BlockPointer, CostModel}; pub const TEST_KEY: &str = "244226452948404D635166546A576E5A7234753778217A25432A462D4A614E64"; pub fn default_cost_model(fee: GRT) -> CostModel { - CostModel::compile(format!("default => {fee};"), "").unwrap() + CostModel::compile(format!("default => {};", fee.0), "").unwrap() } #[track_caller] diff --git a/prelude/Cargo.toml b/prelude/Cargo.toml index a69f05603..216dc45c4 100644 --- a/prelude/Cargo.toml +++ b/prelude/Cargo.toml @@ -4,9 +4,8 @@ name = "prelude" version = "0.0.1" [dependencies] +alloy-primitives.workspace = true anyhow.workspace = true -primitive-types = "0.12.1" -serde.workspace = true siphasher.workspace = true snmalloc-rs = "0.3" tokio = { version = "1.24", features = [ diff --git a/prelude/src/decimal.rs b/prelude/src/decimal.rs index 4fef6a63d..63b8f3d2d 100644 --- a/prelude/src/decimal.rs +++ b/prelude/src/decimal.rs @@ -1,71 +1,75 @@ -use std::{cmp::Ordering, fmt, iter, ops, str}; +use std::{fmt, iter, str}; -// TODO: replace with alloy-primitives::U256 -use primitive_types::U256; +use alloy_primitives::U256; -#[derive(Copy, Clone, Debug, Eq, PartialEq)] -pub enum ParseStrError { - InvalidInput, +const ONE_18: u128 = 1_000_000_000_000_000_000; + +/// Represents a positive decimal value with 18 fractional digits precision. Using U256 as storage. +#[derive(Copy, Clone, Default, Hash, Eq, PartialEq, Ord, PartialOrd)] +pub struct UDecimal18(U256); + +impl From for UDecimal18 { + fn from(value: U256) -> Self { + Self(U256::from(value) * U256::from(ONE_18)) + } } -impl fmt::Display for ParseStrError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "Failed to parse decimal value") +impl From for UDecimal18 { + fn from(value: u128) -> Self { + Self::from(U256::from(value)) } } -/// Represents a positive decimal value with some fractional digit precision, P. -/// Using U256 as storage. -#[derive(Copy, Clone, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct UDecimal { - internal: U256, +impl TryFrom for UDecimal18 { + type Error = >::Error; + fn try_from(value: f64) -> Result { + U256::try_from(value * 1e18).map(Self) + } } -macro_rules! impl_from_uints { - ($($t:ty),+) => {$( - impl std::convert::TryFrom<$t> for UDecimal

{ - type Error = &'static str; - fn try_from(from: $t) -> Result, Self::Error> { - let internal = U256::from(from) - .checked_mul(U256::exp10(P as usize)) - .ok_or("overflow")?; - Ok(UDecimal { internal }) - } - } - )*}; +impl From for f64 { + fn from(value: UDecimal18) -> Self { + f64::from(value.0) * 1e-18 + } } -impl_from_uints!(u8, u16, u32, u64, u128, usize, U256); +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub struct InvalidDecimalString; -impl str::FromStr for UDecimal

{ - type Err = ParseStrError; +impl fmt::Display for InvalidDecimalString { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "invalid decimal string") + } +} + +impl str::FromStr for UDecimal18 { + type Err = InvalidDecimalString; fn from_str(s: &str) -> Result { - use ParseStrError::*; - // We require at least one ASCII digit. Otherwise `U256::from_dec_str` will return 0 for + // We require at least one ASCII digit. Otherwise `U256::from_str_radix` will return 0 for // some inputs we consider invalid. if !s.chars().any(|c: char| -> bool { c.is_ascii_digit() }) { - return Err(InvalidInput); + return Err(InvalidDecimalString); } let (int, frac) = s.split_at(s.chars().position(|c| c == '.').unwrap_or(s.len())); - let p = P as usize; + let p = 18; let digits = int .chars() // append fractional digits (after decimal point) .chain(frac.chars().skip(1).chain(iter::repeat('0')).take(p)) .collect::(); - Ok(UDecimal { - internal: U256::from_dec_str(&digits).map_err(|_| InvalidInput)?, - }) + Ok(UDecimal18( + U256::from_str_radix(&digits, 10).map_err(|_| InvalidDecimalString)?, + )) } } -impl fmt::Display for UDecimal

{ +impl fmt::Display for UDecimal18 { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - if self.internal == 0.into() { + if self.0 == U256::from(0) { return write!(f, "0"); } - let p = P as usize; - let digits = self.internal.to_string().into_bytes(); + let p = 18; + let digits = self.0.to_string().into_bytes(); let ctz = digits .iter() .rev() @@ -96,189 +100,75 @@ impl fmt::Display for UDecimal

{ } } -impl fmt::Debug for UDecimal

{ +impl fmt::Debug for UDecimal18 { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "{self}") } } -// TODO: The following mathematical operations may result in overflow. This is -// fine for our current use-case. But should be handled if we want to release as -// a separate library. +impl UDecimal18 { + /// This will use the value of the given U256 directly, without scaling by 1e18. + pub fn from_raw_u256(value: U256) -> Self { + Self(value) + } -impl ops::Mul for UDecimal

{ - type Output = Self; - fn mul(self, rhs: Self) -> Self::Output { - Self { - internal: (self.internal * rhs.internal) / U256::exp10(P as usize), - } + pub fn raw_u256(&self) -> &U256 { + &self.0 } -} -impl ops::Mul for UDecimal

{ - type Output = Self; - fn mul(self, rhs: U256) -> Self::Output { - Self { - internal: self.internal * rhs, + pub fn as_u128(&self) -> Option { + if self.0 % U256::from(ONE_18) > U256::ZERO { + return None; } + let inner = self.0 / U256::from(ONE_18); + inner.try_into().ok() } -} -impl ops::Div for UDecimal

{ - type Output = Self; - fn div(self, rhs: Self) -> Self::Output { - Self { - internal: (self.internal * U256::exp10(P as usize)) / rhs.internal, - } + pub fn saturating_add(self, rhs: Self) -> Self { + Self(self.0.saturating_add(rhs.0)) } } -impl ops::Add for UDecimal

{ +impl std::ops::Add for UDecimal18 { type Output = Self; fn add(self, rhs: Self) -> Self::Output { - Self { - internal: self.internal + rhs.internal, - } + Self(self.0.add(rhs.0)) } } -impl ops::Sub for UDecimal

{ +impl std::ops::Mul for UDecimal18 { type Output = Self; - fn sub(self, rhs: Self) -> Self::Output { - Self { - internal: self.internal - rhs.internal, - } - } -} - -impl ops::AddAssign for UDecimal

{ - fn add_assign(&mut self, rhs: Self) { - *self = *self + rhs; + fn mul(self, rhs: Self) -> Self::Output { + Self((self.0 * rhs.0) / U256::from(ONE_18)) } } -impl ops::SubAssign for UDecimal

{ - fn sub_assign(&mut self, rhs: Self) { - *self = *self - rhs; +impl std::ops::Div for UDecimal18 { + type Output = Self; + fn div(self, rhs: Self) -> Self::Output { + Self((self.0 * U256::from(ONE_18)) / rhs.0) } } -impl iter::Sum for UDecimal

{ +impl std::iter::Sum for UDecimal18 { fn sum>(iter: I) -> Self { - iter.fold(Self::zero(), |sum, x| sum + x) - } -} - -#[allow(dead_code)] -impl UDecimal

{ - pub fn zero() -> Self { - Self { internal: 0.into() } - } - - pub fn from_little_endian(bytes: &[u8; 32]) -> Self { - Self { - internal: U256::from_little_endian(bytes), - } - } - - pub fn change_precision(self) -> UDecimal { - UDecimal { - internal: match N.cmp(&P) { - Ordering::Greater => self.internal * (U256::exp10((N - P) as usize)), - Ordering::Less => self.internal / (U256::exp10((P - N) as usize)), - Ordering::Equal => self.internal, - }, - } - } - - pub fn shift(self) -> UDecimal { - UDecimal { - internal: self.internal, - } + Self(iter.map(|u| u.0).sum()) } - - pub fn as_u256(&self) -> U256 { - self.internal / U256::exp10(P as usize) - } - - pub fn as_f64(&self) -> f64 { - // Collect the little-endian bytes of the U256 value. - let mut le_u8 = [0u8; 32]; - self.internal.to_little_endian(&mut le_u8); - // Merge the 32 bytes into 4 u64 values to reduce the amount of float - // operations required to calculate the final value. - let mut le_u64 = [0u64; 4]; - for (i, entry) in le_u64.iter_mut().enumerate() { - *entry = u64::from_le_bytes(le_u8[(i * 8)..((i + 1) * 8)].try_into().unwrap()); - } - // Count trailing u64 zero values. This is used to avoid unnecessary - // multiplications by zero. - let ctz = le_u64.iter().rev().take_while(|&&b| b == 0).count(); - // Sum the terms and then divide by 10^P, where each term equals - // 2^(64i) * n. - le_u64 - .iter() - .enumerate() - .take(le_u64.len() - ctz) - .map(|(i, &n)| 2.0f64.powi(i as i32 * 64) * n as f64) - .sum::() - / 10.0f64.powi(P as i32) - } - - pub fn to_little_endian(&self) -> [u8; 32] { - let mut buf = [0u8; 32]; - self.internal.to_little_endian(&mut buf); - buf - } - - pub fn saturating_add(self, other: Self) -> Self { - Self { - internal: self.internal.saturating_add(other.internal), - } - } - - pub fn saturating_sub(self, other: Self) -> Self { - Self { - internal: self.internal.saturating_sub(other.internal), - } - } -} - -impl TryFrom for UDecimal

{ - type Error = FromF64Error; - fn try_from(mut from: f64) -> Result { - if from.is_nan() || (from < 0.0) { - return Err(FromF64Error::InvalidInput); - } - const U128_MAX: f64 = u128::MAX as f64; - from *= 10.0f64.powi(P as i32); - let lower = from.min(U128_MAX); - from -= lower; - let lower = lower as u128; - // This can result in some nasty loss of precision for low (nonzero) values of upper. - let upper = (from / U128_MAX).round() as u128; - let mut le_u8 = [0u8; 32]; - le_u8[0..16].copy_from_slice(&lower.to_le_bytes()); - le_u8[16..32].copy_from_slice(&upper.to_le_bytes()); - Ok(Self { - internal: U256::from_little_endian(&le_u8), - }) - } -} - -#[derive(Copy, Clone, Debug, Eq, PartialEq)] -pub enum FromF64Error { - InvalidInput, } #[cfg(test)] mod test { use super::*; - use std::str::FromStr as _; #[test] - fn udecimal() { - test_udecimal::<6>(&[ + fn u256_from_str() { + assert_eq!("100".parse::().unwrap(), U256::from(100)); + assert_eq!("0x100".parse::().unwrap(), U256::from(256)); + } + + #[test] + fn udecimal18_from_str() { + let tests: &[(&str, Option<(&str, u128)>)] = &[ ("", None), ("?", None), (".", None), @@ -290,42 +180,35 @@ mod test { (".0", Some(("0", 0))), ("0.", Some(("0", 0))), ("00.00", Some(("0", 0))), - ("1", Some(("1", 1_000_000))), - ("1.0", Some(("1", 1_000_000))), - ("1.", Some(("1", 1_000_000))), - ("0.1", Some(("0.1", 100_000))), - (".1", Some(("0.1", 100_000))), - ("0.0000012", Some(("0.000001", 1))), - ("0.001001", Some(("0.001001", 1_001))), - ("0.001", Some(("0.001", 1_000))), - ("100.001", Some(("100.001", 100_001_000))), - ("100.000", Some(("100", 100_000_000))), - ("123.0", Some(("123", 123_000_000))), - ("123", Some(("123", 123_000_000))), + ("1", Some(("1", ONE_18))), + ("1.0", Some(("1", ONE_18))), + ("1.", Some(("1", ONE_18))), + ("0.1", Some(("0.1", ONE_18 / 10))), + (".1", Some(("0.1", ONE_18 / 10))), + ("0.0000000000000000012", Some(("0.000000000000000001", 1))), + ("0.001001", Some(("0.001001", 1_001_000_000_000_000))), + ("0.001", Some(("0.001", ONE_18 / 1_000))), + ("100.001", Some(("100.001", 100_001_000_000_000_000_000))), + ("100.000", Some(("100", 100 * ONE_18))), + ("123.0", Some(("123", 123 * ONE_18))), + ("123", Some(("123", 123 * ONE_18))), ( - "123456789.123456789", - Some(("123456789.123456", 123_456_789_123_456)), + "123456789123456789.123456789123456789123456789", + Some(( + "123456789123456789.123456789123456789", + 123_456_789_123_456_789_123_456_789_123_456_789, + )), ), - ]); - test_udecimal::<0>(&[ - ("0", Some(("0", 0))), - ("1", Some(("1", 1))), - ("0.1", Some(("0", 0))), - ("123456789", Some(("123456789", 123_456_789))), - ("123.1", Some(("123", 123))), - ]); - } - - fn test_udecimal(tests: &[(&str, Option<(&str, u64)>)]) { + ]; for (input, expected) in tests { - println!("input: \"{}\"", input); - let d = UDecimal::

::from_str(input); + let output = input.parse::(); + println!("\"{input}\" => {output:?}"); match expected { &Some((repr, internal)) => { - assert_eq!(d.as_ref().map(|d| d.internal), Ok(internal.into())); - assert_eq!(d.as_ref().map(ToString::to_string), Ok(repr.into())); + assert_eq!(output.as_ref().map(|d| d.0), Ok(U256::from(internal))); + assert_eq!(output.as_ref().map(ToString::to_string), Ok(repr.into())); } - None => assert_eq!(d, Err(ParseStrError::InvalidInput)), + None => assert_eq!(output, Err(InvalidDecimalString)), } } } @@ -340,35 +223,31 @@ mod test { 1.0, 123.456, 1e14, + 1e17, 1e18, + 1e19, 2.0f64.powi(128) - 1.0, 2.0f64.powi(128), 1e26, - // 1e27, // -> 2.085% error @ P = 12 - // 1e28, // -> 1.318% error @ P = 12 + 1e27, + 1e28, 1e29, 1e30, + 1e31, 1e32, ]; for test in tests { - test_udecimal_from_f64::<0>(test); - test_udecimal_from_f64::<1>(test); - test_udecimal_from_f64::<6>(test); - test_udecimal_from_f64::<12>(test); + let expected = (test * 1e18_f64).floor(); + let decimal = UDecimal18::try_from(test).unwrap(); + let output = decimal.0.to_string().parse::().unwrap(); + let error = (expected - output).abs() / expected.max(1e-30); + println!( + "expected: {}\n decimal: {}\n error: {:.3}%\n---", + expected / 1e18, + decimal, + error * 100.0 + ); + assert!(error < 0.005); } } - - fn test_udecimal_from_f64(value: f64) { - let expected = (value * 10.0f64.powi(P as i32)).floor(); - let decimal = UDecimal::

::try_from(value).unwrap(); - let output = decimal.internal.to_string().parse::().unwrap(); - let error = (expected - output).abs() / expected.max(1e-30); - println!( - "expected: {}\n decimal: {}\n error: {:.3}%\n---", - expected / 10.0f64.powi(P as i32), - decimal, - error * 100.0 - ); - assert!(error < 0.005); - } } diff --git a/prelude/src/lib.rs b/prelude/src/lib.rs index a9d904d6b..a5ee70e2f 100644 --- a/prelude/src/lib.rs +++ b/prelude/src/lib.rs @@ -46,24 +46,12 @@ pub fn sip24_hash(value: &impl Hash) -> u64 { hasher.finish() } -/// Decimal Parts-Per-Million with 6 fractional digits -pub type PPM = UDecimal<6>; -/// Decimal USD with 18 fractional digits -pub type USD = UDecimal<18>; -/// Decimal GRT with 18 fractional digits -pub type GRT = UDecimal<18>; -/// Decimal GRT Wei (10^-18 GRT) -pub type GRTWei = UDecimal<0>; - -impl<'de, const P: u8> serde::Deserialize<'de> for UDecimal

{ - fn deserialize>(deserializer: D) -> Result { - let input: &str = serde::Deserialize::deserialize(deserializer)?; - input.parse::().map_err(serde::de::Error::custom) - } -} - -impl serde::Serialize for UDecimal

{ - fn serialize(&self, serializer: S) -> Result { - serializer.serialize_str(&self.to_string()) - } -} +// The following are cumbersome by design. It's better to be forced to think hard about converting +// between these types. + +/// USD with 18 fractional digits +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub struct USD(pub UDecimal18); +/// GRT with 18 fractional digits +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub struct GRT(pub UDecimal18);