From 74400b53280497bd14af0462ac1bd6525af70772 Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 17 Jan 2025 15:37:05 +0000 Subject: [PATCH 01/27] Fetch order app data --- Cargo.lock | 246 +++++++++++++++--- crates/driver/Cargo.toml | 1 + .../driver/src/domain/competition/auction.rs | 74 ++++-- .../src/domain/competition/order/app_data.rs | 111 ++++++++ .../src/domain/competition/order/mod.rs | 49 +++- .../domain/competition/solution/encoding.rs | 2 +- .../src/domain/competition/solution/mod.rs | 2 +- crates/driver/src/domain/quote.rs | 2 + crates/driver/src/infra/api/error.rs | 1 + crates/driver/src/infra/api/mod.rs | 14 +- .../api/routes/solve/dto/solve_request.rs | 3 + .../driver/src/infra/api/routes/solve/mod.rs | 8 +- crates/driver/src/infra/solver/dto/auction.rs | 2 +- 13 files changed, 452 insertions(+), 63 deletions(-) create mode 100644 crates/driver/src/domain/competition/order/app_data.rs diff --git a/Cargo.lock b/Cargo.lock index 9ce09b15ee..7b4421db5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -200,6 +200,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-lock" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" +dependencies = [ + "event-listener 5.4.0", + "event-listener-strategy", + "pin-project-lite", +] + [[package]] name = "async-stream" version = "0.3.5" @@ -1283,7 +1294,7 @@ dependencies = [ "iana-time-zone", "num-traits", "serde", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -1332,6 +1343,15 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "console-api" version = "0.7.0" @@ -1537,6 +1557,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.11" @@ -1820,6 +1849,7 @@ dependencies = [ "mimalloc", "mockall 0.12.1", "model", + "moka", "num", "number", "observe", @@ -2100,6 +2130,27 @@ version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" +[[package]] +name = "event-listener" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1" +dependencies = [ + "event-listener 5.4.0", + "pin-project-lite", +] + [[package]] name = "fastrand" version = "1.9.0" @@ -2324,6 +2375,19 @@ dependencies = [ "web3", ] +[[package]] +name = "generator" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bd114ceda131d3b1d665eba35788690ad37f5916457286b32ab6fd3c438dd" +dependencies = [ + "cfg-if", + "libc", + "log", + "rustversion", + "windows", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -2685,7 +2749,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core", + "windows-core 0.52.0", ] [[package]] @@ -2947,6 +3011,19 @@ version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + [[package]] name = "lru" version = "0.12.3" @@ -3123,6 +3200,28 @@ dependencies = [ "web3", ] +[[package]] +name = "moka" +version = "0.12.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9321642ca94a4282428e6ea4af8cc2ca4eac48ac7a6a4ea8f33f76d0ce70926" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "event-listener 5.4.0", + "futures-util", + "loom", + "parking_lot", + "portable-atomic", + "rustc_version", + "smallvec", + "tagptr", + "thiserror", + "uuid", +] + [[package]] name = "multibase" version = "0.9.1" @@ -3477,6 +3576,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.3" @@ -3497,7 +3602,7 @@ dependencies = [ "libc", "redox_syscall 0.5.1", "smallvec", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -3580,6 +3685,12 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +[[package]] +name = "portable-atomic" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" + [[package]] name = "powerfmt" version = "0.2.0" @@ -4708,7 +4819,7 @@ dependencies = [ "crc", "crossbeam-queue", "either", - "event-listener", + "event-listener 2.5.3", "futures-channel", "futures-core", "futures-intrusive", @@ -4995,6 +5106,12 @@ dependencies = [ "libc", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "tap" version = "1.0.1" @@ -5519,6 +5636,9 @@ name = "uuid" version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" +dependencies = [ + "getrandom", +] [[package]] name = "valuable" @@ -5752,13 +5872,77 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" +dependencies = [ + "windows-core 0.58.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows-core" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-core" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result", + "windows-strings", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-implement" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + +[[package]] +name = "windows-interface" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + +[[package]] +name = "windows-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result", + "windows-targets 0.52.6", ] [[package]] @@ -5776,7 +5960,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -5796,18 +5980,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.5", - "windows_aarch64_msvc 0.52.5", - "windows_i686_gnu 0.52.5", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.5", - "windows_x86_64_gnu 0.52.5", - "windows_x86_64_gnullvm 0.52.5", - "windows_x86_64_msvc 0.52.5", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -5818,9 +6002,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" @@ -5830,9 +6014,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" @@ -5842,15 +6026,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" @@ -5860,9 +6044,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" @@ -5872,9 +6056,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" @@ -5884,9 +6068,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" @@ -5896,9 +6080,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" diff --git a/crates/driver/Cargo.toml b/crates/driver/Cargo.toml index 83c2a1983d..40a30fa6b2 100644 --- a/crates/driver/Cargo.toml +++ b/crates/driver/Cargo.toml @@ -38,6 +38,7 @@ hyper = { workspace = true } indexmap = { workspace = true, features = ["serde"] } itertools = { workspace = true } mimalloc = { workspace = true } +moka = { version = "0.12.10", features = ["future"] } num = { workspace = true } number = { path = "../number" } prometheus = { workspace = true } diff --git a/crates/driver/src/domain/competition/auction.rs b/crates/driver/src/domain/competition/auction.rs index 6014129c46..e49059cc7b 100644 --- a/crates/driver/src/domain/competition/auction.rs +++ b/crates/driver/src/domain/competition/auction.rs @@ -10,8 +10,9 @@ use { infra::{self, blockchain, config::file::OrderPriorityStrategy, observe, Ethereum}, util::{self, Bytes}, }, + app_data::ValidatedAppData, chrono::{Duration, Utc}, - futures::future::{join_all, BoxFuture, FutureExt, Shared}, + futures::future::{join_all, try_join, try_join_all, BoxFuture, FutureExt, Shared}, itertools::Itertools, model::{order::OrderKind, signature::Signature}, shared::signature_validator::{Contracts, SignatureValidating}, @@ -127,7 +128,10 @@ impl Auction { } #[derive(Clone)] -pub struct AuctionProcessor(Arc>); +pub struct AuctionProcessor { + inner: Arc>, + app_data_fetcher: Arc, +} struct Inner { auction: auction::Id, @@ -143,15 +147,43 @@ type BalanceGroup = (order::Trader, eth::TokenAddress, order::SellTokenBalance); type Balances = HashMap; impl AuctionProcessor { - /// Prioritize well priced and filter out unfillable orders from the given - /// auction. - pub async fn prioritize(&self, auction: Auction, solver: ð::H160) -> Auction { - Auction { - orders: self.prioritize_orders(&auction, solver).await, - ..auction + pub async fn process(&self, auction: Auction, solver: ð::H160) -> Result { + let (app_data_by_order, mut prioritized_orders) = try_join( + self.collect_orders_app_data(&auction), + self.prioritize_orders(&auction, solver).map(Ok), + ) + .await?; + + for order in &mut prioritized_orders { + if let Some(Some(app_data)) = app_data_by_order.get(&order.uid) { + order.app_data = app_data.clone().into(); + } } + + Ok(Auction { + orders: prioritized_orders, + ..auction + }) } + async fn collect_orders_app_data( + &self, + auction: &Auction, + ) -> Result>, Error> { + Ok(try_join_all(auction.orders.iter().map(|order| async { + self.app_data_fetcher + .fetch(order.app_data.hash()) + .await + .map(|app_data| (order.uid, app_data)) + .map_err(|err| Error::AppDataFetching(order.uid, err)) + })) + .await? + .into_iter() + .collect::>()) + } + + /// Prioritize well priced and filter out unfillable orders from the given + /// auction. fn prioritize_orders( &self, auction: &Auction, @@ -161,7 +193,7 @@ impl AuctionProcessor { .id() .expect("auctions used for quoting do not have to be prioritized"); - let mut lock = self.0.lock().unwrap(); + let mut lock = self.inner.lock().unwrap(); let current_id = lock.auction; if new_id.0 < current_id.0 { tracing::error!(?current_id, ?new_id, "received an outdated auction"); @@ -367,7 +399,7 @@ impl AuctionProcessor { (*cow_amm.address(), cow_amm.validated_template_order(prices, signature_validator, &domain_separator).await) }), ) - .await; + .await; // Convert results to domain format. let domain_separator = @@ -392,7 +424,7 @@ impl AuctionProcessor { }, kind: order::Kind::Limit, side: template.order.kind.into(), - app_data: order::AppData(Bytes(template.order.app_data.0)), + app_data: order::AppDataHash(Bytes(template.order.app_data.0)).into(), buy_token_balance: template.order.buy_token_balance.into(), sell_token_balance: template.order.sell_token_balance.into(), partial: match template.order.partially_fillable { @@ -449,6 +481,7 @@ impl AuctionProcessor { pub fn new( eth: &infra::Ethereum, order_priority_strategies: Vec, + app_data_fetcher: Arc, ) -> Self { let eth = eth.with_metric_label("auctionPreProcessing".into()); let mut order_sorting_strategies = vec![]; @@ -478,13 +511,16 @@ impl AuctionProcessor { }, ); - Self(Arc::new(Mutex::new(Inner { - auction: Id(0), - fut: futures::future::pending().boxed().shared(), - eth, - order_sorting_strategies, - signature_validator, - }))) + Self { + inner: Arc::new(Mutex::new(Inner { + auction: Id(0), + fut: futures::future::pending().boxed().shared(), + eth, + order_sorting_strategies, + signature_validator, + })), + app_data_fetcher, + } } } @@ -632,4 +668,6 @@ pub enum Error { InvalidAmounts, #[error("blockchain error: {0:?}")] Blockchain(#[from] blockchain::Error), + #[error("order {0:?} app data fetching error: {1}")] + AppDataFetching(order::Uid, order::app_data::FetchingError), } diff --git a/crates/driver/src/domain/competition/order/app_data.rs b/crates/driver/src/domain/competition/order/app_data.rs new file mode 100644 index 0000000000..54a15c8d06 --- /dev/null +++ b/crates/driver/src/domain/competition/order/app_data.rs @@ -0,0 +1,111 @@ +use { + futures::FutureExt, + moka::future::Cache, + reqwest::StatusCode, + shared::request_sharing::BoxRequestSharing, + std::sync::Arc, + thiserror::Error, + url::Url, +}; + +#[derive(Clone)] +pub struct AppDataFetcher(Arc); + +struct Inner { + client: reqwest::Client, + base_url: Url, + request_sharing: BoxRequestSharing< + super::AppDataHash, + Result, FetchingError>, + >, + app_data_validator: app_data::Validator, + cache: Cache>, +} + +impl AppDataFetcher { + pub fn new(orderbook_url: Url) -> Self { + Self(Arc::new(Inner { + client: reqwest::Client::new(), + base_url: orderbook_url, + request_sharing: BoxRequestSharing::labelled("app_data".to_string()), + app_data_validator: app_data::Validator::new(usize::MAX), + cache: Cache::new(2_000), + })) + } + + pub async fn fetch( + &self, + app_data: super::AppDataHash, + ) -> Result, FetchingError> { + if let Some(app_data) = self.0.cache.get(&app_data).await { + return Ok(app_data.clone()); + } + + let app_data_fut = move |app_data: &super::AppDataHash| { + let app_data = *app_data; + let self_ = self.clone(); + + async move { + let url = self_ + .0 + .base_url + .join(&format!("v1/app_data/{:?}", app_data))?; + let response = self_.0.client.get(url).send().await?; + + let validated_app_data = match response.status() { + StatusCode::NOT_FOUND => None, + _ => { + let bytes = response.error_for_status()?.bytes().await?; + Some(self_.0.app_data_validator.validate(&bytes)?) + } + }; + + self_ + .0 + .cache + .insert(app_data, validated_app_data.clone()) + .await; + + Ok(validated_app_data) + } + .boxed() + }; + + self.0 + .request_sharing + .shared_or_else(app_data, app_data_fut) + .await + } +} + +#[derive(Error, Debug)] +pub enum FetchingError { + #[error("unable to send a request: {0}")] + Http(String), + #[error("received invalid app data: {0}")] + InvalidAppData(#[from] anyhow::Error), + #[error("internal error: {0}")] + Internal(String), +} + +impl From for FetchingError { + fn from(err: reqwest::Error) -> Self { + FetchingError::Http(err.to_string()) + } +} + +impl From for FetchingError { + fn from(err: url::ParseError) -> Self { + FetchingError::Http(err.to_string()) + } +} + +impl Clone for FetchingError { + fn clone(&self) -> Self { + match self { + Self::Http(message) => Self::Http(message.clone()), + Self::InvalidAppData(err) => Self::InvalidAppData(shared::clone_anyhow_error(err)), + Self::Internal(message) => Self::Internal(message.clone()), + } + } +} diff --git a/crates/driver/src/domain/competition/order/mod.rs b/crates/driver/src/domain/competition/order/mod.rs index 5b82b1d758..9e7c597161 100644 --- a/crates/driver/src/domain/competition/order/mod.rs +++ b/crates/driver/src/domain/competition/order/mod.rs @@ -12,6 +12,7 @@ use { }; pub use {fees::FeePolicy, signature::Signature}; +pub mod app_data; pub mod fees; pub mod signature; @@ -50,6 +51,42 @@ pub struct Order { pub quote: Option, } +/// The app data associated with an order. +#[derive(Debug, Clone, From)] +pub enum AppData { + /// App data hash. + Hash(AppDataHash), + /// Validated full app data. + Full(Box<::app_data::ValidatedAppData>), +} + +impl Default for AppData { + fn default() -> Self { + Self::Hash(Default::default()) + } +} + +impl AppData { + pub fn hash(&self) -> AppDataHash { + match self { + Self::Hash(hash) => *hash, + Self::Full(data) => AppDataHash(data.hash.0.into()), + } + } +} + +impl From<[u8; APP_DATA_LEN]> for AppData { + fn from(app_data_hash: [u8; APP_DATA_LEN]) -> Self { + Self::Hash(app_data_hash.into()) + } +} + +impl From<::app_data::ValidatedAppData> for AppData { + fn from(value: ::app_data::ValidatedAppData) -> Self { + Self::Full(Box::new(value)) + } +} + /// An amount denominated in the sell token of an [`Order`]. #[derive(Clone, Copy, Debug, Default, Eq, Ord, PartialEq, PartialOrd, From, Into)] pub struct SellAmount(pub eth::U256); @@ -273,17 +310,17 @@ pub const APP_DATA_LEN: usize = 32; /// This is a hash allowing arbitrary user data to be associated with an order. /// While this type holds the hash, the data itself is uploaded to IPFS. This /// hash is signed along with the order. -#[derive(Debug, Default, Clone, Copy)] -pub struct AppData(pub Bytes<[u8; APP_DATA_LEN]>); +#[derive(Debug, Default, Clone, Copy, Hash, PartialEq, Eq)] +pub struct AppDataHash(pub Bytes<[u8; APP_DATA_LEN]>); -impl From<[u8; APP_DATA_LEN]> for AppData { +impl From<[u8; APP_DATA_LEN]> for AppDataHash { fn from(inner: [u8; APP_DATA_LEN]) -> Self { Self(inner.into()) } } -impl From for [u8; APP_DATA_LEN] { - fn from(app_data: AppData) -> Self { +impl From for [u8; APP_DATA_LEN] { + fn from(app_data: AppDataHash) -> Self { app_data.0.into() } } @@ -388,7 +425,7 @@ pub struct Jit { pub receiver: eth::Address, pub valid_to: util::Timestamp, pub partially_fillable: bool, - pub app_data: AppData, + pub app_data: AppDataHash, pub side: Side, pub sell_token_balance: SellTokenBalance, pub buy_token_balance: BuyTokenBalance, diff --git a/crates/driver/src/domain/competition/solution/encoding.rs b/crates/driver/src/domain/competition/solution/encoding.rs index 1872131827..90f0dbc4c1 100644 --- a/crates/driver/src/domain/competition/solution/encoding.rs +++ b/crates/driver/src/domain/competition/solution/encoding.rs @@ -93,7 +93,7 @@ pub fn tx( sell_amount: trade.order().sell.amount.into(), buy_amount: trade.order().buy.amount.into(), valid_to: trade.order().valid_to.into(), - app_data: trade.order().app_data.0 .0.into(), + app_data: trade.order().app_data.hash().0 .0.into(), fee_amount: eth::U256::zero(), flags: Flags { side: trade.order().side, diff --git a/crates/driver/src/domain/competition/solution/mod.rs b/crates/driver/src/domain/competition/solution/mod.rs index 800e3fb892..9fc9bf0db2 100644 --- a/crates/driver/src/domain/competition/solution/mod.rs +++ b/crates/driver/src/domain/competition/solution/mod.rs @@ -95,7 +95,7 @@ impl Solution { .unwrap_or(u32::MIN) .into(), valid_to: jit.order().valid_to, - app_data: jit.order().app_data, + app_data: jit.order().app_data.into(), partial: jit.order().partially_fillable(), pre_interactions: vec![], post_interactions: vec![], diff --git a/crates/driver/src/domain/quote.rs b/crates/driver/src/domain/quote.rs index 75aa0aecf7..67ceeac155 100644 --- a/crates/driver/src/domain/quote.rs +++ b/crates/driver/src/domain/quote.rs @@ -15,6 +15,7 @@ use { }, util, }, + anyhow::anyhow, chrono::Utc, std::{ collections::{HashMap, HashSet}, @@ -182,6 +183,7 @@ impl Order { auction::Error::InvalidTokens => panic!("fake auction with invalid tokens"), auction::Error::InvalidAmounts => panic!("fake auction with invalid amounts"), auction::Error::Blockchain(e) => e.into(), + auction::Error::AppDataFetching(_, e) => Error::Boundary(anyhow!("{}", e)), }) } diff --git a/crates/driver/src/infra/api/error.rs b/crates/driver/src/infra/api/error.rs index e788bbe5bf..5693e125be 100644 --- a/crates/driver/src/infra/api/error.rs +++ b/crates/driver/src/infra/api/error.rs @@ -99,6 +99,7 @@ impl From for (hyper::StatusCode, axum::Json) api::routes::AuctionError::InvalidTokens => Kind::InvalidTokens, api::routes::AuctionError::InvalidAmounts => Kind::InvalidAmounts, api::routes::AuctionError::Blockchain(_) => Kind::Unknown, + api::routes::AuctionError::Internal => Kind::Unknown, }; error.into() } diff --git a/crates/driver/src/infra/api/mod.rs b/crates/driver/src/infra/api/mod.rs index ccbdc5a178..1589e740e5 100644 --- a/crates/driver/src/infra/api/mod.rs +++ b/crates/driver/src/infra/api/mod.rs @@ -1,6 +1,10 @@ use { crate::{ - domain::{self, competition::bad_tokens, Mempools}, + domain::{ + self, + competition::{bad_tokens, order::app_data::AppDataFetcher}, + Mempools, + }, infra::{ self, config::file::OrderPriorityStrategy, @@ -51,8 +55,12 @@ impl Api { ); let tokens = tokens::Fetcher::new(&self.eth); - let pre_processor = - domain::competition::AuctionProcessor::new(&self.eth, order_priority_strategies); + let app_data_fetcher = AppDataFetcher::new("http://localhost:8080".parse().unwrap()); + let pre_processor = domain::competition::AuctionProcessor::new( + &self.eth, + order_priority_strategies, + Arc::new(app_data_fetcher), + ); // Add the metrics and healthz endpoints. app = routes::metrics(app); diff --git a/crates/driver/src/infra/api/routes/solve/dto/solve_request.rs b/crates/driver/src/infra/api/routes/solve/dto/solve_request.rs index b773387715..04ff2cf473 100644 --- a/crates/driver/src/infra/api/routes/solve/dto/solve_request.rs +++ b/crates/driver/src/infra/api/routes/solve/dto/solve_request.rs @@ -177,6 +177,8 @@ pub enum Error { InvalidAmounts, #[error("blockchain error: {0:?}")] Blockchain(#[source] crate::infra::blockchain::Error), + #[error("internal error")] + Internal, } impl From for Error { @@ -191,6 +193,7 @@ impl From for Error { auction::Error::InvalidTokens => Self::InvalidTokens, auction::Error::InvalidAmounts => Self::InvalidAmounts, auction::Error::Blockchain(err) => Self::Blockchain(err), + auction::Error::AppDataFetching(_, _) => Self::Internal, } } } diff --git a/crates/driver/src/infra/api/routes/solve/mod.rs b/crates/driver/src/infra/api/routes/solve/mod.rs index 98faccaa7c..507168d44f 100644 --- a/crates/driver/src/infra/api/routes/solve/mod.rs +++ b/crates/driver/src/infra/api/routes/solve/mod.rs @@ -34,8 +34,12 @@ async fn route( let competition = state.competition(); let auction = state .pre_processor() - .prioritize(auction, &competition.solver.account().address()) - .await; + .process(auction, &competition.solver.account().address()) + .await + .map_err(|err| { + tracing::error!(?err, "unable to pre-process the auction"); + Into::::into(err) + })?; let result = competition.solve(auction).await; competition.ensure_settle_queue_capacity()?; observe::solved(state.solver().name(), &result); diff --git a/crates/driver/src/infra/solver/dto/auction.rs b/crates/driver/src/infra/solver/dto/auction.rs index 2ce99983be..8b993ab17c 100644 --- a/crates/driver/src/infra/solver/dto/auction.rs +++ b/crates/driver/src/infra/solver/dto/auction.rs @@ -158,7 +158,7 @@ impl Auction { .map(FeePolicy::from_domain) .collect(), ), - app_data: AppDataHash(order.app_data.0.into()), + app_data: AppDataHash(order.app_data.hash().0.into()), signature: order.signature.data.clone().into(), signing_scheme: match order.signature.scheme { Scheme::Eip712 => SigningScheme::Eip712, From e67293d69b3e92ac418c0f72f990e77d2a6f9a81 Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 17 Jan 2025 15:56:40 +0000 Subject: [PATCH 02/27] Config --- crates/driver/src/infra/api/mod.rs | 4 +++- crates/driver/src/infra/config/file/load.rs | 1 + crates/driver/src/infra/config/file/mod.rs | 3 +++ crates/driver/src/infra/config/mod.rs | 1 + crates/driver/src/run.rs | 1 + 5 files changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/driver/src/infra/api/mod.rs b/crates/driver/src/infra/api/mod.rs index 1589e740e5..acec2b57f5 100644 --- a/crates/driver/src/infra/api/mod.rs +++ b/crates/driver/src/infra/api/mod.rs @@ -19,6 +19,7 @@ use { futures::Future, std::{net::SocketAddr, sync::Arc}, tokio::sync::oneshot, + url::Url, }; mod error; @@ -44,6 +45,7 @@ impl Api { self, shutdown: impl Future + Send + 'static, order_priority_strategies: Vec, + orderbook_url: Url, ) -> Result<(), hyper::Error> { // Add middleware. let mut app = axum::Router::new().layer( @@ -55,7 +57,7 @@ impl Api { ); let tokens = tokens::Fetcher::new(&self.eth); - let app_data_fetcher = AppDataFetcher::new("http://localhost:8080".parse().unwrap()); + let app_data_fetcher = AppDataFetcher::new(orderbook_url); let pre_processor = domain::competition::AuctionProcessor::new( &self.eth, order_priority_strategies, diff --git a/crates/driver/src/infra/config/file/load.rs b/crates/driver/src/infra/config/file/load.rs index 0c3e2b61f7..189c7d3388 100644 --- a/crates/driver/src/infra/config/file/load.rs +++ b/crates/driver/src/infra/config/file/load.rs @@ -376,5 +376,6 @@ pub async fn load(chain: Chain, path: &Path) -> infra::Config { order_priority_strategies: config.order_priority_strategies, archive_node_url: config.archive_node_url, simulation_bad_token_max_age: config.simulation_bad_token_max_age, + orderbook_url: config.orderbook_url, } } diff --git a/crates/driver/src/infra/config/file/mod.rs b/crates/driver/src/infra/config/file/mod.rs index 39ed135509..6aa2454860 100644 --- a/crates/driver/src/infra/config/file/mod.rs +++ b/crates/driver/src/infra/config/file/mod.rs @@ -73,6 +73,9 @@ struct Config { default = "default_simulation_bad_token_max_age" )] simulation_bad_token_max_age: Duration, + + /// URL of the orderbook API to fetch order's app-data from. + orderbook_url: Url, } #[serde_as] diff --git a/crates/driver/src/infra/config/mod.rs b/crates/driver/src/infra/config/mod.rs index d87d4500f2..0db8ed2bb3 100644 --- a/crates/driver/src/infra/config/mod.rs +++ b/crates/driver/src/infra/config/mod.rs @@ -30,4 +30,5 @@ pub struct Config { pub order_priority_strategies: Vec, pub archive_node_url: Option, pub simulation_bad_token_max_age: Duration, + pub orderbook_url: Url, } diff --git a/crates/driver/src/run.rs b/crates/driver/src/run.rs index bf21134a52..f59632f7e9 100644 --- a/crates/driver/src/run.rs +++ b/crates/driver/src/run.rs @@ -79,6 +79,7 @@ async fn run_with(args: cli::Args, addr_sender: Option Date: Fri, 17 Jan 2025 16:18:24 +0000 Subject: [PATCH 03/27] Docs --- .../driver/src/domain/competition/auction.rs | 14 +++++---- .../src/domain/competition/order/app_data.rs | 29 +++++++++++++++---- crates/driver/src/infra/api/mod.rs | 6 ++-- 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/crates/driver/src/domain/competition/auction.rs b/crates/driver/src/domain/competition/auction.rs index e49059cc7b..3178c25848 100644 --- a/crates/driver/src/domain/competition/auction.rs +++ b/crates/driver/src/domain/competition/auction.rs @@ -130,7 +130,7 @@ impl Auction { #[derive(Clone)] pub struct AuctionProcessor { inner: Arc>, - app_data_fetcher: Arc, + app_data_retriever: Arc, } struct Inner { @@ -147,6 +147,9 @@ type BalanceGroup = (order::Trader, eth::TokenAddress, order::SellTokenBalance); type Balances = HashMap; impl AuctionProcessor { + /// Process the auction by prioritizing the orders and filtering out + /// unfillable orders. Fetches full app data for each order and returns an + /// auction with updated orders. pub async fn process(&self, auction: Auction, solver: ð::H160) -> Result { let (app_data_by_order, mut prioritized_orders) = try_join( self.collect_orders_app_data(&auction), @@ -166,13 +169,14 @@ impl AuctionProcessor { }) } + /// Fetches the app data for all orders in the auction. async fn collect_orders_app_data( &self, auction: &Auction, ) -> Result>, Error> { Ok(try_join_all(auction.orders.iter().map(|order| async { - self.app_data_fetcher - .fetch(order.app_data.hash()) + self.app_data_retriever + .get(order.app_data.hash()) .await .map(|app_data| (order.uid, app_data)) .map_err(|err| Error::AppDataFetching(order.uid, err)) @@ -481,7 +485,7 @@ impl AuctionProcessor { pub fn new( eth: &infra::Ethereum, order_priority_strategies: Vec, - app_data_fetcher: Arc, + app_data_retriever: Arc, ) -> Self { let eth = eth.with_metric_label("auctionPreProcessing".into()); let mut order_sorting_strategies = vec![]; @@ -519,7 +523,7 @@ impl AuctionProcessor { order_sorting_strategies, signature_validator, })), - app_data_fetcher, + app_data_retriever, } } } diff --git a/crates/driver/src/domain/competition/order/app_data.rs b/crates/driver/src/domain/competition/order/app_data.rs index 54a15c8d06..c82d5850c3 100644 --- a/crates/driver/src/domain/competition/order/app_data.rs +++ b/crates/driver/src/domain/competition/order/app_data.rs @@ -8,8 +8,20 @@ use { url::Url, }; +/// A struct for retrieving order's full app-data by its hash from a remote +/// service, with support for caching and deduplicating concurrent requests. +/// +/// Ensures efficient access to application data by: +/// - Caching results to avoid redundant network requests. +/// - Sharing ongoing requests to prevent duplicate fetches for the same +/// `app_data`. +/// - Validating fetched app data. +/// +/// LRU cache is used since only ~2% of app-data is unique across all orders +/// meaning that the cache hit rate is expected to be high, so there is no need +/// for TTL cache. #[derive(Clone)] -pub struct AppDataFetcher(Arc); +pub struct AppDataRetriever(Arc); struct Inner { client: reqwest::Client, @@ -22,18 +34,23 @@ struct Inner { cache: Cache>, } -impl AppDataFetcher { +impl AppDataRetriever { + // According to statistics, the average size of the app-data is ~800 bytes. With + // this constant, the approximate size of the cache will be ~1.6 MB. + const CACHE_SIZE: u64 = 2_000; + pub fn new(orderbook_url: Url) -> Self { Self(Arc::new(Inner { client: reqwest::Client::new(), base_url: orderbook_url, request_sharing: BoxRequestSharing::labelled("app_data".to_string()), app_data_validator: app_data::Validator::new(usize::MAX), - cache: Cache::new(2_000), + cache: Cache::new(Self::CACHE_SIZE), })) } - pub async fn fetch( + /// Retrieves the full app-data for the given `app_data` hash, if exists. + pub async fn get( &self, app_data: super::AppDataHash, ) -> Result, FetchingError> { @@ -80,7 +97,7 @@ impl AppDataFetcher { #[derive(Error, Debug)] pub enum FetchingError { - #[error("unable to send a request: {0}")] + #[error("error while sending a request: {0}")] Http(String), #[error("received invalid app data: {0}")] InvalidAppData(#[from] anyhow::Error), @@ -96,7 +113,7 @@ impl From for FetchingError { impl From for FetchingError { fn from(err: url::ParseError) -> Self { - FetchingError::Http(err.to_string()) + FetchingError::Internal(err.to_string()) } } diff --git a/crates/driver/src/infra/api/mod.rs b/crates/driver/src/infra/api/mod.rs index acec2b57f5..b2ee82f1e0 100644 --- a/crates/driver/src/infra/api/mod.rs +++ b/crates/driver/src/infra/api/mod.rs @@ -2,7 +2,7 @@ use { crate::{ domain::{ self, - competition::{bad_tokens, order::app_data::AppDataFetcher}, + competition::{bad_tokens, order::app_data::AppDataRetriever}, Mempools, }, infra::{ @@ -57,11 +57,11 @@ impl Api { ); let tokens = tokens::Fetcher::new(&self.eth); - let app_data_fetcher = AppDataFetcher::new(orderbook_url); + let app_data_retriever = AppDataRetriever::new(orderbook_url); let pre_processor = domain::competition::AuctionProcessor::new( &self.eth, order_priority_strategies, - Arc::new(app_data_fetcher), + Arc::new(app_data_retriever), ); // Add the metrics and healthz endpoints. From 56e21d41b5bf5cc5ba4ba50a5fb8c02c419d35ed Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 17 Jan 2025 17:11:29 +0000 Subject: [PATCH 04/27] Orderbook mock --- .../src/domain/competition/order/app_data.rs | 2 +- crates/driver/src/infra/cli.rs | 4 +++ crates/driver/src/infra/config/file/load.rs | 1 - crates/driver/src/infra/config/file/mod.rs | 3 -- crates/driver/src/infra/config/mod.rs | 1 - crates/driver/src/run.rs | 2 +- crates/driver/src/tests/setup/driver.rs | 8 +++++- crates/driver/src/tests/setup/mod.rs | 8 +++++- crates/driver/src/tests/setup/orderbook.rs | 28 +++++++++++++++++++ 9 files changed, 48 insertions(+), 9 deletions(-) create mode 100644 crates/driver/src/tests/setup/orderbook.rs diff --git a/crates/driver/src/domain/competition/order/app_data.rs b/crates/driver/src/domain/competition/order/app_data.rs index c82d5850c3..84f51d2208 100644 --- a/crates/driver/src/domain/competition/order/app_data.rs +++ b/crates/driver/src/domain/competition/order/app_data.rs @@ -66,7 +66,7 @@ impl AppDataRetriever { let url = self_ .0 .base_url - .join(&format!("v1/app_data/{:?}", app_data))?; + .join(&format!("v1/app_data/{:?}", app_data.0))?; let response = self_.0.client.get(url).send().await?; let validated_app_data = match response.status() { diff --git a/crates/driver/src/infra/cli.rs b/crates/driver/src/infra/cli.rs index 9159f990f7..b259837a65 100644 --- a/crates/driver/src/infra/cli.rs +++ b/crates/driver/src/infra/cli.rs @@ -21,6 +21,10 @@ pub struct Args { #[clap(long, env)] pub ethrpc: Url, + /// The orderbook API base URL. + #[clap(long, env)] + pub orderbook_url: Url, + /// Path to the driver configuration file. This file should be in TOML /// format. For an example see /// https://github.com/cowprotocol/services/blob/main/crates/driver/example.toml. diff --git a/crates/driver/src/infra/config/file/load.rs b/crates/driver/src/infra/config/file/load.rs index 189c7d3388..0c3e2b61f7 100644 --- a/crates/driver/src/infra/config/file/load.rs +++ b/crates/driver/src/infra/config/file/load.rs @@ -376,6 +376,5 @@ pub async fn load(chain: Chain, path: &Path) -> infra::Config { order_priority_strategies: config.order_priority_strategies, archive_node_url: config.archive_node_url, simulation_bad_token_max_age: config.simulation_bad_token_max_age, - orderbook_url: config.orderbook_url, } } diff --git a/crates/driver/src/infra/config/file/mod.rs b/crates/driver/src/infra/config/file/mod.rs index 6aa2454860..39ed135509 100644 --- a/crates/driver/src/infra/config/file/mod.rs +++ b/crates/driver/src/infra/config/file/mod.rs @@ -73,9 +73,6 @@ struct Config { default = "default_simulation_bad_token_max_age" )] simulation_bad_token_max_age: Duration, - - /// URL of the orderbook API to fetch order's app-data from. - orderbook_url: Url, } #[serde_as] diff --git a/crates/driver/src/infra/config/mod.rs b/crates/driver/src/infra/config/mod.rs index 0db8ed2bb3..d87d4500f2 100644 --- a/crates/driver/src/infra/config/mod.rs +++ b/crates/driver/src/infra/config/mod.rs @@ -30,5 +30,4 @@ pub struct Config { pub order_priority_strategies: Vec, pub archive_node_url: Option, pub simulation_bad_token_max_age: Duration, - pub orderbook_url: Url, } diff --git a/crates/driver/src/run.rs b/crates/driver/src/run.rs index f59632f7e9..f94f479351 100644 --- a/crates/driver/src/run.rs +++ b/crates/driver/src/run.rs @@ -79,7 +79,7 @@ async fn run_with(args: cli::Args, addr_sender: Option, blockchain: &Blockchain, + orderbook: Orderbook, ) -> Self { let (config_file, config_temp_path) = match config.config_file.as_ref() { Some(config_file) => (config_file.to_owned(), None), @@ -46,6 +50,8 @@ impl Driver { "0.0.0.0:0".to_owned(), "--ethrpc".to_owned(), blockchain.web3_url.clone(), + "--orderbook-url".to_owned(), + format!("http://{:?}", orderbook.addr).to_owned(), "--config".to_owned(), config_file.to_str().unwrap().to_owned(), ]; diff --git a/crates/driver/src/tests/setup/mod.rs b/crates/driver/src/tests/setup/mod.rs index ed57f2fb2f..7e480f902a 100644 --- a/crates/driver/src/tests/setup/mod.rs +++ b/crates/driver/src/tests/setup/mod.rs @@ -28,7 +28,10 @@ use { ETH_ORDER_AMOUNT, }, hex_address, - setup::blockchain::{Blockchain, Interaction, Trade}, + setup::{ + blockchain::{Blockchain, Interaction, Trade}, + orderbook::Orderbook, + }, }, }, app_data::AppDataHash, @@ -51,6 +54,7 @@ use { pub mod blockchain; mod driver; pub mod fee; +mod orderbook; mod solver; #[derive(Debug, Clone, Copy)] @@ -957,6 +961,7 @@ impl Setup { (solver.clone(), instance.addr) })) .await; + let orderbook = Orderbook::start(); let driver = Driver::new( &driver::Config { config_file, @@ -966,6 +971,7 @@ impl Setup { }, &solvers_with_address, &blockchain, + orderbook, ) .await; diff --git a/crates/driver/src/tests/setup/orderbook.rs b/crates/driver/src/tests/setup/orderbook.rs new file mode 100644 index 0000000000..33a5d600d1 --- /dev/null +++ b/crates/driver/src/tests/setup/orderbook.rs @@ -0,0 +1,28 @@ +use { + axum::{extract::Path, http::StatusCode, routing::get, Router}, + std::net::SocketAddr, +}; + +pub struct Orderbook { + pub addr: SocketAddr, +} + +impl Orderbook { + pub fn start() -> Self { + let app = Router::new().route("/v1/app_data/:app_data", get(Self::mock_handler)); + let server = + axum::Server::bind(&"0.0.0.0:0".parse().unwrap()).serve(app.into_make_service()); + let addr = server.local_addr(); + println!("Orderbook mock server listening on {}", addr); + + tokio::spawn(server); + + Orderbook { addr } + } + + /// Default mock handler that always returns 404 Not Found. + async fn mock_handler(Path(app_data): Path) -> StatusCode { + println!("Received app_data request: {}", app_data); + StatusCode::NOT_FOUND + } +} From beca1f5aafa08a7a9a7d6fb690bc0ba64980d7c8 Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 17 Jan 2025 17:57:17 +0000 Subject: [PATCH 05/27] Do not discard the whole auction --- .../driver/src/domain/competition/auction.rs | 59 +++++++++++-------- crates/driver/src/domain/quote.rs | 2 - crates/driver/src/infra/api/error.rs | 1 - .../api/routes/solve/dto/solve_request.rs | 3 - .../driver/src/infra/api/routes/solve/mod.rs | 6 +- 5 files changed, 35 insertions(+), 36 deletions(-) diff --git a/crates/driver/src/domain/competition/auction.rs b/crates/driver/src/domain/competition/auction.rs index 3178c25848..5b54b10709 100644 --- a/crates/driver/src/domain/competition/auction.rs +++ b/crates/driver/src/domain/competition/auction.rs @@ -10,9 +10,8 @@ use { infra::{self, blockchain, config::file::OrderPriorityStrategy, observe, Ethereum}, util::{self, Bytes}, }, - app_data::ValidatedAppData, chrono::{Duration, Utc}, - futures::future::{join_all, try_join, try_join_all, BoxFuture, FutureExt, Shared}, + futures::future::{join, join_all, BoxFuture, FutureExt, Shared}, itertools::Itertools, model::{order::OrderKind, signature::Signature}, shared::signature_validator::{Contracts, SignatureValidating}, @@ -150,40 +149,52 @@ impl AuctionProcessor { /// Process the auction by prioritizing the orders and filtering out /// unfillable orders. Fetches full app data for each order and returns an /// auction with updated orders. - pub async fn process(&self, auction: Auction, solver: ð::H160) -> Result { - let (app_data_by_order, mut prioritized_orders) = try_join( + pub async fn process(&self, auction: Auction, solver: ð::H160) -> Auction { + let (app_data_by_order, mut prioritized_orders) = join( self.collect_orders_app_data(&auction), - self.prioritize_orders(&auction, solver).map(Ok), + self.prioritize_orders(&auction, solver), ) - .await?; - - for order in &mut prioritized_orders { - if let Some(Some(app_data)) = app_data_by_order.get(&order.uid) { - order.app_data = app_data.clone().into(); - } - } + .await; + + // Filter out orders that failed to fetch app data. + prioritized_orders.retain_mut(|order| { + app_data_by_order.get(&order.uid).map_or(true, |result| { + match result { + Err(err) => { + tracing::warn!(order_uid=?order.uid, ?err, "failed to fetch app data for order, excluding from auction"); + false + } + Ok(Some(app_data)) => { + order.app_data = app_data.clone().into(); + true + } + Ok(None) => true + } + }) + }); - Ok(Auction { + Auction { orders: prioritized_orders, ..auction - }) + } } /// Fetches the app data for all orders in the auction. + /// Returns a map from order UIDs to the result of fetching the app data. async fn collect_orders_app_data( &self, auction: &Auction, - ) -> Result>, Error> { - Ok(try_join_all(auction.orders.iter().map(|order| async { - self.app_data_retriever - .get(order.app_data.hash()) - .await - .map(|app_data| (order.uid, app_data)) - .map_err(|err| Error::AppDataFetching(order.uid, err)) + ) -> HashMap< + order::Uid, + Result, order::app_data::FetchingError>, + > { + join_all(auction.orders.iter().map(|order| async { + let result = self.app_data_retriever.get(order.app_data.hash()).await; + (order.uid, result) })) - .await? + .await .into_iter() - .collect::>()) + .collect::>() } /// Prioritize well priced and filter out unfillable orders from the given @@ -672,6 +683,4 @@ pub enum Error { InvalidAmounts, #[error("blockchain error: {0:?}")] Blockchain(#[from] blockchain::Error), - #[error("order {0:?} app data fetching error: {1}")] - AppDataFetching(order::Uid, order::app_data::FetchingError), } diff --git a/crates/driver/src/domain/quote.rs b/crates/driver/src/domain/quote.rs index 67ceeac155..75aa0aecf7 100644 --- a/crates/driver/src/domain/quote.rs +++ b/crates/driver/src/domain/quote.rs @@ -15,7 +15,6 @@ use { }, util, }, - anyhow::anyhow, chrono::Utc, std::{ collections::{HashMap, HashSet}, @@ -183,7 +182,6 @@ impl Order { auction::Error::InvalidTokens => panic!("fake auction with invalid tokens"), auction::Error::InvalidAmounts => panic!("fake auction with invalid amounts"), auction::Error::Blockchain(e) => e.into(), - auction::Error::AppDataFetching(_, e) => Error::Boundary(anyhow!("{}", e)), }) } diff --git a/crates/driver/src/infra/api/error.rs b/crates/driver/src/infra/api/error.rs index 5693e125be..e788bbe5bf 100644 --- a/crates/driver/src/infra/api/error.rs +++ b/crates/driver/src/infra/api/error.rs @@ -99,7 +99,6 @@ impl From for (hyper::StatusCode, axum::Json) api::routes::AuctionError::InvalidTokens => Kind::InvalidTokens, api::routes::AuctionError::InvalidAmounts => Kind::InvalidAmounts, api::routes::AuctionError::Blockchain(_) => Kind::Unknown, - api::routes::AuctionError::Internal => Kind::Unknown, }; error.into() } diff --git a/crates/driver/src/infra/api/routes/solve/dto/solve_request.rs b/crates/driver/src/infra/api/routes/solve/dto/solve_request.rs index 04ff2cf473..b773387715 100644 --- a/crates/driver/src/infra/api/routes/solve/dto/solve_request.rs +++ b/crates/driver/src/infra/api/routes/solve/dto/solve_request.rs @@ -177,8 +177,6 @@ pub enum Error { InvalidAmounts, #[error("blockchain error: {0:?}")] Blockchain(#[source] crate::infra::blockchain::Error), - #[error("internal error")] - Internal, } impl From for Error { @@ -193,7 +191,6 @@ impl From for Error { auction::Error::InvalidTokens => Self::InvalidTokens, auction::Error::InvalidAmounts => Self::InvalidAmounts, auction::Error::Blockchain(err) => Self::Blockchain(err), - auction::Error::AppDataFetching(_, _) => Self::Internal, } } } diff --git a/crates/driver/src/infra/api/routes/solve/mod.rs b/crates/driver/src/infra/api/routes/solve/mod.rs index 507168d44f..d72f635b3f 100644 --- a/crates/driver/src/infra/api/routes/solve/mod.rs +++ b/crates/driver/src/infra/api/routes/solve/mod.rs @@ -35,11 +35,7 @@ async fn route( let auction = state .pre_processor() .process(auction, &competition.solver.account().address()) - .await - .map_err(|err| { - tracing::error!(?err, "unable to pre-process the auction"); - Into::::into(err) - })?; + .await; let result = competition.solve(auction).await; competition.ensure_settle_queue_capacity()?; observe::solved(state.solver().name(), &result); From 131b90595c74d4182dd5e83caa32a2152d47544e Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 17 Jan 2025 18:35:34 +0000 Subject: [PATCH 06/27] e2e fix --- crates/e2e/src/setup/colocation.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/e2e/src/setup/colocation.rs b/crates/e2e/src/setup/colocation.rs index beca88e1bd..e4b51244a0 100644 --- a/crates/e2e/src/setup/colocation.rs +++ b/crates/e2e/src/setup/colocation.rs @@ -186,6 +186,7 @@ mempool = "public" let args = vec![ "driver".to_string(), format!("--config={}", config_file.display()), + "--orderbook-url=http://localhost:8080".to_string(), format!("--ethrpc={NODE_HOST}"), ]; From 7917094e6975e6c90916a88ba2f13c912cfbdc7b Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 17 Jan 2025 18:54:04 +0000 Subject: [PATCH 07/27] Comment --- crates/driver/src/domain/competition/auction.rs | 5 +++-- crates/driver/src/domain/competition/order/app_data.rs | 1 - crates/driver/src/tests/setup/orderbook.rs | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/driver/src/domain/competition/auction.rs b/crates/driver/src/domain/competition/auction.rs index 5b54b10709..47e8b2e98b 100644 --- a/crates/driver/src/domain/competition/auction.rs +++ b/crates/driver/src/domain/competition/auction.rs @@ -147,8 +147,9 @@ type Balances = HashMap; impl AuctionProcessor { /// Process the auction by prioritizing the orders and filtering out - /// unfillable orders. Fetches full app data for each order and returns an - /// auction with updated orders. + /// unfillable orders as well as those that failed to fetch app data. + /// Fetches full app data for each order and returns an auction with + /// updated orders. pub async fn process(&self, auction: Auction, solver: ð::H160) -> Auction { let (app_data_by_order, mut prioritized_orders) = join( self.collect_orders_app_data(&auction), diff --git a/crates/driver/src/domain/competition/order/app_data.rs b/crates/driver/src/domain/competition/order/app_data.rs index 84f51d2208..372fbf3c42 100644 --- a/crates/driver/src/domain/competition/order/app_data.rs +++ b/crates/driver/src/domain/competition/order/app_data.rs @@ -68,7 +68,6 @@ impl AppDataRetriever { .base_url .join(&format!("v1/app_data/{:?}", app_data.0))?; let response = self_.0.client.get(url).send().await?; - let validated_app_data = match response.status() { StatusCode::NOT_FOUND => None, _ => { diff --git a/crates/driver/src/tests/setup/orderbook.rs b/crates/driver/src/tests/setup/orderbook.rs index 33a5d600d1..5ca32ae154 100644 --- a/crates/driver/src/tests/setup/orderbook.rs +++ b/crates/driver/src/tests/setup/orderbook.rs @@ -3,6 +3,8 @@ use { std::net::SocketAddr, }; +/// A mocked orderbook service that provides `/v1/app_data/{app_data_hash}` API. +/// Always returns 404 Not Found. pub struct Orderbook { pub addr: SocketAddr, } From 98e788262d3b9711f768f61ef004a7c904a20345 Mon Sep 17 00:00:00 2001 From: ilya Date: Mon, 20 Jan 2025 13:38:11 +0000 Subject: [PATCH 08/27] Review comments --- crates/driver/src/domain/competition/auction.rs | 6 +++--- crates/driver/src/tests/setup/driver.rs | 4 ++-- crates/driver/src/tests/setup/mod.rs | 2 +- crates/driver/src/tests/setup/orderbook.rs | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/driver/src/domain/competition/auction.rs b/crates/driver/src/domain/competition/auction.rs index 47e8b2e98b..e8e2b94b5a 100644 --- a/crates/driver/src/domain/competition/auction.rs +++ b/crates/driver/src/domain/competition/auction.rs @@ -151,7 +151,7 @@ impl AuctionProcessor { /// Fetches full app data for each order and returns an auction with /// updated orders. pub async fn process(&self, auction: Auction, solver: ð::H160) -> Auction { - let (app_data_by_order, mut prioritized_orders) = join( + let (mut app_data_by_order, mut prioritized_orders) = join( self.collect_orders_app_data(&auction), self.prioritize_orders(&auction, solver), ) @@ -159,14 +159,14 @@ impl AuctionProcessor { // Filter out orders that failed to fetch app data. prioritized_orders.retain_mut(|order| { - app_data_by_order.get(&order.uid).map_or(true, |result| { + app_data_by_order.remove(&order.uid).map_or(true, |result| { match result { Err(err) => { tracing::warn!(order_uid=?order.uid, ?err, "failed to fetch app data for order, excluding from auction"); false } Ok(Some(app_data)) => { - order.app_data = app_data.clone().into(); + order.app_data = app_data.into(); true } Ok(None) => true diff --git a/crates/driver/src/tests/setup/driver.rs b/crates/driver/src/tests/setup/driver.rs index c8893a2492..3c8d1bf396 100644 --- a/crates/driver/src/tests/setup/driver.rs +++ b/crates/driver/src/tests/setup/driver.rs @@ -34,7 +34,7 @@ impl Driver { config: &Config, solvers: &Vec<(Solver, SocketAddr)>, blockchain: &Blockchain, - orderbook: Orderbook, + orderbook: &Orderbook, ) -> Self { let (config_file, config_temp_path) = match config.config_file.as_ref() { Some(config_file) => (config_file.to_owned(), None), @@ -51,7 +51,7 @@ impl Driver { "--ethrpc".to_owned(), blockchain.web3_url.clone(), "--orderbook-url".to_owned(), - format!("http://{:?}", orderbook.addr).to_owned(), + format!("http://{}", orderbook.addr).to_owned(), "--config".to_owned(), config_file.to_str().unwrap().to_owned(), ]; diff --git a/crates/driver/src/tests/setup/mod.rs b/crates/driver/src/tests/setup/mod.rs index 7e480f902a..c843e9c2a4 100644 --- a/crates/driver/src/tests/setup/mod.rs +++ b/crates/driver/src/tests/setup/mod.rs @@ -971,7 +971,7 @@ impl Setup { }, &solvers_with_address, &blockchain, - orderbook, + &orderbook, ) .await; diff --git a/crates/driver/src/tests/setup/orderbook.rs b/crates/driver/src/tests/setup/orderbook.rs index 5ca32ae154..fec519ad6f 100644 --- a/crates/driver/src/tests/setup/orderbook.rs +++ b/crates/driver/src/tests/setup/orderbook.rs @@ -24,7 +24,7 @@ impl Orderbook { /// Default mock handler that always returns 404 Not Found. async fn mock_handler(Path(app_data): Path) -> StatusCode { - println!("Received app_data request: {}", app_data); + println!("Orderbook received an app_data request: {}", app_data); StatusCode::NOT_FOUND } } From a7dea13e7648772a4b432839ec0d4cfb1641e85a Mon Sep 17 00:00:00 2001 From: ilya Date: Tue, 21 Jan 2025 12:08:03 +0000 Subject: [PATCH 09/27] Fetch app-data only once for a single auction --- .../driver/src/domain/competition/auction.rs | 101 ++++++++++-------- 1 file changed, 54 insertions(+), 47 deletions(-) diff --git a/crates/driver/src/domain/competition/auction.rs b/crates/driver/src/domain/competition/auction.rs index e8e2b94b5a..6ac3b87498 100644 --- a/crates/driver/src/domain/competition/auction.rs +++ b/crates/driver/src/domain/competition/auction.rs @@ -11,7 +11,7 @@ use { util::{self, Bytes}, }, chrono::{Duration, Utc}, - futures::future::{join, join_all, BoxFuture, FutureExt, Shared}, + futures::future::{join_all, BoxFuture, FutureExt, Shared}, itertools::Itertools, model::{order::OrderKind, signature::Signature}, shared::signature_validator::{Contracts, SignatureValidating}, @@ -127,10 +127,7 @@ impl Auction { } #[derive(Clone)] -pub struct AuctionProcessor { - inner: Arc>, - app_data_retriever: Arc, -} +pub struct AuctionProcessor(Arc>); struct Inner { auction: auction::Id, @@ -140,6 +137,7 @@ struct Inner { /// `order_priority_strategies` from the driver's config. order_sorting_strategies: Vec>, signature_validator: Arc, + app_data_retriever: Arc, } type BalanceGroup = (order::Trader, eth::TokenAddress, order::SellTokenBalance); @@ -151,31 +149,8 @@ impl AuctionProcessor { /// Fetches full app data for each order and returns an auction with /// updated orders. pub async fn process(&self, auction: Auction, solver: ð::H160) -> Auction { - let (mut app_data_by_order, mut prioritized_orders) = join( - self.collect_orders_app_data(&auction), - self.prioritize_orders(&auction, solver), - ) - .await; - - // Filter out orders that failed to fetch app data. - prioritized_orders.retain_mut(|order| { - app_data_by_order.remove(&order.uid).map_or(true, |result| { - match result { - Err(err) => { - tracing::warn!(order_uid=?order.uid, ?err, "failed to fetch app data for order, excluding from auction"); - false - } - Ok(Some(app_data)) => { - order.app_data = app_data.into(); - true - } - Ok(None) => true - } - }) - }); - Auction { - orders: prioritized_orders, + orders: self.prioritize_orders(&auction, solver).await, ..auction } } @@ -183,14 +158,14 @@ impl AuctionProcessor { /// Fetches the app data for all orders in the auction. /// Returns a map from order UIDs to the result of fetching the app data. async fn collect_orders_app_data( - &self, - auction: &Auction, + app_data_retriever: Arc, + orders: &[order::Order], ) -> HashMap< order::Uid, Result, order::app_data::FetchingError>, > { - join_all(auction.orders.iter().map(|order| async { - let result = self.app_data_retriever.get(order.app_data.hash()).await; + join_all(orders.iter().map(|order| async { + let result = app_data_retriever.get(order.app_data.hash()).await; (order.uid, result) })) .await @@ -209,7 +184,7 @@ impl AuctionProcessor { .id() .expect("auctions used for quoting do not have to be prioritized"); - let mut lock = self.inner.lock().unwrap(); + let mut lock = self.0.lock().unwrap(); let current_id = lock.auction; if new_id.0 < current_id.0 { tracing::error!(?current_id, ?new_id, "received an outdated auction"); @@ -228,6 +203,7 @@ impl AuctionProcessor { let mut orders = auction.orders.clone(); let solver = *solver; let order_comparators = lock.order_sorting_strategies.clone(); + let app_data_fetcher = lock.app_data_retriever.clone(); // Use spawn_blocking() because a lot of CPU bound computations are happening // and we don't want to block the runtime for too long. @@ -235,9 +211,17 @@ impl AuctionProcessor { let start = std::time::Instant::now(); orders.extend(rt.block_on(Self::cow_amm_orders(ð, &tokens, &cow_amms, signature_validator.as_ref()))); sorting::sort_orders(&mut orders, &tokens, &solver, &order_comparators); - let mut balances = - rt.block_on(async { Self::fetch_balances(ð, &orders).await }); - Self::filter_orders(&mut balances, &mut orders); + let (mut balances, mut app_data_by_order) = + rt.block_on(async { + tokio::join!( + Self::fetch_balances(ð, &orders), + Self::collect_orders_app_data(app_data_fetcher, &orders), + ) + }); + + Self::filter_orders_by_balances(&mut balances, &mut orders); + Self::filter_orders_by_app_data(&mut app_data_by_order, &mut orders); + tracing::debug!(auction_id = new_id.0, time =? start.elapsed(), "auction preprocessing done"); orders }) @@ -258,7 +242,7 @@ impl AuctionProcessor { } /// Removes orders that cannot be filled due to missing funds of the owner. - fn filter_orders(balances: &mut Balances, orders: &mut Vec) { + fn filter_orders_by_balances(balances: &mut Balances, orders: &mut Vec) { // The auction that we receive from the `autopilot` assumes that there // is sufficient balance to completely cover all the orders. **This is // not the case** (as the protocol should not chose which limit orders @@ -327,6 +311,31 @@ impl AuctionProcessor { }); } + /// Filter out orders that require flashloan but failed to fetch app data. + fn filter_orders_by_app_data( + app_data_by_order: &mut HashMap< + order::Uid, + Result, order::app_data::FetchingError>, + >, + orders: &mut Vec, + ) { + orders.retain_mut(|order| { + app_data_by_order.remove(&order.uid).map_or(true, |result| { + match result { + Err(err) => { + tracing::warn!(order_uid=?order.uid, ?err, "failed to fetch app data for order, excluding from auction"); + false + } + Ok(Some(app_data)) => { + order.app_data = app_data.into(); + true + } + Ok(None) => true + } + }) + }); + } + /// Fetches the tradable balance for every order owner. async fn fetch_balances(ethereum: &infra::Ethereum, orders: &[order::Order]) -> Balances { let ethereum = ethereum.with_metric_label("orderBalances".into()); @@ -527,16 +536,14 @@ impl AuctionProcessor { }, ); - Self { - inner: Arc::new(Mutex::new(Inner { - auction: Id(0), - fut: futures::future::pending().boxed().shared(), - eth, - order_sorting_strategies, - signature_validator, - })), + Self(Arc::new(Mutex::new(Inner { + auction: Id(0), + fut: futures::future::pending().boxed().shared(), + eth, + order_sorting_strategies, + signature_validator, app_data_retriever, - } + }))) } } From 9b4b419c0b0eeaaaa2049dcf9a116c9eaa3b0eb6 Mon Sep 17 00:00:00 2001 From: ilya Date: Tue, 21 Jan 2025 12:17:48 +0000 Subject: [PATCH 10/27] Combine functions --- .../driver/src/domain/competition/auction.rs | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/crates/driver/src/domain/competition/auction.rs b/crates/driver/src/domain/competition/auction.rs index 6ac3b87498..38dcae3425 100644 --- a/crates/driver/src/domain/competition/auction.rs +++ b/crates/driver/src/domain/competition/auction.rs @@ -219,8 +219,7 @@ impl AuctionProcessor { ) }); - Self::filter_orders_by_balances(&mut balances, &mut orders); - Self::filter_orders_by_app_data(&mut app_data_by_order, &mut orders); + Self::filter_orders(&mut balances, &mut app_data_by_order, &mut orders); tracing::debug!(auction_id = new_id.0, time =? start.elapsed(), "auction preprocessing done"); orders @@ -241,8 +240,17 @@ impl AuctionProcessor { fut } - /// Removes orders that cannot be filled due to missing funds of the owner. - fn filter_orders_by_balances(balances: &mut Balances, orders: &mut Vec) { + /// Removes orders that: + /// - Cannot be filled due to missing funds of the owner. + /// - Require flashloan but failed to fetch app data. + fn filter_orders( + balances: &mut Balances, + app_data_by_order: &mut HashMap< + order::Uid, + Result, order::app_data::FetchingError>, + >, + orders: &mut Vec, + ) { // The auction that we receive from the `autopilot` assumes that there // is sufficient balance to completely cover all the orders. **This is // not the case** (as the protocol should not chose which limit orders @@ -307,19 +315,7 @@ impl AuctionProcessor { } remaining_balance.0 -= allocated_balance.0; - true - }); - } - /// Filter out orders that require flashloan but failed to fetch app data. - fn filter_orders_by_app_data( - app_data_by_order: &mut HashMap< - order::Uid, - Result, order::app_data::FetchingError>, - >, - orders: &mut Vec, - ) { - orders.retain_mut(|order| { app_data_by_order.remove(&order.uid).map_or(true, |result| { match result { Err(err) => { From 965204c7c2a0a21bd643bc19a444a01bef170bac Mon Sep 17 00:00:00 2001 From: ilya Date: Tue, 21 Jan 2025 12:18:42 +0000 Subject: [PATCH 11/27] Comment --- crates/driver/src/domain/competition/auction.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/driver/src/domain/competition/auction.rs b/crates/driver/src/domain/competition/auction.rs index 38dcae3425..784e92470a 100644 --- a/crates/driver/src/domain/competition/auction.rs +++ b/crates/driver/src/domain/competition/auction.rs @@ -242,7 +242,7 @@ impl AuctionProcessor { /// Removes orders that: /// - Cannot be filled due to missing funds of the owner. - /// - Require flashloan but failed to fetch app data. + /// - Failed to fetch app data. fn filter_orders( balances: &mut Balances, app_data_by_order: &mut HashMap< From 34a29c35702a7cc123765a51f6fc5c8f200f5008 Mon Sep 17 00:00:00 2001 From: ilya Date: Tue, 21 Jan 2025 15:37:47 +0000 Subject: [PATCH 12/27] Make flashloans configurable --- .../driver/src/domain/competition/auction.rs | 50 ++++++++++--------- .../src/domain/competition/order/app_data.rs | 8 +-- crates/driver/src/infra/api/mod.rs | 6 +-- .../driver/src/infra/api/routes/solve/mod.rs | 2 +- crates/driver/src/infra/config/file/load.rs | 1 + crates/driver/src/infra/config/file/mod.rs | 32 ++++++++++++ crates/driver/src/infra/config/mod.rs | 3 +- crates/driver/src/run.rs | 11 +++- crates/driver/src/tests/setup/driver.rs | 1 + 9 files changed, 77 insertions(+), 37 deletions(-) diff --git a/crates/driver/src/domain/competition/auction.rs b/crates/driver/src/domain/competition/auction.rs index 784e92470a..633c6dc541 100644 --- a/crates/driver/src/domain/competition/auction.rs +++ b/crates/driver/src/domain/competition/auction.rs @@ -137,7 +137,7 @@ struct Inner { /// `order_priority_strategies` from the driver's config. order_sorting_strategies: Vec>, signature_validator: Arc, - app_data_retriever: Arc, + app_data_retriever: Option, } type BalanceGroup = (order::Trader, eth::TokenAddress, order::SellTokenBalance); @@ -148,31 +148,13 @@ impl AuctionProcessor { /// unfillable orders as well as those that failed to fetch app data. /// Fetches full app data for each order and returns an auction with /// updated orders. - pub async fn process(&self, auction: Auction, solver: ð::H160) -> Auction { + pub async fn prioritize(&self, auction: Auction, solver: ð::H160) -> Auction { Auction { orders: self.prioritize_orders(&auction, solver).await, ..auction } } - /// Fetches the app data for all orders in the auction. - /// Returns a map from order UIDs to the result of fetching the app data. - async fn collect_orders_app_data( - app_data_retriever: Arc, - orders: &[order::Order], - ) -> HashMap< - order::Uid, - Result, order::app_data::FetchingError>, - > { - join_all(orders.iter().map(|order| async { - let result = app_data_retriever.get(order.app_data.hash()).await; - (order.uid, result) - })) - .await - .into_iter() - .collect::>() - } - /// Prioritize well priced and filter out unfillable orders from the given /// auction. fn prioritize_orders( @@ -203,7 +185,7 @@ impl AuctionProcessor { let mut orders = auction.orders.clone(); let solver = *solver; let order_comparators = lock.order_sorting_strategies.clone(); - let app_data_fetcher = lock.app_data_retriever.clone(); + let app_data_retriever = lock.app_data_retriever.clone(); // Use spawn_blocking() because a lot of CPU bound computations are happening // and we don't want to block the runtime for too long. @@ -215,7 +197,7 @@ impl AuctionProcessor { rt.block_on(async { tokio::join!( Self::fetch_balances(ð, &orders), - Self::collect_orders_app_data(app_data_fetcher, &orders), + Self::collect_orders_app_data(app_data_retriever, &orders), ) }); @@ -240,6 +222,28 @@ impl AuctionProcessor { fut } + /// Fetches the app data for all orders in the auction. + /// Returns a map from order UIDs to the result of fetching the app data. + async fn collect_orders_app_data( + app_data_retriever: Option, + orders: &[order::Order], + ) -> HashMap< + order::Uid, + Result, order::app_data::FetchingError>, + > { + if let Some(app_data_retriever) = app_data_retriever { + join_all(orders.iter().map(|order| async { + let result = app_data_retriever.get(order.app_data.hash()).await; + (order.uid, result) + })) + .await + .into_iter() + .collect::>() + } else { + Default::default() + } + } + /// Removes orders that: /// - Cannot be filled due to missing funds of the owner. /// - Failed to fetch app data. @@ -502,7 +506,7 @@ impl AuctionProcessor { pub fn new( eth: &infra::Ethereum, order_priority_strategies: Vec, - app_data_retriever: Arc, + app_data_retriever: Option, ) -> Self { let eth = eth.with_metric_label("auctionPreProcessing".into()); let mut order_sorting_strategies = vec![]; diff --git a/crates/driver/src/domain/competition/order/app_data.rs b/crates/driver/src/domain/competition/order/app_data.rs index 372fbf3c42..4fa8961e73 100644 --- a/crates/driver/src/domain/competition/order/app_data.rs +++ b/crates/driver/src/domain/competition/order/app_data.rs @@ -35,17 +35,13 @@ struct Inner { } impl AppDataRetriever { - // According to statistics, the average size of the app-data is ~800 bytes. With - // this constant, the approximate size of the cache will be ~1.6 MB. - const CACHE_SIZE: u64 = 2_000; - - pub fn new(orderbook_url: Url) -> Self { + pub fn new(orderbook_url: Url, cache_size: u64) -> Self { Self(Arc::new(Inner { client: reqwest::Client::new(), base_url: orderbook_url, request_sharing: BoxRequestSharing::labelled("app_data".to_string()), app_data_validator: app_data::Validator::new(usize::MAX), - cache: Cache::new(Self::CACHE_SIZE), + cache: Cache::new(cache_size), })) } diff --git a/crates/driver/src/infra/api/mod.rs b/crates/driver/src/infra/api/mod.rs index b2ee82f1e0..18d6a1e91e 100644 --- a/crates/driver/src/infra/api/mod.rs +++ b/crates/driver/src/infra/api/mod.rs @@ -19,7 +19,6 @@ use { futures::Future, std::{net::SocketAddr, sync::Arc}, tokio::sync::oneshot, - url::Url, }; mod error; @@ -45,7 +44,7 @@ impl Api { self, shutdown: impl Future + Send + 'static, order_priority_strategies: Vec, - orderbook_url: Url, + app_data_retriever: Option, ) -> Result<(), hyper::Error> { // Add middleware. let mut app = axum::Router::new().layer( @@ -57,11 +56,10 @@ impl Api { ); let tokens = tokens::Fetcher::new(&self.eth); - let app_data_retriever = AppDataRetriever::new(orderbook_url); let pre_processor = domain::competition::AuctionProcessor::new( &self.eth, order_priority_strategies, - Arc::new(app_data_retriever), + app_data_retriever, ); // Add the metrics and healthz endpoints. diff --git a/crates/driver/src/infra/api/routes/solve/mod.rs b/crates/driver/src/infra/api/routes/solve/mod.rs index d72f635b3f..98faccaa7c 100644 --- a/crates/driver/src/infra/api/routes/solve/mod.rs +++ b/crates/driver/src/infra/api/routes/solve/mod.rs @@ -34,7 +34,7 @@ async fn route( let competition = state.competition(); let auction = state .pre_processor() - .process(auction, &competition.solver.account().address()) + .prioritize(auction, &competition.solver.account().address()) .await; let result = competition.solve(auction).await; competition.ensure_settle_queue_capacity()?; diff --git a/crates/driver/src/infra/config/file/load.rs b/crates/driver/src/infra/config/file/load.rs index 0c3e2b61f7..e9dcac8b72 100644 --- a/crates/driver/src/infra/config/file/load.rs +++ b/crates/driver/src/infra/config/file/load.rs @@ -376,5 +376,6 @@ pub async fn load(chain: Chain, path: &Path) -> infra::Config { order_priority_strategies: config.order_priority_strategies, archive_node_url: config.archive_node_url, simulation_bad_token_max_age: config.simulation_bad_token_max_age, + flashloans: config.flashloans, } } diff --git a/crates/driver/src/infra/config/file/mod.rs b/crates/driver/src/infra/config/file/mod.rs index 39ed135509..69aa45067d 100644 --- a/crates/driver/src/infra/config/file/mod.rs +++ b/crates/driver/src/infra/config/file/mod.rs @@ -73,6 +73,10 @@ struct Config { default = "default_simulation_bad_token_max_age" )] simulation_bad_token_max_age: Duration, + + /// Configuration for flashloans. + #[serde(default, flatten)] + flashloans: Flashloans, } #[serde_as] @@ -733,6 +737,28 @@ impl Default for BadTokenDetectionConfig { } } +#[serde_as] +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +pub struct Flashloans { + /// Whether the flashloans feature is enabled. + #[serde(default, rename = "flashloans-enabled")] + pub enabled: bool, + + /// The maximum number of app-data entries in the cache. + #[serde( + default = "default_flashloans_cache_size", + rename = "flashloans-cache-size" + )] + pub cache_size: u64, +} + +impl Default for Flashloans { + fn default() -> Self { + serde_json::from_str("{}").expect("Flashloans uses default values") + } +} + fn default_metrics_bad_token_detector_failure_ratio() -> f64 { 0.9 } @@ -755,3 +781,9 @@ fn default_metrics_bad_token_detector_log_only() -> bool { fn default_metrics_bad_token_detector_freeze_time() -> Duration { Duration::from_secs(60 * 10) } + +/// According to statistics, the average size of the app-data is ~800 bytes. +/// With this default, the approximate size of the cache will be ~1.6 MB. +fn default_flashloans_cache_size() -> u64 { + 2000 +} diff --git a/crates/driver/src/infra/config/mod.rs b/crates/driver/src/infra/config/mod.rs index d87d4500f2..8d4ab34ae1 100644 --- a/crates/driver/src/infra/config/mod.rs +++ b/crates/driver/src/infra/config/mod.rs @@ -3,7 +3,7 @@ use { domain::eth, infra::{ blockchain, - config::file::{GasEstimatorType, OrderPriorityStrategy}, + config::file::{Flashloans, GasEstimatorType, OrderPriorityStrategy}, liquidity, mempool, simulator, @@ -30,4 +30,5 @@ pub struct Config { pub order_priority_strategies: Vec, pub archive_node_url: Option, pub simulation_bad_token_max_age: Duration, + pub flashloans: Flashloans, } diff --git a/crates/driver/src/run.rs b/crates/driver/src/run.rs index f94f479351..5ded6a1d06 100644 --- a/crates/driver/src/run.rs +++ b/crates/driver/src/run.rs @@ -1,6 +1,9 @@ use { crate::{ - domain::{competition::bad_tokens, Mempools}, + domain::{ + competition::{bad_tokens, order::app_data::AppDataRetriever}, + Mempools, + }, infra::{ self, blockchain::{self, Ethereum}, @@ -51,6 +54,10 @@ async fn run_with(args: cli::Args, addr_sender: Option Date: Tue, 21 Jan 2025 15:49:00 +0000 Subject: [PATCH 13/27] Simplify logic --- .../driver/src/domain/competition/auction.rs | 42 +++++++------------ 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/crates/driver/src/domain/competition/auction.rs b/crates/driver/src/domain/competition/auction.rs index 633c6dc541..7ea4b5a0a1 100644 --- a/crates/driver/src/domain/competition/auction.rs +++ b/crates/driver/src/domain/competition/auction.rs @@ -19,6 +19,7 @@ use { collections::{HashMap, HashSet}, sync::{Arc, Mutex}, }, + tap::TapFallible, thiserror::Error, }; @@ -227,14 +228,14 @@ impl AuctionProcessor { async fn collect_orders_app_data( app_data_retriever: Option, orders: &[order::Order], - ) -> HashMap< - order::Uid, - Result, order::app_data::FetchingError>, - > { + ) -> HashMap> { if let Some(app_data_retriever) = app_data_retriever { join_all(orders.iter().map(|order| async { - let result = app_data_retriever.get(order.app_data.hash()).await; - (order.uid, result) + let fetched_app_data = app_data_retriever.get(order.app_data.hash()).await.tap_err(|err| { + tracing::warn!(order_uid=?order.uid, ?err, "failed to fetch app data for order"); + }).ok().flatten(); + + (order.uid, fetched_app_data) })) .await .into_iter() @@ -244,15 +245,11 @@ impl AuctionProcessor { } } - /// Removes orders that: - /// - Cannot be filled due to missing funds of the owner. - /// - Failed to fetch app data. + /// Removes orders that cannot be filled due to missing funds of the owner + /// and updates the fetched app data. fn filter_orders( balances: &mut Balances, - app_data_by_order: &mut HashMap< - order::Uid, - Result, order::app_data::FetchingError>, - >, + app_data_by_order: &mut HashMap>, orders: &mut Vec, ) { // The auction that we receive from the `autopilot` assumes that there @@ -320,19 +317,12 @@ impl AuctionProcessor { remaining_balance.0 -= allocated_balance.0; - app_data_by_order.remove(&order.uid).map_or(true, |result| { - match result { - Err(err) => { - tracing::warn!(order_uid=?order.uid, ?err, "failed to fetch app data for order, excluding from auction"); - false - } - Ok(Some(app_data)) => { - order.app_data = app_data.into(); - true - } - Ok(None) => true - } - }) + // Update order app data if it was fetched. + if let Some(fetched_app_data) = app_data_by_order.remove(&order.uid).flatten() { + order.app_data = fetched_app_data.into(); + } + + true }); } From 917761ebbcdb0727c41d4bac53f44259442128de Mon Sep 17 00:00:00 2001 From: ilya Date: Tue, 21 Jan 2025 15:49:52 +0000 Subject: [PATCH 14/27] Formatting --- crates/driver/src/domain/competition/auction.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/crates/driver/src/domain/competition/auction.rs b/crates/driver/src/domain/competition/auction.rs index 7ea4b5a0a1..ccec0b5780 100644 --- a/crates/driver/src/domain/competition/auction.rs +++ b/crates/driver/src/domain/competition/auction.rs @@ -231,9 +231,14 @@ impl AuctionProcessor { ) -> HashMap> { if let Some(app_data_retriever) = app_data_retriever { join_all(orders.iter().map(|order| async { - let fetched_app_data = app_data_retriever.get(order.app_data.hash()).await.tap_err(|err| { - tracing::warn!(order_uid=?order.uid, ?err, "failed to fetch app data for order"); - }).ok().flatten(); + let fetched_app_data = app_data_retriever + .get(order.app_data.hash()) + .await + .tap_err(|err| { + tracing::warn!(order_uid=?order.uid, ?err, "failed to fetch app data for order"); + }) + .ok() + .flatten(); (order.uid, fetched_app_data) })) From 55b18a7739c0a9d2c4821f72256fec2e83054aa1 Mon Sep 17 00:00:00 2001 From: ilya Date: Tue, 21 Jan 2025 15:56:51 +0000 Subject: [PATCH 15/27] Move app data structs --- .../driver/src/domain/competition/auction.rs | 2 +- .../src/domain/competition/order/app_data.rs | 71 +++++++++++++++++-- .../src/domain/competition/order/mod.rs | 61 +--------------- .../api/routes/solve/dto/solve_request.rs | 2 +- .../driver/src/infra/solver/dto/solution.rs | 2 +- 5 files changed, 69 insertions(+), 69 deletions(-) diff --git a/crates/driver/src/domain/competition/auction.rs b/crates/driver/src/domain/competition/auction.rs index ccec0b5780..bb18fa0a84 100644 --- a/crates/driver/src/domain/competition/auction.rs +++ b/crates/driver/src/domain/competition/auction.rs @@ -444,7 +444,7 @@ impl AuctionProcessor { }, kind: order::Kind::Limit, side: template.order.kind.into(), - app_data: order::AppDataHash(Bytes(template.order.app_data.0)).into(), + app_data: order::app_data::AppDataHash(Bytes(template.order.app_data.0)).into(), buy_token_balance: template.order.buy_token_balance.into(), sell_token_balance: template.order.sell_token_balance.into(), partial: match template.order.partially_fillable { diff --git a/crates/driver/src/domain/competition/order/app_data.rs b/crates/driver/src/domain/competition/order/app_data.rs index 4fa8961e73..9dfdbd1224 100644 --- a/crates/driver/src/domain/competition/order/app_data.rs +++ b/crates/driver/src/domain/competition/order/app_data.rs @@ -1,4 +1,6 @@ use { + crate::util::Bytes, + derive_more::From, futures::FutureExt, moka::future::Cache, reqwest::StatusCode, @@ -26,12 +28,10 @@ pub struct AppDataRetriever(Arc); struct Inner { client: reqwest::Client, base_url: Url, - request_sharing: BoxRequestSharing< - super::AppDataHash, - Result, FetchingError>, - >, + request_sharing: + BoxRequestSharing, FetchingError>>, app_data_validator: app_data::Validator, - cache: Cache>, + cache: Cache>, } impl AppDataRetriever { @@ -48,13 +48,13 @@ impl AppDataRetriever { /// Retrieves the full app-data for the given `app_data` hash, if exists. pub async fn get( &self, - app_data: super::AppDataHash, + app_data: AppDataHash, ) -> Result, FetchingError> { if let Some(app_data) = self.0.cache.get(&app_data).await { return Ok(app_data.clone()); } - let app_data_fut = move |app_data: &super::AppDataHash| { + let app_data_fut = move |app_data: &AppDataHash| { let app_data = *app_data; let self_ = self.clone(); @@ -90,6 +90,63 @@ impl AppDataRetriever { } } +/// The app data associated with an order. +#[derive(Debug, Clone, From)] +pub enum AppData { + /// App data hash. + Hash(AppDataHash), + /// Validated full app data. + Full(Box<::app_data::ValidatedAppData>), +} + +impl Default for AppData { + fn default() -> Self { + Self::Hash(Default::default()) + } +} + +impl AppData { + pub fn hash(&self) -> AppDataHash { + match self { + Self::Hash(hash) => *hash, + Self::Full(data) => AppDataHash(data.hash.0.into()), + } + } +} + +impl From<[u8; APP_DATA_LEN]> for AppData { + fn from(app_data_hash: [u8; APP_DATA_LEN]) -> Self { + Self::Hash(app_data_hash.into()) + } +} + +impl From<::app_data::ValidatedAppData> for AppData { + fn from(value: ::app_data::ValidatedAppData) -> Self { + Self::Full(Box::new(value)) + } +} + +/// The length of the app data hash in bytes. +pub const APP_DATA_LEN: usize = 32; + +/// This is a hash allowing arbitrary user data to be associated with an order. +/// While this type holds the hash, the data itself is uploaded to IPFS. This +/// hash is signed along with the order. +#[derive(Debug, Default, Clone, Copy, Hash, PartialEq, Eq)] +pub struct AppDataHash(pub Bytes<[u8; APP_DATA_LEN]>); + +impl From<[u8; APP_DATA_LEN]> for AppDataHash { + fn from(inner: [u8; APP_DATA_LEN]) -> Self { + Self(inner.into()) + } +} + +impl From for [u8; APP_DATA_LEN] { + fn from(app_data: AppDataHash) -> Self { + app_data.0.into() + } +} + #[derive(Error, Debug)] pub enum FetchingError { #[error("error while sending a request: {0}")] diff --git a/crates/driver/src/domain/competition/order/mod.rs b/crates/driver/src/domain/competition/order/mod.rs index 9e7c597161..dffa4c6a55 100644 --- a/crates/driver/src/domain/competition/order/mod.rs +++ b/crates/driver/src/domain/competition/order/mod.rs @@ -30,7 +30,7 @@ pub struct Order { pub sell: eth::Asset, pub side: Side, pub kind: Kind, - pub app_data: AppData, + pub app_data: app_data::AppData, pub partial: Partial, /// The onchain calls to run before sending user funds to the settlement /// contract. @@ -51,42 +51,6 @@ pub struct Order { pub quote: Option, } -/// The app data associated with an order. -#[derive(Debug, Clone, From)] -pub enum AppData { - /// App data hash. - Hash(AppDataHash), - /// Validated full app data. - Full(Box<::app_data::ValidatedAppData>), -} - -impl Default for AppData { - fn default() -> Self { - Self::Hash(Default::default()) - } -} - -impl AppData { - pub fn hash(&self) -> AppDataHash { - match self { - Self::Hash(hash) => *hash, - Self::Full(data) => AppDataHash(data.hash.0.into()), - } - } -} - -impl From<[u8; APP_DATA_LEN]> for AppData { - fn from(app_data_hash: [u8; APP_DATA_LEN]) -> Self { - Self::Hash(app_data_hash.into()) - } -} - -impl From<::app_data::ValidatedAppData> for AppData { - fn from(value: ::app_data::ValidatedAppData) -> Self { - Self::Full(Box::new(value)) - } -} - /// An amount denominated in the sell token of an [`Order`]. #[derive(Clone, Copy, Debug, Default, Eq, Ord, PartialEq, PartialOrd, From, Into)] pub struct SellAmount(pub eth::U256); @@ -304,27 +268,6 @@ impl From for [u8; UID_LEN] { } } -/// The length of the app data hash in bytes. -pub const APP_DATA_LEN: usize = 32; - -/// This is a hash allowing arbitrary user data to be associated with an order. -/// While this type holds the hash, the data itself is uploaded to IPFS. This -/// hash is signed along with the order. -#[derive(Debug, Default, Clone, Copy, Hash, PartialEq, Eq)] -pub struct AppDataHash(pub Bytes<[u8; APP_DATA_LEN]>); - -impl From<[u8; APP_DATA_LEN]> for AppDataHash { - fn from(inner: [u8; APP_DATA_LEN]) -> Self { - Self(inner.into()) - } -} - -impl From for [u8; APP_DATA_LEN] { - fn from(app_data: AppDataHash) -> Self { - app_data.0.into() - } -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Kind { /// Order intended to be immediately executed. This is the "regular" type of @@ -425,7 +368,7 @@ pub struct Jit { pub receiver: eth::Address, pub valid_to: util::Timestamp, pub partially_fillable: bool, - pub app_data: AppDataHash, + pub app_data: app_data::AppDataHash, pub side: Side, pub sell_token_balance: SellTokenBalance, pub buy_token_balance: BuyTokenBalance, diff --git a/crates/driver/src/infra/api/routes/solve/dto/solve_request.rs b/crates/driver/src/infra/api/routes/solve/dto/solve_request.rs index b773387715..f4cd118fa8 100644 --- a/crates/driver/src/infra/api/routes/solve/dto/solve_request.rs +++ b/crates/driver/src/infra/api/routes/solve/dto/solve_request.rs @@ -254,7 +254,7 @@ struct Order { buy_token_balance: BuyTokenBalance, class: Class, #[serde_as(as = "serialize::Hex")] - app_data: [u8; order::APP_DATA_LEN], + app_data: [u8; order::app_data::APP_DATA_LEN], signing_scheme: SigningScheme, #[serde_as(as = "serialize::Hex")] signature: Vec, diff --git a/crates/driver/src/infra/solver/dto/solution.rs b/crates/driver/src/infra/solver/dto/solution.rs index 3a63a9f54d..316f6f1735 100644 --- a/crates/driver/src/infra/solver/dto/solution.rs +++ b/crates/driver/src/infra/solver/dto/solution.rs @@ -291,7 +291,7 @@ struct JitOrder { partially_fillable: bool, valid_to: u32, #[serde_as(as = "serialize::Hex")] - app_data: [u8; order::APP_DATA_LEN], + app_data: [u8; order::app_data::APP_DATA_LEN], kind: Kind, sell_token_balance: SellTokenBalance, buy_token_balance: BuyTokenBalance, From 0fe9fefec268593971cfe97d078a65e51bd3f45c Mon Sep 17 00:00:00 2001 From: ilya Date: Tue, 21 Jan 2025 15:59:43 +0000 Subject: [PATCH 16/27] Comment --- crates/driver/src/domain/competition/auction.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/crates/driver/src/domain/competition/auction.rs b/crates/driver/src/domain/competition/auction.rs index bb18fa0a84..4d54cf30e3 100644 --- a/crates/driver/src/domain/competition/auction.rs +++ b/crates/driver/src/domain/competition/auction.rs @@ -146,9 +146,8 @@ type Balances = HashMap; impl AuctionProcessor { /// Process the auction by prioritizing the orders and filtering out - /// unfillable orders as well as those that failed to fetch app data. - /// Fetches full app data for each order and returns an auction with - /// updated orders. + /// unfillable orders. Fetches full app data for each order and returns an + /// auction with updated orders. pub async fn prioritize(&self, auction: Auction, solver: ð::H160) -> Auction { Auction { orders: self.prioritize_orders(&auction, solver).await, @@ -156,8 +155,6 @@ impl AuctionProcessor { } } - /// Prioritize well priced and filter out unfillable orders from the given - /// auction. fn prioritize_orders( &self, auction: &Auction, @@ -419,7 +416,7 @@ impl AuctionProcessor { (*cow_amm.address(), cow_amm.validated_template_order(prices, signature_validator, &domain_separator).await) }), ) - .await; + .await; // Convert results to domain format. let domain_separator = From c5956d387dff748dc7ecb4f3e2700eef5da96828 Mon Sep 17 00:00:00 2001 From: ilya Date: Tue, 21 Jan 2025 16:08:35 +0000 Subject: [PATCH 17/27] Metrics --- .../driver/src/domain/competition/auction.rs | 18 +++++++++++++++++- crates/driver/src/infra/observe/metrics.rs | 8 ++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/crates/driver/src/domain/competition/auction.rs b/crates/driver/src/domain/competition/auction.rs index 4d54cf30e3..c8fdf9a427 100644 --- a/crates/driver/src/domain/competition/auction.rs +++ b/crates/driver/src/domain/competition/auction.rs @@ -7,7 +7,13 @@ use { liquidity, time, }, - infra::{self, blockchain, config::file::OrderPriorityStrategy, observe, Ethereum}, + infra::{ + self, + blockchain, + config::file::OrderPriorityStrategy, + observe::{self, metrics}, + Ethereum, + }, util::{self, Bytes}, }, chrono::{Duration, Utc}, @@ -227,6 +233,11 @@ impl AuctionProcessor { orders: &[order::Order], ) -> HashMap> { if let Some(app_data_retriever) = app_data_retriever { + let _timer = metrics::get() + .auction_preprocessing + .with_label_values(&["fetch_app_data"]) + .start_timer(); + join_all(orders.iter().map(|order| async { let fetched_app_data = app_data_retriever .get(order.app_data.hash()) @@ -355,6 +366,11 @@ impl AuctionProcessor { }) .collect::>(); + let _timer = metrics::get() + .auction_preprocessing + .with_label_values(&["fetch_balances"]) + .start_timer(); + join_all( traders .into_iter() diff --git a/crates/driver/src/infra/observe/metrics.rs b/crates/driver/src/infra/observe/metrics.rs index 237206c668..3135a4a30e 100644 --- a/crates/driver/src/infra/observe/metrics.rs +++ b/crates/driver/src/infra/observe/metrics.rs @@ -22,6 +22,14 @@ pub struct Metrics { /// How many tokens detected by specific solver and strategy. #[metric(labels("solver", "strategy"))] pub bad_tokens_detected: prometheus::IntCounterVec, + /// Time spent in the auction preprocessing stage. + #[metric( + labels("stage"), + buckets( + 0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 5.0 + ) + )] + pub auction_preprocessing: prometheus::HistogramVec, } /// Setup the metrics registry. From c7c51b038eca4a3829516d41665039419231944c Mon Sep 17 00:00:00 2001 From: ilya Date: Tue, 21 Jan 2025 16:10:29 +0000 Subject: [PATCH 18/27] Total metric --- crates/driver/src/domain/competition/auction.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/driver/src/domain/competition/auction.rs b/crates/driver/src/domain/competition/auction.rs index c8fdf9a427..a1e1de4804 100644 --- a/crates/driver/src/domain/competition/auction.rs +++ b/crates/driver/src/domain/competition/auction.rs @@ -155,6 +155,11 @@ impl AuctionProcessor { /// unfillable orders. Fetches full app data for each order and returns an /// auction with updated orders. pub async fn prioritize(&self, auction: Auction, solver: ð::H160) -> Auction { + let _timer = metrics::get() + .auction_preprocessing + .with_label_values(&["total"]) + .start_timer(); + Auction { orders: self.prioritize_orders(&auction, solver).await, ..auction From a7a09eeaf152665ef96f0b9a53c11ee501d42ffb Mon Sep 17 00:00:00 2001 From: ilya Date: Tue, 21 Jan 2025 18:38:26 +0000 Subject: [PATCH 19/27] Enable in e2e tests --- crates/e2e/src/setup/colocation.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/e2e/src/setup/colocation.rs b/crates/e2e/src/setup/colocation.rs index e4b51244a0..351313031b 100644 --- a/crates/e2e/src/setup/colocation.rs +++ b/crates/e2e/src/setup/colocation.rs @@ -162,6 +162,8 @@ factory = "{:?}" let config_file = config_tmp_file(format!( r#" +flashloans-enabled = true + [contracts] gp-v2-settlement = "{:?}" weth = "{:?}" From 146ababaa9f5237fbbc3dd2437370cb79a721af1 Mon Sep 17 00:00:00 2001 From: ilya Date: Mon, 27 Jan 2025 11:05:41 +0000 Subject: [PATCH 20/27] Optimized collect_orders_app_data --- .../driver/src/domain/competition/auction.rs | 68 +++++++++++-------- .../src/domain/competition/order/app_data.rs | 6 +- 2 files changed, 42 insertions(+), 32 deletions(-) diff --git a/crates/driver/src/domain/competition/auction.rs b/crates/driver/src/domain/competition/auction.rs index a1e1de4804..ba686fc8ac 100644 --- a/crates/driver/src/domain/competition/auction.rs +++ b/crates/driver/src/domain/competition/auction.rs @@ -202,7 +202,7 @@ impl AuctionProcessor { let start = std::time::Instant::now(); orders.extend(rt.block_on(Self::cow_amm_orders(ð, &tokens, &cow_amms, signature_validator.as_ref()))); sorting::sort_orders(&mut orders, &tokens, &solver, &order_comparators); - let (mut balances, mut app_data_by_order) = + let (mut balances, mut app_data_by_hash) = rt.block_on(async { tokio::join!( Self::fetch_balances(ð, &orders), @@ -210,7 +210,7 @@ impl AuctionProcessor { ) }); - Self::filter_orders(&mut balances, &mut app_data_by_order, &mut orders); + Self::filter_orders(&mut balances, &mut app_data_by_hash, &mut orders); tracing::debug!(auction_id = new_id.0, time =? start.elapsed(), "auction preprocessing done"); orders @@ -232,42 +232,52 @@ impl AuctionProcessor { } /// Fetches the app data for all orders in the auction. - /// Returns a map from order UIDs to the result of fetching the app data. + /// Returns a map from app data hash to the fetched app data. async fn collect_orders_app_data( app_data_retriever: Option, orders: &[order::Order], - ) -> HashMap> { - if let Some(app_data_retriever) = app_data_retriever { - let _timer = metrics::get() - .auction_preprocessing - .with_label_values(&["fetch_app_data"]) - .start_timer(); - - join_all(orders.iter().map(|order| async { - let fetched_app_data = app_data_retriever - .get(order.app_data.hash()) - .await - .tap_err(|err| { - tracing::warn!(order_uid=?order.uid, ?err, "failed to fetch app data for order"); - }) - .ok() - .flatten(); + ) -> HashMap { + let Some(app_data_retriever) = app_data_retriever else { + return Default::default(); + }; - (order.uid, fetched_app_data) - })) - .await - .into_iter() - .collect::>() - } else { - Default::default() - } + let _timer = metrics::get() + .auction_preprocessing + .with_label_values(&["fetch_app_data"]) + .start_timer(); + + join_all( + orders + .iter() + .map(|order| order.app_data.hash()) + .unique() + .map(|app_data_hash| { + let app_data_retriever = app_data_retriever.clone(); + async move { + let fetched_app_data = app_data_retriever + .get(&app_data_hash) + .await + .tap_err(|err| { + tracing::warn!(?app_data_hash, ?err, "failed to fetch app data"); + }) + .ok() + .flatten(); + + (app_data_hash, fetched_app_data) + } + }), + ) + .await + .into_iter() + .filter_map(|(app_data_hash, app_data)| app_data.map(|app_data| (app_data_hash, app_data))) + .collect::>() } /// Removes orders that cannot be filled due to missing funds of the owner /// and updates the fetched app data. fn filter_orders( balances: &mut Balances, - app_data_by_order: &mut HashMap>, + app_data_by_hash: &mut HashMap, orders: &mut Vec, ) { // The auction that we receive from the `autopilot` assumes that there @@ -336,7 +346,7 @@ impl AuctionProcessor { remaining_balance.0 -= allocated_balance.0; // Update order app data if it was fetched. - if let Some(fetched_app_data) = app_data_by_order.remove(&order.uid).flatten() { + if let Some(fetched_app_data) = app_data_by_hash.remove(&order.app_data.hash()) { order.app_data = fetched_app_data.into(); } diff --git a/crates/driver/src/domain/competition/order/app_data.rs b/crates/driver/src/domain/competition/order/app_data.rs index 9dfdbd1224..ed2d9cd5dc 100644 --- a/crates/driver/src/domain/competition/order/app_data.rs +++ b/crates/driver/src/domain/competition/order/app_data.rs @@ -48,9 +48,9 @@ impl AppDataRetriever { /// Retrieves the full app-data for the given `app_data` hash, if exists. pub async fn get( &self, - app_data: AppDataHash, + app_data: &AppDataHash, ) -> Result, FetchingError> { - if let Some(app_data) = self.0.cache.get(&app_data).await { + if let Some(app_data) = self.0.cache.get(app_data).await { return Ok(app_data.clone()); } @@ -85,7 +85,7 @@ impl AppDataRetriever { self.0 .request_sharing - .shared_or_else(app_data, app_data_fut) + .shared_or_else(*app_data, app_data_fut) .await } } From 0b0f397bd179b76dbb9ee2eed65763c7a14dff1b Mon Sep 17 00:00:00 2001 From: ilya Date: Mon, 27 Jan 2025 11:13:30 +0000 Subject: [PATCH 21/27] Naming --- crates/driver/src/domain/competition/auction.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/driver/src/domain/competition/auction.rs b/crates/driver/src/domain/competition/auction.rs index ba686fc8ac..30e23b9261 100644 --- a/crates/driver/src/domain/competition/auction.rs +++ b/crates/driver/src/domain/competition/auction.rs @@ -210,7 +210,7 @@ impl AuctionProcessor { ) }); - Self::filter_orders(&mut balances, &mut app_data_by_hash, &mut orders); + Self::update_orders(&mut balances, &mut app_data_by_hash, &mut orders); tracing::debug!(auction_id = new_id.0, time =? start.elapsed(), "auction preprocessing done"); orders @@ -275,7 +275,7 @@ impl AuctionProcessor { /// Removes orders that cannot be filled due to missing funds of the owner /// and updates the fetched app data. - fn filter_orders( + fn update_orders( balances: &mut Balances, app_data_by_hash: &mut HashMap, orders: &mut Vec, From 83e07b6efc2c191a8d821b6665698e79f0cae8ed Mon Sep 17 00:00:00 2001 From: ilya Date: Mon, 27 Jan 2025 12:04:23 +0000 Subject: [PATCH 22/27] Config refactoring --- crates/driver/src/infra/cli.rs | 4 --- crates/driver/src/infra/config/file/mod.rs | 37 ++++++++++++---------- crates/driver/src/run.rs | 11 ++++--- crates/driver/src/tests/setup/driver.rs | 5 ++- crates/driver/src/tests/setup/mod.rs | 2 +- crates/e2e/src/setup/colocation.rs | 2 +- 6 files changed, 31 insertions(+), 30 deletions(-) diff --git a/crates/driver/src/infra/cli.rs b/crates/driver/src/infra/cli.rs index b259837a65..9159f990f7 100644 --- a/crates/driver/src/infra/cli.rs +++ b/crates/driver/src/infra/cli.rs @@ -21,10 +21,6 @@ pub struct Args { #[clap(long, env)] pub ethrpc: Url, - /// The orderbook API base URL. - #[clap(long, env)] - pub orderbook_url: Url, - /// Path to the driver configuration file. This file should be in TOML /// format. For an example see /// https://github.com/cowprotocol/services/blob/main/crates/driver/example.toml. diff --git a/crates/driver/src/infra/config/file/mod.rs b/crates/driver/src/infra/config/file/mod.rs index 69aa45067d..79761edc2d 100644 --- a/crates/driver/src/infra/config/file/mod.rs +++ b/crates/driver/src/infra/config/file/mod.rs @@ -738,25 +738,28 @@ impl Default for BadTokenDetectionConfig { } #[serde_as] -#[derive(Clone, Debug, Deserialize)] -#[serde(rename_all = "kebab-case", deny_unknown_fields)] -pub struct Flashloans { - /// Whether the flashloans feature is enabled. - #[serde(default, rename = "flashloans-enabled")] - pub enabled: bool, +#[derive(Clone, Debug, Deserialize, Default)] +#[serde( + rename_all = "kebab-case", + deny_unknown_fields, + tag = "flashloans-enabled" +)] +pub enum Flashloans { + /// Flashloans feature is disabled + #[serde(rename = "false")] + #[default] + Disabled, - /// The maximum number of app-data entries in the cache. - #[serde( - default = "default_flashloans_cache_size", - rename = "flashloans-cache-size" - )] - pub cache_size: u64, -} + /// Flashloans feature is enabled + #[serde(rename = "true")] + Enabled { + /// The base URL of the orderbook to fetch app-data from + orderbook_url: Url, -impl Default for Flashloans { - fn default() -> Self { - serde_json::from_str("{}").expect("Flashloans uses default values") - } + /// The maximum number of app-data entries in the cache + #[serde(default = "default_flashloans_cache_size")] + cache_size: u64, + }, } fn default_metrics_bad_token_detector_failure_ratio() -> f64 { diff --git a/crates/driver/src/run.rs b/crates/driver/src/run.rs index 5ded6a1d06..8b2ae5e120 100644 --- a/crates/driver/src/run.rs +++ b/crates/driver/src/run.rs @@ -54,10 +54,13 @@ async fn run_with(args: cli::Args, addr_sender: Option Some(AppDataRetriever::new(orderbook_url.clone(), *cache_size)), + config::file::Flashloans::Disabled => None, + }; let serve = Api { solvers: solvers(&config, ð).await, liquidity: liquidity(&config, ð).await, diff --git a/crates/driver/src/tests/setup/driver.rs b/crates/driver/src/tests/setup/driver.rs index 87acb4257f..d1a3a3b274 100644 --- a/crates/driver/src/tests/setup/driver.rs +++ b/crates/driver/src/tests/setup/driver.rs @@ -21,6 +21,7 @@ pub struct Config { pub enable_simulation: bool, pub mempools: Vec, pub order_priority_strategies: Vec, + pub orderbook: Orderbook, } pub struct Driver { @@ -34,7 +35,6 @@ impl Driver { config: &Config, solvers: &Vec<(Solver, SocketAddr)>, blockchain: &Blockchain, - orderbook: &Orderbook, ) -> Self { let (config_file, config_temp_path) = match config.config_file.as_ref() { Some(config_file) => (config_file.to_owned(), None), @@ -50,8 +50,6 @@ impl Driver { "0.0.0.0:0".to_owned(), "--ethrpc".to_owned(), blockchain.web3_url.clone(), - "--orderbook-url".to_owned(), - format!("http://{}", orderbook.addr).to_owned(), "--config".to_owned(), config_file.to_str().unwrap().to_owned(), ]; @@ -211,6 +209,7 @@ async fn create_config_file( }; write!(file, "{simulation}").unwrap(); writeln!(file, "flashloans-enabled = true").unwrap(); + writeln!(file, r#"orderbook-url = "{}""#, config.orderbook.addr).unwrap(); write!( file, r#"[contracts] diff --git a/crates/driver/src/tests/setup/mod.rs b/crates/driver/src/tests/setup/mod.rs index c843e9c2a4..32b7c9a99b 100644 --- a/crates/driver/src/tests/setup/mod.rs +++ b/crates/driver/src/tests/setup/mod.rs @@ -968,10 +968,10 @@ impl Setup { enable_simulation: self.enable_simulation, mempools: self.mempools, order_priority_strategies: self.order_priority_strategies, + orderbook, }, &solvers_with_address, &blockchain, - &orderbook, ) .await; diff --git a/crates/e2e/src/setup/colocation.rs b/crates/e2e/src/setup/colocation.rs index 351313031b..b4b6b473bd 100644 --- a/crates/e2e/src/setup/colocation.rs +++ b/crates/e2e/src/setup/colocation.rs @@ -163,6 +163,7 @@ factory = "{:?}" let config_file = config_tmp_file(format!( r#" flashloans-enabled = true +orderbook-url = "http://localhost:8080" [contracts] gp-v2-settlement = "{:?}" @@ -188,7 +189,6 @@ mempool = "public" let args = vec![ "driver".to_string(), format!("--config={}", config_file.display()), - "--orderbook-url=http://localhost:8080".to_string(), format!("--ethrpc={NODE_HOST}"), ]; From 7bb726a5c0b6225038af9432740f72e8dd5025b3 Mon Sep 17 00:00:00 2001 From: ilya Date: Mon, 27 Jan 2025 12:05:04 +0000 Subject: [PATCH 23/27] Fix --- crates/driver/src/domain/competition/auction.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/driver/src/domain/competition/auction.rs b/crates/driver/src/domain/competition/auction.rs index 30e23b9261..f57ad741ca 100644 --- a/crates/driver/src/domain/competition/auction.rs +++ b/crates/driver/src/domain/competition/auction.rs @@ -346,8 +346,8 @@ impl AuctionProcessor { remaining_balance.0 -= allocated_balance.0; // Update order app data if it was fetched. - if let Some(fetched_app_data) = app_data_by_hash.remove(&order.app_data.hash()) { - order.app_data = fetched_app_data.into(); + if let Some(fetched_app_data) = app_data_by_hash.get(&order.app_data.hash()) { + order.app_data = fetched_app_data.clone().into(); } true From 03ba548bc228c27f61c23d87d2658781a0c8ab41 Mon Sep 17 00:00:00 2001 From: ilya Date: Mon, 27 Jan 2025 12:08:45 +0000 Subject: [PATCH 24/27] Config naming --- crates/driver/src/infra/config/file/mod.rs | 14 +++++++------- crates/driver/src/infra/config/mod.rs | 4 ++-- crates/driver/src/run.rs | 4 ++-- crates/driver/src/tests/setup/driver.rs | 2 +- crates/e2e/src/setup/colocation.rs | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/crates/driver/src/infra/config/file/mod.rs b/crates/driver/src/infra/config/file/mod.rs index 79761edc2d..11bfa5dc0c 100644 --- a/crates/driver/src/infra/config/file/mod.rs +++ b/crates/driver/src/infra/config/file/mod.rs @@ -76,7 +76,7 @@ struct Config { /// Configuration for flashloans. #[serde(default, flatten)] - flashloans: Flashloans, + flashloans: AppDataFetching, } #[serde_as] @@ -742,22 +742,22 @@ impl Default for BadTokenDetectionConfig { #[serde( rename_all = "kebab-case", deny_unknown_fields, - tag = "flashloans-enabled" + tag = "app-data-fetching-enabled" )] -pub enum Flashloans { - /// Flashloans feature is disabled +pub enum AppDataFetching { + /// App-data fetching is disabled #[serde(rename = "false")] #[default] Disabled, - /// Flashloans feature is enabled + /// App-data fetching is enabled #[serde(rename = "true")] Enabled { /// The base URL of the orderbook to fetch app-data from orderbook_url: Url, /// The maximum number of app-data entries in the cache - #[serde(default = "default_flashloans_cache_size")] + #[serde(default = "default_app_data_cache_size")] cache_size: u64, }, } @@ -787,6 +787,6 @@ fn default_metrics_bad_token_detector_freeze_time() -> Duration { /// According to statistics, the average size of the app-data is ~800 bytes. /// With this default, the approximate size of the cache will be ~1.6 MB. -fn default_flashloans_cache_size() -> u64 { +fn default_app_data_cache_size() -> u64 { 2000 } diff --git a/crates/driver/src/infra/config/mod.rs b/crates/driver/src/infra/config/mod.rs index 8d4ab34ae1..e20146059b 100644 --- a/crates/driver/src/infra/config/mod.rs +++ b/crates/driver/src/infra/config/mod.rs @@ -3,7 +3,7 @@ use { domain::eth, infra::{ blockchain, - config::file::{Flashloans, GasEstimatorType, OrderPriorityStrategy}, + config::file::{AppDataFetching, GasEstimatorType, OrderPriorityStrategy}, liquidity, mempool, simulator, @@ -30,5 +30,5 @@ pub struct Config { pub order_priority_strategies: Vec, pub archive_node_url: Option, pub simulation_bad_token_max_age: Duration, - pub flashloans: Flashloans, + pub flashloans: AppDataFetching, } diff --git a/crates/driver/src/run.rs b/crates/driver/src/run.rs index 8b2ae5e120..65308a16de 100644 --- a/crates/driver/src/run.rs +++ b/crates/driver/src/run.rs @@ -55,11 +55,11 @@ async fn run_with(args: cli::Args, addr_sender: Option Some(AppDataRetriever::new(orderbook_url.clone(), *cache_size)), - config::file::Flashloans::Disabled => None, + config::file::AppDataFetching::Disabled => None, }; let serve = Api { solvers: solvers(&config, ð).await, diff --git a/crates/driver/src/tests/setup/driver.rs b/crates/driver/src/tests/setup/driver.rs index d1a3a3b274..57c0b82a1c 100644 --- a/crates/driver/src/tests/setup/driver.rs +++ b/crates/driver/src/tests/setup/driver.rs @@ -208,7 +208,7 @@ async fn create_config_file( "# }; write!(file, "{simulation}").unwrap(); - writeln!(file, "flashloans-enabled = true").unwrap(); + writeln!(file, "app-data-fetching-enabled = true").unwrap(); writeln!(file, r#"orderbook-url = "{}""#, config.orderbook.addr).unwrap(); write!( file, diff --git a/crates/e2e/src/setup/colocation.rs b/crates/e2e/src/setup/colocation.rs index b4b6b473bd..cae8e9d697 100644 --- a/crates/e2e/src/setup/colocation.rs +++ b/crates/e2e/src/setup/colocation.rs @@ -162,7 +162,7 @@ factory = "{:?}" let config_file = config_tmp_file(format!( r#" -flashloans-enabled = true +app-data-fetching-enabled = true orderbook-url = "http://localhost:8080" [contracts] From 0a7eaa2d36d19542c67bfeb823b1285acf668bf4 Mon Sep 17 00:00:00 2001 From: ilya Date: Mon, 27 Jan 2025 12:55:29 +0000 Subject: [PATCH 25/27] Config fix --- crates/driver/src/infra/config/file/mod.rs | 46 ++++++++++++++++------ crates/driver/src/tests/setup/driver.rs | 7 +++- 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/crates/driver/src/infra/config/file/mod.rs b/crates/driver/src/infra/config/file/mod.rs index 11bfa5dc0c..1924df39d0 100644 --- a/crates/driver/src/infra/config/file/mod.rs +++ b/crates/driver/src/infra/config/file/mod.rs @@ -2,7 +2,7 @@ pub use load::load; use { crate::{domain::eth, infra, util::serialize}, reqwest::Url, - serde::{Deserialize, Serialize}, + serde::{Deserialize, Deserializer, Serialize}, serde_with::serde_as, solver::solver::Arn, std::{collections::HashMap, time::Duration}, @@ -737,31 +737,53 @@ impl Default for BadTokenDetectionConfig { } } -#[serde_as] -#[derive(Clone, Debug, Deserialize, Default)] -#[serde( - rename_all = "kebab-case", - deny_unknown_fields, - tag = "app-data-fetching-enabled" -)] +#[derive(Clone, Debug)] pub enum AppDataFetching { /// App-data fetching is disabled - #[serde(rename = "false")] - #[default] Disabled, /// App-data fetching is enabled - #[serde(rename = "true")] Enabled { /// The base URL of the orderbook to fetch app-data from orderbook_url: Url, /// The maximum number of app-data entries in the cache - #[serde(default = "default_app_data_cache_size")] cache_size: u64, }, } +impl<'de> Deserialize<'de> for AppDataFetching { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[serde_as] + #[derive(Deserialize)] + #[serde(rename_all = "kebab-case", deny_unknown_fields)] + struct Helper { + #[serde(default)] + app_data_fetching_enabled: bool, + orderbook_url: Option, + #[serde(default = "default_app_data_cache_size")] + cache_size: u64, + } + + let helper = Helper::deserialize(deserializer)?; + match helper.app_data_fetching_enabled { + false => Ok(AppDataFetching::Disabled), + true => { + let orderbook_url = helper + .orderbook_url + .ok_or_else(|| serde::de::Error::custom("Missing `orderbook_url` field"))?; + Ok(AppDataFetching::Enabled { + orderbook_url, + cache_size: helper.cache_size, + }) + } + } + } +} + fn default_metrics_bad_token_detector_failure_ratio() -> f64 { 0.9 } diff --git a/crates/driver/src/tests/setup/driver.rs b/crates/driver/src/tests/setup/driver.rs index 57c0b82a1c..e2ed7478f4 100644 --- a/crates/driver/src/tests/setup/driver.rs +++ b/crates/driver/src/tests/setup/driver.rs @@ -209,7 +209,12 @@ async fn create_config_file( }; write!(file, "{simulation}").unwrap(); writeln!(file, "app-data-fetching-enabled = true").unwrap(); - writeln!(file, r#"orderbook-url = "{}""#, config.orderbook.addr).unwrap(); + writeln!( + file, + r#"orderbook-url = "http://{}""#, + config.orderbook.addr + ) + .unwrap(); write!( file, r#"[contracts] From 39e3452814da7d58b82c6c636b93257f4dd75fd7 Mon Sep 17 00:00:00 2001 From: ilya Date: Mon, 27 Jan 2025 13:21:09 +0000 Subject: [PATCH 26/27] Naming --- crates/driver/src/infra/config/file/load.rs | 2 +- crates/driver/src/infra/config/file/mod.rs | 4 ++-- crates/driver/src/infra/config/mod.rs | 2 +- crates/driver/src/run.rs | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/driver/src/infra/config/file/load.rs b/crates/driver/src/infra/config/file/load.rs index e9dcac8b72..2e642ead9c 100644 --- a/crates/driver/src/infra/config/file/load.rs +++ b/crates/driver/src/infra/config/file/load.rs @@ -376,6 +376,6 @@ pub async fn load(chain: Chain, path: &Path) -> infra::Config { order_priority_strategies: config.order_priority_strategies, archive_node_url: config.archive_node_url, simulation_bad_token_max_age: config.simulation_bad_token_max_age, - flashloans: config.flashloans, + app_data_fetching: config.app_data_fetching, } } diff --git a/crates/driver/src/infra/config/file/mod.rs b/crates/driver/src/infra/config/file/mod.rs index 1924df39d0..61d5b86401 100644 --- a/crates/driver/src/infra/config/file/mod.rs +++ b/crates/driver/src/infra/config/file/mod.rs @@ -74,9 +74,9 @@ struct Config { )] simulation_bad_token_max_age: Duration, - /// Configuration for flashloans. + /// Configuration for the app-data fetching. #[serde(default, flatten)] - flashloans: AppDataFetching, + app_data_fetching: AppDataFetching, } #[serde_as] diff --git a/crates/driver/src/infra/config/mod.rs b/crates/driver/src/infra/config/mod.rs index e20146059b..9c3dd2c456 100644 --- a/crates/driver/src/infra/config/mod.rs +++ b/crates/driver/src/infra/config/mod.rs @@ -30,5 +30,5 @@ pub struct Config { pub order_priority_strategies: Vec, pub archive_node_url: Option, pub simulation_bad_token_max_age: Duration, - pub flashloans: AppDataFetching, + pub app_data_fetching: AppDataFetching, } diff --git a/crates/driver/src/run.rs b/crates/driver/src/run.rs index 65308a16de..8d62050824 100644 --- a/crates/driver/src/run.rs +++ b/crates/driver/src/run.rs @@ -54,7 +54,7 @@ async fn run_with(args: cli::Args, addr_sender: Option Date: Mon, 27 Jan 2025 16:55:57 +0000 Subject: [PATCH 27/27] Nit --- crates/driver/src/infra/config/file/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/driver/src/infra/config/file/mod.rs b/crates/driver/src/infra/config/file/mod.rs index 61d5b86401..80aa76bf64 100644 --- a/crates/driver/src/infra/config/file/mod.rs +++ b/crates/driver/src/infra/config/file/mod.rs @@ -774,7 +774,7 @@ impl<'de> Deserialize<'de> for AppDataFetching { true => { let orderbook_url = helper .orderbook_url - .ok_or_else(|| serde::de::Error::custom("Missing `orderbook_url` field"))?; + .ok_or_else(|| serde::de::Error::custom("Missing `orderbook-url` field"))?; Ok(AppDataFetching::Enabled { orderbook_url, cache_size: helper.cache_size,