Make subgraph URL's configurable (#2126)

# Description
<!--- Describe your changes to provide context for reviewers, including
why it is needed -->
This PR aims to fix
[2105](#2105)

# Changes
The subgraph base url is read from the environment variables.

## How to test
All unit tests still pass

Co-authored-by: narcis <[email protected]>

crates/autopilot/src/run.rs

@@ -307,6 +307,7 @@ pub async fn run(args: Arguments) {
             .unwrap_or_else(|| BalancerFactoryKind::for_chain(chain_id));
         let contracts = BalancerContracts::new(&web3, factories).await.unwrap();
         match BalancerPoolFetcher::new(
+            &args.shared.graph_api_base_url,
             chain_id,
             block_retriever.clone(),
             token_info_fetcher.clone(),
@@ -335,6 +336,7 @@ pub async fn run(args: Arguments) {
     };
     let uniswap_v3_pool_fetcher = if baseline_sources.contains(&BaselineSource::UniswapV3) {
         match UniswapV3PoolFetcher::new(
+            &args.shared.graph_api_base_url,
             chain_id,
             web3.clone(),
             http_factory.create(),

crates/driver/Cargo.toml

@@ -26,6 +26,7 @@ futures = "0.3"
 hex = "0.4"
 hex-literal = "0.4"
 hyper = "0.14"
+lazy_static = { workspace = true }
 indexmap = { version = "2", features = ["serde"] }
 itertools = "0.11"
 num = "0.4"

crates/driver/example.toml

@@ -34,6 +34,7 @@ base-tokens = [
     "0xDEf1CA1fb7FBcDC777520aa7f396b4E015F497aB",
     "0x6B175474E89094C44Da98b954EedeAC495271d0F",
 ]
+graph-api-base-url = "https://api.thegraph.com/subgraphs/name/"
 
 # [[liquidity.uniswap-v2]] # Uniswap V2 configuration
 # preset = "uniswap-v2" # or "sushi-swap", "honeyswap", "baoswap", "pancake-swap", etc.
@@ -74,4 +75,3 @@ base-tokens = [
 # [[liquidity.uniswap-v3]] # Custom Uniswap V3 configuration
 # router = "0xE592427A0AEce92De3Edee1F18E0157C05861564"
 # max_pools_to_initialize = 100 # how many of the deepest pools to initialise on startup
-

crates/driver/src/boundary/liquidity/balancer/v2/mod.rs

@@ -178,6 +178,7 @@ async fn init_liquidity(
 
     let balancer_pool_fetcher = Arc::new(
         BalancerPoolFetcher::new(
+            &config.graph_api_base_url,
             eth.network().chain.into(),
             block_retriever.clone(),
             token_info_fetcher.clone(),

crates/driver/src/boundary/liquidity/uniswap/v3.rs

@@ -135,6 +135,7 @@ async fn init_liquidity(
 
     let pool_fetcher = Arc::new(
         UniswapV3PoolFetcher::new(
+            &config.graph_api_base_url,
             eth.network().chain.into(),
             web3.clone(),
             boundary::liquidity::http_client(),

crates/driver/src/infra/config/file/load.rs

@@ -4,10 +4,18 @@ use {
         infra::{self, blockchain, config::file, liquidity, mempool, simulator, solver},
     },
     futures::future::join_all,
+    lazy_static::lazy_static,
+    reqwest::Url,
     std::{path::Path, time::Duration},
     tokio::fs,
 };
 
+lazy_static! {
+    pub static ref DEFAULT_GRAPH_API_BASE_URL: Url =
+        Url::parse("https://api.thegraph.com/subgraphs/name/")
+            .expect("invalid default Graph API base URL");
+}
+
 /// Load the driver configuration from a TOML file for the specifed Ethereum
 /// network.
 ///
@@ -35,7 +43,10 @@ pub async fn load(network: &blockchain::Network, path: &Path) -> infra::Config {
         network.chain,
         "The configured chain ID does not match connected Ethereum node"
     );
-
+    let graph_api_base_url = config
+        .liquidity
+        .graph_api_base_url
+        .unwrap_or(DEFAULT_GRAPH_API_BASE_URL.clone());
     infra::Config {
         solvers: join_all(config.solvers.into_iter().map(|config| async move {
             let account = match config.account {
@@ -164,7 +175,10 @@ pub async fn load(network: &blockchain::Network, path: &Path) -> infra::Config {
                         max_pools_to_initialize,
                         ..match preset {
                             file::UniswapV3Preset::UniswapV3 => {
-                                liquidity::config::UniswapV3::uniswap_v3(&network.id)
+                                liquidity::config::UniswapV3::uniswap_v3(
+                                    &graph_api_base_url,
+                                    &network.id,
+                                )
                             }
                         }
                         .expect("no Uniswap V3 preset for current network")
@@ -175,6 +189,7 @@ pub async fn load(network: &blockchain::Network, path: &Path) -> infra::Config {
                     } => liquidity::config::UniswapV3 {
                         router: router.into(),
                         max_pools_to_initialize,
+                        graph_api_base_url: graph_api_base_url.clone(),
                     },
                 })
                 .collect(),
@@ -191,7 +206,10 @@ pub async fn load(network: &blockchain::Network, path: &Path) -> infra::Config {
                         pool_deny_list: pool_deny_list.clone(),
                         ..match preset {
                             file::BalancerV2Preset::BalancerV2 => {
-                                liquidity::config::BalancerV2::balancer_v2(&network.id)
+                                liquidity::config::BalancerV2::balancer_v2(
+                                    &graph_api_base_url,
+                                    &network.id,
+                                )
                             }
                         }
                         .expect("no Balancer V2 preset for current network")
@@ -224,6 +242,7 @@ pub async fn load(network: &blockchain::Network, path: &Path) -> infra::Config {
                             .map(eth::ContractAddress::from)
                             .collect(),
                         pool_deny_list: pool_deny_list.clone(),
+                        graph_api_base_url: graph_api_base_url.clone(),
                     },
                 })
                 .collect(),

crates/driver/src/infra/config/file/mod.rs

@@ -293,6 +293,9 @@ struct LiquidityConfig {
     /// Liquidity provided by a Balancer V2 compatible contract.
     #[serde(default)]
     balancer_v2: Vec<BalancerV2Config>,
+
+    /// The base URL used to connect to subgraph clients.
+    graph_api_base_url: Option<Url>,
 }
 
 #[derive(Clone, Debug, Deserialize)]

crates/driver/src/infra/liquidity/config.rs

@@ -1,6 +1,7 @@
 use {
     crate::{domain::eth, infra::blockchain::contracts::deployment_address},
     hex_literal::hex,
+    reqwest::Url,
     std::{collections::HashSet, time::Duration},
 };
 
@@ -121,22 +122,26 @@ impl Swapr {
 }
 
 /// Uniswap V3 liquidity fetching options.
-#[derive(Clone, Copy, Debug)]
+#[derive(Clone, Debug)]
 pub struct UniswapV3 {
     /// The address of the Uniswap V3 compatible router contract.
     pub router: eth::ContractAddress,
 
     /// How many pools should be initialized during start up.
     pub max_pools_to_initialize: usize,
+
+    /// The base URL used to connect to subgraph clients.
+    pub graph_api_base_url: Url,
 }
 
 impl UniswapV3 {
     /// Returns the liquidity configuration for Uniswap V3.
     #[allow(clippy::self_named_constructors)]
-    pub fn uniswap_v3(network: &eth::NetworkId) -> Option<Self> {
+    pub fn uniswap_v3(graph_api_base_url: &Url, network: &eth::NetworkId) -> Option<Self> {
         Some(Self {
             router: deployment_address(contracts::UniswapV3SwapRouter::raw_contract(), network)?,
             max_pools_to_initialize: 100,
+            graph_api_base_url: graph_api_base_url.clone(),
         })
     }
 }
@@ -168,12 +173,15 @@ pub struct BalancerV2 {
     /// pools to get "bricked". This configuration allows those pools to be
     /// ignored.
     pub pool_deny_list: Vec<eth::H256>,
+
+    /// The base URL used to connect to subgraph clients.
+    pub graph_api_base_url: Url,
 }
 
 impl BalancerV2 {
     /// Returns the liquidity configuration for Balancer V2.
     #[allow(clippy::self_named_constructors)]
-    pub fn balancer_v2(network: &eth::NetworkId) -> Option<Self> {
+    pub fn balancer_v2(graph_api_base_url: &Url, network: &eth::NetworkId) -> Option<Self> {
         let factory_addresses =
             |contracts: &[&ethcontract::Contract]| -> Vec<eth::ContractAddress> {
                 contracts
@@ -205,6 +213,7 @@ impl BalancerV2 {
                 contracts::BalancerV2ComposableStablePoolFactoryV5::raw_contract(),
             ]),
             pool_deny_list: Vec::new(),
+            graph_api_base_url: graph_api_base_url.clone(),
         })
     }
 }

crates/e2e/src/setup/colocation.rs

@@ -51,6 +51,7 @@ account = "0x{}"
 
 [liquidity]
 base-tokens = []
+graph-api-base-url = "https://api.thegraph.com/subgraphs/name/"
 
 [[liquidity.uniswap-v2]]
 router = "{:?}"

crates/orderbook/src/run.rs

@@ -291,6 +291,7 @@ pub async fn run(args: Arguments) {
             .unwrap_or_else(|| BalancerFactoryKind::for_chain(chain_id));
         let contracts = BalancerContracts::new(&web3, factories).await.unwrap();
         match BalancerPoolFetcher::new(
+            &args.shared.graph_api_base_url,
             chain_id,
             block_retriever.clone(),
             token_info_fetcher.clone(),
@@ -319,6 +320,7 @@ pub async fn run(args: Arguments) {
     };
     let uniswap_v3_pool_fetcher = if baseline_sources.contains(&BaselineSource::UniswapV3) {
         match UniswapV3PoolFetcher::new(
+            &args.shared.graph_api_base_url,
             chain_id,
             web3.clone(),
             http_factory.create(),

crates/shared/src/arguments.rs

@@ -162,6 +162,10 @@ pub struct Arguments {
     #[clap(long, env, default_value = "http://localhost:8545")]
     pub node_url: Url,
 
+    /// The base URL used to connect to subgraph clients.
+    #[clap(long, env, default_value = "https://api.thegraph.com/subgraphs/name/")]
+    pub graph_api_base_url: Url,
+
     /// An Ethereum node URL that supports `eth_call`s with state overrides to
     /// be used for simulations.
     #[clap(long, env)]
@@ -414,6 +418,7 @@ impl Display for Arguments {
             self.logging.log_stderr_threshold
         )?;
         writeln!(f, "node_url: {}", self.node_url)?;
+        writeln!(f, "graph_api_base_url: {}", self.graph_api_base_url)?;
         display_option(f, "chain_id", &self.chain_id)?;
         display_option(f, "simulation_node_url", &self.simulation_node_url)?;
         writeln!(f, "gas_estimators: {:?}", self.gas_estimators)?;

crates/shared/src/price_estimation/http.rs

@@ -765,8 +765,10 @@ mod tests {
             current_block_stream(Arc::new(web3.clone()), Duration::from_secs(10))
                 .await
                 .unwrap();
+        let base_url = Url::parse("https://api.thegraph.com/subgraphs/name/").expect("invalid url");
         let balancer_pool_fetcher = Arc::new(
             BalancerPoolFetcher::new(
+                &base_url,
                 chain_id,
                 block_retriever.clone(),
                 token_info.clone(),
@@ -782,6 +784,7 @@ mod tests {
         );
         let uniswap_v3_pool_fetcher = Arc::new(
             UniswapV3PoolFetcher::new(
+                &base_url,
                 chain_id,
                 web3.clone(),
                 client.clone(),

crates/shared/src/sources/balancer_v2/graph_api.rs

@@ -13,7 +13,7 @@ use {
     crate::{event_handling::MAX_REORG_BLOCK_COUNT, subgraph::SubgraphClient},
     anyhow::{bail, Result},
     ethcontract::{H160, H256},
-    reqwest::Client,
+    reqwest::{Client, Url},
     serde::Deserialize,
     serde_json::json,
     serde_with::{serde_as, DisplayFromStr},
@@ -34,14 +34,15 @@ pub struct BalancerSubgraphClient(SubgraphClient);
 
 impl BalancerSubgraphClient {
     /// Creates a new Balancer subgraph client for the specified chain ID.
-    pub fn for_chain(chain_id: u64, client: Client) -> Result<Self> {
+    pub fn for_chain(base_url: &Url, chain_id: u64, client: Client) -> Result<Self> {
         let subgraph_name = match chain_id {
             1 => "balancer-v2",
             5 => "balancer-goerli-v2",
             100 => "balancer-gnosis-chain-v2",
             _ => bail!("unsupported chain {}", chain_id),
         };
         Ok(Self(SubgraphClient::new(
+            base_url,
             "balancer-labs",
             subgraph_name,
             client,
@@ -254,6 +255,11 @@ mod tests {
         std::collections::HashMap,
     };
 
+    pub fn default_for_chain(chain_id: u64, client: Client) -> Result<BalancerSubgraphClient> {
+        let base_url = Url::parse("https://api.thegraph.com/subgraphs/name/").expect("invalid url");
+        BalancerSubgraphClient::for_chain(&base_url, chain_id, client)
+    }
+
     #[test]
     fn decode_pools_data() {
         use pools_query::*;
@@ -485,8 +491,7 @@ mod tests {
     async fn balancer_subgraph_query() {
         for (network_name, chain_id) in [("Mainnet", 1), ("Goerli", 5)] {
             println!("### {network_name}");
-
-            let client = BalancerSubgraphClient::for_chain(chain_id, Client::new()).unwrap();
+            let client = default_for_chain(chain_id, Client::new()).unwrap();
             let result = client.get_registered_pools().await.unwrap();
             println!(
                 "Retrieved {} total pools at block {}",

crates/shared/src/sources/balancer_v2/pool_fetching.rs

@@ -49,7 +49,7 @@ use {
     ethcontract::{dyns::DynInstance, BlockId, Instance, H160, H256},
     ethrpc::current_block::{BlockRetrieving, CurrentBlockStream},
     model::TokenPair,
-    reqwest::Client,
+    reqwest::{Client, Url},
     std::{
         collections::{BTreeMap, HashMap, HashSet},
         sync::Arc,
@@ -279,6 +279,7 @@ impl BalancerContracts {
 impl BalancerPoolFetcher {
     #[allow(clippy::too_many_arguments)]
     pub async fn new(
+        base_url: &Url,
         chain_id: u64,
         block_retriever: Arc<dyn BlockRetrieving>,
         token_infos: Arc<dyn TokenInfoFetching>,
@@ -289,7 +290,7 @@ impl BalancerPoolFetcher {
         contracts: &BalancerContracts,
         deny_listed_pool_ids: Vec<H256>,
     ) -> Result<Self> {
-        let pool_initializer = BalancerSubgraphClient::for_chain(chain_id, client)?;
+        let pool_initializer = BalancerSubgraphClient::for_chain(base_url, chain_id, client)?;
         let fetcher = Arc::new(Cache::new(
             create_aggregate_pool_fetcher(
                 web3,
@@ -532,7 +533,9 @@ mod tests {
             "072f14b85add63488ddad88f855fda4a99d6ac9b000200000000000000000027"
         ))];
         // let deny_list = vec![];
+        let base_url = Url::parse("https://api.thegraph.com/subgraphs/name/").expect("invalid url");
         let pool_fetcher = BalancerPoolFetcher::new(
+            &base_url,
             chain_id,
             Arc::new(web3.clone()),
             token_info_fetcher,
@@ -590,9 +593,9 @@ mod tests {
             ),
             pool_id_deny_list: Default::default(),
         };
-
+        let base_url = Url::parse("https://api.thegraph.com/subgraphs/name/").expect("invalid url");
         // see what the subgraph says.
-        let client = BalancerSubgraphClient::for_chain(chain_id, Client::new()).unwrap();
+        let client = BalancerSubgraphClient::for_chain(&base_url, chain_id, Client::new()).unwrap();
         let subgraph_pools = client.get_registered_pools().await.unwrap();
         let subgraph_token_pairs = subgraph_pools_token_pairs(&subgraph_pools.pools).collect();