From 322ebfe6ca2c71895131409d1867a0991ab404c6 Mon Sep 17 00:00:00 2001 From: Nihar Shah Date: Tue, 9 Apr 2024 21:12:20 -0400 Subject: [PATCH 1/8] feat: allow output option to be passed to dev to generate supergraph --- src/cli.rs | 8 +++++--- src/command/dev/compose.rs | 1 + src/command/dev/do_dev.rs | 3 +++ src/command/dev/protocol/leader.rs | 8 +++++--- src/command/dev/router/config.rs | 7 +++++-- src/options/output.rs | 23 +++++++++++++++++++++-- 6 files changed, 40 insertions(+), 10 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index ab170a544..571fef935 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -181,9 +181,11 @@ impl Rover { match &self.command { Command::Config(command) => command.run(self.get_client_config()?), Command::Contract(command) => command.run(self.get_client_config()?), - Command::Dev(command) => { - command.run(self.get_install_override_path()?, self.get_client_config()?) - } + Command::Dev(command) => command.run( + self.get_install_override_path()?, + self.get_client_config()?, + &self.output_opts, + ), Command::Fed2(command) => command.run(self.get_client_config()?), Command::Supergraph(command) => { command.run(self.get_install_override_path()?, self.get_client_config()?) diff --git a/src/command/dev/compose.rs b/src/command/dev/compose.rs index 486d74d53..3170b1d85 100644 --- a/src/command/dev/compose.rs +++ b/src/command/dev/compose.rs @@ -4,6 +4,7 @@ use std::io::prelude::*; use anyhow::{Context, Error}; use apollo_federation_types::config::{FederationVersion, SupergraphConfig}; use camino::Utf8PathBuf; + use rover_std::{Emoji, Fs}; use crate::command::dev::do_dev::log_err_and_continue; diff --git a/src/command/dev/do_dev.rs b/src/command/dev/do_dev.rs index 36f51ff9f..79fc951c4 100644 --- a/src/command/dev/do_dev.rs +++ b/src/command/dev/do_dev.rs @@ -7,6 +7,7 @@ use super::router::RouterConfigHandler; use super::Dev; use crate::command::dev::protocol::FollowerMessage; +use crate::options::OutputOpts; use crate::utils::client::StudioClientConfig; use crate::{RoverError, RoverOutput, RoverResult}; @@ -22,6 +23,7 @@ impl Dev { &self, override_install_path: Option, client_config: StudioClientConfig, + output_opts: &OutputOpts, ) -> RoverResult { self.opts .plugin_opts @@ -48,6 +50,7 @@ impl Dev { follower_channel.clone(), self.opts.plugin_opts.clone(), router_config_handler, + output_opts, )? { eprintln!("{0}Do not run this command in production! {0}It is intended for local development.", Emoji::Warn); let (ready_sender, ready_receiver) = sync_channel(1); diff --git a/src/command/dev/protocol/leader.rs b/src/command/dev/protocol/leader.rs index 685a40a4a..0fcefb715 100644 --- a/src/command/dev/protocol/leader.rs +++ b/src/command/dev/protocol/leader.rs @@ -5,10 +5,11 @@ use crate::{ router::{RouterConfigHandler, RouterRunner}, OVERRIDE_DEV_COMPOSITION_VERSION, }, - options::PluginOpts, + options::{OutputOpts, PluginOpts}, utils::client::StudioClientConfig, RoverError, RoverErrorSuggestion, RoverResult, PKG_VERSION, }; + use anyhow::{anyhow, Context}; use apollo_federation_types::{ build::SubgraphDefinition, @@ -57,6 +58,7 @@ impl LeaderSession { follower_channel: FollowerChannel, plugin_opts: PluginOpts, router_config_handler: RouterConfigHandler, + output_opts: &OutputOpts, ) -> RoverResult> { let ipc_socket_addr = router_config_handler.get_ipc_address()?; let router_socket_addr = router_config_handler.get_router_address(); @@ -91,12 +93,12 @@ impl LeaderSession { plugin_opts.clone(), override_install_path.clone(), client_config.clone(), - router_config_handler.get_supergraph_schema_path(), + router_config_handler.get_supergraph_schema_path(output_opts.get_output_path()), ); // create a [`RouterRunner`] that we will use to spawn the router when we have a successful composition let mut router_runner = RouterRunner::new( - router_config_handler.get_supergraph_schema_path(), + router_config_handler.get_supergraph_schema_path(output_opts.get_output_file()), router_config_handler.get_router_config_path(), plugin_opts, router_socket_addr, diff --git a/src/command/dev/router/config.rs b/src/command/dev/router/config.rs index 80ad958e2..a5a91c76a 100644 --- a/src/command/dev/router/config.rs +++ b/src/command/dev/router/config.rs @@ -137,8 +137,11 @@ impl RouterConfigHandler { } /// The path to the composed supergraph schema - pub fn get_supergraph_schema_path(&self) -> Utf8PathBuf { - self.tmp_supergraph_schema_path.clone() + pub fn get_supergraph_schema_path(&self, output_path: Option) -> Utf8PathBuf { + return match output_path { + Some(path) => Utf8PathBuf::try_from(path).unwrap(), + _ => self.tmp_supergraph_schema_path.clone(), + }; } /// The path to the patched router config YAML diff --git a/src/options/output.rs b/src/options/output.rs index b0e99968d..4ef955d05 100644 --- a/src/options/output.rs +++ b/src/options/output.rs @@ -8,10 +8,11 @@ use anyhow::Result; use calm_io::{stderrln, stdoutln}; use camino::Utf8PathBuf; use clap::{error::ErrorKind as ClapErrorKind, CommandFactory, Parser, ValueEnum}; -use rover_std::{Emoji, Fs, Style}; use serde::Serialize; use serde_json::{json, Value}; +use rover_std::{Emoji, Fs, Style}; + use crate::{ cli::{Rover, RoverOutputFormatKind}, RoverError, RoverOutput, RoverResult, @@ -116,7 +117,7 @@ impl RoverPrinter for RoverError { } } -#[derive(Debug, Parser, Serialize)] +#[derive(Debug, Parser, Serialize, Clone)] pub struct OutputOpts { /// Specify Rover's format type #[arg(long = "format", global = true)] @@ -167,6 +168,24 @@ impl OutputOpts { rover_command_output.write_or_print(self) } + /// Get the path to the output file + pub fn get_output_path(&self) -> Option { + let output_type = self.output_file.clone(); + let b = match (&self.format_kind, output_type) { + (Some(RoverOutputFormatKind::Plain), Some(OutputOpt::File(path))) => { + RoverOutputDestination::File(path) + } + (Some(RoverOutputFormatKind::Json), Some(OutputOpt::File(path))) => { + RoverOutputDestination::File(path) + } + _ => RoverOutputDestination::Stdout, + }; + match b { + RoverOutputDestination::File(path) => Some(path.into_string()), + _ => None, + } + } + /// Get the format (plain/json) and strategy (stdout/file) pub fn get_format_and_strategy(&self) -> (RoverOutputFormatKind, RoverOutputDestination) { let output_type = self.output_file.clone(); From b65eabc0bf2520aea5c6f488f46122495e17d040 Mon Sep 17 00:00:00 2001 From: Nihar Shah Date: Wed, 10 Apr 2024 10:39:37 -0400 Subject: [PATCH 2/8] chore: fixed a test error --- src/command/dev/protocol/leader.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/command/dev/protocol/leader.rs b/src/command/dev/protocol/leader.rs index 0fcefb715..3e974302e 100644 --- a/src/command/dev/protocol/leader.rs +++ b/src/command/dev/protocol/leader.rs @@ -98,7 +98,7 @@ impl LeaderSession { // create a [`RouterRunner`] that we will use to spawn the router when we have a successful composition let mut router_runner = RouterRunner::new( - router_config_handler.get_supergraph_schema_path(output_opts.get_output_file()), + router_config_handler.get_supergraph_schema_path(output_opts.get_output_path()), router_config_handler.get_router_config_path(), plugin_opts, router_socket_addr, From 4796c90fa73da39f2d610db7793824c99282d375 Mon Sep 17 00:00:00 2001 From: Nihar Shah Date: Wed, 10 Apr 2024 12:58:20 -0400 Subject: [PATCH 3/8] chore: fix pipeline failures --- src/command/dev/no_dev.rs | 2 ++ src/command/dev/router/config.rs | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/command/dev/no_dev.rs b/src/command/dev/no_dev.rs index 0b3d88746..687bd075c 100644 --- a/src/command/dev/no_dev.rs +++ b/src/command/dev/no_dev.rs @@ -2,12 +2,14 @@ use super::Dev; use crate::{utils::client::StudioClientConfig, RoverError, RoverOutput, RoverResult}; use anyhow::anyhow; use camino::Utf8PathBuf; +use crate::options::OutputOpts; impl Dev { pub fn run( &self, _override_install_path: Option, _client_config: StudioClientConfig, + _output_opts: &OutputOpts, ) -> RoverResult { Err(RoverError::new(anyhow!( "rover dev is not supported on this platform" diff --git a/src/command/dev/router/config.rs b/src/command/dev/router/config.rs index a5a91c76a..8512d1b0e 100644 --- a/src/command/dev/router/config.rs +++ b/src/command/dev/router/config.rs @@ -138,10 +138,10 @@ impl RouterConfigHandler { /// The path to the composed supergraph schema pub fn get_supergraph_schema_path(&self, output_path: Option) -> Utf8PathBuf { - return match output_path { - Some(path) => Utf8PathBuf::try_from(path).unwrap(), + match output_path { + Some(path) => Utf8PathBuf::from(path), _ => self.tmp_supergraph_schema_path.clone(), - }; + } } /// The path to the patched router config YAML From 05545bbbbdb3d1c1aed454c0295b884ccf2dc990 Mon Sep 17 00:00:00 2001 From: Nihar Shah Date: Wed, 10 Apr 2024 13:04:36 -0400 Subject: [PATCH 4/8] chore: fix formatting issue --- src/command/dev/no_dev.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/command/dev/no_dev.rs b/src/command/dev/no_dev.rs index 687bd075c..fbce371ef 100644 --- a/src/command/dev/no_dev.rs +++ b/src/command/dev/no_dev.rs @@ -1,8 +1,8 @@ use super::Dev; +use crate::options::OutputOpts; use crate::{utils::client::StudioClientConfig, RoverError, RoverOutput, RoverResult}; use anyhow::anyhow; use camino::Utf8PathBuf; -use crate::options::OutputOpts; impl Dev { pub fn run( From 1f43923bb0023552f670fa413f8edacae66055be Mon Sep 17 00:00:00 2001 From: Nihar Shah Date: Wed, 10 Apr 2024 14:11:12 -0400 Subject: [PATCH 5/8] feat: enable using the output flat to create supergraph from rover dev --- src/cli.rs | 8 +++++--- src/command/dev/compose.rs | 1 + src/command/dev/do_dev.rs | 3 +++ src/command/dev/no_dev.rs | 2 ++ src/command/dev/protocol/leader.rs | 8 +++++--- src/command/dev/router/config.rs | 7 +++++-- src/options/output.rs | 23 +++++++++++++++++++++-- 7 files changed, 42 insertions(+), 10 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index ab170a544..571fef935 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -181,9 +181,11 @@ impl Rover { match &self.command { Command::Config(command) => command.run(self.get_client_config()?), Command::Contract(command) => command.run(self.get_client_config()?), - Command::Dev(command) => { - command.run(self.get_install_override_path()?, self.get_client_config()?) - } + Command::Dev(command) => command.run( + self.get_install_override_path()?, + self.get_client_config()?, + &self.output_opts, + ), Command::Fed2(command) => command.run(self.get_client_config()?), Command::Supergraph(command) => { command.run(self.get_install_override_path()?, self.get_client_config()?) diff --git a/src/command/dev/compose.rs b/src/command/dev/compose.rs index 486d74d53..3170b1d85 100644 --- a/src/command/dev/compose.rs +++ b/src/command/dev/compose.rs @@ -4,6 +4,7 @@ use std::io::prelude::*; use anyhow::{Context, Error}; use apollo_federation_types::config::{FederationVersion, SupergraphConfig}; use camino::Utf8PathBuf; + use rover_std::{Emoji, Fs}; use crate::command::dev::do_dev::log_err_and_continue; diff --git a/src/command/dev/do_dev.rs b/src/command/dev/do_dev.rs index 36f51ff9f..79fc951c4 100644 --- a/src/command/dev/do_dev.rs +++ b/src/command/dev/do_dev.rs @@ -7,6 +7,7 @@ use super::router::RouterConfigHandler; use super::Dev; use crate::command::dev::protocol::FollowerMessage; +use crate::options::OutputOpts; use crate::utils::client::StudioClientConfig; use crate::{RoverError, RoverOutput, RoverResult}; @@ -22,6 +23,7 @@ impl Dev { &self, override_install_path: Option, client_config: StudioClientConfig, + output_opts: &OutputOpts, ) -> RoverResult { self.opts .plugin_opts @@ -48,6 +50,7 @@ impl Dev { follower_channel.clone(), self.opts.plugin_opts.clone(), router_config_handler, + output_opts, )? { eprintln!("{0}Do not run this command in production! {0}It is intended for local development.", Emoji::Warn); let (ready_sender, ready_receiver) = sync_channel(1); diff --git a/src/command/dev/no_dev.rs b/src/command/dev/no_dev.rs index 0b3d88746..fbce371ef 100644 --- a/src/command/dev/no_dev.rs +++ b/src/command/dev/no_dev.rs @@ -1,4 +1,5 @@ use super::Dev; +use crate::options::OutputOpts; use crate::{utils::client::StudioClientConfig, RoverError, RoverOutput, RoverResult}; use anyhow::anyhow; use camino::Utf8PathBuf; @@ -8,6 +9,7 @@ impl Dev { &self, _override_install_path: Option, _client_config: StudioClientConfig, + _output_opts: &OutputOpts, ) -> RoverResult { Err(RoverError::new(anyhow!( "rover dev is not supported on this platform" diff --git a/src/command/dev/protocol/leader.rs b/src/command/dev/protocol/leader.rs index 685a40a4a..3e974302e 100644 --- a/src/command/dev/protocol/leader.rs +++ b/src/command/dev/protocol/leader.rs @@ -5,10 +5,11 @@ use crate::{ router::{RouterConfigHandler, RouterRunner}, OVERRIDE_DEV_COMPOSITION_VERSION, }, - options::PluginOpts, + options::{OutputOpts, PluginOpts}, utils::client::StudioClientConfig, RoverError, RoverErrorSuggestion, RoverResult, PKG_VERSION, }; + use anyhow::{anyhow, Context}; use apollo_federation_types::{ build::SubgraphDefinition, @@ -57,6 +58,7 @@ impl LeaderSession { follower_channel: FollowerChannel, plugin_opts: PluginOpts, router_config_handler: RouterConfigHandler, + output_opts: &OutputOpts, ) -> RoverResult> { let ipc_socket_addr = router_config_handler.get_ipc_address()?; let router_socket_addr = router_config_handler.get_router_address(); @@ -91,12 +93,12 @@ impl LeaderSession { plugin_opts.clone(), override_install_path.clone(), client_config.clone(), - router_config_handler.get_supergraph_schema_path(), + router_config_handler.get_supergraph_schema_path(output_opts.get_output_path()), ); // create a [`RouterRunner`] that we will use to spawn the router when we have a successful composition let mut router_runner = RouterRunner::new( - router_config_handler.get_supergraph_schema_path(), + router_config_handler.get_supergraph_schema_path(output_opts.get_output_path()), router_config_handler.get_router_config_path(), plugin_opts, router_socket_addr, diff --git a/src/command/dev/router/config.rs b/src/command/dev/router/config.rs index 80ad958e2..8512d1b0e 100644 --- a/src/command/dev/router/config.rs +++ b/src/command/dev/router/config.rs @@ -137,8 +137,11 @@ impl RouterConfigHandler { } /// The path to the composed supergraph schema - pub fn get_supergraph_schema_path(&self) -> Utf8PathBuf { - self.tmp_supergraph_schema_path.clone() + pub fn get_supergraph_schema_path(&self, output_path: Option) -> Utf8PathBuf { + match output_path { + Some(path) => Utf8PathBuf::from(path), + _ => self.tmp_supergraph_schema_path.clone(), + } } /// The path to the patched router config YAML diff --git a/src/options/output.rs b/src/options/output.rs index b0e99968d..4ef955d05 100644 --- a/src/options/output.rs +++ b/src/options/output.rs @@ -8,10 +8,11 @@ use anyhow::Result; use calm_io::{stderrln, stdoutln}; use camino::Utf8PathBuf; use clap::{error::ErrorKind as ClapErrorKind, CommandFactory, Parser, ValueEnum}; -use rover_std::{Emoji, Fs, Style}; use serde::Serialize; use serde_json::{json, Value}; +use rover_std::{Emoji, Fs, Style}; + use crate::{ cli::{Rover, RoverOutputFormatKind}, RoverError, RoverOutput, RoverResult, @@ -116,7 +117,7 @@ impl RoverPrinter for RoverError { } } -#[derive(Debug, Parser, Serialize)] +#[derive(Debug, Parser, Serialize, Clone)] pub struct OutputOpts { /// Specify Rover's format type #[arg(long = "format", global = true)] @@ -167,6 +168,24 @@ impl OutputOpts { rover_command_output.write_or_print(self) } + /// Get the path to the output file + pub fn get_output_path(&self) -> Option { + let output_type = self.output_file.clone(); + let b = match (&self.format_kind, output_type) { + (Some(RoverOutputFormatKind::Plain), Some(OutputOpt::File(path))) => { + RoverOutputDestination::File(path) + } + (Some(RoverOutputFormatKind::Json), Some(OutputOpt::File(path))) => { + RoverOutputDestination::File(path) + } + _ => RoverOutputDestination::Stdout, + }; + match b { + RoverOutputDestination::File(path) => Some(path.into_string()), + _ => None, + } + } + /// Get the format (plain/json) and strategy (stdout/file) pub fn get_format_and_strategy(&self) -> (RoverOutputFormatKind, RoverOutputDestination) { let output_type = self.output_file.clone(); From 34522836bebb9c14b5fa955cfb93ddd4a72bbc52 Mon Sep 17 00:00:00 2001 From: Nihar Shah Date: Thu, 11 Apr 2024 22:22:49 -0400 Subject: [PATCH 6/8] chore: use semaphore to fix the failing link check --- xtask/src/tools/lychee.rs | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/xtask/src/tools/lychee.rs b/xtask/src/tools/lychee.rs index 179869560..e208978dc 100644 --- a/xtask/src/tools/lychee.rs +++ b/xtask/src/tools/lychee.rs @@ -7,8 +7,10 @@ use lychee_lib::{ Result as LycheeResult, Uri, }; use reqwest::StatusCode; +use std::sync::Arc; use std::{collections::HashSet, fs, path::PathBuf, time::Duration}; use tokio::runtime::Runtime; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use tokio_stream::StreamExt; pub(crate) struct LycheeRunner { @@ -37,6 +39,8 @@ impl LycheeRunner { pub(crate) fn lint(&self) -> Result<()> { crate::info!("Checking HTTP links in repository"); + let semaphore = Arc::new(Semaphore::new(50)); + let inputs: Vec = get_md_files() .iter() .map(|file| Input { @@ -56,11 +60,16 @@ impl LycheeRunner { .await .collect::>>() .await?; - - let failed_link_futures: Vec<_> = links - .into_iter() - .map(|link| tokio::spawn(get_failed_request(lychee_client.clone(), link))) - .collect(); + let mut failed_link_futures: Vec<_> = Vec::new(); + + for link in links { + let permit = semaphore.clone().acquire_owned().await.unwrap(); + failed_link_futures.push(tokio::spawn(get_failed_request( + lychee_client.clone(), + link, + permit, + ))); + } let links_size = failed_link_futures.len(); @@ -88,11 +97,16 @@ impl LycheeRunner { } } -async fn get_failed_request(lychee_client: Client, link: Request) -> Option { +async fn get_failed_request( + lychee_client: Client, + link: Request, + permit: OwnedSemaphorePermit, +) -> Option { let response = lychee_client .check(link) .await .expect("could not execute lychee request"); + drop(permit); if response.status().is_failure() { Some(response.1.uri) } else { From c6fe980fae7a0c80dc5d41e59add47720481ab0f Mon Sep 17 00:00:00 2001 From: Nihar Shah Date: Thu, 11 Apr 2024 22:48:09 -0400 Subject: [PATCH 7/8] chore: adjust semaphores to 10 --- xtask/src/tools/lychee.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xtask/src/tools/lychee.rs b/xtask/src/tools/lychee.rs index e208978dc..5320ab975 100644 --- a/xtask/src/tools/lychee.rs +++ b/xtask/src/tools/lychee.rs @@ -39,7 +39,7 @@ impl LycheeRunner { pub(crate) fn lint(&self) -> Result<()> { crate::info!("Checking HTTP links in repository"); - let semaphore = Arc::new(Semaphore::new(50)); + let semaphore = Arc::new(Semaphore::new(10)); let inputs: Vec = get_md_files() .iter() From abc3552e14e8cb8d56fdf37800b02bb61dbb2c2a Mon Sep 17 00:00:00 2001 From: Nihar Shah Date: Thu, 11 Apr 2024 22:57:55 -0400 Subject: [PATCH 8/8] chore: reduce to 5 permits to check if build succeeds --- xtask/src/tools/lychee.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xtask/src/tools/lychee.rs b/xtask/src/tools/lychee.rs index 5320ab975..c58992345 100644 --- a/xtask/src/tools/lychee.rs +++ b/xtask/src/tools/lychee.rs @@ -39,7 +39,7 @@ impl LycheeRunner { pub(crate) fn lint(&self) -> Result<()> { crate::info!("Checking HTTP links in repository"); - let semaphore = Arc::new(Semaphore::new(10)); + let semaphore = Arc::new(Semaphore::new(5)); let inputs: Vec = get_md_files() .iter()