From e8bd838ece7d6548b99814a02d7424da261b1558 Mon Sep 17 00:00:00 2001 From: shikai liu Date: Mon, 27 May 2024 08:42:37 +0800 Subject: [PATCH 1/8] Update the version. --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 7fb91eb..0eb1c04 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rcurl" -version = "0.0.23" +version = "0.0.24" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 2afe327830cb0415f70694e901bdfe80ac48b59b Mon Sep 17 00:00:00 2001 From: shikai liu Date: Mon, 27 May 2024 09:48:59 +0800 Subject: [PATCH 2/8] Fix the bug of http headder. --- src/http/handler.rs | 44 +++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/src/http/handler.rs b/src/http/handler.rs index 54e9d0a..6d4c3e0 100644 --- a/src/http/handler.rs +++ b/src/http/handler.rs @@ -43,6 +43,7 @@ use http_body_util::BodyExt; use http::response::Parts; use http_body_util::combinators::BoxBody; use hyper::body::Buf; +use hyper::HeaderMap; use hyper_util::client::legacy::Client; use rustls::crypto::ring::DEFAULT_CIPHER_SUITES; use rustls::crypto::CryptoProvider; @@ -218,14 +219,6 @@ pub async fn http_request( .set_certificate_verifier(Arc::new(NoCertificateVerification::new(default_provider()))); }; let uri: hyper::Uri = cli.url.parse()?; - let host = uri.host().expect("uri has no host"); - let default_port = if let Some(port) = uri.port_u16() { - port - } else if uri.scheme_str() == Some("https") { - 443 - } else { - 80 - }; let mut method = String::from("GET"); let mut content_type_option = None; @@ -239,12 +232,12 @@ pub async fn http_request( let mut request_builder = Request::builder() .method(method.as_str()) .uri(cli.url.clone()); + let mut header_map = HeaderMap::new(); if let Some(content_type) = content_type_option { - request_builder = - request_builder.header(CONTENT_TYPE, HeaderValue::from_str(&content_type)?); + header_map.insert(CONTENT_TYPE, HeaderValue::from_str(&content_type)?); } - request_builder = request_builder.header(ACCEPT, HeaderValue::from_str("*/*")?); - request_builder = request_builder.header( + header_map.insert(ACCEPT, HeaderValue::from_str("*/*")?); + header_map.insert( HOST, HeaderValue::from_str(uri.host().ok_or(anyhow!("no host"))?)?, ); @@ -252,16 +245,16 @@ pub async fn http_request( .user_agent_option .clone() .unwrap_or(format!("rcur/{}", env!("CARGO_PKG_VERSION").to_string())); - request_builder = request_builder.header(USER_AGENT, HeaderValue::from_str(&user_agent)?); + header_map.insert(USER_AGENT, HeaderValue::from_str(&user_agent)?); if let Some(cookie) = cli.cookie_option.clone() { - request_builder = request_builder.header(COOKIE, HeaderValue::from_str(&cookie)?); + header_map.insert(COOKIE, HeaderValue::from_str(&cookie)?); } if let Some(range) = cli.range_option.clone() { let ranges_format = format!("bytes={}", range); - request_builder = request_builder.header(RANGE, HeaderValue::from_str(&ranges_format)?); + header_map.insert(RANGE, HeaderValue::from_str(&ranges_format)?); } if let Some(refer) = cli.refer_option.clone() { - request_builder = request_builder.header(REFERER, HeaderValue::from_str(&refer)?); + header_map.insert(REFERER, HeaderValue::from_str(&refer)?); } let mut body_bytes = Bytes::new(); if cli.form_option.len() != 0 { @@ -302,11 +295,13 @@ pub async fn http_request( } for x in cli.headers.clone() { let split: Vec = x.splitn(2, ':').map(|s| s.to_string()).collect(); + if split.len() == 2 { let key = &split[0]; let value = &split[1]; let new_value = value.trim_start(); - request_builder = request_builder.header( + + header_map.insert( HeaderName::from_str(key.as_str())?, HeaderValue::from_str(new_value)?, ); @@ -314,6 +309,9 @@ pub async fn http_request( return Err(anyhow!("header error")); } } + for (key, val) in header_map { + request_builder = request_builder.header(key.ok_or(anyhow!(""))?, val); + } let content_length = body_bytes.len(); let body = Full::new(body_bytes); let request = request_builder.body(body)?; @@ -331,11 +329,11 @@ pub async fn http_request( println!("> Content-Length: {}", content_length); } - let port = uri.port_u16().unwrap_or(default_port); - let addr = format!("{}:{}", host, port); - let stream = TcpStream::connect(addr.clone()).await?; - let remote_addr = stream.peer_addr()?.to_string(); - let local_addr = stream.local_addr()?.to_string(); + // let port = uri.port_u16().unwrap_or(default_port); + // let addr = format!("{}:{}", host, port); + // let stream = TcpStream::connect(addr.clone()).await?; + // let remote_addr = stream.peer_addr()?.to_string(); + // let local_addr = stream.local_addr()?.to_string(); let span = tracing::info_span!("Rcurl"); let _enter = span.enter(); let request_future = { @@ -344,7 +342,7 @@ pub async fn http_request( let https = hyper_rustls::HttpsConnectorBuilder::new() .with_tls_config(tls_config) .https_only() - .enable_http1() + .enable_http2() .build(); let https_clientt: Client<_, Full> = Client::builder(TokioExecutor::new()).build(https); From 9bb784aec1c45cbc9f5c51106ad8abe6b0ff56db Mon Sep 17 00:00:00 2001 From: shikai liu Date: Mon, 27 May 2024 11:23:28 +0800 Subject: [PATCH 3/8] Remove the logger. --- Cargo.toml | 2 +- src/ftp/handler.rs | 10 +--------- src/http/handler.rs | 13 ++++--------- src/main.rs | 27 +++++++++++++++++++++++---- 4 files changed, 29 insertions(+), 23 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0eb1c04..7c2dd87 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ futures-util = { version = "0.3.1", default-features = false } log = "^0.4" http = "1.1.0" http-body-util = { version = "0.1" } -hyper = { version = "1.3.1", features = ["full", "tracing", "http2", "client"] } +hyper = { version = "1.3.1", features = ["full"] } hyper-util = { version = "0.1.4", features = ["full"] } indicatif = "0.17.7" mime_guess = "2.0.4" diff --git a/src/ftp/handler.rs b/src/ftp/handler.rs index d998b15..42965ad 100644 --- a/src/ftp/handler.rs +++ b/src/ftp/handler.rs @@ -9,9 +9,9 @@ use futures::io::BufReader; use indicatif::ProgressBar; use indicatif::ProgressStyle; use log::LevelFilter; -use suppaftp::{AsyncNativeTlsConnector, AsyncNativeTlsFtpStream}; use suppaftp::async_native_tls::TlsConnector; use suppaftp::types::FileType; +use suppaftp::{AsyncNativeTlsConnector, AsyncNativeTlsFtpStream}; use tokio::fs::OpenOptions; use tokio::io::AsyncWriteExt; @@ -40,14 +40,6 @@ impl futures::io::AsyncRead for ProgressBarIt } pub async fn ftp_request(cli: Cli, scheme: &str) -> Result<(), anyhow::Error> { - let log_level_hyper = if cli.debug { - LevelFilter::Trace - } else { - LevelFilter::Info - }; - - // init logger - let _ = Builder::new().filter_level(log_level_hyper).try_init(); let uri: hyper::Uri = cli.url.parse()?; let host = uri.host().ok_or(anyhow!(""))?; let port = uri.port_u16().unwrap_or(21); diff --git a/src/http/handler.rs b/src/http/handler.rs index 6d4c3e0..f12fabc 100644 --- a/src/http/handler.rs +++ b/src/http/handler.rs @@ -1,6 +1,7 @@ use crate::cli::app_config::Cli; use bytes::Bytes; +use env_logger::Builder; use http::header::ACCEPT; use http_body_util::Full; use hyper::header::CONTENT_TYPE; @@ -9,6 +10,7 @@ use hyper::header::HOST; use hyper::header::REFERER; use hyper::header::{HeaderName, HeaderValue}; use hyper::Request; +use hyper::Version; use hyper_util::rt::TokioExecutor; use hyper_util::rt::TokioIo; use mime_guess::mime; @@ -35,6 +37,7 @@ use hyper::header::CONTENT_LENGTH; use hyper::Response; use indicatif::ProgressBar; use indicatif::ProgressStyle; +use log::LevelFilter; use rustls::crypto::ring::default_provider; use futures::StreamExt; @@ -180,14 +183,6 @@ pub async fn http_request( cli: Cli, scheme: &str, ) -> Result>, anyhow::Error> { - let log_level_hyper = if cli.debug { Level::TRACE } else { Level::INFO }; - - let subscriber = tracing_subscriber::fmt() - .with_level(true) - .with_max_level(log_level_hyper) - .finish(); - let _ = tracing::subscriber::set_global_default(subscriber); - let mut root_store = RootCertStore::empty(); if let Some(file_path) = cli.certificate_path_option.clone() { @@ -342,7 +337,7 @@ pub async fn http_request( let https = hyper_rustls::HttpsConnectorBuilder::new() .with_tls_config(tls_config) .https_only() - .enable_http2() + .enable_all_versions() .build(); let https_clientt: Client<_, Full> = Client::builder(TokioExecutor::new()).build(https); diff --git a/src/main.rs b/src/main.rs index 3d8ddd9..e1060dd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,9 +2,9 @@ extern crate anyhow; #[macro_use] extern crate tracing; - use clap::CommandFactory; use clap::Parser; +use env_logger::Builder; use http_body_util::BodyExt; use http::handler::http_request; @@ -12,6 +12,8 @@ use http::handler::http_request; use crate::cli::app_config::Cli; use crate::ftp::handler::ftp_request; use crate::response::res::RcurlResponse; +use log::LevelFilter; +use tracing::Level; mod ftp; @@ -31,6 +33,21 @@ async fn main() { } async fn do_request(cli: Cli) -> Result { + let log_level_hyper = if cli.debug { Level::TRACE } else { Level::INFO }; + + let subscriber = tracing_subscriber::fmt() + .with_level(true) + .with_max_level(log_level_hyper) + .finish(); + let _ = tracing::subscriber::set_global_default(subscriber); + let log_level_hyper = if cli.debug { + LevelFilter::Trace + } else { + LevelFilter::Info + }; + + // init logger + let _ = Builder::new().filter_level(log_level_hyper).try_init(); let url = cli.url.clone(); let uri: hyper::Uri = url.parse()?; if let Some(scheme) = uri.scheme() { @@ -57,8 +74,8 @@ async fn do_request(cli: Cli) -> Result { mod tests { use ::http::StatusCode; - use crate::{cli::app_config::Cli, do_request}; use crate::response::res::RcurlResponse; + use crate::{cli::app_config::Cli, do_request}; use super::*; @@ -269,7 +286,8 @@ mod tests { let result = do_request(cli).await; assert!(result.is_ok()); let rcurl_response = result.unwrap(); - if let RcurlResponse::Ftp(response) = rcurl_response {} else { + if let RcurlResponse::Ftp(response) = rcurl_response { + } else { assert!(false); } } @@ -285,7 +303,8 @@ mod tests { let result = do_request(cli).await; assert!(result.is_ok()); let rcurl_response = result.unwrap(); - if let RcurlResponse::Ftp(response) = rcurl_response {} else { + if let RcurlResponse::Ftp(response) = rcurl_response { + } else { assert!(false); } } From 1a566c2b63c60116143f2e1f2a6aa0e60249fbea Mon Sep 17 00:00:00 2001 From: Shikai Liu Date: Mon, 27 May 2024 21:24:25 +0800 Subject: [PATCH 4/8] Fix the bug of https with http1 or http2. --- src/http/handler.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/http/handler.rs b/src/http/handler.rs index f12fabc..408f3f9 100644 --- a/src/http/handler.rs +++ b/src/http/handler.rs @@ -232,10 +232,10 @@ pub async fn http_request( header_map.insert(CONTENT_TYPE, HeaderValue::from_str(&content_type)?); } header_map.insert(ACCEPT, HeaderValue::from_str("*/*")?); - header_map.insert( - HOST, - HeaderValue::from_str(uri.host().ok_or(anyhow!("no host"))?)?, - ); + // header_map.insert( + // HOST, + // HeaderValue::from_str(uri.host().ok_or(anyhow!("no host"))?)?, + // ); let user_agent = cli .user_agent_option .clone() From ed166e2585ea9e887a260ac7a46a74e659372e2b Mon Sep 17 00:00:00 2001 From: shikai liu Date: Tue, 28 May 2024 09:00:55 +0800 Subject: [PATCH 5/8] Remove unuseful code. --- src/http/handler.rs | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/src/http/handler.rs b/src/http/handler.rs index 408f3f9..85243b2 100644 --- a/src/http/handler.rs +++ b/src/http/handler.rs @@ -324,11 +324,6 @@ pub async fn http_request( println!("> Content-Length: {}", content_length); } - // let port = uri.port_u16().unwrap_or(default_port); - // let addr = format!("{}:{}", host, port); - // let stream = TcpStream::connect(addr.clone()).await?; - // let remote_addr = stream.peer_addr()?.to_string(); - // let local_addr = stream.local_addr()?.to_string(); let span = tracing::info_span!("Rcurl"); let _enter = span.enter(); let request_future = { @@ -341,30 +336,7 @@ pub async fn http_request( .build(); let https_clientt: Client<_, Full> = Client::builder(TokioExecutor::new()).build(https); - // let domain = pki_types::ServerName::try_from(host) - // .map_err(|e| anyhow!("{}", e))? - // .to_owned(); - // let tls_stream = connector.connect(domain, stream).await?; - // let stream_io = TokioIo::new(tls_stream); - - // let (mut sender, conn) = hyper::client::conn::http1::handshake(stream_io) - // .instrument(info_span!("Https Handshake")) - // .await?; - // tokio::task::spawn(async move { - // if let Err(err) = conn - // .instrument(info_span!( - // "rcurl", - // localAddr=%local_addr, - // remoteAddr=remote_addr, - - // )) - // .await - // { - // println!("Connection failed: {:?}", err); - // } - // }); https_clientt.request(request) - // sender.send_request(request) } else { let http_client = Client::builder(TokioExecutor::new()) .http1_title_case_headers(true) From e96458b2c1d173a1abcdef21721f3ca24e1c2e74 Mon Sep 17 00:00:00 2001 From: shikai liu Date: Tue, 28 May 2024 14:34:51 +0800 Subject: [PATCH 6/8] Update the version of some crates. --- Cargo.toml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7c2dd87..776c3d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,14 +8,14 @@ edition = "2021" [dependencies] anyhow = "1.0" async-std = "1.12.0" -async-tls = "0.8" +async-tls = "0.13.0" bytes = "1" -chrono = "0.4.31" +chrono = "0.4.38" clap = { version = "4.5.4", features = ["derive", "string"] } env_logger = "^0.11" form-data-builder = "1.0.1" -futures = "0.3.29" -futures-util = { version = "0.3.1", default-features = false } +futures = "0.3.30" +futures-util = { version = "0.3.30", default-features = false } log = "^0.4" http = "1.1.0" http-body-util = { version = "0.1" } @@ -23,18 +23,18 @@ hyper = { version = "1.3.1", features = ["full"] } hyper-util = { version = "0.1.4", features = ["full"] } indicatif = "0.17.7" mime_guess = "2.0.4" -openssl = { version = "0.10", features = ["vendored"] } +openssl = { version = "0.10.64", features = ["vendored"] } pki-types = { package = "rustls-pki-types", version = "1" } http-range-header = "0.4.1" -rustls = { version = "0.23.5", default-features = false, features = [ +rustls = { version = "0.23.8", default-features = false, features = [ "logging", "ring", "tls12", ] } -rustls-pemfile = "2" -serde = { version = "1.0.190", features = ["derive"] } -serde_json = "1.0.108" -suppaftp = { version = "6.0.0", features = ["async-native-tls", "native-tls"] } +rustls-pemfile = "2.1.2" +serde = { version = "1.0.203", features = ["derive"] } +serde_json = "1.0.117" +suppaftp = { version = "6.0.1", features = ["async-native-tls", "native-tls"] } tokio = { version = "1.37.0", features = ["full"] } tokio-rustls = { version = "0.26.0", default-features = false, features = [ @@ -42,10 +42,10 @@ tokio-rustls = { version = "0.26.0", default-features = false, features = [ "ring", ] } tokio-util = { version = "0.7.11", features = ["full", "time"] } -tracing = "0.1.4" +tracing = "0.1.40" tracing-appender = "0.2.3" tracing-subscriber = "0.3.18" -webpki-roots = "0.26.0" +webpki-roots = "0.26.1" hyper-rustls = { version = "0.27.1", default-features = false, features = [ "logging", "ring", From fb678cd24cdfdc872b09a2612bfe12f911a5ea68 Mon Sep 17 00:00:00 2001 From: shikai liu Date: Tue, 28 May 2024 17:05:11 +0800 Subject: [PATCH 7/8] Fix the bug of error agent. --- src/http/handler.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/http/handler.rs b/src/http/handler.rs index 85243b2..035aeaf 100644 --- a/src/http/handler.rs +++ b/src/http/handler.rs @@ -239,7 +239,7 @@ pub async fn http_request( let user_agent = cli .user_agent_option .clone() - .unwrap_or(format!("rcur/{}", env!("CARGO_PKG_VERSION").to_string())); + .unwrap_or(format!("rcurl/{}", env!("CARGO_PKG_VERSION").to_string())); header_map.insert(USER_AGENT, HeaderValue::from_str(&user_agent)?); if let Some(cookie) = cli.cookie_option.clone() { header_map.insert(COOKIE, HeaderValue::from_str(&cookie)?); From c9413f65ec4ab2b354c093967e364d979f62f9a5 Mon Sep 17 00:00:00 2001 From: shikai liu Date: Wed, 29 May 2024 10:39:18 +0800 Subject: [PATCH 8/8] Add the http2 and http2-prior-knowledge. --- README.md | 3 +++ src/cli/app_config.rs | 14 +++++++------- src/http/handler.rs | 34 +++++++++++++++++++++++++--------- 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 67ba01b..334a656 100644 --- a/README.md +++ b/README.md @@ -24,10 +24,13 @@ Options: -e, --referer Referrer URL -o, --output Write to file instead of stdout -T, --upload-file Transfer local FILE to destination + -Q, --quote Send command(s) to server before transfer -k, --insecure Allow insecure server connections -I, --head Show document info only -r, --range Retrieve only the bytes within RANGE -v, --verbose Make the operation more talkative + --http2 + --http2-prior-knowledge -h, --help Print help -V, --version Print version ``` diff --git a/src/cli/app_config.rs b/src/cli/app_config.rs index 5539cff..d3203e9 100644 --- a/src/cli/app_config.rs +++ b/src/cli/app_config.rs @@ -28,12 +28,7 @@ pub struct Cli { #[arg(short = 'A', long = "user-agent", value_name = "name")] pub user_agent_option: Option, /// The Cookie option. - #[arg( - short = 'b', - long = "cookie", - value_name = "data|filename", - hide_short_help = true - )] + #[arg(short = 'b', long = "cookie", value_name = "data|filename")] pub cookie_option: Option, /// Referrer URL #[arg(short = 'e', long = "referer", value_name = "URL")] @@ -47,7 +42,6 @@ pub struct Cli { default_missing_value = "none" )] pub file_path_option: Option, - /// Transfer local FILE to destination #[arg(long = "upload-file", short = 'T', value_name = "file")] pub uploadfile_option: Option, @@ -68,6 +62,10 @@ pub struct Cli { /// Make the operation more talkative #[arg(short = 'v', long = "verbose")] pub debug: bool, + #[arg(long = "http2")] + pub http2: bool, + #[arg(long = "http2-prior-knowledge")] + pub http2_prior_knowledge: bool, // /// Print help // #[arg(short, long, action = clap::ArgAction::Help)] // pub help: bool, @@ -95,6 +93,8 @@ impl Cli { skip_certificate_validate: false, header_option: false, range_option: None, + http2: false, + http2_prior_knowledge: false, debug: false, // help: false, // help_all: false, diff --git a/src/http/handler.rs b/src/http/handler.rs index 035aeaf..2c4f2f0 100644 --- a/src/http/handler.rs +++ b/src/http/handler.rs @@ -227,6 +227,9 @@ pub async fn http_request( let mut request_builder = Request::builder() .method(method.as_str()) .uri(cli.url.clone()); + if cli.http2_prior_knowledge { + request_builder = request_builder.version(hyper::Version::HTTP_2); + } let mut header_map = HeaderMap::new(); if let Some(content_type) = content_type_option { header_map.insert(CONTENT_TYPE, HeaderValue::from_str(&content_type)?); @@ -329,20 +332,33 @@ pub async fn http_request( let request_future = { trace!("Start request"); let fut = if scheme == "https" { - let https = hyper_rustls::HttpsConnectorBuilder::new() + let connector_builder = hyper_rustls::HttpsConnectorBuilder::new() .with_tls_config(tls_config) - .https_only() - .enable_all_versions() - .build(); + .https_only(); + let https_connector = if cli.http2 { + connector_builder.enable_all_versions().build() + } else if cli.http2_prior_knowledge { + connector_builder.enable_http2().build() + } else { + connector_builder.enable_http1().build() + }; + let https_clientt: Client<_, Full> = - Client::builder(TokioExecutor::new()).build(https); + Client::builder(TokioExecutor::new()).build(https_connector); https_clientt.request(request) } else { - let http_client = Client::builder(TokioExecutor::new()) + let mut http_client_builder = Client::builder(TokioExecutor::new()); + http_client_builder .http1_title_case_headers(true) - .http1_preserve_header_case(true) - .build_http(); - http_client.request(request) + .http1_preserve_header_case(true); + let https_connector = if cli.http2 { + http_client_builder.http2_only(false).build_http() + } else if cli.http2_prior_knowledge { + http_client_builder.http2_only(true).build_http() + } else { + http_client_builder.build_http() + }; + https_connector.request(request) }; fut };