Merge pull request #38 from lsk569937453/dev

Add the http2 and http2-prior-knowledge.

Cargo.toml

@@ -1,51 +1,51 @@
 [package]
 name = "rcurl"
-version = "0.0.23"
+version = "0.0.24"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 anyhow = "1.0"
 async-std = "1.12.0"
-async-tls = "0.8"
+async-tls = "0.13.0"
 bytes = "1"
-chrono = "0.4.31"
+chrono = "0.4.38"
 clap = { version = "4.5.4", features = ["derive", "string"] }
 env_logger = "^0.11"
 form-data-builder = "1.0.1"
-futures = "0.3.29"
-futures-util = { version = "0.3.1", default-features = false }
+futures = "0.3.30"
+futures-util = { version = "0.3.30", default-features = false }
 log = "^0.4"
 http = "1.1.0"
 http-body-util = { version = "0.1" }
-hyper = { version = "1.3.1", features = ["full", "tracing", "http2", "client"] }
+hyper = { version = "1.3.1", features = ["full"] }
 hyper-util = { version = "0.1.4", features = ["full"] }
 indicatif = "0.17.7"
 mime_guess = "2.0.4"
-openssl = { version = "0.10", features = ["vendored"] }
+openssl = { version = "0.10.64", features = ["vendored"] }
 pki-types = { package = "rustls-pki-types", version = "1" }
 http-range-header = "0.4.1"
-rustls = { version = "0.23.5", default-features = false, features = [
+rustls = { version = "0.23.8", default-features = false, features = [
     "logging",
     "ring",
     "tls12",
 ] }
-rustls-pemfile = "2"
-serde = { version = "1.0.190", features = ["derive"] }
-serde_json = "1.0.108"
-suppaftp = { version = "6.0.0", features = ["async-native-tls", "native-tls"] }
+rustls-pemfile = "2.1.2"
+serde = { version = "1.0.203", features = ["derive"] }
+serde_json = "1.0.117"
+suppaftp = { version = "6.0.1", features = ["async-native-tls", "native-tls"] }
 
 tokio = { version = "1.37.0", features = ["full"] }
 tokio-rustls = { version = "0.26.0", default-features = false, features = [
     "logging",
     "ring",
 ] }
 tokio-util = { version = "0.7.11", features = ["full", "time"] }
-tracing = "0.1.4"
+tracing = "0.1.40"
 tracing-appender = "0.2.3"
 tracing-subscriber = "0.3.18"
-webpki-roots = "0.26.0"
+webpki-roots = "0.26.1"
 hyper-rustls = { version = "0.27.1", default-features = false, features = [
     "logging",
     "ring",

README.md

@@ -24,10 +24,13 @@ Options:
   -e, --referer <URL>                                      Referrer URL
   -o, --output <file>                                      Write to file instead of stdout
   -T, --upload-file <file>                                 Transfer local FILE to destination
+  -Q, --quote <command>                                    Send command(s) to server before transfer
   -k, --insecure                                           Allow insecure server connections
   -I, --head                                               Show document info only
   -r, --range <range>                                      Retrieve only the bytes within RANGE
   -v, --verbose                                            Make the operation more talkative
+      --http2
+      --http2-prior-knowledge
   -h, --help                                               Print help
   -V, --version                                            Print version
 ```

src/cli/app_config.rs

@@ -28,12 +28,7 @@ pub struct Cli {
     #[arg(short = 'A', long = "user-agent", value_name = "name")]
     pub user_agent_option: Option<String>,
     /// The Cookie option.
-    #[arg(
-        short = 'b',
-        long = "cookie",
-        value_name = "data|filename",
-        hide_short_help = true
-    )]
+    #[arg(short = 'b', long = "cookie", value_name = "data|filename")]
     pub cookie_option: Option<String>,
     ///  Referrer URL
     #[arg(short = 'e', long = "referer", value_name = "URL")]
@@ -47,7 +42,6 @@ pub struct Cli {
         default_missing_value = "none"
     )]
     pub file_path_option: Option<String>,
-
     ///  Transfer local FILE to destination
     #[arg(long = "upload-file", short = 'T', value_name = "file")]
     pub uploadfile_option: Option<String>,
@@ -68,6 +62,10 @@ pub struct Cli {
     ///  Make the operation more talkative
     #[arg(short = 'v', long = "verbose")]
     pub debug: bool,
+    #[arg(long = "http2")]
+    pub http2: bool,
+    #[arg(long = "http2-prior-knowledge")]
+    pub http2_prior_knowledge: bool,
     // /// Print help
     // #[arg(short, long, action = clap::ArgAction::Help)]
     // pub help: bool,
@@ -95,6 +93,8 @@ impl Cli {
             skip_certificate_validate: false,
             header_option: false,
             range_option: None,
+            http2: false,
+            http2_prior_knowledge: false,
             debug: false,
             // help: false,
             // help_all: false,

src/ftp/handler.rs

@@ -9,9 +9,9 @@ use futures::io::BufReader;
 use indicatif::ProgressBar;
 use indicatif::ProgressStyle;
 use log::LevelFilter;
-use suppaftp::{AsyncNativeTlsConnector, AsyncNativeTlsFtpStream};
 use suppaftp::async_native_tls::TlsConnector;
 use suppaftp::types::FileType;
+use suppaftp::{AsyncNativeTlsConnector, AsyncNativeTlsFtpStream};
 use tokio::fs::OpenOptions;
 use tokio::io::AsyncWriteExt;
 
@@ -40,14 +40,6 @@ impl<W: futures::io::AsyncRead + Unpin> futures::io::AsyncRead for ProgressBarIt
 }
 
 pub async fn ftp_request(cli: Cli, scheme: &str) -> Result<(), anyhow::Error> {
-    let log_level_hyper = if cli.debug {
-        LevelFilter::Trace
-    } else {
-        LevelFilter::Info
-    };
-
-    // init logger
-    let _ = Builder::new().filter_level(log_level_hyper).try_init();
     let uri: hyper::Uri = cli.url.parse()?;
     let host = uri.host().ok_or(anyhow!(""))?;
     let port = uri.port_u16().unwrap_or(21);

src/http/handler.rs

@@ -1,6 +1,7 @@
 use crate::cli::app_config::Cli;
 use bytes::Bytes;
 
+use env_logger::Builder;
 use http::header::ACCEPT;
 use http_body_util::Full;
 use hyper::header::CONTENT_TYPE;
@@ -9,6 +10,7 @@ use hyper::header::HOST;
 use hyper::header::REFERER;
 use hyper::header::{HeaderName, HeaderValue};
 use hyper::Request;
+use hyper::Version;
 use hyper_util::rt::TokioExecutor;
 use hyper_util::rt::TokioIo;
 use mime_guess::mime;
@@ -35,6 +37,7 @@ use hyper::header::CONTENT_LENGTH;
 use hyper::Response;
 use indicatif::ProgressBar;
 use indicatif::ProgressStyle;
+use log::LevelFilter;
 use rustls::crypto::ring::default_provider;
 
 use futures::StreamExt;
@@ -43,6 +46,7 @@ use http_body_util::BodyExt;
 use http::response::Parts;
 use http_body_util::combinators::BoxBody;
 use hyper::body::Buf;
+use hyper::HeaderMap;
 use hyper_util::client::legacy::Client;
 use rustls::crypto::ring::DEFAULT_CIPHER_SUITES;
 use rustls::crypto::CryptoProvider;
@@ -179,14 +183,6 @@ pub async fn http_request(
     cli: Cli,
     scheme: &str,
 ) -> Result<Response<BoxBody<Bytes, Infallible>>, anyhow::Error> {
-    let log_level_hyper = if cli.debug { Level::TRACE } else { Level::INFO };
-
-    let subscriber = tracing_subscriber::fmt()
-        .with_level(true)
-        .with_max_level(log_level_hyper)
-        .finish();
-    let _ = tracing::subscriber::set_global_default(subscriber);
-
     let mut root_store = RootCertStore::empty();
 
     if let Some(file_path) = cli.certificate_path_option.clone() {
@@ -218,14 +214,6 @@ pub async fn http_request(
             .set_certificate_verifier(Arc::new(NoCertificateVerification::new(default_provider())));
     };
     let uri: hyper::Uri = cli.url.parse()?;
-    let host = uri.host().expect("uri has no host");
-    let default_port = if let Some(port) = uri.port_u16() {
-        port
-    } else if uri.scheme_str() == Some("https") {
-        443
-    } else {
-        80
-    };
     let mut method = String::from("GET");
     let mut content_type_option = None;
 
@@ -239,29 +227,32 @@ pub async fn http_request(
     let mut request_builder = Request::builder()
         .method(method.as_str())
         .uri(cli.url.clone());
+    if cli.http2_prior_knowledge {
+        request_builder = request_builder.version(hyper::Version::HTTP_2);
+    }
+    let mut header_map = HeaderMap::new();
     if let Some(content_type) = content_type_option {
-        request_builder =
-            request_builder.header(CONTENT_TYPE, HeaderValue::from_str(&content_type)?);
+        header_map.insert(CONTENT_TYPE, HeaderValue::from_str(&content_type)?);
     }
-    request_builder = request_builder.header(ACCEPT, HeaderValue::from_str("*/*")?);
-    request_builder = request_builder.header(
-        HOST,
-        HeaderValue::from_str(uri.host().ok_or(anyhow!("no host"))?)?,
-    );
+    header_map.insert(ACCEPT, HeaderValue::from_str("*/*")?);
+    // header_map.insert(
+    //     HOST,
+    //     HeaderValue::from_str(uri.host().ok_or(anyhow!("no host"))?)?,
+    // );
     let user_agent = cli
         .user_agent_option
         .clone()
-        .unwrap_or(format!("rcur/{}", env!("CARGO_PKG_VERSION").to_string()));
-    request_builder = request_builder.header(USER_AGENT, HeaderValue::from_str(&user_agent)?);
+        .unwrap_or(format!("rcurl/{}", env!("CARGO_PKG_VERSION").to_string()));
+    header_map.insert(USER_AGENT, HeaderValue::from_str(&user_agent)?);
     if let Some(cookie) = cli.cookie_option.clone() {
-        request_builder = request_builder.header(COOKIE, HeaderValue::from_str(&cookie)?);
+        header_map.insert(COOKIE, HeaderValue::from_str(&cookie)?);
     }
     if let Some(range) = cli.range_option.clone() {
         let ranges_format = format!("bytes={}", range);
-        request_builder = request_builder.header(RANGE, HeaderValue::from_str(&ranges_format)?);
+        header_map.insert(RANGE, HeaderValue::from_str(&ranges_format)?);
     }
     if let Some(refer) = cli.refer_option.clone() {
-        request_builder = request_builder.header(REFERER, HeaderValue::from_str(&refer)?);
+        header_map.insert(REFERER, HeaderValue::from_str(&refer)?);
     }
     let mut body_bytes = Bytes::new();
     if cli.form_option.len() != 0 {
@@ -302,18 +293,23 @@ pub async fn http_request(
     }
     for x in cli.headers.clone() {
         let split: Vec<String> = x.splitn(2, ':').map(|s| s.to_string()).collect();
+
         if split.len() == 2 {
             let key = &split[0];
             let value = &split[1];
             let new_value = value.trim_start();
-            request_builder = request_builder.header(
+
+            header_map.insert(
                 HeaderName::from_str(key.as_str())?,
                 HeaderValue::from_str(new_value)?,
             );
         } else {
             return Err(anyhow!("header error"));
         }
     }
+    for (key, val) in header_map {
+        request_builder = request_builder.header(key.ok_or(anyhow!(""))?, val);
+    }
     let content_length = body_bytes.len();
     let body = Full::new(body_bytes);
     let request = request_builder.body(body)?;
@@ -331,53 +327,38 @@ pub async fn http_request(
         println!("> Content-Length: {}", content_length);
     }
 
-    let port = uri.port_u16().unwrap_or(default_port);
-    let addr = format!("{}:{}", host, port);
-    let stream = TcpStream::connect(addr.clone()).await?;
-    let remote_addr = stream.peer_addr()?.to_string();
-    let local_addr = stream.local_addr()?.to_string();
     let span = tracing::info_span!("Rcurl");
     let _enter = span.enter();
     let request_future = {
         trace!("Start request");
         let fut = if scheme == "https" {
-            let https = hyper_rustls::HttpsConnectorBuilder::new()
+            let connector_builder = hyper_rustls::HttpsConnectorBuilder::new()
                 .with_tls_config(tls_config)
-                .https_only()
-                .enable_http1()
-                .build();
+                .https_only();
+            let https_connector = if cli.http2 {
+                connector_builder.enable_all_versions().build()
+            } else if cli.http2_prior_knowledge {
+                connector_builder.enable_http2().build()
+            } else {
+                connector_builder.enable_http1().build()
+            };
+
             let https_clientt: Client<_, Full<Bytes>> =
-                Client::builder(TokioExecutor::new()).build(https);
-            // let domain = pki_types::ServerName::try_from(host)
-            //     .map_err(|e| anyhow!("{}", e))?
-            //     .to_owned();
-            // let tls_stream = connector.connect(domain, stream).await?;
-            // let stream_io = TokioIo::new(tls_stream);
-
-            // let (mut sender, conn) = hyper::client::conn::http1::handshake(stream_io)
-            //     .instrument(info_span!("Https Handshake"))
-            //     .await?;
-            // tokio::task::spawn(async move {
-            //     if let Err(err) = conn
-            //         .instrument(info_span!(
-            //             "rcurl",
-            //             localAddr=%local_addr,
-            //              remoteAddr=remote_addr,
-
-            //         ))
-            //         .await
-            //     {
-            //         println!("Connection failed: {:?}", err);
-            //     }
-            // });
+                Client::builder(TokioExecutor::new()).build(https_connector);
             https_clientt.request(request)
-            // sender.send_request(request)
         } else {
-            let http_client = Client::builder(TokioExecutor::new())
+            let mut http_client_builder = Client::builder(TokioExecutor::new());
+            http_client_builder
                 .http1_title_case_headers(true)
-                .http1_preserve_header_case(true)
-                .build_http();
-            http_client.request(request)
+                .http1_preserve_header_case(true);
+            let https_connector = if cli.http2 {
+                http_client_builder.http2_only(false).build_http()
+            } else if cli.http2_prior_knowledge {
+                http_client_builder.http2_only(true).build_http()
+            } else {
+                http_client_builder.build_http()
+            };
+            https_connector.request(request)
         };
         fut
     };