From f85194ac1ef932809a7e8fc2dff18fa72a1be78f Mon Sep 17 00:00:00 2001 From: zuisong Date: Tue, 21 Jan 2025 17:52:05 +0800 Subject: [PATCH 1/6] support compress request body --- completions/_xh | 7 ++- completions/_xh.ps1 | 3 + completions/xh.bash | 2 +- completions/xh.elv | 3 + completions/xh.fish | 2 + completions/xh.nu | 4 +- doc/xh.1 | 17 +++++- src/cli.rs | 5 ++ src/download.rs | 2 +- src/main.rs | 27 ++++++++- src/to_curl.rs | 2 + tests/cli.rs | 142 +++++++++++++++++++++++++++++++++++++++++++- 12 files changed, 205 insertions(+), 11 deletions(-) diff --git a/completions/_xh b/completions/_xh index 258f21d0..50c83257 100644 --- a/completions/_xh +++ b/completions/_xh @@ -49,7 +49,7 @@ none\:"Disable both coloring and formatting"))' \ '--http-version=[HTTP version to use]:VERSION:(1.0 1.1 2 2-prior-knowledge)' \ '*--resolve=[Override DNS resolution for specific domain to a custom IP]:HOST:ADDRESS:_default' \ '--interface=[Bind to a network interface or local IP address]:NAME:_default' \ -'--generate=[Generate shell completions or man pages]:KIND:(complete-bash complete-elvish complete-fish complete-nushell complete-powershell complete-zsh man)' \ +'()--generate=[Generate shell completions or man pages]:KIND:(complete-bash complete-elvish complete-fish complete-nushell complete-powershell complete-zsh man)' \ '-j[(default) Serialize data items from the command line as a JSON object]' \ '--json[(default) Serialize data items from the command line as a JSON object]' \ '-f[Serialize data items from the command line as form fields]' \ @@ -69,6 +69,8 @@ none\:"Disable both coloring and formatting"))' \ '*--quiet[Do not print to stdout or stderr]' \ '-S[Always stream the response body]' \ '--stream[Always stream the response body]' \ +'*-x[Content compressed (encoded) with Deflate algorithm. The Content-Encoding header is set to deflate]' \ +'*--compress[Content compressed (encoded) with Deflate algorithm. The Content-Encoding header is set to deflate]' \ '-d[Download the body to a file instead of printing it]' \ '--download[Download the body to a file instead of printing it]' \ '-c[Resume an interrupted download. Requires --download and --output]' \ @@ -108,6 +110,7 @@ none\:"Disable both coloring and formatting"))' \ '--no-history-print[]' \ '--no-quiet[]' \ '--no-stream[]' \ +'--no-compress[]' \ '--no-output[]' \ '--no-download[]' \ '--no-continue[]' \ @@ -142,7 +145,7 @@ none\:"Disable both coloring and formatting"))' \ '--no-help[]' \ '-V[Print version]' \ '--version[Print version]' \ -'::raw_method_or_url -- The request URL, preceded by an optional HTTP method:_default' \ +':raw_method_or_url -- The request URL, preceded by an optional HTTP method:_default' \ '*::raw_rest_args -- Optional key-value pairs to be included in the request.:_default' \ && ret=0 } diff --git a/completions/_xh.ps1 b/completions/_xh.ps1 index 27ed4af9..d37ef272 100644 --- a/completions/_xh.ps1 +++ b/completions/_xh.ps1 @@ -72,6 +72,8 @@ Register-ArgumentCompleter -Native -CommandName 'xh' -ScriptBlock { [CompletionResult]::new('--quiet', '--quiet', [CompletionResultType]::ParameterName, 'Do not print to stdout or stderr') [CompletionResult]::new('-S', '-S ', [CompletionResultType]::ParameterName, 'Always stream the response body') [CompletionResult]::new('--stream', '--stream', [CompletionResultType]::ParameterName, 'Always stream the response body') + [CompletionResult]::new('-x', '-x', [CompletionResultType]::ParameterName, 'Content compressed (encoded) with Deflate algorithm. The Content-Encoding header is set to deflate') + [CompletionResult]::new('--compress', '--compress', [CompletionResultType]::ParameterName, 'Content compressed (encoded) with Deflate algorithm. The Content-Encoding header is set to deflate') [CompletionResult]::new('-d', '-d', [CompletionResultType]::ParameterName, 'Download the body to a file instead of printing it') [CompletionResult]::new('--download', '--download', [CompletionResultType]::ParameterName, 'Download the body to a file instead of printing it') [CompletionResult]::new('-c', '-c', [CompletionResultType]::ParameterName, 'Resume an interrupted download. Requires --download and --output') @@ -111,6 +113,7 @@ Register-ArgumentCompleter -Native -CommandName 'xh' -ScriptBlock { [CompletionResult]::new('--no-history-print', '--no-history-print', [CompletionResultType]::ParameterName, 'no-history-print') [CompletionResult]::new('--no-quiet', '--no-quiet', [CompletionResultType]::ParameterName, 'no-quiet') [CompletionResult]::new('--no-stream', '--no-stream', [CompletionResultType]::ParameterName, 'no-stream') + [CompletionResult]::new('--no-compress', '--no-compress', [CompletionResultType]::ParameterName, 'no-compress') [CompletionResult]::new('--no-output', '--no-output', [CompletionResultType]::ParameterName, 'no-output') [CompletionResult]::new('--no-download', '--no-download', [CompletionResultType]::ParameterName, 'no-download') [CompletionResult]::new('--no-continue', '--no-continue', [CompletionResultType]::ParameterName, 'no-continue') diff --git a/completions/xh.bash b/completions/xh.bash index 0637dc97..d94c9e51 100644 --- a/completions/xh.bash +++ b/completions/xh.bash @@ -19,7 +19,7 @@ _xh() { case "${cmd}" in xh) - opts="-j -f -s -p -h -b -m -v -P -q -S -o -d -c -A -a -F -4 -6 -I -V --json --form --multipart --raw --pretty --format-options --style --response-charset --response-mime --print --headers --body --meta --verbose --debug --all --history-print --quiet --stream --output --download --continue --session --session-read-only --auth-type --auth --bearer --ignore-netrc --offline --check-status --follow --max-redirects --timeout --proxy --verify --cert --cert-key --ssl --native-tls --default-scheme --https --http-version --resolve --interface --ipv4 --ipv6 --ignore-stdin --curl --curl-long --generate --help --no-json --no-form --no-multipart --no-raw --no-pretty --no-format-options --no-style --no-response-charset --no-response-mime --no-print --no-headers --no-body --no-meta --no-verbose --no-debug --no-all --no-history-print --no-quiet --no-stream --no-output --no-download --no-continue --no-session --no-session-read-only --no-auth-type --no-auth --no-bearer --no-ignore-netrc --no-offline --no-check-status --no-follow --no-max-redirects --no-timeout --no-proxy --no-verify --no-cert --no-cert-key --no-ssl --no-native-tls --no-default-scheme --no-https --no-http-version --no-resolve --no-interface --no-ipv4 --no-ipv6 --no-ignore-stdin --no-curl --no-curl-long --no-generate --no-help --version [[METHOD] URL] [REQUEST_ITEM]..." + opts="-j -f -s -p -h -b -m -v -P -q -S -x -o -d -c -A -a -F -4 -6 -I -V --json --form --multipart --raw --pretty --format-options --style --response-charset --response-mime --print --headers --body --meta --verbose --debug --all --history-print --quiet --stream --compress --output --download --continue --session --session-read-only --auth-type --auth --bearer --ignore-netrc --offline --check-status --follow --max-redirects --timeout --proxy --verify --cert --cert-key --ssl --native-tls --default-scheme --https --http-version --resolve --interface --ipv4 --ipv6 --ignore-stdin --curl --curl-long --generate --help --no-json --no-form --no-multipart --no-raw --no-pretty --no-format-options --no-style --no-response-charset --no-response-mime --no-print --no-headers --no-body --no-meta --no-verbose --no-debug --no-all --no-history-print --no-quiet --no-stream --no-compress --no-output --no-download --no-continue --no-session --no-session-read-only --no-auth-type --no-auth --no-bearer --no-ignore-netrc --no-offline --no-check-status --no-follow --no-max-redirects --no-timeout --no-proxy --no-verify --no-cert --no-cert-key --no-ssl --no-native-tls --no-default-scheme --no-https --no-http-version --no-resolve --no-interface --no-ipv4 --no-ipv6 --no-ignore-stdin --no-curl --no-curl-long --no-generate --no-help --version <[METHOD] URL> [REQUEST_ITEM]..." if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) return 0 diff --git a/completions/xh.elv b/completions/xh.elv index 028561d0..e13f31a6 100644 --- a/completions/xh.elv +++ b/completions/xh.elv @@ -69,6 +69,8 @@ set edit:completion:arg-completer[xh] = {|@words| cand --quiet 'Do not print to stdout or stderr' cand -S 'Always stream the response body' cand --stream 'Always stream the response body' + cand -x 'Content compressed (encoded) with Deflate algorithm. The Content-Encoding header is set to deflate' + cand --compress 'Content compressed (encoded) with Deflate algorithm. The Content-Encoding header is set to deflate' cand -d 'Download the body to a file instead of printing it' cand --download 'Download the body to a file instead of printing it' cand -c 'Resume an interrupted download. Requires --download and --output' @@ -108,6 +110,7 @@ set edit:completion:arg-completer[xh] = {|@words| cand --no-history-print 'no-history-print' cand --no-quiet 'no-quiet' cand --no-stream 'no-stream' + cand --no-compress 'no-compress' cand --no-output 'no-output' cand --no-download 'no-download' cand --no-continue 'no-continue' diff --git a/completions/xh.fish b/completions/xh.fish index 2f876cf4..16270098 100644 --- a/completions/xh.fish +++ b/completions/xh.fish @@ -35,6 +35,7 @@ complete -c xh -l debug -d 'Print full error stack traces and debug log messages complete -c xh -l all -d 'Show any intermediary requests/responses while following redirects with --follow' complete -c xh -s q -l quiet -d 'Do not print to stdout or stderr' complete -c xh -s S -l stream -d 'Always stream the response body' +complete -c xh -s x -l compress -d 'Content compressed (encoded) with Deflate algorithm. The Content-Encoding header is set to deflate' complete -c xh -s d -l download -d 'Download the body to a file instead of printing it' complete -c xh -s c -l continue -d 'Resume an interrupted download. Requires --download and --output' complete -c xh -l ignore-netrc -d 'Do not use credentials from .netrc' @@ -68,6 +69,7 @@ complete -c xh -l no-all complete -c xh -l no-history-print complete -c xh -l no-quiet complete -c xh -l no-stream +complete -c xh -l no-compress complete -c xh -l no-output complete -c xh -l no-download complete -c xh -l no-continue diff --git a/completions/xh.nu b/completions/xh.nu index 33e2767b..564c0551 100644 --- a/completions/xh.nu +++ b/completions/xh.nu @@ -45,6 +45,7 @@ module completions { --history-print(-P): string # The same as --print but applies only to intermediary requests/responses --quiet(-q) # Do not print to stdout or stderr --stream(-S) # Always stream the response body + --compress(-x) # Content compressed (encoded) with Deflate algorithm. The Content-Encoding header is set to deflate --output(-o): string # Save output to FILE instead of stdout --download(-d) # Download the body to a file instead of printing it --continue(-c) # Resume an interrupted download. Requires --download and --output @@ -77,7 +78,7 @@ module completions { --curl-long # Use the long versions of curl's flags --generate: string@"nu-complete xh generate" # Generate shell completions or man pages --help # Print help - raw_method_or_url?: string # The request URL, preceded by an optional HTTP method + raw_method_or_url: string # The request URL, preceded by an optional HTTP method ...raw_rest_args: string # Optional key-value pairs to be included in the request. --no-json --no-form @@ -98,6 +99,7 @@ module completions { --no-history-print --no-quiet --no-stream + --no-compress --no-output --no-download --no-continue diff --git a/doc/xh.1 b/doc/xh.1 index e4f44c1c..444cb9d8 100644 --- a/doc/xh.1 +++ b/doc/xh.1 @@ -1,4 +1,4 @@ -.TH XH 1 2025-01-04 0.23.1 "User Commands" +.TH XH 1 2025-01-22 0.23.1 "User Commands" .SH NAME xh \- Friendly and fast tool for sending HTTP requests @@ -188,6 +188,9 @@ Using quiet twice i.e. \-qq will suppress warnings as well. \fB\-S\fR, \fB\-\-stream\fR Always stream the response body. .TP 4 +\fB\-x\fR, \fB\-\-compress\fR +Content compressed (encoded) with Deflate algorithm. The Content\-Encoding header is set to deflate. +.TP 4 \fB\-o\fR, \fB\-\-output\fR=\fIFILE\fR Save output to FILE instead of stdout. .TP 4 @@ -321,9 +324,17 @@ For translating the other way, try https://curl2httpie.online/. Use the long versions of curl's flags. .TP 4 \fB\-\-generate\fR=\fIKIND\fR -Generate shell completions or man pages. +Generate shell completions or man pages. Possible values are: + + complete\-bash + complete\-elvish + complete\-fish + complete\-nushell + complete\-powershell + complete\-zsh + man -[possible values: complete\-bash, complete\-elvish, complete\-fish, complete\-nushell, complete\-powershell, complete\-zsh, man] +Example: xh \-\-generate=complete\-bash > xh.bash. .TP 4 \fB\-\-help\fR Print help. diff --git a/src/cli.rs b/src/cli.rs index 8d227a18..1c2bf554 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -182,6 +182,11 @@ Example: --print=Hb" #[clap(short = 'S', long = "stream", name = "stream")] pub stream_raw: bool, + /// Content compressed (encoded) with Deflate algorithm. + /// The Content-Encoding header is set to deflate. + #[clap(short = 'x', long = "compress", name = "compress", action = ArgAction::Count)] + pub compress: u8, + #[clap(skip)] pub stream: Option, diff --git a/src/download.rs b/src/download.rs index d4cd6678..f9d8127c 100644 --- a/src/download.rs +++ b/src/download.rs @@ -60,7 +60,7 @@ fn get_file_name(response: &Response, orig_url: &reqwest::Url) -> String { .or_else(|| from_url(orig_url)) .unwrap_or_else(|| "index".to_string()); - let filename = filename.split(std::path::is_separator).last().unwrap(); + let filename = filename.split(std::path::is_separator).next_back().unwrap(); let mut filename = filename.trim().trim_start_matches('.').to_string(); diff --git a/src/main.rs b/src/main.rs index 2caac847..72ab96e3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -19,7 +19,7 @@ mod utils; use std::env; use std::fs::File; -use std::io::{self, IsTerminal, Read}; +use std::io::{self, IsTerminal, Read, Write as _}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use std::path::PathBuf; use std::process; @@ -28,8 +28,10 @@ use std::sync::Arc; use anyhow::{anyhow, Context, Result}; use cookie_store::{CookieStore, RawCookie}; +use flate2::write::ZlibEncoder; +use hyper::header::CONTENT_ENCODING; use redirect::RedirectFollower; -use reqwest::blocking::Client; +use reqwest::blocking::{Body as ReqwestBody, Client}; use reqwest::header::{ HeaderValue, ACCEPT, ACCEPT_ENCODING, CONNECTION, CONTENT_TYPE, COOKIE, RANGE, USER_AGENT, }; @@ -508,6 +510,27 @@ fn run(args: Cli) -> Result { let mut request = request_builder.headers(headers).build()?; + if args.compress >= 1 && request.headers().get(CONTENT_ENCODING).is_none() { + let mut compressed = false; + if let Some(body) = request.body_mut() { + if let Some(body_bytes) = body.as_bytes() { + let mut encoder = ZlibEncoder::new(Vec::new(), Default::default()); + encoder.write_all(body_bytes)?; + let output = encoder.finish()?; + if output.len() < body_bytes.len() || args.compress >= 2 { + let _ = std::mem::replace(body, ReqwestBody::from(output)); + compressed = true; + } + } + } + if compressed { + request + .headers_mut() + .entry(CONTENT_ENCODING) + .or_insert(HeaderValue::from_static("deflate")); + } + } + for header in &headers_to_unset { request.headers_mut().remove(header); } diff --git a/src/to_curl.rs b/src/to_curl.rs index 744970e4..adeebf1f 100644 --- a/src/to_curl.rs +++ b/src/to_curl.rs @@ -97,6 +97,8 @@ pub fn translate(args: Cli) -> Result { // No equivalent (args.style.is_some(), "-s/--style"), // No equivalent + (args.compress > 0, "-x/--compress"), + // No equivalent (args.response_charset.is_some(), "--response-charset"), // No equivalent (args.response_mime.is_some(), "--response-mime"), diff --git a/tests/cli.rs b/tests/cli.rs index c2008a64..a5243165 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -6,7 +6,7 @@ mod server; use std::collections::{HashMap, HashSet}; use std::fs::{self, File, OpenOptions}; use std::future::Future; -use std::io::Write; +use std::io::{Read as _, Write}; use std::iter::FromIterator; use std::net::IpAddr; use std::pin::Pin; @@ -3483,6 +3483,146 @@ fn zstd() { "#}); } +#[test] +fn compress_request_body() { + fn zlib_decode(bytes: Vec) -> std::io::Result { + let mut z = flate2::read::ZlibDecoder::new(&bytes[..]); + let mut s = String::new(); + z.read_to_string(&mut s)?; + Ok(s) + } + + let server = server::http(|req| async move { + match req.uri().path() { + "/deflate" => { + assert_eq!( + req.headers().get(hyper::header::CONTENT_ENCODING), + Some(HeaderValue::from_static("deflate")).as_ref() + ); + + let compressed_body = req.body().await; + let body = zlib_decode(compressed_body).unwrap(); + hyper::Response::builder() + .header("date", "N/A") + .header("Content-Type", "text/plain") + .body(body.into()) + .unwrap() + } + "/normal" => { + let body = req.body_as_string().await; + hyper::Response::builder() + .header("date", "N/A") + .header("Content-Type", "text/plain") + .body(body.into()) + .unwrap() + } + _ => panic!("unknown path"), + } + }); + + get_command() + .arg(format!("{}/deflate", server.base_url())) + .args([ + &format!("key={}", "1".repeat(1000)), + "-x", + "-j", + "--pretty=none", + ]) + .assert() + .stdout(indoc::formatdoc! {r#" + HTTP/1.1 200 OK + Date: N/A + Content-Type: text/plain + Content-Length: 1010 + + {{"key":"{c}"}} + "#, c = "1".repeat(1000),}); + + get_command() + .arg(format!("{}/deflate", server.base_url())) + .args([ + &format!("key={}", "1".repeat(1000)), + "-x", + "-x", + "-f", + "--pretty=none", + ]) + .assert() + .stdout(indoc::formatdoc! {r#" + HTTP/1.1 200 OK + Date: N/A + Content-Type: text/plain + Content-Length: 1004 + + key={c} + "#, c = "1".repeat(1000),}); + + get_command() + .arg(format!("{}/deflate", server.base_url())) + .args([ + &format!("key={}", "1".repeat(1000)), + "-x", + "-x", + "-f", + "--pretty=none", + ]) + .assert() + .stdout(indoc::formatdoc! {r#" + HTTP/1.1 200 OK + Date: N/A + Content-Type: text/plain + Content-Length: 1004 + + key={c} + "#, c = "1".repeat(1000),}); + + get_command() + .arg(format!("{}/normal", server.base_url())) + .args([&format!("key={}", "1"), "-x", "-f", "--pretty=none"]) + .assert() + .stdout(indoc::formatdoc! {r#" + HTTP/1.1 200 OK + Date: N/A + Content-Type: text/plain + Content-Length: 5 + + key={c} + "#, c = "1"}); + + // force compress + get_command() + .arg(format!("{}/deflate", server.base_url())) + .args([&format!("key={}", "1"), "-xx", "-f", "--pretty=none"]) + .assert() + .stdout(indoc::formatdoc! {r#" + HTTP/1.1 200 OK + Date: N/A + Content-Type: text/plain + Content-Length: 5 + + key={c} + "#, c = "1"}); + // dont compress_request_body_if_content_encoding_have_value + get_command() + .arg(format!("{}/normal", server.base_url())) + .args([ + &format!("key={}", "1".repeat(1000)), + "content-encoding:gzip", + "-x", + "-f", + "--pretty=none", + ]) + .assert() + .stdout(indoc::formatdoc! {r#" + HTTP/1.1 200 OK + Date: N/A + Content-Type: text/plain + Content-Length: 1004 + + key={c} + "#, c = "1".repeat(1000),}); +} + #[test] fn empty_response_with_content_encoding() { let server = server::http(|_req| async move { From 9b26a314158c120c938fe5d572b3665cbf70491c Mon Sep 17 00:00:00 2001 From: zuisong Date: Wed, 22 Jan 2025 21:35:38 +0800 Subject: [PATCH 2/6] support compress stream data --- src/main.rs | 2 +- tests/cli.rs | 40 +++++++++++++++++++++++++++++++++------- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/src/main.rs b/src/main.rs index 72ab96e3..e0b26dd7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -513,7 +513,7 @@ fn run(args: Cli) -> Result { if args.compress >= 1 && request.headers().get(CONTENT_ENCODING).is_none() { let mut compressed = false; if let Some(body) = request.body_mut() { - if let Some(body_bytes) = body.as_bytes() { + if let Ok(body_bytes) = body.buffer() { let mut encoder = ZlibEncoder::new(Vec::new(), Default::default()); encoder.write_all(body_bytes)?; let output = encoder.finish()?; diff --git a/tests/cli.rs b/tests/cli.rs index a5243165..6ea68dd4 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -3484,14 +3484,40 @@ fn zstd() { } #[test] -fn compress_request_body() { - fn zlib_decode(bytes: Vec) -> std::io::Result { - let mut z = flate2::read::ZlibDecoder::new(&bytes[..]); - let mut s = String::new(); - z.read_to_string(&mut s)?; - Ok(s) - } +fn compress_body_from_file() { + let server = server::http(|req| async move { + assert_eq!("Hello world\n", zlib_decode(req.body().await).unwrap()); + hyper::Response::default() + }); + let dir = tempfile::tempdir().unwrap(); + let filename = dir.path().join("input.txt"); + OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&filename) + .unwrap() + .write_all(b"Hello world\n") + .unwrap(); + + get_command() + .arg(server.base_url()) + .arg("-xx") + .arg(format!("@{}", filename.to_string_lossy())) + .assert() + .success(); +} + +fn zlib_decode(bytes: Vec) -> std::io::Result { + let mut z = flate2::read::ZlibDecoder::new(&bytes[..]); + let mut s = String::new(); + z.read_to_string(&mut s)?; + Ok(s) +} + +#[test] +fn compress_request_body() { let server = server::http(|req| async move { match req.uri().path() { "/deflate" => { From 70ad491c81eedc5bf4f323e836279ae3160a44d7 Mon Sep 17 00:00:00 2001 From: zuisong Date: Wed, 22 Jan 2025 22:04:04 +0800 Subject: [PATCH 3/6] compress request body online test --- tests/cli.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/cli.rs b/tests/cli.rs index 6ea68dd4..7c266563 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -19,6 +19,7 @@ use indoc::indoc; use predicates::function::function; use predicates::str::contains; use reqwest::header::HeaderValue; +use serde_json::Value; use tempfile::{tempdir, NamedTempFile, TempDir}; pub trait RequestExt { @@ -779,6 +780,25 @@ fn successful_digest_auth() { .stdout(contains("HTTP/1.1 200 OK")); } +#[cfg(feature = "online-tests")] +#[test] +fn compress_request_body_online() { + get_command() + .arg("https://postman-echo.com/post") + .args(["-xx", "--body", &format!("a={}", "1".repeat(1000))]) + .assert() + .stdout(function(|body: &str| { + let json: Value = serde_json::from_str(body).unwrap(); + assert_eq!(json["json"]["a"], Value::String("1".repeat(1000))); + if let Some(request_body_length) = json["headers"]["content-length"].as_str() { + let length: i32 = request_body_length.parse().unwrap(); + assert!(length < 1000) + } + + true + })); +} + #[cfg(feature = "online-tests")] #[test] fn unsuccessful_digest_auth() { From 2a1fba66f7bde77df6bb34c3318ace83a64b9bf9 Mon Sep 17 00:00:00 2001 From: zuisong Date: Wed, 22 Jan 2025 22:23:50 +0800 Subject: [PATCH 4/6] add more test --- tests/cli.rs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/cli.rs b/tests/cli.rs index 7c266563..809130c3 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -3529,6 +3529,32 @@ fn compress_body_from_file() { .success(); } +#[test] +fn compress_body_from_file_unless_compress_rate_less_1() { + let server = server::http(|req| async move { + assert_eq!("Hello world\n", req.body_as_string().await); + hyper::Response::default() + }); + + let dir = tempfile::tempdir().unwrap(); + let filename = dir.path().join("input.txt"); + OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&filename) + .unwrap() + .write_all(b"Hello world\n") + .unwrap(); + + get_command() + .arg(server.base_url()) + .arg("-x") + .arg(format!("@{}", filename.to_string_lossy())) + .assert() + .success(); +} + fn zlib_decode(bytes: Vec) -> std::io::Result { let mut z = flate2::read::ZlibDecoder::new(&bytes[..]); let mut s = String::new(); From 1ec0267a189cac554a6651615fb7fd6839b916cc Mon Sep 17 00:00:00 2001 From: zuisong Date: Thu, 23 Jan 2025 09:13:19 +0800 Subject: [PATCH 5/6] apply suggestion --- src/main.rs | 25 ++++++++++--------------- tests/cli.rs | 28 +++++++++++++++++++++++----- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/src/main.rs b/src/main.rs index e0b26dd7..bbd26ee7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -511,24 +511,19 @@ fn run(args: Cli) -> Result { let mut request = request_builder.headers(headers).build()?; if args.compress >= 1 && request.headers().get(CONTENT_ENCODING).is_none() { - let mut compressed = false; if let Some(body) = request.body_mut() { - if let Ok(body_bytes) = body.buffer() { - let mut encoder = ZlibEncoder::new(Vec::new(), Default::default()); - encoder.write_all(body_bytes)?; - let output = encoder.finish()?; - if output.len() < body_bytes.len() || args.compress >= 2 { - let _ = std::mem::replace(body, ReqwestBody::from(output)); - compressed = true; - } + let body_bytes = body.buffer()?; + let mut encoder = ZlibEncoder::new(Vec::new(), Default::default()); + encoder.write_all(body_bytes)?; + let output = encoder.finish()?; + if output.len() < body_bytes.len() || args.compress >= 2 { + let _ = std::mem::replace(body, ReqwestBody::from(output)); + request + .headers_mut() + .entry(CONTENT_ENCODING) + .or_insert(HeaderValue::from_static("deflate")); } } - if compressed { - request - .headers_mut() - .entry(CONTENT_ENCODING) - .or_insert(HeaderValue::from_static("deflate")); - } } for header in &headers_to_unset { diff --git a/tests/cli.rs b/tests/cli.rs index 809130c3..7af3eb99 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -785,15 +785,33 @@ fn successful_digest_auth() { fn compress_request_body_online() { get_command() .arg("https://postman-echo.com/post") - .args(["-xx", "--body", &format!("a={}", "1".repeat(1000))]) + .args(["--body", "-f", &format!("a={}", "1".repeat(1000))]) .assert() .stdout(function(|body: &str| { let json: Value = serde_json::from_str(body).unwrap(); assert_eq!(json["json"]["a"], Value::String("1".repeat(1000))); - if let Some(request_body_length) = json["headers"]["content-length"].as_str() { - let length: i32 = request_body_length.parse().unwrap(); - assert!(length < 1000) - } + let length: i32 = json["headers"]["content-length"] + .as_str() + .unwrap() + .parse() + .unwrap(); + assert_eq!(length, 1002); + + true + })); + get_command() + .arg("https://postman-echo.com/post") + .args(["-x", "--body", "-f", &format!("a={}", "1".repeat(1000))]) + .assert() + .stdout(function(|body: &str| { + let json: Value = serde_json::from_str(body).unwrap(); + assert_eq!(json["json"]["a"], Value::String("1".repeat(1000))); + let length: i32 = json["headers"]["content-length"] + .as_str() + .unwrap() + .parse() + .unwrap(); + assert!(length < 1000); true })); From c4bbdba60e9dce25d4d29161f29e222d18f4ed3f Mon Sep 17 00:00:00 2001 From: zuisong Date: Thu, 23 Jan 2025 10:51:53 +0800 Subject: [PATCH 6/6] update help doc --- doc/xh.1 | 4 +++- src/cli.rs | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/xh.1 b/doc/xh.1 index 444cb9d8..fa225260 100644 --- a/doc/xh.1 +++ b/doc/xh.1 @@ -1,4 +1,4 @@ -.TH XH 1 2025-01-22 0.23.1 "User Commands" +.TH XH 1 2025-01-23 0.23.1 "User Commands" .SH NAME xh \- Friendly and fast tool for sending HTTP requests @@ -190,6 +190,8 @@ Always stream the response body. .TP 4 \fB\-x\fR, \fB\-\-compress\fR Content compressed (encoded) with Deflate algorithm. The Content\-Encoding header is set to deflate. + +Compression is skipped if it appears that compression ratio is negative. Compression can be forced by repeating this option. .TP 4 \fB\-o\fR, \fB\-\-output\fR=\fIFILE\fR Save output to FILE instead of stdout. diff --git a/src/cli.rs b/src/cli.rs index 1c2bf554..cd60b553 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -184,6 +184,9 @@ Example: --print=Hb" /// Content compressed (encoded) with Deflate algorithm. /// The Content-Encoding header is set to deflate. + /// + /// Compression is skipped if it appears that compression ratio is negative. + /// Compression can be forced by repeating this option. #[clap(short = 'x', long = "compress", name = "compress", action = ArgAction::Count)] pub compress: u8,