diff --git a/lib/curl_req.ex b/lib/curl_req.ex index 077a318..36e217e 100644 --- a/lib/curl_req.ex +++ b/lib/curl_req.ex @@ -121,153 +121,20 @@ defmodule CurlReq do ] @spec to_curl(Req.Request.t(), to_curl_opts()) :: String.t() def to_curl(req, options \\ []) do - opts = Keyword.validate!(options, flags: :short, run_steps: true, flavor: nil, flavour: :curl) - flavor = opts[:flavor] || opts[:flavour] - flag_style = opts[:flags] - run_steps = opts[:run_steps] - - available_steps = step_names(req, run_steps) - req = run_steps(req, available_steps) - - cookies = - case Map.get(req.headers, "cookie") do - nil -> [] - [cookies] -> [cookie_flag(flag_style), cookies] - end - - headers = - req.headers - |> Enum.reject(fn {key, _val} -> key == "cookie" end) - |> Enum.flat_map(&map_header(&1, flag_style, flavor)) - - body = - case req.body do - nil -> [] - body -> [data_flag(flag_style), body] - end - options = - case req.options do - %{redirect: true} -> - [location_flag(flag_style)] - - # avoids duplicate compression argument - %{compressed: true} -> - if :compressed in available_steps, do: [], else: [compressed_flag()] - - %{connect_options: connect_options} -> - proxy = - case Keyword.get(connect_options, :proxy) do - nil -> - [] - - {scheme, host, port, _} -> - [proxy_flag(flag_style), "#{scheme}://#{host}:#{port}"] - end - - case Keyword.get(connect_options, :proxy_headers) do - [{"proxy-authorization", "Basic " <> encoded_creds}] -> - proxy ++ [proxy_user_flag(flag_style), Base.decode64!(encoded_creds)] - - _ -> - proxy - end - - _ -> - [] - end + Keyword.validate!(options, flags: :short, run_steps: true, flavor: nil, flavour: :curl) - auth = - with %{auth: scheme} <- req.options do - case scheme do - {:bearer, token} -> - [header_flag(flag_style), "authorization: Bearer #{token}"] + # flavor = opts[:flavor] || opts[:flavour] + # flag_style = opts[:flags] + run_steps = options[:run_steps] - {:basic, userinfo} -> - [user_flag(flag_style), userinfo] ++ [basic_auth_flag()] - - :netrc -> - [netrc_flag(flag_style)] - - {:netrc, filepath} -> - [netrc_file_flag(flag_style), filepath] - - _ -> - [] - end - else - _ -> - [] - end - - method = - case req.method do - nil -> [request_flag(flag_style), "GET"] - :head -> [head_flag(flag_style)] - m -> [request_flag(flag_style), String.upcase(to_string(m))] - end - - url = [to_string(req.url)] - - CurlReq.Shell.cmd_to_string( - "curl", - auth ++ headers ++ cookies ++ body ++ options ++ method ++ url - ) - end + available_steps = step_names(req, run_steps) + req = run_steps(req, available_steps) - @typep header :: {String.t(), list(String.t())} - @spec map_header(header(), flags(), flavor()) :: list() - defp map_header({"accept-encoding", [compression]}, _flag_style, :curl) - when compression in ["gzip", "br", "zstd"] do - [compressed_flag()] + CurlReq.Req.decode(req) + |> CurlReq.Curl.encode(options) end - # filter out auth header because we expect it to be set as an auth step option - defp map_header({"authorization", _}, _flag_style, :curl), - do: [] - - # filter out user agent when mode is :curl - defp map_header({"user-agent", ["req/" <> _]}, _, :curl), do: [] - - defp map_header({key, value}, flag_style, _), - do: [header_flag(flag_style), "#{key}: #{value}"] - - defp cookie_flag(:short), do: "-b" - defp cookie_flag(:long), do: "--cookie" - - defp header_flag(:short), do: "-H" - defp header_flag(:long), do: "--header" - - defp data_flag(:short), do: "-d" - defp data_flag(:long), do: "--data" - - defp head_flag(:short), do: "-I" - defp head_flag(:long), do: "--head" - - defp request_flag(:short), do: "-X" - defp request_flag(:long), do: "--request" - - defp location_flag(:short), do: "-L" - defp location_flag(:long), do: "--location" - - defp user_flag(:short), do: "-u" - defp user_flag(:long), do: "--user" - - defp basic_auth_flag(), do: "--basic" - - defp compressed_flag(), do: "--compressed" - - defp proxy_flag(:short), do: "-x" - defp proxy_flag(:long), do: "--proxy" - - defp proxy_user_flag(:short), do: "-U" - defp proxy_user_flag(:long), do: "--proxy-user" - - defp netrc_flag(:short), do: "-n" - defp netrc_flag(:long), do: "--netrc" - - defp netrc_file_flag(_), do: "--netrc-file" - @doc """ Transforms a curl command into a Req request. @@ -308,7 +175,11 @@ defmodule CurlReq do @doc since: "0.98.4" @spec from_curl(String.t()) :: Req.Request.t() - def from_curl(curl_command), do: CurlReq.Macro.parse(curl_command) + def from_curl(curl_command) do + curl_command + |> CurlReq.Curl.decode() + |> CurlReq.Req.encode() + end @doc """ Same as `from_curl/1` but as a sigil. The benefit here is, that the Req.Request struct will be created at compile time and you don't need to escape the string @@ -335,7 +206,7 @@ defmodule CurlReq do defmacro sigil_CURL({:<<>>, _line_info, [command]}, _extra) do command - |> CurlReq.Macro.parse() + |> from_curl() |> Macro.escape() end end diff --git a/lib/curl_req/curl.ex b/lib/curl_req/curl.ex new file mode 100644 index 0000000..c62f0f7 --- /dev/null +++ b/lib/curl_req/curl.ex @@ -0,0 +1,400 @@ +defmodule CurlReq.Curl do + # TODO: docs + @behaviour CurlReq.Request + + @impl CurlReq.Request + @spec decode(String.t()) :: CurlReq.Request.t() + def decode(command, _opts \\ []) when is_binary(command) do + command = + command + |> String.trim() + |> String.trim_leading("curl") + |> String.replace("\\\n", " ") + |> String.replace("\n", " ") + + {options, rest, invalid} = + command + |> OptionParser.split() + |> OptionParser.parse( + strict: [ + header: :keep, + request: :string, + data: :keep, + data_raw: :keep, + data_ascii: :keep, + cookie: :string, + head: :boolean, + form: :keep, + location: :boolean, + user: :string, + compressed: :boolean, + proxy: :string, + proxy_user: :string, + netrc: :boolean, + netrc_file: :string + ], + aliases: [ + H: :header, + X: :request, + d: :data, + b: :cookie, + I: :head, + F: :form, + L: :location, + u: :user, + x: :proxy, + U: :proxy_user, + n: :netrc + ] + ) + + if invalid != [] do + errors = + Enum.map(invalid, fn + {flag, nil} -> "Unknown #{inspect(flag)}" + {flag, value} -> "Invalid value #{inspect(value)} for #{inspect(flag)}" + end) + |> Enum.join("\n") + + raise ArgumentError, """ + + Command: \'curl #{command}\" + Unsupported or invalid flag(s) encountered: + + #{errors} + + Please remove the unknown flags and open an issue at https://github.com/derekkraan/curl_req + """ + end + + [url] = + rest + |> List.flatten() + + url = URI.parse(url) + + %CurlReq.Request{} + |> CurlReq.Request.put_url(url) + |> add_header(options) + |> add_method(options) + |> add_body(options) + |> add_cookie(options) + |> add_form(options) + |> add_auth(options) + |> add_compression(options) + |> add_proxy(options) + |> configure_redirects(options) + end + + defp add_header(request, options) do + headers = Keyword.get_values(options, :header) + + Enum.reduce(headers, request, fn header, acc -> + [key, value] = + header + |> String.split(":", parts: 2) + |> Enum.map(&String.trim/1) + + CurlReq.Request.put_header(acc, key, value) + end) + end + + defp add_method(request, options) do + method = + if Keyword.get(options, :head, false) do + :head + else + Keyword.get(options, :request, "GET") + end + + CurlReq.Request.put_method(request, method) + end + + defp add_body(request, options) do + body = + Enum.flat_map([:data, :data_ascii, :data_raw], fn key -> + case Keyword.get_values(options, key) do + [] -> [] + values -> Enum.map(values, &String.trim_leading(&1, "$")) + end + end) + |> Enum.join("&") + + if body != "" do + CurlReq.Request.put_body(request, body) + else + request + end + end + + defp add_cookie(request, options) do + case Keyword.get(options, :cookie) do + nil -> + request + + cookie -> + String.split(cookie, ";") + |> Enum.reduce(request, fn cookie, acc -> + [key, value] = + String.split(cookie, "=", parts: 2) + |> Enum.map(&String.trim/1) + + CurlReq.Request.put_cookie(acc, key, value) + end) + end + end + + defp add_form(request, options) do + case Keyword.get_values(options, :form) do + [] -> + request + + formdata -> + form = + for fd <- formdata, reduce: %{} do + map -> + [key, value] = String.split(fd, "=", parts: 2) + Map.put(map, key, value) + end + + request + |> CurlReq.Request.put_body(form) + |> CurlReq.Request.put_encoding(:form) + end + end + + defp add_auth(request, options) do + request = + case Keyword.get(options, :user) do + nil -> + request + + userinfo -> + CurlReq.Request.put_auth(request, :basic, userinfo) + end + + request = + case Keyword.get(options, :netrc) do + nil -> + request + + _ -> + CurlReq.Request.put_auth(request, :netrc) + end + + case Keyword.get(options, :netrc_file) do + nil -> + request + + path -> + CurlReq.Request.put_auth(request, :netrc, path) + end + end + + defp add_compression(request, options) do + case Keyword.get(options, :compressed) do + nil -> + request + + bool -> + CurlReq.Request.put_compression(request, bool) + end + end + + defp add_proxy(request, options) do + proxy = Keyword.get(options, :proxy) + proxy_user = Keyword.get(options, :proxy_user) + + case {proxy, proxy_user} do + {nil, _} -> + request + + {proxy, nil} -> + proxy = validate_proxy_uri(proxy) + CurlReq.Request.put_proxy(request, proxy) + + {proxy, proxy_user} -> + proxy = validate_proxy_uri(proxy) + proxy = %{proxy | userinfo: proxy_user} + CurlReq.Request.put_proxy(request, proxy) + end + end + + defp validate_proxy_uri("http://" <> _rest = uri), do: URI.parse(uri) + defp validate_proxy_uri("https://" <> _rest = uri), do: URI.parse(uri) + + defp validate_proxy_uri(uri) do + case String.split(uri, "://") do + [scheme, _uri] -> + raise ArgumentError, "Unsupported scheme #{scheme} for proxy in #{uri}" + + [uri] -> + URI.parse("http://" <> uri) + end + end + + defp configure_redirects(request, options) do + case Keyword.get(options, :location) do + nil -> request + bool -> CurlReq.Request.put_redirect(request, bool) + end + end + + @impl CurlReq.Request + @spec encode(CurlReq.Request.t(), Keyword.t()) :: String.t() + def encode(%CurlReq.Request{} = request, options \\ []) do + flag_style = Keyword.get(options, :flags, :short) + # TODO: implement flavor + _flavor = Keyword.get(options, :flavor, nil) || Keyword.get(options, :flavour, :curl) + + cookies = + if map_size(request.cookies) != 0 do + request.cookies + |> Enum.map(fn {key, val} -> "#{key}=#{val}" end) + |> Enum.join(";") + else + [] + end + + cookies = emit_if(cookies != [], [cookie_flag(flag_style, cookies)]) + + headers = + for {key, values} <- request.headers, reduce: [] do + headers -> + [headers, header_flag(flag_style, [key, ": ", Enum.intersperse(values, ";")])] + end + + headers = + case request.encoding do + :raw -> + headers + + :json -> + headers ++ [header_flag(flag_style, "content-type: application/json")] + + :form -> + headers ++ [header_flag(flag_style, "content-type: application/x-www-form-urlencoded")] + end + + headers = Enum.intersperse(headers, " ") + + body = + emit_if(request.body, fn -> + case request.encoding do + :json -> [data_flag(flag_style, Jason.encode!(request.body))] + _ -> [data_flag(flag_style, request.body)] + end + end) + + redirect = emit_if(request.redirect, [location_flag(flag_style)]) + compressed = emit_if(request.compression != :none, [compressed_flag(flag_style)]) + + auth = + case request.auth do + :none -> + [] + + {:basic, userinfo} -> + user_flag(flag_style, userinfo) + + {:bearer, token} -> + [header_flag(flag_style, ["authorization: Bearer ", token])] + + :netrc -> + [netrc_flag(flag_style)] + + {:netrc, filepath} -> + [netrc_file_flag(flag_style, filepath)] + end + + method = + case request.method do + :head -> [head_flag(flag_style)] + m -> [request_flag(flag_style, String.upcase(to_string(m)))] + end + + proxy = + if request.proxy do + proxy_flag(flag_style, URI.to_string(request.proxy_url)) + else + [] + end + + proxy_auth = + case request.proxy_auth do + :none -> [] + {:basic, userinfo} -> proxy_user_flag(flag_style, userinfo) + _ -> [] + end + + url = [to_string(request.url)] + + IO.iodata_to_binary( + [ + "curl", + compressed, + auth, + headers, + cookies, + body, + proxy, + proxy_auth, + redirect, + method, + url + ] + |> Enum.reject(fn part -> part == [] end) + |> Enum.intersperse(" ") + ) + end + + defp emit_if(bool, fun) when is_function(fun) do + if bool, do: fun.(), else: [] + end + + defp emit_if(bool, value) do + if bool, do: value, else: [] + end + + defp escape(value) when is_list(value) do + IO.iodata_to_binary(value) |> escape() + end + + defp escape(value) when is_binary(value) do + CurlReq.Shell.escape(value) + end + + defp cookie_flag(:short, value), do: ["-b ", escape(value)] + defp cookie_flag(:long, value), do: ["--cookie ", escape(value)] + + defp header_flag(:short, value), do: ["-H ", escape(value)] + defp header_flag(:long, value), do: ["--header ", escape(value)] + + defp data_flag(:short, value), do: ["-d ", escape(value)] + defp data_flag(:long, value), do: ["--data ", escape(value)] + + defp head_flag(:short), do: "-I" + defp head_flag(:long), do: "--head" + + defp request_flag(:short, value), do: ["-X ", escape(value)] + defp request_flag(:long, value), do: ["--request ", escape(value)] + + defp location_flag(:short), do: "-L" + defp location_flag(:long), do: "--location" + + defp user_flag(:short, value), do: ["-u ", escape(value)] + defp user_flag(:long, value), do: ["--user ", escape(value)] + + defp netrc_flag(:short), do: "-n" + defp netrc_flag(:long), do: "--netrc" + + defp netrc_file_flag(_, value), do: ["--netrc-file ", escape(value)] + + defp compressed_flag(_), do: "--compressed" + + defp proxy_flag(:short, value), do: ["-x ", escape(value)] + defp proxy_flag(:long, value), do: ["--proxy ", escape(value)] + + defp proxy_user_flag(:short, value), do: ["-U ", escape(value)] + defp proxy_user_flag(:long, value), do: ["--proxy-user ", escape(value)] +end diff --git a/lib/curl_req/req.ex b/lib/curl_req/req.ex new file mode 100644 index 0000000..6ea5582 --- /dev/null +++ b/lib/curl_req/req.ex @@ -0,0 +1,182 @@ +defmodule CurlReq.Req do + # TODO: docs + @behaviour CurlReq.Request + + @impl CurlReq.Request + @spec decode(Req.Request.t()) :: CurlReq.Request.t() + def decode(%Req.Request{} = req, _opts \\ []) do + request = + %CurlReq.Request{} + |> put_header(req) + |> CurlReq.Request.put_auth(req.options[:auth]) + |> CurlReq.Request.put_redirect(req.options[:redirect]) + |> CurlReq.Request.put_compression(req.options[:compressed]) + |> CurlReq.Request.put_user_agent(:req) + |> CurlReq.Request.put_body(req.body) + |> CurlReq.Request.put_url(req.url) + |> CurlReq.Request.put_method(req.method) + + request = + case req.options[:connect_options] do + nil -> + request + + connect_options -> + userinfo = + case Keyword.get(connect_options, :proxy_headers) do + [{"proxy-authorization", "Basic " <> encoded_userinfo}] -> + case Base.decode64(encoded_userinfo) do + {:ok, userinfo} -> userinfo + _ -> encoded_userinfo + end + + _ -> + nil + end + + case Keyword.get(connect_options, :proxy) do + {scheme, host, port, _} -> + CurlReq.Request.put_proxy(request, %URI{ + scheme: Atom.to_string(scheme), + host: host, + port: port, + userinfo: userinfo + }) + + _ -> + request + end + end + + request + end + + defp put_header(%CurlReq.Request{} = request, %Req.Request{} = req) do + for {key, val} <- req.headers, reduce: request do + request -> CurlReq.Request.put_header(request, key, val) + end + end + + @impl CurlReq.Request + @spec encode(CurlReq.Request.t()) :: Req.Request.t() + def encode(%CurlReq.Request{} = request, _opts \\ []) do + req = + %Req.Request{} + |> Req.merge(url: request.url) + |> Req.merge(method: request.method) + + cookies = + request.cookies + |> Enum.map(fn {key, val} -> "#{key}=#{val}" end) + |> Enum.join(";") + + req = + case request.user_agent do + :req -> req + :curl -> req + other -> Req.Request.put_header(req, "user-agent", other) + end + + req = + case request.encoding do + :raw -> + Req.merge(req, body: request.body) + + :form -> + req + |> Req.Request.register_options([:form]) + |> Req.Request.prepend_request_steps(encode_body: &Req.Steps.encode_body/1) + |> Req.merge(form: request.body) + + :json -> + req + |> Req.Request.register_options([:json]) + |> Req.Request.prepend_request_steps(encode_body: &Req.Steps.encode_body/1) + |> Req.merge(json: request.body) + end + + req = + case request.auth do + :none -> + req + + auth -> + req + |> Req.Request.register_options([:auth]) + |> Req.Request.prepend_request_steps(auth: &Req.Steps.auth/1) + |> Req.merge(auth: auth) + end + + req = + case request.compression do + :none -> + req + + _ -> + req + |> Req.Request.register_options([:compressed]) + |> Req.Request.prepend_request_steps(compressed: &Req.Steps.compressed/1) + |> Req.merge(compressed: true) + end + + req = + case request.redirect do + false -> + req + + _ -> + req + |> Req.Request.register_options([:redirect]) + |> Req.Request.prepend_response_steps(redirect: &Req.Steps.redirect/1) + |> Req.merge(redirect: true) + end + + req = + for {key, values} <- request.headers, reduce: req do + req -> Req.Request.put_header(req, key, values) + end + + req = + if cookies != "" do + Req.Request.put_header(req, "cookie", cookies) + else + req + end + + req = + if request.proxy do + %URI{scheme: scheme, host: host, port: port} = request.proxy_url + + connect_options = + [ + proxy: {String.to_existing_atom(scheme), host, port, []} + ] + + connect_options = + case request.proxy_auth do + :none -> + connect_options + + {:basic, userinfo} -> + Keyword.merge(connect_options, + proxy_headers: [ + {"proxy-authorization", "Basic " <> Base.encode64(userinfo)} + ] + ) + + _ -> + connect_options + end + + req + |> Req.Request.register_options([ + :connect_options + ]) + |> Req.merge(connect_options: connect_options) + else + req + end + + req + end +end diff --git a/lib/curl_req/request.ex b/lib/curl_req/request.ex index d8ebec5..3c0ff93 100644 --- a/lib/curl_req/request.ex +++ b/lib/curl_req/request.ex @@ -287,7 +287,7 @@ defmodule CurlReq.Request do iex> request = %CurlReq.Request{} |> CurlReq.Request.put_proxy("https://example.com") iex> request.url - URI.parse("https://example.com") + URI.new!("https://example.com") """ @spec put_proxy(__MODULE__.t(), URI.t() | String.t()) :: __MODULE__.t() def put_proxy(%__MODULE__{} = request, uri) do diff --git a/lib/curl_req/shell.ex b/lib/curl_req/shell.ex index 906348f..7a9a056 100644 --- a/lib/curl_req/shell.ex +++ b/lib/curl_req/shell.ex @@ -12,20 +12,7 @@ defmodule CurlReq.Shell do {~S(~), ~S(\~)} ] - @doc """ - This function takes the same arguments as `System.cmd/3`, but returns - the command in string form instead of running the command. - """ @no_quotes ~r/^[a-zA-Z-,._+:@%\/]*$/ - def cmd_to_string(cmd, args) do - final_args = - args - |> Enum.map(&IO.iodata_to_binary/1) - |> Enum.map(&escape/1) - |> Enum.join(" ") - - "#{cmd} #{final_args}" |> String.trim_trailing() - end @doc ~S""" Examples: diff --git a/mix.exs b/mix.exs index c26e262..a7db916 100644 --- a/mix.exs +++ b/mix.exs @@ -26,6 +26,7 @@ defmodule CurlReq.MixProject do defp deps do [ {:req, "~> 0.4.0 or ~> 0.5.0"}, + {:jason, "~> 1.4"}, {:ex_doc, ">= 0.0.0", only: :dev}, {:blend, "~> 0.4.1", only: :dev} ] diff --git a/test/curl_req/macro_test.exs b/test/curl_req/macro_test.exs deleted file mode 100644 index 59b5d35..0000000 --- a/test/curl_req/macro_test.exs +++ /dev/null @@ -1,377 +0,0 @@ -defmodule CurlReq.MacroTest do - use ExUnit.Case, async: true - - import CurlReq - - describe "macro" do - test "single header" do - assert ~CURL(curl -H "user-agent: req/0.4.14" -X GET https://example.com/fact) == - %Req.Request{ - method: :get, - headers: %{"user-agent" => ["req/0.4.14"]}, - url: URI.parse("https://example.com/fact") - } - end - - test "post method" do - assert ~CURL(curl -X POST https://example.com) == - %Req.Request{ - method: :post, - url: URI.parse("https://example.com") - } - end - - test "head method" do - assert ~CURL(curl -I https://example.com) == - %Req.Request{ - method: :head, - url: URI.parse("https://example.com") - } - end - - test "multiple headers with body" do - assert ~CURL(curl -H "accept-encoding: gzip" -H "authorization: Bearer 6e8f18e6-141b-4d12-8397-7e7791d92ed4:lon" -H "content-type: application/json" -H "user-agent: req/0.4.14" -d "{\"input\":[{\"leadFormFields\":{\"Company\":\"k\",\"Country\":\"DZ\",\"Email\":\"k\",\"FirstName\":\"k\",\"Industry\":\"CTO\",\"LastName\":\"k\",\"Phone\":\"k\",\"PostalCode\":\"1234ZZ\",\"jobspecialty\":\"engineer\",\"message\":\"I would like to know if Roche delivers to The Netherlands.\"}}],\"formId\":4318}" -X POST "https://example.com/rest/v1/leads/submitForm.json") == - %Req.Request{ - method: :post, - url: URI.parse("https://example.com/rest/v1/leads/submitForm.json"), - headers: %{ - "accept-encoding" => ["gzip"], - "content-type" => ["application/json"], - "user-agent" => ["req/0.4.14"] - }, - registered_options: MapSet.new([:auth]), - options: %{auth: {:bearer, "6e8f18e6-141b-4d12-8397-7e7791d92ed4:lon"}}, - current_request_steps: [:auth], - request_steps: [auth: &Req.Steps.auth/1], - body: - "{\"input\":[{\"leadFormFields\":{\"Company\":\"k\",\"Country\":\"DZ\",\"Email\":\"k\",\"FirstName\":\"k\",\"Industry\":\"CTO\",\"LastName\":\"k\",\"Phone\":\"k\",\"PostalCode\":\"1234ZZ\",\"jobspecialty\":\"engineer\",\"message\":\"I would like to know if Roche delivers to The Netherlands.\"}}],\"formId\":4318}" - } - end - - test "without curl prefix" do - assert ~CURL(http://example.com) == - %Req.Request{ - method: :get, - url: URI.parse("http://example.com") - } - end - - test "multiple data flags" do - assert ~CURL(curl http://example.com -d name=foo -d mail=bar) == - %Req.Request{ - url: URI.parse("http://example.com"), - body: "name=foo&mail=bar" - } - end - - test "cookie" do - assert ~CURL(http://example.com -b "name1=value1") == - %Req.Request{ - url: URI.parse("http://example.com"), - headers: %{"cookie" => ["name1=value1"]} - } - - assert ~CURL(http://example.com -b "name1=value1; name2=value2") == - %Req.Request{ - url: URI.parse("http://example.com"), - headers: %{"cookie" => ["name1=value1; name2=value2"]} - } - end - - test "formdata" do - assert ~CURL(curl http://example.com -F name=foo -F mail=bar) == - %Req.Request{ - url: URI.parse("http://example.com"), - body: nil, - registered_options: MapSet.new([:form]), - options: %{form: %{"name" => "foo", "mail" => "bar"}}, - current_request_steps: [:encode_body], - request_steps: [encode_body: &Req.Steps.encode_body/1] - } - end - - test "data raw" do - assert ~CURL""" - curl 'https://example.com/graphql' \ - -X POST \ - -H 'Accept: application/graphql-response+json'\ - --data-raw '{"operationName":"get","query":"query get {name}"}' - """ == - %Req.Request{ - method: :post, - url: URI.parse("https://example.com/graphql"), - headers: %{"accept" => ["application/graphql-response+json"]}, - body: "{\"operationName\":\"get\",\"query\":\"query get {name}\"}", - options: %{}, - halted: false, - adapter: &Req.Steps.run_finch/1, - request_steps: [], - response_steps: [], - error_steps: [], - private: %{} - } - end - - test "data raw with ansii escape" do - assert ~CURL""" - curl 'https://example.com/employees/107'\ - -X PATCH\ - -H 'Accept: application/vnd.api+json'\ - --data-raw $'{"data":{"attributes":{"first-name":"Adam"}}}' - """ == - %Req.Request{ - method: :patch, - url: URI.parse("https://example.com/employees/107"), - headers: %{"accept" => ["application/vnd.api+json"]}, - body: "{\"data\":{\"attributes\":{\"first-name\":\"Adam\"}}}", - options: %{}, - halted: false, - adapter: &Req.Steps.run_finch/1, - request_steps: [], - response_steps: [], - error_steps: [], - private: %{} - } - end - - test "basic auth" do - assert ~CURL(curl http://example.com -u user:pass) == - %Req.Request{ - url: URI.parse("http://example.com"), - body: nil, - registered_options: MapSet.new([:auth]), - options: %{auth: {:basic, "user:pass"}}, - current_request_steps: [:auth], - request_steps: [auth: &Req.Steps.auth/1] - } - end - - test "bearer token auth" do - curl = ~CURL""" - curl -L \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer " \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://example.com/users - """ - - assert curl == - %Req.Request{ - url: URI.parse("https://example.com/users"), - body: nil, - headers: %{ - "accept" => ["application/vnd.github+json"], - "x-github-api-version" => ["2022-11-28"] - }, - registered_options: MapSet.new([:auth, :redirect]), - options: %{auth: {:bearer, ""}, redirect: true}, - current_request_steps: [:auth], - request_steps: [auth: &Req.Steps.auth/1], - response_steps: [redirect: &Req.Steps.redirect/1] - } - end - - test "netrc auth" do - assert ~CURL(curl http://example.com -n) == - %Req.Request{ - url: URI.parse("http://example.com"), - body: nil, - registered_options: MapSet.new([:auth]), - options: %{auth: :netrc}, - current_request_steps: [:auth], - request_steps: [auth: &Req.Steps.auth/1] - } - end - - test "netrc file auth" do - assert ~CURL(curl http://example.com --netrc-file "./mynetrc") == - %Req.Request{ - url: URI.parse("http://example.com"), - body: nil, - registered_options: MapSet.new([:auth]), - options: %{auth: {:netrc, "./mynetrc"}}, - current_request_steps: [:auth], - request_steps: [auth: &Req.Steps.auth/1] - } - end - - test "compressed" do - assert ~CURL(curl --compressed http://example.com) == - %Req.Request{ - url: URI.parse("http://example.com"), - body: nil, - registered_options: MapSet.new([:compressed]), - options: %{compressed: true}, - current_request_steps: [:compressed], - request_steps: [compressed: &Req.Steps.compressed/1] - } - end - - test "redirect" do - assert ~CURL(curl -L http://example.com) == - %Req.Request{ - url: URI.parse("http://example.com"), - registered_options: MapSet.new([:redirect]), - options: %{redirect: true}, - response_steps: [redirect: &Req.Steps.redirect/1] - } - end - - test "cookie, formadata, auth and redirect" do - assert ~CURL(curl -L -u user:pass -F name=foo -b name=bar http://example.com) == - %Req.Request{ - url: URI.parse("http://example.com"), - headers: %{"cookie" => ["name=bar"]}, - current_request_steps: [:auth, :encode_body], - registered_options: MapSet.new([:redirect, :auth, :form]), - options: %{redirect: true, auth: {:basic, "user:pass"}, form: %{"name" => "foo"}}, - request_steps: [auth: &Req.Steps.auth/1, encode_body: &Req.Steps.encode_body/1], - response_steps: [redirect: &Req.Steps.redirect/1] - } - end - - test "proxy" do - assert ~CURL(curl --proxy my.proxy.com:22225 http://example.com) == - %Req.Request{ - url: URI.parse("http://example.com"), - registered_options: MapSet.new([:connect_options]), - options: %{ - connect_options: [proxy: {:http, "my.proxy.com", 22225, []}] - } - } - end - - test "proxy with basic auth" do - assert ~CURL(curl --proxy https://my.proxy.com:22225 --proxy-user foo:bar http://example.com) == - %Req.Request{ - url: URI.parse("http://example.com"), - registered_options: MapSet.new([:connect_options]), - options: %{ - connect_options: [ - proxy: {:https, "my.proxy.com", 22225, []}, - proxy_headers: [ - {"proxy-authorization", "Basic " <> Base.encode64("foo:bar")} - ] - ] - } - } - end - - test "proxy with inline basic auth" do - assert ~CURL(curl --proxy https://foo:bar@my.proxy.com:22225 http://example.com) == - %Req.Request{ - url: URI.parse("http://example.com"), - registered_options: MapSet.new([:connect_options]), - options: %{ - connect_options: [ - proxy: {:https, "my.proxy.com", 22225, []}, - proxy_headers: [ - {"proxy-authorization", "Basic " <> Base.encode64("foo:bar")} - ] - ] - } - } - end - - test "proxy raises on non http scheme uri" do - assert_raise( - ArgumentError, - "Unsupported scheme ssh for proxy in ssh://my.proxy.com:22225", - fn -> - CurlReq.Macro.parse("curl --proxy ssh://my.proxy.com:22225 http://example.com") - end - ) - end - end - - describe "newlines" do - test "sigil_CURL supports newlines" do - curl = ~CURL""" - curl -X POST \ - --location \ - https://example.com - """ - - assert curl == %Req.Request{ - method: :post, - url: URI.parse("https://example.com"), - registered_options: MapSet.new([:redirect]), - options: %{redirect: true}, - response_steps: [redirect: &Req.Steps.redirect/1] - } - end - - test "from_curl supports newlines" do - curl = - from_curl(""" - curl -X POST \ - --location \ - https://example.com - """) - - assert curl == %Req.Request{ - method: :post, - url: URI.parse("https://example.com"), - registered_options: MapSet.new([:redirect]), - options: %{redirect: true}, - response_steps: [redirect: &Req.Steps.redirect/1] - } - end - - test "accepts newlines ending in backslash" do - uri = URI.parse("https://example.com/api/2024-07/graphql.json") - - assert %Req.Request{ - method: :post, - url: ^uri, - headers: %{"content-type" => ["application/json"]} - } = ~CURL""" - curl -X POST \ - https://example.com/api/2024-07/graphql.json \ - -H 'Content-Type: application/json' \ - -H 'X-Shopify-Storefront-Access-Token: ABCDEF' \ - -d '{ - "query": "{ - products(first: 3) { - edges { - node { - id - title - } - } - } - }" - }' - """ - - assert %Req.Request{ - method: :post, - url: ^uri, - headers: %{"content-type" => ["application/json"]} - } = ~CURL""" - curl -X POST - https://example.com/api/2024-07/graphql.json - -H 'Content-Type: application/json' - -H 'X-Shopify-Storefront-Access-Token: ABCDEF' - -d '{ - "query": "{ - products(first: 3) { - edges { - node { - id - title - } - } - } - }" - }' - """ - end - - test "raises on unsupported flag" do - assert_raise ArgumentError, ~r/Unknown "--foo"/, fn -> - CurlReq.Macro.parse(~s(curl --foo https://example.com)) - end - end - end -end diff --git a/test/curl_req_test.exs b/test/curl_req_test.exs index bb1ce1a..f2612f2 100644 --- a/test/curl_req_test.exs +++ b/test/curl_req_test.exs @@ -1,6 +1,8 @@ defmodule CurlReqTest do use ExUnit.Case, async: true + doctest CurlReq + import CurlReq import ExUnit.CaptureIO @@ -13,7 +15,7 @@ defmodule CurlReqTest do end test "with label" do - assert capture_io(fn -> + assert capture_io(:stdio, fn -> Req.new(url: "/with_label", base_url: "https://example.com/") |> CurlReq.inspect(label: "MY REQ") end) === "MY REQ: curl --compressed -X GET https://example.com/with_label\n" @@ -30,7 +32,7 @@ defmodule CurlReqTest do test "cookies get extracted from header" do assert Req.new(url: "http://example.com", headers: %{"cookie" => ["name1=value1"]}) |> CurlReq.to_curl() == - "curl --compressed -b \"name1=value1\" -X GET http://example.com" + ~s|curl --compressed -b "name1=value1" -X GET http://example.com| end test "redirect flag gets set" do @@ -53,13 +55,13 @@ defmodule CurlReqTest do headers: %{"cookie" => ["name1=value1"], "content-type" => ["application/json"]} ) |> CurlReq.to_curl(flags: :long) == - "curl --compressed --header \"content-type: application/json\" --cookie \"name1=value1\" --location --head http://example.com" + ~S|curl --compressed --header "content-type: application/json" --cookie "name1=value1" --location --head http://example.com| end test "formdata flags get set with correct headers and body" do assert Req.new(url: "http://example.com", form: [key1: "value1", key2: "value2"]) |> CurlReq.to_curl() == - "curl --compressed -H \"content-type: application/x-www-form-urlencoded\" -d \"key1=value1&key2=value2\" -X GET http://example.com" + ~S|curl --compressed -H "content-type: application/x-www-form-urlencoded" -d "key1=value1&key2=value2" -X GET http://example.com| end test "works when body is iodata" do @@ -80,13 +82,13 @@ defmodule CurlReqTest do end test "req flavor with explicit headers" do - assert "curl -H \"accept-encoding: gzip\" -H \"user-agent: req/#{req_version()}\" -X GET https://example.com" == + assert ~s|curl -H "accept-encoding: gzip" -H "user-agent: req/#{req_version()}" -X GET https://example.com| == Req.new(url: "https://example.com") |> CurlReq.to_curl(flavor: :req) end test "proxy" do - assert ~S(curl --compressed -x "http://my.proxy.com:80" -X GET https://example.com) == + assert ~S(curl --compressed -x http://my.proxy.com -X GET https://example.com) == Req.new( url: "https://example.com", connect_options: [proxy: {:http, "my.proxy.com", 80, []}] @@ -95,7 +97,7 @@ defmodule CurlReqTest do end test "proxy user" do - assert ~S(curl --compressed -x "http://my.proxy.com:80" -U foo:bar -X GET https://example.com) == + assert ~S(curl --compressed -x http://my.proxy.com -U foo:bar -X GET https://example.com) == Req.new( url: "https://example.com", connect_options: [ @@ -109,13 +111,13 @@ defmodule CurlReqTest do end test "basic auth option" do - assert "curl -u user:pass --basic --compressed -X GET https://example.com" == + assert "curl --compressed -u user:pass -X GET https://example.com" == Req.new(url: "https://example.com", auth: {:basic, "user:pass"}) |> CurlReq.to_curl() end test "bearer auth option" do - assert ~S(curl -H "authorization: Bearer foo123bar" --compressed -X GET https://example.com) == + assert ~S(curl --compressed -H "authorization: Bearer foo123bar" -X GET https://example.com) == Req.new(url: "https://example.com", auth: {:bearer, "foo123bar"}) |> CurlReq.to_curl() end @@ -133,7 +135,7 @@ defmodule CurlReqTest do File.write(netrc_path, credentials) System.put_env("NETRC", netrc_path) - assert "curl -n --compressed -X GET https://example.com" == + assert "curl --compressed -n -X GET https://example.com" == Req.new(url: "https://example.com", auth: :netrc) |> CurlReq.to_curl() end @@ -150,12 +152,12 @@ defmodule CurlReqTest do netrc_path = Path.join(tmp_dir, "my_netrc") File.write(netrc_path, credentials) - assert ~s(curl --netrc-file "#{netrc_path}" --compressed -X GET https://example.com) == + assert ~s(curl --compressed --netrc-file "#{netrc_path}" -X GET https://example.com) == Req.new(url: "https://example.com", auth: {:netrc, netrc_path}) |> CurlReq.to_curl() end - test "include `encode_body` does not run `comporessed` or other steps" do + test "include `encode_body` does not run `compressed` or other steps" do assert ~S(curl -H "accept: application/json" -H "content-type: application/json" -d "{\"key\":\"val\"}" -X GET https://example.com) == Req.new(url: "https://example.com", json: %{key: "val"}) |> CurlReq.to_curl(run_steps: [only: [:encode_body]]) @@ -167,4 +169,375 @@ defmodule CurlReqTest do |> CurlReq.to_curl(run_steps: [except: [:compressed, :encode_body]]) end end + + describe "from_curl" do + test "single header" do + assert ~CURL(curl -H "user-agent: req/0.4.14" -X GET https://example.com/fact) == + %Req.Request{ + method: :get, + headers: %{"user-agent" => ["req/0.4.14"]}, + url: URI.parse("https://example.com/fact") + } + end + + test "post method" do + assert ~CURL(curl -X POST https://example.com) == + %Req.Request{ + method: :post, + url: URI.parse("https://example.com") + } + end + + test "head method" do + assert ~CURL(curl -I https://example.com) == + %Req.Request{ + method: :head, + url: URI.parse("https://example.com") + } + end + + test "multiple headers with body" do + assert ~CURL(curl -H "accept-encoding: gzip" -H "authorization: Bearer 6e8f18e6-141b-4d12-8397-7e7791d92ed4:lon" -H "content-type: application/json" -H "user-agent: req/0.4.14" -d "{\"input\":[{\"leadFormFields\":{\"Company\":\"k\",\"Country\":\"DZ\",\"Email\":\"k\",\"FirstName\":\"k\",\"Industry\":\"CTO\",\"LastName\":\"k\",\"Phone\":\"k\",\"PostalCode\":\"1234ZZ\",\"jobspecialty\":\"engineer\",\"message\":\"I would like to know if Roche delivers to The Netherlands.\"}}],\"formId\":4318}" -X POST "https://example.com/rest/v1/leads/submitForm.json") == + %Req.Request{ + method: :post, + url: URI.parse("https://example.com/rest/v1/leads/submitForm.json"), + headers: %{ + "user-agent" => ["req/0.4.14"] + }, + registered_options: MapSet.new([:compressed, :auth, :json]), + options: %{ + compressed: true, + auth: {:bearer, "6e8f18e6-141b-4d12-8397-7e7791d92ed4:lon"}, + json: %{ + "formId" => 4318, + "input" => [ + %{ + "leadFormFields" => %{ + "Company" => "k", + "Country" => "DZ", + "Email" => "k", + "FirstName" => "k", + "Industry" => "CTO", + "LastName" => "k", + "Phone" => "k", + "PostalCode" => "1234ZZ", + "jobspecialty" => "engineer", + "message" => + "I would like to know if Roche delivers to The Netherlands." + } + } + ] + } + }, + current_request_steps: [:compressed, :auth, :encode_body], + request_steps: [ + compressed: &Req.Steps.compressed/1, + auth: &Req.Steps.auth/1, + encode_body: &Req.Steps.encode_body/1 + ] + } + end + + test "without curl prefix" do + assert ~CURL(http://example.com) == + %Req.Request{ + method: :get, + url: URI.parse("http://example.com") + } + end + + test "multiple data flags" do + assert ~CURL(curl http://example.com -d name=foo -d mail=bar) == + %Req.Request{ + url: URI.parse("http://example.com"), + body: "name=foo&mail=bar" + } + end + + test "cookie" do + assert ~CURL(http://example.com -b "name1=value1") == + %Req.Request{ + url: URI.parse("http://example.com"), + headers: %{"cookie" => ["name1=value1"]} + } + + assert ~CURL(http://example.com -b "name1=value1; name2=value2") == + %Req.Request{ + url: URI.parse("http://example.com"), + headers: %{"cookie" => ["name1=value1;name2=value2"]} + } + end + + test "formdata" do + assert ~CURL(curl http://example.com -F name=foo -F mail=bar) == + %Req.Request{ + url: URI.parse("http://example.com"), + body: nil, + registered_options: MapSet.new([:form]), + options: %{form: %{"name" => "foo", "mail" => "bar"}}, + current_request_steps: [:encode_body], + request_steps: [encode_body: &Req.Steps.encode_body/1] + } + end + + test "data raw" do + assert ~CURL""" + curl 'https://example.com/graphql' \ + -X POST \ + -H 'Accept: application/graphql-response+json'\ + --data-raw '{"operationName":"get","query":"query get {name}"}' + """ == + %Req.Request{ + method: :post, + url: URI.parse("https://example.com/graphql"), + headers: %{"accept" => ["application/graphql-response+json"]}, + body: "{\"operationName\":\"get\",\"query\":\"query get {name}\"}", + options: %{}, + halted: false, + adapter: &Req.Steps.run_finch/1, + request_steps: [], + response_steps: [], + error_steps: [], + private: %{} + } + end + + test "data raw with ansii escape" do + assert ~CURL""" + curl 'https://example.com/employees/107'\ + -X PATCH\ + -H 'Accept: application/vnd.api+json'\ + --data-raw $'{"data":{"attributes":{"first-name":"Adam"}}}' + """ == + %Req.Request{ + method: :patch, + url: URI.parse("https://example.com/employees/107"), + headers: %{"accept" => ["application/vnd.api+json"]}, + body: "{\"data\":{\"attributes\":{\"first-name\":\"Adam\"}}}", + options: %{}, + halted: false, + adapter: &Req.Steps.run_finch/1, + request_steps: [], + response_steps: [], + error_steps: [], + private: %{} + } + end + + test "auth" do + assert ~CURL(curl http://example.com -u user:pass) == + %Req.Request{ + url: URI.parse("http://example.com"), + body: nil, + registered_options: MapSet.new([:auth]), + options: %{auth: {:basic, "user:pass"}}, + current_request_steps: [:auth], + request_steps: [auth: &Req.Steps.auth/1] + } + end + + test "bearer token auth" do + curl = ~CURL""" + curl -L \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer " \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://example.com/users + """ + + assert curl == + %Req.Request{ + url: URI.parse("https://example.com/users"), + body: nil, + headers: %{ + "accept" => ["application/vnd.github+json"], + "x-github-api-version" => ["2022-11-28"] + }, + registered_options: MapSet.new([:auth, :redirect]), + options: %{auth: {:bearer, ""}, redirect: true}, + current_request_steps: [:auth], + request_steps: [auth: &Req.Steps.auth/1], + response_steps: [redirect: &Req.Steps.redirect/1] + } + end + + test "compressed" do + assert ~CURL(curl --compressed http://example.com) == + %Req.Request{ + url: URI.parse("http://example.com"), + body: nil, + registered_options: MapSet.new([:compressed]), + options: %{compressed: true}, + current_request_steps: [:compressed], + request_steps: [compressed: &Req.Steps.compressed/1] + } + end + + test "redirect" do + assert ~CURL(curl -L http://example.com) == + %Req.Request{ + url: URI.parse("http://example.com"), + registered_options: MapSet.new([:redirect]), + options: %{redirect: true}, + response_steps: [redirect: &Req.Steps.redirect/1] + } + end + + test "cookie, formadata, auth and redirect" do + assert ~CURL(curl -L -u user:pass -F name=foo -b name=bar http://example.com) == + %Req.Request{ + url: URI.parse("http://example.com"), + headers: %{"cookie" => ["name=bar"]}, + current_request_steps: [:auth, :encode_body], + registered_options: MapSet.new([:redirect, :auth, :form]), + options: %{redirect: true, auth: {:basic, "user:pass"}, form: %{"name" => "foo"}}, + request_steps: [auth: &Req.Steps.auth/1, encode_body: &Req.Steps.encode_body/1], + response_steps: [redirect: &Req.Steps.redirect/1] + } + end + + test "proxy" do + assert ~CURL(curl --proxy my.proxy.com:22225 http://example.com) == + %Req.Request{ + url: URI.parse("http://example.com"), + registered_options: MapSet.new([:connect_options]), + options: %{ + connect_options: [proxy: {:http, "my.proxy.com", 22225, []}] + } + } + end + + test "proxy with basic auth" do + assert ~CURL(curl --proxy https://my.proxy.com:22225 --proxy-user foo:bar http://example.com) == + %Req.Request{ + url: URI.parse("http://example.com"), + registered_options: MapSet.new([:connect_options]), + options: %{ + connect_options: [ + proxy: {:https, "my.proxy.com", 22225, []}, + proxy_headers: [ + {"proxy-authorization", "Basic " <> Base.encode64("foo:bar")} + ] + ] + } + } + end + + test "proxy with inline basic auth" do + assert ~CURL(curl --proxy https://foo:bar@my.proxy.com:22225 http://example.com) == + %Req.Request{ + url: URI.parse("http://example.com"), + registered_options: MapSet.new([:connect_options]), + options: %{ + connect_options: [ + proxy: {:https, "my.proxy.com", 22225, []}, + proxy_headers: [ + {"proxy-authorization", "Basic " <> Base.encode64("foo:bar")} + ] + ] + } + } + end + + test "proxy raises on non http scheme uri" do + assert_raise( + ArgumentError, + "Unsupported scheme ssh for proxy in ssh://my.proxy.com:22225", + fn -> + CurlReq.Curl.decode("curl --proxy ssh://my.proxy.com:22225 http://example.com") + end + ) + end + end + + describe "newlines" do + test "sigil_CURL supports newlines" do + curl = ~CURL""" + curl -X POST \ + --location \ + https://example.com + """ + + assert curl == %Req.Request{ + method: :post, + url: URI.parse("https://example.com"), + registered_options: MapSet.new([:redirect]), + options: %{redirect: true}, + response_steps: [redirect: &Req.Steps.redirect/1] + } + end + + test "from_curl supports newlines" do + curl = + from_curl(""" + curl -X POST \ + --location \ + https://example.com + """) + + assert curl == %Req.Request{ + method: :post, + url: URI.parse("https://example.com"), + registered_options: MapSet.new([:redirect]), + options: %{redirect: true}, + response_steps: [redirect: &Req.Steps.redirect/1] + } + end + + test "accepts newlines ending in backslash" do + uri = URI.parse("https://example.com/api/2024-07/graphql.json") + + assert %Req.Request{ + method: :post, + url: ^uri, + options: %{json: %{"query" => _}} + } = ~CURL""" + curl -X POST \ + https://example.com/api/2024-07/graphql.json \ + -H 'Content-Type: application/json' \ + -H 'X-Shopify-Storefront-Access-Token: ABCDEF' \ + -d '{ + "query": "{ + products(first: 3) { + edges { + node { + id + title + } + } + } + }" + }' + """ + + assert %Req.Request{ + method: :post, + url: ^uri, + options: %{json: %{"query" => _}} + } = ~CURL""" + curl -X POST + https://example.com/api/2024-07/graphql.json + -H 'Content-Type: application/json' + -H 'X-Shopify-Storefront-Access-Token: ABCDEF' + -d '{ + "query": "{ + products(first: 3) { + edges { + node { + id + title + } + } + } + }" + }' + """ + end + + test "raises on unsupported flag" do + assert_raise ArgumentError, ~r/Unknown "--foo"/, fn -> + CurlReq.Curl.decode(~s(curl --foo https://example.com)) + end + end + end end