From d17ad33bc20a6fea0d4b4733c34b8ec4a3118a0a Mon Sep 17 00:00:00 2001 From: Kevin Schweikert Date: Tue, 24 Dec 2024 01:48:10 +0100 Subject: [PATCH 1/8] introduce internal request structure --- lib/curl_req/request.ex | 402 +++++++++++++++++++++++++++++++++ test/curl_req/request_test.exs | 4 + 2 files changed, 406 insertions(+) create mode 100644 lib/curl_req/request.ex create mode 100644 test/curl_req/request_test.exs diff --git a/lib/curl_req/request.ex b/lib/curl_req/request.ex new file mode 100644 index 0000000..d8ebec5 --- /dev/null +++ b/lib/curl_req/request.ex @@ -0,0 +1,402 @@ +defmodule CurlReq.Request do + @moduledoc since: "0.100" + @moduledoc """ + This struct is a general abstraction over an HTTP request of an HTTP client. + It acts as an intermediate representation to convert from and into the desired formats. + """ + + @doc "encode from the custom type to #{__MODULE__}" + @callback encode(term(), Keyword.t()) :: __MODULE__.t() + @doc "decode from #{__MODULE__} to the destination type" + @callback decode(__MODULE__.t(), Keyword.t()) :: term() + + @type t() :: %__MODULE__{ + user_agent: user_agent(), + headers: header(), + cookies: cookie(), + method: method(), + url: URI.t(), + compression: compression(), + redirect: boolean(), + proxy: boolean(), + proxy_auth: auth(), + auth: auth(), + encoding: encoding(), + body: term() + } + + @type user_agent() :: :curl | :req | String.t() + @type header() :: %{optional(String.t()) => [String.t()]} + @type cookie() :: %{optional(String.t()) => String.t()} + @type auth() :: {auth_option(), String.t()} | auth_option() + @type auth_option() :: :none | :basic | :bearer | :netrc + @type encoding() :: :raw | :form | :json + @type method() :: :get | :head | :put | :post | :delete | :patch + @type compression() :: :none | :gzip | :br | :zstd + + @derive {Inspect, except: [:auth]} + defstruct user_agent: :curl, + headers: %{}, + cookies: %{}, + method: :get, + url: URI.parse(""), + compression: :none, + redirect: false, + proxy: false, + proxy_url: URI.parse(""), + proxy_auth: :none, + auth: :none, + encoding: :raw, + body: nil + + @doc """ + Puts the header into the CurlReq.Request struct. Special headers like encoding, authorization or user-agent are stored in their respective field in the #{__MODULE__} struct instead of a general header. + + ## Examples + + iex> request = %CurlReq.Request{} |> CurlReq.Request.put_header("X-GitHub-Api-Version", "2022-11-28") + iex> request.headers + %{"x-github-api-version" => ["2022-11-28"]} + + iex> request = %CurlReq.Request{} |> CurlReq.Request.put_header("Content-Type", "application/json") + iex> request.encoding + :json + + iex> request = %CurlReq.Request{} |> CurlReq.Request.put_header("Authorization", "Bearer foobar") + iex> request.auth + {:bearer, "foobar"} + """ + @spec put_header(__MODULE__.t(), String.t(), String.t() | [String.t()]) :: __MODULE__.t() + def put_header(%__MODULE__{} = request, key, [val]) do + put_header(request, key, val) + end + + def put_header(%__MODULE__{} = request, key, val) when is_binary(val) do + key = String.downcase(key) + + case {key, val} do + {"authorization", "Bearer " <> token} -> + %{request | auth: {:bearer, token}} + + {"authorization", "Basic " <> userinfo} -> + %{request | auth: {:basic, userinfo}} + + {"accept-encoding", compression} + when compression in ["gzip", "br", "zstd"] -> + put_compression(request, String.to_existing_atom(compression)) + + {"content-type", "application/json"} -> + %{request | encoding: :json} + + {"content-type", "application/x-www-form-urlencoded"} -> + %{request | encoding: :form} + + {"user-agent", user_agent} -> + put_user_agent(request, user_agent) + + {"cookie", cookies} -> + for cookie <- String.split(cookies, ";"), reduce: request do + request -> + [key, value] = String.split(cookie, "=") + put_cookie(request, key, value) + end + + {key, val} -> + headers = Map.put(request.headers, key, String.split(val, ";")) + %{request | headers: headers} + end + end + + @doc """ + Puts the cookie into the CurlReq.Request struct + + ## Examples + + iex> request = %CurlReq.Request{} |> CurlReq.Request.put_cookie("key1", "value1") + iex> request.cookies + %{"key1" => "value1"} + """ + @spec put_cookie(__MODULE__.t(), String.t(), String.t()) :: __MODULE__.t() + def put_cookie(%__MODULE__{} = request, key, value) + when is_binary(key) and is_binary(value) do + cookies = Map.put(request.cookies, key, value) + %{request | cookies: cookies} + end + + @doc """ + Puts the body and optional encoding into the CurlReq.Request struct + It will immediately transform the input to the specified encoding when previously set. + + ## Examples + + iex> request = %CurlReq.Request{} |> CurlReq.Request.put_body("some body") + iex> request.encoding + :raw + iex> request.body + "some body" + + iex> request = %CurlReq.Request{} |> CurlReq.Request.put_body(%{some: "body"}) + iex> request.body + %{some: "body"} + + iex> request = %CurlReq.Request{} + ...> |> CurlReq.Request.put_encoding(:json) + ...> |> CurlReq.Request.put_body(~S|{"some": "body"}|) + iex> request.body + %{"some" => "body"} + """ + @spec put_body(__MODULE__.t(), term()) :: __MODULE__.t() + def put_body(%__MODULE__{} = request, nil), do: request + + def put_body(%__MODULE__{} = request, body) do + body = + case request.encoding do + :json -> + with true <- is_binary(body) or is_list(body), + {:ok, json} <- Jason.decode(body) do + json + else + _ -> body + end + + _ -> + body + end + + %{request | body: body} + end + + @doc """ + Puts the body and optional encoding into the CurlReq.Request struct + It will immediately transform the input to the specified encoding when previously set. + + ## Examples + + iex> request = %CurlReq.Request{} |> CurlReq.Request.put_encoding(:json) + iex> request.encoding + :json + + iex> request = %CurlReq.Request{} |> CurlReq.Request.put_encoding(:form) + iex> request.encoding + :form + + iex> request = %CurlReq.Request{} |> CurlReq.Request.put_encoding(:raw) + iex> request.encoding + :raw + """ + @spec put_encoding(__MODULE__.t(), encoding()) :: __MODULE__.t() + def put_encoding(%__MODULE__{body: body} = request, encoding) + when encoding in [:raw, :json, :form] do + body = + case encoding do + :json -> + with true <- is_binary(body) or is_list(body), + {:ok, json} <- Jason.decode(body) do + json + else + _ -> body + end + + _ -> + body + end + + %{request | body: body, encoding: encoding} + end + + @doc """ + Puts authorization into the CurlReq.Request struct + + ## Examples + + iex> request = %CurlReq.Request{} |> CurlReq.Request.put_auth({:basic, "barbaz"}) + iex> request.auth + {:basic, "barbaz"} + """ + @spec put_auth(__MODULE__.t(), {:bearer | :basic | :netrc, String.t()} | :netrc | nil) :: + __MODULE__.t() + def put_auth(%__MODULE__{} = request, nil) do + request + end + + def put_auth(%__MODULE__{} = request, :netrc) do + %{request | auth: :netrc} + end + + def put_auth(%__MODULE__{} = request, {type, credentials}) + when type in [:netrc, :basic, :bearer] do + %{request | auth: {type, credentials}} + end + + @doc """ + Puts authorization into the CurlReq.Request struct + + ## Examples + + iex> request = %CurlReq.Request{} |> CurlReq.Request.put_auth(:bearer, "foobar") + iex> request.auth + {:bearer, "foobar"} + """ + @spec put_auth(__MODULE__.t(), :bearer | :basic | :netrc, String.t()) :: __MODULE__.t() + def put_auth(%__MODULE__{} = request, type, credentials) + when type in [:netrc, :basic, :bearer] do + %{request | auth: {type, credentials}} + end + + @doc """ + Puts the url into the CurlReq.Request struct, + It either accepts a binary or an URI struct + + ## Examples + + iex> request = %CurlReq.Request{} |> CurlReq.Request.put_url("https://example.com") + iex> request.url + URI.parse("https://example.com") + """ + @spec put_url(__MODULE__.t(), URI.t() | String.t()) :: __MODULE__.t() + def put_url(%__MODULE__{} = request, %URI{scheme: scheme, userinfo: userinfo} = uri) + when scheme in ["http", "https"] do + request = %{request | url: uri} + + case userinfo do + nil -> request + userinfo -> %{request | auth: {:basic, userinfo}} + end + end + + def put_url(%__MODULE__{} = request, uri) when is_binary(uri) do + with %URI{scheme: scheme, userinfo: userinfo} when scheme in ["http", "https"] <- + URI.parse(uri) do + request = %{ + request + | url: URI.parse(uri) |> Map.put(:userinfo, nil) + } + + case userinfo do + nil -> request + userinfo -> %{request | auth: {:basic, userinfo}} + end + end + end + + @doc """ + Puts the proxy url into the CurlReq.Request struct, + It either accepts a binary or an URI struct + + ## Examples + + iex> request = %CurlReq.Request{} |> CurlReq.Request.put_proxy("https://example.com") + iex> request.url + URI.parse("https://example.com") + """ + @spec put_proxy(__MODULE__.t(), URI.t() | String.t()) :: __MODULE__.t() + def put_proxy(%__MODULE__{} = request, uri) do + with %URI{scheme: scheme, userinfo: userinfo} when scheme in ["http", "https"] <- + URI.parse(uri) do + request = %{ + request + | proxy_url: URI.parse(uri) |> Map.put(:userinfo, nil), + proxy: true + } + + case userinfo do + nil -> request + userinfo -> %{request | proxy_auth: {:basic, userinfo}} + end + else + _ -> + require Logger + Logger.error(inspect(uri)) + request + end + end + + @doc """ + Puts the method into the CurlReq.Request struct + + ## Examples + + iex> request = %CurlReq.Request{} |> CurlReq.Request.put_method("PUT") + iex> request.method + :put + + iex> request = %CurlReq.Request{} |> CurlReq.Request.put_method(:post) + iex> request.method + :post + """ + @spec put_method(__MODULE__.t(), method() | String.t()) :: __MODULE__.t() + def put_method(%__MODULE__{} = request, method) + when method in [:get, :head, :put, :post, :delete, :patch] do + %{request | method: method} + end + + def put_method(%__MODULE__{} = request, method) when is_binary(method) do + method + |> String.downcase() + |> String.to_existing_atom() + |> then(&put_method(request, &1)) + end + + @doc """ + Sets the compression option in the Curl.Request struct + + ## Examples + + iex> request = %CurlReq.Request{} |> CurlReq.Request.put_compression(true) + iex> request.compression + :gzip + + iex> request = %CurlReq.Request{} |> CurlReq.Request.put_compression(:br) + iex> request.compression + :br + """ + @spec put_compression(__MODULE__.t(), compression() | boolean() | nil) :: __MODULE__.t() + def put_compression(%__MODULE__{} = request, nil), do: request + + def put_compression(%__MODULE__{} = request, true) do + %{request | compression: :gzip} + end + + def put_compression(%__MODULE__{} = request, false) do + %{request | compression: :none} + end + + def put_compression(%__MODULE__{} = request, type) when type in [:gzip, :br, :zstd] do + %{request | compression: type} + end + + @doc """ + Sets the redirect option in the Curl.Request struct + + ## Examples + + iex> request = %CurlReq.Request{} |> CurlReq.Request.put_redirect(true) + iex> request.redirect + true + """ + @spec put_redirect(__MODULE__.t(), boolean() | nil) :: __MODULE__.t() + def put_redirect(%__MODULE__{} = request, nil), do: request + + def put_redirect(%__MODULE__{} = request, enabled) when is_boolean(enabled) do + %{request | redirect: enabled} + end + + @doc """ + Sets the user agent in the Curl.Request struct + + ## Examples + + iex> request = %CurlReq.Request{} |> CurlReq.Request.put_user_agent(:req) + iex> request.user_agent + :req + + iex> request = %CurlReq.Request{} |> CurlReq.Request.put_user_agent("some user agent") + iex> request.user_agent + "some user agent" + """ + @spec put_redirect(__MODULE__.t(), :req | :curl | String.t()) :: __MODULE__.t() + def put_user_agent(%__MODULE{} = request, user_agent) + when user_agent in [:curl, :req] or is_binary(user_agent) do + %{request | user_agent: user_agent} + end +end diff --git a/test/curl_req/request_test.exs b/test/curl_req/request_test.exs new file mode 100644 index 0000000..443c788 --- /dev/null +++ b/test/curl_req/request_test.exs @@ -0,0 +1,4 @@ +defmodule CurlReq.RequestTest do + use ExUnit.Case, async: true + doctest CurlReq.Request +end From c8b759c36aaf3b02ca94b0302983e0215d8e0c74 Mon Sep 17 00:00:00 2001 From: Kevin Schweikert Date: Tue, 24 Dec 2024 02:12:24 +0100 Subject: [PATCH 2/8] implement new request behaviour for curl and req --- lib/curl_req.ex | 157 ++------------ lib/curl_req/curl.ex | 400 +++++++++++++++++++++++++++++++++++ lib/curl_req/req.ex | 182 ++++++++++++++++ lib/curl_req/request.ex | 2 +- lib/curl_req/shell.ex | 13 -- mix.exs | 1 + test/curl_req/macro_test.exs | 377 --------------------------------- test/curl_req_test.exs | 397 ++++++++++++++++++++++++++++++++-- 8 files changed, 983 insertions(+), 546 deletions(-) create mode 100644 lib/curl_req/curl.ex create mode 100644 lib/curl_req/req.ex delete mode 100644 test/curl_req/macro_test.exs diff --git a/lib/curl_req.ex b/lib/curl_req.ex index 077a318..36e217e 100644 --- a/lib/curl_req.ex +++ b/lib/curl_req.ex @@ -121,153 +121,20 @@ defmodule CurlReq do ] @spec to_curl(Req.Request.t(), to_curl_opts()) :: String.t() def to_curl(req, options \\ []) do - opts = Keyword.validate!(options, flags: :short, run_steps: true, flavor: nil, flavour: :curl) - flavor = opts[:flavor] || opts[:flavour] - flag_style = opts[:flags] - run_steps = opts[:run_steps] - - available_steps = step_names(req, run_steps) - req = run_steps(req, available_steps) - - cookies = - case Map.get(req.headers, "cookie") do - nil -> [] - [cookies] -> [cookie_flag(flag_style), cookies] - end - - headers = - req.headers - |> Enum.reject(fn {key, _val} -> key == "cookie" end) - |> Enum.flat_map(&map_header(&1, flag_style, flavor)) - - body = - case req.body do - nil -> [] - body -> [data_flag(flag_style), body] - end - options = - case req.options do - %{redirect: true} -> - [location_flag(flag_style)] - - # avoids duplicate compression argument - %{compressed: true} -> - if :compressed in available_steps, do: [], else: [compressed_flag()] - - %{connect_options: connect_options} -> - proxy = - case Keyword.get(connect_options, :proxy) do - nil -> - [] - - {scheme, host, port, _} -> - [proxy_flag(flag_style), "#{scheme}://#{host}:#{port}"] - end - - case Keyword.get(connect_options, :proxy_headers) do - [{"proxy-authorization", "Basic " <> encoded_creds}] -> - proxy ++ [proxy_user_flag(flag_style), Base.decode64!(encoded_creds)] - - _ -> - proxy - end - - _ -> - [] - end + Keyword.validate!(options, flags: :short, run_steps: true, flavor: nil, flavour: :curl) - auth = - with %{auth: scheme} <- req.options do - case scheme do - {:bearer, token} -> - [header_flag(flag_style), "authorization: Bearer #{token}"] + # flavor = opts[:flavor] || opts[:flavour] + # flag_style = opts[:flags] + run_steps = options[:run_steps] - {:basic, userinfo} -> - [user_flag(flag_style), userinfo] ++ [basic_auth_flag()] - - :netrc -> - [netrc_flag(flag_style)] - - {:netrc, filepath} -> - [netrc_file_flag(flag_style), filepath] - - _ -> - [] - end - else - _ -> - [] - end - - method = - case req.method do - nil -> [request_flag(flag_style), "GET"] - :head -> [head_flag(flag_style)] - m -> [request_flag(flag_style), String.upcase(to_string(m))] - end - - url = [to_string(req.url)] - - CurlReq.Shell.cmd_to_string( - "curl", - auth ++ headers ++ cookies ++ body ++ options ++ method ++ url - ) - end + available_steps = step_names(req, run_steps) + req = run_steps(req, available_steps) - @typep header :: {String.t(), list(String.t())} - @spec map_header(header(), flags(), flavor()) :: list() - defp map_header({"accept-encoding", [compression]}, _flag_style, :curl) - when compression in ["gzip", "br", "zstd"] do - [compressed_flag()] + CurlReq.Req.decode(req) + |> CurlReq.Curl.encode(options) end - # filter out auth header because we expect it to be set as an auth step option - defp map_header({"authorization", _}, _flag_style, :curl), - do: [] - - # filter out user agent when mode is :curl - defp map_header({"user-agent", ["req/" <> _]}, _, :curl), do: [] - - defp map_header({key, value}, flag_style, _), - do: [header_flag(flag_style), "#{key}: #{value}"] - - defp cookie_flag(:short), do: "-b" - defp cookie_flag(:long), do: "--cookie" - - defp header_flag(:short), do: "-H" - defp header_flag(:long), do: "--header" - - defp data_flag(:short), do: "-d" - defp data_flag(:long), do: "--data" - - defp head_flag(:short), do: "-I" - defp head_flag(:long), do: "--head" - - defp request_flag(:short), do: "-X" - defp request_flag(:long), do: "--request" - - defp location_flag(:short), do: "-L" - defp location_flag(:long), do: "--location" - - defp user_flag(:short), do: "-u" - defp user_flag(:long), do: "--user" - - defp basic_auth_flag(), do: "--basic" - - defp compressed_flag(), do: "--compressed" - - defp proxy_flag(:short), do: "-x" - defp proxy_flag(:long), do: "--proxy" - - defp proxy_user_flag(:short), do: "-U" - defp proxy_user_flag(:long), do: "--proxy-user" - - defp netrc_flag(:short), do: "-n" - defp netrc_flag(:long), do: "--netrc" - - defp netrc_file_flag(_), do: "--netrc-file" - @doc """ Transforms a curl command into a Req request. @@ -308,7 +175,11 @@ defmodule CurlReq do @doc since: "0.98.4" @spec from_curl(String.t()) :: Req.Request.t() - def from_curl(curl_command), do: CurlReq.Macro.parse(curl_command) + def from_curl(curl_command) do + curl_command + |> CurlReq.Curl.decode() + |> CurlReq.Req.encode() + end @doc """ Same as `from_curl/1` but as a sigil. The benefit here is, that the Req.Request struct will be created at compile time and you don't need to escape the string @@ -335,7 +206,7 @@ defmodule CurlReq do defmacro sigil_CURL({:<<>>, _line_info, [command]}, _extra) do command - |> CurlReq.Macro.parse() + |> from_curl() |> Macro.escape() end end diff --git a/lib/curl_req/curl.ex b/lib/curl_req/curl.ex new file mode 100644 index 0000000..c62f0f7 --- /dev/null +++ b/lib/curl_req/curl.ex @@ -0,0 +1,400 @@ +defmodule CurlReq.Curl do + # TODO: docs + @behaviour CurlReq.Request + + @impl CurlReq.Request + @spec decode(String.t()) :: CurlReq.Request.t() + def decode(command, _opts \\ []) when is_binary(command) do + command = + command + |> String.trim() + |> String.trim_leading("curl") + |> String.replace("\\\n", " ") + |> String.replace("\n", " ") + + {options, rest, invalid} = + command + |> OptionParser.split() + |> OptionParser.parse( + strict: [ + header: :keep, + request: :string, + data: :keep, + data_raw: :keep, + data_ascii: :keep, + cookie: :string, + head: :boolean, + form: :keep, + location: :boolean, + user: :string, + compressed: :boolean, + proxy: :string, + proxy_user: :string, + netrc: :boolean, + netrc_file: :string + ], + aliases: [ + H: :header, + X: :request, + d: :data, + b: :cookie, + I: :head, + F: :form, + L: :location, + u: :user, + x: :proxy, + U: :proxy_user, + n: :netrc + ] + ) + + if invalid != [] do + errors = + Enum.map(invalid, fn + {flag, nil} -> "Unknown #{inspect(flag)}" + {flag, value} -> "Invalid value #{inspect(value)} for #{inspect(flag)}" + end) + |> Enum.join("\n") + + raise ArgumentError, """ + + Command: \'curl #{command}\" + Unsupported or invalid flag(s) encountered: + + #{errors} + + Please remove the unknown flags and open an issue at https://github.com/derekkraan/curl_req + """ + end + + [url] = + rest + |> List.flatten() + + url = URI.parse(url) + + %CurlReq.Request{} + |> CurlReq.Request.put_url(url) + |> add_header(options) + |> add_method(options) + |> add_body(options) + |> add_cookie(options) + |> add_form(options) + |> add_auth(options) + |> add_compression(options) + |> add_proxy(options) + |> configure_redirects(options) + end + + defp add_header(request, options) do + headers = Keyword.get_values(options, :header) + + Enum.reduce(headers, request, fn header, acc -> + [key, value] = + header + |> String.split(":", parts: 2) + |> Enum.map(&String.trim/1) + + CurlReq.Request.put_header(acc, key, value) + end) + end + + defp add_method(request, options) do + method = + if Keyword.get(options, :head, false) do + :head + else + Keyword.get(options, :request, "GET") + end + + CurlReq.Request.put_method(request, method) + end + + defp add_body(request, options) do + body = + Enum.flat_map([:data, :data_ascii, :data_raw], fn key -> + case Keyword.get_values(options, key) do + [] -> [] + values -> Enum.map(values, &String.trim_leading(&1, "$")) + end + end) + |> Enum.join("&") + + if body != "" do + CurlReq.Request.put_body(request, body) + else + request + end + end + + defp add_cookie(request, options) do + case Keyword.get(options, :cookie) do + nil -> + request + + cookie -> + String.split(cookie, ";") + |> Enum.reduce(request, fn cookie, acc -> + [key, value] = + String.split(cookie, "=", parts: 2) + |> Enum.map(&String.trim/1) + + CurlReq.Request.put_cookie(acc, key, value) + end) + end + end + + defp add_form(request, options) do + case Keyword.get_values(options, :form) do + [] -> + request + + formdata -> + form = + for fd <- formdata, reduce: %{} do + map -> + [key, value] = String.split(fd, "=", parts: 2) + Map.put(map, key, value) + end + + request + |> CurlReq.Request.put_body(form) + |> CurlReq.Request.put_encoding(:form) + end + end + + defp add_auth(request, options) do + request = + case Keyword.get(options, :user) do + nil -> + request + + userinfo -> + CurlReq.Request.put_auth(request, :basic, userinfo) + end + + request = + case Keyword.get(options, :netrc) do + nil -> + request + + _ -> + CurlReq.Request.put_auth(request, :netrc) + end + + case Keyword.get(options, :netrc_file) do + nil -> + request + + path -> + CurlReq.Request.put_auth(request, :netrc, path) + end + end + + defp add_compression(request, options) do + case Keyword.get(options, :compressed) do + nil -> + request + + bool -> + CurlReq.Request.put_compression(request, bool) + end + end + + defp add_proxy(request, options) do + proxy = Keyword.get(options, :proxy) + proxy_user = Keyword.get(options, :proxy_user) + + case {proxy, proxy_user} do + {nil, _} -> + request + + {proxy, nil} -> + proxy = validate_proxy_uri(proxy) + CurlReq.Request.put_proxy(request, proxy) + + {proxy, proxy_user} -> + proxy = validate_proxy_uri(proxy) + proxy = %{proxy | userinfo: proxy_user} + CurlReq.Request.put_proxy(request, proxy) + end + end + + defp validate_proxy_uri("http://" <> _rest = uri), do: URI.parse(uri) + defp validate_proxy_uri("https://" <> _rest = uri), do: URI.parse(uri) + + defp validate_proxy_uri(uri) do + case String.split(uri, "://") do + [scheme, _uri] -> + raise ArgumentError, "Unsupported scheme #{scheme} for proxy in #{uri}" + + [uri] -> + URI.parse("http://" <> uri) + end + end + + defp configure_redirects(request, options) do + case Keyword.get(options, :location) do + nil -> request + bool -> CurlReq.Request.put_redirect(request, bool) + end + end + + @impl CurlReq.Request + @spec encode(CurlReq.Request.t(), Keyword.t()) :: String.t() + def encode(%CurlReq.Request{} = request, options \\ []) do + flag_style = Keyword.get(options, :flags, :short) + # TODO: implement flavor + _flavor = Keyword.get(options, :flavor, nil) || Keyword.get(options, :flavour, :curl) + + cookies = + if map_size(request.cookies) != 0 do + request.cookies + |> Enum.map(fn {key, val} -> "#{key}=#{val}" end) + |> Enum.join(";") + else + [] + end + + cookies = emit_if(cookies != [], [cookie_flag(flag_style, cookies)]) + + headers = + for {key, values} <- request.headers, reduce: [] do + headers -> + [headers, header_flag(flag_style, [key, ": ", Enum.intersperse(values, ";")])] + end + + headers = + case request.encoding do + :raw -> + headers + + :json -> + headers ++ [header_flag(flag_style, "content-type: application/json")] + + :form -> + headers ++ [header_flag(flag_style, "content-type: application/x-www-form-urlencoded")] + end + + headers = Enum.intersperse(headers, " ") + + body = + emit_if(request.body, fn -> + case request.encoding do + :json -> [data_flag(flag_style, Jason.encode!(request.body))] + _ -> [data_flag(flag_style, request.body)] + end + end) + + redirect = emit_if(request.redirect, [location_flag(flag_style)]) + compressed = emit_if(request.compression != :none, [compressed_flag(flag_style)]) + + auth = + case request.auth do + :none -> + [] + + {:basic, userinfo} -> + user_flag(flag_style, userinfo) + + {:bearer, token} -> + [header_flag(flag_style, ["authorization: Bearer ", token])] + + :netrc -> + [netrc_flag(flag_style)] + + {:netrc, filepath} -> + [netrc_file_flag(flag_style, filepath)] + end + + method = + case request.method do + :head -> [head_flag(flag_style)] + m -> [request_flag(flag_style, String.upcase(to_string(m)))] + end + + proxy = + if request.proxy do + proxy_flag(flag_style, URI.to_string(request.proxy_url)) + else + [] + end + + proxy_auth = + case request.proxy_auth do + :none -> [] + {:basic, userinfo} -> proxy_user_flag(flag_style, userinfo) + _ -> [] + end + + url = [to_string(request.url)] + + IO.iodata_to_binary( + [ + "curl", + compressed, + auth, + headers, + cookies, + body, + proxy, + proxy_auth, + redirect, + method, + url + ] + |> Enum.reject(fn part -> part == [] end) + |> Enum.intersperse(" ") + ) + end + + defp emit_if(bool, fun) when is_function(fun) do + if bool, do: fun.(), else: [] + end + + defp emit_if(bool, value) do + if bool, do: value, else: [] + end + + defp escape(value) when is_list(value) do + IO.iodata_to_binary(value) |> escape() + end + + defp escape(value) when is_binary(value) do + CurlReq.Shell.escape(value) + end + + defp cookie_flag(:short, value), do: ["-b ", escape(value)] + defp cookie_flag(:long, value), do: ["--cookie ", escape(value)] + + defp header_flag(:short, value), do: ["-H ", escape(value)] + defp header_flag(:long, value), do: ["--header ", escape(value)] + + defp data_flag(:short, value), do: ["-d ", escape(value)] + defp data_flag(:long, value), do: ["--data ", escape(value)] + + defp head_flag(:short), do: "-I" + defp head_flag(:long), do: "--head" + + defp request_flag(:short, value), do: ["-X ", escape(value)] + defp request_flag(:long, value), do: ["--request ", escape(value)] + + defp location_flag(:short), do: "-L" + defp location_flag(:long), do: "--location" + + defp user_flag(:short, value), do: ["-u ", escape(value)] + defp user_flag(:long, value), do: ["--user ", escape(value)] + + defp netrc_flag(:short), do: "-n" + defp netrc_flag(:long), do: "--netrc" + + defp netrc_file_flag(_, value), do: ["--netrc-file ", escape(value)] + + defp compressed_flag(_), do: "--compressed" + + defp proxy_flag(:short, value), do: ["-x ", escape(value)] + defp proxy_flag(:long, value), do: ["--proxy ", escape(value)] + + defp proxy_user_flag(:short, value), do: ["-U ", escape(value)] + defp proxy_user_flag(:long, value), do: ["--proxy-user ", escape(value)] +end diff --git a/lib/curl_req/req.ex b/lib/curl_req/req.ex new file mode 100644 index 0000000..6ea5582 --- /dev/null +++ b/lib/curl_req/req.ex @@ -0,0 +1,182 @@ +defmodule CurlReq.Req do + # TODO: docs + @behaviour CurlReq.Request + + @impl CurlReq.Request + @spec decode(Req.Request.t()) :: CurlReq.Request.t() + def decode(%Req.Request{} = req, _opts \\ []) do + request = + %CurlReq.Request{} + |> put_header(req) + |> CurlReq.Request.put_auth(req.options[:auth]) + |> CurlReq.Request.put_redirect(req.options[:redirect]) + |> CurlReq.Request.put_compression(req.options[:compressed]) + |> CurlReq.Request.put_user_agent(:req) + |> CurlReq.Request.put_body(req.body) + |> CurlReq.Request.put_url(req.url) + |> CurlReq.Request.put_method(req.method) + + request = + case req.options[:connect_options] do + nil -> + request + + connect_options -> + userinfo = + case Keyword.get(connect_options, :proxy_headers) do + [{"proxy-authorization", "Basic " <> encoded_userinfo}] -> + case Base.decode64(encoded_userinfo) do + {:ok, userinfo} -> userinfo + _ -> encoded_userinfo + end + + _ -> + nil + end + + case Keyword.get(connect_options, :proxy) do + {scheme, host, port, _} -> + CurlReq.Request.put_proxy(request, %URI{ + scheme: Atom.to_string(scheme), + host: host, + port: port, + userinfo: userinfo + }) + + _ -> + request + end + end + + request + end + + defp put_header(%CurlReq.Request{} = request, %Req.Request{} = req) do + for {key, val} <- req.headers, reduce: request do + request -> CurlReq.Request.put_header(request, key, val) + end + end + + @impl CurlReq.Request + @spec encode(CurlReq.Request.t()) :: Req.Request.t() + def encode(%CurlReq.Request{} = request, _opts \\ []) do + req = + %Req.Request{} + |> Req.merge(url: request.url) + |> Req.merge(method: request.method) + + cookies = + request.cookies + |> Enum.map(fn {key, val} -> "#{key}=#{val}" end) + |> Enum.join(";") + + req = + case request.user_agent do + :req -> req + :curl -> req + other -> Req.Request.put_header(req, "user-agent", other) + end + + req = + case request.encoding do + :raw -> + Req.merge(req, body: request.body) + + :form -> + req + |> Req.Request.register_options([:form]) + |> Req.Request.prepend_request_steps(encode_body: &Req.Steps.encode_body/1) + |> Req.merge(form: request.body) + + :json -> + req + |> Req.Request.register_options([:json]) + |> Req.Request.prepend_request_steps(encode_body: &Req.Steps.encode_body/1) + |> Req.merge(json: request.body) + end + + req = + case request.auth do + :none -> + req + + auth -> + req + |> Req.Request.register_options([:auth]) + |> Req.Request.prepend_request_steps(auth: &Req.Steps.auth/1) + |> Req.merge(auth: auth) + end + + req = + case request.compression do + :none -> + req + + _ -> + req + |> Req.Request.register_options([:compressed]) + |> Req.Request.prepend_request_steps(compressed: &Req.Steps.compressed/1) + |> Req.merge(compressed: true) + end + + req = + case request.redirect do + false -> + req + + _ -> + req + |> Req.Request.register_options([:redirect]) + |> Req.Request.prepend_response_steps(redirect: &Req.Steps.redirect/1) + |> Req.merge(redirect: true) + end + + req = + for {key, values} <- request.headers, reduce: req do + req -> Req.Request.put_header(req, key, values) + end + + req = + if cookies != "" do + Req.Request.put_header(req, "cookie", cookies) + else + req + end + + req = + if request.proxy do + %URI{scheme: scheme, host: host, port: port} = request.proxy_url + + connect_options = + [ + proxy: {String.to_existing_atom(scheme), host, port, []} + ] + + connect_options = + case request.proxy_auth do + :none -> + connect_options + + {:basic, userinfo} -> + Keyword.merge(connect_options, + proxy_headers: [ + {"proxy-authorization", "Basic " <> Base.encode64(userinfo)} + ] + ) + + _ -> + connect_options + end + + req + |> Req.Request.register_options([ + :connect_options + ]) + |> Req.merge(connect_options: connect_options) + else + req + end + + req + end +end diff --git a/lib/curl_req/request.ex b/lib/curl_req/request.ex index d8ebec5..3c0ff93 100644 --- a/lib/curl_req/request.ex +++ b/lib/curl_req/request.ex @@ -287,7 +287,7 @@ defmodule CurlReq.Request do iex> request = %CurlReq.Request{} |> CurlReq.Request.put_proxy("https://example.com") iex> request.url - URI.parse("https://example.com") + URI.new!("https://example.com") """ @spec put_proxy(__MODULE__.t(), URI.t() | String.t()) :: __MODULE__.t() def put_proxy(%__MODULE__{} = request, uri) do diff --git a/lib/curl_req/shell.ex b/lib/curl_req/shell.ex index 906348f..7a9a056 100644 --- a/lib/curl_req/shell.ex +++ b/lib/curl_req/shell.ex @@ -12,20 +12,7 @@ defmodule CurlReq.Shell do {~S(~), ~S(\~)} ] - @doc """ - This function takes the same arguments as `System.cmd/3`, but returns - the command in string form instead of running the command. - """ @no_quotes ~r/^[a-zA-Z-,._+:@%\/]*$/ - def cmd_to_string(cmd, args) do - final_args = - args - |> Enum.map(&IO.iodata_to_binary/1) - |> Enum.map(&escape/1) - |> Enum.join(" ") - - "#{cmd} #{final_args}" |> String.trim_trailing() - end @doc ~S""" Examples: diff --git a/mix.exs b/mix.exs index c26e262..a7db916 100644 --- a/mix.exs +++ b/mix.exs @@ -26,6 +26,7 @@ defmodule CurlReq.MixProject do defp deps do [ {:req, "~> 0.4.0 or ~> 0.5.0"}, + {:jason, "~> 1.4"}, {:ex_doc, ">= 0.0.0", only: :dev}, {:blend, "~> 0.4.1", only: :dev} ] diff --git a/test/curl_req/macro_test.exs b/test/curl_req/macro_test.exs deleted file mode 100644 index 59b5d35..0000000 --- a/test/curl_req/macro_test.exs +++ /dev/null @@ -1,377 +0,0 @@ -defmodule CurlReq.MacroTest do - use ExUnit.Case, async: true - - import CurlReq - - describe "macro" do - test "single header" do - assert ~CURL(curl -H "user-agent: req/0.4.14" -X GET https://example.com/fact) == - %Req.Request{ - method: :get, - headers: %{"user-agent" => ["req/0.4.14"]}, - url: URI.parse("https://example.com/fact") - } - end - - test "post method" do - assert ~CURL(curl -X POST https://example.com) == - %Req.Request{ - method: :post, - url: URI.parse("https://example.com") - } - end - - test "head method" do - assert ~CURL(curl -I https://example.com) == - %Req.Request{ - method: :head, - url: URI.parse("https://example.com") - } - end - - test "multiple headers with body" do - assert ~CURL(curl -H "accept-encoding: gzip" -H "authorization: Bearer 6e8f18e6-141b-4d12-8397-7e7791d92ed4:lon" -H "content-type: application/json" -H "user-agent: req/0.4.14" -d "{\"input\":[{\"leadFormFields\":{\"Company\":\"k\",\"Country\":\"DZ\",\"Email\":\"k\",\"FirstName\":\"k\",\"Industry\":\"CTO\",\"LastName\":\"k\",\"Phone\":\"k\",\"PostalCode\":\"1234ZZ\",\"jobspecialty\":\"engineer\",\"message\":\"I would like to know if Roche delivers to The Netherlands.\"}}],\"formId\":4318}" -X POST "https://example.com/rest/v1/leads/submitForm.json") == - %Req.Request{ - method: :post, - url: URI.parse("https://example.com/rest/v1/leads/submitForm.json"), - headers: %{ - "accept-encoding" => ["gzip"], - "content-type" => ["application/json"], - "user-agent" => ["req/0.4.14"] - }, - registered_options: MapSet.new([:auth]), - options: %{auth: {:bearer, "6e8f18e6-141b-4d12-8397-7e7791d92ed4:lon"}}, - current_request_steps: [:auth], - request_steps: [auth: &Req.Steps.auth/1], - body: - "{\"input\":[{\"leadFormFields\":{\"Company\":\"k\",\"Country\":\"DZ\",\"Email\":\"k\",\"FirstName\":\"k\",\"Industry\":\"CTO\",\"LastName\":\"k\",\"Phone\":\"k\",\"PostalCode\":\"1234ZZ\",\"jobspecialty\":\"engineer\",\"message\":\"I would like to know if Roche delivers to The Netherlands.\"}}],\"formId\":4318}" - } - end - - test "without curl prefix" do - assert ~CURL(http://example.com) == - %Req.Request{ - method: :get, - url: URI.parse("http://example.com") - } - end - - test "multiple data flags" do - assert ~CURL(curl http://example.com -d name=foo -d mail=bar) == - %Req.Request{ - url: URI.parse("http://example.com"), - body: "name=foo&mail=bar" - } - end - - test "cookie" do - assert ~CURL(http://example.com -b "name1=value1") == - %Req.Request{ - url: URI.parse("http://example.com"), - headers: %{"cookie" => ["name1=value1"]} - } - - assert ~CURL(http://example.com -b "name1=value1; name2=value2") == - %Req.Request{ - url: URI.parse("http://example.com"), - headers: %{"cookie" => ["name1=value1; name2=value2"]} - } - end - - test "formdata" do - assert ~CURL(curl http://example.com -F name=foo -F mail=bar) == - %Req.Request{ - url: URI.parse("http://example.com"), - body: nil, - registered_options: MapSet.new([:form]), - options: %{form: %{"name" => "foo", "mail" => "bar"}}, - current_request_steps: [:encode_body], - request_steps: [encode_body: &Req.Steps.encode_body/1] - } - end - - test "data raw" do - assert ~CURL""" - curl 'https://example.com/graphql' \ - -X POST \ - -H 'Accept: application/graphql-response+json'\ - --data-raw '{"operationName":"get","query":"query get {name}"}' - """ == - %Req.Request{ - method: :post, - url: URI.parse("https://example.com/graphql"), - headers: %{"accept" => ["application/graphql-response+json"]}, - body: "{\"operationName\":\"get\",\"query\":\"query get {name}\"}", - options: %{}, - halted: false, - adapter: &Req.Steps.run_finch/1, - request_steps: [], - response_steps: [], - error_steps: [], - private: %{} - } - end - - test "data raw with ansii escape" do - assert ~CURL""" - curl 'https://example.com/employees/107'\ - -X PATCH\ - -H 'Accept: application/vnd.api+json'\ - --data-raw $'{"data":{"attributes":{"first-name":"Adam"}}}' - """ == - %Req.Request{ - method: :patch, - url: URI.parse("https://example.com/employees/107"), - headers: %{"accept" => ["application/vnd.api+json"]}, - body: "{\"data\":{\"attributes\":{\"first-name\":\"Adam\"}}}", - options: %{}, - halted: false, - adapter: &Req.Steps.run_finch/1, - request_steps: [], - response_steps: [], - error_steps: [], - private: %{} - } - end - - test "basic auth" do - assert ~CURL(curl http://example.com -u user:pass) == - %Req.Request{ - url: URI.parse("http://example.com"), - body: nil, - registered_options: MapSet.new([:auth]), - options: %{auth: {:basic, "user:pass"}}, - current_request_steps: [:auth], - request_steps: [auth: &Req.Steps.auth/1] - } - end - - test "bearer token auth" do - curl = ~CURL""" - curl -L \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer " \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://example.com/users - """ - - assert curl == - %Req.Request{ - url: URI.parse("https://example.com/users"), - body: nil, - headers: %{ - "accept" => ["application/vnd.github+json"], - "x-github-api-version" => ["2022-11-28"] - }, - registered_options: MapSet.new([:auth, :redirect]), - options: %{auth: {:bearer, ""}, redirect: true}, - current_request_steps: [:auth], - request_steps: [auth: &Req.Steps.auth/1], - response_steps: [redirect: &Req.Steps.redirect/1] - } - end - - test "netrc auth" do - assert ~CURL(curl http://example.com -n) == - %Req.Request{ - url: URI.parse("http://example.com"), - body: nil, - registered_options: MapSet.new([:auth]), - options: %{auth: :netrc}, - current_request_steps: [:auth], - request_steps: [auth: &Req.Steps.auth/1] - } - end - - test "netrc file auth" do - assert ~CURL(curl http://example.com --netrc-file "./mynetrc") == - %Req.Request{ - url: URI.parse("http://example.com"), - body: nil, - registered_options: MapSet.new([:auth]), - options: %{auth: {:netrc, "./mynetrc"}}, - current_request_steps: [:auth], - request_steps: [auth: &Req.Steps.auth/1] - } - end - - test "compressed" do - assert ~CURL(curl --compressed http://example.com) == - %Req.Request{ - url: URI.parse("http://example.com"), - body: nil, - registered_options: MapSet.new([:compressed]), - options: %{compressed: true}, - current_request_steps: [:compressed], - request_steps: [compressed: &Req.Steps.compressed/1] - } - end - - test "redirect" do - assert ~CURL(curl -L http://example.com) == - %Req.Request{ - url: URI.parse("http://example.com"), - registered_options: MapSet.new([:redirect]), - options: %{redirect: true}, - response_steps: [redirect: &Req.Steps.redirect/1] - } - end - - test "cookie, formadata, auth and redirect" do - assert ~CURL(curl -L -u user:pass -F name=foo -b name=bar http://example.com) == - %Req.Request{ - url: URI.parse("http://example.com"), - headers: %{"cookie" => ["name=bar"]}, - current_request_steps: [:auth, :encode_body], - registered_options: MapSet.new([:redirect, :auth, :form]), - options: %{redirect: true, auth: {:basic, "user:pass"}, form: %{"name" => "foo"}}, - request_steps: [auth: &Req.Steps.auth/1, encode_body: &Req.Steps.encode_body/1], - response_steps: [redirect: &Req.Steps.redirect/1] - } - end - - test "proxy" do - assert ~CURL(curl --proxy my.proxy.com:22225 http://example.com) == - %Req.Request{ - url: URI.parse("http://example.com"), - registered_options: MapSet.new([:connect_options]), - options: %{ - connect_options: [proxy: {:http, "my.proxy.com", 22225, []}] - } - } - end - - test "proxy with basic auth" do - assert ~CURL(curl --proxy https://my.proxy.com:22225 --proxy-user foo:bar http://example.com) == - %Req.Request{ - url: URI.parse("http://example.com"), - registered_options: MapSet.new([:connect_options]), - options: %{ - connect_options: [ - proxy: {:https, "my.proxy.com", 22225, []}, - proxy_headers: [ - {"proxy-authorization", "Basic " <> Base.encode64("foo:bar")} - ] - ] - } - } - end - - test "proxy with inline basic auth" do - assert ~CURL(curl --proxy https://foo:bar@my.proxy.com:22225 http://example.com) == - %Req.Request{ - url: URI.parse("http://example.com"), - registered_options: MapSet.new([:connect_options]), - options: %{ - connect_options: [ - proxy: {:https, "my.proxy.com", 22225, []}, - proxy_headers: [ - {"proxy-authorization", "Basic " <> Base.encode64("foo:bar")} - ] - ] - } - } - end - - test "proxy raises on non http scheme uri" do - assert_raise( - ArgumentError, - "Unsupported scheme ssh for proxy in ssh://my.proxy.com:22225", - fn -> - CurlReq.Macro.parse("curl --proxy ssh://my.proxy.com:22225 http://example.com") - end - ) - end - end - - describe "newlines" do - test "sigil_CURL supports newlines" do - curl = ~CURL""" - curl -X POST \ - --location \ - https://example.com - """ - - assert curl == %Req.Request{ - method: :post, - url: URI.parse("https://example.com"), - registered_options: MapSet.new([:redirect]), - options: %{redirect: true}, - response_steps: [redirect: &Req.Steps.redirect/1] - } - end - - test "from_curl supports newlines" do - curl = - from_curl(""" - curl -X POST \ - --location \ - https://example.com - """) - - assert curl == %Req.Request{ - method: :post, - url: URI.parse("https://example.com"), - registered_options: MapSet.new([:redirect]), - options: %{redirect: true}, - response_steps: [redirect: &Req.Steps.redirect/1] - } - end - - test "accepts newlines ending in backslash" do - uri = URI.parse("https://example.com/api/2024-07/graphql.json") - - assert %Req.Request{ - method: :post, - url: ^uri, - headers: %{"content-type" => ["application/json"]} - } = ~CURL""" - curl -X POST \ - https://example.com/api/2024-07/graphql.json \ - -H 'Content-Type: application/json' \ - -H 'X-Shopify-Storefront-Access-Token: ABCDEF' \ - -d '{ - "query": "{ - products(first: 3) { - edges { - node { - id - title - } - } - } - }" - }' - """ - - assert %Req.Request{ - method: :post, - url: ^uri, - headers: %{"content-type" => ["application/json"]} - } = ~CURL""" - curl -X POST - https://example.com/api/2024-07/graphql.json - -H 'Content-Type: application/json' - -H 'X-Shopify-Storefront-Access-Token: ABCDEF' - -d '{ - "query": "{ - products(first: 3) { - edges { - node { - id - title - } - } - } - }" - }' - """ - end - - test "raises on unsupported flag" do - assert_raise ArgumentError, ~r/Unknown "--foo"/, fn -> - CurlReq.Macro.parse(~s(curl --foo https://example.com)) - end - end - end -end diff --git a/test/curl_req_test.exs b/test/curl_req_test.exs index bb1ce1a..f2612f2 100644 --- a/test/curl_req_test.exs +++ b/test/curl_req_test.exs @@ -1,6 +1,8 @@ defmodule CurlReqTest do use ExUnit.Case, async: true + doctest CurlReq + import CurlReq import ExUnit.CaptureIO @@ -13,7 +15,7 @@ defmodule CurlReqTest do end test "with label" do - assert capture_io(fn -> + assert capture_io(:stdio, fn -> Req.new(url: "/with_label", base_url: "https://example.com/") |> CurlReq.inspect(label: "MY REQ") end) === "MY REQ: curl --compressed -X GET https://example.com/with_label\n" @@ -30,7 +32,7 @@ defmodule CurlReqTest do test "cookies get extracted from header" do assert Req.new(url: "http://example.com", headers: %{"cookie" => ["name1=value1"]}) |> CurlReq.to_curl() == - "curl --compressed -b \"name1=value1\" -X GET http://example.com" + ~s|curl --compressed -b "name1=value1" -X GET http://example.com| end test "redirect flag gets set" do @@ -53,13 +55,13 @@ defmodule CurlReqTest do headers: %{"cookie" => ["name1=value1"], "content-type" => ["application/json"]} ) |> CurlReq.to_curl(flags: :long) == - "curl --compressed --header \"content-type: application/json\" --cookie \"name1=value1\" --location --head http://example.com" + ~S|curl --compressed --header "content-type: application/json" --cookie "name1=value1" --location --head http://example.com| end test "formdata flags get set with correct headers and body" do assert Req.new(url: "http://example.com", form: [key1: "value1", key2: "value2"]) |> CurlReq.to_curl() == - "curl --compressed -H \"content-type: application/x-www-form-urlencoded\" -d \"key1=value1&key2=value2\" -X GET http://example.com" + ~S|curl --compressed -H "content-type: application/x-www-form-urlencoded" -d "key1=value1&key2=value2" -X GET http://example.com| end test "works when body is iodata" do @@ -80,13 +82,13 @@ defmodule CurlReqTest do end test "req flavor with explicit headers" do - assert "curl -H \"accept-encoding: gzip\" -H \"user-agent: req/#{req_version()}\" -X GET https://example.com" == + assert ~s|curl -H "accept-encoding: gzip" -H "user-agent: req/#{req_version()}" -X GET https://example.com| == Req.new(url: "https://example.com") |> CurlReq.to_curl(flavor: :req) end test "proxy" do - assert ~S(curl --compressed -x "http://my.proxy.com:80" -X GET https://example.com) == + assert ~S(curl --compressed -x http://my.proxy.com -X GET https://example.com) == Req.new( url: "https://example.com", connect_options: [proxy: {:http, "my.proxy.com", 80, []}] @@ -95,7 +97,7 @@ defmodule CurlReqTest do end test "proxy user" do - assert ~S(curl --compressed -x "http://my.proxy.com:80" -U foo:bar -X GET https://example.com) == + assert ~S(curl --compressed -x http://my.proxy.com -U foo:bar -X GET https://example.com) == Req.new( url: "https://example.com", connect_options: [ @@ -109,13 +111,13 @@ defmodule CurlReqTest do end test "basic auth option" do - assert "curl -u user:pass --basic --compressed -X GET https://example.com" == + assert "curl --compressed -u user:pass -X GET https://example.com" == Req.new(url: "https://example.com", auth: {:basic, "user:pass"}) |> CurlReq.to_curl() end test "bearer auth option" do - assert ~S(curl -H "authorization: Bearer foo123bar" --compressed -X GET https://example.com) == + assert ~S(curl --compressed -H "authorization: Bearer foo123bar" -X GET https://example.com) == Req.new(url: "https://example.com", auth: {:bearer, "foo123bar"}) |> CurlReq.to_curl() end @@ -133,7 +135,7 @@ defmodule CurlReqTest do File.write(netrc_path, credentials) System.put_env("NETRC", netrc_path) - assert "curl -n --compressed -X GET https://example.com" == + assert "curl --compressed -n -X GET https://example.com" == Req.new(url: "https://example.com", auth: :netrc) |> CurlReq.to_curl() end @@ -150,12 +152,12 @@ defmodule CurlReqTest do netrc_path = Path.join(tmp_dir, "my_netrc") File.write(netrc_path, credentials) - assert ~s(curl --netrc-file "#{netrc_path}" --compressed -X GET https://example.com) == + assert ~s(curl --compressed --netrc-file "#{netrc_path}" -X GET https://example.com) == Req.new(url: "https://example.com", auth: {:netrc, netrc_path}) |> CurlReq.to_curl() end - test "include `encode_body` does not run `comporessed` or other steps" do + test "include `encode_body` does not run `compressed` or other steps" do assert ~S(curl -H "accept: application/json" -H "content-type: application/json" -d "{\"key\":\"val\"}" -X GET https://example.com) == Req.new(url: "https://example.com", json: %{key: "val"}) |> CurlReq.to_curl(run_steps: [only: [:encode_body]]) @@ -167,4 +169,375 @@ defmodule CurlReqTest do |> CurlReq.to_curl(run_steps: [except: [:compressed, :encode_body]]) end end + + describe "from_curl" do + test "single header" do + assert ~CURL(curl -H "user-agent: req/0.4.14" -X GET https://example.com/fact) == + %Req.Request{ + method: :get, + headers: %{"user-agent" => ["req/0.4.14"]}, + url: URI.parse("https://example.com/fact") + } + end + + test "post method" do + assert ~CURL(curl -X POST https://example.com) == + %Req.Request{ + method: :post, + url: URI.parse("https://example.com") + } + end + + test "head method" do + assert ~CURL(curl -I https://example.com) == + %Req.Request{ + method: :head, + url: URI.parse("https://example.com") + } + end + + test "multiple headers with body" do + assert ~CURL(curl -H "accept-encoding: gzip" -H "authorization: Bearer 6e8f18e6-141b-4d12-8397-7e7791d92ed4:lon" -H "content-type: application/json" -H "user-agent: req/0.4.14" -d "{\"input\":[{\"leadFormFields\":{\"Company\":\"k\",\"Country\":\"DZ\",\"Email\":\"k\",\"FirstName\":\"k\",\"Industry\":\"CTO\",\"LastName\":\"k\",\"Phone\":\"k\",\"PostalCode\":\"1234ZZ\",\"jobspecialty\":\"engineer\",\"message\":\"I would like to know if Roche delivers to The Netherlands.\"}}],\"formId\":4318}" -X POST "https://example.com/rest/v1/leads/submitForm.json") == + %Req.Request{ + method: :post, + url: URI.parse("https://example.com/rest/v1/leads/submitForm.json"), + headers: %{ + "user-agent" => ["req/0.4.14"] + }, + registered_options: MapSet.new([:compressed, :auth, :json]), + options: %{ + compressed: true, + auth: {:bearer, "6e8f18e6-141b-4d12-8397-7e7791d92ed4:lon"}, + json: %{ + "formId" => 4318, + "input" => [ + %{ + "leadFormFields" => %{ + "Company" => "k", + "Country" => "DZ", + "Email" => "k", + "FirstName" => "k", + "Industry" => "CTO", + "LastName" => "k", + "Phone" => "k", + "PostalCode" => "1234ZZ", + "jobspecialty" => "engineer", + "message" => + "I would like to know if Roche delivers to The Netherlands." + } + } + ] + } + }, + current_request_steps: [:compressed, :auth, :encode_body], + request_steps: [ + compressed: &Req.Steps.compressed/1, + auth: &Req.Steps.auth/1, + encode_body: &Req.Steps.encode_body/1 + ] + } + end + + test "without curl prefix" do + assert ~CURL(http://example.com) == + %Req.Request{ + method: :get, + url: URI.parse("http://example.com") + } + end + + test "multiple data flags" do + assert ~CURL(curl http://example.com -d name=foo -d mail=bar) == + %Req.Request{ + url: URI.parse("http://example.com"), + body: "name=foo&mail=bar" + } + end + + test "cookie" do + assert ~CURL(http://example.com -b "name1=value1") == + %Req.Request{ + url: URI.parse("http://example.com"), + headers: %{"cookie" => ["name1=value1"]} + } + + assert ~CURL(http://example.com -b "name1=value1; name2=value2") == + %Req.Request{ + url: URI.parse("http://example.com"), + headers: %{"cookie" => ["name1=value1;name2=value2"]} + } + end + + test "formdata" do + assert ~CURL(curl http://example.com -F name=foo -F mail=bar) == + %Req.Request{ + url: URI.parse("http://example.com"), + body: nil, + registered_options: MapSet.new([:form]), + options: %{form: %{"name" => "foo", "mail" => "bar"}}, + current_request_steps: [:encode_body], + request_steps: [encode_body: &Req.Steps.encode_body/1] + } + end + + test "data raw" do + assert ~CURL""" + curl 'https://example.com/graphql' \ + -X POST \ + -H 'Accept: application/graphql-response+json'\ + --data-raw '{"operationName":"get","query":"query get {name}"}' + """ == + %Req.Request{ + method: :post, + url: URI.parse("https://example.com/graphql"), + headers: %{"accept" => ["application/graphql-response+json"]}, + body: "{\"operationName\":\"get\",\"query\":\"query get {name}\"}", + options: %{}, + halted: false, + adapter: &Req.Steps.run_finch/1, + request_steps: [], + response_steps: [], + error_steps: [], + private: %{} + } + end + + test "data raw with ansii escape" do + assert ~CURL""" + curl 'https://example.com/employees/107'\ + -X PATCH\ + -H 'Accept: application/vnd.api+json'\ + --data-raw $'{"data":{"attributes":{"first-name":"Adam"}}}' + """ == + %Req.Request{ + method: :patch, + url: URI.parse("https://example.com/employees/107"), + headers: %{"accept" => ["application/vnd.api+json"]}, + body: "{\"data\":{\"attributes\":{\"first-name\":\"Adam\"}}}", + options: %{}, + halted: false, + adapter: &Req.Steps.run_finch/1, + request_steps: [], + response_steps: [], + error_steps: [], + private: %{} + } + end + + test "auth" do + assert ~CURL(curl http://example.com -u user:pass) == + %Req.Request{ + url: URI.parse("http://example.com"), + body: nil, + registered_options: MapSet.new([:auth]), + options: %{auth: {:basic, "user:pass"}}, + current_request_steps: [:auth], + request_steps: [auth: &Req.Steps.auth/1] + } + end + + test "bearer token auth" do + curl = ~CURL""" + curl -L \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer " \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://example.com/users + """ + + assert curl == + %Req.Request{ + url: URI.parse("https://example.com/users"), + body: nil, + headers: %{ + "accept" => ["application/vnd.github+json"], + "x-github-api-version" => ["2022-11-28"] + }, + registered_options: MapSet.new([:auth, :redirect]), + options: %{auth: {:bearer, ""}, redirect: true}, + current_request_steps: [:auth], + request_steps: [auth: &Req.Steps.auth/1], + response_steps: [redirect: &Req.Steps.redirect/1] + } + end + + test "compressed" do + assert ~CURL(curl --compressed http://example.com) == + %Req.Request{ + url: URI.parse("http://example.com"), + body: nil, + registered_options: MapSet.new([:compressed]), + options: %{compressed: true}, + current_request_steps: [:compressed], + request_steps: [compressed: &Req.Steps.compressed/1] + } + end + + test "redirect" do + assert ~CURL(curl -L http://example.com) == + %Req.Request{ + url: URI.parse("http://example.com"), + registered_options: MapSet.new([:redirect]), + options: %{redirect: true}, + response_steps: [redirect: &Req.Steps.redirect/1] + } + end + + test "cookie, formadata, auth and redirect" do + assert ~CURL(curl -L -u user:pass -F name=foo -b name=bar http://example.com) == + %Req.Request{ + url: URI.parse("http://example.com"), + headers: %{"cookie" => ["name=bar"]}, + current_request_steps: [:auth, :encode_body], + registered_options: MapSet.new([:redirect, :auth, :form]), + options: %{redirect: true, auth: {:basic, "user:pass"}, form: %{"name" => "foo"}}, + request_steps: [auth: &Req.Steps.auth/1, encode_body: &Req.Steps.encode_body/1], + response_steps: [redirect: &Req.Steps.redirect/1] + } + end + + test "proxy" do + assert ~CURL(curl --proxy my.proxy.com:22225 http://example.com) == + %Req.Request{ + url: URI.parse("http://example.com"), + registered_options: MapSet.new([:connect_options]), + options: %{ + connect_options: [proxy: {:http, "my.proxy.com", 22225, []}] + } + } + end + + test "proxy with basic auth" do + assert ~CURL(curl --proxy https://my.proxy.com:22225 --proxy-user foo:bar http://example.com) == + %Req.Request{ + url: URI.parse("http://example.com"), + registered_options: MapSet.new([:connect_options]), + options: %{ + connect_options: [ + proxy: {:https, "my.proxy.com", 22225, []}, + proxy_headers: [ + {"proxy-authorization", "Basic " <> Base.encode64("foo:bar")} + ] + ] + } + } + end + + test "proxy with inline basic auth" do + assert ~CURL(curl --proxy https://foo:bar@my.proxy.com:22225 http://example.com) == + %Req.Request{ + url: URI.parse("http://example.com"), + registered_options: MapSet.new([:connect_options]), + options: %{ + connect_options: [ + proxy: {:https, "my.proxy.com", 22225, []}, + proxy_headers: [ + {"proxy-authorization", "Basic " <> Base.encode64("foo:bar")} + ] + ] + } + } + end + + test "proxy raises on non http scheme uri" do + assert_raise( + ArgumentError, + "Unsupported scheme ssh for proxy in ssh://my.proxy.com:22225", + fn -> + CurlReq.Curl.decode("curl --proxy ssh://my.proxy.com:22225 http://example.com") + end + ) + end + end + + describe "newlines" do + test "sigil_CURL supports newlines" do + curl = ~CURL""" + curl -X POST \ + --location \ + https://example.com + """ + + assert curl == %Req.Request{ + method: :post, + url: URI.parse("https://example.com"), + registered_options: MapSet.new([:redirect]), + options: %{redirect: true}, + response_steps: [redirect: &Req.Steps.redirect/1] + } + end + + test "from_curl supports newlines" do + curl = + from_curl(""" + curl -X POST \ + --location \ + https://example.com + """) + + assert curl == %Req.Request{ + method: :post, + url: URI.parse("https://example.com"), + registered_options: MapSet.new([:redirect]), + options: %{redirect: true}, + response_steps: [redirect: &Req.Steps.redirect/1] + } + end + + test "accepts newlines ending in backslash" do + uri = URI.parse("https://example.com/api/2024-07/graphql.json") + + assert %Req.Request{ + method: :post, + url: ^uri, + options: %{json: %{"query" => _}} + } = ~CURL""" + curl -X POST \ + https://example.com/api/2024-07/graphql.json \ + -H 'Content-Type: application/json' \ + -H 'X-Shopify-Storefront-Access-Token: ABCDEF' \ + -d '{ + "query": "{ + products(first: 3) { + edges { + node { + id + title + } + } + } + }" + }' + """ + + assert %Req.Request{ + method: :post, + url: ^uri, + options: %{json: %{"query" => _}} + } = ~CURL""" + curl -X POST + https://example.com/api/2024-07/graphql.json + -H 'Content-Type: application/json' + -H 'X-Shopify-Storefront-Access-Token: ABCDEF' + -d '{ + "query": "{ + products(first: 3) { + edges { + node { + id + title + } + } + } + }" + }' + """ + end + + test "raises on unsupported flag" do + assert_raise ArgumentError, ~r/Unknown "--foo"/, fn -> + CurlReq.Curl.decode(~s(curl --foo https://example.com)) + end + end + end end From ece1b5f73296c78af911ed781236fc4091b650ff Mon Sep 17 00:00:00 2001 From: Kevin Schweikert Date: Tue, 24 Dec 2024 16:45:27 +0100 Subject: [PATCH 3/8] Implement missing features --- lib/curl_req.ex | 11 ++-- lib/curl_req/curl.ex | 109 ++++++++++++++++++++++------------------ lib/curl_req/request.ex | 4 +- lib/curl_req/shell.ex | 33 ++++++++++-- 4 files changed, 99 insertions(+), 58 deletions(-) diff --git a/lib/curl_req.ex b/lib/curl_req.ex index 36e217e..c937cd7 100644 --- a/lib/curl_req.ex +++ b/lib/curl_req.ex @@ -85,6 +85,7 @@ defmodule CurlReq do * `-u`/`--user` * `-n`/`--netrc` * `--netrc-file` + * `--compressed` Options: @@ -124,15 +125,17 @@ defmodule CurlReq do options = Keyword.validate!(options, flags: :short, run_steps: true, flavor: nil, flavour: :curl) - # flavor = opts[:flavor] || opts[:flavour] - # flag_style = opts[:flags] + flavor = options[:flavor] || options[:flavour] + flags = options[:flags] run_steps = options[:run_steps] available_steps = step_names(req, run_steps) req = run_steps(req, available_steps) + curl_options = [flavor: flavor, flags: flags] + CurlReq.Req.decode(req) - |> CurlReq.Curl.encode(options) + |> CurlReq.Curl.encode(curl_options) end @doc """ @@ -150,6 +153,8 @@ defmodule CurlReq do * `-u`/`--user` * `-x`/`--proxy` * `-U`/`--proxy-user` + * `-n`/`--netrc` + * `--netrc_file` * `--compressed` The `curl` command prefix is optional diff --git a/lib/curl_req/curl.ex b/lib/curl_req/curl.ex index c62f0f7..ba5f595 100644 --- a/lib/curl_req/curl.ex +++ b/lib/curl_req/curl.ex @@ -9,8 +9,7 @@ defmodule CurlReq.Curl do command |> String.trim() |> String.trim_leading("curl") - |> String.replace("\\\n", " ") - |> String.replace("\n", " ") + |> CurlReq.Shell.remove_newlines() {options, rest, invalid} = command @@ -243,9 +242,11 @@ defmodule CurlReq.Curl do @impl CurlReq.Request @spec encode(CurlReq.Request.t(), Keyword.t()) :: String.t() def encode(%CurlReq.Request{} = request, options \\ []) do - flag_style = Keyword.get(options, :flags, :short) - # TODO: implement flavor - _flavor = Keyword.get(options, :flavor, nil) || Keyword.get(options, :flavour, :curl) + options = + Keyword.validate!(options, flags: :short, flavor: :curl) + + flag_style = options[:flags] + flavor = options[:flavor] cookies = if map_size(request.cookies) != 0 do @@ -276,7 +277,11 @@ defmodule CurlReq.Curl do headers ++ [header_flag(flag_style, "content-type: application/x-www-form-urlencoded")] end - headers = Enum.intersperse(headers, " ") + headers = + case flavor do + :curl -> headers + :req -> headers ++ header_flag(flag_style, ["user-agent: req/", CurlReq.req_version()]) + end body = emit_if(request.body, fn -> @@ -287,7 +292,19 @@ defmodule CurlReq.Curl do end) redirect = emit_if(request.redirect, [location_flag(flag_style)]) - compressed = emit_if(request.compression != :none, [compressed_flag(flag_style)]) + + compressed = + emit_if(request.compression != :none, fn -> + case flavor do + :curl -> + [compressed_flag(flag_style)] + + :req -> + [ + header_flag(flag_style, ["accept-encoding: ", Atom.to_string(request.compression)]) + ] + end + end) auth = case request.auth do @@ -327,25 +344,21 @@ defmodule CurlReq.Curl do _ -> [] end - url = [to_string(request.url)] - - IO.iodata_to_binary( - [ - "curl", - compressed, - auth, - headers, - cookies, - body, - proxy, - proxy_auth, - redirect, - method, - url - ] - |> Enum.reject(fn part -> part == [] end) - |> Enum.intersperse(" ") - ) + url = [" ", to_string(request.url)] + + IO.iodata_to_binary([ + "curl", + compressed, + auth, + headers, + cookies, + body, + proxy, + proxy_auth, + redirect, + method, + url + ]) end defp emit_if(bool, fun) when is_function(fun) do @@ -364,37 +377,37 @@ defmodule CurlReq.Curl do CurlReq.Shell.escape(value) end - defp cookie_flag(:short, value), do: ["-b ", escape(value)] - defp cookie_flag(:long, value), do: ["--cookie ", escape(value)] + defp cookie_flag(:short, value), do: [" -b ", escape(value)] + defp cookie_flag(:long, value), do: [" --cookie ", escape(value)] - defp header_flag(:short, value), do: ["-H ", escape(value)] - defp header_flag(:long, value), do: ["--header ", escape(value)] + defp header_flag(:short, value), do: [" -H ", escape(value)] + defp header_flag(:long, value), do: [" --header ", escape(value)] - defp data_flag(:short, value), do: ["-d ", escape(value)] - defp data_flag(:long, value), do: ["--data ", escape(value)] + defp data_flag(:short, value), do: [" -d ", escape(value)] + defp data_flag(:long, value), do: [" --data ", escape(value)] - defp head_flag(:short), do: "-I" - defp head_flag(:long), do: "--head" + defp head_flag(:short), do: " -I" + defp head_flag(:long), do: " --head" - defp request_flag(:short, value), do: ["-X ", escape(value)] - defp request_flag(:long, value), do: ["--request ", escape(value)] + defp request_flag(:short, value), do: [" -X ", escape(value)] + defp request_flag(:long, value), do: [" --request ", escape(value)] - defp location_flag(:short), do: "-L" - defp location_flag(:long), do: "--location" + defp location_flag(:short), do: " -L" + defp location_flag(:long), do: " --location" - defp user_flag(:short, value), do: ["-u ", escape(value)] - defp user_flag(:long, value), do: ["--user ", escape(value)] + defp user_flag(:short, value), do: [" -u ", escape(value)] + defp user_flag(:long, value), do: [" --user ", escape(value)] - defp netrc_flag(:short), do: "-n" - defp netrc_flag(:long), do: "--netrc" + defp netrc_flag(:short), do: " -n" + defp netrc_flag(:long), do: " --netrc" - defp netrc_file_flag(_, value), do: ["--netrc-file ", escape(value)] + defp netrc_file_flag(_, value), do: [" --netrc-file ", escape(value)] - defp compressed_flag(_), do: "--compressed" + defp compressed_flag(_), do: " --compressed" - defp proxy_flag(:short, value), do: ["-x ", escape(value)] - defp proxy_flag(:long, value), do: ["--proxy ", escape(value)] + defp proxy_flag(:short, value), do: [" -x ", escape(value)] + defp proxy_flag(:long, value), do: [" --proxy ", escape(value)] - defp proxy_user_flag(:short, value), do: ["-U ", escape(value)] - defp proxy_user_flag(:long, value), do: ["--proxy-user ", escape(value)] + defp proxy_user_flag(:short, value), do: [" -U ", escape(value)] + defp proxy_user_flag(:long, value), do: [" --proxy-user ", escape(value)] end diff --git a/lib/curl_req/request.ex b/lib/curl_req/request.ex index 3c0ff93..7f3a498 100644 --- a/lib/curl_req/request.ex +++ b/lib/curl_req/request.ex @@ -286,8 +286,8 @@ defmodule CurlReq.Request do ## Examples iex> request = %CurlReq.Request{} |> CurlReq.Request.put_proxy("https://example.com") - iex> request.url - URI.new!("https://example.com") + iex> request.proxy_url + URI.parse("https://example.com") """ @spec put_proxy(__MODULE__.t(), URI.t() | String.t()) :: __MODULE__.t() def put_proxy(%__MODULE__{} = request, uri) do diff --git a/lib/curl_req/shell.ex b/lib/curl_req/shell.ex index 7a9a056..ad44ee0 100644 --- a/lib/curl_req/shell.ex +++ b/lib/curl_req/shell.ex @@ -15,12 +15,15 @@ defmodule CurlReq.Shell do @no_quotes ~r/^[a-zA-Z-,._+:@%\/]*$/ @doc ~S""" - Examples: - iex> CurlReq.Shell.escape(~s(abc def)) - ~s("abc def") + Escapes the value into a shell representable version - iex> CurlReq.Shell.escape(~s({"json":"is_cool"})) - ~S("{\"json\":\"is_cool\"}") + ## Examples + + iex> CurlReq.Shell.escape(~s(abc def)) + ~s("abc def") + + iex> CurlReq.Shell.escape(~s({"json":"is_cool"})) + ~S("{\"json\":\"is_cool\"}") """ def escape(arg) do if String.match?(arg, @no_quotes) do @@ -34,4 +37,24 @@ defmodule CurlReq.Shell do ~s("#{arg}") end end + + @doc """ + Removes newlines from the input string + + ## Examples + + iex> \""" + ...> curl + ...> -X GET example.com + ...> \""" + ...> |> CurlReq.Shell.remove_newlines() + "curl -X GET example.com" + """ + def remove_newlines(input) when is_binary(input) do + input + |> String.replace("\\\n", " ") + |> String.replace("\n", " ") + |> String.replace("\r", "") + |> String.trim() + end end From 24aae34c642c6e708f921ee740960e6b5cab4a78 Mon Sep 17 00:00:00 2001 From: Kevin Schweikert Date: Tue, 24 Dec 2024 16:54:51 +0100 Subject: [PATCH 4/8] update CHANGELOG --- CHANGELOG.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba34725..8f30fb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,17 @@ # Changelog -## 0.99.1 +## 0.100.0 + +- Switch some flag positions in the generated cURL command +- Some bugfixes regarding the constructed Req.Request struct when multiple request steps have to be set +- [BREAKING]: From cURL to Req the body gets encoded in the specified encoding and set in the correct Req option ## 0.99.0 - Add new supported flags: `--proxy` and `--proxy-user` ([#26](https://github.com/derekkraan/curl_req/pull/26)) - Add more supported auth steps: `netrc` and `netrc_file` ([#19](https://github.com/derekkraan/curl_req/pull/19)) - Add option to exclude `req` steps to run when generating the cURL command -- [BREAKING] Raise on unrecognized `curl` flags ([#27](https://github.com/derekkraan/curl_req/pull/27)) +- Raise on unrecognized `curl` flags ([#27](https://github.com/derekkraan/curl_req/pull/27)) ## 0.98.6 - Handle `--data-raw` and `--data-ascii` ([#16](https://github.com/derekkraan/curl_req/pull/16)) From 9af1da6d81bbd08fced0eab3d0ed4117b620bfeb Mon Sep 17 00:00:00 2001 From: Kevin Schweikert Date: Tue, 24 Dec 2024 16:55:03 +0100 Subject: [PATCH 5/8] simplify auth helper function --- lib/curl_req/curl.ex | 4 ++-- lib/curl_req/request.ex | 15 --------------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/lib/curl_req/curl.ex b/lib/curl_req/curl.ex index ba5f595..962a422 100644 --- a/lib/curl_req/curl.ex +++ b/lib/curl_req/curl.ex @@ -169,7 +169,7 @@ defmodule CurlReq.Curl do request userinfo -> - CurlReq.Request.put_auth(request, :basic, userinfo) + CurlReq.Request.put_auth(request, {:basic, userinfo}) end request = @@ -186,7 +186,7 @@ defmodule CurlReq.Curl do request path -> - CurlReq.Request.put_auth(request, :netrc, path) + CurlReq.Request.put_auth(request, {:netrc, path}) end end diff --git a/lib/curl_req/request.ex b/lib/curl_req/request.ex index 7f3a498..c14e823 100644 --- a/lib/curl_req/request.ex +++ b/lib/curl_req/request.ex @@ -228,21 +228,6 @@ defmodule CurlReq.Request do %{request | auth: {type, credentials}} end - @doc """ - Puts authorization into the CurlReq.Request struct - - ## Examples - - iex> request = %CurlReq.Request{} |> CurlReq.Request.put_auth(:bearer, "foobar") - iex> request.auth - {:bearer, "foobar"} - """ - @spec put_auth(__MODULE__.t(), :bearer | :basic | :netrc, String.t()) :: __MODULE__.t() - def put_auth(%__MODULE__{} = request, type, credentials) - when type in [:netrc, :basic, :bearer] do - %{request | auth: {type, credentials}} - end - @doc """ Puts the url into the CurlReq.Request struct, It either accepts a binary or an URI struct From 646114cd01a5ed0ebbe04b75e9303da7cb70238d Mon Sep 17 00:00:00 2001 From: Kevin Schweikert Date: Tue, 24 Dec 2024 17:12:32 +0100 Subject: [PATCH 6/8] simplify some function calls --- lib/curl_req/curl.ex | 51 ++++++----------------------------------- lib/curl_req/request.ex | 6 +++++ 2 files changed, 13 insertions(+), 44 deletions(-) diff --git a/lib/curl_req/curl.ex b/lib/curl_req/curl.ex index 962a422..468e180 100644 --- a/lib/curl_req/curl.ex +++ b/lib/curl_req/curl.ex @@ -99,14 +99,7 @@ defmodule CurlReq.Curl do end defp add_method(request, options) do - method = - if Keyword.get(options, :head, false) do - :head - else - Keyword.get(options, :request, "GET") - end - - CurlReq.Request.put_method(request, method) + CurlReq.Request.put_method(request, (options[:head] && :head) || options[:request]) end defp add_body(request, options) do @@ -163,41 +156,14 @@ defmodule CurlReq.Curl do end defp add_auth(request, options) do - request = - case Keyword.get(options, :user) do - nil -> - request - - userinfo -> - CurlReq.Request.put_auth(request, {:basic, userinfo}) - end - - request = - case Keyword.get(options, :netrc) do - nil -> - request - - _ -> - CurlReq.Request.put_auth(request, :netrc) - end - - case Keyword.get(options, :netrc_file) do - nil -> - request - - path -> - CurlReq.Request.put_auth(request, {:netrc, path}) - end + request + |> CurlReq.Request.put_auth({:basic, options[:user]}) + |> CurlReq.Request.put_auth(options[:netrct]) + |> CurlReq.Request.put_auth({:netrc, options[:netrc_file]}) end defp add_compression(request, options) do - case Keyword.get(options, :compressed) do - nil -> - request - - bool -> - CurlReq.Request.put_compression(request, bool) - end + CurlReq.Request.put_compression(request, options[:compressed]) end defp add_proxy(request, options) do @@ -233,10 +199,7 @@ defmodule CurlReq.Curl do end defp configure_redirects(request, options) do - case Keyword.get(options, :location) do - nil -> request - bool -> CurlReq.Request.put_redirect(request, bool) - end + CurlReq.Request.put_redirect(request, options[:location]) end @impl CurlReq.Request diff --git a/lib/curl_req/request.ex b/lib/curl_req/request.ex index c14e823..63fe4ef 100644 --- a/lib/curl_req/request.ex +++ b/lib/curl_req/request.ex @@ -223,6 +223,10 @@ defmodule CurlReq.Request do %{request | auth: :netrc} end + def put_auth(%__MODULE__{} = request, {_type, nil}) do + request + end + def put_auth(%__MODULE__{} = request, {type, credentials}) when type in [:netrc, :basic, :bearer] do %{request | auth: {type, credentials}} @@ -310,6 +314,8 @@ defmodule CurlReq.Request do :post """ @spec put_method(__MODULE__.t(), method() | String.t()) :: __MODULE__.t() + def put_method(%__MODULE__{} = request, nil), do: request + def put_method(%__MODULE__{} = request, method) when method in [:get, :head, :put, :post, :delete, :patch] do %{request | method: method} From 74ea4b6e4b8b9a547064841dcc3b07e237be954c Mon Sep 17 00:00:00 2001 From: Kevin Schweikert Date: Tue, 24 Dec 2024 17:13:46 +0100 Subject: [PATCH 7/8] add docs --- lib/curl_req/curl.ex | 5 ++++- lib/curl_req/req.ex | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/curl_req/curl.ex b/lib/curl_req/curl.ex index 468e180..065be7b 100644 --- a/lib/curl_req/curl.ex +++ b/lib/curl_req/curl.ex @@ -1,5 +1,8 @@ defmodule CurlReq.Curl do - # TODO: docs + @moduledoc """ + Implements the CurlReq.Request behaviour for a cURL command string + """ + @behaviour CurlReq.Request @impl CurlReq.Request diff --git a/lib/curl_req/req.ex b/lib/curl_req/req.ex index 6ea5582..90c598c 100644 --- a/lib/curl_req/req.ex +++ b/lib/curl_req/req.ex @@ -1,5 +1,8 @@ defmodule CurlReq.Req do - # TODO: docs + @moduledoc """ + Implements the CurlReq.Request behaviour for a Req.Request struct + """ + @behaviour CurlReq.Request @impl CurlReq.Request From 4aaae6a3342c5be4e4047833d17ead5dd3afc3d8 Mon Sep 17 00:00:00 2001 From: Kevin Schweikert Date: Wed, 25 Dec 2024 18:29:06 +0100 Subject: [PATCH 8/8] save raw body before to undo encoding --- lib/curl_req/request.ex | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/lib/curl_req/request.ex b/lib/curl_req/request.ex index 63fe4ef..68b7e6b 100644 --- a/lib/curl_req/request.ex +++ b/lib/curl_req/request.ex @@ -22,7 +22,8 @@ defmodule CurlReq.Request do proxy_auth: auth(), auth: auth(), encoding: encoding(), - body: term() + body: term(), + raw_body: term() } @type user_agent() :: :curl | :req | String.t() @@ -47,7 +48,8 @@ defmodule CurlReq.Request do proxy_auth: :none, auth: :none, encoding: :raw, - body: nil + body: nil, + raw_body: nil @doc """ Puts the header into the CurlReq.Request struct. Special headers like encoding, authorization or user-agent are stored in their respective field in the #{__MODULE__} struct instead of a general header. @@ -148,22 +150,22 @@ defmodule CurlReq.Request do @spec put_body(__MODULE__.t(), term()) :: __MODULE__.t() def put_body(%__MODULE__{} = request, nil), do: request - def put_body(%__MODULE__{} = request, body) do + def put_body(%__MODULE__{} = request, input) do body = case request.encoding do :json -> - with true <- is_binary(body) or is_list(body), - {:ok, json} <- Jason.decode(body) do + with true <- is_binary(input) or is_list(input), + {:ok, json} <- Jason.decode(input) do json else - _ -> body + _ -> input end _ -> - body + input end - %{request | body: body} + %{request | body: body, raw_body: input} end @doc """ @@ -183,22 +185,33 @@ defmodule CurlReq.Request do iex> request = %CurlReq.Request{} |> CurlReq.Request.put_encoding(:raw) iex> request.encoding :raw + + iex> request = %CurlReq.Request{} + ...> |> CurlReq.Request.put_body(~S|{"foo": "bar"}|) + ...> |> CurlReq.Request.put_encoding(:json) + iex> request.encoding + :json + iex> request.body + %{"foo" => "bar"} + iex> request = request |> CurlReq.Request.put_encoding(:raw) + iex> request.body + ~S|{"foo": "bar"}| """ @spec put_encoding(__MODULE__.t(), encoding()) :: __MODULE__.t() - def put_encoding(%__MODULE__{body: body} = request, encoding) + def put_encoding(%__MODULE__{raw_body: raw_body} = request, encoding) when encoding in [:raw, :json, :form] do body = case encoding do :json -> - with true <- is_binary(body) or is_list(body), - {:ok, json} <- Jason.decode(body) do + with true <- is_binary(raw_body) or is_list(raw_body), + {:ok, json} <- Jason.decode(raw_body) do json else - _ -> body + _ -> raw_body end _ -> - body + raw_body end %{request | body: body, encoding: encoding}