From 03f450042a6f68bf74ca1550e558cbc009a8315d Mon Sep 17 00:00:00 2001 From: Kevin Schweikert Date: Tue, 10 Dec 2024 01:50:23 +0100 Subject: [PATCH 1/4] fix typo --- lib/curl_req/macro.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/curl_req/macro.ex b/lib/curl_req/macro.ex index 4abe676..9203f3b 100644 --- a/lib/curl_req/macro.ex +++ b/lib/curl_req/macro.ex @@ -50,7 +50,7 @@ defmodule CurlReq.Macro do %Req.Request{} # Req would accept an URI struct but here we use to_string/1 because Req uses URI.parse/1 which sets the deprecated `authority` field which upsets the test assertions. - # Can be removed the Req uses URI.new/1 + # Can be removed when Req uses URI.new/1 |> Req.merge(url: URI.to_string(url)) |> add_header(options) |> add_method(options) From f71046c84b45a8f48615bcef422ca2014f4e3766 Mon Sep 17 00:00:00 2001 From: Kevin Schweikert Date: Tue, 10 Dec 2024 02:28:48 +0100 Subject: [PATCH 2/4] add proxy handling --- lib/curl_req.ex | 28 +++++++++++++++++ lib/curl_req/macro.ex | 59 ++++++++++++++++++++++++++++++++++-- test/curl_req/macro_test.exs | 37 ++++++++++++++++++++++ test/curl_req_test.exs | 23 ++++++++++++++ 4 files changed, 145 insertions(+), 2 deletions(-) diff --git a/lib/curl_req.ex b/lib/curl_req.ex index e9d53c1..6bf83be 100644 --- a/lib/curl_req.ex +++ b/lib/curl_req.ex @@ -58,6 +58,8 @@ defmodule CurlReq do * `-I`/`--head` * `-d`/`--data`/`--data-ascii` * `--data-raw` + * `-x`/`--proxy` + * `-U`/`--proxy-user` Options: @@ -127,6 +129,24 @@ defmodule CurlReq do %{auth: {:basic, credentials}} -> [user_flag(flag_style), credentials] ++ [basic_auth_flag()] + %{connect_options: connect_options} -> + proxy = + case Keyword.get(connect_options, :proxy) do + nil -> + [] + + {scheme, host, port, _} -> + [proxy_flag(flag_style), "#{scheme}://#{host}:#{port}"] + end + + case Keyword.get(connect_options, :proxy_headers) do + [{"proxy-authorization", "Basic " <> encoded_creds}] -> + proxy ++ [proxy_user_flag(flag_style), Base.decode64!(encoded_creds)] + + _ -> + proxy + end + _ -> [] end @@ -188,6 +208,12 @@ defmodule CurlReq do defp compressed_flag(), do: "--compressed" + defp proxy_flag(:short), do: "-x" + defp proxy_flag(:long), do: "--proxy" + + defp proxy_user_flag(:short), do: "-U" + defp proxy_user_flag(:long), do: "--proxy-user" + @doc """ Transforms a curl command into a Req request. @@ -201,6 +227,8 @@ defmodule CurlReq do * `-F`/`--form` * `-L`/`--location` * `-u`/`--user` + * `-x`/`--proxy` + * `-U`/`--proxy-user` * `--compressed` The `curl` command prefix is optional diff --git a/lib/curl_req/macro.ex b/lib/curl_req/macro.ex index 9203f3b..be6d05c 100644 --- a/lib/curl_req/macro.ex +++ b/lib/curl_req/macro.ex @@ -25,7 +25,9 @@ defmodule CurlReq.Macro do form: :keep, location: :boolean, user: :string, - compressed: :boolean + compressed: :boolean, + proxy: :string, + proxy_user: :string ], aliases: [ H: :header, @@ -35,7 +37,9 @@ defmodule CurlReq.Macro do I: :head, F: :form, L: :location, - u: :user + u: :user, + x: :proxy, + U: :proxy_user ] ) @@ -59,6 +63,7 @@ defmodule CurlReq.Macro do |> add_form(options) |> add_auth(options) |> add_compression(options) + |> add_proxy(options) |> configure_redirects(options) end @@ -169,6 +174,56 @@ defmodule CurlReq.Macro do end end + defp add_proxy(req, options) do + with proxy when is_binary(proxy) <- Keyword.get(options, :proxy), + %URI{scheme: scheme, port: port, host: host} when scheme in ["http", "https"] <- + validate_proxy_uri(proxy) do + connect_options = + [ + proxy: {String.to_existing_atom(scheme), host, port, []} + ] + |> maybe_add_proxy_auth(options) + + req + |> Req.Request.register_options([ + :connect_options + ]) + |> Req.merge(connect_options: connect_options) + else + _ -> req + end + end + + defp validate_proxy_uri("http://" <> _rest = uri), do: URI.parse(uri) + defp validate_proxy_uri("https://" <> _rest = uri), do: URI.parse(uri) + + defp validate_proxy_uri(uri) do + case String.split(uri, "://") do + [scheme, _uri] -> + raise ArgumentError, "Unsupported scheme #{scheme} for proxy in #{uri}" + + [uri] -> + URI.parse("http://" <> uri) + end + end + + defp maybe_add_proxy_auth(connect_options, options) do + proxy_headers = + case Keyword.get(options, :proxy_user) do + nil -> + [] + + credentials -> + [ + proxy_headers: [ + {"proxy-authorization", "Basic " <> Base.encode64(credentials)} + ] + ] + end + + Keyword.merge(connect_options, proxy_headers) + end + defp configure_redirects(req, options) do if Keyword.get(options, :location, false) do req diff --git a/test/curl_req/macro_test.exs b/test/curl_req/macro_test.exs index c792555..d716d7d 100644 --- a/test/curl_req/macro_test.exs +++ b/test/curl_req/macro_test.exs @@ -205,6 +205,43 @@ defmodule CurlReq.MacroTest do response_steps: [redirect: &Req.Steps.redirect/1] } end + + test "proxy" do + assert ~CURL(curl --proxy my.proxy.com:22225 http://example.com) == + %Req.Request{ + url: URI.parse("http://example.com"), + registered_options: MapSet.new([:connect_options]), + options: %{ + connect_options: [proxy: {:http, "my.proxy.com", 22225, []}] + } + } + end + + test "proxy with basic auth" do + assert ~CURL(curl --proxy https://my.proxy.com:22225 --proxy-user foo:bar http://example.com) == + %Req.Request{ + url: URI.parse("http://example.com"), + registered_options: MapSet.new([:connect_options]), + options: %{ + connect_options: [ + proxy: {:https, "my.proxy.com", 22225, []}, + proxy_headers: [ + {"proxy-authorization", "Basic " <> Base.encode64("foo:bar")} + ] + ] + } + } + end + + test "proxy raises on non http scheme uri" do + assert_raise( + ArgumentError, + "Unsupported scheme ssh for proxy in ssh://my.proxy.com:22225", + fn -> + CurlReq.Macro.parse("curl --proxy ssh://my.proxy.com:22225 http://example.com") + end + ) + end end describe "newlines" do diff --git a/test/curl_req_test.exs b/test/curl_req_test.exs index 83e5b6a..ce34e26 100644 --- a/test/curl_req_test.exs +++ b/test/curl_req_test.exs @@ -90,5 +90,28 @@ defmodule CurlReqTest do Req.new(url: "https://example.com", auth: {:basic, "user:pass"}) |> CurlReq.to_curl() end + + test "proxy" do + assert ~S(curl --compressed -x "http://my.proxy.com:80" -X GET https://example.com) == + Req.new( + url: "https://example.com", + connect_options: [proxy: {:http, "my.proxy.com", 80, []}] + ) + |> CurlReq.to_curl() + end + + test "proxy user" do + assert ~S(curl --compressed -x "http://my.proxy.com:80" -U foo:bar -X GET https://example.com) == + Req.new( + url: "https://example.com", + connect_options: [ + proxy: {:http, "my.proxy.com", 80, []}, + proxy_headers: [ + {"proxy-authorization", "Basic " <> Base.encode64("foo:bar")} + ] + ] + ) + |> CurlReq.to_curl() + end end end From d2856893705906e8f30e432f170284cf48ca8935 Mon Sep 17 00:00:00 2001 From: Kevin Schweikert Date: Tue, 10 Dec 2024 02:30:41 +0100 Subject: [PATCH 3/4] add changes to changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c32cc77..4331ede 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.98.7 + +- Add new supported flags: `--proxy` and `--proxy-user` + ## 0.98.6 - Handle `--data-raw` and `--data-ascii` ([#16](https://github.com/derekkraan/curl_req/pull/16)) - Strip `$` as necessary From 5846656a73f649cc4ed6335cd9c25f18ffb2d3b5 Mon Sep 17 00:00:00 2001 From: Kevin Schweikert Date: Tue, 10 Dec 2024 10:02:12 +0100 Subject: [PATCH 4/4] support inline basic auth for proxy --- lib/curl_req/macro.ex | 17 ++++++++++++++--- test/curl_req/macro_test.exs | 16 ++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/lib/curl_req/macro.ex b/lib/curl_req/macro.ex index be6d05c..4e2f92f 100644 --- a/lib/curl_req/macro.ex +++ b/lib/curl_req/macro.ex @@ -176,13 +176,14 @@ defmodule CurlReq.Macro do defp add_proxy(req, options) do with proxy when is_binary(proxy) <- Keyword.get(options, :proxy), - %URI{scheme: scheme, port: port, host: host} when scheme in ["http", "https"] <- + %URI{scheme: scheme, port: port, host: host, userinfo: userinfo} + when scheme in ["http", "https"] <- validate_proxy_uri(proxy) do connect_options = [ proxy: {String.to_existing_atom(scheme), host, port, []} ] - |> maybe_add_proxy_auth(options) + |> maybe_add_proxy_auth(options, userinfo) req |> Req.Request.register_options([ @@ -207,7 +208,7 @@ defmodule CurlReq.Macro do end end - defp maybe_add_proxy_auth(connect_options, options) do + defp maybe_add_proxy_auth(connect_options, options, nil) do proxy_headers = case Keyword.get(options, :proxy_user) do nil -> @@ -224,6 +225,16 @@ defmodule CurlReq.Macro do Keyword.merge(connect_options, proxy_headers) end + defp maybe_add_proxy_auth(connect_options, _options, userinfo) do + proxy_headers = [ + proxy_headers: [ + {"proxy-authorization", "Basic " <> Base.encode64(userinfo)} + ] + ] + + Keyword.merge(connect_options, proxy_headers) + end + defp configure_redirects(req, options) do if Keyword.get(options, :location, false) do req diff --git a/test/curl_req/macro_test.exs b/test/curl_req/macro_test.exs index d716d7d..800ec64 100644 --- a/test/curl_req/macro_test.exs +++ b/test/curl_req/macro_test.exs @@ -233,6 +233,22 @@ defmodule CurlReq.MacroTest do } end + test "proxy with inline basic auth" do + assert ~CURL(curl --proxy https://foo:bar@my.proxy.com:22225 http://example.com) == + %Req.Request{ + url: URI.parse("http://example.com"), + registered_options: MapSet.new([:connect_options]), + options: %{ + connect_options: [ + proxy: {:https, "my.proxy.com", 22225, []}, + proxy_headers: [ + {"proxy-authorization", "Basic " <> Base.encode64("foo:bar")} + ] + ] + } + } + end + test "proxy raises on non http scheme uri" do assert_raise( ArgumentError,