This commit is contained in:
2026-07-14 22:50:46 +04:00
parent 27143319e3
commit bd19e0682b
3116 changed files with 467189 additions and 0 deletions

1392
phoenix/deps/req/lib/req.ex Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,20 @@
defmodule Req.Application do
@moduledoc false
use Application
@impl true
def start(_type, _args) do
children = [
{Finch,
name: Req.Finch,
pools: %{
default: Req.Finch.pool_options(%{})
}},
{DynamicSupervisor, strategy: :one_for_one, name: Req.FinchSupervisor},
{Req.Test.Ownership, name: Req.Test.Ownership}
]
Supervisor.start_link(children, strategy: :one_for_one)
end
end

View File

@@ -0,0 +1,22 @@
defmodule Req.ArchiveError do
@moduledoc """
Represents an error when unpacking archives fails, returned by `Req.Steps.decode_body/1`.
"""
defexception [:format, :data, :reason]
@impl true
def message(%{format: :tar, reason: reason}) do
"tar unpacking failed: #{:erl_tar.format_error(reason)}"
end
@impl true
def message(%{format: format, reason: nil}) do
"#{format} unpacking failed"
end
@impl true
def message(%{format: format, reason: reason}) do
"#{format} unpacking failed: #{inspect(reason)}"
end
end

View File

@@ -0,0 +1,16 @@
defmodule Req.ChecksumMismatchError do
@moduledoc """
Represents a checksum mismatch error returned by `Req.Steps.checksum/1`.
"""
defexception [:expected, :actual]
@impl true
def message(%{expected: expected, actual: actual}) do
"""
checksum mismatch
expected: #{expected}
actual: #{actual}\
"""
end
end

View File

@@ -0,0 +1,17 @@
defmodule Req.DecompressError do
@moduledoc """
Represents an error when decompression fails, returned by `Req.Steps.decompress_body/1`.
"""
defexception [:format, :data, :reason]
@impl true
def message(%{format: format, reason: nil}) do
"#{format} decompression failed"
end
@impl true
def message(%{format: format, reason: reason}) do
"#{format} decompression failed, reason: #{inspect(reason)}"
end
end

View File

@@ -0,0 +1,233 @@
defmodule Req.Fields do
@moduledoc false
# Conveniences for working with HTTP Fields, i.e. HTTP Headers and HTTP Trailers.
@legacy? Req.MixProject.legacy_headers_as_lists?()
# Legacy behaviour previously used in Req.Request.new.
# I plan to use Req.Fields.new, i.e. normalize.
def new_without_normalize(enumerable)
if @legacy? do
def new_without_normalize(enumerable) do
Enum.to_list(enumerable)
end
else
def new_without_normalize(enumerable) do
Map.new(enumerable, fn {key, value} ->
{key, List.wrap(value)}
end)
end
end
# Legacy behaviour previously used in Req.Response.new.
# I plan to use Req.Fields.new, i.e. normalize.
def new_without_normalize_with_duplicates(enumerable)
if @legacy? do
def new_without_normalize_with_duplicates(enumerable) do
Enum.to_list(enumerable)
end
else
def new_without_normalize_with_duplicates(enumerable) do
Enum.reduce(enumerable, %{}, fn {name, value}, acc ->
Map.update(acc, name, List.wrap(value), &(&1 ++ List.wrap(value)))
end)
end
end
@doc """
Returns fields from a given enumerable.
## Examples
iex> Req.Fields.new(a: 1, b: [1, 2])
%{"a" => ["1"], "b" => ["1", "2"]}
iex> Req.Fields.new(%{"a" => ["1"], "b" => ["1", "2"]})
%{"a" => ["1"], "b" => ["1", "2"]}
"""
if @legacy? do
def new(enumerable) do
for {name, value} <- enumerable do
{normalize_name(name), normalize_value(value)}
end
end
else
def new(enumerable) do
Enum.reduce(enumerable, %{}, fn {name, value}, acc ->
Map.update(
acc,
normalize_name(name),
normalize_values(List.wrap(value)),
&(&1 ++ normalize_values(List.wrap(value)))
)
end)
end
defp normalize_values([value | rest]) do
[normalize_value(value) | normalize_values(rest)]
end
defp normalize_values([]) do
[]
end
end
defp normalize_name(name) when is_atom(name) do
name |> Atom.to_string() |> String.replace("_", "-") |> ensure_name_downcase()
end
defp normalize_name(name) when is_binary(name) do
ensure_name_downcase(name)
end
defp normalize_value(%DateTime{} = datetime) do
datetime |> DateTime.shift_zone!("Etc/UTC") |> Req.Utils.format_http_date()
end
defp normalize_value(%NaiveDateTime{} = datetime) do
IO.warn("setting field to %NaiveDateTime{} is deprecated, use %DateTime{} instead")
Req.Utils.format_http_date(datetime)
end
defp normalize_value(value) when is_binary(value) do
value
end
defp normalize_value(value) when is_integer(value) do
Integer.to_string(value)
end
defp normalize_value(value) do
IO.warn(
"setting field to value other than string, integer, or %DateTime{} is deprecated," <>
" got: #{inspect(value)}"
)
String.Chars.to_string(value)
end
@doc """
Merges `fields1` and `fields2`.
## Examples
iex> Req.Fields.merge(%{"a" => ["1"]}, %{"a" => ["2"], "b" => ["2"]})
%{"a" => ["2"], "b" => ["2"]}
"""
def merge(fields1, fields2)
def merge(old_fields, new_fields) do
if unquote(@legacy?) do
new_fields = new(new_fields)
new_field_names = Enum.map(new_fields, &elem(&1, 0))
Enum.reject(old_fields, &(elem(&1, 0) in new_field_names)) ++ new_fields
else
Map.merge(old_fields, new(new_fields))
end
end
def ensure_name_downcase(name) do
String.downcase(name, :ascii)
end
@doc """
Returns field values.
"""
def get_values(fields, name)
if @legacy? do
def get_values(fields, name) when is_binary(name) do
name = ensure_name_downcase(name)
for {^name, value} <- fields do
value
end
end
else
def get_values(fields, name) when is_binary(name) do
name = ensure_name_downcase(name)
Map.get(fields, name, [])
end
end
@doc """
Adds a new field `name` with the given `value` if not present,
otherwise replaces previous value with `value`.
"""
def put(fields, name, value)
if @legacy? do
def put(fields, name, value) when is_binary(name) and is_binary(value) do
name = ensure_name_downcase(name)
List.keystore(fields, name, 0, {name, value})
end
else
def put(fields, name, value) when is_binary(name) and is_binary(value) do
name = ensure_name_downcase(name)
put_in(fields[name], List.wrap(value))
end
end
@doc """
Adds a field `name` unless already present.
"""
def put_new(fields, name, value)
if @legacy? do
def put_new(fields, name, value) when is_binary(name) and is_binary(value) do
case get_values(fields, name) do
[] ->
put(fields, name, value)
_ ->
fields
end
end
else
def put_new(fields, name, value) when is_binary(name) and is_binary(value) do
name = ensure_name_downcase(name)
Map.put_new(fields, name, List.wrap(value))
end
end
@doc """
Deletes the field given by `name`.
"""
def delete(fields, name)
if @legacy? do
def delete(fields, name) when is_binary(name) do
name_to_delete = ensure_name_downcase(name)
for {name, value} <- fields,
name != name_to_delete do
{name, value}
end
end
else
def delete(fields, name) when is_binary(name) do
name = ensure_name_downcase(name)
Map.delete(fields, name)
end
end
@doc """
Returns fields as list.
"""
def get_list(fields)
if @legacy? do
def get_list(fields) do
fields
end
else
def get_list(fields) do
for {name, values} <- fields,
value <- values do
{name, value}
end
end
end
end

View File

@@ -0,0 +1,435 @@
defmodule Req.Finch do
@moduledoc false
@doc """
Runs the request using `Finch`.
"""
def run(request) do
# URI.parse removes `[` and `]` so we can't check for these. The host
# should not have `:` so it should be safe to check for it.
request =
if !request.options[:inet6] and
(request.url.host || "") =~ ":" do
request = put_in(request.options[:inet6], true)
# ...and have to put them back for host header.
Req.Request.put_new_header(request, "host", "[#{request.url.host}]")
else
request
end
finch_name = finch_name(request)
request_headers =
if unquote(Req.MixProject.legacy_headers_as_lists?()) do
request.headers
else
for {name, values} <- request.headers,
value <- values do
{name, value}
end
end
body =
case request.body do
iodata when is_binary(iodata) or is_list(iodata) ->
iodata
nil ->
nil
enumerable ->
{:stream, enumerable}
end
finch_request =
Finch.build(request.method, request.url, request_headers, body)
|> Map.replace!(:unix_socket, request.options[:unix_socket])
|> add_private_options(request.options[:finch_private])
finch_options =
request.options |> Map.take([:receive_timeout, :pool_timeout]) |> Enum.to_list()
run(request, finch_request, finch_name, finch_options)
end
defp run(req, finch_req, finch_name, finch_options) do
case req.options[:finch_request] do
fun when is_function(fun, 4) ->
fun.(req, finch_req, finch_name, finch_options)
deprecated_fun when is_function(deprecated_fun, 1) ->
IO.warn(
"passing a :finch_request function accepting a single argument is deprecated. " <>
"See Req.Steps.run_finch/1 for more information."
)
{req, run_finch_request(deprecated_fun.(finch_req), finch_name, finch_options)}
nil ->
case req.into do
nil ->
{req, run_finch_request(finch_req, finch_name, finch_options)}
fun when is_function(fun, 2) ->
finch_stream_into_fun(req, finch_req, finch_name, finch_options, fun)
:legacy_self ->
finch_stream_into_legacy_self(req, finch_req, finch_name, finch_options)
:self ->
finch_stream_into_self(req, finch_req, finch_name, finch_options)
collectable ->
finch_stream_into_collectable(req, finch_req, finch_name, finch_options, collectable)
end
end
end
defp finch_stream_into_fun(req, finch_req, finch_name, finch_options, fun) do
resp = Req.Response.new()
fun = fn
{:status, status}, {req, resp} ->
{:cont, {req, %{resp | status: status}}}
{:headers, fields}, {req, resp} ->
resp =
Enum.reduce(fields, resp, fn {name, value}, resp ->
Req.Response.put_header(resp, name, value)
end)
{:cont, {req, resp}}
{:data, data}, acc ->
fun.({:data, data}, acc)
{:trailers, fields}, {req, resp} ->
fields = finch_fields_to_map(fields)
resp = update_in(resp.trailers, &Map.merge(&1, fields))
{:cont, {req, resp}}
end
case Finch.stream_while(finch_req, finch_name, {req, resp}, fun, finch_options) do
{:ok, acc} ->
acc
# TODO: remove when we require Finch 0.20
{:error, exception} ->
{req, normalize_error(exception)}
{:error, exception, _acc} ->
{req, normalize_error(exception)}
end
end
defp finch_stream_into_collectable(req, finch_req, finch_name, finch_options, collectable) do
resp = Req.Response.new()
fun = fn
{:status, 200}, {nil, req, resp} ->
{acc, collector} = Collectable.into(collectable)
{{acc, collector}, req, %{resp | status: 200}}
{:status, status}, {nil, req, resp} ->
{acc, collector} = Collectable.into("")
{{acc, collector}, req, %{resp | status: status}}
{:headers, fields}, {acc, req, resp} ->
resp =
Enum.reduce(fields, resp, fn {name, value}, resp ->
Req.Response.put_header(resp, name, value)
end)
{acc, req, resp}
{:data, data}, {{acc, collector}, req, resp} ->
acc = collector.(acc, {:cont, data})
{{acc, collector}, req, resp}
{:trailers, fields}, {acc, req, resp} ->
fields = finch_fields_to_map(fields)
resp = update_in(resp.trailers, &Map.merge(&1, fields))
{acc, req, resp}
end
case Finch.stream(finch_req, finch_name, {nil, req, resp}, fun, finch_options) do
{:ok, {{acc, collector}, req, resp}} ->
acc = collector.(acc, :done)
{req, %{resp | body: acc}}
# TODO: remove when we require Finch 0.20
{:error, exception} ->
{req, normalize_error(exception)}
{:error, exception, {nil, _req, _resp}} ->
{req, normalize_error(exception)}
{:error, exception, {{acc, collector}, _req, _resp}} ->
collector.(acc, :halt)
{req, normalize_error(exception)}
end
end
defp normalize_error(%Mint.TransportError{reason: reason}) do
%Req.TransportError{reason: reason}
end
defp normalize_error(%Mint.HTTPError{module: Mint.HTTP1, reason: reason}) do
%Req.HTTPError{protocol: :http1, reason: reason}
end
defp normalize_error(%Mint.HTTPError{module: Mint.HTTP2, reason: reason}) do
%Req.HTTPError{protocol: :http2, reason: reason}
end
defp normalize_error(%Finch.Error{reason: reason}) do
%Req.HTTPError{protocol: :http2, reason: reason}
end
defp normalize_error(error) do
error
end
defp finch_stream_into_legacy_self(req, finch_req, finch_name, finch_options) do
ref = Finch.async_request(finch_req, finch_name, finch_options)
{:status, status} =
receive do
{^ref, message} ->
message
end
headers =
receive do
{^ref, message} ->
{:headers, headers} = message
handle_finch_headers(headers)
end
async = %Req.Response.Async{
pid: self(),
ref: ref,
stream_fun: &parse_message/2,
cancel_fun: &cancel/1
}
req = put_in(req.async, async)
resp = Req.Response.new(status: status, headers: headers)
{req, resp}
end
defp finch_stream_into_self(req, finch_req, finch_name, finch_options) do
ref = Finch.async_request(finch_req, finch_name, finch_options)
with {:status, status} <- recv_status(req, ref),
{:headers, headers} <- recv_headers(req, ref) do
# TODO: handle trailers
headers = handle_finch_headers(headers)
async = %Req.Response.Async{
pid: self(),
ref: ref,
stream_fun: &parse_message/2,
cancel_fun: &cancel/1
}
resp = Req.Response.new(status: status, headers: headers, body: async)
{req, resp}
end
end
defp recv_status(req, ref) do
receive do
{^ref, {:status, status}} ->
{:status, status}
{^ref, {:error, exception}} ->
{req, normalize_error(exception)}
end
end
defp recv_headers(req, ref) do
receive do
{^ref, {:headers, headers}} ->
{:headers, headers}
{^ref, {:error, exception}} ->
{req, normalize_error(exception)}
end
end
defp run_finch_request(finch_request, finch_name, finch_options) do
case Finch.request(finch_request, finch_name, finch_options) do
{:ok, response} ->
Req.Response.new(response)
{:error, exception} ->
normalize_error(exception)
end
end
defp add_private_options(finch_request, nil) do
finch_request
end
defp add_private_options(finch_request, private_options)
when is_list(private_options) or is_map(private_options) do
Enum.reduce(private_options, finch_request, fn {k, v}, acc_finch_req ->
Finch.Request.put_private(acc_finch_req, k, v)
end)
end
if Req.MixProject.legacy_headers_as_lists?() do
defp handle_finch_headers(headers), do: headers
else
defp handle_finch_headers(headers), do: finch_fields_to_map(headers)
end
defp finch_fields_to_map(fields) do
Enum.reduce(fields, %{}, fn {name, value}, acc ->
Map.update(acc, name, [value], &(&1 ++ [value]))
end)
end
defp parse_message(ref, {ref, {:data, data}}) do
{:ok, [data: data]}
end
defp parse_message(ref, {ref, :done}) do
{:ok, [:done]}
end
defp parse_message(ref, {ref, {:trailers, trailers}}) do
{:ok, [trailers: trailers]}
end
defp parse_message(ref, {ref, {:error, reason}}) do
{:error, reason}
end
defp parse_message(_, _) do
:unknown
end
defp cancel(ref) do
Finch.cancel_async_request(ref)
clean_responses(ref)
:ok
end
defp clean_responses(ref) do
receive do
{^ref, _} -> clean_responses(ref)
after
0 -> :ok
end
end
defp finch_name(request) do
custom_options? =
Map.has_key?(request.options, :connect_options) or Map.has_key?(request.options, :inet6)
cond do
name = request.options[:finch] ->
if custom_options? do
raise ArgumentError, "cannot set both :finch and :connect_options"
else
name
end
custom_options? ->
pool_options = pool_options(request.options)
name =
pool_options
|> :erlang.term_to_binary()
|> :erlang.md5()
|> Base.url_encode64(padding: false)
name = Module.concat(Req.FinchSupervisor, "Pool_#{name}")
case DynamicSupervisor.start_child(
Req.FinchSupervisor,
{Finch, name: name, pools: %{default: pool_options}}
) do
{:ok, _} ->
name
{:error, {:already_started, _}} ->
name
end
true ->
Req.Finch
end
end
@doc """
Returns Finch pool options for the given Req `options`.
"""
def pool_options(options) when is_map(options) do
connect_options = options[:connect_options] || []
inet6_options = options |> Map.take([:inet6]) |> Enum.to_list()
pool_options = options |> Map.take([:pool_max_idle_time]) |> Enum.to_list()
Req.Request.validate_options(
connect_options,
MapSet.new([
:timeout,
:protocols,
:transport_opts,
:proxy_headers,
:proxy,
:client_settings,
:hostname,
# TODO: Remove on Req v1.0
:protocol
])
)
transport_opts =
Keyword.merge(
Keyword.take(connect_options, [:timeout]) ++ inet6_options,
Keyword.get(connect_options, :transport_opts, [])
)
conn_opts =
Keyword.take(connect_options, [:hostname, :proxy, :proxy_headers, :client_settings]) ++
if transport_opts != [] do
[transport_opts: transport_opts]
else
[]
end
protocols =
cond do
protocols = connect_options[:protocols] ->
protocols
protocol = connect_options[:protocol] ->
IO.warn([
"setting `connect_options: [protocol: protocol]` is deprecated, ",
"use `connect_options: [protocols: protocols]` instead"
])
[protocol]
true ->
[:http1]
end
pool_options ++
[protocols: protocols] ++
if conn_opts != [] do
[conn_opts: conn_opts]
else
[]
end
end
def pool_options(options) when is_list(options) do
pool_options(Req.new(options).options)
end
end

View File

@@ -0,0 +1,27 @@
defmodule Req.HTTPError do
@moduledoc """
Represents an HTTP protocol error.
This is a standardised exception that all Req adapters should use for HTTP-protocol-related
errors.
This exception is based on `Mint.HTTPError`.
"""
defexception [:protocol, :reason]
@impl true
def message(%{protocol: :http1, reason: reason}) do
Mint.HTTP1.format_error(reason)
rescue
FunctionClauseError ->
"http1 error: #{inspect(reason)}"
end
def message(%{protocol: :http2, reason: reason}) do
Mint.HTTP2.format_error(reason)
rescue
FunctionClauseError ->
"http2 error: #{inspect(reason)}"
end
end

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,269 @@
defmodule Req.Response do
@moduledoc """
The response struct.
Fields:
* `:status` - the HTTP status code.
* `:headers` - the HTTP response headers. The header names should be downcased.
See also "Headers" section in `Req` module documentation.
* `:body` - the HTTP response body.
* `:trailers` - the HTTP response trailers. The trailer names must be downcased.
* `:private` - a map reserved for libraries and frameworks to use.
Prefix the keys with the name of your project to avoid any future
conflicts. Only accepts `t:atom/0` keys.
"""
@type t() :: %__MODULE__{
status: non_neg_integer(),
headers: %{optional(binary()) => [binary()]},
body: binary() | %Req.Response.Async{} | term(),
trailers: %{optional(binary()) => [binary()]},
private: map()
}
defstruct status: 200,
headers: Req.Fields.new([]),
body: "",
trailers: Req.Fields.new([]),
private: %{}
@doc """
Returns a new response.
Expects a keyword list, map, or struct containing the response keys.
## Example
iex> Req.Response.new(status: 200, body: "body")
%Req.Response{status: 200, headers: %{}, body: "body"}
iex> finch_response = %Finch.Response{status: 200, headers: [{"content-type", "text/html"}]}
iex> Req.Response.new(finch_response)
%Req.Response{status: 200, headers: %{"content-type" => ["text/html"]}, body: ""}
"""
@spec new(options :: keyword() | map() | struct()) :: t()
def new(options \\ [])
def new(%{} = options) do
options =
Map.take(options, [:status, :headers, :body, :trailers])
|> Map.update(
:headers,
Req.Fields.new([]),
&Req.Fields.new_without_normalize_with_duplicates/1
)
|> Map.update(
:trailers,
Req.Fields.new([]),
&Req.Fields.new_without_normalize_with_duplicates/1
)
struct!(__MODULE__, options)
end
def new(options) when is_list(options) do
new(Map.new(options))
end
@doc """
Converts response to a map for interoperability with other libraries.
The resulting map has the folowing fields:
* `:status`
* `:headers`
* `:trailers`
* `:body`
Note, `body` can be any term since Req built-in and custom steps usually transform it.
## Examples
iex> resp = Req.Response.new(status: 200, headers: %{"server" => ["test"]}, body: "hello")
iex> Req.Response.to_map(resp)
%{status: 200, body: "hello", headers: [{"server", "test"}], trailers: []}
"""
@spec to_map(t()) :: %{
status: non_neg_integer(),
headers: [{binary(), binary()}],
trailers: [{binary(), binary()}],
body: term()
}
def to_map(%Req.Response{} = resp) do
%{
status: resp.status,
headers: Req.Fields.get_list(resp.headers),
trailers: Req.Fields.get_list(resp.trailers),
body: resp.body
}
end
@doc """
Builds or updates a response with JSON body.
## Example
iex> Req.Response.json(%{hello: 42})
%Req.Response{
status: 200,
headers: %{"content-type" => ["application/json"]},
body: ~s|{"hello":42}|
}
iex> resp = Req.Response.new()
iex> Req.Response.json(resp, %{hello: 42})
%Req.Response{
status: 200,
headers: %{"content-type" => ["application/json"]},
body: ~s|{"hello":42}|
}
If the request already contains a 'content-type' header, it is kept as is:
iex> Req.Response.new()
iex> |> Req.Response.put_header("content-type", "application/vnd.api+json; charset=utf-8")
iex> |> Req.Response.json(%{hello: 42})
%Req.Response{
status: 200,
headers: %{"content-type" => ["application/vnd.api+json; charset=utf-8"]},
body: ~s|{"hello":42}|
}
"""
@spec json(t(), body :: term()) :: t()
def json(response \\ new(), body) do
response =
update_in(response.headers, &Req.Fields.put_new(&1, "content-type", "application/json"))
Map.replace!(response, :body, Jason.encode!(body))
end
@doc """
Gets the value for a specific private `key`.
"""
@spec get_private(t(), key :: atom(), default :: term()) :: term()
def get_private(%Req.Response{} = response, key, default \\ nil) when is_atom(key) do
Map.get(response.private, key, default)
end
@doc """
Assigns a private `key` to `value`.
"""
@spec put_private(t(), key :: atom(), value :: term()) :: t()
def put_private(%Req.Response{} = response, key, value) when is_atom(key) do
put_in(response.private[key], value)
end
@doc """
Updates private `key` with the given function.
If `key` is present in request private map then the existing value is passed to `fun` and its
result is used as the updated value of `key`. If `key` is not present, `default` is inserted
as the value of `key`. The default value will not be passed through the update function.
## Examples
iex> resp = %Req.Response{private: %{a: 1}}
iex> Req.Response.update_private(resp, :a, 11, & &1 + 1).private
%{a: 2}
iex> Req.Response.update_private(resp, :b, 11, & &1 + 1).private
%{a: 1, b: 11}
"""
@spec update_private(t(), key :: atom(), default :: term(), (atom() -> term())) :: t()
def update_private(%Req.Response{} = response, key, initial, fun)
when is_atom(key) and is_function(fun, 1) do
update_in(response.private, &Map.update(&1, key, initial, fun))
end
@doc """
Returns the values of the header specified by `name`.
See also "Headers" section in `Req` module documentation.
## Examples
iex> Req.Response.get_header(response, "content-type")
["application/json"]
"""
@spec get_header(t(), binary()) :: [binary()]
def get_header(%Req.Response{} = resp, name) when is_binary(name) do
Req.Fields.get_values(resp.headers, name)
end
@doc """
Adds a new response header `name` if not present, otherwise replaces the
previous value of that header with `value`.
See also "Headers" section in `Req` module documentation.
## Examples
iex> resp = Req.Response.put_header(%Req.Response{}, "content-type", "application/json")
iex> resp.headers
%{"content-type" => ["application/json"]}
"""
@spec put_header(t(), binary(), binary()) :: t()
def put_header(%Req.Response{} = resp, name, value) when is_binary(name) and is_binary(value) do
update_in(resp.headers, &Req.Fields.put(&1, name, value))
end
@doc """
Deletes the header given by `name`.
All occurrences of the header are deleted, in case the header is repeated multiple times.
See also "Headers" section in `Req` module documentation.
## Examples
iex> Req.Response.get_header(resp, "cache-control")
["max-age=600", "no-transform"]
iex> resp = Req.Response.delete_header(resp, "cache-control")
iex> Req.Response.get_header(resp, "cache-control")
[]
"""
def delete_header(%Req.Response{} = resp, name) when is_binary(name) do
update_in(resp.headers, &Req.Fields.delete(&1, name))
end
@doc """
Returns the `retry-after` header delay value in seconds.
Returns `nil` if the header is not found or the computed number of seconds is negative.
"""
@spec get_retry_after(t()) :: integer() | nil
def get_retry_after(response) do
case get_header(response, "retry-after") do
[delay] ->
retry_delay_in_ms(delay)
[] ->
nil
end
end
defp retry_delay_in_ms(delay_value) do
case Integer.parse(delay_value) do
{seconds, ""} ->
if seconds >= 0 do
:timer.seconds(seconds)
end
:error ->
milliseconds =
delay_value
|> Req.Utils.parse_http_date!()
|> DateTime.diff(DateTime.utc_now(), :millisecond)
if milliseconds >= 0 do
milliseconds
end
end
end
end

View File

@@ -0,0 +1,90 @@
defmodule Req.Response.Async do
@moduledoc """
Asynchronous response body.
This is the `response.body` when making a request with `into: :self`, that is,
streaming response body chunks to the current process mailbox.
This struct implements the `Enumerable` protocol where each element is a body chunk received
from the current process mailbox. HTTP Trailer fields are ignored.
If the request is sent using HTTP/1, an extra process is spawned to consume messages from the
underlying socket. On both HTTP/1 and HTTP/2 the messages are sent to the current process as
soon as they arrive, as a firehose. If you wish to maximize request rate or have more control
over how messages are streamed, use `into: fun` or `into: collectable` instead.
**Note:** This feature is currently experimental and it may change in future releases.
## Examples
iex> resp = Req.get!("https://reqbin.org/ndjson?delay=1000", into: :self)
iex> resp.body
#Req.Response.Async<...>
iex> Enum.each(resp.body, &IO.puts/1)
# {"id":0}
# {"id":1}
# {"id":2}
:ok
"""
@derive {Inspect, only: []}
defstruct [:pid, :ref, :stream_fun, :cancel_fun]
defimpl Enumerable do
def count(_async), do: {:error, __MODULE__}
def member?(_async, _value), do: {:error, __MODULE__}
def slice(_async), do: {:error, __MODULE__}
def reduce(async, {:halt, acc}, _fun) do
cancel(async)
{:halted, acc}
end
def reduce(async, {:suspend, acc}, fun) do
{:suspended, acc, &reduce(async, &1, fun)}
end
def reduce(async, {:cont, acc}, fun) do
if async.pid != self() do
raise "expected to read body chunk in the process #{inspect(async.pid)} which made the request, got: #{inspect(self())}"
end
ref = async.ref
receive do
{^ref, _} = message ->
case async.stream_fun.(async.ref, message) do
{:ok, [data: data]} ->
result =
try do
fun.(data, acc)
rescue
e ->
cancel(async)
reraise e, __STACKTRACE__
end
reduce(async, result, fun)
{:ok, [:done]} ->
{:done, acc}
{:ok, [trailers: _trailers]} ->
reduce(async, {:cont, acc}, fun)
{:error, e} ->
raise e
other ->
raise "unexpected message: #{inspect(other)}"
end
end
end
defp cancel(async) do
async.cancel_fun.(async.ref)
end
end
end

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,795 @@
defmodule Req.Test do
@moduledoc """
Req testing conveniences.
Req is composed of:
* `Req` - the high-level API
* `Req.Request` - the low-level API and the request struct
* `Req.Steps` - the collection of built-in steps
* `Req.Test` - the testing conveniences (you're here!)
Req already has built-in support for different variants of stubs via `:plug`, `:adapter`,
and (indirectly) `:base_url` options. With this module you can:
* Create request stubs using [`Req.Test.stub(name, plug)`](`stub/2`) and mocks
using [`Req.Test.expect(name, count, plug)`](`expect/3`). Both can be used in concurrent
tests.
* Configure Req to run requests through mocks/stubs by setting `plug: {Req.Test, name}`.
This works because `Req.Test` itself is a plug whose job is to fetch the mocks/stubs under
`name`.
* Easily create JSON responses with [`Req.Test.json(conn, body)`](`json/2`),
HTML responses with [`Req.Test.html(conn, body)`](`html/2`), and
text responses with [`Req.Test.text(conn, body)`](`text/2`).
* Simulate network errors with [`Req.Test.transport_error(conn, reason)`](`transport_error/2`).
Mocks and stubs are using the same ownership model of
[nimble_ownership](https://hex.pm/packages/nimble_ownership), also used by
[Mox](https://hex.pm/packages/mox). This allows `Req.Test` to be used in concurrent tests.
## Example
Imagine we're building an app that displays weather for a given location using an HTTP weather
service:
defmodule MyApp.Weather do
def get_rating(location) do
case get_temperature(location) do
{:ok, %{status: 200, body: %{"celsius" => celsius}}} ->
cond do
celsius < 18.0 -> {:ok, :too_cold}
celsius < 30.0 -> {:ok, :nice}
true -> {:ok, :too_hot}
end
_ ->
:error
end
end
def get_temperature(location) do
[
base_url: "https://weather-service",
params: [location: location]
]
|> Keyword.merge(Application.get_env(:myapp, :weather_req_options, []))
|> Req.request()
end
end
We configure it for production:
# config/runtime.exs
config :myapp, weather_req_options: [
auth: {:bearer, System.fetch_env!("MYAPP_WEATHER_API_KEY")}
]
In tests, instead of hitting the network, we make the request against
a [plug](`Req.Steps.run_plug/1`) _stub_ named `MyApp.Weather`:
# config/test.exs
config :myapp, weather_req_options: [
plug: {Req.Test, MyApp.Weather}
]
Now we can control our stubs **in concurrent tests**:
use ExUnit.Case, async: true
test "nice weather" do
Req.Test.stub(MyApp.Weather, fn conn ->
Req.Test.json(conn, %{"celsius" => 25.0})
end)
assert MyApp.Weather.get_rating("Krakow, Poland") == {:ok, :nice}
end
## Concurrency and Allowances
The example above works in concurrent tests because `MyApp.Weather.get_rating/1` calls
directly to `Req.request/1` *in the same process*. It also works in many cases where the
request happens in a spawned process, such as a `Task`, `GenServer`, and more.
However, if you are encountering issues with stubs not being available in spawned processes,
it's likely that you'll need **explicit allowances**. For example, if
`MyApp.Weather.get_rating/1` was calling `Req.request/1` in a process spawned with `spawn/1`,
the stub would not be available in the spawned process:
# With code like this, the stub would not be available in the spawned task:
def get_rating_async(location) do
spawn(fn -> get_rating(location) end)
end
To make stubs defined in the test process available in other processes, you can use
`allow/3`. For example, imagine that the call to `MyApp.Weather.get_rating/1`
was happening in a spawned GenServer:
test "nice weather" do
{:ok, pid} = start_gen_server(...)
Req.Test.stub(MyApp.Weather, fn conn ->
Req.Test.json(conn, %{"celsius" => 25.0})
end)
Req.Test.allow(MyApp.Weather, self(), pid)
assert get_weather(pid, "Krakow, Poland") == {:ok, :nice}
end
## Broadway
If you're using `Req.Test` with [Broadway](https://hex.pm/broadway), you may need to use
`allow/3` to make stubs available in the Broadway processors. A great way to do that is
to hook into the [Telemetry](https://hex.pm/telemetry) events that Broadway publishes to
manually allow the processors and batch processors to access the stubs. This approach is
similar to what is [documented in Broadway
itself](https://hexdocs.pm/broadway/Broadway.html#module-testing-with-ecto).
First, you should add the test PID (which is allowed to use the Req stub) to the metadata
for the test events you're publishing:
Broadway.test_message(MyApp.Pipeline, message, metadata: %{req_stub_owner: self()})
Then, you'll need to define a test helper to hook into the Telemetry events. For example,
in your `test/test_helper.exs` file:
defmodule BroadwayReqStubs do
def attach(stub) do
events = [
[:broadway, :processor, :start],
[:broadway, :batch_processor, :start],
]
:telemetry.attach_many({__MODULE__, stub}, events, &__MODULE__.handle_event/4, %{stub: stub})
end
def handle_event(_event_name, _event_measurement, %{messages: messages}, %{stub: stub}) do
with [%Broadway.Message{metadata: %{req_stub_owner: pid}} | _] <- messages do
:ok = Req.Test.allow(stub, pid, self())
end
:ok
end
end
Last but not least, attach the helper in your `test/test_helper.exs`:
BroadwayReqStubs.attach(MyStub)
"""
@typep name() :: term()
if Code.ensure_loaded?(Plug.Conn) do
@typep plug() ::
module()
| {module(), term()}
| (Plug.Conn.t() -> Plug.Conn.t())
| (Plug.Conn.t(), term() -> Plug.Conn.t())
else
@typep plug() ::
module()
| {module, term()}
| (conn :: term() -> term())
| (conn :: term(), term() -> term())
end
@ownership Req.Test.Ownership
@doc """
Sends JSON response.
## Examples
iex> plug = fn conn ->
...> Req.Test.json(conn, %{celsius: 25.0})
...> end
iex>
iex> resp = Req.get!(plug: plug)
iex> resp.headers["content-type"]
["application/json; charset=utf-8"]
iex> resp.body
%{"celsius" => 25.0}
"""
if Code.ensure_loaded?(Plug.Test) do
@spec json(Plug.Conn.t(), term()) :: Plug.Conn.t()
def json(%Plug.Conn{} = conn, data) do
send_resp(conn, conn.status || 200, "application/json", Jason.encode_to_iodata!(data))
end
defp send_resp(conn, default_status, default_content_type, body) do
conn
|> ensure_resp_content_type(default_content_type)
|> Plug.Conn.send_resp(conn.status || default_status, body)
end
defp ensure_resp_content_type(%Plug.Conn{resp_headers: resp_headers} = conn, content_type) do
if List.keyfind(resp_headers, "content-type", 0) do
conn
else
content_type = content_type <> "; charset=utf-8"
%{conn | resp_headers: [{"content-type", content_type} | resp_headers]}
end
end
else
def json(_conn, _data) do
require Logger
Logger.error("""
Could not find plug dependency.
Please add :plug to your dependencies:
{:plug, "~> 1.0"}
""")
raise "missing plug dependency"
end
end
@doc """
Sends HTML response.
## Examples
iex> plug = fn conn ->
...> Req.Test.html(conn, "<h1>Hello, World!</h1>")
...> end
iex>
iex> resp = Req.get!(plug: plug)
iex> resp.headers["content-type"]
["text/html; charset=utf-8"]
iex> resp.body
"<h1>Hello, World!</h1>"
"""
if Code.ensure_loaded?(Plug.Test) do
@spec html(Plug.Conn.t(), iodata()) :: Plug.Conn.t()
def html(%Plug.Conn{} = conn, data) do
send_resp(conn, conn.status || 200, "text/html", data)
end
else
def html(_conn, _data) do
require Logger
Logger.error("""
Could not find plug dependency.
Please add :plug to your dependencies:
{:plug, "~> 1.0"}
""")
raise "missing plug dependency"
end
end
@doc """
Sends text response.
## Examples
iex> plug = fn conn ->
...> Req.Test.text(conn, "Hello, World!")
...> end
iex>
iex> resp = Req.get!(plug: plug)
iex> resp.headers["content-type"]
["text/plain; charset=utf-8"]
iex> resp.body
"Hello, World!"
"""
if Code.ensure_loaded?(Plug.Test) do
@spec text(Plug.Conn.t(), iodata()) :: Plug.Conn.t()
def text(%Plug.Conn{} = conn, data) do
send_resp(conn, conn.status || 200, "text/plain", data)
end
else
def text(_conn, _data) do
require Logger
Logger.error("""
Could not find plug dependency.
Please add :plug to your dependencies:
{:plug, "~> 1.0"}
""")
raise "missing plug dependency"
end
end
@doc """
Sends redirect response to the given url.
This function is adapted from [`Phoenix.Controller.redirect/2`](https://hexdocs.pm/phoenix/Phoenix.Controller.html#redirect/2).
For security, `:to` only accepts paths. Use the `:external`
option to redirect to any URL.
The response will be sent with the status code defined within
the connection, via `Plug.Conn.put_status/2`. If no status
code is set, a 302 response is sent.
## Examples
iex> plug = fn
...> conn when conn.request_path == nil ->
...> Req.Test.redirect(conn, to: "/hello")
...>
...> conn when conn.request_path == "/hello" ->
...> Req.Test.text(conn, "Hello, World!")
...> conn -> dbg(conn)
...> end
iex>
iex> resp = Req.get!(plug: plug, url: "http://example.com")
# 14:53:06.101 [debug] redirecting to /hello
iex> resp.body
"Hello, World!"
"""
def redirect(conn, opts)
if Code.ensure_loaded?(Plug.Conn) do
def redirect(conn, opts) when is_list(opts) do
url = url(opts)
html = Plug.HTML.html_escape(url)
body = "<html><body>You are being <a href=\"#{html}\">redirected</a>.</body></html>"
conn =
if List.keyfind(conn.resp_headers, "content-type", 0) do
conn
else
content_type = "text/html; charset=utf-8"
update_in(conn.resp_headers, &[{"content-type", content_type} | &1])
end
conn
|> Plug.Conn.put_resp_header("location", url)
|> Plug.Conn.send_resp(conn.status || 302, body)
end
defp url(opts) do
cond do
to = opts[:to] -> validate_local_url(to)
external = opts[:external] -> external
true -> raise ArgumentError, "expected :to or :external option in redirect/2"
end
end
@invalid_local_url_chars ["\\", "/%09", "/\t"]
defp validate_local_url("//" <> _ = to), do: raise_invalid_url(to)
defp validate_local_url("/" <> _ = to) do
if String.contains?(to, @invalid_local_url_chars) do
raise ArgumentError, "unsafe characters detected for local redirect in URL #{inspect(to)}"
else
to
end
end
defp validate_local_url(to), do: raise_invalid_url(to)
defp raise_invalid_url(url) do
raise ArgumentError, "the :to option in redirect expects a path but was #{inspect(url)}"
end
else
def redirect(_conn, _opts) do
require Logger
Logger.error("""
Could not find plug dependency.
Please add :plug to your dependencies:
{:plug, "~> 1.0"}
""")
raise "missing plug dependency"
end
end
@doc """
Simulates a network transport error.
## Examples
iex> plug = fn conn ->
...> Req.Test.transport_error(conn, :timeout)
...> end
iex>
iex> Req.get(plug: plug, retry: false)
{:error, %Req.TransportError{reason: :timeout}}
"""
@doc since: "0.5.0"
def transport_error(conn, reason)
if Code.ensure_loaded?(Plug.Conn) do
@spec transport_error(Plug.Conn.t(), reason :: atom()) :: Plug.Conn.t()
def transport_error(%Plug.Conn{} = conn, reason) do
validate_transport_error!(reason)
exception = Req.TransportError.exception(reason: reason)
put_in(conn.private[:req_test_exception], exception)
end
defp validate_transport_error!(:protocol_not_negotiated), do: :ok
defp validate_transport_error!({:bad_alpn_protocol, _}), do: :ok
defp validate_transport_error!(:closed), do: :ok
defp validate_transport_error!(:timeout), do: :ok
defp validate_transport_error!(reason) do
case :ssl.format_error(reason) do
~c"Unexpected error:" ++ _ ->
raise ArgumentError, """
unexpected Req.TransportError reason: #{inspect(reason)}
This function only accepts error reasons that can happen
in production, for example: `:closed`, `:timeout`,
`:econnrefused`, etc.
"""
_ ->
:ok
end
end
else
def transport_error(_conn, _reason) do
require Logger
Logger.error("""
Could not find plug dependency.
Please add :plug to your dependencies:
{:plug, "~> 1.0"}
""")
raise "missing plug dependency"
end
end
@doc false
@deprecated "Don't manually fetch stubs. See the documentation for Req.Test instead."
def stub(name) do
__fetch_plug__(name)
end
def __fetch_plug__(name) do
case Req.Test.Ownership.fetch_owner(@ownership, callers(), name) do
{tag, owner} when is_pid(owner) and tag in [:ok, :shared_owner] ->
result =
Req.Test.Ownership.get_and_update(@ownership, owner, name, fn
%{expectations: [value | rest]} = map ->
{{:ok, value}, put_in(map[:expectations], rest)}
%{stub: value} = map ->
{{:ok, value}, map}
%{expectations: []} = map ->
{{:error, :no_expectations_and_no_stub}, map}
end)
case result do
{:ok, {:ok, value}} ->
value
{:ok, {:error, :no_expectations_and_no_stub}} ->
raise "no mock or stub for #{inspect(name)}"
end
:error ->
raise "cannot find mock/stub #{inspect(name)} in process #{inspect(self())}"
end
end
defguardp is_plug(value)
when is_function(value, 1) or
is_function(value, 2) or
is_atom(value) or
(is_tuple(value) and tuple_size(value) == 2 and is_atom(elem(value, 0)))
@doc """
Creates a request stub with the given `name` and `plug`.
Req allows running requests against _plugs_ (instead of over the network) using the
[`:plug`](`Req.Steps.run_plug/1`) option. However, passing the `:plug` value throughout the
system can be cumbersome. Instead, you can tell Req to find plugs by `name` by setting
`plug: {Req.Test, name}`, and register plug stubs for that `name` by calling
`Req.Test.stub(name, plug)`. In other words, multiple concurrent tests can register test stubs
under the same `name`, and when Req makes the request, it will find the appropriate
implementation, even when invoked from different processes than the test process.
The `name` can be any term.
The `plug` can be one of:
* A _function_ plug: a `fun(conn)` or `fun(conn, options)` function that takes a
`Plug.Conn` and returns a `Plug.Conn`.
* A _module_ plug: a `module` name or a `{module, options}` tuple.
## Examples
iex> Req.Test.stub(MyStub, fn conn ->
...> send(self(), :req_happened)
...> Req.Test.json(conn, %{})
...> end)
:ok
iex> Req.get!(plug: {Req.Test, MyStub}).body
%{}
iex> receive do
...> :req_happened -> :ok
...> end
:ok
"""
@doc type: :mock
@spec stub(name(), plug()) :: :ok
def stub(name, plug) when is_plug(plug) do
{:ok, :ok} =
Req.Test.Ownership.get_and_update(@ownership, self(), name, fn map_or_nil ->
{:ok, put_in(map_or_nil || %{}, [:stub], plug)}
end)
:ok
end
@doc """
Creates a request expectation with the given `name` and `plug`, expected to be fetched at
most `n` times, **in order**.
This function allows you to expect a `n` number of request and handle them **in order** via the
given `plug`. It is safe to use in concurrent tests. If you fetch the value under `name` more
than `n` times, this function raises a `RuntimeError`.
The `name` can be any term.
The `plug` can be one of:
* A _function_ plug: a `fun(conn)` or `fun(conn, options)` function that takes a
`Plug.Conn` and returns a `Plug.Conn`.
* A _module_ plug: a `module` name or a `{module, options}` tuple.
See `stub/2` and module documentation for more information.
`verify_on_exit!/1` can be used to ensure that all defined expectations have been called.
## Examples
Let's simulate a server that is having issues: on the first request it is not responding
and on the following two requests it returns an HTTP 500. Only on the third request it returns
an HTTP 200. Req by default automatically retries transient errors (using `Req.Steps.retry/1`)
so it will make multiple requests exercising all of our request expectations:
iex> Req.Test.expect(MyStub, &Req.Test.transport_error(&1, :econnrefused))
iex> Req.Test.expect(MyStub, 2, &Plug.Conn.send_resp(&1, 500, "internal server error"))
iex> Req.Test.expect(MyStub, &Plug.Conn.send_resp(&1, 200, "ok"))
iex> Req.get!(plug: {Req.Test, MyStub}).body
# 15:57:06.309 [warning] retry: got exception, will retry in 1000ms, 3 attempts left
# 15:57:06.309 [warning] ** (Req.TransportError) connection refused
# 15:57:07.310 [warning] retry: got response with status 500, will retry in 2000ms, 2 attempts left
# 15:57:09.311 [warning] retry: got response with status 500, will retry in 4000ms, 1 attempt left
"ok"
iex> Req.request!(plug: {Req.Test, MyStub})
** (RuntimeError) no mock or stub for MyStub
"""
@doc since: "0.4.15"
@doc type: :mock
@spec expect(name(), pos_integer(), plug()) :: name()
def expect(name, n \\ 1, plug) when is_integer(n) and n > 0 do
plugs = List.duplicate(plug, n)
{:ok, :ok} =
Req.Test.Ownership.get_and_update(@ownership, self(), name, fn map_or_nil ->
{:ok, Map.update(map_or_nil || %{}, :expectations, plugs, &(&1 ++ plugs))}
end)
name
end
@doc """
Allows `pid_to_allow` to access `name` provided that `owner` is already allowed.
"""
@doc type: :mock
@spec allow(name(), pid(), pid() | (-> pid())) :: :ok | {:error, Exception.t()}
def allow(name, owner, pid_to_allow) when is_pid(owner) do
Req.Test.Ownership.allow(@ownership, owner, pid_to_allow, name)
end
@doc """
Sets the `Req.Test` mode to "global", meaning that the stubs are shared across all tests
and cannot be used concurrently.
"""
@doc since: "0.5.0"
@doc type: :mock
@spec set_req_test_to_shared(ex_unit_context :: term()) :: :ok
def set_req_test_to_shared(_context \\ %{}) do
Req.Test.Ownership.set_mode_to_shared(@ownership, self())
end
@doc """
Sets the `Req.Test` mode to "private", meaning that stubs can be shared across
tests concurrently.
"""
@doc type: :mock
@doc since: "0.5.0"
@spec set_req_test_to_private(ex_unit_context :: term()) :: :ok
def set_req_test_to_private(_context \\ %{}) do
Req.Test.Ownership.set_mode_to_private(@ownership)
end
@doc """
Sets the `Req.Test` mode based on the given `ExUnit` context.
This works as a ExUnit callback:
setup :set_req_test_from_context
"""
@doc since: "0.5.0"
@doc type: :mock
@spec set_req_test_from_context(ex_unit_context :: term()) :: :ok
def set_req_test_from_context(_context \\ %{})
def set_req_test_from_context(%{async: true} = context), do: set_req_test_to_private(context)
def set_req_test_from_context(context), do: set_req_test_to_shared(context)
@doc """
Sets a ExUnit callback to verify the expectations on exit.
Similar to calling `verify!/0` at the end of your test.
This works as a ExUnit callback:
setup {Req.Test, :verify_on_exit!}
"""
@doc since: "0.5.0"
@doc type: :mock
@spec verify_on_exit!(term()) :: :ok
def verify_on_exit!(_context \\ %{}) do
pid = self()
Req.Test.Ownership.set_owner_to_manual_cleanup(@ownership, pid)
ExUnit.Callbacks.on_exit(Req.Test, fn ->
verify(pid, :all)
Req.Test.Ownership.cleanup_owner(@ownership, pid)
end)
end
@doc """
Verifies that all the plugs expected to be executed within any scope have been executed.
"""
@doc since: "0.5.0"
@doc type: :mock
@spec verify!() :: :ok
def verify! do
verify(self(), :all)
end
@doc """
Verifies that all the plugs expected to be executed within the scope of `name` have been
executed.
"""
@doc type: :mock
@doc since: "0.5.0"
@spec verify!(name()) :: :ok
def verify!(name) do
verify(self(), name)
end
defp verify(owner_pid, mock_or_all) do
messages =
for {name, stubs_and_expecs} <-
Req.Test.Ownership.get_owned(@ownership, owner_pid, _default = %{}, 5000),
name == mock_or_all or mock_or_all == :all,
pending_count = stubs_and_expecs |> Map.get(:expectations, []) |> length(),
pending_count > 0 do
" * expected #{inspect(name)} to be still used #{pending_count} more times"
end
if messages != [] do
raise "error while verifying Req.Test expectations for #{inspect(owner_pid)}:\n\n" <>
Enum.join(messages, "\n")
end
:ok
end
## Helpers
defp callers do
[self() | Process.get(:"$callers") || []]
end
## Plug callbacks
if Code.ensure_loaded?(Plug) do
@behaviour Plug
end
@doc false
def init(name) do
name
end
@doc false
def call(conn, name) do
case __fetch_plug__(name) do
fun when is_function(fun, 1) ->
fun.(conn)
fun when is_function(fun, 2) ->
fun.(conn, [])
module when is_atom(module) ->
module.call(conn, module.init([]))
{module, options} when is_atom(module) ->
module.call(conn, module.init(options))
other ->
raise """
expected plug to be one of:
* fun(conn)
* fun(conn, options)
* module
* {module, options}
got: #{inspect(other)}\
"""
end
end
@doc """
Reads the raw request body from a plug request.
## Examples
iex> echo = fn conn ->
...> body = Req.Test.raw_body(conn)
...> Plug.Conn.send_resp(conn, 200, body)
...> end
iex>
iex> resp = Req.post!(plug: echo, json: %{hello: "world"})
iex> resp.body
"{\\"hello\\":\\"world\\"}"
"""
if Code.ensure_loaded?(Plug.Test) do
@spec raw_body(Plug.Conn.t()) :: iodata()
def raw_body(%Plug.Conn{} = conn) do
{Req.Test.Adapter, %{raw_body: raw_body}} = conn.adapter
raw_body
end
else
def raw_body(_conn) do
require Logger
Logger.error("""
Could not find plug dependency.
Please add :plug to your dependencies:
{:plug, "~> 1.0"}
""")
raise "missing plug dependency"
end
end
end

View File

@@ -0,0 +1,49 @@
if Code.ensure_loaded?(Plug.Conn) do
defmodule Req.Test.Adapter do
@behaviour Plug.Conn.Adapter
@moduledoc false
## Test helpers
def conn(conn, method, uri, body) when is_binary(body) do
conn = Plug.Adapters.Test.Conn.conn(conn, method, uri, body)
{_, state} = conn.adapter
state = Map.merge(state, %{body_read: false, has_more_body: false, raw_body: body})
%{conn | adapter: {__MODULE__, state}}
end
## Connection adapter
def read_req_body(state, opts \\ []) do
# We restore the body for the first automatic read for backwards
# compatability with Req 0.5.10 and below.
# TODO: remove in 0.6 if we allow opting out
case Plug.Adapters.Test.Conn.read_req_body(state, opts) do
{:more, body, state} ->
{:more, body, %{state | has_more_body: true}}
{:ok, body, %{has_more_body: true} = state} ->
{:ok, body, state}
{:ok, body, %{body_read: true} = state} ->
{:ok, body, state}
{:ok, body, state} ->
{:ok, body, %{state | req_body: body}}
end
end
defdelegate send_resp(state, status, headers, body), to: Plug.Adapters.Test.Conn
defdelegate send_file(state, status, headers, path, offset, length),
to: Plug.Adapters.Test.Conn
defdelegate send_chunked(state, status, headers), to: Plug.Adapters.Test.Conn
defdelegate chunk(state, body), to: Plug.Adapters.Test.Conn
defdelegate inform(state, status, headers), to: Plug.Adapters.Test.Conn
defdelegate upgrade(state, protocol, opts), to: Plug.Adapters.Test.Conn
defdelegate push(state, path, headers), to: Plug.Adapters.Test.Conn
defdelegate get_peer_data(payload), to: Plug.Adapters.Test.Conn
defdelegate get_http_protocol(payload), to: Plug.Adapters.Test.Conn
end
end

View File

@@ -0,0 +1,368 @@
# Vendored from nimble_ownership v1.0.1, replacing NimbleOwnership.Error with
# Req.Test.OwnershipError.
#
# Check changes with:
#
# git diff --no-index lib/req/test/ownership.ex ../nimble_ownership/lib/nimble_ownership.ex
defmodule Req.Test.Ownership do
@moduledoc false
defguardp is_timeout(val) when (is_integer(val) and val > 0) or val == :infinity
use GenServer
alias Req.Test.OwnershipError, as: Error
@typedoc "Ownership server."
@type server() :: GenServer.server()
@typedoc "Arbitrary key."
@type key() :: term()
@typedoc "Arbitrary metadata associated with an owned `t:key/0`."
@type metadata() :: term()
@genserver_opts [
:name,
:timeout,
:debug,
:spawn_opt,
:hibernate_after
]
@spec start_link(keyword()) :: GenServer.on_start()
def start_link(options \\ []) when is_list(options) do
{genserver_opts, other_opts} = Keyword.split(options, @genserver_opts)
if other_opts != [] do
raise ArgumentError, "unknown options: #{inspect(Keyword.keys(other_opts))}"
end
GenServer.start_link(__MODULE__, [], genserver_opts)
end
@spec allow(server(), pid(), pid() | (-> resolved_pid), key()) ::
:ok | {:error, Error.t()}
when resolved_pid: pid() | [pid()]
def allow(ownership_server, pid_with_access, pid_to_allow, key, timeout \\ 5000)
when is_pid(pid_with_access) and (is_pid(pid_to_allow) or is_function(pid_to_allow, 0)) and
is_timeout(timeout) do
GenServer.call(ownership_server, {:allow, pid_with_access, pid_to_allow, key}, timeout)
end
@spec get_and_update(server(), pid(), key(), fun, timeout()) ::
{:ok, get_value} | {:error, Error.t()}
when fun: (nil | metadata() -> {get_value, updated_metadata :: metadata()}),
get_value: term()
def get_and_update(ownership_server, owner_pid, key, fun, timeout \\ 5000)
when is_pid(owner_pid) and is_function(fun, 1) and is_timeout(timeout) do
case GenServer.call(ownership_server, {:get_and_update, owner_pid, key, fun}, timeout) do
{:ok, get_value} -> {:ok, get_value}
{:error, %Error{} = error} -> {:error, error}
{:__raise__, error} when is_exception(error) -> raise error
end
end
@spec fetch_owner(server(), [pid(), ...], key(), timeout()) ::
{:ok, owner :: pid()}
| {:shared_owner, shared_owner :: pid()}
| :error
def fetch_owner(ownership_server, [_ | _] = callers, key, timeout \\ 5000)
when is_timeout(timeout) do
GenServer.call(ownership_server, {:fetch_owner, callers, key}, timeout)
end
@spec get_owned(server(), pid(), default, timeout()) :: %{key() => metadata()} | default
when default: term()
def get_owned(ownership_server, owner_pid, default \\ nil, timeout \\ 5000)
when is_pid(owner_pid) and is_timeout(timeout) do
GenServer.call(ownership_server, {:get_owned, owner_pid, default}, timeout)
end
@spec set_mode_to_private(server()) :: :ok
def set_mode_to_private(ownership_server) do
GenServer.call(ownership_server, {:set_mode, :private})
end
@spec set_mode_to_shared(server(), pid()) :: :ok
def set_mode_to_shared(ownership_server, shared_owner) when is_pid(shared_owner) do
GenServer.call(ownership_server, {:set_mode, {:shared, shared_owner}})
end
@spec set_owner_to_manual_cleanup(server(), pid()) :: :ok
def set_owner_to_manual_cleanup(ownership_server, owner_pid) do
GenServer.call(ownership_server, {:set_owner_to_manual_cleanup, owner_pid})
end
@spec cleanup_owner(server(), pid()) :: :ok
def cleanup_owner(ownership_server, owner_pid) when is_pid(owner_pid) do
GenServer.call(ownership_server, {:cleanup_owner, owner_pid})
end
## State
defstruct [
# The mode can be either :private, or {:shared, shared_owner_pid}.
mode: :private,
# This is a map of %{owner_pid => %{key => metadata}}. Its purpose is to track the metadata
# under each key that a owner owns.
owners: %{},
# This tracks what to do when each owner goes down. It's a map of
# %{owner_pid => :auto | :manual}.
owner_cleanup: %{},
# This is a map of %{allowed_pid => %{key => owner_pid}}. Its purpose is to track the keys
# that a PID is allowed to access, alongside which the owner of those keys is.
allowances: %{},
# This is used to track which PIDs we're monitoring, to avoid double-monitoring.
monitored_pids: MapSet.new()
]
## Callbacks
@impl true
def init([]) do
{:ok, %__MODULE__{}}
end
@impl true
def handle_call(call, from, state)
def handle_call(
{:allow, _pid_with_access, _pid_to_allow, key},
_from,
%__MODULE__{mode: {:shared, _shared_owner}} = state
) do
error = %Error{key: key, reason: :cant_allow_in_shared_mode}
{:reply, {:error, error}, state}
end
def handle_call(
{:allow, pid_with_access, pid_to_allow, key},
_from,
%__MODULE__{mode: :private} = state
) do
if state.owners[pid_to_allow][key] do
error = %Error{key: key, reason: :already_an_owner}
throw({:reply, {:error, error}, state})
end
owner_pid =
cond do
owner_pid = state.allowances[pid_with_access][key] ->
owner_pid
_meta = state.owners[pid_with_access][key] ->
pid_with_access
true ->
throw({:reply, {:error, %Error{key: key, reason: :not_allowed}}, state})
end
case state.allowances[pid_to_allow][key] do
# There's already another owner PID that is allowing "pid_to_allow" to use "key".
other_owner_pid when is_pid(other_owner_pid) and other_owner_pid != owner_pid ->
error = %Error{key: key, reason: {:already_allowed, other_owner_pid}}
{:reply, {:error, error}, state}
# "pid_to_allow" is already allowed access to "key" through the same "owner_pid",
# so this is a no-op.
^owner_pid ->
{:reply, :ok, state}
nil ->
state =
state
|> maybe_monitor_pid(pid_with_access)
|> put_in([Access.key!(:allowances), Access.key(pid_to_allow, %{}), key], owner_pid)
{:reply, :ok, state}
end
end
def handle_call({:get_and_update, owner_pid, key, fun}, _from, %__MODULE__{} = state) do
case state.mode do
{:shared, shared_owner_pid} when shared_owner_pid != owner_pid ->
error = %Error{key: key, reason: {:not_shared_owner, shared_owner_pid}}
throw({:reply, {:error, error}, state})
_ ->
:ok
end
state = resolve_lazy_calls_for_key(state, key)
if other_owner = state.allowances[owner_pid][key] do
throw({:reply, {:error, %Error{key: key, reason: {:already_allowed, other_owner}}}, state})
end
case fun.(_meta_or_nil = state.owners[owner_pid][key]) do
{get_value, new_meta} ->
state = put_in(state, [Access.key!(:owners), Access.key(owner_pid, %{}), key], new_meta)
# We should also monitor the new owner, if it hasn't already been monitored. That
# can happen if that owner is already the owner of another key. We ALWAYS monitor,
# so if owner_pid is already an owner we're already monitoring it.
state =
if not Map.has_key?(state.owner_cleanup, owner_pid) do
_ref = Process.monitor(owner_pid)
put_in(state.owner_cleanup[owner_pid], :auto)
else
state
end
{:reply, {:ok, get_value}, state}
other ->
message = """
invalid return value from callback function. Expected nil or a tuple of the form \
{get_value, update_value} (see the function's @spec), instead got: #{inspect(other)}\
"""
{:reply, {:__raise__, %ArgumentError{message: message}}, state}
end
end
def handle_call(
{:fetch_owner, _callers, _key},
_from,
%__MODULE__{mode: {:shared, shared_owner_pid}} = state
) do
{:reply, {:shared_owner, shared_owner_pid}, state}
end
def handle_call({:fetch_owner, callers, key}, _from, %__MODULE__{mode: :private} = state) do
{owner, state} =
case fetch_owner_once(state, callers, key) do
nil ->
state = resolve_lazy_calls_for_key(state, key)
{fetch_owner_once(state, callers, key), state}
owner ->
{owner, state}
end
if is_nil(owner) do
{:reply, :error, state}
else
{:reply, {:ok, owner}, state}
end
end
def handle_call({:get_owned, owner_pid, default}, _from, %__MODULE__{} = state) do
{:reply, state.owners[owner_pid] || default, state}
end
def handle_call({:set_mode, {:shared, shared_owner_pid}}, _from, %__MODULE__{} = state) do
state = maybe_monitor_pid(state, shared_owner_pid)
state = %{state | mode: {:shared, shared_owner_pid}}
{:reply, :ok, state}
end
def handle_call({:set_mode, :private}, _from, %__MODULE__{} = state) do
{:reply, :ok, %{state | mode: :private}}
end
def handle_call({:set_owner_to_manual_cleanup, owner_pid}, _from, %__MODULE__{} = state) do
{:reply, :ok, put_in(state.owner_cleanup[owner_pid], :manual)}
end
def handle_call({:cleanup_owner, pid}, _from, %__MODULE__{} = state) do
{:reply, :ok, pop_owner_and_clean_up_allowances(state, pid)}
end
@impl true
def handle_info(msg, state)
# The global owner went down, so we go back to private mode.
def handle_info({:DOWN, _, _, down_pid, _}, %__MODULE__{mode: {:shared, down_pid}} = state) do
{:noreply, %{state | mode: :private}}
end
# An owner went down, so we need to clean up all of its allowances as well as all its keys.
def handle_info({:DOWN, _ref, _, down_pid, _}, state)
when is_map_key(state.owners, down_pid) do
case state.owner_cleanup[down_pid] || :auto do
:manual ->
{:noreply, state}
:auto ->
state = pop_owner_and_clean_up_allowances(state, down_pid)
{:noreply, state}
end
end
# A PID that we were monitoring went down. Let's just clean up all its allowances.
def handle_info({:DOWN, _, _, down_pid, _}, state) do
{_keys_and_values, state} = pop_in(state.allowances[down_pid])
state = update_in(state.monitored_pids, &MapSet.delete(&1, down_pid))
{:noreply, state}
end
## Helpers
defp pop_owner_and_clean_up_allowances(state, target_pid) do
{_, state} = pop_in(state.owners[target_pid])
{_, state} = pop_in(state.owner_cleanup[target_pid])
allowances =
Enum.reduce(state.allowances, state.allowances, fn {pid, allowances}, acc ->
new_allowances =
for {key, owner_pid} <- allowances,
owner_pid != target_pid,
into: %{},
do: {key, owner_pid}
Map.put(acc, pid, new_allowances)
end)
%{state | allowances: allowances}
end
defp maybe_monitor_pid(state, pid) do
if pid in state.monitored_pids do
state
else
Process.monitor(pid)
update_in(state.monitored_pids, &MapSet.put(&1, pid))
end
end
defp fetch_owner_once(state, callers, key) do
Enum.find_value(callers, fn caller ->
case state do
%{owners: %{^caller => %{^key => _meta}}} -> caller
%{allowances: %{^caller => %{^key => owner_pid}}} -> owner_pid
_ -> nil
end
end)
end
defp resolve_lazy_calls_for_key(state, key) do
updated_allowances =
Enum.reduce(state.allowances, state.allowances, fn
{fun, value}, allowances when is_function(fun, 0) and is_map_key(value, key) ->
result =
fun.()
|> List.wrap()
|> Enum.group_by(&is_pid/1)
allowances =
result
|> Map.get(true, [])
|> Enum.reduce(allowances, fn pid, allowances ->
Map.update(allowances, pid, value, &Map.merge(&1, value))
end)
if Map.has_key?(allowances, false), do: Map.delete(allowances, fun), else: allowances
_, allowances ->
allowances
end)
%{state | allowances: updated_allowances}
end
end

View File

@@ -0,0 +1,31 @@
# Vendored from nimble_ownership. See Req.Test.Ownership.
defmodule Req.Test.OwnershipError do
defexception [:reason, :key]
@impl true
def message(%__MODULE__{key: key, reason: reason}) do
format_reason(key, reason)
end
## Helpers
defp format_reason(key, {:already_allowed, other_owner_pid}) do
"this PID is already allowed to access key #{inspect(key)} via other owner PID #{inspect(other_owner_pid)}"
end
defp format_reason(key, :not_allowed) do
"this PID is not allowed to access key #{inspect(key)}"
end
defp format_reason(key, :already_an_owner) do
"this PID is already an owner of key #{inspect(key)}"
end
defp format_reason(_key, {:not_shared_owner, pid}) do
"#{inspect(pid)} is not the shared owner, so it cannot update keys"
end
defp format_reason(_key, :cant_allow_in_shared_mode) do
"cannot allow PIDs in shared mode"
end
end

View File

@@ -0,0 +1,12 @@
defmodule Req.TooManyRedirectsError do
@moduledoc """
Represents an error when too many redirects occured, returned by `Req.Steps.redirect/1`.
"""
defexception [:max_redirects]
@impl true
def message(%{max_redirects: max_redirects}) do
"too many redirects (#{max_redirects})"
end
end

View File

@@ -0,0 +1,17 @@
defmodule Req.TransportError do
@moduledoc """
Represents an error with the transport used by an HTTP connection.
This is a standardised exception that all Req adapters should use for transport-layer-related
errors.
This exception is based on `Mint.TransportError`.
"""
defexception [:reason]
@impl true
def message(%__MODULE__{reason: reason}) do
Mint.TransportError.message(%Mint.TransportError{reason: reason})
end
end

View File

@@ -0,0 +1,843 @@
defmodule Req.Utils do
@moduledoc false
defmacrop iodata({:<<>>, _, parts}) do
Enum.map(parts, &to_iodata/1)
end
defp to_iodata(binary) when is_binary(binary) do
binary
end
defp to_iodata(
{:"::", _, [{{:., _, [Kernel, :to_string]}, _, [interpolation]}, {:binary, _, nil}]}
) do
interpolation
end
@doc """
Create AWS Signature v4.
https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
"""
def aws_sigv4_headers(options) do
{access_key_id, options} = Keyword.pop!(options, :access_key_id)
{secret_access_key, options} = Keyword.pop!(options, :secret_access_key)
{security_token, options} = Keyword.pop(options, :token)
{region, options} = Keyword.pop!(options, :region)
{service, options} = Keyword.pop!(options, :service)
{datetime, options} = Keyword.pop!(options, :datetime)
{method, options} = Keyword.pop!(options, :method)
{url, options} = Keyword.pop!(options, :url)
{headers, options} = Keyword.pop!(options, :headers)
{body, options} = Keyword.pop!(options, :body)
Keyword.validate!(options, [:body_digest])
datetime = DateTime.truncate(datetime, :second)
datetime_string = DateTime.to_iso8601(datetime, :basic)
date_string = Date.to_iso8601(datetime, :basic)
url = normalize_url(url)
body_digest = options[:body_digest] || hex(sha256(body))
service = to_string(service)
method = method |> Atom.to_string() |> String.upcase()
headers = canonical_host_header(headers, url)
aws_headers = [
{"x-amz-content-sha256", body_digest},
{"x-amz-date", datetime_string}
]
aws_headers =
if security_token do
aws_headers ++ [{"x-amz-security-token", security_token}]
else
aws_headers
end
canonical_headers = headers ++ aws_headers
## canonical_headers needs to be sorted for canonical_request construction
canonical_headers = Enum.sort(canonical_headers)
signed_headers =
Enum.map_intersperse(
Enum.sort(canonical_headers),
";",
&String.downcase(elem(&1, 0), :ascii)
)
canonical_headers =
Enum.map_intersperse(canonical_headers, "\n", fn {name, value} -> [name, ":", value] end)
path = URI.encode(url.path || "/", &(&1 == ?/ or URI.char_unreserved?(&1)))
canonical_query = canonical_query(url.query)
canonical_request = """
#{method}
#{path}
#{canonical_query}
#{canonical_headers}
#{signed_headers}
#{body_digest}\
"""
string_to_sign =
iodata("""
AWS4-HMAC-SHA256
#{datetime_string}
#{date_string}/#{region}/#{service}/aws4_request
#{hex(sha256(canonical_request))}\
""")
signature =
aws_sigv4(
string_to_sign,
date_string,
region,
service,
secret_access_key
)
credential = "#{access_key_id}/#{date_string}/#{region}/#{service}/aws4_request"
authorization =
"AWS4-HMAC-SHA256 Credential=#{credential},SignedHeaders=#{signed_headers},Signature=#{signature}"
[{"authorization", authorization}] ++ aws_headers ++ headers
end
defp canonical_query(query) when query in [nil, ""] do
query
end
defp canonical_query(query) do
for item <- String.split(query, "&", trim: true) do
case String.split(item, "=") do
[name, value] -> [name, "=", value]
[name] -> [name, "="]
end
end
|> Enum.sort()
|> Enum.intersperse("&")
end
@doc """
Create AWS Signature v4 URL.
https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
"""
def aws_sigv4_url(options) do
{access_key_id, options} = Keyword.pop!(options, :access_key_id)
{secret_access_key, options} = Keyword.pop!(options, :secret_access_key)
{region, options} = Keyword.pop!(options, :region)
{service, options} = Keyword.pop!(options, :service)
{datetime, options} = Keyword.pop!(options, :datetime)
{method, options} = Keyword.pop!(options, :method)
{url, options} = Keyword.pop!(options, :url)
{expires, options} = Keyword.pop(options, :expires, 86400)
{headers, options} = Keyword.pop(options, :headers, [])
{query, options} = Keyword.pop(options, :query, [])
[] = options
datetime = DateTime.truncate(datetime, :second)
datetime_string = DateTime.to_iso8601(datetime, :basic)
date_string = Date.to_iso8601(datetime, :basic)
url = normalize_url(url)
service = to_string(service)
canonical_headers =
headers
|> canonical_host_header(url)
|> format_canonical_headers()
signed_headers = Enum.map_join(canonical_headers, ";", &elem(&1, 0))
canonical_query_string =
format_canonical_query_params(
[
{"X-Amz-Algorithm", "AWS4-HMAC-SHA256"},
{"X-Amz-Credential",
"#{access_key_id}/#{date_string}/#{region}/#{service}/aws4_request"},
{"X-Amz-Date", datetime_string},
{"X-Amz-Expires", expires},
{"X-Amz-SignedHeaders", signed_headers}
] ++ query
)
path = URI.encode(url.path || "/", &(&1 == ?/ or URI.char_unreserved?(&1)))
true = url.query in [nil, ""]
method = method |> Atom.to_string() |> String.upcase()
canonical_headers =
Enum.map_intersperse(canonical_headers, "\n", fn {name, value} -> [name, ":", value] end)
canonical_request = """
#{method}
#{path}
#{canonical_query_string}
#{canonical_headers}
#{signed_headers}
UNSIGNED-PAYLOAD\
"""
string_to_sign =
iodata("""
AWS4-HMAC-SHA256
#{datetime_string}
#{date_string}/#{region}/#{service}/aws4_request
#{hex(sha256(canonical_request))}\
""")
signature =
aws_sigv4(
string_to_sign,
date_string,
region,
service,
secret_access_key
)
%{url | path: path, query: canonical_query_string <> "&X-Amz-Signature=#{signature}"}
end
# Try decoding the path in case it was encoded earlier to prevent double encoding,
# as the path is encoded later in the corresponding function.
defp normalize_url(url) do
url = URI.parse(url)
case url.path do
nil -> url
path -> %{url | path: URI.decode(path)}
end
end
defp canonical_host_header(headers, %URI{} = url) do
{_host_headers, headers} = Enum.split_with(headers, &match?({"host", _value}, &1))
host_value =
if is_nil(url.port) or URI.default_port(url.scheme) == url.port do
url.host
else
"#{url.host}:#{url.port}"
end
[{"host", host_value} | headers]
end
# Headers must be sorted alphabetically by name
# Header names must be lower case
# Header values must be trimmed
# See https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
defp format_canonical_headers(headers) do
headers
|> Enum.map(&format_canonical_header/1)
|> Enum.sort(fn {name_1, _}, {name_2, _} -> name_1 < name_2 end)
end
defp format_canonical_header({name, value}) do
name =
name
|> to_string()
|> String.downcase(:ascii)
value =
value
|> to_string()
|> String.trim()
{name, value}
end
# Query params must be sorted alphabetically by name
# Query param name and values must be URI-encoded individually
# Query params must be sorted after encoding
# See https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
defp format_canonical_query_params(query_params) do
query_params
|> Enum.map(&format_canonical_query_param/1)
|> Enum.sort(&canonical_query_param_sorter/2)
|> Enum.map_join("&", fn {name, value} -> "#{name}=#{value}" end)
end
# Spaces must be encoded as %20, not as "+".
defp format_canonical_query_param({name, value}) do
name =
name
|> to_string()
|> URI.encode(&URI.char_unreserved?/1)
value =
value
|> to_string()
|> URI.encode(&URI.char_unreserved?/1)
{name, value}
end
defp canonical_query_param_sorter({name, value_1}, {name, value_2}), do: value_1 < value_2
defp canonical_query_param_sorter({name_1, _}, {name_2, _}), do: name_1 < name_2
def aws_sigv4(
string_to_sign,
date_string,
region,
service,
secret_access_key
) do
signature =
["AWS4", secret_access_key]
|> hmac(date_string)
|> hmac(region)
|> hmac(service)
|> hmac("aws4_request")
|> hmac(string_to_sign)
|> hex()
signature
end
defp hex(data) do
Base.encode16(data, case: :lower)
end
defp sha256(data) do
:crypto.hash(:sha256, data)
end
defp hmac(key, data) do
:crypto.mac(:hmac, :sha256, key, data)
end
@doc """
Formats a datetime as "HTTP Date".
## Examples
iex> Req.Utils.format_http_date(~U[2024-01-01 09:00:00Z])
"Mon, 01 Jan 2024 09:00:00 GMT"
"""
def format_http_date(datetime) do
Calendar.strftime(datetime, "%a, %d %b %Y %H:%M:%S GMT")
end
@doc """
Parses "HTTP Date" as datetime.
## Examples
iex> Req.Utils.parse_http_date("Mon, 01 Jan 2024 09:00:00 GMT")
{:ok, ~U[2024-01-01 09:00:00Z]}
"""
def parse_http_date(<<
day_name::binary-size(3),
", ",
day::binary-size(2),
" ",
month_name::binary-size(3),
" ",
year::binary-size(4),
" ",
time::binary-size(8),
" GMT"
>>) do
with {:ok, day_of_week} <- parse_day_name(day_name),
{day, ""} <- Integer.parse(day),
{:ok, month} <- parse_month_name(month_name),
{year, ""} <- Integer.parse(year),
{:ok, time} <- Time.from_iso8601(time),
{:ok, date} <- Date.new(year, month, day),
true <- day_of_week == Date.day_of_week(date) do
DateTime.new(date, time)
else
{:error, _} = e ->
e
_ ->
{:error, :invalid_format}
end
end
def parse_http_date(binary) when is_binary(binary) do
{:error, :invalid_format}
end
defp parse_month_name("Jan"), do: {:ok, 1}
defp parse_month_name("Feb"), do: {:ok, 2}
defp parse_month_name("Mar"), do: {:ok, 3}
defp parse_month_name("Apr"), do: {:ok, 4}
defp parse_month_name("May"), do: {:ok, 5}
defp parse_month_name("Jun"), do: {:ok, 6}
defp parse_month_name("Jul"), do: {:ok, 7}
defp parse_month_name("Aug"), do: {:ok, 8}
defp parse_month_name("Sep"), do: {:ok, 9}
defp parse_month_name("Oct"), do: {:ok, 10}
defp parse_month_name("Nov"), do: {:ok, 11}
defp parse_month_name("Dec"), do: {:ok, 12}
defp parse_month_name(_), do: :error
defp parse_day_name("Mon"), do: {:ok, 1}
defp parse_day_name("Tue"), do: {:ok, 2}
defp parse_day_name("Wed"), do: {:ok, 3}
defp parse_day_name("Thu"), do: {:ok, 4}
defp parse_day_name("Fri"), do: {:ok, 5}
defp parse_day_name("Sat"), do: {:ok, 6}
defp parse_day_name("Sun"), do: {:ok, 7}
defp parse_day_name(_), do: :error
@doc """
Parses "HTTP Date" as datetime or raises an error.
## Examples
iex> Req.Utils.parse_http_date!("Mon, 01 Jan 2024 09:00:00 GMT")
~U[2024-01-01 09:00:00Z]
iex> Req.Utils.parse_http_date!("Mon")
** (ArgumentError) cannot parse "Mon" as HTTP date, reason: :invalid_format
"""
def parse_http_date!(binary) do
case parse_http_date(binary) do
{:ok, datetime} ->
datetime
{:error, reason} ->
raise ArgumentError,
"cannot parse #{inspect(binary)} as HTTP date, reason: #{inspect(reason)}"
end
end
@doc """
Returns a stream where each element is gzipped.
## Examples
iex> gzipped = Req.Utils.stream_gzip(~w[foo bar baz]) |> Enum.to_list()
iex> :zlib.gunzip(gzipped)
"foobarbaz"
"""
def stream_gzip(enumerable) do
Stream.transform(
enumerable,
# start_fun
fn ->
z = :zlib.open()
# copied from :zlib.gzip/1
:ok = :zlib.deflateInit(z, :default, :deflated, 16 + 15, 8, :default)
z
end,
# reducer
fn chunk, z ->
case :zlib.deflate(z, chunk) do
# optimization: avoid emitting empty chunks
[] -> {[], z}
compressed -> {[compressed], z}
end
end,
# last_fun
fn z ->
last = :zlib.deflate(z, [], :finish)
:ok = :zlib.deflateEnd(z)
{[last], z}
end,
# after_fun
fn z -> :ok = :zlib.close(z) end
)
end
defmodule CollectWithHash do
@moduledoc false
defstruct [:collectable, :type]
defimpl Collectable do
def into(%{collectable: collectable, type: type}) do
{acc, collector} = Collectable.into(collectable)
new_collector = fn
{acc, hash}, {:cont, element} ->
hash = :crypto.hash_update(hash, element)
{collector.(acc, {:cont, element}), hash}
{acc, hash}, :done ->
hash = :crypto.hash_final(hash)
{collector.(acc, :done), hash}
{acc, hash}, :halt ->
{collector.(acc, :halt), hash}
end
hash = hash_init(type)
{{acc, hash}, new_collector}
end
defp hash_init(:sha1), do: :crypto.hash_init(:sha)
defp hash_init(type), do: :crypto.hash_init(type)
end
end
@doc """
Returns a collectable with hash.
## Examples
iex> collectable = Req.Utils.collect_with_hash([], :md5)
iex> Enum.into(Stream.duplicate("foo", 2), collectable)
{~w[foo foo], :erlang.md5("foofoo")}
"""
def collect_with_hash(collectable, type) do
%CollectWithHash{collectable: collectable, type: type}
end
@crlf "\r\n"
@doc """
Encodes fields into "multipart/form-data" format.
"""
def encode_form_multipart(fields, options \\ []) do
options = Keyword.validate!(options, [:boundary])
boundary =
options[:boundary] ||
Base.encode16(:crypto.strong_rand_bytes(16), padding: false, case: :lower)
footer = [["--", boundary, "--", @crlf]]
{body, size} =
fields
|> Enum.reduce({[], 0}, &add_form_parts(&2, encode_form_part(&1, boundary)))
|> add_form_parts({footer, IO.iodata_length(footer)})
%{
size: size,
content_type: "multipart/form-data; boundary=#{boundary}",
body: body
}
end
defp add_sizes(_, nil), do: nil
defp add_sizes(nil, _), do: nil
defp add_sizes(size1, size2), do: size1 + size2
defp add_form_parts({parts1, size1}, {parts2, size2})
when is_list(parts1) and is_list(parts2) do
{[parts1, parts2], add_sizes(size1, size2)}
end
defp add_form_parts({parts1, size1}, {parts2, size2}) do
{Stream.concat(parts1, parts2), add_sizes(size1, size2)}
end
defp encode_form_part({name, {value, options}}, boundary) do
options = Keyword.validate!(options, [:filename, :content_type, :size])
{parts, parts_size, options} =
case value do
integer when is_integer(integer) ->
part = Integer.to_string(integer)
{[part], byte_size(part), options}
value when is_binary(value) or is_list(value) ->
{[value], IO.iodata_length(value), options}
stream = %File.Stream{} ->
filename = Path.basename(stream.path)
# TODO: Simplify when we require Elixir v1.15
size =
if not Map.has_key?(stream, :node) or stream.node == node() do
File.stat!(stream.path).size
else
:erpc.call(stream.node, fn -> File.stat!(stream.path).size end)
end
options =
options
|> Keyword.put_new(:filename, filename)
|> Keyword.put_new_lazy(:content_type, fn ->
MIME.from_path(filename)
end)
{stream, size, options}
enum ->
size = Keyword.get(options, :size)
{enum, size, options}
end
params =
if filename = options[:filename] do
["; filename=\"", filename, "\""]
else
[]
end
headers =
if content_type = options[:content_type] do
["content-type: ", content_type, @crlf]
else
[]
end
headers = ["content-disposition: form-data; name=\"#{name}\"", params, @crlf, headers]
header = [["--", boundary, @crlf, headers, @crlf]]
{header, IO.iodata_length(header)}
|> add_form_parts({parts, parts_size})
|> add_form_parts({[@crlf], 2})
end
defp encode_form_part({name, value}, boundary) do
encode_form_part({name, {value, []}}, boundary)
end
@doc """
Loads .netrc file.
## Examples
iex> {:ok, pid} = StringIO.open(\"""
...> machine localhost
...> login foo
...> password bar
...> \""")
iex> Req.Utils.load_netrc(pid)
%{"localhost" => {"foo", "bar"}}
"""
def load_netrc(path_or_device) do
case read_netrc(path_or_device) do
{:ok, ""} ->
raise ".netrc file is empty"
{:ok, contents} ->
contents
|> String.trim()
|> String.split()
|> parse_netrc()
{:error, reason} ->
raise "error reading .netrc file: #{:file.format_error(reason)}"
end
end
defp read_netrc(path) when is_binary(path) do
File.read(path)
end
defp read_netrc(pid) when is_pid(pid) do
<<content::binary>> = IO.read(pid, :eof)
{:ok, content}
end
defp parse_netrc(credentials), do: parse_netrc(credentials, %{})
defp parse_netrc([], acc), do: acc
defp parse_netrc([_, machine, _, login, _, password | tail], acc) do
acc = Map.put(acc, String.trim(machine), {String.trim(login), String.trim(password)})
parse_netrc(tail, acc)
end
defp parse_netrc(_, _), do: raise("error parsing .netrc file")
@doc """
Parses HTTP Digest authentication header (RFC 7616).
## Examples
iex> Req.Utils.parse_http_digest(~s(Digest realm="example", nonce="abc123", qop="auth"))
%{"realm" => "example", "nonce" => "abc123", "qop" => "auth"}
"""
def parse_http_digest("Digest " <> rest) do
rest
|> String.split(",")
|> Enum.map(&String.trim/1)
|> Map.new(&parse_digest_header_part/1)
end
@doc """
Generates HTTP Digest authentication header (RFC 7616).
Takes a challenge header from the server's `WWW-Authenticate` response and generates
the corresponding `Authorization` header value for digest authentication.
Returns `{:ok, header_value}` on success or `{:error, reason}` on failure.
## Options
* `:count` - The nonce count (default: 1). Used for tracking request count with same nonce.
## Examples
challenge = ~s(Digest realm="example", nonce="abc123", qop="auth")
{:ok, value} = Req.Utils.http_digest_auth(challenge, "user", "pass", :get, "/path")
#=> {:ok, "Digest response=..."}
"""
def http_digest_auth(challenge_header, username, password, method, uri, opts \\ []) do
count = Keyword.get(opts, :count, 1)
challenge = parse_http_digest(challenge_header)
generate_digest_auth_header(challenge, username, password, method, uri, count)
end
defp parse_digest_header_part(part) do
case String.split(part, "=", parts: 2) do
[key, value] ->
{String.trim(key), unquote_digest_value(value)}
[key] ->
{String.trim(key), ""}
end
end
defp unquote_digest_value(value) do
value = String.trim(value)
if String.starts_with?(value, ~s(")) and String.ends_with?(value, ~s(")) do
value
|> String.slice(1..-2//1)
|> Macro.unescape_string()
else
value
end
end
defp quote_digest_value(str) when is_binary(str) do
inspect(str, printable_limit: :infinity)
end
defp digest_count_to_nc(count) do
String.pad_leading(Integer.to_string(count, 16), 8, "0")
end
defp generate_digest_auth_header(challenge, username, password, method, uri, count) do
algorithm = Map.get(challenge, "algorithm", "MD5")
realm = Map.get(challenge, "realm", "")
nonce = Map.get(challenge, "nonce", "")
opaque = Map.get(challenge, "opaque")
qop = Map.get(challenge, "qop")
cnonce = generate_digest_cnonce()
nc = digest_count_to_nc(count)
with {:ok, hash_func, sess?} <- digest_hash_function(algorithm) do
ha1 = hash_func.("#{username}:#{realm}:#{password}")
ha1 =
if sess? do
hash_func.("#{ha1}:#{nonce}:#{cnonce}")
else
ha1
end
method_str = method |> to_string() |> String.upcase()
ha2 = hash_func.("#{method_str}:#{uri}")
response =
if qop in ["auth", "auth-int"] do
hash_func.("#{ha1}:#{nonce}:#{nc}:#{cnonce}:#{qop}:#{ha2}")
else
hash_func.("#{ha1}:#{nonce}:#{ha2}")
end
build_digest_auth_header(
username: username,
realm: realm,
nonce: nonce,
uri: uri,
response: response,
qop: qop,
nc: nc,
cnonce: cnonce,
opaque: opaque,
algorithm: algorithm
)
end
end
defp digest_md5_hash(data) do
:md5
|> :crypto.hash(data)
|> Base.encode16(case: :lower)
end
defp digest_sha256_hash(data) do
:sha256
|> :crypto.hash(data)
|> Base.encode16(case: :lower)
end
defp generate_digest_cnonce do
16
|> :crypto.strong_rand_bytes()
|> Base.encode16(case: :lower)
end
defp digest_hash_function(algorithm) do
case String.upcase(algorithm) do
"MD5" -> {:ok, &digest_md5_hash/1, false}
"MD5-SESS" -> {:ok, &digest_md5_hash/1, true}
"SHA-256" -> {:ok, &digest_sha256_hash/1, false}
"SHA-256-SESS" -> {:ok, &digest_sha256_hash/1, true}
_ -> {:error, {:unsupported_digest_algorithm, algorithm}}
end
end
defp build_digest_auth_header(opts) do
opts =
Keyword.validate!(opts, [
:username,
:realm,
:nonce,
:uri,
:response,
:nc,
:cnonce,
:algorithm,
opaque: nil,
qop: nil
])
header_parts = Keyword.take(opts, [:username, :realm, :nonce, :uri, :response])
qop_parts =
if is_binary(opts[:qop]) do
Keyword.take(opts, [:qop, :nc, :cnonce])
end
opaque_part =
if is_binary(opts[:opaque]) do
Keyword.take(opts, [:opaque])
end
algorithm_parts =
if opts[:algorithm] |> String.upcase() |> String.ends_with?("-SESS") do
Keyword.take(opts, [:algorithm, :cnonce])
else
Keyword.take(opts, [:algorithm])
end
header =
header_parts
|> Keyword.merge(qop_parts || [])
|> Keyword.merge(opaque_part || [])
|> Keyword.merge(algorithm_parts)
|> Enum.map_join(", ", &encode_digest_header_part/1)
{:ok, "Digest #{header}"}
end
defp encode_digest_header_part({key, value}) when key in [:algorithm, :qop, :nc] do
"#{key}=#{value}"
end
defp encode_digest_header_part({key, value}) do
"#{key}=#{quote_digest_value(value)}"
end
end