diff --git a/.formatter.exs b/.formatter.exs index ca892dfb..fd297560 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -1,8 +1,5 @@ [ - inputs: [ - "*.exs", - "config/*.exs", - "lib/**/*.ex", - "test/**/*.{ex,exs}" - ] + inputs: + ["*.exs", "config/*.exs", "test/**/*.{ex,exs}"] ++ + (Path.wildcard("lib/**/*.ex") -- Path.wildcard("lib/hex/mint/**/*.ex")) ] diff --git a/lib/hex/application.ex b/lib/hex/application.ex index 18412b06..739ec11d 100644 --- a/lib/hex/application.ex +++ b/lib/hex/application.ex @@ -9,8 +9,6 @@ defmodule Hex.Application do Mix.SCM.append(Hex.SCM) Mix.RemoteConverger.register(Hex.RemoteConverger) - start_httpc() - opts = [strategy: :one_for_one, name: Hex.Supervisor] Supervisor.start_link(children(), opts) end @@ -33,18 +31,6 @@ defmodule Hex.Application do defp dev_setup, do: :ok end - defp start_httpc() do - :inets.start(:httpc, profile: :hex) - - opts = [ - max_sessions: 8, - max_keep_alive_length: 4, - keep_alive_timeout: 120_000 - ] - - :httpc.set_options(opts, :hex) - end - if Mix.env() == :test do defp children do [ @@ -52,6 +38,7 @@ defmodule Hex.Application do Hex.OAuth, Hex.Repo, Hex.State, + Hex.HTTP.Pool, Hex.Server, {Hex.Parallel, [:hex_fetcher]} ] @@ -63,6 +50,7 @@ defmodule Hex.Application do Hex.OAuth, Hex.Repo, Hex.State, + Hex.HTTP.Pool, Hex.Server, {Hex.Parallel, [:hex_fetcher]}, Hex.Registry.Server, diff --git a/lib/hex/http.ex b/lib/hex/http.ex index 7e8cd89a..0453b579 100644 --- a/lib/hex/http.ex +++ b/lib/hex/http.ex @@ -8,6 +8,8 @@ defmodule Hex.HTTP do @request_retries 2 @chunk_size 10_000 + alias Hex.HTTP.Pool + @spec config() :: :mix_hex_core.config() def config do %{ @@ -28,27 +30,14 @@ defmodule Hex.HTTP do @impl :mix_hex_http def request(method, url, headers, body, adapter_config) when is_map(adapter_config) do - {method, url, request, http_opts, timeout, profile} = + {method, url, headers, body, timeout} = prepare_request(method, url, headers, body, adapter_config) Hex.Shell.debug("Hex.HTTP.request(#{inspect(method)}, #{inspect(url)})") - result = - retry(method, request, http_opts, @request_retries, profile, fn request, http_opts -> - redirect(request, http_opts, @request_redirects, fn request, http_opts -> - timeout(request, http_opts, timeout, fn request, http_opts -> - :httpc.request(method, request, http_opts, [body_format: :binary], profile) - |> handle_response(method, url) - end) - end) - end) - - # Convert to hex_core expected format - case result do - {:ok, status, headers, body} -> - # Convert headers to map with binary keys/values for hex_core - headers = Map.new(headers, fn {k, v} -> {to_string(k), to_string(v)} end) - {:ok, {status, headers, body}} + case run_request(method, url, headers, body, timeout, nil) do + {:ok, status, headers_map, resp_body} -> + {:ok, {status, headers_map, resp_body}} {:error, reason} -> {:error, reason} @@ -58,42 +47,24 @@ defmodule Hex.HTTP do @impl :mix_hex_http def request_to_file(method, url, headers, body, filename, adapter_config) when is_map(adapter_config) do - {method, url, request, http_opts, timeout, profile} = + {method, url, headers, body, timeout} = prepare_request(method, url, headers, body, adapter_config) Hex.Shell.debug("Hex.HTTP.request_to_file(#{inspect(method)}, #{inspect(url)})") - filename_charlist = String.to_charlist(filename) - - result = - retry(method, request, http_opts, @request_retries, profile, fn request, http_opts -> - redirect(request, http_opts, @request_redirects, fn request, http_opts -> - timeout(request, http_opts, timeout, fn request, http_opts -> - :httpc.request(method, request, http_opts, [{:stream, filename_charlist}], profile) - |> handle_response_to_file(method, url) - end) - end) - end) - - # Convert to hex_core expected format - case result do - {:ok, status, headers} -> - # Convert headers to map with binary keys/values for hex_core - headers = Map.new(headers, fn {k, v} -> {to_string(k), to_string(v)} end) - {:ok, {status, headers}} + case run_request(method, url, headers, body, timeout, filename) do + {:ok, status, headers_map, _nil} -> + {:ok, {status, headers_map}} {:error, reason} -> + _ = File.rm(filename) {:error, reason} end end defp prepare_request(method, url, headers, body, adapter_config) do - # Convert method to atom if it's not already method = if is_binary(method), do: String.to_atom(method), else: method - # Convert URL to string if it's binary url = if is_binary(url), do: url, else: to_string(url) - - # Convert headers from map to our format headers = if is_map(headers), do: headers, else: Map.new(headers) headers = add_basic_auth_via_netrc(headers, url) @@ -103,306 +74,308 @@ defmodule Hex.HTTP do Hex.State.fetch!(:http_timeout, fn val -> if is_integer(val), do: val * 1000 end) || @request_timeout - # Handle progress callback for uploads progress_callback = Map.get(adapter_config, :progress_callback) {body, extra_headers} = wrap_body_with_progress(body, progress_callback) headers = Map.merge(headers, extra_headers) - # Work around httpc bug: disable connection reuse when using Expect: 100-continue - # httpc doesn't properly handle connection state when receiving final status (401) - # instead of 100 Continue response - headers = - if headers["expect"] == "100-continue" do - Map.put(headers, "connection", "close") - else - headers - end + {method, url, headers, body, timeout} + end - http_opts = build_http_opts(url, timeout) - request = build_request(url, headers, body) - profile = Hex.State.fetch!(:httpc_profile) + defp run_request(method, url, headers, body, timeout, filename) do + # Start with :inet (IPv4); retry will swap to :inet6 on network failures. + inet = :inet - {method, url, request, http_opts, timeout, profile} + retry(method, url, headers, body, timeout, inet, @request_retries, fn url, headers, inet -> + redirect(method, url, headers, body, timeout, inet, @request_redirects, fn url, + headers, + inet -> + timeout(timeout, fn -> + do_request(method, url, headers, body, timeout, inet, filename) + end) + end) + end) end - defp handle_response_to_file({:ok, :saved_to_file}, method, url) do - Hex.Shell.debug("Hex.HTTP.request_to_file(#{inspect(method)}, #{inspect(url)}) => 200") - {:ok, 200, %{}} - end + ## Core request via Mint-backed pool - defp handle_response_to_file({:ok, {{_version, code, _reason}, headers, _body}}, method, url) do - Hex.Shell.debug("Hex.HTTP.request_to_file(#{inspect(method)}, #{inspect(url)}) => #{code}") - headers = Map.new(headers, &decode_header/1) - handle_hex_message(headers["x-hex-message"]) - {:ok, code, headers} - end + defp do_request(method, url, headers, body, timeout, inet, filename) do + connect_opts = build_connect_opts(url, inet) - defp handle_response_to_file({:error, term}, method, url) do - Hex.Shell.debug( - "Hex.HTTP.request_to_file(#{inspect(method)}, #{inspect(url)}) => #{inspect(term, limit: :infinity, pretty: true)}" - ) + mint_method = method |> Atom.to_string() |> String.upcase() - {:error, term} - end + {mint_headers, mint_body} = build_mint_request(headers, body) + + pool_opts = [timeout: timeout, connect_opts: connect_opts] + + pool_result = + case filename do + nil -> + Pool.request(url, mint_method, mint_headers, mint_body, pool_opts) + + filename when is_binary(filename) -> + Pool.request_to_file(url, mint_method, mint_headers, mint_body, filename, pool_opts) + end - defp fallback(:inet), do: :inet6 - defp fallback(:inet6), do: :inet + case pool_result do + {:ok, status, resp_headers, resp_body} -> + headers_map = headers_to_map(resp_headers) + handle_hex_message(headers_map["x-hex-message"]) + Hex.Shell.debug("Hex.HTTP.request(#{inspect(method)}, #{inspect(url)}) => #{status}") + {:ok, status, headers_map, maybe_unzip(resp_body, headers_map)} - defp build_http_opts(url, timeout) do - [ - relaxed: true, - timeout: timeout, - ssl: Hex.HTTP.SSL.ssl_opts(url), - autoredirect: false - ] ++ proxy_config(url) + {:error, reason} -> + Hex.Shell.debug( + "Hex.HTTP.request(#{inspect(method)}, #{inspect(url)}) => #{inspect(reason, limit: :infinity, pretty: true)}" + ) + + {:error, reason} + end end - defp build_request(url, headers, body) do - url = String.to_charlist(url) - headers = Enum.map(headers, &encode_header/1) + defp maybe_unzip(nil, _headers), do: nil + defp maybe_unzip(body, headers) when is_binary(body), do: unzip(body, headers) - case body do - {content_type, body} -> - # content_type might already be a charlist from hex_core - content_type = - if is_binary(content_type) do - String.to_charlist(content_type) - else - content_type - end + defp build_mint_request(headers, body) do + base_headers = Enum.map(headers, fn {k, v} -> {to_string(k), to_string(v)} end) - {url, headers, content_type, body} + case body do + {content_type, binary} when is_binary(binary) -> + ct = if is_binary(content_type), do: content_type, else: to_string(content_type) + {[{"content-type", ct} | base_headers], binary} + + {content_type, {fun, initial_offset}} when is_function(fun, 1) -> + # Progress-callback streaming: send headers first, then feed the + # producer function chunk-by-chunk through the pool so the callback + # fires as bytes actually go out on the wire (for `mix hex.publish` + # progress output). + ct = if is_binary(content_type), do: content_type, else: to_string(content_type) + {[{"content-type", ct} | base_headers], {:stream, fun, initial_offset}} nil -> - {url, headers} + {base_headers, nil} :undefined -> - {url, headers} + {base_headers, nil} end end - defp encode_header({name, value}) do - {String.to_charlist(name), String.to_charlist(value)} - end + defp headers_to_map(headers) when is_list(headers) do + Enum.reduce(headers, %{}, fn {name, value}, acc -> + name = String.downcase(to_string(name)) + value = to_string(value) - defp retry(:get, request, http_opts, times, profile, fun) do - result = - case fun.(request, http_opts) do - {:http_error, _, _} = error -> - {:retry, error} + Map.update(acc, name, value, fn existing -> existing <> ", " <> value end) + end) + end - {:error, :socket_closed_remotely} = error -> - {:retry, error} + defp build_connect_opts(url, inet) do + uri = URI.parse(url) - {:error, {:failed_connect, [{:to_address, to_addr}, {inet, inet_l, reason}]}} - when inet in [:inet, :inet6] and - reason in [:ehostunreach, :enetunreach, :eprotonosupport, :nxdomain] -> - :httpc.set_options([ipfamily: fallback(inet)], profile) - {:retry, {:error, {:failed_connect, [{:to_address, to_addr}, {inet, inet_l, reason}]}}} + # Convert our :inet / :inet6 marker into Mint's transport flags. + inet_opts = + case inet do + :inet -> [inet4: true, inet6: false] + :inet6 -> [inet4: false, inet6: true] + end - other -> - {:noretry, other} + transport_opts = + case uri.scheme do + "https" -> inet_opts ++ Hex.HTTP.SSL.ssl_opts(url) + _ -> inet_opts end - case result do - {:retry, _} when times > 0 -> - retry(:get, request, http_opts, times - 1, profile, fun) + [transport_opts: transport_opts] ++ proxy_connect_opts(uri) + end + + ## Retry + + defp retry(:get, url, headers, body, timeout, inet, times, fun) do + case fun.(url, headers, inet) do + {:error, reason} = error -> + if retryable?(reason) and times > 0 do + new_inet = fallback_inet(reason, inet) + retry(:get, url, headers, body, timeout, new_inet, times - 1, fun) + else + error + end - {_other, result} -> - result + other -> + other end end - defp retry(_method, request, http_opts, _times, _profile, fun), do: fun.(request, http_opts) + defp retry(_method, url, headers, _body, _timeout, inet, _times, fun) do + fun.(url, headers, inet) + end - defp redirect(request, http_opts, times, fun) do - case fun.(request, http_opts) do - {:ok, code, headers, body} -> - case handle_redirect(code, headers) do - {:ok, location} when times > 0 -> - do_redirect(request, http_opts, location, times, fun) + defp retryable?(%Hex.Mint.TransportError{reason: :closed}), do: true + defp retryable?(%Hex.Mint.TransportError{reason: :timeout}), do: true + defp retryable?(%Hex.Mint.TransportError{reason: :econnrefused}), do: true + defp retryable?(%Hex.Mint.TransportError{reason: :ehostunreach}), do: true + defp retryable?(%Hex.Mint.TransportError{reason: :enetunreach}), do: true + defp retryable?(%Hex.Mint.TransportError{reason: :eprotonosupport}), do: true + defp retryable?(%Hex.Mint.TransportError{reason: :nxdomain}), do: true + defp retryable?(:disconnected), do: true + defp retryable?(:socket_closed_remotely), do: true + defp retryable?(_), do: false - {:ok, _location} -> - Mix.raise("Too many redirects") + defp fallback_inet(%Hex.Mint.TransportError{reason: reason}, inet) + when reason in [:ehostunreach, :enetunreach, :eprotonosupport, :nxdomain] do + case inet do + :inet -> :inet6 + :inet6 -> :inet + end + end - :error -> - {:ok, code, headers, body} - end + defp fallback_inet(_reason, inet), do: inet - {:ok, code, headers} -> - case handle_redirect(code, headers) do + ## Redirect + + defp redirect(method, url, headers, body, timeout, inet, times, fun) do + case fun.(url, headers, inet) do + {:ok, code, resp_headers, _resp_body} = resp -> + case handle_redirect(code, resp_headers) do {:ok, location} when times > 0 -> - do_redirect(request, http_opts, location, times, fun) + location = resolve_location(url, location) + redirect(method, location, headers, body, timeout, inet, times - 1, fun) - {:ok, _location} -> + {:ok, _} -> Mix.raise("Too many redirects") :error -> - {:ok, code, headers} + resp end - {:error, reason} -> - {:error, reason} + other -> + other end end - defp do_redirect(request, http_opts, location, times, fun) do - ssl_opts = Hex.HTTP.SSL.ssl_opts(to_string(location)) - http_opts = Keyword.put(http_opts, :ssl, ssl_opts) - - request - |> update_request(location) - |> redirect(http_opts, times - 1, fun) - end - - defp handle_redirect(code, headers) - when code in [301, 302, 303, 307, 308] do - if location = headers["location"] do - {:ok, location} - else - :error + defp handle_redirect(code, headers) when code in [301, 302, 303, 307, 308] do + case headers["location"] do + nil -> :error + loc -> {:ok, loc} end end - defp handle_redirect(_, _) do - :error - end + defp handle_redirect(_, _), do: :error - defp update_request({_url, headers, content_type, body}, new_url) do - {new_url, headers, content_type, body} - end + defp resolve_location(base, location) do + case URI.parse(location) do + %URI{host: nil} = uri -> + %URI{} = base_uri = URI.parse(base) + URI.to_string(%{base_uri | path: uri.path, query: uri.query, fragment: uri.fragment}) - defp update_request({_url, headers}, new_url) do - {new_url, headers} + _ -> + location + end end - defp timeout(request, http_opts, timeout, fun) do - Task.async(fn -> fun.(request, http_opts) end) - |> task_await(:timeout, timeout) - end + ## Timeout - defp task_await(%Task{ref: ref, pid: pid} = task, reason, timeout) do - receive do - {^ref, result} -> - Process.demonitor(ref, [:flush]) - result + defp timeout(timeout, fun) do + task = Task.async(fun) - {:DOWN, ^ref, _, _, _reason} -> + try do + Task.await(task, timeout) + catch + :exit, {:timeout, _} -> + Process.unlink(task.pid) + Process.exit(task.pid, :kill) {:error, :timeout} - after - timeout -> - Process.unlink(pid) - Process.exit(pid, reason) - task_await(task, :kill, timeout) end end - defp handle_response({:ok, {{_version, code, _reason}, headers, body}}, method, url) do - Hex.Shell.debug("Hex.HTTP.request(#{inspect(method)}, #{inspect(url)}) => #{code}") - - headers = Map.new(headers, &decode_header/1) - handle_hex_message(headers["x-hex-message"]) - {:ok, code, headers, unzip(body, headers)} - end - - defp handle_response({:error, term}, method, url) do - Hex.Shell.debug( - "Hex.HTTP.request(#{inspect(method)}, #{inspect(url)}) => #{inspect(term, limit: :infinity, pretty: true)}" - ) - - {:error, term} - end - - defp decode_header({name, value}) do - {List.to_string(name), List.to_string(value)} - end + ## Response helpers defp unzip(body, headers) do - content_encoding = headers["content-encoding"] || "" + encoding = headers["content-encoding"] || "" - if String.contains?(content_encoding, "gzip") do + if String.contains?(encoding, "gzip") do :zlib.gunzip(body) else body end end + ## Proxy + def proxy_config(url) do - {http_proxy, https_proxy} = proxy_setup() - proxy_auth(URI.parse(url), http_proxy, https_proxy) + uri = URI.parse(url) + proxy_connect_opts(uri) end - defp proxy_setup do + defp proxy_connect_opts(%URI{host: host, scheme: scheme} = uri) do no_proxy = no_proxy() - http_proxy = (proxy = Hex.State.fetch!(:http_proxy)) && proxy(:http, proxy, no_proxy) - https_proxy = (proxy = Hex.State.fetch!(:https_proxy)) && proxy(:https, proxy, no_proxy) - {http_proxy, https_proxy} - end - defp proxy(scheme, proxy, no_proxy) do - uri = URI.parse(proxy) + if host_in_no_proxy?(host, no_proxy) do + [] + else + case proxy_uri_for(scheme) do + nil -> + [] - if uri.host && uri.port do - host = String.to_charlist(uri.host) - :httpc.set_options([{proxy_scheme(scheme), {{host, uri.port}, no_proxy}}], :hex) - end + %URI{host: phost, port: pport} = proxy when not is_nil(phost) and not is_nil(pport) -> + proxy_opts = + case proxy.userinfo do + nil -> + [] - uri - end + userinfo -> + encoded = Base.encode64(userinfo) + [proxy_headers: [{"proxy-authorization", "Basic #{encoded}"}]] + end + + [proxy: {:http, phost, pport, proxy_opts}] - defp proxy_scheme(scheme) do - case scheme do - :http -> :proxy - :https -> :https_proxy + _ -> + [] + end + |> Kernel.++(maybe_proxy_ignored(uri)) end end - defp proxy_auth(%URI{scheme: "http"}, http_proxy, _https_proxy) do - proxy_auth(http_proxy) - end + defp proxy_connect_opts(_), do: [] - defp proxy_auth(%URI{scheme: "https"}, _http_proxy, https_proxy) do - proxy_auth(https_proxy) - end + defp maybe_proxy_ignored(_), do: [] - defp proxy_auth(nil) do - [] + defp proxy_uri_for("http") do + case Hex.State.fetch!(:http_proxy) do + nil -> nil + str -> URI.parse(str) + end end - defp proxy_auth(%URI{userinfo: nil}) do - [] + defp proxy_uri_for("https") do + case Hex.State.fetch!(:https_proxy) do + nil -> nil + str -> URI.parse(str) + end end - defp proxy_auth(%URI{userinfo: auth}) do - destructure [user, pass], String.split(auth, ":", parts: 2) + defp proxy_uri_for(_), do: nil - user = String.to_charlist(user) - pass = String.to_charlist(pass || "") - [proxy_auth: {user, pass}] - end - - defp no_proxy() do + defp no_proxy do (Hex.State.fetch!(:no_proxy) || "") |> String.split(",", trim: true) |> Enum.map(&String.trim/1) - |> Enum.map(&normalize_no_proxy_domain_desc/1) - |> Enum.map(&String.to_charlist/1) end - defp normalize_no_proxy_domain_desc(<<".", rest::binary>>) do - "*." <> rest - end + defp host_in_no_proxy?(_host, []), do: false - defp normalize_no_proxy_domain_desc(host) do - host + defp host_in_no_proxy?(host, patterns) do + Enum.any?(patterns, fn pattern -> + pattern = String.trim_leading(pattern, ".") + host == pattern or String.ends_with?(host, "." <> pattern) + end) end - def handle_hex_message(nil) do - :ok - end + ## Hex messages + + def handle_hex_message(nil), do: :ok - def handle_hex_message(header) do - {message, level} = :binary.list_to_bin(header) |> parse_hex_message + def handle_hex_message(header) when is_binary(header) do + {message, level} = parse_hex_message(header) case level do "warn" -> Hex.Shell.info("API warning: " <> message) @@ -411,6 +384,10 @@ defmodule Hex.HTTP do end end + def handle_hex_message(header) when is_list(header) do + handle_hex_message(:binary.list_to_bin(header)) + end + @space [?\s, ?\t] defp parse_hex_message(message) do @@ -430,9 +407,7 @@ defmodule Hex.HTTP do skip_trail_ws(rest, <>, "") end - defp skip_trail_ws("", str, _ws) do - str - end + defp skip_trail_ws("", str, _ws), do: str defp quoted("\"" <> rest), do: do_quoted(rest, "") @@ -451,6 +426,8 @@ defmodule Hex.HTTP do |> skip_trail_ws("", "") end + ## Auth + defp add_basic_auth_via_netrc(%{"authorization" => _} = headers, _url), do: headers defp add_basic_auth_via_netrc(%{} = headers, url) do @@ -466,6 +443,8 @@ defmodule Hex.HTTP do end end + ## Progress callback + defp wrap_body_with_progress(body, progress_callback) do case body do {content_type, binary_body} diff --git a/lib/hex/http/pool.ex b/lib/hex/http/pool.ex new file mode 100644 index 00000000..71415b32 --- /dev/null +++ b/lib/hex/http/pool.ex @@ -0,0 +1,150 @@ +defmodule Hex.HTTP.Pool do + @moduledoc false + + # Top-level Mint-based HTTP connection pool. + # + # Owns a Registry and a DynamicSupervisor. Each {scheme, host, port} maps to + # one `Hex.HTTP.Pool.Host` child which owns its own pool of + # `Hex.HTTP.Pool.Conn` processes. Protocol (HTTP/1 vs HTTP/2) is discovered + # inside the Conn on its first connect, so this layer is protocol-agnostic. + + use Supervisor + + alias Hex.HTTP.Pool.Host + + @registry Hex.HTTP.Pool.Registry + @dyn_sup Hex.HTTP.Pool.DynamicSupervisor + + def start_link(_opts \\ []) do + Supervisor.start_link(__MODULE__, [], name: __MODULE__) + end + + @impl true + def init(_) do + children = [ + {Registry, keys: :unique, name: @registry}, + {DynamicSupervisor, name: @dyn_sup, strategy: :one_for_one} + ] + + Supervisor.init(children, strategy: :rest_for_one) + end + + @doc """ + Performs an HTTP request via the appropriate per-host pool. + + Args: + * `url` - request URL (binary) + * `method` - binary method (e.g. "GET") + * `headers` - list of `{name, value}` tuples + * `body` - binary body or nil + * `opts` - keyword list, supports: + * `:timeout` - total timeout ms (default 15_000) + * `:connect_opts` - keyword list passed to `Hex.Mint.HTTP.connect/4` + + The pool key includes the IPv4/IPv6 variant (derived from `connect_opts`' + `:inet4`/`:inet6` transport flags) so `Hex.HTTP`'s IPv4↔IPv6 retry fallback + actually takes effect: a retry with the opposite inet variant gets its own + Host pool and connects with the new flags, rather than being routed to the + existing pool that still carries the original variant's connect_opts. + """ + def request(url, method, headers, body, opts) do + {scheme, host, port, path} = parse_url(url) + connect_opts = Keyword.get(opts, :connect_opts, []) + inet = inet_variant(connect_opts) + key = {scheme, host, port, inet} + + timeout = Keyword.get(opts, :timeout, 15_000) + + pid = get_or_start_host(key, connect_opts) + Host.request(pid, method, path, headers, body, timeout) + end + + @doc """ + Like `request/5` but streams the response body directly to `filename`. + + Returns `{:ok, status, headers, nil}` on success (body already on disk), + `{:error, reason}` on failure. On error or redirect the partial file is + truncated/removed so the caller never sees a half-written payload. + """ + def request_to_file(url, method, headers, body, filename, opts) do + {scheme, host, port, path} = parse_url(url) + connect_opts = Keyword.get(opts, :connect_opts, []) + inet = inet_variant(connect_opts) + key = {scheme, host, port, inet} + + timeout = Keyword.get(opts, :timeout, 15_000) + + pid = get_or_start_host(key, connect_opts) + Host.request_to_file(pid, method, path, headers, body, filename, timeout) + end + + defp inet_variant(connect_opts) do + transport_opts = Keyword.get(connect_opts, :transport_opts, []) + + case {Keyword.get(transport_opts, :inet4, true), Keyword.get(transport_opts, :inet6, false)} do + {true, false} -> :inet + {false, true} -> :inet6 + _ -> :default + end + end + + defp get_or_start_host(key, connect_opts) do + case Registry.lookup(@registry, key) do + [{pid, _}] -> pid + [] -> start_host(key, connect_opts) + end + end + + defp start_host(key, connect_opts) do + via = {:via, Registry, {@registry, key}} + arg = {key, connect_opts, [name: via]} + + spec = %{ + id: {Host, key}, + start: {Host, :start_link, [arg]}, + restart: :temporary + } + + case DynamicSupervisor.start_child(@dyn_sup, spec) do + {:ok, pid} -> + pid + + {:error, {:already_started, pid}} -> + pid + + {:error, reason} -> + case Registry.lookup(@registry, key) do + [{pid, _}] -> + pid + + [] -> + raise "failed to start Hex HTTP host pool for #{inspect(key)}: #{inspect(reason)}" + end + end + end + + ## URL parsing + + defp parse_url(url) when is_binary(url) do + uri = URI.parse(url) + scheme = String.to_atom(uri.scheme) + host = uri.host + port = uri.port || default_port(scheme) + + path = + (uri.path || "/") <> + case uri.query do + nil -> "" + q -> "?" <> q + end + + {scheme, host, port, path} + end + + defp default_port(:http), do: 80 + defp default_port(:https), do: 443 + + # For tests / debugging + @doc false + def __registry__, do: @registry +end diff --git a/lib/hex/http/pool/conn.ex b/lib/hex/http/pool/conn.ex new file mode 100644 index 00000000..cd337599 --- /dev/null +++ b/lib/hex/http/pool/conn.ex @@ -0,0 +1,376 @@ +defmodule Hex.HTTP.Pool.Conn do + @moduledoc false + + # A single Mint connection owned by its own GenServer. + # + # Because the connection is opened inside this process via `MintHTTP.connect/4`, + # the socket is owned by us from the start and no `controlling_process/2` + # transfer is ever needed. + # + # On first successful connect, we report the negotiated protocol and per-conn + # request capacity back to the parent `Hex.HTTP.Pool.Host`. Requests arrive + # as casts carrying the caller's `from` tuple; when the request completes we + # reply directly to that `from` via `GenServer.reply/2` and notify the host + # so it can decrement its in-flight count for load-based dispatch. + + use GenServer + + alias Hex.Mint.HTTP, as: MintHTTP + + @initial_backoff 1_000 + @max_backoff 30_000 + + def start_link({host_pid, key, connect_opts}) do + GenServer.start_link(__MODULE__, {host_pid, key, connect_opts}) + end + + @impl true + def init({host_pid, key, connect_opts}) do + state = %{ + host_pid: host_pid, + key: key, + connect_opts: connect_opts, + conn: nil, + protocol: nil, + capacity: 0, + requests: %{}, + backoff_ms: 0, + ready: false + } + + {:ok, state, {:continue, :connect}} + end + + @impl true + def handle_continue(:connect, state), do: do_connect(state) + + @impl true + def handle_cast( + {:request, from, method, path, headers, {:stream, fun, offset}}, + %{ready: true} = state + ) do + case MintHTTP.request(state.conn, method, path, headers, :stream) do + {:ok, conn, ref} -> + case stream_body(conn, ref, fun, offset) do + {:ok, conn} -> + req = %{from: from, status: nil, headers: [], data: []} + state = %{state | conn: conn, requests: Map.put(state.requests, ref, req)} + {:noreply, state} + + {:error, conn, reason} -> + GenServer.reply(from, {:error, reason}) + GenServer.cast(state.host_pid, {:req_done, self()}) + close_and_reconnect(%{state | conn: conn}) + end + + {:error, conn, reason} -> + request_error(from, reason, %{state | conn: conn}) + end + end + + def handle_cast({:request, from, method, path, headers, body}, %{ready: true} = state) do + case MintHTTP.request(state.conn, method, path, headers, body) do + {:ok, conn, ref} -> + req = %{from: from, status: nil, headers: [], data: [], sink: nil} + state = %{state | conn: conn, requests: Map.put(state.requests, ref, req)} + {:noreply, state} + + {:error, conn, reason} -> + request_error(from, reason, %{state | conn: conn}) + end + end + + def handle_cast({:request, from, _method, _path, _headers, _body}, state) do + # Host shouldn't dispatch to a non-ready conn; reply defensively. + GenServer.reply(from, {:error, :disconnected}) + GenServer.cast(state.host_pid, {:req_done, self()}) + {:noreply, state} + end + + def handle_cast( + {:request_to_file, from, method, path, headers, body, filename}, + %{ready: true} = state + ) do + case File.open(filename, [:write, :raw, :binary]) do + {:ok, fd} -> + case MintHTTP.request(state.conn, method, path, headers, body) do + {:ok, conn, ref} -> + req = %{ + from: from, + status: nil, + headers: [], + data: [], + sink: %{fd: fd, filename: filename} + } + + state = %{state | conn: conn, requests: Map.put(state.requests, ref, req)} + {:noreply, state} + + {:error, conn, reason} -> + _ = File.close(fd) + _ = File.rm(filename) + request_error(from, reason, %{state | conn: conn}) + end + + {:error, reason} -> + GenServer.reply(from, {:error, reason}) + GenServer.cast(state.host_pid, {:req_done, self()}) + {:noreply, state} + end + end + + def handle_cast({:request_to_file, from, _method, _path, _headers, _body, _filename}, state) do + GenServer.reply(from, {:error, :disconnected}) + GenServer.cast(state.host_pid, {:req_done, self()}) + {:noreply, state} + end + + @impl true + def handle_info(:reconnect, state), do: do_connect(state) + + def handle_info(message, %{conn: conn} = state) when conn != nil do + case MintHTTP.stream(conn, message) do + {:ok, conn, responses} -> + state = %{state | conn: conn} + state = Enum.reduce(responses, state, &process_response/2) + {:noreply, maybe_draining(state)} + + {:error, conn, reason, responses} -> + state = %{state | conn: conn} + state = Enum.reduce(responses, state, &process_response/2) + state = fail_in_flight(state, reason) + close_and_reconnect(state) + + :unknown -> + {:noreply, state} + end + end + + def handle_info(_, state), do: {:noreply, state} + + @impl true + def terminate(_reason, state) do + _ = fail_in_flight(state, :terminated) + if state.conn, do: safe_close(state.conn) + :ok + end + + defp stream_body(conn, ref, fun, offset) do + case fun.(offset) do + :eof -> + MintHTTP.stream_request_body(conn, ref, :eof) + + {:ok, chunk, next_offset} -> + case MintHTTP.stream_request_body(conn, ref, chunk) do + {:ok, conn} -> stream_body(conn, ref, fun, next_offset) + {:error, _conn, _reason} = err -> err + end + end + end + + defp request_error(from, reason, state) do + GenServer.reply(from, {:error, reason}) + GenServer.cast(state.host_pid, {:req_done, self()}) + + if MintHTTP.open?(state.conn, :write) do + {:noreply, state} + else + close_and_reconnect(state) + end + end + + ## Connect / reconnect + + defp do_connect(%{key: {scheme, host, port, _inet}, connect_opts: opts} = state) do + # Negotiate HTTP/2 via ALPN when the server supports it; fall back to HTTP/1. + # Both protocols are equivalent on `mix deps.get` wall time and HTTP/2 uses + # slightly less CPU (fewer TLS handshakes). Mint's default HTTP/2 receive + # windows (4 MB per stream, 16 MB per connection) are already tuned for bulk + # downloads, so no extra tuning is needed here. + opts = Keyword.merge([protocols: [:http1, :http2]], opts) + + case MintHTTP.connect(scheme, host, port, opts) do + {:ok, conn} -> + protocol = MintHTTP.protocol(conn) + capacity = compute_capacity(conn, protocol) + GenServer.cast(state.host_pid, {:conn_ready, self(), protocol, capacity}) + + {:noreply, + %{ + state + | conn: conn, + protocol: protocol, + capacity: capacity, + ready: true, + backoff_ms: 0 + }} + + {:error, reason} -> + schedule_reconnect(reason, %{state | conn: nil, ready: false}) + end + end + + defp close_and_reconnect(state) do + if state.conn, do: safe_close(state.conn) + schedule_reconnect(:closed, %{state | conn: nil, ready: false}) + end + + defp schedule_reconnect(reason, state) do + GenServer.cast(state.host_pid, {:conn_down, self(), reason}) + backoff = next_backoff(state.backoff_ms) + Process.send_after(self(), :reconnect, backoff) + {:noreply, %{state | backoff_ms: backoff}} + end + + defp next_backoff(0), do: @initial_backoff + defp next_backoff(n), do: min(n * 2, @max_backoff) + + defp compute_capacity(_conn, :http1), do: 1 + + defp compute_capacity(conn, :http2) do + case Hex.Mint.HTTP2.get_server_setting(conn, :max_concurrent_streams) do + n when is_integer(n) and n > 0 -> n + _ -> 100 + end + end + + ## Draining (server sent GOAWAY) + + defp maybe_draining(%{ready: true, conn: conn} = state) do + if MintHTTP.open?(conn, :write) do + state + else + GenServer.cast(state.host_pid, {:conn_draining, self()}) + drain_if_done(%{state | ready: false}) + end + end + + defp maybe_draining(state), do: drain_if_done(state) + + defp drain_if_done(%{ready: false, requests: reqs, conn: conn} = state) + when reqs == %{} and conn != nil do + safe_close(conn) + GenServer.cast(state.host_pid, {:conn_down, self(), :drained}) + Process.send_after(self(), :reconnect, 0) + %{state | conn: nil, backoff_ms: 0} + end + + defp drain_if_done(state), do: state + + ## Response handling + + defp process_response({:status, ref, status}, state) do + # A new status line starts a new response. Reset accumulated headers/data + # so that 1xx informational responses (100 Continue, 103 Early Hints) don't + # bleed headers into the final response that follows on the same ref. + # When streaming to a file, rewind the sink so any partial writes from a + # prior response on this ref are discarded. + update_in(state.requests[ref], fn + nil -> + nil + + %{sink: %{fd: fd}} = req -> + _ = :file.position(fd, 0) + _ = :file.truncate(fd) + %{req | status: status, headers: [], data: []} + + req -> + %{req | status: status, headers: [], data: []} + end) + end + + defp process_response({:headers, ref, headers}, state) do + update_in(state.requests[ref], fn req -> + req && %{req | headers: req.headers ++ headers} + end) + end + + defp process_response({:data, ref, chunk}, state) do + case state.requests[ref] do + %{sink: %{fd: fd}} -> + case :file.write(fd, chunk) do + :ok -> state + {:error, reason} -> abort_request(state, ref, reason) + end + + %{} = _req -> + update_in(state.requests[ref], fn req -> %{req | data: [req.data | chunk]} end) + + nil -> + state + end + end + + defp process_response({:done, ref}, state) do + case Map.pop(state.requests, ref) do + {nil, _} -> + state + + {%{sink: %{fd: fd}} = req, requests} -> + _ = File.close(fd) + GenServer.reply(req.from, {:ok, req.status, req.headers, nil}) + GenServer.cast(state.host_pid, {:req_done, self()}) + %{state | requests: requests} + + {req, requests} -> + body = IO.iodata_to_binary(req.data) + GenServer.reply(req.from, {:ok, req.status, req.headers, body}) + GenServer.cast(state.host_pid, {:req_done, self()}) + %{state | requests: requests} + end + end + + defp process_response({:error, ref, reason}, state) do + case Map.pop(state.requests, ref) do + {nil, _} -> + state + + {req, requests} -> + close_sink(req) + GenServer.reply(req.from, {:error, reason}) + GenServer.cast(state.host_pid, {:req_done, self()}) + %{state | requests: requests} + end + end + + defp process_response(_other, state), do: state + + defp abort_request(state, ref, reason) do + case Map.pop(state.requests, ref) do + {nil, _} -> + state + + {req, requests} -> + close_sink(req) + GenServer.reply(req.from, {:error, reason}) + GenServer.cast(state.host_pid, {:req_done, self()}) + %{state | requests: requests} + end + end + + defp close_sink(%{sink: %{fd: fd, filename: filename}}) do + _ = File.close(fd) + _ = File.rm(filename) + :ok + end + + defp close_sink(_), do: :ok + + defp fail_in_flight(state, reason) do + Enum.each(state.requests, fn {_ref, req} -> + close_sink(req) + GenServer.reply(req.from, {:error, reason}) + GenServer.cast(state.host_pid, {:req_done, self()}) + end) + + %{state | requests: %{}} + end + + defp safe_close(conn) do + try do + MintHTTP.close(conn) + catch + _, _ -> :ok + end + end +end diff --git a/lib/hex/http/pool/host.ex b/lib/hex/http/pool/host.ex new file mode 100644 index 00000000..1279c14e --- /dev/null +++ b/lib/hex/http/pool/host.ex @@ -0,0 +1,218 @@ +defmodule Hex.HTTP.Pool.Host do + @moduledoc false + + # Per-host pool. One GenServer per {scheme, host, port}. + # + # On start we spawn two probe `Conn` processes. When the first probe reports + # back with the negotiated protocol we decide the target pool size: + # + # * `:http1` → scale up to 8 conns to match the previous `:httpc` + # `max_sessions` behaviour (one in-flight request per conn). This is + # hex's default and what benchmarks show to be fastest for its workload. + # * `:http2` → stay at 2 conns (HTTP/2 multiplexes, and a second conn + # gives us redundancy while one is draining under GOAWAY). Kept for + # completeness; `Conn` currently pins `protocols: [:http1]` so HTTP/2 + # is not negotiated, but the capacity logic still handles it if a caller + # overrides `:protocols` in `:connect_opts`. + # + # Dispatch picks the conn with the fewest in-flight requests that still has + # free capacity (capacity is 1 for HTTP/1 and the server's advertised + # `max_concurrent_streams` for HTTP/2). When no conn has capacity, callers + # are enqueued and drained as requests finish. + # + # Requests are forwarded to the chosen `Conn` via cast with the caller's + # `from` tuple; the Conn replies directly to the caller and casts `:req_done` + # back here so we can decrement its in-flight count. + # + # Conns stay alive for the life of the BEAM. Hex runs as a CLI that exits + # at the end of the Mix task, at which point the supervisor terminates the + # pool and each Conn closes its socket in terminate/2 — so an idle-timeout + # reap path would only add complexity without saving real resources. + + use GenServer + + alias Hex.HTTP.Pool.Conn + + @initial_size 2 + @http1_size 8 + + def start_link({key, connect_opts, opts}) do + case Keyword.fetch(opts, :name) do + {:ok, name} -> GenServer.start_link(__MODULE__, {key, connect_opts}, name: name) + :error -> GenServer.start_link(__MODULE__, {key, connect_opts}) + end + end + + def request(pid, method, path, headers, body, timeout) do + GenServer.call(pid, {:request, method, path, headers, body}, timeout) + catch + :exit, {:timeout, _} -> {:error, :timeout} + :exit, {reason, _} -> {:error, reason} + end + + def request_to_file(pid, method, path, headers, body, filename, timeout) do + GenServer.call(pid, {:request_to_file, method, path, headers, body, filename}, timeout) + catch + :exit, {:timeout, _} -> {:error, :timeout} + :exit, {reason, _} -> {:error, reason} + end + + @impl true + def init({key, connect_opts}) do + Process.flag(:trap_exit, true) + + state = %{ + key: key, + connect_opts: connect_opts, + protocol: nil, + target_size: @initial_size, + conns: %{}, + waiters: :queue.new() + } + + state = Enum.reduce(1..@initial_size, state, fn _, s -> start_conn(s) end) + {:ok, state} + end + + @impl true + def handle_call({:request, method, path, headers, body}, from, state) do + dispatch(state, from, {:request, from, method, path, headers, body}) + end + + def handle_call({:request_to_file, method, path, headers, body, filename}, from, state) do + dispatch(state, from, {:request_to_file, from, method, path, headers, body, filename}) + end + + @impl true + def handle_cast({:conn_ready, conn_pid, protocol, capacity}, state) do + state = + update_conn(state, conn_pid, fn info -> + %{info | ready: true, capacity: capacity, in_flight: 0} + end) + + state = maybe_set_protocol(state, protocol) + state = drain_waiters(state) + {:noreply, state} + end + + def handle_cast({:conn_draining, conn_pid}, state) do + state = update_conn(state, conn_pid, fn info -> %{info | ready: false} end) + {:noreply, state} + end + + def handle_cast({:conn_down, conn_pid, _reason}, state) do + state = + update_conn(state, conn_pid, fn info -> + %{info | ready: false, in_flight: 0} + end) + + {:noreply, state} + end + + def handle_cast({:req_done, conn_pid}, state) do + state = + update_conn(state, conn_pid, fn info -> + %{info | in_flight: max(info.in_flight - 1, 0)} + end) + + {:noreply, drain_waiters(state)} + end + + @impl true + def handle_info({:EXIT, pid, _reason}, state) do + case Map.pop(state.conns, pid) do + {nil, _} -> + {:noreply, state} + + {_info, conns} -> + state = %{state | conns: conns} + {:noreply, start_conn(state)} + end + end + + def handle_info(_, state), do: {:noreply, state} + + ## Conn lifecycle + + defp start_conn(state) do + case Conn.start_link({self(), state.key, state.connect_opts}) do + {:ok, pid} -> + info = %{ready: false, in_flight: 0, capacity: 0} + %{state | conns: Map.put(state.conns, pid, info)} + + {:error, _reason} -> + state + end + end + + defp update_conn(state, pid, fun) do + case Map.get(state.conns, pid) do + nil -> state + info -> %{state | conns: Map.put(state.conns, pid, fun.(info))} + end + end + + defp maybe_set_protocol(%{protocol: nil} = state, :http1) do + state = %{state | protocol: :http1, target_size: @http1_size} + needed = @http1_size - map_size(state.conns) + + if needed > 0 do + Enum.reduce(1..needed, state, fn _, s -> start_conn(s) end) + else + state + end + end + + defp maybe_set_protocol(%{protocol: nil} = state, :http2), + do: %{state | protocol: :http2} + + defp maybe_set_protocol(state, _), do: state + + ## Dispatch + + defp pick_conn(state) do + best = + state.conns + |> Enum.filter(fn {_pid, info} -> + info.ready and info.in_flight < info.capacity + end) + |> Enum.min_by(fn {_pid, info} -> info.in_flight end, fn -> nil end) + + case best do + nil -> + :no_capacity + + {pid, info} -> + conns = Map.put(state.conns, pid, %{info | in_flight: info.in_flight + 1}) + {:ok, pid, %{state | conns: conns}} + end + end + + defp dispatch(state, from, cast_msg) do + case pick_conn(state) do + {:ok, conn_pid, state} -> + GenServer.cast(conn_pid, cast_msg) + {:noreply, state} + + :no_capacity -> + waiters = :queue.in({from, cast_msg}, state.waiters) + {:noreply, %{state | waiters: waiters}} + end + end + + defp drain_waiters(state) do + if :queue.is_empty(state.waiters) do + state + else + case pick_conn(state) do + {:ok, conn_pid, state} -> + {{:value, {_from, cast_msg}}, waiters} = :queue.out(state.waiters) + GenServer.cast(conn_pid, cast_msg) + drain_waiters(%{state | waiters: waiters}) + + :no_capacity -> + state + end + end + end +end diff --git a/lib/hex/mint/core/conn.ex b/lib/hex/mint/core/conn.ex new file mode 100644 index 00000000..fe882088 --- /dev/null +++ b/lib/hex/mint/core/conn.ex @@ -0,0 +1,67 @@ +# Vendored from mint v1.7.1 (d30d2cf), do not edit manually + +defmodule Hex.Mint.Core.Conn do + @moduledoc false + + alias Hex.Mint.Types + + @type conn() :: term() + + @callback initiate( + module(), + Hex.Mint.Types.socket(), + String.t(), + :inet.port_number(), + keyword() + ) :: {:ok, conn()} | {:error, Types.error()} + + @callback open?(conn(), :read | :write) :: boolean() + + @callback close(conn()) :: {:ok, conn()} + + @callback request( + conn(), + method :: String.t(), + path :: String.t(), + Types.headers(), + body :: iodata() | nil | :stream + ) :: + {:ok, conn(), Types.request_ref()} + | {:error, conn(), Types.error()} + + @callback stream_request_body( + conn(), + Types.request_ref(), + body_chunk :: iodata() | :eof | {:eof, trailer_headers :: Types.headers()} + ) :: + {:ok, conn()} | {:error, conn(), Types.error()} + + @callback stream(conn(), term()) :: + {:ok, conn(), [Types.response()]} + | {:error, conn(), Types.error(), [Types.response()]} + | :unknown + + @callback open_request_count(conn()) :: non_neg_integer() + + @callback recv(conn(), byte_count :: non_neg_integer(), timeout()) :: + {:ok, conn(), [Types.response()]} + | {:error, conn(), Types.error(), [Types.response()]} + + @callback set_mode(conn(), :active | :passive) :: {:ok, conn()} | {:error, Types.error()} + + @callback controlling_process(conn(), pid()) :: {:ok, conn()} | {:error, Types.error()} + + @callback put_private(conn(), key :: atom(), value :: term()) :: conn() + + @callback get_private(conn(), key :: atom(), default_value :: term()) :: term() + + @callback delete_private(conn(), key :: atom()) :: conn() + + @callback get_socket(conn()) :: Hex.Mint.Types.socket() + + @callback get_proxy_headers(conn()) :: Hex.Mint.Types.headers() + + @callback put_proxy_headers(conn(), Hex.Mint.Types.headers()) :: conn() + + @callback put_log(conn(), boolean()) :: conn() +end diff --git a/lib/hex/mint/core/headers.ex b/lib/hex/mint/core/headers.ex new file mode 100644 index 00000000..01d5ec36 --- /dev/null +++ b/lib/hex/mint/core/headers.ex @@ -0,0 +1,138 @@ +# Vendored from mint v1.7.1 (d30d2cf), do not edit manually + +defmodule Hex.Mint.Core.Headers do + @moduledoc false + + @type canonical() :: + {original_name :: String.t(), canonical_name :: String.t(), value :: String.t()} + @type raw() :: {original_name :: String.t(), value :: String.t()} + + @unallowed_trailers MapSet.new([ + "content-encoding", + "content-length", + "content-range", + "content-type", + "trailer", + "transfer-encoding", + + # Control headers (https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7231.html#rfc.section.5.1) + "cache-control", + "expect", + "host", + "max-forwards", + "pragma", + "range", + "te", + + # Conditionals (https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7231.html#rfc.section.5.2) + "if-match", + "if-none-match", + "if-modified-since", + "if-unmodified-since", + "if-range", + + # Authentication/authorization (https://tools.ietf.org/html/rfc7235#section-5.3) + "authorization", + "proxy-authenticate", + "proxy-authorization", + "www-authenticate", + + # Cookie management (https://tools.ietf.org/html/rfc6265) + "cookie", + "set-cookie", + + # Control data (https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7231.html#rfc.section.7.1) + "age", + "cache-control", + "expires", + "date", + "location", + "retry-after", + "vary", + "warning" + ]) + + @spec from_raw([raw()]) :: [canonical()] + def from_raw(headers) do + Enum.map(headers, fn {name, value} -> {name, lower_raw(name), value} end) + end + + @spec to_raw([canonical()], boolean()) :: [raw()] + def to_raw(headers, _case_sensitive = true) do + Enum.map(headers, fn {name, _canonical_name, value} -> {name, value} end) + end + + def to_raw(headers, _case_sensitive = false) do + Enum.map(headers, fn {_name, canonical_name, value} -> + {canonical_name, value} + end) + end + + @spec find([canonical()], String.t()) :: {String.t(), String.t()} | nil + def find(headers, name) do + case List.keyfind(headers, name, 1) do + nil -> nil + {name, _canonical_name, value} -> {name, value} + end + end + + @spec replace([canonical()], String.t(), String.t(), String.t()) :: + [canonical()] + def replace(headers, new_name, canonical_name, value) do + List.keyreplace(headers, canonical_name, 1, {new_name, canonical_name, value}) + end + + @spec has?([canonical()], String.t()) :: boolean() + def has?(headers, name) do + List.keymember?(headers, name, 1) + end + + @spec put_new([canonical()], String.t(), String.t(), String.t() | nil) :: + [canonical()] + def put_new(headers, _name, _canonical_name, nil) do + headers + end + + def put_new(headers, name, canonical_name, value) do + if List.keymember?(headers, canonical_name, 1) do + headers + else + [{name, canonical_name, value} | headers] + end + end + + @spec put_new([canonical()], String.t(), String.t(), (-> String.t())) :: + [canonical()] + def put_new_lazy(headers, name, canonical_name, fun) do + if List.keymember?(headers, canonical_name, 1) do + headers + else + [{name, canonical_name, fun.()} | headers] + end + end + + @spec find_unallowed_trailer([canonical()]) :: String.t() | nil + def find_unallowed_trailer(headers) do + Enum.find_value(headers, fn + {raw_name, canonical_name, _value} -> + if canonical_name in @unallowed_trailers do + raw_name + end + end) + end + + @spec remove_unallowed_trailer([raw()]) :: [raw()] + def remove_unallowed_trailer(headers) do + Enum.reject(headers, fn {name, _value} -> name in @unallowed_trailers end) + end + + @spec lower_raw(String.t()) :: String.t() + def lower_raw(name) do + String.downcase(name, :ascii) + end + + @spec lower_raws([raw()]) :: [raw()] + def lower_raws(headers) do + Enum.map(headers, fn {name, value} -> {lower_raw(name), value} end) + end +end diff --git a/lib/hex/mint/core/transport.ex b/lib/hex/mint/core/transport.ex new file mode 100644 index 00000000..24b59dcc --- /dev/null +++ b/lib/hex/mint/core/transport.ex @@ -0,0 +1,38 @@ +# Vendored from mint v1.7.1 (d30d2cf), do not edit manually + +defmodule Hex.Mint.Core.Transport do + @moduledoc false + + @type error() :: {:error, %Hex.Mint.TransportError{}} + + alias Hex.Mint.Types + + @callback connect(address :: Types.address(), port :: :inet.port_number(), opts :: keyword()) :: + {:ok, Types.socket()} | error() + + @callback upgrade( + Types.socket(), + original_scheme :: Types.scheme(), + hostname :: String.t(), + :inet.port_number(), + opts :: keyword() + ) :: {:ok, Types.socket()} | error() + + @callback negotiated_protocol(Types.socket()) :: + {:ok, protocol :: binary()} | {:error, :protocol_not_negotiated} + + @callback send(Types.socket(), payload :: iodata()) :: :ok | error() + + @callback close(Types.socket()) :: :ok | error() + + @callback recv(Types.socket(), bytes :: non_neg_integer(), timeout()) :: + {:ok, binary()} | error() + + @callback controlling_process(Types.socket(), pid()) :: :ok | error() + + @callback setopts(Types.socket(), opts :: keyword()) :: :ok | error() + + @callback getopts(Types.socket(), opts :: keyword()) :: {:ok, opts :: keyword()} | error() + + @callback wrap_error(reason :: term()) :: %Hex.Mint.TransportError{} +end diff --git a/lib/hex/mint/core/transport/ssl.ex b/lib/hex/mint/core/transport/ssl.ex new file mode 100644 index 00000000..620bc443 --- /dev/null +++ b/lib/hex/mint/core/transport/ssl.ex @@ -0,0 +1,752 @@ +# Vendored from mint v1.7.1 (d30d2cf), do not edit manually + +defmodule Hex.Mint.Core.Transport.SSL do + @moduledoc false + + require Record + + @behaviour Hex.Mint.Core.Transport + + # From RFC7540 appendix A + @blocked_ciphers MapSet.new([ + {:null, :null, :null}, + {:rsa, :null, :md5}, + {:rsa, :null, :sha}, + {:rsa_export, :rc4_40, :md5}, + {:rsa, :rc4_128, :md5}, + {:rsa, :rc4_128, :sha}, + {:rsa_export, :rc2_cbc_40, :md5}, + {:rsa, :idea_cbc, :sha}, + {:rsa_export, :des40_cbc, :sha}, + {:rsa, :des_cbc, :sha}, + {:rsa, :"3des_ede_cbc", :sha}, + {:dh_dss_export, :des40_cbc, :sha}, + {:dh_dss, :des_cbc, :sha}, + {:dh_dss, :"3des_ede_cbc", :sha}, + {:dh_rsa_export, :des40_cbc, :sha}, + {:dh_rsa, :des_cbc, :sha}, + {:dh_rsa, :"3des_ede_cbc", :sha}, + {:dhe_dss_export, :des40_cbc, :sha}, + {:dhe_dss, :des_cbc, :sha}, + {:dhe_dss, :"3des_ede_cbc", :sha}, + {:dhe_rsa_export, :des40_cbc, :sha}, + {:dhe_rsa, :des_cbc, :sha}, + {:dhe_rsa, :"3des_ede_cbc", :sha}, + {:dh_anon_export, :rc4_40, :md5}, + {:dh_anon, :rc4_128, :md5}, + {:dh_anon_export, :des40_cbc, :sha}, + {:dh_anon, :des_cbc, :sha}, + {:dh_anon, :"3des_ede_cbc", :sha}, + {:krb5, :des_cbc, :sha}, + {:krb5, :"3des_ede_cbc", :sha}, + {:krb5, :rc4_128, :sha}, + {:krb5, :idea_cbc, :sha}, + {:krb5, :des_cbc, :md5}, + {:krb5, :"3des_ede_cbc", :md5}, + {:krb5, :rc4_128, :md5}, + {:krb5, :idea_cbc, :md5}, + {:krb5_export, :des_cbc_40, :sha}, + {:krb5_export, :rc2_cbc_40, :sha}, + {:krb5_export, :rc4_40, :sha}, + {:krb5_export, :des_cbc_40, :md5}, + {:krb5_export, :rc2_cbc_40, :md5}, + {:krb5_export, :rc4_40, :md5}, + {:psk, :null, :sha}, + {:dhe_psk, :null, :sha}, + {:rsa_psk, :null, :sha}, + {:rsa, :aes_128_cbc, :sha}, + {:dh_dss, :aes_128_cbc, :sha}, + {:dh_rsa, :aes_128_cbc, :sha}, + {:dhe_dss, :aes_128_cbc, :sha}, + {:dhe_rsa, :aes_128_cbc, :sha}, + {:dh_anon, :aes_128_cbc, :sha}, + {:rsa, :aes_256_cbc, :sha}, + {:dh_dss, :aes_256_cbc, :sha}, + {:dh_rsa, :aes_256_cbc, :sha}, + {:dhe_dss, :aes_256_cbc, :sha}, + {:dhe_rsa, :aes_256_cbc, :sha}, + {:dh_anon, :aes_256_cbc, :sha}, + {:rsa, :null, :sha256}, + {:rsa, :aes_128_cbc, :sha256}, + {:rsa, :aes_256_cbc, :sha256}, + {:dh_dss, :aes_128_cbc, :sha256}, + {:dh_rsa, :aes_128_cbc, :sha256}, + {:dhe_dss, :aes_128_cbc, :sha256}, + {:rsa, :camellia_128_cbc, :sha}, + {:dh_dss, :camellia_128_cbc, :sha}, + {:dh_rsa, :camellia_128_cbc, :sha}, + {:dhe_dss, :camellia_128_cbc, :sha}, + {:dhe_rsa, :camellia_128_cbc, :sha}, + {:dh_anon, :camellia_128_cbc, :sha}, + {:dhe_rsa, :aes_128_cbc, :sha256}, + {:dh_dss, :aes_256_cbc, :sha256}, + {:dh_rsa, :aes_256_cbc, :sha256}, + {:dhe_dss, :aes_256_cbc, :sha256}, + {:dhe_rsa, :aes_256_cbc, :sha256}, + {:dh_anon, :aes_128_cbc, :sha256}, + {:dh_anon, :aes_256_cbc, :sha256}, + {:rsa, :camellia_256_cbc, :sha}, + {:dh_dss, :camellia_256_cbc, :sha}, + {:dh_rsa, :camellia_256_cbc, :sha}, + {:dhe_dss, :camellia_256_cbc, :sha}, + {:dhe_rsa, :camellia_256_cbc, :sha}, + {:dh_anon, :camellia_256_cbc, :sha}, + {:psk, :rc4_128, :sha}, + {:psk, :"3des_ede_cbc", :sha}, + {:psk, :aes_128_cbc, :sha}, + {:psk, :aes_256_cbc, :sha}, + {:dhe_psk, :rc4_128, :sha}, + {:dhe_psk, :"3des_ede_cbc", :sha}, + {:dhe_psk, :aes_128_cbc, :sha}, + {:dhe_psk, :aes_256_cbc, :sha}, + {:rsa_psk, :rc4_128, :sha}, + {:rsa_psk, :"3des_ede_cbc", :sha}, + {:rsa_psk, :aes_128_cbc, :sha}, + {:rsa_psk, :aes_256_cbc, :sha}, + {:rsa, :seed_cbc, :sha}, + {:dh_dss, :seed_cbc, :sha}, + {:dh_rsa, :seed_cbc, :sha}, + {:dhe_dss, :seed_cbc, :sha}, + {:dhe_rsa, :seed_cbc, :sha}, + {:dh_anon, :seed_cbc, :sha}, + {:rsa, :aes_128_gcm, :sha256}, + {:rsa, :aes_256_gcm, :sha384}, + {:dh_rsa, :aes_128_gcm, :sha256}, + {:dh_rsa, :aes_256_gcm, :sha384}, + {:dh_dss, :aes_128_gcm, :sha256}, + {:dh_dss, :aes_256_gcm, :sha384}, + {:dh_anon, :aes_128_gcm, :sha256}, + {:dh_anon, :aes_256_gcm, :sha384}, + {:psk, :aes_128_gcm, :sha256}, + {:psk, :aes_256_gcm, :sha384}, + {:rsa_psk, :aes_128_gcm, :sha256}, + {:rsa_psk, :aes_256_gcm, :sha384}, + {:psk, :aes_128_cbc, :sha256}, + {:psk, :aes_256_cbc, :sha384}, + {:psk, :null, :sha256}, + {:psk, :null, :sha384}, + {:dhe_psk, :aes_128_cbc, :sha256}, + {:dhe_psk, :aes_256_cbc, :sha384}, + {:dhe_psk, :null, :sha256}, + {:dhe_psk, :null, :sha384}, + {:rsa_psk, :aes_128_cbc, :sha256}, + {:rsa_psk, :aes_256_cbc, :sha384}, + {:rsa_psk, :null, :sha256}, + {:rsa_psk, :null, :sha384}, + {:rsa, :camellia_128_cbc, :sha256}, + {:dh_dss, :camellia_128_cbc, :sha256}, + {:dh_rsa, :camellia_128_cbc, :sha256}, + {:dhe_dss, :camellia_128_cbc, :sha256}, + {:dhe_rsa, :camellia_128_cbc, :sha256}, + {:dh_anon, :camellia_128_cbc, :sha256}, + {:rsa, :camellia_256_cbc, :sha256}, + {:dh_dss, :camellia_256_cbc, :sha256}, + {:dh_rsa, :camellia_256_cbc, :sha256}, + {:dhe_dss, :camellia_256_cbc, :sha256}, + {:dhe_rsa, :camellia_256_cbc, :sha256}, + {:dh_anon, :camellia_256_cbc, :sha256}, + {:ecdh_ecdsa, :null, :sha}, + {:ecdh_ecdsa, :rc4_128, :sha}, + {:ecdh_ecdsa, :"3des_ede_cbc", :sha}, + {:ecdh_ecdsa, :aes_128_cbc, :sha}, + {:ecdh_ecdsa, :aes_256_cbc, :sha}, + {:ecdhe_ecdsa, :null, :sha}, + {:ecdhe_ecdsa, :rc4_128, :sha}, + {:ecdhe_ecdsa, :"3des_ede_cbc", :sha}, + {:ecdhe_ecdsa, :aes_128_cbc, :sha}, + {:ecdhe_ecdsa, :aes_256_cbc, :sha}, + {:ecdh_rsa, :null, :sha}, + {:ecdh_rsa, :rc4_128, :sha}, + {:ecdh_rsa, :"3des_ede_cbc", :sha}, + {:ecdh_rsa, :aes_128_cbc, :sha}, + {:ecdh_rsa, :aes_256_cbc, :sha}, + {:ecdhe_rsa, :null, :sha}, + {:ecdhe_rsa, :rc4_128, :sha}, + {:ecdhe_rsa, :"3des_ede_cbc", :sha}, + {:ecdhe_rsa, :aes_128_cbc, :sha}, + {:ecdhe_rsa, :aes_256_cbc, :sha}, + {:ecdh_anon, :null, :sha}, + {:ecdh_anon, :rc4_128, :sha}, + {:ecdh_anon, :"3des_ede_cbc", :sha}, + {:ecdh_anon, :aes_128_cbc, :sha}, + {:ecdh_anon, :aes_256_cbc, :sha}, + {:srp_sha, :"3des_ede_cbc", :sha}, + {:srp_sha_rsa, :"3des_ede_cbc", :sha}, + {:srp_sha_dss, :"3des_ede_cbc", :sha}, + {:srp_sha, :aes_128_cbc, :sha}, + {:srp_sha_rsa, :aes_128_cbc, :sha}, + {:srp_sha_dss, :aes_128_cbc, :sha}, + {:srp_sha, :aes_256_cbc, :sha}, + {:srp_sha_rsa, :aes_256_cbc, :sha}, + {:srp_sha_dss, :aes_256_cbc, :sha}, + {:ecdhe_ecdsa, :aes_128_cbc, :sha256}, + {:ecdhe_ecdsa, :aes_256_cbc, :sha384}, + {:ecdh_ecdsa, :aes_128_cbc, :sha256}, + {:ecdh_ecdsa, :aes_256_cbc, :sha384}, + {:ecdhe_rsa, :aes_128_cbc, :sha256}, + {:ecdhe_rsa, :aes_256_cbc, :sha384}, + {:ecdh_rsa, :aes_128_cbc, :sha256}, + {:ecdh_rsa, :aes_256_cbc, :sha384}, + {:ecdh_ecdsa, :aes_128_gcm, :sha256}, + {:ecdh_ecdsa, :aes_256_gcm, :sha384}, + {:ecdh_rsa, :aes_128_gcm, :sha256}, + {:ecdh_rsa, :aes_256_gcm, :sha384}, + {:ecdhe_psk, :rc4_128, :sha}, + {:ecdhe_psk, :"3des_ede_cbc", :sha}, + {:ecdhe_psk, :aes_128_cbc, :sha}, + {:ecdhe_psk, :aes_256_cbc, :sha}, + {:ecdhe_psk, :aes_128_cbc, :sha256}, + {:ecdhe_psk, :aes_256_cbc, :sha384}, + {:ecdhe_psk, :null, :sha}, + {:ecdhe_psk, :null, :sha256}, + {:ecdhe_psk, :null, :sha384}, + {:rsa, :aria_128_cbc, :sha256}, + {:rsa, :aria_256_cbc, :sha384}, + {:dh_dss, :aria_128_cbc, :sha256}, + {:dh_dss, :aria_256_cbc, :sha384}, + {:dh_rsa, :aria_128_cbc, :sha256}, + {:dh_rsa, :aria_256_cbc, :sha384}, + {:dhe_dss, :aria_128_cbc, :sha256}, + {:dhe_dss, :aria_256_cbc, :sha384}, + {:dhe_rsa, :aria_128_cbc, :sha256}, + {:dhe_rsa, :aria_256_cbc, :sha384}, + {:dh_anon, :aria_128_cbc, :sha256}, + {:dh_anon, :aria_256_cbc, :sha384}, + {:ecdhe_ecdsa, :aria_128_cbc, :sha256}, + {:ecdhe_ecdsa, :aria_256_cbc, :sha384}, + {:ecdh_ecdsa, :aria_128_cbc, :sha256}, + {:ecdh_ecdsa, :aria_256_cbc, :sha384}, + {:ecdhe_rsa, :aria_128_cbc, :sha256}, + {:ecdhe_rsa, :aria_256_cbc, :sha384}, + {:ecdh_rsa, :aria_128_cbc, :sha256}, + {:ecdh_rsa, :aria_256_cbc, :sha384}, + {:rsa, :aria_128_gcm, :sha256}, + {:rsa, :aria_256_gcm, :sha384}, + {:dh_rsa, :aria_128_gcm, :sha256}, + {:dh_rsa, :aria_256_gcm, :sha384}, + {:dh_dss, :aria_128_gcm, :sha256}, + {:dh_dss, :aria_256_gcm, :sha384}, + {:dh_anon, :aria_128_gcm, :sha256}, + {:dh_anon, :aria_256_gcm, :sha384}, + {:ecdh_ecdsa, :aria_128_gcm, :sha256}, + {:ecdh_ecdsa, :aria_256_gcm, :sha384}, + {:ecdh_rsa, :aria_128_gcm, :sha256}, + {:ecdh_rsa, :aria_256_gcm, :sha384}, + {:psk, :aria_128_cbc, :sha256}, + {:psk, :aria_256_cbc, :sha384}, + {:dhe_psk, :aria_128_cbc, :sha256}, + {:dhe_psk, :aria_256_cbc, :sha384}, + {:rsa_psk, :aria_128_cbc, :sha256}, + {:rsa_psk, :aria_256_cbc, :sha384}, + {:psk, :aria_128_gcm, :sha256}, + {:psk, :aria_256_gcm, :sha384}, + {:rsa_psk, :aria_128_gcm, :sha256}, + {:rsa_psk, :aria_256_gcm, :sha384}, + {:ecdhe_psk, :aria_128_cbc, :sha256}, + {:ecdhe_psk, :aria_256_cbc, :sha384}, + {:ecdhe_ecdsa, :camellia_128_cbc, :sha256}, + {:ecdhe_ecdsa, :camellia_256_cbc, :sha384}, + {:ecdh_ecdsa, :camellia_128_cbc, :sha256}, + {:ecdh_ecdsa, :camellia_256_cbc, :sha384}, + {:ecdhe_rsa, :camellia_128_cbc, :sha256}, + {:ecdhe_rsa, :camellia_256_cbc, :sha384}, + {:ecdh_rsa, :camellia_128_cbc, :sha256}, + {:ecdh_rsa, :camellia_256_cbc, :sha384}, + {:rsa, :camellia_128_gcm, :sha256}, + {:rsa, :camellia_256_gcm, :sha384}, + {:dh_rsa, :camellia_128_gcm, :sha256}, + {:dh_rsa, :camellia_256_gcm, :sha384}, + {:dh_dss, :camellia_128_gcm, :sha256}, + {:dh_dss, :camellia_256_gcm, :sha384}, + {:dh_anon, :camellia_128_gcm, :sha256}, + {:dh_anon, :camellia_256_gcm, :sha384}, + {:ecdh_ecdsa, :camellia_128_gcm, :sha256}, + {:ecdh_ecdsa, :camellia_256_gcm, :sha384}, + {:ecdh_rsa, :camellia_128_gcm, :sha256}, + {:ecdh_rsa, :camellia_256_gcm, :sha384}, + {:psk, :camellia_128_gcm, :sha256}, + {:psk, :camellia_256_gcm, :sha384}, + {:rsa_psk, :camellia_128_gcm, :sha256}, + {:rsa_psk, :camellia_256_gcm, :sha384}, + {:psk, :camellia_128_cbc, :sha256}, + {:psk, :camellia_256_cbc, :sha384}, + {:dhe_psk, :camellia_128_cbc, :sha256}, + {:dhe_psk, :camellia_256_cbc, :sha384}, + {:rsa_psk, :camellia_128_cbc, :sha256}, + {:rsa_psk, :camellia_256_cbc, :sha384}, + {:ecdhe_psk, :camellia_128_cbc, :sha256}, + {:ecdhe_psk, :camellia_256_cbc, :sha384}, + {:rsa, :aes_128, :ccm}, + {:rsa, :aes_256, :ccm}, + {:rsa, :aes_128, :ccm_8}, + {:rsa, :aes_256, :ccm_8}, + {:psk, :aes_128, :ccm}, + {:psk, :aes_256, :ccm}, + {:psk, :aes_128, :ccm_8}, + {:psk, :aes_256, :ccm_8} + ]) + + @transport_opts [ + packet: :raw, + mode: :binary, + active: false + ] + + @default_versions [:"tlsv1.3", :"tlsv1.2"] + @default_timeout 30_000 + + Record.defrecordp( + :certificate, + :Certificate, + Record.extract(:Certificate, from_lib: "public_key/include/OTP-PUB-KEY.hrl") + ) + + Record.defrecordp( + :tbs_certificate, + :OTPTBSCertificate, + Record.extract(:OTPTBSCertificate, from_lib: "public_key/include/OTP-PUB-KEY.hrl") + ) + + # TODO: Document how to enable revocation checking: + # crl_check: true + # crl_cache: {:ssl_crl_cache, {:internal, [http: 30_000]}} + + @impl true + def connect(address, port, opts) do + hostname = Hex.Mint.Core.Util.hostname(opts, address) + opts = Keyword.delete(opts, :hostname) + + connect(address, hostname, port, opts) + end + + defp connect(address, hostname, port, opts) when is_binary(address), + do: connect(String.to_charlist(address), hostname, port, opts) + + defp connect(address, hostname, port, opts) do + timeout = Keyword.get(opts, :timeout, @default_timeout) + inet4? = Keyword.get(opts, :inet4, true) + inet6? = Keyword.get(opts, :inet6, false) + + opts = ssl_opts(String.to_charlist(hostname), opts) + + if inet6? do + # Try inet6 first, then fall back to the defaults provided by + # ssl/gen_tcp if connection fails. + case :ssl.connect(address, port, [:inet6 | opts], timeout) do + {:ok, sslsocket} -> + {:ok, sslsocket} + + _error when inet4? -> + wrap_err(:ssl.connect(address, port, opts, timeout)) + + error -> + wrap_err(error) + end + else + # Use the defaults provided by ssl/gen_tcp. + wrap_err(:ssl.connect(address, port, opts, timeout)) + end + end + + @impl true + def upgrade(socket, :http, hostname, _port, opts) do + hostname = String.to_charlist(hostname) + timeout = Keyword.get(opts, :timeout, @default_timeout) + + # Seems like this is not set in :ssl.connect/2 correctly, so set it explicitly + Hex.Mint.Core.Transport.TCP.setopts(socket, active: false) + + wrap_err(:ssl.connect(socket, ssl_opts(hostname, opts), timeout)) + end + + def upgrade(_socket, :https, _hostname, _port, _opts) do + raise "nested SSL sessions are not supported" + end + + @impl true + def negotiated_protocol(socket) do + wrap_err(:ssl.negotiated_protocol(socket)) + end + + @impl true + def send(socket, payload) do + wrap_err(:ssl.send(socket, payload)) + end + + @impl true + def close(socket) do + wrap_err(:ssl.close(socket)) + end + + @impl true + def recv(socket, bytes, timeout) do + wrap_err(:ssl.recv(socket, bytes, timeout)) + end + + @impl true + def controlling_process(socket, pid) do + # We do this dance because it's what gen_tcp does in Erlang. However, ssl + # doesn't do this so we need to do it ourselves. Implementation roughly + # taken from this: + # https://github.com/erlang/otp/blob/fc1f0444e32b039194189af97fb3d5358a2b91e3/lib/kernel/src/inet.erl#L1696-L1754 + with {:ok, active: active} <- getopts(socket, [:active]), + :ok <- setopts(socket, active: false), + :ok <- forward_messages_to_new_controlling_process(socket, pid), + :ok <- wrap_err(:ssl.controlling_process(socket, pid)) do + if(active == :once, do: setopts(socket, active: :once), else: :ok) + end + end + + defp forward_messages_to_new_controlling_process(socket, pid) do + receive do + {:ssl, ^socket, _data} = message -> + Kernel.send(pid, message) + forward_messages_to_new_controlling_process(socket, pid) + + {:ssl_error, ^socket, error} -> + {:error, error} + + {:ssl_closed, ^socket} -> + {:error, :closed} + after + 0 -> + :ok + end + end + + @impl true + def setopts(socket, opts) do + wrap_err(:ssl.setopts(socket, opts)) + end + + @impl true + def getopts(socket, opts) do + wrap_err(:ssl.getopts(socket, opts)) + end + + @impl true + def wrap_error(reason) do + %Hex.Mint.TransportError{reason: reason} + end + + defp ssl_opts(hostname, opts) do + default_ssl_opts(hostname) + |> Keyword.merge(opts) + |> Keyword.merge(@transport_opts) + |> Keyword.drop([:timeout, :inet4, :inet6]) + |> add_verify_opts(hostname) + |> remove_incompatible_ssl_opts() + |> add_ciphers_opt() + end + + defp add_verify_opts(opts, hostname) do + verify = Keyword.get(opts, :verify) + + if verify == :verify_peer do + opts + |> add_cacerts() + |> add_partial_chain_fun() + |> customize_hostname_check(hostname) + else + opts + end + end + + defp remove_incompatible_ssl_opts(opts) do + # These are the TLS versions that are compatible with :reuse_sessions and :secure_renegotiate + # If none of the compatible TLS versions are present in the transport options, then + # :reuse_sessions and :secure_renegotiate will be removed from the transport options. + compatible_versions = [:tlsv1, :"tlsv1.1", :"tlsv1.2"] + versions_opt = Keyword.get(opts, :versions, []) + + if Enum.any?(compatible_versions, &(&1 in versions_opt)) do + opts + else + opts + |> Keyword.delete(:reuse_sessions) + |> Keyword.delete(:secure_renegotiate) + end + end + + defp customize_hostname_check(opts, host_or_ip) do + if ssl_version() >= [9, 0] do + # From OTP 20.0 use built-in support for custom hostname checks + add_customize_hostname_check(opts) + else + # Before OTP 20.0 use mint_shims for hostname check, from a custom + # verify_fun + add_verify_fun(opts, host_or_ip) + end + end + + defp add_customize_hostname_check(opts) do + Keyword.put_new(opts, :customize_hostname_check, match_fun: &match_fun/2) + end + + defp add_verify_fun(opts, host_or_ip) do + Keyword.put_new_lazy(opts, :verify_fun, fn -> + reference_ids = [dns_id: host_or_ip, ip: host_or_ip] + {&verify_fun/3, reference_ids} + end) + end + + def verify_fun(_, {:bad_cert, _} = reason, _), do: {:fail, reason} + def verify_fun(_, {:extension, _}, state), do: {:unknown, state} + def verify_fun(_, :valid, state), do: {:valid, state} + + def verify_fun(cert, :valid_peer, state) do + if :hex_mint_shims.pkix_verify_hostname(cert, state, match_fun: &match_fun/2) do + {:valid, state} + else + {:fail, {:bad_cert, :hostname_check_failed}} + end + end + + # Wildcard domain handling for DNS ID entries in the subjectAltName X.509 + # extension. Note that this is a subset of the wildcard patterns implemented + # by OTP when matching against the subject CN attribute, but this is the only + # wildcard usage defined by the CA/Browser Forum's Baseline Requirements, and + # therefore the only pattern used in commercially issued certificates. + defp match_fun({:dns_id, reference}, {:dNSName, [?*, ?. | presented]}) do + case domain_without_host(reference) do + ~c"" -> :default + domain -> :string.casefold(domain) == :string.casefold(presented) + end + end + + # Workaround for a bug that was fixed in OTP 27: + # Before OTP 27 when connecting to an IP address and the server offers a + # certificate with its IP address in the "subject alternate names" extension, + # the TLS handshake fails with a `{:bad_cert, :hostname_check_failed}`. + # This clause can be removed once we depend on OTP 27+. + defp match_fun({:dns_id, hostname}, {:iPAddress, ip}) do + with {:ok, ip_tuple} <- :inet.parse_address(hostname), + ^ip <- Tuple.to_list(ip_tuple) do + true + else + _ -> :default + end + end + + defp match_fun(_reference, _presented), do: :default + + defp domain_without_host([]), do: [] + defp domain_without_host([?. | domain]), do: domain + defp domain_without_host([_ | more]), do: domain_without_host(more) + + defp add_ciphers_opt(opts) do + Keyword.put_new_lazy(opts, :ciphers, fn -> + versions = opts[:versions] + get_ciphers_for_versions(versions) + end) + end + + defp default_ssl_opts(hostname) do + # TODO: Add revocation check + + # Note: the :ciphers option is added once the :versions option + # has been merged with the user-specified value + [ + server_name_indication: hostname, + versions: ssl_versions(), + verify: :verify_peer, + depth: 4, + secure_renegotiate: true, + reuse_sessions: true + ] + end + + @doc false + def ssl_versions() do + available_versions = :ssl.versions()[:available] + versions = Enum.filter(@default_versions, &(&1 in available_versions)) + + # Remove buggy TLS 1.3 versions + if ssl_version() < [10, 0] do + versions -- [:"tlsv1.3"] + else + versions + end + end + + defp add_cacerts(opts) do + if Keyword.has_key?(opts, :cacertfile) or Keyword.has_key?(opts, :cacerts) do + opts + else + try do + Keyword.put(opts, :cacerts, :public_key.cacerts_get()) + rescue + _ -> + raise_on_missing_castore!() + Keyword.put(opts, :cacertfile, CAStore.file_path()) + end + end + end + + defp add_partial_chain_fun(opts) do + if Keyword.has_key?(opts, :partial_chain) do + opts + else + case Keyword.fetch(opts, :cacerts) do + {:ok, cacerts} -> + cacerts = decode_cacerts(cacerts) + fun = &partial_chain(cacerts, &1) + Keyword.put(opts, :partial_chain, fun) + + :error -> + path = Keyword.fetch!(opts, :cacertfile) + cacerts = get_cacertfile(path) + fun = &partial_chain(cacerts, &1) + Keyword.put(opts, :partial_chain, fun) + end + end + end + + defp get_cacertfile(path) do + case :persistent_term.get({:hex_mint, {:cacertfile, path}}, :error) do + {:ok, cacerts} -> + cacerts + + :error -> + cacerts = decode_cacertfile(path) + :persistent_term.put({:hex_mint, {:cacertfile, path}}, {:ok, cacerts}) + cacerts + end + end + + defp decode_cacertfile(path) do + path + |> File.read!() + |> :public_key.pem_decode() + |> Enum.filter(&match?({:Certificate, _, :not_encrypted}, &1)) + |> Enum.map(&:public_key.pem_entry_decode/1) + end + + defp decode_cacerts(certs) do + Enum.map(certs, fn + cert when is_binary(cert) -> :public_key.pkix_decode_cert(cert, :plain) + {:cert, _, otp_certificate} -> otp_certificate + end) + end + + def partial_chain(cacerts, certs) do + # TODO: Shim this with OTP 21.1 implementation? + + certs = + certs + |> Enum.map(&{&1, :public_key.pkix_decode_cert(&1, :plain)}) + |> Enum.drop_while(&cert_expired?/1) + + trusted = + Enum.find_value(certs, fn {der, cert} -> + trusted? = + Enum.find(cacerts, fn cacert -> + extract_public_key_info(cacert) == extract_public_key_info(cert) + end) + + if trusted?, do: der + end) + + if trusted do + {:trusted_ca, trusted} + else + :unknown_ca + end + end + + defp cert_expired?({_der, cert}) do + now = DateTime.utc_now() + {not_before, not_after} = extract_validity(cert) + + DateTime.compare(now, not_before) == :lt or + DateTime.compare(now, not_after) == :gt + end + + defp extract_validity(cert) do + {:Validity, not_before, not_after} = + cert + |> certificate(:tbsCertificate) + |> tbs_certificate(:validity) + + {to_datetime!(not_before), to_datetime!(not_after)} + end + + defp extract_public_key_info(cert) do + cert + |> certificate(:tbsCertificate) + |> tbs_certificate(:subjectPublicKeyInfo) + end + + defp to_datetime!({:utcTime, time}) do + "20#{time}" + |> to_datetime!() + end + + defp to_datetime!({:generalTime, time}) do + time + |> to_string() + |> to_datetime!() + end + + defp to_datetime!( + <> + ) do + {:ok, datetime, _} = + DateTime.from_iso8601("#{year}-#{month}-#{day}T#{hour}:#{minute}:#{second}Z") + + datetime + end + + defp blocked_cipher?(%{cipher: cipher, key_exchange: kex, prf: prf}), + do: blocked_cipher?({kex, cipher, prf}) + + defp blocked_cipher?({kex, cipher, _mac, prf}), do: blocked_cipher?({kex, cipher, prf}) + defp blocked_cipher?({_kex, _cipher, _prf} = suite), do: suite in @blocked_ciphers + + if Code.ensure_loaded?(CAStore) do + defp raise_on_missing_castore! do + :ok + end + else + defp raise_on_missing_castore! do + raise """ + default CA trust store not available; please add `:castore` to your project's \ + dependencies or specify the trust store using the :cacertfile/:cacerts option \ + within :transport_options. From OTP 25, you can also use: + + * :public_key.cacerts_get/0 to get certificates that you loaded from files or + * from the OS with :public_key.cacerts_load/0,1 + + See: https://www.erlang.org/blog/my-otp-25-highlights/#ca-certificates-can-be-fetched-from-the-os-standard-place + """ + end + end + + defp wrap_err({:error, reason}), do: {:error, wrap_error(reason)} + defp wrap_err(other), do: other + + @doc false + def ssl_version() do + Application.spec(:ssl, :vsn) + |> List.to_string() + |> String.split(".") + |> Enum.map(&String.to_integer/1) + end + + # Dialyzer warns on :ssl.cipher_suites/1 for now. + @dialyzer {:nowarn_function, get_ciphers_for_versions: 1} + + @doc false + def get_ciphers_for_versions(versions) do + if ssl_version() >= [8, 2, 4] do + # :ssl.filter_cipher_suites/2 is available in ssl v8.2.4+ + versions + |> Enum.flat_map(&:ssl.filter_cipher_suites(:ssl.cipher_suites(:all, &1), [])) + |> Enum.uniq() + else + :ssl.cipher_suites(:all) + end + |> Enum.reject(&blocked_cipher?/1) + end +end diff --git a/lib/hex/mint/core/transport/tcp.ex b/lib/hex/mint/core/transport/tcp.ex new file mode 100644 index 00000000..e981d71a --- /dev/null +++ b/lib/hex/mint/core/transport/tcp.ex @@ -0,0 +1,94 @@ +# Vendored from mint v1.7.1 (d30d2cf), do not edit manually + +defmodule Hex.Mint.Core.Transport.TCP do + @moduledoc false + + @behaviour Hex.Mint.Core.Transport + + @transport_opts [ + packet: :raw, + mode: :binary, + active: false + ] + + @default_timeout 30_000 + + @impl true + def connect(address, port, opts) when is_binary(address), + do: connect(String.to_charlist(address), port, opts) + + def connect(address, port, opts) do + opts = Keyword.delete(opts, :hostname) + + timeout = Keyword.get(opts, :timeout, @default_timeout) + inet4? = Keyword.get(opts, :inet4, true) + inet6? = Keyword.get(opts, :inet6, false) + + opts = + opts + |> Keyword.merge(@transport_opts) + |> Keyword.drop([:alpn_advertised_protocols, :timeout, :inet4, :inet6]) + + if inet6? do + # Try inet6 first, then fall back to the defaults provided by + # gen_tcp if connection fails. + case :gen_tcp.connect(address, port, [:inet6 | opts], timeout) do + {:ok, socket} -> + {:ok, socket} + + _error when inet4? -> + wrap_err(:gen_tcp.connect(address, port, opts, timeout)) + + error -> + wrap_err(error) + end + else + # Use the defaults provided by gen_tcp. + wrap_err(:gen_tcp.connect(address, port, opts, timeout)) + end + end + + @impl true + def upgrade(socket, _scheme, _hostname, _port, _opts) do + {:ok, socket} + end + + @impl true + def negotiated_protocol(_socket), do: wrap_err({:error, :protocol_not_negotiated}) + + @impl true + def send(socket, payload) do + wrap_err(:gen_tcp.send(socket, payload)) + end + + @impl true + defdelegate close(socket), to: :gen_tcp + + @impl true + def recv(socket, bytes, timeout) do + wrap_err(:gen_tcp.recv(socket, bytes, timeout)) + end + + @impl true + def controlling_process(socket, pid) do + wrap_err(:gen_tcp.controlling_process(socket, pid)) + end + + @impl true + def setopts(socket, opts) do + wrap_err(:inet.setopts(socket, opts)) + end + + @impl true + def getopts(socket, opts) do + wrap_err(:inet.getopts(socket, opts)) + end + + @impl true + def wrap_error(reason) do + %Hex.Mint.TransportError{reason: reason} + end + + defp wrap_err({:error, reason}), do: {:error, wrap_error(reason)} + defp wrap_err(other), do: other +end diff --git a/lib/hex/mint/core/util.ex b/lib/hex/mint/core/util.ex new file mode 100644 index 00000000..75e70f46 --- /dev/null +++ b/lib/hex/mint/core/util.ex @@ -0,0 +1,73 @@ +# Vendored from mint v1.7.1 (d30d2cf), do not edit manually + +defmodule Hex.Mint.Core.Util do + @moduledoc false + + alias Hex.Mint.Types + + @spec hostname(keyword(), String.t()) :: String.t() + def hostname(opts, address) when is_list(opts) do + case Keyword.fetch(opts, :hostname) do + {:ok, hostname} -> + hostname + + :error when is_binary(address) -> + address + + :error -> + raise ArgumentError, "the :hostname option is required when address is not a binary" + end + end + + @spec inet_opts(:gen_tcp | :ssl, :gen_tcp.socket() | :ssl.sslsocket()) :: :ok | {:error, term()} + def inet_opts(transport, socket) do + with {:ok, opts} <- transport.getopts(socket, [:sndbuf, :recbuf, :buffer]), + buffer = calculate_buffer(opts), + :ok <- transport.setopts(socket, buffer: buffer) do + :ok + end + end + + @spec scheme_to_transport(atom()) :: module() + def scheme_to_transport(:http), do: Hex.Mint.Core.Transport.TCP + def scheme_to_transport(:https), do: Hex.Mint.Core.Transport.SSL + def scheme_to_transport(module) when is_atom(module), do: module + + defp calculate_buffer(opts) do + Keyword.fetch!(opts, :buffer) + |> max(Keyword.fetch!(opts, :sndbuf)) + |> max(Keyword.fetch!(opts, :recbuf)) + end + + # Adds a header to the list of headers unless it's nil or it's already there. + @spec put_new_header(Types.headers(), String.t(), String.t() | nil) :: Types.headers() + def put_new_header(headers, name, value) + + def put_new_header(headers, _name, nil) do + headers + end + + def put_new_header(headers, name, value) do + if List.keymember?(headers, name, 0) do + headers + else + [{name, value} | headers] + end + end + + @spec put_new_header_lazy(Types.headers(), String.t(), (-> String.t())) :: Types.headers() + def put_new_header_lazy(headers, name, fun) do + if List.keymember?(headers, name, 0) do + headers + else + [{name, fun.()} | headers] + end + end + + # If the buffer is empty, reusing the incoming data saves + # a potentially large allocation of memory. + # This should be fixed in a subsequent OTP release. + @spec maybe_concat(binary(), binary()) :: binary() + def maybe_concat(<<>>, data), do: data + def maybe_concat(buffer, data) when is_binary(buffer), do: buffer <> data +end diff --git a/lib/hex/mint/hpax.ex b/lib/hex/mint/hpax.ex new file mode 100644 index 00000000..4b3de8ce --- /dev/null +++ b/lib/hex/mint/hpax.ex @@ -0,0 +1,362 @@ +# Vendored from hpax v1.0.3, do not edit manually + +defmodule Hex.Mint.HPAX do + _ = """ + Support for the HPACK header compression algorithm. + + This module provides support for the HPACK header compression algorithm used mainly in HTTP/2. + + ## Encoding and decoding contexts + + The HPACK algorithm requires both + + * an encoding context on the encoder side + * a decoding context on the decoder side + + These contexts are semantically different but structurally the same. In HPACK they are + implemented as **HPACK tables**. This library uses the name "tables" everywhere internally + + HPACK tables can be created through the `new/1` function. + """ + + alias Hex.Mint.HPAX.{Table, Types} + + @typedoc """ + An HPACK table. + + This can be used for encoding or decoding. + """ + @typedoc since: "0.2.0" + @opaque table() :: Table.t() + + @typedoc """ + An HPACK header name. + """ + @type header_name() :: binary() + + @typedoc """ + An HPACK header value. + """ + @type header_value() :: binary() + + @valid_header_actions [:store, :store_name, :no_store, :never_store] + + @doc """ + Creates a new HPACK table. + + Same as `new/2` with default options. + """ + @spec new(non_neg_integer()) :: table() + def new(max_table_size), do: new(max_table_size, []) + + @doc """ + Create a new HPACK table that can be used as encoding or decoding context. + + See the "Encoding and decoding contexts" section in the module documentation. + + `max_table_size` is the maximum table size (in bytes) for the newly created table. + + ## Options + + This function accepts the following `options`: + + * `:huffman_encoding` - (since 0.2.0) `:always` or `:never`. If `:always`, + then Hex.Mint.HPAX will always encode headers using Huffman encoding. If `:never`, + Hex.Mint.HPAX will not use any Huffman encoding. Defaults to `:never`. + + ## Examples + + encoding_context = Hex.Mint.HPAX.new(4096) + + """ + @doc since: "0.2.0" + @spec new(non_neg_integer(), [keyword()]) :: table() + def new(max_table_size, options) + when is_integer(max_table_size) and max_table_size >= 0 and is_list(options) do + options = Keyword.put_new(options, :huffman_encoding, :never) + + Enum.each(options, fn + {:huffman_encoding, _huffman_encoding} -> :ok + {key, _value} -> raise ArgumentError, "unknown option: #{inspect(key)}" + end) + + Table.new(max_table_size, Keyword.fetch!(options, :huffman_encoding)) + end + + @doc """ + Resizes the given table to the given maximum size. + + This is intended for use where the overlying protocol has signaled a change to the table's + maximum size, such as when an HTTP/2 `SETTINGS` frame is received. + + If the indicated size is less than the table's current size, entries + will be evicted as needed to fit within the specified size, and the table's + maximum size will be decreased to the specified value. A flag will also be + set which will enqueue a "dynamic table size update" command to be prefixed + to the next block encoded with this table, per + [RFC9113§4.3.1](https://www.rfc-editor.org/rfc/rfc9113.html#section-4.3.1). + + If the indicated size is greater than or equal to the table's current max size, no entries are evicted + and the table's maximum size changes to the specified value. + + ## Examples + + decoding_context = Hex.Mint.HPAX.new(4096) + Hex.Mint.HPAX.resize(decoding_context, 8192) + + """ + @spec resize(table(), non_neg_integer()) :: table() + defdelegate resize(table, new_max_size), to: Table + + @doc """ + Decodes a header block fragment (HBF) through a given table. + + If decoding is successful, this function returns a `{:ok, headers, updated_table}` tuple where + `headers` is a list of decoded headers, and `updated_table` is the updated table. If there's + an error in decoding, this function returns `{:error, reason}`. + + ## Examples + + decoding_context = Hex.Mint.HPAX.new(1000) + hbf = get_hbf_from_somewhere() + Hex.Mint.HPAX.decode(hbf, decoding_context) + #=> {:ok, [{":method", "GET"}], decoding_context} + + """ + @spec decode(binary(), table()) :: + {:ok, [{header_name(), header_value()}], table()} | {:error, term()} + + # Dynamic resizes must occur only at the start of a block + # https://datatracker.ietf.org/doc/html/rfc7541#section-4.2 + def decode(<<0b001::3, rest::bitstring>>, %Table{} = table) do + {new_max_size, rest} = decode_integer(rest, 5) + + # Dynamic resizes must be less than protocol max table size + # https://datatracker.ietf.org/doc/html/rfc7541#section-6.3 + if new_max_size <= table.protocol_max_table_size do + decode(rest, Table.dynamic_resize(table, new_max_size)) + else + {:error, :protocol_error} + end + end + + def decode(block, %Table{} = table) when is_binary(block) do + decode_headers(block, table, _acc = []) + catch + :throw, {:hpax, error} -> {:error, error} + end + + @doc """ + Encodes a list of headers through the given table. + + Returns a two-element tuple where the first element is a binary representing the encoded headers + and the second element is an updated table. + + ## Examples + + headers = [{:store, ":authority", "https://example.com"}] + encoding_context = Hex.Mint.HPAX.new(1000) + Hex.Mint.HPAX.encode(headers, encoding_context) + #=> {iodata, updated_encoding_context} + + """ + @spec encode([header], table()) :: {iodata(), table()} + when header: {action, header_name(), header_value()}, + action: :store | :store_name | :no_store | :never_store + def encode(headers, %Table{} = table) when is_list(headers) do + {table, pending_resizes} = Table.pop_pending_resizes(table) + acc = Enum.map(pending_resizes, &[<<0b001::3, Types.encode_integer(&1, 5)::bitstring>>]) + encode_headers(headers, table, acc) + end + + @doc """ + Encodes a list of headers through the given table, applying the same `action` to all of them. + + This function is the similar to `encode/2`, but `headers` are `{name, value}` tuples instead, + and the same `action` is applied to all headers. + + ## Examples + + headers = [{":authority", "https://example.com"}] + encoding_context = Hex.Mint.HPAX.new(1000) + Hex.Mint.HPAX.encode(:store, headers, encoding_context) + #=> {iodata, updated_encoding_context} + + """ + @doc since: "0.2.0" + @spec encode(action, [header], table()) :: {iodata(), table()} + when action: :store | :store_name | :no_store | :never_store, + header: {header_name(), header_value()} + def encode(action, headers, %Table{} = table) + when is_list(headers) and action in [:store, :store_name, :no_store, :never_store] do + headers + |> Enum.map(fn {name, value} -> {action, name, value} end) + |> encode(table) + end + + ## Helpers + + defp decode_headers(<<>>, table, acc) do + {:ok, Enum.reverse(acc), table} + end + + # Indexed header field + # http://httpwg.org/specs/rfc7541.html#rfc.section.6.1 + defp decode_headers(<<0b1::1, rest::bitstring>>, table, acc) do + {index, rest} = decode_integer(rest, 7) + decode_headers(rest, table, [lookup_by_index!(table, index) | acc]) + end + + # Literal header field with incremental indexing + # http://httpwg.org/specs/rfc7541.html#rfc.section.6.2.1 + defp decode_headers(<<0b01::2, rest::bitstring>>, table, acc) do + {name, value, rest} = + case rest do + # The header name is a string. + <<0::6, rest::binary>> -> + {name, rest} = decode_binary(rest) + {value, rest} = decode_binary(rest) + {name, value, rest} + + # The header name is an index to be looked up in the table. + _other -> + {index, rest} = decode_integer(rest, 6) + {value, rest} = decode_binary(rest) + {name, _value} = lookup_by_index!(table, index) + {name, value, rest} + end + + decode_headers(rest, Table.add(table, name, value), [{name, value} | acc]) + end + + # Literal header field without indexing + # http://httpwg.org/specs/rfc7541.html#rfc.section.6.2.2 + defp decode_headers(<<0b0000::4, rest::bitstring>>, table, acc) do + {name, value, rest} = + case rest do + <<0::4, rest::binary>> -> + {name, rest} = decode_binary(rest) + {value, rest} = decode_binary(rest) + {name, value, rest} + + _other -> + {index, rest} = decode_integer(rest, 4) + {value, rest} = decode_binary(rest) + {name, _value} = lookup_by_index!(table, index) + {name, value, rest} + end + + decode_headers(rest, table, [{name, value} | acc]) + end + + # Literal header field never indexed + # http://httpwg.org/specs/rfc7541.html#rfc.section.6.2.3 + defp decode_headers(<<0b0001::4, rest::bitstring>>, table, acc) do + {name, value, rest} = + case rest do + <<0::4, rest::binary>> -> + {name, rest} = decode_binary(rest) + {value, rest} = decode_binary(rest) + {name, value, rest} + + _other -> + {index, rest} = decode_integer(rest, 4) + {value, rest} = decode_binary(rest) + {name, _value} = lookup_by_index!(table, index) + {name, value, rest} + end + + # TODO: enforce the "never indexed" part somehow. + decode_headers(rest, table, [{name, value} | acc]) + end + + defp decode_headers(_other, _table, _acc) do + throw({:hpax, :protocol_error}) + end + + defp lookup_by_index!(table, index) do + case Table.lookup_by_index(table, index) do + {:ok, header} -> header + :error -> throw({:hpax, {:index_not_found, index}}) + end + end + + defp decode_integer(bitstring, prefix) do + case Types.decode_integer(bitstring, prefix) do + {:ok, int, rest} -> {int, rest} + :error -> throw({:hpax, :bad_integer_encoding}) + end + end + + defp decode_binary(binary) do + case Types.decode_binary(binary) do + {:ok, binary, rest} -> {binary, rest} + :error -> throw({:hpax, :bad_binary_encoding}) + end + end + + defp encode_headers([], table, acc) do + {acc, table} + end + + defp encode_headers([{action, name, value} | rest], table, acc) + when action in @valid_header_actions and is_binary(name) and is_binary(value) do + huffman? = table.huffman_encoding == :always + + {encoded, table} = + case Table.lookup_by_header(table, name, value) do + {:full, index} -> + {encode_indexed_header(index), table} + + {:name, index} when action == :store -> + {encode_literal_header_with_indexing(index, value, huffman?), + Table.add(table, name, value)} + + {:name, index} when action in [:store_name, :no_store] -> + {encode_literal_header_without_indexing(index, value, huffman?), table} + + {:name, index} when action == :never_store -> + {encode_literal_header_never_indexed(index, value, huffman?), table} + + :not_found when action in [:store, :store_name] -> + {encode_literal_header_with_indexing(name, value, huffman?), + Table.add(table, name, value)} + + :not_found when action == :no_store -> + {encode_literal_header_without_indexing(name, value, huffman?), table} + + :not_found when action == :never_store -> + {encode_literal_header_never_indexed(name, value, huffman?), table} + end + + encode_headers(rest, table, [acc, encoded]) + end + + defp encode_indexed_header(index) do + <<1::1, Types.encode_integer(index, 7)::bitstring>> + end + + defp encode_literal_header_with_indexing(index, value, huffman?) when is_integer(index) do + [<<1::2, Types.encode_integer(index, 6)::bitstring>>, Types.encode_binary(value, huffman?)] + end + + defp encode_literal_header_with_indexing(name, value, huffman?) when is_binary(name) do + [<<1::2, 0::6>>, Types.encode_binary(name, huffman?), Types.encode_binary(value, huffman?)] + end + + defp encode_literal_header_without_indexing(index, value, huffman?) when is_integer(index) do + [<<0::4, Types.encode_integer(index, 4)::bitstring>>, Types.encode_binary(value, huffman?)] + end + + defp encode_literal_header_without_indexing(name, value, huffman?) when is_binary(name) do + [<<0::4, 0::4>>, Types.encode_binary(name, huffman?), Types.encode_binary(value, huffman?)] + end + + defp encode_literal_header_never_indexed(index, value, huffman?) when is_integer(index) do + [<<1::4, Types.encode_integer(index, 4)::bitstring>>, Types.encode_binary(value, huffman?)] + end + + defp encode_literal_header_never_indexed(name, value, huffman?) when is_binary(name) do + [<<1::4, 0::4>>, Types.encode_binary(name, huffman?), Types.encode_binary(value, huffman?)] + end +end diff --git a/lib/hex/mint/hpax/huffman.ex b/lib/hex/mint/hpax/huffman.ex new file mode 100644 index 00000000..c94934f8 --- /dev/null +++ b/lib/hex/mint/hpax/huffman.ex @@ -0,0 +1,96 @@ +# Vendored from hpax v1.0.3, do not edit manually + +defmodule Hex.Mint.HPAX.Huffman do + @moduledoc false + + import Bitwise, only: [>>>: 2] + + # This file is downloaded from the spec directly. + # http://httpwg.org/specs/rfc7541.html#huffman.code + table_file = Path.absname("huffman_table", __DIR__) + @external_resource table_file + + entries = + Enum.map(File.stream!(table_file), fn line -> + [byte_value, bits, _hex, bit_count] = + line + |> case do + <> -> rest + "EOS " <> rest -> rest + _other -> line + end + |> String.replace(["|", "(", ")", "[", "]"], "") + |> String.split() + + byte_value = String.to_integer(byte_value) + bits = String.to_integer(bits, 2) + bit_count = String.to_integer(bit_count) + + {byte_value, bits, bit_count} + end) + + {regular_entries, [eos_entry]} = Enum.split(entries, -1) + {_eos_byte_value, eos_bits, eos_bit_count} = eos_entry + + ## Encoding + + @spec encode(binary()) :: binary() + def encode(binary) do + encode(binary, _acc = <<>>) + end + + for {byte_value, bits, bit_count} <- regular_entries do + defp encode(<>, acc) do + encode(rest, <>) + end + end + + defp encode(<<>>, acc) do + overflowing_bits = rem(bit_size(acc), 8) + + if overflowing_bits == 0 do + acc + else + bits_to_add = 8 - overflowing_bits + + value_of_bits_to_add = + take_significant_bits(unquote(eos_bits), unquote(eos_bit_count), bits_to_add) + + <> + end + end + + ## Decoding + + @spec decode(binary()) :: binary() + def decode(binary) + + for {byte_value, bits, bit_count} <- regular_entries do + def decode(<>) do + <> + end + end + + def decode(<<>>) do + <<>> + end + + # Use binary syntax for single match context optimization. + def decode(<>) when bit_size(padding) in 1..7 do + padding_size = bit_size(padding) + <> = padding + + if take_significant_bits(unquote(eos_bits), unquote(eos_bit_count), padding_size) == padding do + <<>> + else + throw({:hpax, {:protocol_error, :invalid_huffman_encoding}}) + end + end + + ## Helpers + + @compile {:inline, take_significant_bits: 3} + defp take_significant_bits(value, bit_count, bits_to_take) do + value >>> (bit_count - bits_to_take) + end +end diff --git a/lib/hex/mint/hpax/huffman_table b/lib/hex/mint/hpax/huffman_table new file mode 100644 index 00000000..b116ba3a --- /dev/null +++ b/lib/hex/mint/hpax/huffman_table @@ -0,0 +1,257 @@ +( 0) |11111111|11000 1ff8 [13] +( 1) |11111111|11111111|1011000 7fffd8 [23] +( 2) |11111111|11111111|11111110|0010 fffffe2 [28] +( 3) |11111111|11111111|11111110|0011 fffffe3 [28] +( 4) |11111111|11111111|11111110|0100 fffffe4 [28] +( 5) |11111111|11111111|11111110|0101 fffffe5 [28] +( 6) |11111111|11111111|11111110|0110 fffffe6 [28] +( 7) |11111111|11111111|11111110|0111 fffffe7 [28] +( 8) |11111111|11111111|11111110|1000 fffffe8 [28] +( 9) |11111111|11111111|11101010 ffffea [24] +( 10) |11111111|11111111|11111111|111100 3ffffffc [30] +( 11) |11111111|11111111|11111110|1001 fffffe9 [28] +( 12) |11111111|11111111|11111110|1010 fffffea [28] +( 13) |11111111|11111111|11111111|111101 3ffffffd [30] +( 14) |11111111|11111111|11111110|1011 fffffeb [28] +( 15) |11111111|11111111|11111110|1100 fffffec [28] +( 16) |11111111|11111111|11111110|1101 fffffed [28] +( 17) |11111111|11111111|11111110|1110 fffffee [28] +( 18) |11111111|11111111|11111110|1111 fffffef [28] +( 19) |11111111|11111111|11111111|0000 ffffff0 [28] +( 20) |11111111|11111111|11111111|0001 ffffff1 [28] +( 21) |11111111|11111111|11111111|0010 ffffff2 [28] +( 22) |11111111|11111111|11111111|111110 3ffffffe [30] +( 23) |11111111|11111111|11111111|0011 ffffff3 [28] +( 24) |11111111|11111111|11111111|0100 ffffff4 [28] +( 25) |11111111|11111111|11111111|0101 ffffff5 [28] +( 26) |11111111|11111111|11111111|0110 ffffff6 [28] +( 27) |11111111|11111111|11111111|0111 ffffff7 [28] +( 28) |11111111|11111111|11111111|1000 ffffff8 [28] +( 29) |11111111|11111111|11111111|1001 ffffff9 [28] +( 30) |11111111|11111111|11111111|1010 ffffffa [28] +( 31) |11111111|11111111|11111111|1011 ffffffb [28] +' ' ( 32) |010100 14 [ 6] +'!' ( 33) |11111110|00 3f8 [10] +'"' ( 34) |11111110|01 3f9 [10] +'#' ( 35) |11111111|1010 ffa [12] +'$' ( 36) |11111111|11001 1ff9 [13] +'%' ( 37) |010101 15 [ 6] +'&' ( 38) |11111000 f8 [ 8] +''' ( 39) |11111111|010 7fa [11] +'(' ( 40) |11111110|10 3fa [10] +')' ( 41) |11111110|11 3fb [10] +'*' ( 42) |11111001 f9 [ 8] +'+' ( 43) |11111111|011 7fb [11] +',' ( 44) |11111010 fa [ 8] +'-' ( 45) |010110 16 [ 6] +'.' ( 46) |010111 17 [ 6] +'/' ( 47) |011000 18 [ 6] +'0' ( 48) |00000 0 [ 5] +'1' ( 49) |00001 1 [ 5] +'2' ( 50) |00010 2 [ 5] +'3' ( 51) |011001 19 [ 6] +'4' ( 52) |011010 1a [ 6] +'5' ( 53) |011011 1b [ 6] +'6' ( 54) |011100 1c [ 6] +'7' ( 55) |011101 1d [ 6] +'8' ( 56) |011110 1e [ 6] +'9' ( 57) |011111 1f [ 6] +':' ( 58) |1011100 5c [ 7] +';' ( 59) |11111011 fb [ 8] +'<' ( 60) |11111111|1111100 7ffc [15] +'=' ( 61) |100000 20 [ 6] +'>' ( 62) |11111111|1011 ffb [12] +'?' ( 63) |11111111|00 3fc [10] +'@' ( 64) |11111111|11010 1ffa [13] +'A' ( 65) |100001 21 [ 6] +'B' ( 66) |1011101 5d [ 7] +'C' ( 67) |1011110 5e [ 7] +'D' ( 68) |1011111 5f [ 7] +'E' ( 69) |1100000 60 [ 7] +'F' ( 70) |1100001 61 [ 7] +'G' ( 71) |1100010 62 [ 7] +'H' ( 72) |1100011 63 [ 7] +'I' ( 73) |1100100 64 [ 7] +'J' ( 74) |1100101 65 [ 7] +'K' ( 75) |1100110 66 [ 7] +'L' ( 76) |1100111 67 [ 7] +'M' ( 77) |1101000 68 [ 7] +'N' ( 78) |1101001 69 [ 7] +'O' ( 79) |1101010 6a [ 7] +'P' ( 80) |1101011 6b [ 7] +'Q' ( 81) |1101100 6c [ 7] +'R' ( 82) |1101101 6d [ 7] +'S' ( 83) |1101110 6e [ 7] +'T' ( 84) |1101111 6f [ 7] +'U' ( 85) |1110000 70 [ 7] +'V' ( 86) |1110001 71 [ 7] +'W' ( 87) |1110010 72 [ 7] +'X' ( 88) |11111100 fc [ 8] +'Y' ( 89) |1110011 73 [ 7] +'Z' ( 90) |11111101 fd [ 8] +'[' ( 91) |11111111|11011 1ffb [13] +'\' ( 92) |11111111|11111110|000 7fff0 [19] +']' ( 93) |11111111|11100 1ffc [13] +'^' ( 94) |11111111|111100 3ffc [14] +'_' ( 95) |100010 22 [ 6] +'`' ( 96) |11111111|1111101 7ffd [15] +'a' ( 97) |00011 3 [ 5] +'b' ( 98) |100011 23 [ 6] +'c' ( 99) |00100 4 [ 5] +'d' (100) |100100 24 [ 6] +'e' (101) |00101 5 [ 5] +'f' (102) |100101 25 [ 6] +'g' (103) |100110 26 [ 6] +'h' (104) |100111 27 [ 6] +'i' (105) |00110 6 [ 5] +'j' (106) |1110100 74 [ 7] +'k' (107) |1110101 75 [ 7] +'l' (108) |101000 28 [ 6] +'m' (109) |101001 29 [ 6] +'n' (110) |101010 2a [ 6] +'o' (111) |00111 7 [ 5] +'p' (112) |101011 2b [ 6] +'q' (113) |1110110 76 [ 7] +'r' (114) |101100 2c [ 6] +'s' (115) |01000 8 [ 5] +'t' (116) |01001 9 [ 5] +'u' (117) |101101 2d [ 6] +'v' (118) |1110111 77 [ 7] +'w' (119) |1111000 78 [ 7] +'x' (120) |1111001 79 [ 7] +'y' (121) |1111010 7a [ 7] +'z' (122) |1111011 7b [ 7] +'{' (123) |11111111|1111110 7ffe [15] +'|' (124) |11111111|100 7fc [11] +'}' (125) |11111111|111101 3ffd [14] +'~' (126) |11111111|11101 1ffd [13] +(127) |11111111|11111111|11111111|1100 ffffffc [28] +(128) |11111111|11111110|0110 fffe6 [20] +(129) |11111111|11111111|010010 3fffd2 [22] +(130) |11111111|11111110|0111 fffe7 [20] +(131) |11111111|11111110|1000 fffe8 [20] +(132) |11111111|11111111|010011 3fffd3 [22] +(133) |11111111|11111111|010100 3fffd4 [22] +(134) |11111111|11111111|010101 3fffd5 [22] +(135) |11111111|11111111|1011001 7fffd9 [23] +(136) |11111111|11111111|010110 3fffd6 [22] +(137) |11111111|11111111|1011010 7fffda [23] +(138) |11111111|11111111|1011011 7fffdb [23] +(139) |11111111|11111111|1011100 7fffdc [23] +(140) |11111111|11111111|1011101 7fffdd [23] +(141) |11111111|11111111|1011110 7fffde [23] +(142) |11111111|11111111|11101011 ffffeb [24] +(143) |11111111|11111111|1011111 7fffdf [23] +(144) |11111111|11111111|11101100 ffffec [24] +(145) |11111111|11111111|11101101 ffffed [24] +(146) |11111111|11111111|010111 3fffd7 [22] +(147) |11111111|11111111|1100000 7fffe0 [23] +(148) |11111111|11111111|11101110 ffffee [24] +(149) |11111111|11111111|1100001 7fffe1 [23] +(150) |11111111|11111111|1100010 7fffe2 [23] +(151) |11111111|11111111|1100011 7fffe3 [23] +(152) |11111111|11111111|1100100 7fffe4 [23] +(153) |11111111|11111110|11100 1fffdc [21] +(154) |11111111|11111111|011000 3fffd8 [22] +(155) |11111111|11111111|1100101 7fffe5 [23] +(156) |11111111|11111111|011001 3fffd9 [22] +(157) |11111111|11111111|1100110 7fffe6 [23] +(158) |11111111|11111111|1100111 7fffe7 [23] +(159) |11111111|11111111|11101111 ffffef [24] +(160) |11111111|11111111|011010 3fffda [22] +(161) |11111111|11111110|11101 1fffdd [21] +(162) |11111111|11111110|1001 fffe9 [20] +(163) |11111111|11111111|011011 3fffdb [22] +(164) |11111111|11111111|011100 3fffdc [22] +(165) |11111111|11111111|1101000 7fffe8 [23] +(166) |11111111|11111111|1101001 7fffe9 [23] +(167) |11111111|11111110|11110 1fffde [21] +(168) |11111111|11111111|1101010 7fffea [23] +(169) |11111111|11111111|011101 3fffdd [22] +(170) |11111111|11111111|011110 3fffde [22] +(171) |11111111|11111111|11110000 fffff0 [24] +(172) |11111111|11111110|11111 1fffdf [21] +(173) |11111111|11111111|011111 3fffdf [22] +(174) |11111111|11111111|1101011 7fffeb [23] +(175) |11111111|11111111|1101100 7fffec [23] +(176) |11111111|11111111|00000 1fffe0 [21] +(177) |11111111|11111111|00001 1fffe1 [21] +(178) |11111111|11111111|100000 3fffe0 [22] +(179) |11111111|11111111|00010 1fffe2 [21] +(180) |11111111|11111111|1101101 7fffed [23] +(181) |11111111|11111111|100001 3fffe1 [22] +(182) |11111111|11111111|1101110 7fffee [23] +(183) |11111111|11111111|1101111 7fffef [23] +(184) |11111111|11111110|1010 fffea [20] +(185) |11111111|11111111|100010 3fffe2 [22] +(186) |11111111|11111111|100011 3fffe3 [22] +(187) |11111111|11111111|100100 3fffe4 [22] +(188) |11111111|11111111|1110000 7ffff0 [23] +(189) |11111111|11111111|100101 3fffe5 [22] +(190) |11111111|11111111|100110 3fffe6 [22] +(191) |11111111|11111111|1110001 7ffff1 [23] +(192) |11111111|11111111|11111000|00 3ffffe0 [26] +(193) |11111111|11111111|11111000|01 3ffffe1 [26] +(194) |11111111|11111110|1011 fffeb [20] +(195) |11111111|11111110|001 7fff1 [19] +(196) |11111111|11111111|100111 3fffe7 [22] +(197) |11111111|11111111|1110010 7ffff2 [23] +(198) |11111111|11111111|101000 3fffe8 [22] +(199) |11111111|11111111|11110110|0 1ffffec [25] +(200) |11111111|11111111|11111000|10 3ffffe2 [26] +(201) |11111111|11111111|11111000|11 3ffffe3 [26] +(202) |11111111|11111111|11111001|00 3ffffe4 [26] +(203) |11111111|11111111|11111011|110 7ffffde [27] +(204) |11111111|11111111|11111011|111 7ffffdf [27] +(205) |11111111|11111111|11111001|01 3ffffe5 [26] +(206) |11111111|11111111|11110001 fffff1 [24] +(207) |11111111|11111111|11110110|1 1ffffed [25] +(208) |11111111|11111110|010 7fff2 [19] +(209) |11111111|11111111|00011 1fffe3 [21] +(210) |11111111|11111111|11111001|10 3ffffe6 [26] +(211) |11111111|11111111|11111100|000 7ffffe0 [27] +(212) |11111111|11111111|11111100|001 7ffffe1 [27] +(213) |11111111|11111111|11111001|11 3ffffe7 [26] +(214) |11111111|11111111|11111100|010 7ffffe2 [27] +(215) |11111111|11111111|11110010 fffff2 [24] +(216) |11111111|11111111|00100 1fffe4 [21] +(217) |11111111|11111111|00101 1fffe5 [21] +(218) |11111111|11111111|11111010|00 3ffffe8 [26] +(219) |11111111|11111111|11111010|01 3ffffe9 [26] +(220) |11111111|11111111|11111111|1101 ffffffd [28] +(221) |11111111|11111111|11111100|011 7ffffe3 [27] +(222) |11111111|11111111|11111100|100 7ffffe4 [27] +(223) |11111111|11111111|11111100|101 7ffffe5 [27] +(224) |11111111|11111110|1100 fffec [20] +(225) |11111111|11111111|11110011 fffff3 [24] +(226) |11111111|11111110|1101 fffed [20] +(227) |11111111|11111111|00110 1fffe6 [21] +(228) |11111111|11111111|101001 3fffe9 [22] +(229) |11111111|11111111|00111 1fffe7 [21] +(230) |11111111|11111111|01000 1fffe8 [21] +(231) |11111111|11111111|1110011 7ffff3 [23] +(232) |11111111|11111111|101010 3fffea [22] +(233) |11111111|11111111|101011 3fffeb [22] +(234) |11111111|11111111|11110111|0 1ffffee [25] +(235) |11111111|11111111|11110111|1 1ffffef [25] +(236) |11111111|11111111|11110100 fffff4 [24] +(237) |11111111|11111111|11110101 fffff5 [24] +(238) |11111111|11111111|11111010|10 3ffffea [26] +(239) |11111111|11111111|1110100 7ffff4 [23] +(240) |11111111|11111111|11111010|11 3ffffeb [26] +(241) |11111111|11111111|11111100|110 7ffffe6 [27] +(242) |11111111|11111111|11111011|00 3ffffec [26] +(243) |11111111|11111111|11111011|01 3ffffed [26] +(244) |11111111|11111111|11111100|111 7ffffe7 [27] +(245) |11111111|11111111|11111101|000 7ffffe8 [27] +(246) |11111111|11111111|11111101|001 7ffffe9 [27] +(247) |11111111|11111111|11111101|010 7ffffea [27] +(248) |11111111|11111111|11111101|011 7ffffeb [27] +(249) |11111111|11111111|11111111|1110 ffffffe [28] +(250) |11111111|11111111|11111101|100 7ffffec [27] +(251) |11111111|11111111|11111101|101 7ffffed [27] +(252) |11111111|11111111|11111101|110 7ffffee [27] +(253) |11111111|11111111|11111101|111 7ffffef [27] +(254) |11111111|11111111|11111110|000 7fffff0 [27] +(255) |11111111|11111111|11111011|10 3ffffee [26] +EOS (256) |11111111|11111111|11111111|111111 3fffffff [30] diff --git a/lib/hex/mint/hpax/table.ex b/lib/hex/mint/hpax/table.ex new file mode 100644 index 00000000..60cc9bfc --- /dev/null +++ b/lib/hex/mint/hpax/table.ex @@ -0,0 +1,350 @@ +# Vendored from hpax v1.0.3, do not edit manually + +defmodule Hex.Mint.HPAX.Table do + @moduledoc false + + @enforce_keys [:max_table_size, :huffman_encoding] + defstruct [ + :protocol_max_table_size, + :max_table_size, + :huffman_encoding, + entries: [], + size: 0, + length: 0, + pending_minimum_resize: nil + ] + + @type huffman_encoding() :: :always | :never + + @type t() :: %__MODULE__{ + protocol_max_table_size: non_neg_integer(), + max_table_size: non_neg_integer(), + huffman_encoding: huffman_encoding(), + entries: [{binary(), binary()}], + size: non_neg_integer(), + length: non_neg_integer(), + pending_minimum_resize: non_neg_integer() | nil + } + + @static_table [ + {":authority", nil}, + {":method", "GET"}, + {":method", "POST"}, + {":path", "/"}, + {":path", "/index.html"}, + {":scheme", "http"}, + {":scheme", "https"}, + {":status", "200"}, + {":status", "204"}, + {":status", "206"}, + {":status", "304"}, + {":status", "400"}, + {":status", "404"}, + {":status", "500"}, + {"accept-charset", nil}, + {"accept-encoding", "gzip, deflate"}, + {"accept-language", nil}, + {"accept-ranges", nil}, + {"accept", nil}, + {"access-control-allow-origin", nil}, + {"age", nil}, + {"allow", nil}, + {"authorization", nil}, + {"cache-control", nil}, + {"content-disposition", nil}, + {"content-encoding", nil}, + {"content-language", nil}, + {"content-length", nil}, + {"content-location", nil}, + {"content-range", nil}, + {"content-type", nil}, + {"cookie", nil}, + {"date", nil}, + {"etag", nil}, + {"expect", nil}, + {"expires", nil}, + {"from", nil}, + {"host", nil}, + {"if-match", nil}, + {"if-modified-since", nil}, + {"if-none-match", nil}, + {"if-range", nil}, + {"if-unmodified-since", nil}, + {"last-modified", nil}, + {"link", nil}, + {"location", nil}, + {"max-forwards", nil}, + {"proxy-authenticate", nil}, + {"proxy-authorization", nil}, + {"range", nil}, + {"referer", nil}, + {"refresh", nil}, + {"retry-after", nil}, + {"server", nil}, + {"set-cookie", nil}, + {"strict-transport-security", nil}, + {"transfer-encoding", nil}, + {"user-agent", nil}, + {"vary", nil}, + {"via", nil}, + {"www-authenticate", nil} + ] + + @static_table_size length(@static_table) + @dynamic_table_start @static_table_size + 1 + + @doc """ + Creates a new HPACK table with the given maximum size. + + The maximum size is not the maximum number of entries but rather the maximum size as defined in + http://httpwg.org/specs/rfc7541.html#maximum.table.size. + """ + @spec new(non_neg_integer(), huffman_encoding()) :: t() + def new(protocol_max_table_size, huffman_encoding) + when is_integer(protocol_max_table_size) and protocol_max_table_size >= 0 and + huffman_encoding in [:always, :never] do + %__MODULE__{ + protocol_max_table_size: protocol_max_table_size, + max_table_size: protocol_max_table_size, + huffman_encoding: huffman_encoding + } + end + + @doc """ + Adds the given header to the given table. + + If the new entry does not fit within the max table size then the oldest entries will be evicted. + + Header names should be lowercase when added to the HPACK table + as per the [HTTP/2 spec](https://http2.github.io/http2-spec/#rfc.section.8.1.2): + + > header field names MUST be converted to lowercase prior to their encoding in HTTP/2 + + """ + @spec add(t(), binary(), binary()) :: t() + def add(%__MODULE__{} = table, name, value) do + %{max_table_size: max_table_size, size: size} = table + entry_size = entry_size(name, value) + + cond do + # An attempt to add an entry larger than the maximum size causes the table to be emptied of + # all existing entries and results in an empty table. + entry_size > max_table_size -> + %{table | entries: [], size: 0, length: 0} + + size + entry_size > max_table_size -> + table + |> evict_to_size(max_table_size - entry_size) + |> add_header(name, value, entry_size) + + true -> + add_header(table, name, value, entry_size) + end + end + + defp add_header(%__MODULE__{} = table, name, value, entry_size) do + %{entries: entries, size: size, length: length} = table + %{table | entries: [{name, value} | entries], size: size + entry_size, length: length + 1} + end + + @doc """ + Looks up a header by index `index` in the given `table`. + + Returns `{:ok, {name, value}}` if a header is found at the given `index`, otherwise returns + `:error`. `value` can be a binary in case both the header name and value are present in the + table, or `nil` if only the name is present (this can only happen in the static table). + """ + @spec lookup_by_index(t(), pos_integer()) :: {:ok, {binary(), binary() | nil}} | :error + def lookup_by_index(table, index) + + # Static table + for {header, index} <- Enum.with_index(@static_table, 1) do + def lookup_by_index(%__MODULE__{}, unquote(index)), do: {:ok, unquote(header)} + end + + def lookup_by_index(%__MODULE__{length: 0}, _index) do + :error + end + + def lookup_by_index(%__MODULE__{entries: entries, length: length}, index) + when index >= @dynamic_table_start and index <= @dynamic_table_start + length - 1 do + {:ok, Enum.at(entries, index - @dynamic_table_start)} + end + + def lookup_by_index(%__MODULE__{}, _index) do + :error + end + + @doc """ + Looks up the index of a header by its name and value. + + It returns: + + * `{:full, index}` if the full header (name and value) are present in the table at `index` + + * `{:name, index}` if `name` is present in the table but with a different value than `value` + + * `:not_found` if the header name is not in the table at all + + Header names should be lowercase when looked up in the HPACK table + as per the [HTTP/2 spec](https://http2.github.io/http2-spec/#rfc.section.8.1.2): + + > header field names MUST be converted to lowercase prior to their encoding in HTTP/2 + + """ + @spec lookup_by_header(t(), binary(), binary() | nil) :: + {:full, pos_integer()} | {:name, pos_integer()} | :not_found + def lookup_by_header(table, name, value) + + def lookup_by_header(%__MODULE__{entries: entries}, name, value) do + case static_lookup_by_header(name, value) do + {:full, _index} = result -> + result + + {:name, index} -> + # Check if we get full match in the dynamic tabble + case dynamic_lookup_by_header(entries, name, value, @dynamic_table_start, nil) do + {:full, _index} = result -> result + _other -> {:name, index} + end + + :not_found -> + dynamic_lookup_by_header(entries, name, value, @dynamic_table_start, nil) + end + end + + for {{name, value}, index} when is_binary(value) <- Enum.with_index(@static_table, 1) do + defp static_lookup_by_header(unquote(name), unquote(value)) do + {:full, unquote(index)} + end + end + + static_table_names = + @static_table + |> Enum.map(&elem(&1, 0)) + |> Enum.with_index(1) + |> Enum.uniq_by(&elem(&1, 0)) + + for {name, index} <- static_table_names do + defp static_lookup_by_header(unquote(name), _value) do + {:name, unquote(index)} + end + end + + defp static_lookup_by_header(_name, _value) do + :not_found + end + + defp dynamic_lookup_by_header([{name, value} | _rest], name, value, index, _name_index) do + {:full, index} + end + + defp dynamic_lookup_by_header([{name, _} | rest], name, value, index, _name_index) do + dynamic_lookup_by_header(rest, name, value, index + 1, index) + end + + defp dynamic_lookup_by_header([_other | rest], name, value, index, name_index) do + dynamic_lookup_by_header(rest, name, value, index + 1, name_index) + end + + defp dynamic_lookup_by_header([], _name, _value, _index, name_index) do + if name_index, do: {:name, name_index}, else: :not_found + end + + @doc """ + Changes the table's protocol negotiated maximum size, possibly evicting entries as needed to satisfy. + + If the indicated size is less than the table's current max size, entries + will be evicted as needed to fit within the specified size, and the table's + maximum size will be decreased to the specified value. An will also be + set which will enqueue a 'dynamic table size update' command to be prefixed + to the next block encoded with this table, per RFC9113§4.3.1. + + If the indicated size is greater than or equal to the table's current max size, no entries are evicted + and the table's maximum size changes to the specified value. + + In all cases, the table's `:protocol_max_table_size` is updated accordingly + """ + @spec resize(t(), non_neg_integer()) :: t() + def resize(%__MODULE__{} = table, new_protocol_max_table_size) do + pending_minimum_resize = + case table.pending_minimum_resize do + nil -> new_protocol_max_table_size + current -> min(current, new_protocol_max_table_size) + end + + %{ + evict_to_size(table, new_protocol_max_table_size) + | protocol_max_table_size: new_protocol_max_table_size, + max_table_size: new_protocol_max_table_size, + pending_minimum_resize: pending_minimum_resize + } + end + + def dynamic_resize(%__MODULE__{} = table, new_max_table_size) do + %{ + evict_to_size(table, new_max_table_size) + | max_table_size: new_max_table_size + } + end + + @doc """ + Returns (and clears) any pending resize events on the table which will need to be signalled to + the decoder via dynamic table size update messages. Intended to be called at the start of any + block encode to prepend such dynamic table size update(s) as needed. The value of + `pending_minimum_resize` indicates the smallest maximum size of this table which has not yet + been signalled to the decoder, and is always included in the list returned if it is set. + Additionally, if the current max table size is larger than this value, it is also included int + the list, per https://www.rfc-editor.org/rfc/rfc7541#section-4.2 + """ + def pop_pending_resizes(%__MODULE__{pending_minimum_resize: nil} = table), do: {table, []} + + def pop_pending_resizes(%__MODULE__{} = table) do + pending_resizes = + if table.max_table_size > table.pending_minimum_resize, + do: [table.pending_minimum_resize, table.max_table_size], + else: [table.pending_minimum_resize] + + {%{table | pending_minimum_resize: nil}, pending_resizes} + end + + # Removes records as necessary to have the total size of entries within the table be less than + # or equal to the specified value. Does not change the table's max size. + defp evict_to_size(%__MODULE__{size: size} = table, new_size) when size <= new_size, do: table + + defp evict_to_size(%__MODULE__{entries: entries, size: size} = table, new_size) do + {new_entries_reversed, new_size} = + evict_towards_size(Enum.reverse(entries), size, new_size) + + %{ + table + | entries: Enum.reverse(new_entries_reversed), + size: new_size, + length: length(new_entries_reversed) + } + end + + defp evict_towards_size([{name, value} | rest], size, max_target_size) do + new_size = size - entry_size(name, value) + + if new_size <= max_target_size do + {rest, new_size} + else + evict_towards_size(rest, new_size, max_target_size) + end + end + + defp evict_towards_size([], 0, _max_target_size) do + {[], 0} + end + + defp entry_size(name, value) do + byte_size(name) + byte_size(value) + 32 + end + + # Made public to be used in tests. + @doc false + def __static_table__() do + @static_table + end +end diff --git a/lib/hex/mint/hpax/types.ex b/lib/hex/mint/hpax/types.ex new file mode 100644 index 00000000..100213a7 --- /dev/null +++ b/lib/hex/mint/hpax/types.ex @@ -0,0 +1,91 @@ +# Vendored from hpax v1.0.3, do not edit manually + +defmodule Hex.Mint.HPAX.Types do + @moduledoc false + + import Bitwise, only: [<<<: 2] + + alias Hex.Mint.HPAX.Huffman + + # This is used as a macro and not an inlined function because we want to be able to use it in + # guards. + defmacrop power_of_two(n) do + quote do: 1 <<< unquote(n) + end + + ## Encoding + + @spec encode_integer(non_neg_integer(), 1..8) :: bitstring() + def encode_integer(integer, prefix) + + def encode_integer(integer, prefix) when integer < power_of_two(prefix) - 1 do + <> + end + + def encode_integer(integer, prefix) do + initial = power_of_two(prefix) - 1 + remaining = integer - initial + <> + end + + defp encode_remaining_integer(remaining) when remaining >= 128 do + first = rem(remaining, 128) + 128 + <> + end + + defp encode_remaining_integer(remaining) do + <> + end + + @spec encode_binary(binary(), boolean()) :: iodata() + def encode_binary(binary, huffman?) do + binary = if huffman?, do: Huffman.encode(binary), else: binary + huffman_bit = if huffman?, do: 1, else: 0 + binary_size = encode_integer(byte_size(binary), 7) + [<>, binary] + end + + ## Decoding + + @spec decode_integer(bitstring, 1..8) :: {:ok, non_neg_integer(), binary()} | :error + def decode_integer(bitstring, prefix) when is_bitstring(bitstring) and prefix in 1..8 do + with <> <- bitstring do + if value < power_of_two(prefix) - 1 do + {:ok, value, rest} + else + decode_remaining_integer(rest, value, 0) + end + else + _ -> :error + end + end + + defp decode_remaining_integer(<<0::1, value::7, rest::binary>>, int, m) do + {:ok, int + (value <<< m), rest} + end + + defp decode_remaining_integer(<<1::1, value::7, rest::binary>>, int, m) do + decode_remaining_integer(rest, int + (value <<< m), m + 7) + end + + defp decode_remaining_integer(_, _, _) do + :error + end + + @spec decode_binary(binary) :: {:ok, binary(), binary()} | :error + def decode_binary(binary) when is_binary(binary) do + with <> <- binary, + {:ok, length, rest} <- decode_integer(rest, 7), + <> <- rest do + contents = + case huffman_bit do + 0 -> contents + 1 -> Huffman.decode(contents) + end + + {:ok, contents, rest} + else + _ -> :error + end + end +end diff --git a/lib/hex/mint/http.ex b/lib/hex/mint/http.ex new file mode 100644 index 00000000..409e622a --- /dev/null +++ b/lib/hex/mint/http.ex @@ -0,0 +1,1086 @@ +# Vendored from mint v1.7.1 (d30d2cf), do not edit manually + +defmodule Hex.Mint.HTTP do + _ = """ + Process-less HTTP connection data structure and functions. + + Single interface for `Hex.Mint.HTTP1` and `Hex.Mint.HTTP2` with support for version + negotiation and proxies. + + ## Usage + + To establish a connection with a given server, use `connect/4`. This will + return an opaque data structure that represents the connection + to the server. To send a request, you can use `request/5`. Sending a request + does not take care of the response to that request, instead we use `Hex.Mint.HTTP.stream/2` + to process the response, which we will look at in just a bit. The connection is a + wrapper around a TCP (`:gen_tcp` module) or SSL (`:ssl` module) socket that is + set in **active mode** (with `active: :once`). This means that TCP/SSL messages + will be delivered to the process that started the connection. + + The process that owns the connection is responsible for receiving the messages + (for example, a GenServer is responsible for defining `handle_info/2`). However, + `Hex.Mint.HTTP` makes it easy to identify TCP/SSL messages that are coming from the + connection with the server with the `stream/2` function. This function takes the + connection and a term and returns `:unknown` if the term is not a TCP/SSL message + belonging to the connection. If the term *is* a message for the connection, then + a response and a new connection are returned. It's important to store the new + returned connection data structure over the old one since the connection is an + immutable data structure. + + Let's see an example of a common workflow of connecting to a server, sending a + request, and processing the response. We start by using `connect/3` to connect + to a server. + + {:ok, conn} = Hex.Mint.HTTP.connect(:http, "httpbin.org", 80) + + `conn` is a data structure that represents the connection. + + To send a request, we use `request/5`. + + {:ok, conn, request_ref} = Hex.Mint.HTTP.request(conn, "GET", "/", [], nil) + + As you can see, sending a request returns a new updated `conn` struct and a + `request_ref`. The updated connection struct is returned because the connection + is an immutable structure keeping the connection state, so every action we do on it must return a new, + possibly updated, connection that we're responsible for storing over the old + one. `request_ref` is a unique reference that can be used to identify which + request a given response belongs to. + + Now that we sent our request, we're responsible for receiving the messages that + the TCP/SSL socket will send to our process. For example, in a GenServer + we would do that with a `handle_info/2` callback. In our case, we're going to + use a simple `receive`. `Hex.Mint.HTTP` provides a way to tell if a message comes + from the socket wrapped by our connection or not: the `stream/2` function. If + the message we pass to it is not destined for our connection, this function returns + `:unknown`. Otherwise, it returns an updated connection and one or more responses. + + receive do + message -> + case Hex.Mint.HTTP.stream(conn, message) do + :unknown -> handle_normal_message(message) + {:ok, conn, responses} -> handle_responses(conn, responses) + end + end + + `responses` is a list of possible responses. The most common responses are: + + * `{:status, request_ref, status_code}` for the status code + * `{:headers, request_ref, headers}` for the response headers + * `{:data, request_ref, binary}` for pieces of the response body + * `{:done, request_ref}` for the end of the response + + As you can see, all responses have the unique request reference as the second + element of the tuple, so that we know which request the response belongs to. + See `t:Hex.Mint.Types.response/0` for the full list of responses returned by `Hex.Mint.HTTP.stream/2`. + + ## Architecture + + A processless architecture like the one here requires a few modifications to how + we use this HTTP client. Usually, you will want to create this data structure + in a process that acts as *connection manager*. Sometimes, you might want to + have a single process responsible for multiple connections, either to just one + host or multiple hosts. For more discussion on architectures based off of this + HTTP client, see the [*Architecture*](architecture.html) page in the docs. + + ## SSL certificates + + When using SSL, you can pass in your own CA certificate store or use one provided by Hex.Mint. Mint + doesn't ship with the certificate store itself, but it has an optional dependency on + [CAStore](https://github.com/elixir-mint/castore), which provides an up-to-date certificate store. If + you don't want to use your own certificate store, just add `:castore` to your dependencies. + + Starting [from OTP + 25](https://www.erlang.org/blog/my-otp-25-highlights/#ca-certificates-can-be-fetched-from-the-os-standard-place), + you can also load certificates from a file + ([`:public_key.cacerts_load/1`](https://www.erlang.org/doc/man/public_key.html#cacerts_load-1)). + You can also get certificate from the OS trust store using + [`:public_key.cacerts_get/0`](https://www.erlang.org/doc/man/public_key.html#cacerts_get-0). + If you are using OTP 25+ it is recommended to set this option. + + Hex.Mint.HTTP.connect(:https, host, port, transport_opts: [cacerts: :public_key.cacerts_get()]) + + ## Mode + + By default Mint operates in **active mode** meaning that the process that started the + connection receives socket messages. Mint also supports **passive mode**, where no messages + are sent to the process and the process needs to fetch data out of the socket manually. + The mode can be controlled at connection time through the `:mode` option in `connect/4` + or changed dynamically through `set_mode/2`. Passive mode is generally only recommended + for special use cases. + + ## Logging + + Mint uses the `Logger` module to log information about the connection. Most logs are + emitted *since version 1.5.0*. The logs are not emitted by default, since we consider + Mint to be too low level. However, you can enable logging by passing `log: true` to + `connect/4`. + + > #### Changes to the Format of Logs {: .warning} + > + > The format of logs emitted by Mint might change without notice between any versions, + > without it being considered a breaking change. You are only meant to control what + > gets logged by using the `Logger` API and Erlang's `:logger` module. + """ + + alias Hex.Mint.{Types, TunnelProxy, UnsafeProxy} + alias Hex.Mint.Core.{Transport, Util} + + @behaviour Hex.Mint.Core.Conn + + @opaque t() :: Hex.Mint.HTTP1.t() | Hex.Mint.HTTP2.t() + + defguardp is_data_message(message) + when elem(message, 0) in [:ssl, :tcp] and tuple_size(message) == 3 + + defguardp is_closed_message(message) + when elem(message, 0) in [:ssl_closed, :tcp_closed] and tuple_size(message) == 2 + + defguardp is_error_message(message) + when elem(message, 0) in [:ssl_error, :tcp_error] and tuple_size(message) == 3 + + defguardp is_non_proxy_connection_message(conn, message) + when is_struct(conn) and + is_tuple(message) and + is_map_key(conn, :socket) and + elem(message, 1) == :erlang.map_get(:socket, conn) and + (is_data_message(message) or is_closed_message(message) or + is_error_message(message)) + + defguardp is_proxy_conn(conn) when is_struct(conn, Hex.Mint.UnsafeProxy) + + @doc """ + Macro to check that a given received `message` is intended for the given connection `conn`. + + This guard is useful in `receive` loops or in callbacks that handle generic messages (such as a + `c:GenServer.handle_info/2` callback) so that you don't have to hand the `message` to + `Hex.Mint.HTTP.stream/2` and check for the `:unknown_message` return value. + + This macro can be used in guards. + + **Note**: this macro is only available if you compile Mint with Elixir 1.10.0 or greater (and + OTP 21+, which is required by Elixir 1.10.0 and on). + + ## Examples + + require Hex.Mint.HTTP + + {:ok, conn, request_ref} = Hex.Mint.HTTP.request(conn, "POST", "/", headers, "") + + receive do + message when Hex.Mint.HTTP.is_connection_message(conn, message) -> + Hex.Mint.HTTP.stream(conn, message) + + other -> + # This message is related to something else or to some other connection + end + + """ + @doc since: "1.1.0" + defguard is_connection_message(conn, message) + when (is_proxy_conn(conn) and + is_non_proxy_connection_message( + :erlang.map_get(:state, conn), + message + )) or is_non_proxy_connection_message(conn, message) + + @doc """ + Creates a new connection to a given server. + + Creates a new connection struct and establishes the connection to the given server, + identified by the given `host` and `port` combination. Both HTTP and HTTPS are supported + by passing respectively `:http` and `:https` as the `scheme`. + + The connection struct wraps a socket, which is created once the connection + is established inside this function. If HTTP is used, then the created socket is a TCP + socket and the `:gen_tcp` module is used to create that socket. If HTTPS is used, then + the created socket is an SSL socket and the `:ssl` module is used to create that socket. + The socket is created in active mode (with `active: :once`), which is why it is important + to know the type of the socket: messages from the socket will be delivered directly to the + process that creates the connection and tagged appropriately by the socket module (see the + `:gen_tcp` and `:ssl` modules). See `stream/2` for more information on the messages and + how to process them and on the socket mode. + + ## Options + + * `:hostname` - (string) explicitly provide the hostname used for the `Host` header, + hostname verification, SNI, and so on. **Required when `address` is not a string.** + + * `:transport_opts` - (keyword) options to be given to the transport being used. + These options will be merged with some default options that cannot be overridden. + For more details, refer to the "Transport options" section below. + + * `:mode` - (`:active` or `:passive`) whether to set the socket to active or + passive mode. See the "Mode" section in the module documentation and `set_mode/2`. + + * `:protocols` - (list of atoms) a list of protocols to try when connecting to the + server. The possible values in the list are `:http1` for HTTP/1 and HTTP/1.1 and + `:http2` for HTTP/2. If only one protocol is present in the list, then the connection + will be forced to use that protocol. If both `:http1` and `:http2` are present in the + list, then Mint will negotiate the protocol. See the section "Protocol negotiation" + below for more information. Defaults to `[:http1, :http2]`. + + * `:proxy_headers` - a list of headers (`t:Hex.Mint.Types.headers/0`) to pass when using + a proxy. They will be used for the `CONNECT` request in tunnel proxies or merged + with every request for forward proxies. + + * `:log` - (boolean) whether this connection logs or not. See the ["Logging" + section](#module-logging) in the module documentation. Defaults to `false`. + *Available since v1.5.0*. + + The following options are HTTP/1-specific and will force the connection + to be an HTTP/1 connection. + + * `:proxy` - a `{scheme, address, port, opts}` tuple that identifies a proxy to + connect to. See the "Proxying" section below for more information. + + The following options are HTTP/2-specific and will only be used on HTTP/2 connections. + + * `:client_settings` - (keyword) a list of client HTTP/2 settings to send to the + server. See `Hex.Mint.HTTP2.put_settings/2` for more information. This is only used + in HTTP/2 connections. + + * `:connection_window_size` - (integer) the initial size of the connection-level + HTTP/2 receive window, in bytes. Sent to the server as a `WINDOW_UPDATE` frame + on stream 0 as part of the connection preface. Defaults to 16 MB. Can be + raised later with `Hex.Mint.HTTP2.set_window_size/3`. + + * `:receive_window_update_threshold` - (integer) the minimum number of bytes of receive + window that must remain on a connection or stream before a `WINDOW_UPDATE` + frame is sent to refill it. Lower values send more frequent, smaller updates; + higher values batch updates into fewer, larger ones. Defaults to 160_000 + (approximately 10× the default max frame size). + + There may be further protocol specific options that only take effect when the corresponding + connection is established. Check `Hex.Mint.HTTP1.connect/4` and `Hex.Mint.HTTP2.connect/4` for + details. + + ## Protocol negotiation + + If both `:http1` and `:http2` are present in the list passed in the `:protocols` option, + the protocol negotiation happens in the following way: + + * If the scheme used to connect to the server is `:http`, then HTTP/1 or HTTP/1.1 is used. + + * If the scheme is `:https`, then ALPN negotiation is used to determine the right + protocol. This means that the server will decide whether to use HTTP/1 or + HTTP/2. If the server doesn't support protocol negotiation, we will fall back to + HTTP/1. If the server negotiates a protocol that we don't know how to handle, + `{:error, {:bad_alpn_protocol, protocol}}` is returned. + + ## Proxying + + You can set up proxying through the `:proxy` option, which is a tuple + `{scheme, address, port, opts}` that identifies the proxy to connect to. + Once a proxied connection is returned, the proxy is transparent to you and you + can use the connection like a normal HTTP/1 connection. + + If the `scheme` is `:http`, we will connect to the host in the most compatible + way, supporting older proxy servers. Data will be sent in clear text. + + If the connection scheme is `:https`, we will connect to the host with a tunnel + through the proxy. Using `:https` for both the proxy and the connection scheme + is not supported, it is recommended to use `:https` for the end host connection + instead of the proxy. + + ## Transport options + + The options specified in `:transport_opts` are passed to the module that + implements the socket interface: `:gen_tcp` when the scheme is `:http`, and + `:ssl` when the scheme is `:https`. Please refer to the documentation for those + modules, as well as for `:inet.setopts/2`, for a detailed description of all + available options. + + The behaviour of some options is modified by Mint, as described below. + + A special case is the `:timeout` option, which is passed to the transport + module's `connect` function to limit the amount of time to wait for the + network connection to be established. + + Common options for `:http` and `:https`: + + * `:active` - controlled by the `:mode` option. Cannot be overridden. + + * `:mode` - set to `:binary`. Cannot be overridden. + + * `:packet` - set to `:raw`. Cannot be overridden. + + * `:timeout` - connect timeout in milliseconds. Defaults to `30_000` (30 + seconds), and may be overridden by the caller. Set to `:infinity` to + disable the connect timeout. + + * `:inet6` - if set to `true` enables IPv6 connection. Defaults to `false` + and may be overridden by the caller. + + * `:inet4` - if set to `true` falls back to IPv4 if IPv6 connection fails. + Defaults to `true` and may be overridden by the caller. *Available since + v1.6.0*. + + Options for `:https` only: + + * `:alpn_advertised_protocols` - managed by Hex.Mint. Cannot be overridden. + + * `:cacerts` - certificates of types `:ssl.client_cacerts()`. + If `:verify` is set to `:verify_peer` (the default) and + no CA trust store is specified using the `:cacertfile` or `:cacerts` + option, Mint will attempt to use the trust store from the + [CAStore](https://github.com/elixir-mint/castore) package or raise an + exception if this package is not available. It is recommended to set this + option to `:public_key.cacerts_get()`. + + * `:cacertfile` - path to a file containing PEM-encoded CA certificates. + See the `:cacerts` option for the defaults to this value. + + * `:ciphers` - defaults to the lists returned by + `:ssl.filter_cipher_suites(:ssl.cipher_suites(:all, version), [])` + where `version` is each value in the `:versions` setting. This list is + then filtered according to the blocklist in + [RFC7540 appendix A](https://tools.ietf.org/html/rfc7540#appendix-A); + May be overridden by the caller. See the "Supporting older cipher suites" + section below for some examples. + + * `:depth` - defaults to `4`. May be overridden by the caller. + + * `:partial_chain` - unless a custom `:partial_chain` function is specified, + Mint will enable its own partial chain handler, which accepts server + certificate chains containing a certificate that was issued by a + CA certificate in the CA trust store, even if that certificate is not + last in the chain. This improves interoperability with some servers + (for example, with a cross-signed intermediate CA or some misconfigured servers), + but is a less strict interpretation of the TLS specification than the + Erlang/OTP default behaviour. + + * `:reuse_sessions` - defaults to `true`. May be overridden by the caller. If + `:"tlsv1.3"` is the only TLS version specified, `:reuse_sessions` will be + removed from the options. + + * `:secure_renegotiate` - defaults to `true`. May be overridden by the + caller. If `:"tlsv1.3"` is the only TLS version specified, `:secure_renegotiate` + will be removed from the options. + + * `:server_name_indication` - defaults to specified destination hostname. + May be overridden by the caller. + + * `:verify` - defaults to `:verify_peer`. May be overridden by the caller. + + * `:verify_fun` - unless a custom `:verify_fun` is specified, or `:verify` + is set to `:verify_none`, Mint will enable hostname verification with + support for wildcards in the server's 'SubjectAltName' extension, similar + to the behaviour implemented in + `:public_key.pkix_verify_hostname_match_fun(:https)` in recent Erlang/OTP + releases. This improves compatibility with recently issued wildcard + certificates also on older Erlang/OTP releases. + + * `:versions` - defaults to `[:"tlsv1.2"]` (TLS v1.2 only). May be + overridden by the caller. + + ### Supporting older cipher suites + + By default only a small list of modern cipher suites is enabled, in compliance + with the HTTP/2 specification. Some servers, in particular HTTP/1 servers, may + not support any of these cipher suites, resulting in TLS handshake failures or + closed connections. + + To select the default cipher suites of Erlang/OTP (including for example + AES-CBC), use the following `:transport_opts`: + + # Erlang/OTP 20.3 or later: + transport_opts: [ciphers: :ssl.cipher_suites(:default, :"tlsv1.2")] + # Older versions: + transport_opts: [ciphers: :ssl.cipher_suites()] + + Recent Erlang/OTP releases do not enable RSA key exchange by default, due to + known weaknesses. If necessary, you can build a cipher list with RSA exchange + and use it in `:transport_opts`: + + ciphers = + :ssl.cipher_suites(:all, :"tlsv1.2") + |> :ssl.filter_cipher_suites( + key_exchange: &(&1 == :rsa), + cipher: &(&1 in [:aes_256_gcm, :aes_128_gcm, :aes_256_cbc, :aes_128_cbc]) + ) + |> :ssl.append_cipher_suites(:ssl.cipher_suites(:default, :"tlsv1.2")) + + ## Examples + + {:ok, conn} = Hex.Mint.HTTP.connect(:http, "httpbin.org", 80) + + Using a proxy: + + proxy = {:http, "myproxy.example.com", 80, []} + {:ok, conn} = Hex.Mint.HTTP.connect(:https, "httpbin.org", 443, proxy: proxy) + + Forcing the connection to be an HTTP/2 connection: + + {:ok, conn} = Hex.Mint.HTTP.connect(:https, "httpbin.org", 443, protocols: [:http2]) + + Enable all default cipher suites of Erlang/OTP (release 20.3 or later): + + opts = [transport_opts: [ciphers: :ssl.cipher_suites(:default, :"tlsv1.2")]] + {:ok, conn} = Hex.Mint.HTTP.connect(:https, "httpbin.org", 443, opts) + + """ + @spec connect(Types.scheme(), Types.address(), :inet.port_number(), keyword()) :: + {:ok, t()} | {:error, Types.error()} + def connect(scheme, address, port, opts \\ []) do + case Keyword.fetch(opts, :proxy) do + {:ok, {proxy_scheme, proxy_address, proxy_port, proxy_opts}} -> + case Util.scheme_to_transport(scheme) do + Transport.TCP -> + proxy = {proxy_scheme, proxy_address, proxy_port} + host = {scheme, address, port} + opts = Keyword.merge(opts, proxy_opts) + UnsafeProxy.connect(proxy, host, opts) + + Transport.SSL -> + proxy = {proxy_scheme, proxy_address, proxy_port, proxy_opts} + host = {scheme, address, port, opts} + TunnelProxy.connect(proxy, host) + end + + :error -> + Hex.Mint.Negotiate.connect(scheme, address, port, opts) + end + end + + @doc false + @spec upgrade( + module(), + Hex.Mint.Types.socket(), + Types.scheme(), + String.t(), + :inet.port_number(), + keyword() + ) :: {:ok, t()} | {:error, Types.error()} + def upgrade(old_transport, transport_state, scheme, hostname, port, opts), + do: Hex.Mint.Negotiate.upgrade(old_transport, transport_state, scheme, hostname, port, opts) + + @doc """ + Returns the protocol used by the current connection. + + ## Examples + + iex> Hex.Mint.HTTP.protocol(%Hex.Mint.HTTP1{}) + :http1 + + iex> Hex.Mint.HTTP.protocol(%Hex.Mint.HTTP2{}) + :http2 + + """ + @doc since: "1.4.0" + @spec protocol(t()) :: :http1 | :http2 + def protocol(conn) + + def protocol(%Hex.Mint.HTTP1{}), do: :http1 + def protocol(%Hex.Mint.HTTP2{}), do: :http2 + def protocol(%Hex.Mint.UnsafeProxy{state: internal_conn}), do: protocol(internal_conn) + + @doc false + @impl true + @spec initiate( + module(), + Types.socket(), + String.t(), + :inet.port_number(), + keyword() + ) :: {:ok, t()} | {:error, Types.error()} + def initiate(transport, transport_state, hostname, port, opts), + do: Hex.Mint.Negotiate.initiate(transport, transport_state, hostname, port, opts) + + @doc """ + Closes the given connection. + + This function closes the socket wrapped by the given connection. Once the socket + is closed, the connection goes into the "closed" state and `open?/1` returns `false`. + You can throw away a closed connection. + + Closing a connection does not guarantee that data that is in flight gets delivered + to the server. + + Always returns `{:ok, conn}` where `conn` is the updated connection. + + ## Examples + + {:ok, conn} = Hex.Mint.HTTP.close(conn) + + """ + @impl true + @spec close(t()) :: {:ok, t()} + def close(conn), do: conn_apply(conn, :close, [conn]) + + @doc """ + Checks whether the connection is open. + + This function returns `true` if the connection is open for the given `type`, + `false` otherwise. It should be used to check that a connection is open before + sending requests or performing operations that involve talking to the server. + + The `type` argument can be used to tell whether the connection is open for both reading + and writing, only open for reading, or closed for both. In HTTP/1, a connection is always + either open, or closed (for both reading and writing). In HTTP/2, the connection can be closed only + for writing but not for reading, meaning that you cannot send any more data to the + server but you can still receive data from the server. In this case, `Hex.Mint.HTTP.open?(conn, :read)` + would return `true` but `Hex.Mint.HTTP.open?(conn, :write)` would return `false`. + See the "Closed connection" section in the module documentation of `Hex.Mint.HTTP2`. + + If a connection is *completely closed* (that is, `Hex.Mint.HTTP.open?(conn, :read)` returns `false`), + it has become useless and you should get rid of it. If you still need a connection + to the server, start a new connection with `connect/4`. + + > #### The default value of `type` is `:write` {: .warning} + > + > With the default value of `type` being `:write`, a call to + > `Hex.Mint.HTTP.open?(conn)` will return `false` if `conn` was closed for writing + > but is still open for reading. If you need to make sure the connection is + > completely closed, check that `Hex.Mint.HTTP.open?(conn, :read)` returns `false`. + + ## Examples + + {:ok, conn} = Hex.Mint.HTTP.connect(:http, "httpbin.org", 80) + Hex.Mint.HTTP.open?(conn) + #=> true + + """ + @impl true + @spec open?(t(), :read | :write) :: boolean() + def open?(conn, type \\ :write), do: conn_apply(conn, :open?, [conn, type]) + + @doc """ + Sends a request to the connected server. + + This function sends a new request to the server that `conn` is connected to. + `method` is a string representing the method for the request, such as `"GET"` + or `"POST"`. `path` is the path on the host to send the request to. `headers` + is a list of request headers in the form `{header_name, header_value}` with + `header_name` and `header_value` being strings. `body` can have one of three + values: + + * `nil` - no body is sent with the request. + + * iodata - the body to send for the request. + + * `:stream` - when the value of the body is `:stream` the request + body can be streamed on the connection. See `stream_request_body/3`. + In HTTP/1, you can't open a request if the body of another request is + streaming. + + If the request is sent correctly, this function returns `{:ok, conn, request_ref}`. + `conn` is an updated connection that should be stored over the old connection. + `request_ref` is a unique reference that can be used to match on responses for this + request that are returned by `stream/2`. See `stream/2` for more information. + + If there's an error with sending the request, `{:error, conn, reason}` is returned. + `reason` is the cause of the error. `conn` is an updated connection. It's important + to store the returned connection over the old connection in case of errors too, because + the state of the connection might change when there are errors as well. An error when + sending a request **does not** necessarily mean that the connection is closed. Use + `open?/1` to verify that the connection is open. + + Requests can be pipelined so the full response does not have to received + before the next request can be sent. It is up to users to verify that the + server supports pipelining and that the request is safe to pipeline. + + In HTTP/1, you can't open a request if the body of another request is streaming. + See `Hex.Mint.HTTP1.request/5` for more information. + + For a quick discussion on HTTP/2 streams and requests, see the `Hex.Mint.HTTP2` module and + `Hex.Mint.HTTP2.request/5`. + + ## The `content-length` header + + If you don't set the `content-length` header and you send a body with the request (that + is, not `nil` and not `:stream`), then Mint will add a default `content-length` header + to your request. If you're using HTTP/2 and streaming the request, you may provide the + `content-length` header yourself. If you're using HTTP/1, Mint will do chunked + transfer-encoding when a content-length is not provided (see `Hex.Mint.HTTP1.request/5`). + + ## Examples + + Hex.Mint.HTTP.request(conn, "GET", "/", _headers = [], _body = nil) + Hex.Mint.HTTP.request(conn, "POST", "/path", [{"content-type", "application/json"}], "{}") + + """ + @impl true + @spec request( + t(), + method :: String.t(), + path :: String.t(), + Types.headers(), + body :: iodata() | nil | :stream + ) :: + {:ok, t(), Types.request_ref()} + | {:error, t(), Types.error()} + + def request(conn, method, path, headers, body), + do: conn_apply(conn, :request, [conn, method, path, headers, body]) + + @doc """ + Streams a chunk of the request body on the connection or signals the end of the body. + + If a request is opened (through `request/5`) with the body as `:stream`, then the + body can be streamed through this function. The function takes a `conn`, a + `request_ref` returned by `request/5` to identify the request to stream the body for, + and a chunk of body to stream. The value of chunk can be: + + * iodata - a chunk of iodata is transmitted to the server as part of the body + of the request. If the chunk is empty, in HTTP/1 it's a no-op, while in HTTP/2 + a `DATA` frame will be sent. + + * `:eof` - signals the end of the streaming of the request body for the given + request. Usually the server won't send any reply until this is sent. + + * `{:eof, trailer_headers}` - sends **trailer headers** and signals the end + of the streaming of the request body for the given request. This behaves the + same way as `:eof` but first sends the trailer headers. See the + [*Trailer headers*](#module-trailer-headers) section below. + + This function always returns an updated connection to be stored over the old connection. + + For information about transfer encoding and content length in HTTP/1, see + `Hex.Mint.HTTP1.stream_request_body/3`. + + ## Trailer headers + + HTTP trailer headers can be sent after the body of a request. trailer headers are described + [in RFC 9110](https://www.rfc-editor.org/rfc/rfc9110#section-6.5). + + The behaviour is slightly different for HTTP/1 and HTTP/2: + + * In HTTP/1, trailer headers are only supported if the transfer encoding is set to + `chunked`. See `Hex.Mint.HTTP1.stream_request_body/3` for more information on chunked + transfer encoding. + + * In HTTP/2, trailer headers behave like normal headers. You don't need to care + about the transfer encoding. + + ### The `trailer` header + + As specified in [section 4.4 of RFC 7230](https://tools.ietf.org/html/rfc7230#section-4.4), + in HTTP/1 you need to specify which headers you're going to send as traoler + headers using the `trailer` header. The `trailer` header applies to both HTTP/1 + and HTTP/2. See the examples below for more information. + + ### The `te` header + + As specified in [section 4.3 of RFC 7230](https://tools.ietf.org/html/rfc7230#section-4.3), + the `te` (or `TE`) header is used to specify which transfer-encodings the client + is willing to accept (besides `chunked`). Mint supports decoding of trailer headers, + but if you want to notify the server that you are accepting trailer headers, + use the `trailers` value in the `te` header. For example: + + Hex.Mint.HTTP.request(conn, "GET", "/", [{"te", "trailers"}], "some body") + + Note that the `te` header can also be used to communicate which encodings you + support to the server. + + ## Examples + + Let's see an example of streaming an empty JSON object (`{}`) by streaming one curly + brace at a time. + + headers = [{"content-type", "application/json"}, {"content-length", "2"}] + {:ok, conn, request_ref} = Hex.Mint.HTTP.request(conn, "POST", "/", headers, :stream) + {:ok, conn} = Hex.Mint.HTTP.stream_request_body(conn, request_ref, "{") + {:ok, conn} = Hex.Mint.HTTP.stream_request_body(conn, request_ref, "}") + {:ok, conn} = Hex.Mint.HTTP.stream_request_body(conn, request_ref, :eof) + + Here's an example of sending trailer headers: + + headers = [{"content-type", "application/json"}, {"trailer", "my-trailer, x-expires"}] + {:ok, conn, request_ref} = Hex.Mint.HTTP.request(conn, "POST", "/", headers, :stream) + + {:ok, conn} = Hex.Mint.HTTP.stream_request_body(conn, request_ref, "{}") + + trailer_headers = [{"my-trailer", "xxx"}, {"x-expires", "10 days"}] + {:ok, conn} = Hex.Mint.HTTP.stream_request_body(conn, request_ref, {:eof, trailer_headers}) + + """ + @impl true + @spec stream_request_body( + t(), + Types.request_ref(), + iodata() | :eof | {:eof, trailer_headers :: Types.headers()} + ) :: + {:ok, t()} | {:error, t(), Types.error()} + def stream_request_body(conn, ref, body), + do: conn_apply(conn, :stream_request_body, [conn, ref, body]) + + @doc """ + Streams the next batch of responses from the given `message`. + + This function processes a "message" which can be any term, but should be + a message received by the process that owns the connection. **Processing** + a message means that this function will parse it and check if it's a message + that is directed to this connection, that is, a TCP/SSL message received on the + connection's socket. If it is, then this function will parse the message, + turn it into a list of responses, and possibly take action given the responses. + As an example of an action that this function could perform, if the server sends + a ping request this function will transparently take care of pinging the server back. + + If there's no error, this function returns `{:ok, conn, responses}` where `conn` is + the updated connection and `responses` is a list of responses. See the "Responses" + section below. If there's an error, `{:error, conn, reason, responses}` is returned, + where `conn` is the updated connection, `reason` is the error reason, and `responses` + is a list of responses that were correctly parsed before the error. + + > #### Graceful Close {: .tip} + > + > If this function returns `{:ok, conn, responses}`, it doesn't *necessarily* mean + > that the connection is still open. For example, TCP/SSL **close** messages are treated + > as errors only if there are in-flight requests. If there are no in-flight requests, + > the connection is closed gracefully and `{:ok, conn, responses}` is returned. + > Always check with `open?/1` to see if the connection is still open. + + If the given `message` is not from the connection's socket, + this function returns `:unknown`. + + > #### Receiving Multiple Messages {: .warning} + > + > Your connection and the HTTP server can exchange multiple **protocol-specific messages** + > on the socket that don't necessarily *produce responses*. For example, the HTTP server + > might tell the connection to update some internal settings. For this reason, you + > should always receive as many messages coming to your process as possible, for example + > by using `receive` recursively. You can see an example of this approach in the + > ["Usage Examples" documentation](architecture.html#usage-examples). + + ## Socket mode + + Mint sets the socket in `active: :once` mode. This means that a single socket + message at a time is delivered to the process that owns the connection. After + a message is delivered, then no other messages are delivered (we say the socket + goes in *passive* mode). When `stream/2` is called to process the message that + was received, Mint sets the socket back to `active: :once`. This is good to know + in order to understand how the socket is handled by Mint, but in normal usage + it just means that you will process one message at a time with `stream/2` and not + pay too much attention to the socket mode. + + Mint also supports passive mode to avoid receiving messages. See the "Mode" section + in the module documentation. + + ## Responses + + Each possible response returned by this function is a tuple with two or more elements. + The first element is always an atom that identifies the kind of response. The second + element is a unique reference `t:Hex.Mint.Types.request_ref/0` that identifies the request + that the response belongs to. This is the term returned by `request/5`. After these + two elements, there can be response-specific terms as well, documented below. + + These are the possible responses that can be returned. + + * `{:status, request_ref, status_code}` - returned when the server replied + with a response status code. The status code is a non-negative integer. + You can have zero or more `1xx` `:status` and `:headers` responses for a + single request, but they all precede a single non-`1xx` `:status` response. + + * `{:status_reason, request_ref, reason_phrase}` - returned when the server replied + with a response status code and a reason-phrase. The reason-phrase is a string. + Returned when the `:optional_responses` option is passed to `connect/4`, with + `:status_reason` in the list. See `Hex.Mint.HTTP1.connect/4` for more information. + This is only available for HTTP/1.1 connections. *Available since v1.8.0*. + + * `{:headers, request_ref, headers}` - returned when the server replied + with a list of headers. Headers are in the form `{header_name, header_value}` + with `header_name` and `header_value` being strings. A single `:headers` response + will come after the `:status` response. A single `:headers` response may come + after all the `:data` responses if **trailer headers** are present. + + * `{:data, request_ref, binary}` - returned when the server replied with + a chunk of response body (as a binary). The request shouldn't be considered done + when a piece of body is received because multiple chunks could be received. The + request is done when the `:done` response is returned. + + * `{:done, request_ref}` - returned when the server signaled the request + as done. When this is received, the response body and headers can be considered + complete and it can be assumed that no more responses will be received for this + request. This means that for example, you can stop holding on to the request ref + for this request. + + * `{:error, request_ref, reason}` - returned when there is an error that + only affects the request and not the whole connection. For example, if the + server sends bad data on a given request, that request will be closed and an error + for that request will be returned among the responses, but the connection will + remain alive and well. + + * `{:pong, request_ref}` - returned when a server replies to a ping + request sent by the client. This response type is HTTP/2-specific + and will never be returned by an HTTP/1 connection. See `Hex.Mint.HTTP2.ping/2` + for more information. + + * `{:push_promise, request_ref, promised_request_ref, headers}` - returned when + the server sends a server push to the client. This response type is HTTP/2 specific + and will never be returned by an HTTP/1 connection. See `Hex.Mint.HTTP2` for more + information on server pushes. + + ## Examples + + Let's assume we have a function called `receive_next_and_stream/1` that takes + a connection and then receives the next message, calls `stream/2` with that message + as an argument, and then returns the result of `stream/2`: + + defp receive_next_and_stream(conn) do + receive do + message -> Hex.Mint.HTTP.stream(conn, message) + end + end + + Now, we can see an example of a workflow involving `stream/2`. + + {:ok, conn, request_ref} = Hex.Mint.HTTP.request(conn, "GET", "/", _headers = []) + + {:ok, conn, responses} = receive_next_and_stream(conn) + responses + #=> [{:status, ^request_ref, 200}] + + {:ok, conn, responses} = receive_next_and_stream(conn) + responses + #=> [{:headers, ^request_ref, [{"Content-Type", "application/json"}]}, + #=> {:data, ^request_ref, "{"}] + + {:ok, conn, responses} = receive_next_and_stream(conn) + responses + #=> [{:data, ^request_ref, "}"}, {:done, ^request_ref}] + + """ + @impl true + @spec stream(t(), term()) :: + {:ok, t(), [Types.response()]} + | {:error, t(), Types.error(), [Types.response()]} + | :unknown + def stream(conn, message), do: conn_apply(conn, :stream, [conn, message]) + + @doc """ + Returns the number of open requests. + + Open requests are requests that have not yet received a `:done` response. + This function returns the number of open requests for both HTTP/1 and HTTP/2, + but for HTTP/2 only client-initiated requests are considered as open requests. + See `Hex.Mint.HTTP2.open_request_count/1` for more information. + + ## Examples + + {:ok, conn, _ref} = Hex.Mint.HTTP.request(conn, "GET", "/", []) + Hex.Mint.HTTP.open_request_count(conn) + #=> 1 + + """ + @impl true + @spec open_request_count(t()) :: non_neg_integer() + def open_request_count(conn), do: conn_apply(conn, :open_request_count, [conn]) + + @doc """ + Receives data from the socket in a blocking way. + + By default Mint operates in active mode, meaning that messages are delivered + to the process that started the connection. However, Mint also supports passive + mode (see the "Mode" section in the module documentation). + + In passive mode, you'll need to manually get bytes out of the socket. You can + do that with this function. + + `byte_count` is the number of bytes you want out of the socket. If `byte_count` + is `0`, all available bytes will be returned. + + `timeout` is the maximum time to wait before returning an error. + + This function will raise an error if the socket is in active mode. + + > #### Hanging Waiting for Bytes {: .warning} + > + > If `byte_count` is greater than `0` and the socket doesn't receive + > *at least* `byte_count` bytes withing the `timeout`, then the function + > will block for the duration of `timeout` and then return a timeout error. + > This behavior is the same as the `recv` function in [`:gen_tcp`](`:gen_tcp`) + > and [`:ssl`](`:ssl`). + + ## Examples + + {:ok, conn, responses} = Hex.Mint.HTTP.recv(conn, 0, 5000) + + """ + @impl true + @spec recv(t(), non_neg_integer(), timeout()) :: + {:ok, t(), [Types.response()]} + | {:error, t(), Types.error(), [Types.response()]} + def recv(conn, byte_count, timeout), do: conn_apply(conn, :recv, [conn, byte_count, timeout]) + + @doc """ + Changes the mode of the underlying socket. + + To use the connection in *active mode*, where the process that started the + connection receives socket messages, set the mode to `:active` (see also `stream/2`). + To use the connection in *passive mode*, where you need to manually receive data + from the socket, set the mode to `:passive` (see also `recv/3`). + + The mode can also be controlled at connection time by the `:mode` option passed + to `connect/4`. + + Note that if you're switching from active to passive mode, you still might have + socket messages in the process mailbox that you need to consume before doing + any other operation on the connection. + + See the "Mode" section in the module documentation for more information on modes. + + ## Examples + + {:ok, conn} = Hex.Mint.HTTP.set_mode(conn, :passive) + + """ + @impl true + @spec set_mode(t(), :active | :passive) :: {:ok, t()} | {:error, Types.error()} + def set_mode(conn, mode), do: conn_apply(conn, :set_mode, [conn, mode]) + + @doc """ + Changes the *controlling process* of the given connection to `new_pid`. + + The **controlling process** is a concept that comes from the Erlang TCP and + SSL implementations. The controlling process of a connection is the process + that started the connection and that receives the messages for that connection. + You can change the controlling process of a connection through this function. + + This function also takes care of "transferring" all the connection messages + that are in the mailbox of the current controlling process to the new + controlling process. + + Remember that the connection is a data structure, so if you + change the controlling process it doesn't mean you "transferred" the + connection data structure itself to the other process, which you have + to do manually (for example by sending the connection data structure to the + new controlling process). If you do that, be careful of race conditions + and be sure to retrieve the connection in the new controlling process + before accepting connection messages in the new controlling process. + In fact, this function is guaranteed to return the connection unchanged, + so you are free to ignore the connection entry returned in `{:ok, conn}`. + + ## Examples + + send(new_pid, {:conn, conn}) + {:ok, conn} = Hex.Mint.HTTP.controlling_process(conn, new_pid) + + # In the "new_pid" process + receive do + {:conn, conn} -> + # Will receive connection messages. + end + + """ + @impl true + @spec controlling_process(t(), pid()) :: {:ok, t()} | {:error, Types.error()} + def controlling_process(conn, new_pid), + do: conn_apply(conn, :controlling_process, [conn, new_pid]) + + @doc """ + Assigns a new private key and value in the connection. + + This storage is meant to be used to associate metadata with the connection and + it can be useful when handling multiple connections. + + The given `key` must be an atom, while the given `value` can be an arbitrary + term. The return value of this function is an updated connection. + + See also `get_private/3` and `delete_private/2`. + + ## Examples + + Let's see an example of putting a value and then getting it: + + conn = Hex.Mint.HTTP.put_private(conn, :client_name, "Mint") + Hex.Mint.HTTP.get_private(conn, :client_name) + #=> "Mint" + + """ + @impl true + @spec put_private(t(), atom(), term()) :: t() + def put_private(conn, key, value), do: conn_apply(conn, :put_private, [conn, key, value]) + + @doc """ + Gets a private value from the connection. + + Retrieves a private value previously set with `put_private/3` from the connection. + `key` is the key under which the value to retrieve is stored. `default` is a default + value returned in case there's no value under the given key. + + See also `put_private/3` and `delete_private/2`. + + ## Examples + + conn = Hex.Mint.HTTP.put_private(conn, :client_name, "Mint") + + Hex.Mint.HTTP.get_private(conn, :client_name) + #=> "Mint" + + Hex.Mint.HTTP.get_private(conn, :non_existent) + #=> nil + + """ + @impl true + @spec get_private(t(), atom(), term()) :: term() + def get_private(conn, key, default \\ nil), + do: conn_apply(conn, :get_private, [conn, key, default]) + + @doc """ + Deletes a value in the private store. + + Deletes the private value stored under `key` in the connection. Returns the + updated connection. + + See also `put_private/3` and `get_private/3`. + + ## Examples + + conn = Hex.Mint.HTTP.put_private(conn, :client_name, "Mint") + + Hex.Mint.HTTP.get_private(conn, :client_name) + #=> "Mint" + + conn = Hex.Mint.HTTP.delete_private(conn, :client_name) + Hex.Mint.HTTP.get_private(conn, :client_name) + #=> nil + + """ + @impl true + @spec delete_private(t(), atom()) :: t() + def delete_private(conn, key), do: conn_apply(conn, :delete_private, [conn, key]) + + @doc """ + Gets the socket associated with the connection. + + Do not use the returned socket to change its internal state. Only read information from the socket. + For instance, use `:ssl.connection_information/2` to retrieve TLS-specific information from the + socket. + """ + @impl true + @spec get_socket(t()) :: Hex.Mint.Types.socket() + def get_socket(conn), do: conn_apply(conn, :get_socket, [conn]) + + @doc """ + Sets whether the connection should log information or not. + + See the ["Logging" section](#module-logging) in the module documentation for more information. + """ + @doc since: "1.5.0" + @impl true + @spec put_log(t(), boolean()) :: t() + def put_log(conn, log?), do: conn_apply(conn, :put_log, [conn, log?]) + + @doc """ + Gets the proxy headers associated with the connection in the `CONNECT` method. + + When using tunnel proxy and HTTPs, the only way to exchange data with + the proxy is through headers in the `CONNECT` method. + """ + @doc since: "1.4.0" + @impl true + @spec get_proxy_headers(t()) :: Hex.Mint.Types.headers() + def get_proxy_headers(conn), do: conn_apply(conn, :get_proxy_headers, [conn]) + + # Made public since the struct is opaque. + @doc false + @impl true + def put_proxy_headers(conn, headers), do: conn_apply(conn, :put_proxy_headers, [conn, headers]) + + ## Helpers + + defp conn_apply(%UnsafeProxy{}, fun, args), do: apply(UnsafeProxy, fun, args) + defp conn_apply(%Hex.Mint.HTTP1{}, fun, args), do: apply(Hex.Mint.HTTP1, fun, args) + defp conn_apply(%Hex.Mint.HTTP2{}, fun, args), do: apply(Hex.Mint.HTTP2, fun, args) +end diff --git a/lib/hex/mint/http1.ex b/lib/hex/mint/http1.ex new file mode 100644 index 00000000..f2d90bd1 --- /dev/null +++ b/lib/hex/mint/http1.ex @@ -0,0 +1,1217 @@ +# Vendored from mint v1.7.1 (d30d2cf), do not edit manually + +defmodule Hex.Mint.HTTP1 do + _ = """ + Process-less HTTP/1.1 client connection. + + This module provides a data structure that represents an HTTP/1 or HTTP/1.1 connection to + a given server. The connection is represented as an opaque struct `%Hex.Mint.HTTP1{}`. + The connection is a data structure and is not backed by a process, and all the + connection handling happens in the process that creates the struct. + + This module and data structure work exactly like the ones described in the `Mint` + module, with the exception that `Hex.Mint.HTTP1` specifically deals with HTTP/1 and HTTP/1.1 while + `Mint` deals seamlessly with HTTP/1, HTTP/1.1, and HTTP/2. For more information on + how to use the data structure and client architecture, see `Mint`. + """ + + alias Hex.Mint.Core.{Headers, Util} + + alias Hex.Mint.HTTP1.{Parse, Request, Response} + alias Hex.Mint.{HTTPError, TransportError, Types} + + require Logger + + @behaviour Hex.Mint.Core.Conn + + @typedoc """ + A Mint HTTP/1 connection struct. + + The struct's fields are private. + """ + @opaque t() :: %__MODULE__{} + + @user_agent "mint/" <> "1.7.1" + + @typedoc """ + An HTTP/1-specific error reason. + + The values can be: + + * `:closed` - when you try to make a request or stream a body chunk but the connection + is closed. + + * `:request_body_is_streaming` - when you call `request/5` to send a new + request but another request is already streaming. + + * `{:unexpected_data, data}` - when unexpected data is received from the server. + + * `:invalid_status_line` - when the HTTP/1 status line is invalid. + + * `{:invalid_request_target, target}` - when the request target is invalid. + + * `:invalid_header` - when headers can't be parsed correctly. + + * `{:invalid_header_name, name}` - when a header name is invalid. + + * `{:invalid_header_value, name, value}` - when a header value is invalid. `name` + is the name of the header and `value` is the invalid value. + + * `:invalid_chunk_size` - when the chunk size is invalid. + + * `:missing_crlf_after_chunk` - when the CRLF after a chunk is missing. + + * `:invalid_trailer_header` - when trailer headers can't be parsed. + + * `:more_than_one_content_length_header` - when more than one `content-length` + headers are in the response. + + * `:transfer_encoding_and_content_length` - when both the `content-length` as well + as the `transfer-encoding` headers are in the response. + + * `{:invalid_content_length_header, value}` - when the value of the `content-length` + header is invalid, that is, is not an non-negative integer. + + * `:empty_token_list` - when a header that is supposed to contain a list of tokens + (such as the `connection` header) doesn't contain any. + + * `{:invalid_token_list, string}` - when a header that is supposed to contain a list + of tokens (such as the `connection` header) contains a malformed list of tokens. + + * `:trailing_headers_but_not_chunked_encoding` - when you try to send trailer + headers through `stream_request_body/3` but the transfer encoding of the request + was not `chunked`. + + """ + @type error_reason() :: term() + + @optional_responses_opts [:status_reason] + + defstruct [ + :host, + :port, + :request, + :streaming_request, + :socket, + :transport, + :mode, + :scheme_as_string, + :case_sensitive_headers, + :skip_target_validation, + requests: :queue.new(), + state: :closed, + buffer: "", + proxy_headers: [], + private: %{}, + log: false, + optional_responses: [] + ] + + defmacrop log(conn, level, message) do + quote do + conn = unquote(conn) + + if conn.log do + Logger.log(unquote(level), unquote(message)) + else + :ok + end + end + end + + @doc """ + Same as `Hex.Mint.HTTP.connect/4`, but forces an HTTP/1 or HTTP/1.1 connection. + + This function doesn't support proxying. + + ## Additional Options + + * `:case_sensitive_headers` - (boolean) if set to `true` the case of the supplied + headers in requests will be preserved. The default is to lowercase the headers + because HTTP/1.1 header names are case-insensitive. *Available since v1.6.0*. + * `:skip_target_validation` - (boolean) if set to `true` the target of a request + will not be validated. You might want this if you deal with non standard- + conforming URIs but need to preserve them. The default is to validate the request + target. *Available since v1.7.0*. + * `:optional_responses` - (list of atoms) a list of optional responses to return. + Defaults to `[]`. The allowed values in the list are: + * `:status_reason`: includes the + [reason-phrase](https://datatracker.ietf.org/doc/html/rfc9112#name-status-line) + for the status code if it is returned by the server in the status-line. + This is only available for HTTP/1.1 connections. *Available since v1.8.0*. + + """ + @spec connect(Types.scheme(), Types.address(), :inet.port_number(), keyword()) :: + {:ok, t()} | {:error, Types.error()} + def connect(scheme, address, port, opts \\ []) do + # TODO: Also ALPN negotiate HTTP1? + + hostname = Hex.Mint.Core.Util.hostname(opts, address) + transport = Util.scheme_to_transport(scheme) + + transport_opts = + Keyword.get(opts, :transport_opts, []) + |> Keyword.put(:hostname, hostname) + + with {:ok, socket} <- transport.connect(address, port, transport_opts) do + initiate(scheme, socket, hostname, port, opts) + end + end + + @doc false + @spec upgrade( + Types.scheme(), + Types.socket(), + Types.scheme(), + String.t(), + :inet.port_number(), + keyword() + ) :: {:ok, t()} | {:error, Types.error()} + def upgrade(old_scheme, socket, new_scheme, hostname, port, opts) do + # TODO: Also ALPN negotiate HTTP1? + + transport = Util.scheme_to_transport(new_scheme) + + transport_opts = + Keyword.get(opts, :transport_opts, []) + |> Keyword.put(:hostname, hostname) + + with {:ok, socket} <- transport.upgrade(socket, old_scheme, hostname, port, transport_opts) do + initiate(new_scheme, socket, hostname, port, opts) + end + end + + @doc false + @impl true + @spec initiate( + Types.scheme(), + Types.socket(), + String.t(), + :inet.port_number(), + keyword() + ) :: {:ok, t()} | {:error, Types.error()} + def initiate(scheme, socket, hostname, port, opts) do + transport = Util.scheme_to_transport(scheme) + mode = Keyword.get(opts, :mode, :active) + log? = Keyword.get(opts, :log, false) + + unless mode in [:active, :passive] do + raise ArgumentError, + "the :mode option must be either :active or :passive, got: #{inspect(mode)}" + end + + unless is_boolean(log?) do + raise ArgumentError, + "the :log option must be a boolean, got: #{inspect(log?)}" + end + + with :ok <- Util.inet_opts(transport, socket), + :ok <- if(mode == :active, do: transport.setopts(socket, active: :once), else: :ok) do + conn = %__MODULE__{ + transport: transport, + socket: socket, + mode: mode, + host: hostname, + port: port, + scheme_as_string: Atom.to_string(scheme), + state: :open, + log: log?, + case_sensitive_headers: Keyword.get(opts, :case_sensitive_headers, false), + skip_target_validation: Keyword.get(opts, :skip_target_validation, false), + optional_responses: validate_optional_response_values(opts) + } + + {:ok, conn} + else + {:error, reason} -> + :ok = transport.close(socket) + {:error, reason} + end + end + + defp validate_optional_response_values(opts) do + opts + |> Keyword.get(:optional_responses, []) + |> Enum.map(fn opt -> + if opt not in @optional_responses_opts do + raise ArgumentError, """ + invalid :optional_responses value #{inspect(opt)}, the allowed values are: \ + #{inspect(@optional_responses_opts)}\ + """ + end + + opt + end) + end + + @doc """ + See `Hex.Mint.HTTP.close/1`. + """ + @impl true + @spec close(t()) :: {:ok, t()} + def close(conn) + + def close(%__MODULE__{state: :open} = conn) do + conn = internal_close(conn) + {:ok, conn} + end + + def close(%__MODULE__{state: :closed} = conn) do + {:ok, conn} + end + + @doc """ + See `Hex.Mint.HTTP.open?/1`. + """ + @impl true + @spec open?(t(), :read | :write) :: boolean() + def open?(conn, type \\ :write) + + # TODO: hard-deprecate :read_write in 1.7. + def open?(%__MODULE__{state: state}, type) when type in [:read, :write, :read_write] do + state == :open + end + + @doc """ + See `Hex.Mint.HTTP.request/5`. + + In HTTP/1 and HTTP/1.1, you can't open a new request if you're streaming the body of + another request. If you try, an error will be returned. + """ + @impl true + @spec request( + t(), + method :: String.t(), + path :: String.t(), + Types.headers(), + body :: iodata() | nil | :stream + ) :: + {:ok, t(), Types.request_ref()} + | {:error, t(), Types.error()} + def request(conn, method, path, headers, body) + + def request(%__MODULE__{state: :closed} = conn, _method, _path, _headers, _body) do + {:error, conn, wrap_error(:closed)} + end + + def request(%__MODULE__{streaming_request: %{}} = conn, _method, _path, _headers, _body) do + {:error, conn, wrap_error(:request_body_is_streaming)} + end + + def request(%__MODULE__{} = conn, method, path, headers, body) + when is_binary(method) and is_binary(path) and is_list(headers) do + %__MODULE__{transport: transport, socket: socket} = conn + + headers = + headers + |> Headers.from_raw() + |> add_default_headers(conn) + + with {:ok, headers, encoding} <- add_content_length_or_transfer_encoding(headers, body), + :ok <- validate_request_target(path, conn.skip_target_validation), + {:ok, iodata} <- + Request.encode( + method, + path, + Headers.to_raw(headers, conn.case_sensitive_headers), + body + ), + :ok <- transport.send(socket, iodata) do + request_ref = make_ref() + request = new_request(request_ref, method, body, encoding) + + case request.state do + {:stream_request, _} -> + conn = %{conn | streaming_request: request} + {:ok, conn, request_ref} + + _ -> + conn = enqueue_request(conn, request) + {:ok, conn, request_ref} + end + else + {:error, %TransportError{reason: :closed} = error} -> + {:error, %{conn | state: :closed}, error} + + {:error, %error_module{} = error} when error_module in [HTTPError, TransportError] -> + {:error, conn, error} + + {:error, reason} -> + {:error, conn, wrap_error(reason)} + end + end + + @doc """ + See `Hex.Mint.HTTP.stream_request_body/3`. + + In HTTP/1, sending an empty chunk is a no-op. + + ## Transfer encoding and content length + + When streaming the request body, Mint cannot send a precalculated `content-length` + request header because it doesn't know the body that you'll stream. However, Mint + will transparently handle the presence of a `content-length` header using this logic: + + * if you specifically set a `content-length` header, then transfer encoding and + making sure the content length is correct for what you'll stream is up to you. + + * if you specifically set the transfer encoding (`transfer-encoding` header) + to `chunked`, then it's up to you to + [properly encode chunks](https://en.wikipedia.org/wiki/Chunked_transfer_encoding). + + * if you don't set the transfer encoding to `chunked` and don't provide a + `content-length` header, Mint will do implicit `chunked` transfer encoding + (setting the `transfer-encoding` header appropriately) and will take care + of properly encoding the chunks. + + """ + @impl true + @spec stream_request_body( + t(), + Types.request_ref(), + iodata() | :eof | {:eof, trailer_headers :: Types.headers()} + ) :: + {:ok, t()} | {:error, t(), Types.error()} + def stream_request_body( + %__MODULE__{streaming_request: %{state: {:stream_request, :identity}, ref: ref}} = conn, + ref, + :eof + ) do + request = %{conn.streaming_request | state: :status} + conn = enqueue_request(%{conn | streaming_request: nil}, request) + {:ok, conn} + end + + def stream_request_body( + %__MODULE__{streaming_request: %{state: {:stream_request, :identity}, ref: ref}} = conn, + ref, + {:eof, _trailer_headers} + ) do + {:error, conn, wrap_error(:trailing_headers_but_not_chunked_encoding)} + end + + def stream_request_body( + %__MODULE__{streaming_request: %{state: {:stream_request, :identity}, ref: ref}} = conn, + ref, + body + ) do + case conn.transport.send(conn.socket, body) do + :ok -> + {:ok, conn} + + {:error, %TransportError{reason: :closed} = error} -> + {:error, %{conn | state: :closed}, error} + + {:error, error} -> + {:error, conn, error} + end + end + + def stream_request_body( + %__MODULE__{streaming_request: %{state: {:stream_request, :chunked}, ref: ref}} = conn, + ref, + chunk + ) do + with {:ok, chunk} <- validate_chunk(conn, chunk), + :ok <- conn.transport.send(conn.socket, Request.encode_chunk(chunk)) do + case chunk do + :eof -> + request = %{conn.streaming_request | state: :status} + conn = enqueue_request(%{conn | streaming_request: nil}, request) + {:ok, conn} + + {:eof, _trailer_headers} -> + request = %{conn.streaming_request | state: :status} + conn = enqueue_request(%{conn | streaming_request: nil}, request) + {:ok, conn} + + _other -> + {:ok, conn} + end + else + :empty_chunk -> + {:ok, conn} + + {:error, %TransportError{reason: :closed} = error} -> + {:error, %{conn | state: :closed}, error} + + {:error, error} -> + {:error, conn, error} + end + end + + defp validate_chunk(conn, {:eof, trailers}) do + trailers = Headers.from_raw(trailers) + + if unallowed_header = Headers.find_unallowed_trailer(trailers) do + {:error, wrap_error({:unallowed_trailing_header, unallowed_header})} + else + {:ok, {:eof, Headers.to_raw(trailers, conn.case_sensitive_headers)}} + end + end + + defp validate_chunk(_conn, :eof) do + {:ok, :eof} + end + + defp validate_chunk(_conn, chunk) do + if IO.iodata_length(chunk) == 0 do + :empty_chunk + else + {:ok, chunk} + end + end + + @doc """ + See `Hex.Mint.HTTP.stream/2`. + """ + @impl true + @spec stream(t(), term()) :: + {:ok, t(), [Types.response()]} + | {:error, t(), Types.error(), [Types.response()]} + | :unknown + def stream(conn, message) + + def stream(%__MODULE__{transport: transport, socket: socket} = conn, {tag, socket, data}) + when tag in [:tcp, :ssl] do + case handle_data(conn, data) do + {:ok, %{mode: mode, state: state} = conn, responses} + when mode == :active and state != :closed -> + case transport.setopts(socket, active: :once) do + :ok -> {:ok, conn, responses} + {:error, reason} -> {:error, put_in(conn.state, :closed), reason, responses} + end + + other -> + other + end + end + + def stream(%__MODULE__{socket: socket} = conn, {tag, socket}) + when tag in [:tcp_closed, :ssl_closed] do + handle_close(conn) + end + + def stream(%__MODULE__{socket: socket} = conn, {tag, socket, reason}) + when tag in [:tcp_error, :ssl_error] do + handle_transport_error(conn, conn.transport.wrap_error(reason)) + end + + def stream(%__MODULE__{}, _message) do + :unknown + end + + defp handle_data(%__MODULE__{request: nil} = conn, data) do + conn = internal_close(conn) + {:error, conn, wrap_error({:unexpected_data, data}), []} + end + + defp handle_data(%__MODULE__{request: request} = conn, data) do + data = Util.maybe_concat(conn.buffer, data) + + case decode(request.state, conn, data, []) do + {:ok, conn, responses} -> + {:ok, conn, Enum.reverse(responses)} + + {:error, conn, reason, responses} -> + conn = put_in(conn.state, :closed) + {:error, conn, reason, responses} + end + end + + defp handle_close(%__MODULE__{request: request} = conn) do + conn = put_in(conn.state, :closed) + conn = request_done(conn) + + if request && request.body == :until_closed do + conn = put_in(conn.state, :closed) + {:ok, conn, [{:done, request.ref}]} + else + {:error, conn, conn.transport.wrap_error(:closed), []} + end + end + + defp handle_transport_error(conn, error) do + # The socket *should* be closed in this case, but it might not be, so let's still + # close it to make sure. + _ = conn.transport.close(conn.socket) + + conn = put_in(conn.state, :closed) + {:error, conn, error, []} + end + + @doc """ + See `Hex.Mint.HTTP.recv/3`. + """ + @impl true + @spec recv(t(), non_neg_integer(), timeout()) :: + {:ok, t(), [Types.response()]} + | {:error, t(), Types.error(), [Types.response()]} + def recv(conn, byte_count, timeout) + + def recv(%__MODULE__{mode: :passive} = conn, byte_count, timeout) do + case conn.transport.recv(conn.socket, byte_count, timeout) do + {:ok, data} -> handle_data(conn, data) + {:error, %Hex.Mint.TransportError{reason: :closed}} -> handle_close(conn) + {:error, error} -> handle_transport_error(conn, error) + end + end + + def recv(_conn, _byte_count, _timeout) do + raise ArgumentError, + "can't use recv/3 to synchronously receive data when the mode is :active. " <> + "Use Hex.Mint.HTTP.set_mode/2 to set the connection to passive mode" + end + + @doc """ + See `Hex.Mint.HTTP.set_mode/2`. + """ + @impl true + @spec set_mode(t(), :active | :passive) :: {:ok, t()} | {:error, Types.error()} + def set_mode(%__MODULE__{} = conn, mode) when mode in [:active, :passive] do + active = + case mode do + :active -> :once + :passive -> false + end + + with :ok <- conn.transport.setopts(conn.socket, active: active) do + {:ok, put_in(conn.mode, mode)} + end + end + + @doc """ + See `Hex.Mint.HTTP.controlling_process/2`. + """ + @impl true + @spec controlling_process(t(), pid()) :: {:ok, t()} | {:error, Types.error()} + def controlling_process(%__MODULE__{} = conn, new_pid) when is_pid(new_pid) do + with :ok <- conn.transport.controlling_process(conn.socket, new_pid) do + {:ok, conn} + end + end + + @doc """ + See `Hex.Mint.HTTP.open_request_count/1`. + + In HTTP/1, the number of open requests is the number of pipelined requests. + """ + @impl true + @spec open_request_count(t()) :: non_neg_integer() + def open_request_count(%__MODULE__{} = conn) do + case conn do + %{request: nil, streaming_request: nil} -> 0 + %{request: nil} -> 1 + %{streaming_request: nil} -> 1 + :queue.len(conn.requests) + _ -> 2 + :queue.len(conn.requests) + end + end + + @doc """ + See `Hex.Mint.HTTP.put_private/3`. + """ + @impl true + @spec put_private(t(), atom(), term()) :: t() + def put_private(%__MODULE__{private: private} = conn, key, value) when is_atom(key) do + %{conn | private: Map.put(private, key, value)} + end + + @doc """ + See `Hex.Mint.HTTP.get_private/3`. + """ + @impl true + @spec get_private(t(), atom(), term()) :: term() + def get_private(%__MODULE__{private: private} = _conn, key, default \\ nil) when is_atom(key) do + Map.get(private, key, default) + end + + @doc """ + See `Hex.Mint.HTTP.delete_private/2`. + """ + @impl true + @spec delete_private(t(), atom()) :: t() + def delete_private(%__MODULE__{private: private} = conn, key) when is_atom(key) do + %{conn | private: Map.delete(private, key)} + end + + @doc """ + See `Hex.Mint.HTTP.get_socket/1`. + """ + @impl true + @spec get_socket(t()) :: Hex.Mint.Types.socket() + def get_socket(%__MODULE__{socket: socket} = _conn) do + socket + end + + @doc """ + See `Hex.Mint.HTTP.put_log/2`. + """ + @doc since: "1.5.0" + @impl true + @spec put_log(t(), boolean()) :: t() + def put_log(%__MODULE__{} = conn, log?) when is_boolean(log?) do + %{conn | log: log?} + end + + @doc """ + See `Hex.Mint.HTTP.get_proxy_headers/1`. + """ + @doc since: "1.4.0" + @impl true + @spec get_proxy_headers(t()) :: Hex.Mint.Types.headers() + def get_proxy_headers(%__MODULE__{proxy_headers: proxy_headers}), do: proxy_headers + + # Made public since the %Hex.Mint.HTTP1{} struct is opaque. + @doc false + @impl true + @spec put_proxy_headers(t(), Hex.Mint.Types.headers()) :: t() + def put_proxy_headers(%__MODULE__{} = conn, headers) when is_list(headers) do + %{conn | proxy_headers: headers} + end + + ## Helpers + + defp decode(:status, %{request: request} = conn, data, responses) do + case Response.decode_status_line(data) do + {:ok, {version, status, status_reason}, rest} -> + request = %{request | version: version, status: status, state: :headers} + conn = %{conn | request: request} + responses = [{:status, request.ref, status} | responses] + + responses = + if :status_reason in conn.optional_responses do + [{:status_reason, request.ref, status_reason} | responses] + else + responses + end + + decode(:headers, conn, rest, responses) + + :more -> + conn = put_in(conn.buffer, data) + {:ok, conn, responses} + + :error -> + {:error, conn, wrap_error(:invalid_status_line), responses} + end + end + + defp decode(:headers, %{request: request} = conn, data, responses) do + decode_headers(conn, request, data, responses, request.headers_buffer) + end + + defp decode(:body, conn, data, responses) do + case message_body(conn.request) do + {:ok, body} -> + conn = put_in(conn.request.body, body) + decode_body(body, conn, data, conn.request.ref, responses) + + {:error, reason} -> + {:error, conn, wrap_error(reason), responses} + end + end + + defp decode_headers(conn, request, data, responses, headers) do + case Response.decode_header(data) do + {:ok, {name, value}, rest} -> + headers = [{name, value} | headers] + + case store_header(request, name, value) do + {:ok, request} -> decode_headers(conn, request, rest, responses, headers) + {:error, reason} -> {:error, conn, wrap_error(reason), responses} + end + + {:ok, :eof, rest} -> + responses = [{:headers, request.ref, Enum.reverse(headers)} | responses] + request = %{request | state: :body, headers_buffer: []} + conn = %{conn | buffer: "", request: request} + decode(:body, conn, rest, responses) + + :more -> + request = %{request | headers_buffer: headers} + conn = %{conn | buffer: data, request: request} + {:ok, conn, responses} + + :error -> + {:error, conn, wrap_error(:invalid_header), responses} + end + end + + defp decode_body(:none, conn, data, request_ref, responses) do + conn = put_in(conn.buffer, data) + conn = request_done(conn) + responses = [{:done, request_ref} | responses] + {:ok, conn, responses} + end + + # Informational (1xx) responses have no body and must not finalize the + # request; the final response follows on the same request ref. Reset the + # request's response-side fields and continue parsing without popping it. + defp decode_body(:informational, conn, data, _request_ref, responses) do + request = %{ + conn.request + | state: :status, + version: nil, + status: nil, + headers_buffer: [], + data_buffer: [], + content_length: nil, + connection: [], + transfer_encoding: [], + body: nil + } + + conn = %{conn | request: request, buffer: ""} + decode(:status, conn, data, responses) + end + + defp decode_body(:single, conn, data, request_ref, responses) do + {conn, responses} = add_body(conn, data, responses) + conn = request_done(conn) + responses = [{:done, request_ref} | responses] + {:ok, conn, responses} + end + + defp decode_body(:until_closed, conn, data, _request_ref, responses) do + {conn, responses} = add_body(conn, data, responses) + {:ok, conn, responses} + end + + defp decode_body({:content_length, length}, conn, data, request_ref, responses) do + cond do + length > byte_size(data) -> + conn = put_in(conn.request.body, {:content_length, length - byte_size(data)}) + {conn, responses} = add_body(conn, data, responses) + {:ok, conn, responses} + + length <= byte_size(data) -> + <> = data + {conn, responses} = add_body(conn, body, responses) + conn = request_done(conn) + responses = [{:done, request_ref} | responses] + next_request(conn, rest, responses) + end + end + + defp decode_body({:chunked, nil}, conn, "", _request_ref, responses) do + conn = put_in(conn.buffer, "") + conn = put_in(conn.request.body, {:chunked, nil}) + {:ok, conn, responses} + end + + defp decode_body({:chunked, nil}, conn, data, request_ref, responses) do + case Integer.parse(data, 16) do + {_size, ""} -> + conn = put_in(conn.buffer, data) + conn = put_in(conn.request.body, {:chunked, nil}) + {:ok, conn, responses} + + {0, rest} -> + # Manually collapse the body buffer since we're done with the body + {conn, responses} = collapse_body_buffer(conn, responses) + decode_body({:chunked, :metadata, :trailer}, conn, rest, request_ref, responses) + + {size, rest} when size > 0 -> + decode_body({:chunked, :metadata, size}, conn, rest, request_ref, responses) + + _other -> + {:error, conn, wrap_error(:invalid_chunk_size), responses} + end + end + + defp decode_body({:chunked, :metadata, size}, conn, data, request_ref, responses) do + case Parse.ignore_until_crlf(data) do + {:ok, rest} -> + decode_body({:chunked, size}, conn, rest, request_ref, responses) + + :more -> + conn = put_in(conn.buffer, data) + conn = put_in(conn.request.body, {:chunked, :metadata, size}) + {:ok, conn, responses} + end + end + + defp decode_body({:chunked, :trailer}, conn, data, _request_ref, responses) do + decode_trailer_headers(conn, data, responses, conn.request.headers_buffer) + end + + defp decode_body({:chunked, :crlf}, conn, data, request_ref, responses) do + case data do + <<"\r\n", rest::binary>> -> + conn = put_in(conn.request.body, {:chunked, nil}) + decode_body({:chunked, nil}, conn, rest, request_ref, responses) + + _other when byte_size(data) < 2 -> + conn = put_in(conn.buffer, data) + {:ok, conn, responses} + + _other -> + {:error, conn, wrap_error(:missing_crlf_after_chunk), responses} + end + end + + defp decode_body({:chunked, length}, conn, data, request_ref, responses) do + cond do + length > byte_size(data) -> + conn = put_in(conn.buffer, "") + conn = put_in(conn.request.body, {:chunked, length - byte_size(data)}) + conn = add_body_to_buffer(conn, data) + {:ok, conn, responses} + + length <= byte_size(data) -> + <> = data + {conn, responses} = add_body(conn, body, responses) + conn = put_in(conn.request.body, {:chunked, :crlf}) + decode_body({:chunked, :crlf}, conn, rest, request_ref, responses) + end + end + + defp decode_trailer_headers(conn, data, responses, headers) do + case Response.decode_header(data) do + {:ok, {name, value}, rest} -> + headers = [{name, value} | headers] + decode_trailer_headers(conn, rest, responses, headers) + + {:ok, :eof, rest} -> + headers = Headers.remove_unallowed_trailer(headers) + + responses = [ + {:done, conn.request.ref} + | add_trailer_headers(headers, conn.request.ref, responses) + ] + + conn = request_done(conn) + next_request(conn, rest, responses) + + :more -> + request = %{conn.request | body: {:chunked, :trailer}, headers_buffer: headers} + conn = %{conn | buffer: data, request: request} + {:ok, conn, responses} + + :error -> + {:error, conn, wrap_error(:invalid_trailer_header), responses} + end + end + + defp next_request(%{request: nil} = conn, data, responses) do + # TODO: Figure out if we should keep buffering even though there are no + # requests in flight + {:ok, %{conn | buffer: data}, responses} + end + + defp next_request(conn, data, responses) do + decode(:status, %{conn | state: :status}, data, responses) + end + + defp add_trailer_headers([], _request_ref, responses), do: responses + + defp add_trailer_headers(headers, request_ref, responses), + do: [{:headers, request_ref, Enum.reverse(headers)} | responses] + + defp add_body(conn, data, responses) do + conn = add_body_to_buffer(conn, data) + collapse_body_buffer(conn, responses) + end + + defp add_body_to_buffer(conn, data) do + update_in(conn.request.data_buffer, &[&1 | data]) + end + + defp collapse_body_buffer(conn, responses) do + case IO.iodata_to_binary(conn.request.data_buffer) do + "" -> + {conn, responses} + + data -> + conn = put_in(conn.request.data_buffer, []) + {conn, [{:data, conn.request.ref, data} | responses]} + end + end + + defp store_header(%{content_length: nil} = request, "content-length", value) do + with {:ok, content_length} <- Parse.content_length_header(value), + do: {:ok, %{request | content_length: content_length}} + end + + defp store_header(%{connection: connection} = request, "connection", value) do + with {:ok, connection_header} <- Parse.connection_header(value), + do: {:ok, %{request | connection: connection ++ connection_header}} + end + + defp store_header(%{transfer_encoding: transfer_encoding} = request, "transfer-encoding", value) do + with {:ok, transfer_encoding_header} <- Parse.transfer_encoding_header(value), + do: {:ok, %{request | transfer_encoding: transfer_encoding ++ transfer_encoding_header}} + end + + defp store_header(_request, "content-length", _value) do + {:error, :more_than_one_content_length_header} + end + + defp store_header(request, _name, _value) do + {:ok, request} + end + + defp request_done(%{request: request} = conn) do + conn = pop_request(conn) + + cond do + !request -> conn + "close" in request.connection -> internal_close(conn) + request.version >= {1, 1} -> conn + "keep-alive" in request.connection -> conn + true -> internal_close(conn) + end + end + + defp pop_request(conn) do + case :queue.out(conn.requests) do + {{:value, request}, requests} -> + %{conn | request: request, requests: requests} + + {:empty, requests} -> + %{conn | request: nil, requests: requests} + end + end + + defp enqueue_request(%{request: nil} = conn, request) do + %{conn | request: request} + end + + defp enqueue_request(conn, request) do + requests = :queue.in(request, conn.requests) + %{conn | requests: requests} + end + + defp internal_close(conn) do + if conn.buffer != "" do + log(conn, :debug, ["Connection closed with data left in the buffer: ", inspect(conn.buffer)]) + end + + _ = conn.transport.close(conn.socket) + %{conn | state: :closed} + end + + # RFC7230 3.3.3: + # > If a message is received with both a Transfer-Encoding and a + # > Content-Length header field, the Transfer-Encoding overrides the + # > Content-Length. Such a message might indicate an attempt to + # > perform request smuggling (Section 9.5) or response splitting + # > (Section 9.4) and ought to be handled as an error. A sender MUST + # > remove the received Content-Length field prior to forwarding such + # > a message downstream. + defp message_body(%{body: nil, method: method, status: status} = request) do + cond do + status == 101 -> + {:ok, :single} + + status in 100..199 -> + {:ok, :informational} + + method == "HEAD" or status in [204, 304] -> + {:ok, :none} + + # method == "CONNECT" and status in 200..299 -> nil + + request.transfer_encoding != [] && request.content_length -> + {:error, :transfer_encoding_and_content_length} + + "chunked" == List.first(request.transfer_encoding) -> + {:ok, {:chunked, nil}} + + request.content_length -> + {:ok, {:content_length, request.content_length}} + + true -> + {:ok, :until_closed} + end + end + + defp message_body(%{body: body}) do + {:ok, body} + end + + defp validate_request_target(target, skip_validation?) + defp validate_request_target(target, false), do: validate_target(target) + defp validate_request_target(_, true), do: :ok + + # Percent-encoding is not case sensitive so we have to account for lowercase and uppercase. + @hex_characters ~c"0123456789abcdefABCDEF" + + defp validate_target(<<>> = empty_target), do: {:error, {:invalid_request_target, empty_target}} + defp validate_target(target), do: validate_target(target, target) + + defp validate_target(<>, original_target) + when char1 in @hex_characters and char2 in @hex_characters do + validate_target(rest, original_target) + end + + defp validate_target(<>, original_target) do + if URI.char_unescaped?(char) do + validate_target(rest, original_target) + else + {:error, {:invalid_request_target, original_target}} + end + end + + defp validate_target(<<>>, _original_target) do + :ok + end + + defp new_request(ref, method, body, encoding) do + state = + if body == :stream do + {:stream_request, encoding} + else + :status + end + + %{ + ref: ref, + state: state, + method: method, + version: nil, + status: nil, + headers_buffer: [], + data_buffer: [], + content_length: nil, + connection: [], + transfer_encoding: [], + body: nil + } + end + + defp add_default_headers(headers, conn) do + headers + |> Headers.put_new("User-Agent", "user-agent", @user_agent) + |> Headers.put_new("Host", "host", default_host_header(conn)) + end + + # If the port is the default for the scheme, don't add it to the host header + defp default_host_header(%__MODULE__{scheme_as_string: scheme, host: host, port: port}) do + if URI.default_port(scheme) == port do + host + else + "#{host}:#{port}" + end + end + + defp add_content_length_or_transfer_encoding(headers, :stream) do + cond do + Headers.has?(headers, "content-length") -> + {:ok, headers, :identity} + + found = Headers.find(headers, "transfer-encoding") -> + {raw_name, value} = found + + with {:ok, tokens} <- Parse.transfer_encoding_header(value) do + if "chunked" in tokens or "identity" in tokens do + {:ok, headers, :identity} + else + headers = + Headers.replace(headers, raw_name, "transfer-encoding", value <> ",chunked") + + {:ok, headers, :chunked} + end + end + + # If no content-length or transfer-encoding are present, assume + # chunked transfer-encoding and handle the encoding ourselves. + true -> + headers = + Headers.put_new(headers, "Transfer-Encoding", "transfer-encoding", "chunked") + + {:ok, headers, :chunked} + end + end + + defp add_content_length_or_transfer_encoding(headers, nil) do + {:ok, headers, :identity} + end + + defp add_content_length_or_transfer_encoding(headers, body) do + length_fun = fn -> body |> IO.iodata_length() |> Integer.to_string() end + + {:ok, Headers.put_new_lazy(headers, "Content-Length", "content-length", length_fun), + :identity} + end + + defp wrap_error(reason) do + %HTTPError{reason: reason, module: __MODULE__} + end + + @doc false + def format_error(reason) + + def format_error(:closed) do + "the connection is closed" + end + + def format_error(:request_body_is_streaming) do + "a request body is currently streaming, so no new requests can be issued" + end + + def format_error({:unexpected_data, data}) do + "received unexpected data: " <> inspect(data) + end + + def format_error(:invalid_status_line) do + "invalid status line" + end + + def format_error(:invalid_header) do + "invalid header" + end + + def format_error({:invalid_request_target, target}) do + "invalid request target: #{inspect(target)}" + end + + def format_error({:invalid_header_name, name}) do + "invalid header name: #{inspect(name)}" + end + + def format_error({:invalid_header_value, name, value}) do + "invalid value for header (only printable ASCII characters are allowed) " <> + "#{inspect(name)}: #{inspect(value)}" + end + + def format_error(:invalid_chunk_size) do + "invalid chunk size" + end + + def format_error(:missing_crlf_after_chunk) do + "missing CRLF after chunk" + end + + def format_error(:invalid_trailer_header) do + "invalid trailer header" + end + + def format_error(:more_than_one_content_length_header) do + "the response contains two or more Content-Length headers" + end + + def format_error(:transfer_encoding_and_content_length) do + "the response contained both a Transfer-Encoding header as well as a Content-Length header" + end + + def format_error({:invalid_content_length_header, value}) do + "invalid Content-Length header: #{inspect(value)}" + end + + def format_error(:empty_token_list) do + "header should contain a list of values, but it doesn't" + end + + def format_error({:invalid_token_list, string}) do + "header contains invalid tokens: #{inspect(string)}" + end + + def format_error(:trailing_headers_but_not_chunked_encoding) do + "trailer headers can only be sent when using chunked transfer-encoding" + end + + def format_error({:unallowed_trailing_header, {name, value}}) do + "header #{inspect(name)} (with value #{inspect(value)}) is not allowed as a trailer header" + end +end diff --git a/lib/hex/mint/http1/parse.ex b/lib/hex/mint/http1/parse.ex new file mode 100644 index 00000000..04479aad --- /dev/null +++ b/lib/hex/mint/http1/parse.ex @@ -0,0 +1,74 @@ +# Vendored from mint v1.7.1 (d30d2cf), do not edit manually + +defmodule Hex.Mint.HTTP1.Parse do + @moduledoc false + + defmacro is_digit(char), do: quote(do: unquote(char) in ?0..?9) + defmacro is_alpha(char), do: quote(do: unquote(char) in ?a..?z or unquote(char) in ?A..?Z) + defmacro is_whitespace(char), do: quote(do: unquote(char) in ~c"\s\t") + defmacro is_comma(char), do: quote(do: unquote(char) == ?,) + defmacro is_vchar(char), do: quote(do: unquote(char) in 33..126) + + defmacro is_tchar(char) do + quote do + is_digit(unquote(char)) or is_alpha(unquote(char)) or unquote(char) in ~c"!#$%&'*+-.^_`|~" + end + end + + def ignore_until_crlf(<<>>), do: :more + def ignore_until_crlf(<<"\r\n", rest::binary>>), do: {:ok, rest} + def ignore_until_crlf(<<_char, rest::binary>>), do: ignore_until_crlf(rest) + + def content_length_header(string) do + case Integer.parse(String.trim_trailing(string)) do + {length, ""} when length >= 0 -> {:ok, length} + _other -> {:error, {:invalid_content_length_header, string}} + end + end + + def connection_header(string) do + split_into_downcase_tokens(string) + end + + def transfer_encoding_header(string) do + split_into_downcase_tokens(string) + end + + defp split_into_downcase_tokens(string) do + case token_list_downcase(string) do + {:ok, []} -> {:error, :empty_token_list} + {:ok, list} -> {:ok, list} + :error -> {:error, {:invalid_token_list, string}} + end + end + + # Made public for testing. + def token_list_downcase(string), do: token_list_downcase(string, []) + + defp token_list_downcase(<<>>, acc), do: {:ok, :lists.reverse(acc)} + + # Skip all whitespace and commas. + defp token_list_downcase(<>, acc) + when is_whitespace(char) or is_comma(char), + do: token_list_downcase(rest, acc) + + defp token_list_downcase(rest, acc), do: token_downcase(rest, _token_acc = <<>>, acc) + + defp token_downcase(<>, token_acc, acc) when is_tchar(char), + do: token_downcase(rest, <>, acc) + + defp token_downcase(rest, token_acc, acc), do: token_list_sep_downcase(rest, [token_acc | acc]) + + defp token_list_sep_downcase(<<>>, acc), do: {:ok, :lists.reverse(acc)} + + defp token_list_sep_downcase(<>, acc) when is_whitespace(char), + do: token_list_sep_downcase(rest, acc) + + defp token_list_sep_downcase(<>, acc) when is_comma(char), + do: token_list_downcase(rest, acc) + + defp token_list_sep_downcase(_rest, _acc), do: :error + + defp downcase_ascii_char(char) when char in ?A..?Z, do: char + 32 + defp downcase_ascii_char(char) when char in 0..127, do: char +end diff --git a/lib/hex/mint/http1/request.ex b/lib/hex/mint/http1/request.ex new file mode 100644 index 00000000..fd33667e --- /dev/null +++ b/lib/hex/mint/http1/request.ex @@ -0,0 +1,71 @@ +# Vendored from mint v1.7.1 (d30d2cf), do not edit manually + +defmodule Hex.Mint.HTTP1.Request do + @moduledoc false + + import Hex.Mint.HTTP1.Parse + + def encode(method, target, headers, body) do + body = [ + encode_request_line(method, target), + encode_headers(headers), + "\r\n", + encode_body(body) + ] + + {:ok, body} + catch + {:hex_mint, reason} -> {:error, reason} + end + + defp encode_request_line(method, target) do + [method, ?\s, target, " HTTP/1.1\r\n"] + end + + defp encode_headers(headers) do + Enum.reduce(headers, "", fn {name, value}, acc -> + validate_header_name!(name) + validate_header_value!(name, value) + [acc, name, ": ", value, "\r\n"] + end) + end + + defp encode_body(nil), do: "" + defp encode_body(:stream), do: "" + defp encode_body(body), do: body + + def encode_chunk(:eof) do + "0\r\n\r\n" + end + + def encode_chunk({:eof, trailing_headers}) do + ["0\r\n", encode_headers(trailing_headers), "\r\n"] + end + + def encode_chunk(chunk) do + length = IO.iodata_length(chunk) + [Integer.to_string(length, 16), "\r\n", chunk, "\r\n"] + end + + defp validate_header_name!(name) do + _ = + for <> do + unless is_tchar(char) do + throw({:hex_mint, {:invalid_header_name, name}}) + end + end + + :ok + end + + defp validate_header_value!(name, value) do + _ = + for <> do + unless is_vchar(char) or char in ~c"\s\t" do + throw({:hex_mint, {:invalid_header_value, name, value}}) + end + end + + :ok + end +end diff --git a/lib/hex/mint/http1/response.ex b/lib/hex/mint/http1/response.ex new file mode 100644 index 00000000..b02ee818 --- /dev/null +++ b/lib/hex/mint/http1/response.ex @@ -0,0 +1,45 @@ +# Vendored from mint v1.7.1 (d30d2cf), do not edit manually + +defmodule Hex.Mint.HTTP1.Response do + @moduledoc false + + alias Hex.Mint.Core.Headers + + def decode_status_line(binary) do + case :erlang.decode_packet(:http_bin, binary, []) do + {:ok, {:http_response, version, status, reason}, rest} -> + {:ok, {version, status, reason}, rest} + + {:ok, _other, _rest} -> + :error + + {:more, _length} -> + :more + + {:error, _reason} -> + :error + end + end + + def decode_header(binary) do + case :erlang.decode_packet(:httph_bin, binary, []) do + {:ok, {:http_header, _unused, name, _reserved, value}, rest} -> + {:ok, {header_name(name), value}, rest} + + {:ok, :http_eoh, rest} -> + {:ok, :eof, rest} + + {:ok, _other, _rest} -> + :error + + {:more, _length} -> + :more + + {:error, _reason} -> + :error + end + end + + defp header_name(atom) when is_atom(atom), do: atom |> Atom.to_string() |> header_name() + defp header_name(binary) when is_binary(binary), do: Headers.lower_raw(binary) +end diff --git a/lib/hex/mint/http2.ex b/lib/hex/mint/http2.ex new file mode 100644 index 00000000..5bd53d7e --- /dev/null +++ b/lib/hex/mint/http2.ex @@ -0,0 +1,2544 @@ +# Vendored from mint v1.7.1 (d30d2cf), do not edit manually + +defmodule Hex.Mint.HTTP2 do + _ = """ + Process-less HTTP/2 client connection. + + This module provides a data structure that represents an HTTP/2 connection to + a given server. The connection is represented as an opaque struct `%Hex.Mint.HTTP2{}`. + The connection is a data structure and is not backed by a process, and all the + connection handling happens in the process that creates the struct. + + This module and data structure work exactly like the ones described in the `Hex.Mint.HTTP` + module, with the exception that `Hex.Mint.HTTP2` specifically deals with HTTP/2 while + `Hex.Mint.HTTP` deals seamlessly with HTTP/1.1 and HTTP/2. For more information on + how to use the data structure and client architecture, see `Hex.Mint.HTTP`. + + ## HTTP/2 Streams and Requests + + HTTP/2 introduces the concept of **streams**. A stream is an isolated conversation + between the client and the server. Each stream is unique and identified by a unique + **stream ID**, which means that there's no order when data comes on different streams + since they can be identified uniquely. A stream closely corresponds to a request, so + in this documentation and client we will mostly refer to streams as "requests". + We mentioned data on streams can come in arbitrary order, and streams are requests, + so the practical effect of this is that performing request A and then request B + does not mean that the response to request A will come before the response to request B. + This is why we identify each request with a unique reference returned by `request/5`. + See `request/5` for more information. + + ## Closed Connection + + In HTTP/2, the connection can either be open, closed, or only closed for writing. + When a connection is closed for writing, the client cannot send requests or stream + body chunks, but it can still read data that the server might be sending. When the + connection gets closed on the writing side, a `:server_closed_connection` error is + returned. `{:error, request_ref, error}` is returned for requests that haven't been + processed by the server, with the reason of `error` being `:unprocessed`. + These requests are safe to retry. + + ## HTTP/2 Settings + + HTTP/2 supports settings negotiation between servers and clients. The server advertises + its settings to the client and the client advertises its settings to the server. A peer + (server or client) has to acknowledge the settings advertised by the other peer before + those settings come into action (that's why it's called a negotiation). + + A first settings negotiation happens right when the connection starts. + Servers and clients can renegotiate settings at any time during the life of the + connection. + + Mint users don't need to care about settings acknowledgements directly since they're + handled transparently by `stream/2`. + + To retrieve the server settings, you can use `get_server_setting/2`. Doing so is often + useful to be able to tune your requests based on the server settings. + + To communicate client settings to the server, use `put_settings/2` or pass them when + starting up a connection with `connect/4`. Note that the server needs to acknowledge + the settings sent through `put_setting/2` before those settings come into effect. The + server ack is processed transparently by `stream/2`, but this means that if you change + a setting through `put_settings/2` and try to retrieve the value of that setting right + after with `get_client_setting/2`, you'll likely get the old value of that setting. Once + the server acknowledges the new settings, the updated value will be returned by + `get_client_setting/2`. + + ## Server Push + + HTTP/2 supports [server push](https://en.wikipedia.org/wiki/HTTP/2_Server_Push), which + is a way for a server to send a response to a client without the client needing to make + the corresponding request. The server sends a `:push_promise` response to a normal request: + this creates a new request reference. Then, the server sends normal responses for the newly + created request reference. + + Let's see an example. We will ask the server for `"/index.html"` and the server will + send us a push promise for `"/style.css"`. + + {:ok, conn} = Hex.Mint.HTTP2.connect(:https, "example.com", 443) + {:ok, conn, request_ref} = Hex.Mint.HTTP2.request(conn, "GET", "/index.html", _headers = [], _body = "") + + next_message = + receive do + msg -> msg + end + + {:ok, conn, responses} = Hex.Mint.HTTP2.stream(conn, next_message) + + [ + {:push_promise, ^request_ref, promised_request_ref, promised_headers}, + {:status, ^request_ref, 200}, + {:headers, ^request_ref, []}, + {:data, ^request_ref, "..."}, + {:done, ^request_ref} + ] = responses + + promised_headers + #=> [{":method", "GET"}, {":path", "/style.css"}] + + As you can see in the example above, when the server sends a push promise then a + `:push_promise` response is returned as a response to a request. The `:push_promise` + response contains a `promised_request_ref` and some `promised_headers`. The + `promised_request_ref` is the new request ref that pushed responses will be tagged with. + `promised_headers` are headers that tell the client *what request* the promised response + will respond to. The idea is that the server tells the client a request the client will + want to make and then preemptively sends a response for that request. Promised headers + will always include `:method`, `:path`, and `:authority`. + + next_message = + receive do + msg -> msg + end + + {:ok, conn, responses} = Hex.Mint.HTTP2.stream(conn, next_message) + + [ + {:status, ^promised_request_ref, 200}, + {:headers, ^promised_request_ref, []}, + {:data, ^promised_request_ref, "body { ... }"}, + {:done, ^promised_request_ref} + ] + + The response to a promised request is like a response to any normal request. + + > #### Disabling Server Pushes {: .tip} + > + > HTTP/2 exposes a boolean setting for enabling or disabling server pushes with `:enable_push`. + > You can pass this option when connecting or in `put_settings/2`. By default server push + > is enabled. + """ + + import Hex.Mint.HTTP2.Frame, except: [encode: 1, decode_next: 1, inspect: 1] + + alias Hex.Mint.{HTTPError, TransportError} + alias Hex.Mint.Types + alias Hex.Mint.Core.{Headers, Util} + alias Hex.Mint.HTTP2.Frame + + require Logger + require Integer + + @behaviour Hex.Mint.Core.Conn + + ## Constants + + @connection_preface "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" + @transport_opts [alpn_advertised_protocols: ["h2"]] + + @default_window_size 65_535 + @default_connection_window_size 16 * 1024 * 1024 + @default_stream_window_size 4 * 1024 * 1024 + @max_window_size 2_147_483_647 + + # Defer refilling the receive window until it has dropped to this many + # bytes — roughly 10× the default 16 KB max frame size, so the server + # has a safety margin before the window would starve it. See + # `refill_client_windows/3`. + @default_receive_window_update_threshold 160_000 + + @default_max_frame_size 16_384 + @valid_max_frame_size_range @default_max_frame_size..16_777_215 + + @valid_client_settings [ + :max_concurrent_streams, + :initial_window_size, + :max_frame_size, + :enable_push, + :max_header_list_size + ] + + @user_agent "mint/" <> "1.7.1" + + # HTTP/2 connection struct. + defstruct [ + # Transport things. + :transport, + :socket, + :mode, + + # Host things. + :hostname, + :port, + :scheme, + :authority, + + # Connection state (open, closed, and so on). + :state, + + # Fields of the connection. + buffer: "", + # `send_window_size` is the client *send* window for the connection + # — how much request-body data we're allowed to send to the server + # before it refills the window with a WINDOW_UPDATE frame. + send_window_size: @default_window_size, + # `receive_window_size` is the client *receive* window for the + # connection — the peak size we've advertised to the server via + # `WINDOW_UPDATE` frames on stream 0. Initialized to the configured + # connection window during `initiate/5`, which also sends the + # matching WINDOW_UPDATE to bring the server's view from the spec + # default of 65_535 up to the advertised peak. + receive_window_size: @default_window_size, + # `receive_window` is the server's current view of our receive + # window — decremented by DATA frame sizes as they arrive, bumped + # back up to `receive_window_size` whenever we send a + # WINDOW_UPDATE. When it drops to `receive_window_update_threshold`, we + # refill it back to the peak in one frame. + receive_window: @default_window_size, + # Minimum remaining receive window before we send a WINDOW_UPDATE. + # Configurable via the `:receive_window_update_threshold` connect option. + receive_window_update_threshold: @default_receive_window_update_threshold, + encode_table: Hex.Mint.HPAX.new(4096), + decode_table: Hex.Mint.HPAX.new(4096), + + # Queue for sent PING frames. + ping_queue: :queue.new(), + + # Queue for sent SETTINGS frames. + client_settings_queue: :queue.new(), + + # Stream-set-related things. + next_stream_id: 3, + streams: %{}, + open_client_stream_count: 0, + open_server_stream_count: 0, + ref_to_stream_id: %{}, + + # Settings that the server communicates to the client. + server_settings: %{ + enable_push: true, + max_concurrent_streams: 100, + initial_window_size: @default_window_size, + max_frame_size: @default_max_frame_size, + max_header_list_size: :infinity, + # Only supported by the server: https://www.rfc-editor.org/rfc/rfc8441.html#section-3 + enable_connect_protocol: false + }, + + # Settings that the client communicates to the server. + client_settings: %{ + max_concurrent_streams: 100, + initial_window_size: @default_stream_window_size, + max_header_list_size: :infinity, + max_frame_size: @default_max_frame_size, + enable_push: true + }, + + # Headers being processed (when headers are split into multiple frames with CONTINUATIONS, all + # the continuation frames must come one right after the other). + headers_being_processed: nil, + + # Stores the headers returned by the proxy in the `CONNECT` method + proxy_headers: [], + + # Private store. + private: %{}, + + # Logging + log: false + ] + + defmacrop log(conn, level, message) do + quote do + conn = unquote(conn) + + if conn.log do + Logger.log(unquote(level), unquote(message)) + else + :ok + end + end + end + + ## Types + + @typedoc """ + HTTP/2 setting with its value. + + This type represents both server settings as well as client settings. To retrieve + server settings use `get_server_setting/2` and to retrieve client settings use + `get_client_setting/2`. To send client settings to the server, see `put_settings/2`. + + The supported settings are the following: + + * `:header_table_size` - corresponds to `SETTINGS_HEADER_TABLE_SIZE`. + + * `:enable_push` - corresponds to `SETTINGS_ENABLE_PUSH`. Sets whether + push promises are supported. If you don't want to support push promises, + use `put_settings/2` to tell the server that your client doesn't want push promises. + + * `:max_concurrent_streams` - corresponds to `SETTINGS_MAX_CONCURRENT_STREAMS`. + Tells what is the maximum number of streams that the peer sending this (client or server) + supports. As mentioned in the module documentation, HTTP/2 streams are equivalent to + requests, so knowing the maximum number of streams that the server supports can be useful + to know how many concurrent requests can be open at any time. Use `get_server_setting/2` + to find out how many concurrent streams the server supports. + + * `:initial_window_size` - corresponds to `SETTINGS_INITIAL_WINDOW_SIZE`. + Tells what is the value of the initial HTTP/2 window size for the peer + that sends this setting. + + * `:max_frame_size` - corresponds to `SETTINGS_MAX_FRAME_SIZE`. Tells what is the + maximum size of an HTTP/2 frame for the peer that sends this setting. + + * `:max_header_list_size` - corresponds to `SETTINGS_MAX_HEADER_LIST_SIZE`. + + * `:enable_connect_protocol` - corresponds to `SETTINGS_ENABLE_CONNECT_PROTOCOL`. + Sets whether the client may invoke the extended connect protocol which is used to + bootstrap WebSocket connections. + + """ + @type setting() :: + {:enable_push, boolean()} + | {:header_table_size, non_neg_integer()} + | {:max_concurrent_streams, pos_integer()} + | {:initial_window_size, 1..2_147_483_647} + | {:max_frame_size, 16_384..16_777_215} + | {:max_header_list_size, :infinity | pos_integer()} + | {:enable_connect_protocol, boolean()} + + @typedoc """ + HTTP/2 settings. + + See `t:setting/0`. + """ + @type settings() :: [setting()] + + @typedoc """ + An HTTP/2-specific error reason. + + The values can be: + + * `:closed` - when you try to make a request or stream a body chunk but the connection + is closed. + + * `:closed_for_writing` - when you try to make a request or stream a body chunk but + the connection is closed for writing. This means you cannot issue any more requests. + See the "Closed connection" section in the module documentation for more information. + + * `:too_many_concurrent_requests` - when the maximum number of concurrent requests + allowed by the server is reached. To find out what this limit is, use `get_setting/2` + with the `:max_concurrent_streams` setting name. + + * `{:max_header_list_size_exceeded, size, max_size}` - when the maximum size of + the header list is reached. `size` is the actual value of the header list size, + `max_size` is the maximum value allowed. See `get_setting/2` to retrieve the + value of the max size. + + * `{:exceeds_window_size, what, window_size}` - when the data you're trying to send + exceeds the window size of the connection (if `what` is `:connection`) or of a request + (if `what` is `:request`). `window_size` is the allowed window size. See + `get_window_size/2`. + + * `{:stream_not_found, stream_id}` - when the given request is not found. + + * `:unknown_request_to_stream` - when you're trying to stream data on an unknown + request. + + * `:request_is_not_streaming` - when you try to send data (with `stream_request_body/3`) + on a request that is not open for streaming. + + * `:unprocessed` - when a request was closed because it was not processed by the server. + When this error is returned, it means that the server hasn't processed the request at all, + so it's safe to retry the given request on a different or new connection. + + * `{:server_closed_request, error_code}` - when the server closes the request. + `error_code` is the reason why the request was closed. + + * `{:server_closed_connection, reason, debug_data}` - when the server closes the connection + gracefully or because of an error. In HTTP/2, this corresponds to a `GOAWAY` frame. + `error` is the reason why the connection was closed. `debug_data` is additional debug data. + + * `{:frame_size_error, frame}` - when there's an error with the size of a frame. + `frame` is the frame type, such as `:settings` or `:window_update`. + + * `{:protocol_error, debug_data}` - when there's a protocol error. + `debug_data` is a string that explains the nature of the error. + + * `{:compression_error, debug_data}` - when there's a header compression error. + `debug_data` is a string that explains the nature of the error. + + * `{:flow_control_error, debug_data}` - when there's a flow control error. + `debug_data` is a string that explains the nature of the error. + + """ + @type error_reason() :: term() + + @typedoc """ + A Mint HTTP/2 connection struct. + + The struct's fields are private. + """ + @opaque t() :: %__MODULE__{} + + ## Public interface + + @doc """ + Same as `Hex.Mint.HTTP.connect/4`, but forces a HTTP/2 connection. + """ + @spec connect(Types.scheme(), Types.address(), :inet.port_number(), keyword()) :: + {:ok, t()} | {:error, Types.error()} + def connect(scheme, address, port, opts \\ []) do + hostname = Hex.Mint.Core.Util.hostname(opts, address) + + transport_opts = + opts + |> Keyword.get(:transport_opts, []) + |> Keyword.merge(@transport_opts) + |> Keyword.put(:hostname, hostname) + + case negotiate(address, port, scheme, transport_opts) do + {:ok, socket} -> + initiate(scheme, socket, hostname, port, opts) + + {:error, reason} -> + {:error, reason} + end + end + + @doc false + @spec upgrade( + Types.scheme(), + Hex.Mint.Types.socket(), + Types.scheme(), + String.t(), + :inet.port_number(), + keyword() + ) :: {:ok, t()} | {:error, Types.error()} + def upgrade(old_scheme, socket, new_scheme, hostname, port, opts) do + transport = Util.scheme_to_transport(new_scheme) + + transport_opts = + opts + |> Keyword.get(:transport_opts, []) + |> Keyword.merge(@transport_opts) + + with {:ok, socket} <- transport.upgrade(socket, old_scheme, hostname, port, transport_opts) do + initiate(new_scheme, socket, hostname, port, opts) + end + end + + @doc """ + See `Hex.Mint.HTTP.close/1`. + """ + @impl true + @spec close(t()) :: {:ok, t()} + def close(conn) + + def close(%__MODULE__{state: :open} = conn) do + send_connection_error!(conn, :no_error, "connection peacefully closed by client") + catch + {:hex_mint, conn, %HTTPError{reason: {:no_error, _}}} -> + {:ok, conn} + end + + def close(%__MODULE__{state: {:goaway, _error_code, _debug_data}} = conn) do + _ = conn.transport.close(conn.socket) + {:ok, put_in(conn.state, :closed)} + end + + def close(%__MODULE__{state: :handshaking} = conn) do + _ = conn.transport.close(conn.socket) + {:ok, put_in(conn.state, :closed)} + end + + def close(%__MODULE__{state: :closed} = conn) do + {:ok, conn} + end + + @doc """ + See `Hex.Mint.HTTP.open?/1`. + """ + @impl true + @spec open?(t(), :read | :write) :: boolean() + def open?(%__MODULE__{state: state} = _conn, type \\ :write) + when type in [:read, :write, :read_write] do + case state do + :handshaking -> true + :open -> true + {:goaway, _error_code, _debug_data} -> type == :read + :closed -> false + end + end + + @doc """ + See `Hex.Mint.HTTP.request/5`. + + In HTTP/2, opening a request means opening a new HTTP/2 stream (see the + module documentation). This means that a request could fail because the + maximum number of concurrent streams allowed by the server has been reached. + In that case, the error reason `:too_many_concurrent_requests` is returned. + If you want to avoid incurring in this error, you can retrieve the value of + the maximum number of concurrent streams supported by the server through + `get_server_setting/2` (passing in the `:max_concurrent_streams` setting name). + + ## Header list size + + In HTTP/2, the server can optionally specify a maximum header list size that + the client needs to respect when sending headers. The header list size is calculated + by summing the length (in bytes) of each header name plus value, plus 32 bytes for + each header. Note that pseudo-headers (like `:path` or `:method`) count towards + this size. If the size is exceeded, an error is returned. To check what the size + is, use `get_server_setting/2`. + + ## Request body size + + If the request body size will exceed the window size of the HTTP/2 stream created by the + request or the window size of the connection Mint will return a `:exceeds_window_size` + error. + + To ensure you do not exceed the window size it is recommended to stream the request + body by initially passing `:stream` as the body and sending the body in chunks using + `stream_request_body/3` and using `get_window_size/2` to get the window size of the + request and connection. + """ + @impl true + @spec request( + t(), + method :: String.t(), + path :: String.t(), + Types.headers(), + body :: iodata() | nil | :stream + ) :: + {:ok, t(), Types.request_ref()} + | {:error, t(), Types.error()} + def request(conn, method, path, headers, body) + + def request(%__MODULE__{state: :closed} = conn, _method, _path, _headers, _body) do + {:error, conn, wrap_error(:closed)} + end + + def request( + %__MODULE__{state: {:goaway, _error_code, _debug_data}} = conn, + _method, + _path, + _headers, + _body + ) do + {:error, conn, wrap_error(:closed_for_writing)} + end + + def request(%__MODULE__{} = conn, method, path, headers, body) + when is_binary(method) and is_binary(path) and is_list(headers) do + headers = + headers + |> Headers.lower_raws() + |> add_pseudo_headers(conn, method, path) + |> add_default_headers(body) + |> sort_pseudo_headers_to_front() + + {conn, stream_id, ref} = open_stream(conn) + {conn, payload} = encode_request_payload(conn, stream_id, headers, body) + conn = send!(conn, payload) + {:ok, conn, ref} + catch + :throw, {:hex_mint, _conn, reason} -> + # The stream is invalid and "_conn" may be tracking it, so we return the original connection instead. + {:error, conn, reason} + end + + @doc """ + See `Hex.Mint.HTTP.stream_request_body/3`. + """ + @impl true + @spec stream_request_body( + t(), + Types.request_ref(), + iodata() | :eof | {:eof, trailer_headers :: Types.headers()} + ) :: {:ok, t()} | {:error, t(), Types.error()} + def stream_request_body(conn, request_ref, chunk) + + def stream_request_body(%__MODULE__{state: :closed} = conn, _request_ref, _chunk) do + {:error, conn, wrap_error(:closed)} + end + + def stream_request_body( + %__MODULE__{state: {:goaway, _error_code, _debug_data}} = conn, + _request_ref, + _chunk + ) do + {:error, conn, wrap_error(:closed_for_writing)} + end + + def stream_request_body(%__MODULE__{} = conn, request_ref, chunk) + when is_reference(request_ref) do + case Map.fetch(conn.ref_to_stream_id, request_ref) do + {:ok, stream_id} -> + {conn, payload} = encode_stream_body_request_payload(conn, stream_id, chunk) + conn = send!(conn, payload) + {:ok, conn} + + :error -> + {:error, conn, wrap_error(:unknown_request_to_stream)} + end + catch + :throw, {:hex_mint, _conn, reason} -> + # The stream is invalid and "_conn" may be tracking it, so we return the original connection instead. + {:error, conn, reason} + end + + @doc """ + Pings the server. + + This function is specific to HTTP/2 connections. It sends a **ping** request to + the server `conn` is connected to. A `{:ok, conn, request_ref}` tuple is returned, + where `conn` is the updated connection and `request_ref` is a unique reference that + identifies this ping request. The response to a ping request is returned by `stream/2` + as a `{:pong, request_ref}` tuple. If there's an error, this function returns + `{:error, conn, reason}` where `conn` is the updated connection and `reason` is the + error reason. + + `payload` must be an 8-byte binary with arbitrary content. When the server responds to + a ping request, it will use that same payload. By default, the payload is an 8-byte + binary with all bits set to `0`. + + Pinging can be used to measure the latency with the server and to ensure the connection + is alive and well. + + ## Examples + + {:ok, conn, ref} = Hex.Mint.HTTP2.ping(conn) + + """ + @spec ping(t(), <<_::8>>) :: {:ok, t(), Types.request_ref()} | {:error, t(), Types.error()} + def ping(%__MODULE__{} = conn, payload \\ :binary.copy(<<0>>, 8)) + when byte_size(payload) == 8 do + {conn, ref} = send_ping(conn, payload) + {:ok, conn, ref} + catch + :throw, {:hex_mint, conn, error} -> {:error, conn, error} + end + + @doc """ + Communicates the given **client settings** to the server. + + This function is HTTP/2-specific. + + This function takes a connection and a keyword list of HTTP/2 settings and sends + the values of those settings to the server. The settings won't be effective until + the server acknowledges them, which will be handled transparently by `stream/2`. + + This function returns `{:ok, conn}` when sending the settings to the server is + successful, with `conn` being the updated connection. If there's an error, this + function returns `{:error, conn, reason}` with `conn` being the updated connection + and `reason` being the reason of the error. + + ## Supported Settings + + See `t:setting/0` for the supported settings. You can see the meaning + of these settings [in the corresponding section in the HTTP/2 + RFC](https://httpwg.org/specs/rfc7540.html#SettingValues). + + See the "HTTP/2 settings" section in the module documentation for more information. + + ## Examples + + {:ok, conn} = Hex.Mint.HTTP2.put_settings(conn, max_frame_size: 100) + + """ + @spec put_settings(t(), settings()) :: {:ok, t()} | {:error, t(), Types.error()} + def put_settings(%__MODULE__{} = conn, settings) when is_list(settings) do + conn = send_settings(conn, settings) + {:ok, conn} + catch + :throw, {:hex_mint, conn, error} -> {:error, conn, error} + end + + @doc """ + Gets the value of the given HTTP/2 server settings. + + This function returns the value of the given HTTP/2 setting that the server + advertised to the client. This function is HTTP/2 specific. + For more information on HTTP/2 settings, see [the related section in + the RFC](https://httpwg.org/specs/rfc7540.html#SettingValues). + + See the "HTTP/2 settings" section in the module documentation for more information. + + ## Supported settings + + The possible settings that can be retrieved are described in `t:setting/0`. + Any other atom passed as `name` will raise an error. + + ## Examples + + Hex.Mint.HTTP2.get_server_setting(conn, :max_concurrent_streams) + #=> 500 + + """ + @spec get_server_setting(t(), atom()) :: term() + def get_server_setting(%__MODULE__{} = conn, name) when is_atom(name) do + get_setting(conn.server_settings, name) + end + + @doc """ + Gets the value of the given HTTP/2 client setting. + + This function returns the value of the given HTTP/2 setting that the client + advertised to the server. Client settings can be advertised through `put_settings/2` + or when starting up a connection. + + Client settings have to be acknowledged by the server before coming into effect. + + This function is HTTP/2 specific. For more information on HTTP/2 settings, see + [the related section in the RFC](https://httpwg.org/specs/rfc7540.html#SettingValues). + + See the "HTTP/2 settings" section in the module documentation for more information. + + ## Supported settings + + The possible settings that can be retrieved are described in `t:setting/0`. + Any other atom passed as `name` will raise an error. + + ## Examples + + Hex.Mint.HTTP2.get_client_setting(conn, :max_concurrent_streams) + #=> 500 + + """ + @spec get_client_setting(t(), atom()) :: term() + def get_client_setting(%__MODULE__{} = conn, name) when is_atom(name) do + get_setting(conn.client_settings, name) + end + + defp get_setting(settings, name) do + case Map.fetch(settings, name) do + {:ok, value} -> value + :error -> raise ArgumentError, "unknown HTTP/2 setting: #{inspect(name)}" + end + end + + @doc """ + Cancels an in-flight request. + + This function is HTTP/2 specific. It cancels an in-flight request. The server could have + already sent responses for the request you want to cancel: those responses will be parsed + by the connection but not returned to the user. No more responses + to a request will be returned after you call `cancel_request/2` on that request. + + If there's no error in canceling the request, `{:ok, conn}` is returned where `conn` is + the updated connection. If there's an error, `{:error, conn, reason}` is returned where + `conn` is the updated connection and `reason` is the error reason. + + ## Examples + + {:ok, conn, ref} = Hex.Mint.HTTP2.request(conn, "GET", "/", _headers = []) + {:ok, conn} = Hex.Mint.HTTP2.cancel_request(conn, ref) + + """ + @spec cancel_request(t(), Types.request_ref()) :: {:ok, t()} | {:error, t(), Types.error()} + def cancel_request(%__MODULE__{} = conn, request_ref) when is_reference(request_ref) do + case Map.fetch(conn.ref_to_stream_id, request_ref) do + {:ok, stream_id} -> + conn = close_stream!(conn, stream_id, _error_code = :cancel) + {:ok, conn} + + :error -> + {:ok, conn} + end + catch + :throw, {:hex_mint, conn, error} -> {:error, conn, error} + end + + @doc """ + Returns the client **send** window size for the connection or a request. + + > #### Send vs receive windows {: .warning} + > + > This function returns the *send* window — how much body data this client + > is still permitted to send to the server before being throttled. It is + > decremented by `request/5` and `stream_request_body/3` and refilled by + > the server, which `stream/2` handles transparently. + > + > It does **not** return the client *receive* window (how much the server + > is permitted to send us). To influence that, use `set_window_size/3`. + + This function is HTTP/2 specific. It returns the send window of either the + connection if `connection_or_request` is `:connection` or of a single request + if `connection_or_request` is `{:request, request_ref}`. + + Use this function to check the window size of the connection before sending a + full request. Also use this function to check the window size of both the + connection and of a request if you want to stream body chunks on that request. + + For more information on flow control and window sizes in HTTP/2, see the section + below. + + ## HTTP/2 Flow Control + + In HTTP/2, flow control is implemented through a window size. When the client + sends data to the server, the window size is decreased and the server needs + to "refill" it on the client side, which `stream/2` handles transparently. + Symmetrically, the server's outbound flow toward the client is bounded by a + receive window the client advertises and refills — see `set_window_size/3`. + + A window size is kept for the entire connection and all requests affect this + window size. A window size is also kept per request. + + The only thing that affects the send window size is the body of a request, + regardless of whether it's a full request sent with `request/5` or body chunks + sent through `stream_request_body/3`. That means that if we make a request with + a body that is five bytes long, like `"hello"`, the send window size of the + connection and the send window size of that particular request will decrease + by five bytes. + + If we use all the send window size before the server refills it, functions like + `request/5` will return an error. + + ## Examples + + On the connection: + + HTTP2.get_window_size(conn, :connection) + #=> 65_536 + + On a single streamed request: + + {:ok, conn, request_ref} = HTTP2.request(conn, "GET", "/", [], :stream) + HTTP2.get_window_size(conn, {:request, request_ref}) + #=> 65_536 + + {:ok, conn} = HTTP2.stream_request_body(conn, request_ref, "hello") + HTTP2.get_window_size(conn, {:request, request_ref}) + #=> 65_531 + + """ + @spec get_window_size(t(), :connection | {:request, Types.request_ref()}) :: non_neg_integer() + def get_window_size(conn, connection_or_request) + + def get_window_size(%__MODULE__{} = conn, :connection) do + conn.send_window_size + end + + def get_window_size(%__MODULE__{} = conn, {:request, request_ref}) do + case Map.fetch(conn.ref_to_stream_id, request_ref) do + {:ok, stream_id} -> + conn.streams[stream_id].send_window_size + + :error -> + raise ArgumentError, + "request with request reference #{inspect(request_ref)} was not found" + end + end + + @doc """ + Advertises a larger client **receive** window to the server. + + > #### Receive vs send windows {: .warning} + > + > This function sets the *receive* window — the peak amount of body data + > the server is permitted to send us before being throttled. It does + > **not** set the *send* window (how much body data we're permitted to + > send to the server) — the server controls that. See `get_window_size/2` + > for the send window. + + Without calling this, `stream/2` refills the receive window in small + increments as response body data is consumed. Each refill costs a + round-trip before the server can send more, so bulk throughput is capped + at roughly `window / RTT`; on higher-latency links the default 64 KB + window makes that cap well below the link bandwidth. Raising the window + removes those pauses and is the main HTTP/2 tuning knob for bulk or + highly parallel downloads. + + Mint exposes the per-stream initial window as the `:initial_window_size` + client setting passed to `connect/4`, but there is no connection-level + equivalent — use this function for the connection window, and for any + per-stream adjustment after a request has started. + + `connection_or_request` is `:connection` for the whole connection or + `{:request, request_ref}` for a single request. `new_size` must be in + `1..2_147_483_647`. Windows can only grow: `new_size` smaller than the + current receive window returns + `{:error, conn, %Hex.Mint.HTTPError{reason: :window_size_too_small}}`, and + `new_size` equal to the current window is a no-op. + + For more information on flow control and window sizes in HTTP/2, see the + section below. + + ## HTTP/2 Flow Control + + See `get_window_size/2` for a description of the client *send* window. + The client *receive* window is the symmetric bound on the server's + outbound flow: it starts at 64 KB for the connection and for each new + request, is decremented by response body bytes, and is refilled by + `stream/2` as the body is consumed. A window size is kept for the entire + connection and all responses affect this window size; a window size is + also kept per request. + + This function raises the *advertised* receive window — the peak the + server is allowed to fill before pausing. It does not pre-allocate any + buffers; it only permits the server to send further ahead of the + client's reads. + + ## Examples + + Bump the connection-level receive window right after connect so the server + can stream multi-MB bodies without flow-control pauses: + + {:ok, conn} = Hex.Mint.HTTP2.connect(:https, host, 443) + {:ok, conn} = Hex.Mint.HTTP2.set_window_size(conn, :connection, 8_000_000) + + Give one specific request a bigger window than the per-stream default: + + {:ok, conn, ref} = Hex.Mint.HTTP2.request(conn, "GET", "/huge", [], nil) + {:ok, conn} = Hex.Mint.HTTP2.set_window_size(conn, {:request, ref}, 16_000_000) + + """ + @spec set_window_size(t(), :connection | {:request, Types.request_ref()}, pos_integer()) :: + {:ok, t()} | {:error, t(), Types.error()} + def set_window_size(conn, connection_or_request, new_size) + + def set_window_size(%__MODULE__{} = _conn, _target, new_size) + when not (is_integer(new_size) and new_size >= 1 and new_size <= @max_window_size) do + raise ArgumentError, + "new window size must be an integer in 1..#{@max_window_size}, got: #{inspect(new_size)}" + end + + def set_window_size(%__MODULE__{} = conn, :connection, new_size) do + do_set_window_size(conn, 0, conn.receive_window_size, new_size, fn conn, size -> + conn = put_in(conn.receive_window_size, size) + put_in(conn.receive_window, size) + end) + catch + :throw, {:hex_mint, conn, error} -> {:error, conn, error} + end + + def set_window_size(%__MODULE__{} = conn, {:request, request_ref}, new_size) do + case Map.fetch(conn.ref_to_stream_id, request_ref) do + {:ok, stream_id} -> + current = conn.streams[stream_id].receive_window_size + + do_set_window_size(conn, stream_id, current, new_size, fn conn, size -> + conn = put_in(conn.streams[stream_id].receive_window_size, size) + put_in(conn.streams[stream_id].receive_window, size) + end) + + :error -> + {:error, conn, wrap_error({:unknown_request_to_stream, request_ref})} + end + catch + :throw, {:hex_mint, conn, error} -> {:error, conn, error} + end + + defp do_set_window_size(conn, _stream_id, current, new_size, _update) when new_size == current do + {:ok, conn} + end + + defp do_set_window_size(conn, _stream_id, current, new_size, _update) when new_size < current do + {:error, conn, wrap_error({:window_size_too_small, current, new_size})} + end + + defp do_set_window_size(conn, stream_id, current, new_size, update) do + increment = new_size - current + frame = window_update(stream_id: stream_id, window_size_increment: increment) + conn = send!(conn, Frame.encode(frame)) + {:ok, update.(conn, new_size)} + end + + @doc """ + See `Hex.Mint.HTTP.stream/2`. + """ + @impl true + @spec stream(t(), term()) :: + {:ok, t(), [Types.response()]} + | {:error, t(), Types.error(), [Types.response()]} + | :unknown + def stream(conn, message) + + def stream(%__MODULE__{socket: socket} = conn, {tag, socket, reason}) + when tag in [:tcp_error, :ssl_error] do + error = conn.transport.wrap_error(reason) + {:error, %{conn | state: :closed}, error, _responses = []} + end + + def stream(%__MODULE__{socket: socket} = conn, {tag, socket}) + when tag in [:tcp_closed, :ssl_closed] do + handle_closed(conn) + end + + def stream(%__MODULE__{transport: transport, socket: socket} = conn, {tag, socket, data}) + when tag in [:tcp, :ssl] do + case maybe_concat_and_handle_new_data(conn, data) do + {:ok, %{mode: mode, state: state} = conn, responses} + when mode == :active and state != :closed -> + case transport.setopts(socket, active: :once) do + :ok -> {:ok, conn, responses} + {:error, reason} -> {:error, put_in(conn.state, :closed), reason, responses} + end + + other -> + other + end + catch + :throw, {:hex_mint, conn, error, responses} -> {:error, conn, error, responses} + end + + def stream(%__MODULE__{}, _message) do + :unknown + end + + @doc """ + See `Hex.Mint.HTTP.open_request_count/1`. + + In HTTP/2, the number of open requests is the number of requests **opened by the client** + that have not yet received a `:done` response. It's important to note that only + requests opened by the client (with `request/5`) count towards the number of open + requests, as requests opened from the server with server pushes (see the "Server push" + section in the module documentation) are not considered open requests. We do this because + clients might need to know how many open requests there are because the server limits + the number of concurrent requests the client can open. To know how many requests the client + can open, see `get_server_setting/2` with the `:max_concurrent_streams` setting. + """ + @impl true + @spec open_request_count(t()) :: non_neg_integer() + def open_request_count(%__MODULE__{} = conn) do + conn.open_client_stream_count + end + + @doc """ + See `Hex.Mint.HTTP.recv/3`. + """ + @impl true + @spec recv(t(), non_neg_integer(), timeout()) :: + {:ok, t(), [Types.response()]} + | {:error, t(), Types.error(), [Types.response()]} + def recv(conn, byte_count, timeout) + + def recv(%__MODULE__{mode: :passive} = conn, byte_count, timeout) do + case conn.transport.recv(conn.socket, byte_count, timeout) do + {:ok, data} -> + maybe_concat_and_handle_new_data(conn, data) + + {:error, %TransportError{reason: :closed}} -> + handle_closed(conn) + + {:error, error} -> + {:error, %{conn | state: :closed}, error, _responses = []} + end + catch + :throw, {:hex_mint, conn, error, responses} -> {:error, conn, error, responses} + end + + def recv(_conn, _byte_count, _timeout) do + raise ArgumentError, + "can't use recv/3 to synchronously receive data when the mode is :active. " <> + "Use Hex.Mint.HTTP.set_mode/2 to set the connection to passive mode" + end + + @doc """ + See `Hex.Mint.HTTP.set_mode/2`. + """ + @impl true + @spec set_mode(t(), :active | :passive) :: {:ok, t()} | {:error, Types.error()} + def set_mode(%__MODULE__{} = conn, mode) when mode in [:active, :passive] do + active = + case mode do + :active -> :once + :passive -> false + end + + with :ok <- conn.transport.setopts(conn.socket, active: active) do + {:ok, put_in(conn.mode, mode)} + end + end + + @doc """ + See `Hex.Mint.HTTP.controlling_process/2`. + """ + @impl true + @spec controlling_process(t(), pid()) :: {:ok, t()} | {:error, Types.error()} + def controlling_process(%__MODULE__{} = conn, new_pid) when is_pid(new_pid) do + with :ok <- conn.transport.controlling_process(conn.socket, new_pid) do + {:ok, conn} + end + end + + @doc """ + See `Hex.Mint.HTTP.put_private/3`. + """ + @impl true + @spec put_private(t(), atom(), term()) :: t() + def put_private(%__MODULE__{private: private} = conn, key, value) when is_atom(key) do + %{conn | private: Map.put(private, key, value)} + end + + @doc """ + See `Hex.Mint.HTTP.get_private/3`. + """ + @impl true + @spec get_private(t(), atom(), term()) :: term() + def get_private(%__MODULE__{private: private} = _conn, key, default \\ nil) when is_atom(key) do + Map.get(private, key, default) + end + + @doc """ + See `Hex.Mint.HTTP.delete_private/2`. + """ + @impl true + @spec delete_private(t(), atom()) :: t() + def delete_private(%__MODULE__{private: private} = conn, key) when is_atom(key) do + %{conn | private: Map.delete(private, key)} + end + + @doc """ + See `Hex.Mint.HTTP.put_log/2`. + """ + @doc since: "1.5.0" + @impl true + @spec put_log(t(), boolean()) :: t() + def put_log(%__MODULE__{} = conn, log?) when is_boolean(log?) do + %{conn | log: log?} + end + + # http://httpwg.org/specs/rfc7540.html#rfc.section.6.5 + # SETTINGS parameters are not negotiated. We keep client settings and server settings separate. + @doc false + @impl true + @spec initiate( + Types.scheme(), + Types.socket(), + String.t(), + :inet.port_number(), + keyword() + ) :: {:ok, t()} | {:error, Types.error()} + def initiate(scheme, socket, hostname, port, opts) do + transport = Util.scheme_to_transport(scheme) + scheme_string = Atom.to_string(scheme) + mode = Keyword.get(opts, :mode, :active) + log? = Keyword.get(opts, :log, false) + connection_window_size = Keyword.get(opts, :connection_window_size, @default_connection_window_size) + validate_window_size!(:connection_window_size, connection_window_size) + receive_window_update_threshold = Keyword.get(opts, :receive_window_update_threshold, @default_receive_window_update_threshold) + validate_receive_window_update_threshold!(receive_window_update_threshold) + client_settings_params = Keyword.get(opts, :client_settings, []) + + client_settings_params = + Keyword.put_new(client_settings_params, :initial_window_size, @default_stream_window_size) + + validate_client_settings!(client_settings_params) + # If the port is the default for the scheme, don't add it to the :authority pseudo-header + authority = + if URI.default_port(scheme_string) == port do + hostname + else + "#{hostname}:#{port}" + end + + unless mode in [:active, :passive] do + raise ArgumentError, + "the :mode option must be either :active or :passive, got: #{inspect(mode)}" + end + + unless is_boolean(log?) do + raise ArgumentError, + "the :log option must be a boolean, got: #{inspect(log?)}" + end + + conn = %__MODULE__{ + hostname: hostname, + port: port, + authority: authority, + transport: Util.scheme_to_transport(scheme), + socket: socket, + mode: mode, + scheme: scheme_string, + state: :handshaking, + log: log?, + receive_window_size: connection_window_size, + receive_window: connection_window_size, + receive_window_update_threshold: receive_window_update_threshold + } + + preface = build_preface(client_settings_params, connection_window_size) + + with :ok <- Util.inet_opts(transport, socket), + :ok <- transport.send(socket, preface), + conn = update_in(conn.client_settings_queue, &:queue.in(client_settings_params, &1)), + conn = put_in(conn.socket, socket), + :ok <- if(mode == :active, do: transport.setopts(socket, active: :once), else: :ok) do + {:ok, conn} + else + error -> + transport.close(socket) + error + end + end + + defp build_preface(client_settings_params, connection_window_size) do + settings_frame = Frame.encode(settings(stream_id: 0, params: client_settings_params)) + + if connection_window_size > @default_window_size do + increment = connection_window_size - @default_window_size + update_frame = Frame.encode(window_update(stream_id: 0, window_size_increment: increment)) + [@connection_preface, settings_frame, update_frame] + else + [@connection_preface, settings_frame] + end + end + + defp validate_window_size!(name, value) do + unless is_integer(value) and value >= @default_window_size and value <= @max_window_size do + raise ArgumentError, + "the :#{name} option must be an integer in " <> + "#{@default_window_size}..#{@max_window_size}, got: #{inspect(value)}" + end + end + + defp validate_receive_window_update_threshold!(value) do + unless is_integer(value) and value >= 1 and value <= @max_window_size do + raise ArgumentError, + "the :receive_window_update_threshold option must be a positive integer no larger than " <> + "#{@max_window_size}, got: #{inspect(value)}" + end + end + + @doc """ + See `Hex.Mint.HTTP.get_socket/1`. + """ + @impl true + @spec get_socket(t()) :: Hex.Mint.Types.socket() + def get_socket(%__MODULE__{socket: socket} = _conn) do + socket + end + + @doc """ + See `Hex.Mint.HTTP.get_proxy_headers/1`. + """ + @doc since: "1.4.0" + @impl true + @spec get_proxy_headers(t()) :: Hex.Mint.Types.headers() + def get_proxy_headers(%__MODULE__{proxy_headers: proxy_headers} = _conn), do: proxy_headers + + # Made public since the %Hex.Mint.HTTP2{} struct is opaque. + @doc false + @impl true + def put_proxy_headers(%__MODULE__{} = conn, headers) when is_list(headers) do + %{conn | proxy_headers: headers} + end + + ## Helpers + + defp handle_closed(conn) do + conn = put_in(conn.state, :closed) + + if conn.open_client_stream_count > 0 or conn.open_server_stream_count > 0 do + error = conn.transport.wrap_error(:closed) + {:error, conn, error, _responses = []} + else + {:ok, conn, _responses = []} + end + end + + defp negotiate(address, port, :http, transport_opts) do + # We don't support protocol negotiation for TCP connections + # so currently we just assume the HTTP/2 protocol + transport = Util.scheme_to_transport(:http) + transport.connect(address, port, transport_opts) + end + + defp negotiate(address, port, :https, transport_opts) do + transport = Util.scheme_to_transport(:https) + + with {:ok, socket} <- transport.connect(address, port, transport_opts), + {:ok, protocol} <- transport.negotiated_protocol(socket) do + if protocol == "h2" do + {:ok, socket} + else + {:error, transport.wrap_error({:bad_alpn_protocol, protocol})} + end + end + end + + defp open_stream(conn) do + max_concurrent_streams = conn.server_settings.max_concurrent_streams + + if conn.open_client_stream_count >= max_concurrent_streams do + throw({:hex_mint, conn, wrap_error(:too_many_concurrent_requests)}) + end + + stream = %{ + id: conn.next_stream_id, + ref: make_ref(), + state: :idle, + # Client send window — decremented as we send body bytes, refilled + # by incoming WINDOW_UPDATE frames from the server. Bounded initially + # by the server's SETTINGS_INITIAL_WINDOW_SIZE. + send_window_size: conn.server_settings.initial_window_size, + # Client receive window — the peak we've advertised to the server + # for this stream. Starts at whatever we told the server via our + # SETTINGS_INITIAL_WINDOW_SIZE; can be bumped per-stream with + # `set_window_size/3`. + receive_window_size: conn.client_settings.initial_window_size, + # Current remaining receive window for this stream, tracked + # independently from the peak so that refills can be batched. + receive_window: conn.client_settings.initial_window_size, + received_first_headers?: false + } + + conn = put_in(conn.streams[stream.id], stream) + conn = put_in(conn.ref_to_stream_id[stream.ref], stream.id) + conn = update_in(conn.next_stream_id, &(&1 + 2)) + {conn, stream.id, stream.ref} + end + + defp encode_stream_body_request_payload(conn, stream_id, :eof) do + encode_data(conn, stream_id, "", [:end_stream]) + end + + defp encode_stream_body_request_payload(conn, stream_id, {:eof, trailers}) do + trailers = Headers.from_raw(trailers) + + if unallowed_trailer_header = Headers.find_unallowed_trailer(trailers) do + error = wrap_error({:unallowed_trailing_header, unallowed_trailer_header}) + throw({:hex_mint, conn, error}) + end + + trailer_headers = Headers.to_raw(trailers, _case_sensitive = false) + encode_headers(conn, stream_id, trailer_headers, [:end_headers, :end_stream]) + end + + defp encode_stream_body_request_payload(conn, stream_id, iodata) do + encode_data(conn, stream_id, iodata, []) + end + + defp encode_request_payload(conn, stream_id, headers, :stream) do + encode_headers(conn, stream_id, headers, [:end_headers]) + end + + defp encode_request_payload(conn, stream_id, headers, nil) do + encode_headers(conn, stream_id, headers, [:end_stream, :end_headers]) + end + + defp encode_request_payload(conn, stream_id, headers, iodata) do + {conn, headers_payload} = encode_headers(conn, stream_id, headers, [:end_headers]) + {conn, data_payload} = encode_data(conn, stream_id, iodata, [:end_stream]) + {conn, [headers_payload, data_payload]} + end + + defp encode_headers(conn, stream_id, headers, enabled_flags) do + assert_headers_smaller_than_max_header_list_size(conn, headers) + + headers = Enum.map(headers, fn {name, value} -> {:store_name, name, value} end) + {hbf, conn} = get_and_update_in(conn.encode_table, &Hex.Mint.HPAX.encode(headers, &1)) + + payload = headers_to_encoded_frames(conn, stream_id, hbf, enabled_flags) + + stream_state = if :end_stream in enabled_flags, do: :half_closed_local, else: :open + + conn = put_in(conn.streams[stream_id].state, stream_state) + conn = update_in(conn.open_client_stream_count, &(&1 + 1)) + + {conn, payload} + end + + defp assert_headers_smaller_than_max_header_list_size( + %{server_settings: %{max_header_list_size: :infinity}}, + _headers + ) do + :ok + end + + defp assert_headers_smaller_than_max_header_list_size(conn, headers) do + # The value is based on the uncompressed size of header fields, including the length + # of the name and value in octets plus an overhead of 32 octets for each header field. + total_size = + Enum.reduce(headers, 0, fn {name, value}, acc -> + acc + byte_size(name) + byte_size(value) + 32 + end) + + max_header_list_size = conn.server_settings.max_header_list_size + + if total_size <= max_header_list_size do + :ok + else + error = wrap_error({:max_header_list_size_exceeded, total_size, max_header_list_size}) + throw({:hex_mint, conn, error}) + end + end + + defp headers_to_encoded_frames(conn, stream_id, hbf, enabled_flags) do + if IO.iodata_length(hbf) > conn.server_settings.max_frame_size do + hbf + |> IO.iodata_to_binary() + |> split_payload_in_chunks(conn.server_settings.max_frame_size) + |> split_hbf_to_encoded_frames(stream_id, enabled_flags) + else + Frame.encode( + headers(stream_id: stream_id, hbf: hbf, flags: set_flags(:headers, enabled_flags)) + ) + end + end + + defp split_hbf_to_encoded_frames({[first_chunk | chunks], last_chunk}, stream_id, enabled_flags) do + flags = set_flags(:headers, enabled_flags -- [:end_headers]) + first_frame = Frame.encode(headers(stream_id: stream_id, hbf: first_chunk, flags: flags)) + + middle_frames = + Enum.map(chunks, fn chunk -> + Frame.encode(continuation(stream_id: stream_id, hbf: chunk)) + end) + + flags = + if :end_headers in enabled_flags do + set_flags(:continuation, [:end_headers]) + else + set_flags(:continuation, []) + end + + last_frame = Frame.encode(continuation(stream_id: stream_id, hbf: last_chunk, flags: flags)) + + [first_frame, middle_frames, last_frame] + end + + defp encode_data(conn, stream_id, data, enabled_flags) do + stream = fetch_stream!(conn, stream_id) + + if stream.state != :open do + error = wrap_error(:request_is_not_streaming) + throw({:hex_mint, conn, error}) + end + + data_size = IO.iodata_length(data) + + cond do + data_size > stream.send_window_size -> + throw({:hex_mint, conn, wrap_error({:exceeds_window_size, :request, stream.send_window_size})}) + + data_size > conn.send_window_size -> + throw({:hex_mint, conn, wrap_error({:exceeds_window_size, :connection, conn.send_window_size})}) + + # If the data size is greater than the max frame size, we chunk automatically based + # on the max frame size. + data_size > conn.server_settings.max_frame_size -> + {chunks, last_chunk} = + data + |> IO.iodata_to_binary() + |> split_payload_in_chunks(conn.server_settings.max_frame_size) + + {encoded_chunks, conn} = + Enum.map_reduce(chunks, conn, fn chunk, acc -> + {acc, encoded} = encode_data_chunk(acc, stream_id, chunk, []) + {encoded, acc} + end) + + {conn, encoded_last_chunk} = encode_data_chunk(conn, stream_id, last_chunk, enabled_flags) + {conn, [encoded_chunks, encoded_last_chunk]} + + true -> + encode_data_chunk(conn, stream_id, data, enabled_flags) + end + end + + defp encode_data_chunk(%__MODULE__{} = conn, stream_id, chunk, enabled_flags) + when is_integer(stream_id) and is_list(enabled_flags) do + chunk_size = IO.iodata_length(chunk) + frame = data(stream_id: stream_id, flags: set_flags(:data, enabled_flags), data: chunk) + conn = update_in(conn.streams[stream_id].send_window_size, &(&1 - chunk_size)) + conn = update_in(conn.send_window_size, &(&1 - chunk_size)) + + conn = + if :end_stream in enabled_flags do + put_in(conn.streams[stream_id].state, :half_closed_local) + else + conn + end + + {conn, Frame.encode(frame)} + end + + defp split_payload_in_chunks(binary, chunk_size), + do: split_payload_in_chunks(binary, chunk_size, []) + + defp split_payload_in_chunks(chunk, chunk_size, acc) when byte_size(chunk) <= chunk_size do + {Enum.reverse(acc), chunk} + end + + defp split_payload_in_chunks(binary, chunk_size, acc) do + <> = binary + split_payload_in_chunks(rest, chunk_size, [chunk | acc]) + end + + defp send_ping(conn, payload) do + frame = Frame.ping(stream_id: 0, opaque_data: payload) + conn = send!(conn, Frame.encode(frame)) + ref = make_ref() + conn = update_in(conn.ping_queue, &:queue.in({ref, payload}, &1)) + {conn, ref} + end + + defp send_settings(conn, settings) do + validate_client_settings!(settings) + frame = settings(stream_id: 0, params: settings) + conn = send!(conn, Frame.encode(frame)) + conn = update_in(conn.client_settings_queue, &:queue.in(settings, &1)) + conn + end + + defp validate_client_settings!(settings) do + unless Keyword.keyword?(settings) do + raise ArgumentError, "settings must be a keyword list" + end + + Enum.each(settings, fn + {:header_table_size, value} -> + unless is_integer(value) do + raise ArgumentError, ":header_table_size must be an integer, got: #{inspect(value)}" + end + + {:enable_push, value} -> + unless is_boolean(value) do + raise ArgumentError, ":enable_push must be a boolean, got: #{inspect(value)}" + end + + {:max_concurrent_streams, value} -> + unless is_integer(value) do + raise ArgumentError, + ":max_concurrent_streams must be an integer, got: #{inspect(value)}" + end + + {:initial_window_size, value} -> + unless is_integer(value) and value <= @max_window_size do + raise ArgumentError, + ":initial_window_size must be an integer < #{@max_window_size}, " <> + "got: #{inspect(value)}" + end + + {:max_frame_size, value} -> + unless is_integer(value) and value in @valid_max_frame_size_range do + raise ArgumentError, + ":max_frame_size must be an integer in #{inspect(@valid_max_frame_size_range)}, " <> + "got: #{inspect(value)}" + end + + {:max_header_list_size, value} -> + unless is_integer(value) do + raise ArgumentError, ":max_header_list_size must be an integer, got: #{inspect(value)}" + end + + {:enable_connect_protocol, _value} -> + raise ArgumentError, ":enable_connect_protocol is only valid for server settings" + + {name, _value} -> + raise ArgumentError, "unknown setting parameter #{inspect(name)}" + end) + end + + defp add_default_headers(headers, body) do + headers + |> Util.put_new_header("user-agent", @user_agent) + |> add_default_content_length_header(body) + end + + defp add_default_content_length_header(headers, body) when body in [nil, :stream] do + headers + end + + defp add_default_content_length_header(headers, body) do + Util.put_new_header_lazy(headers, "content-length", fn -> + body |> IO.iodata_length() |> Integer.to_string() + end) + end + + defp add_pseudo_headers(headers, conn, method, path) do + if same_method?(method, "CONNECT") do + [ + {":method", method}, + {":authority", conn.authority} + | headers + ] + else + [ + {":method", method}, + {":path", path}, + {":scheme", conn.scheme}, + {":authority", conn.authority} + | headers + ] + end + end + + # same_method?/2 is pretty optimized, so bench before changing. + + # Same binary, which is a common case. + defp same_method?(bin, bin), do: true + + # Get out early if the size is different, these can't be the same. + defp same_method?(bin1, bin2) when byte_size(bin1) != byte_size(bin2), do: false + + defp same_method?(<>, <>), do: same_method?(rest1, rest2) + + defp same_method?(<>, <>) when lower - 32 == char, + do: same_method?(rest1, rest2) + + defp same_method?(_method1, _method2), do: false + + defp sort_pseudo_headers_to_front(headers) do + Enum.sort_by(headers, fn {key, _value} -> + not String.starts_with?(key, ":") + end) + end + + ## Frame handling + + defp maybe_concat_and_handle_new_data(conn, data) do + data = Util.maybe_concat(conn.buffer, data) + {conn, responses} = handle_new_data(conn, data, []) + {:ok, conn, Enum.reverse(responses)} + end + + defp handle_new_data(%__MODULE__{} = conn, data, responses) do + case Frame.decode_next(data, conn.client_settings.max_frame_size) do + {:ok, frame, rest} -> + log(conn, :debug, "Received frame: #{Frame.inspect(frame)}") + conn = validate_frame(conn, frame) + {conn, responses} = handle_frame(conn, frame, responses) + handle_new_data(conn, rest, responses) + + :more -> + conn = put_in(conn.buffer, data) + handle_consumed_all_frames(conn, responses) + + {:error, :payload_too_big} -> + debug_data = "frame payload exceeds connection's max frame size" + send_connection_error!(conn, :frame_size_error, debug_data) + + {:error, {:frame_size_error, frame}} -> + debug_data = "error with size of frame: #{inspect(frame)}" + send_connection_error!(conn, :frame_size_error, debug_data) + + {:error, {:protocol_error, info}} -> + debug_data = "error when decoding frame: #{inspect(info)}" + send_connection_error!(conn, :protocol_error, debug_data) + end + catch + :throw, {:hex_mint, conn, error} -> throw({:hex_mint, conn, error, responses}) + :throw, {:hex_mint, _conn, _error, _responses} = thrown -> throw(thrown) + end + + defp handle_consumed_all_frames(%{state: state} = conn, responses) do + case state do + {:goaway, :no_error, _debug_data} -> + {conn, responses} + + {:goaway, error_code, debug_data} -> + error = wrap_error({:server_closed_connection, error_code, debug_data}) + throw({:hex_mint, conn, error, responses}) + + _ -> + {conn, responses} + end + end + + defp validate_frame(conn, unknown()) do + # Unknown frames MUST be ignored: + # https://datatracker.ietf.org/doc/html/rfc7540#section-4.1 + conn + end + + defp validate_frame(conn, frame) do + type = elem(frame, 0) + stream_id = elem(frame, 1) + + # The SETTINGS frame MUST be the first frame that the server sends. + # https://www.rfc-editor.org/rfc/rfc7540#section-3.5 + # > The server connection preface consists of a potentially empty SETTINGS frame + # > that MUST be the first frame the server sends in the HTTP/2 connection. + conn = + cond do + conn.state == :handshaking and type == :goaway -> + goaway(error_code: error_code, debug_data: debug_data) = frame + error = wrap_error({:server_closed_connection, error_code, debug_data}) + throw({:hex_mint, %{conn | state: :closed}, error, []}) + + conn.state == :handshaking and type != :settings -> + debug_data = "received invalid frame #{type} during handshake" + send_connection_error!(conn, :protocol_error, debug_data) + + conn.state == :handshaking -> + %{conn | state: :open} + + true -> + conn + end + + assert_frame_on_right_level(conn, elem(frame, 0), stream_id) + assert_stream_id_is_allowed(conn, stream_id) + assert_frame_doesnt_interrupt_header_streaming(conn, frame) + conn + end + + # http://httpwg.org/specs/rfc7540.html#HttpSequence + defp assert_frame_doesnt_interrupt_header_streaming(conn, frame) do + case {conn.headers_being_processed, frame} do + {nil, continuation()} -> + debug_data = "CONTINUATION received outside of headers streaming" + send_connection_error!(conn, :protocol_error, debug_data) + + {nil, _frame} -> + :ok + + {{stream_id, _, _}, continuation(stream_id: stream_id)} -> + :ok + + _other -> + debug_data = + "headers are streaming but got a #{inspect(elem(frame, 0))} frame instead " <> + "of a CONTINUATION frame" + + send_connection_error!(conn, :protocol_error, debug_data) + end + end + + stream_level_frames = [:data, :headers, :priority, :rst_stream, :push_promise, :continuation] + connection_level_frames = [:settings, :ping, :goaway] + + defp assert_frame_on_right_level(conn, frame, _stream_id = 0) + when frame in unquote(stream_level_frames) do + debug_data = "frame #{inspect(frame)} not allowed at the connection level (stream_id = 0)" + send_connection_error!(conn, :protocol_error, debug_data) + end + + defp assert_frame_on_right_level(conn, frame, stream_id) + when frame in unquote(connection_level_frames) and stream_id != 0 do + debug_data = "frame #{inspect(frame)} only allowed at the connection level" + send_connection_error!(conn, :protocol_error, debug_data) + end + + defp assert_frame_on_right_level(_conn, _frame, _stream_id) do + :ok + end + + defp assert_stream_id_is_allowed(conn, stream_id) do + if Integer.is_odd(stream_id) and stream_id >= conn.next_stream_id do + debug_data = "frame with stream ID #{inspect(stream_id)} has not been opened yet" + send_connection_error!(conn, :protocol_error, debug_data) + else + :ok + end + end + + for frame_name <- stream_level_frames ++ connection_level_frames ++ [:window_update, :unknown] do + function_name = :"handle_#{frame_name}" + + defp handle_frame(conn, Frame.unquote(frame_name)() = frame, responses) do + unquote(function_name)(conn, frame, responses) + end + end + + defp handle_unknown(conn, _frame, responses) do + # Implementations MUST ignore and discard any frame that has a type that is unknown. + # see: https://datatracker.ietf.org/doc/html/rfc7540#section-4.1 + + {conn, responses} + end + + # DATA + + defp handle_data(conn, frame, responses) do + data(stream_id: stream_id, flags: flags, data: data, padding: padding) = frame + + # Regardless of whether we have the stream or not, we need to abide by flow + # control rules so we still refill the client window for the stream_id we got. + window_size_increment = byte_size(data) + byte_size(padding || "") + + conn = + if window_size_increment > 0 do + refill_client_windows(conn, stream_id, window_size_increment) + else + conn + end + + case Map.fetch(conn.streams, stream_id) do + {:ok, stream} -> + assert_stream_in_state(conn, stream, [:open, :half_closed_local]) + responses = [{:data, stream.ref, data} | responses] + + if flag_set?(flags, :data, :end_stream) do + conn = close_stream!(conn, stream.id, :remote_end_stream) + {conn, [{:done, stream.ref} | responses]} + else + {conn, responses} + end + + :error -> + log(conn, :debug, "Received DATA frame on closed stream ID #{stream_id}") + {conn, responses} + end + end + + # Accounts for `data_size` bytes arriving on the connection and on + # `stream_id`. Sends a WINDOW_UPDATE for either window only once its + # remaining receive credit drops to `conn.receive_window_update_threshold`; + # previously we sent one per DATA frame, so an adversarial server + # emitting many small frames could amplify its inbound bytes into a + # WINDOW_UPDATE flood of outbound frames. Batching caps that ratio at + # roughly one update per `(receive_window_size - threshold)` bytes + # consumed. + defp refill_client_windows(conn, stream_id, data_size) do + conn = update_in(conn.receive_window, &(&1 - data_size)) + + conn = + case Map.fetch(conn.streams, stream_id) do + {:ok, _stream} -> + update_in(conn.streams[stream_id].receive_window, &(&1 - data_size)) + + :error -> + conn + end + + frames = + [] + |> maybe_refill_stream(conn, stream_id) + |> maybe_refill_conn(conn) + + if frames != [] and open?(conn) do + conn = send!(conn, Enum.map(frames, &Frame.encode/1)) + apply_refills(conn, frames) + else + conn + end + end + + defp maybe_refill_conn(frames, conn) do + if conn.receive_window <= conn.receive_window_update_threshold do + increment = conn.receive_window_size - conn.receive_window + [window_update(stream_id: 0, window_size_increment: increment) | frames] + else + frames + end + end + + defp maybe_refill_stream(frames, conn, stream_id) do + case Map.fetch(conn.streams, stream_id) do + {:ok, stream} -> + if stream.receive_window <= conn.receive_window_update_threshold do + increment = stream.receive_window_size - stream.receive_window + + [ + window_update(stream_id: stream_id, window_size_increment: increment) | frames + ] + else + frames + end + + :error -> + frames + end + end + + defp apply_refills(conn, frames) do + Enum.reduce(frames, conn, fn + window_update(stream_id: 0), conn -> + put_in(conn.receive_window, conn.receive_window_size) + + window_update(stream_id: stream_id), conn -> + put_in( + conn.streams[stream_id].receive_window, + conn.streams[stream_id].receive_window_size + ) + end) + end + + # HEADERS + + defp handle_headers(conn, frame, responses) do + headers(stream_id: stream_id, flags: flags, hbf: hbf) = frame + + stream = Map.get(conn.streams, stream_id) + end_stream? = flag_set?(flags, :headers, :end_stream) + + if stream do + assert_stream_in_state(conn, stream, [:open, :half_closed_local, :reserved_remote]) + end + + if flag_set?(flags, :headers, :end_headers) do + decode_hbf_and_add_responses(conn, responses, hbf, stream, end_stream?) + else + callback = &decode_hbf_and_add_responses(&1, &2, &3, &4, end_stream?) + conn = put_in(conn.headers_being_processed, {stream_id, hbf, callback}) + {conn, responses} + end + end + + # Here, "stream" can be nil in case the stream was closed. In that case, we + # still need to process the hbf so that the HPACK table is updated, but then + # we don't add any responses. + defp decode_hbf_and_add_responses(conn, responses, hbf, stream, end_stream?) do + {conn, headers} = decode_hbf(conn, hbf) + + if stream do + handle_decoded_headers_for_stream(conn, responses, stream, headers, end_stream?) + else + log(conn, :debug, "Received HEADERS frame on closed stream ID") + {conn, responses} + end + end + + defp handle_decoded_headers_for_stream(conn, responses, stream, headers, end_stream?) do + %{ref: ref, received_first_headers?: received_first_headers?} = stream + + case headers do + # Interim response (1xx), which is made of only one HEADERS plus zero or more CONTINUATIONs. + # There can be zero or more interim responses before a "proper" response. + # https://httpwg.org/specs/rfc9113.html#HttpFraming + [{":status", <> = status} | headers] -> + cond do + received_first_headers? -> + conn = close_stream!(conn, stream.id, :protocol_error) + + debug_data = + "informational response (1xx) must appear before final response, got a #{status} status" + + error = wrap_error({:protocol_error, debug_data}) + responses = [{:error, stream.ref, error} | responses] + {conn, responses} + + end_stream? -> + conn = close_stream!(conn, stream.id, :protocol_error) + debug_data = "informational response (1xx) must not have the END_STREAM flag set" + error = wrap_error({:protocol_error, debug_data}) + responses = [{:error, stream.ref, error} | responses] + {conn, responses} + + true -> + assert_stream_in_state(conn, stream, [:open, :half_closed_local]) + status = String.to_integer(status) + headers = join_cookie_headers(headers) + new_responses = [{:headers, ref, headers}, {:status, ref, status} | responses] + {conn, new_responses} + end + + [{":status", status} | headers] when not received_first_headers? -> + conn = put_in(conn.streams[stream.id].received_first_headers?, true) + status = String.to_integer(status) + headers = join_cookie_headers(headers) + new_responses = [{:headers, ref, headers}, {:status, ref, status} | responses] + + cond do + # :reserved_remote means that this was a promised stream. As soon as headers come, + # the stream goes in the :half_closed_local state (unless it's not allowed because + # of the client's max concurrent streams limit, or END_STREAM is set). + stream.state == :reserved_remote -> + cond do + conn.open_server_stream_count >= conn.client_settings.max_concurrent_streams -> + conn = close_stream!(conn, stream.id, :refused_stream) + {conn, responses} + + end_stream? -> + conn = close_stream!(conn, stream.id, :remote_end_stream) + {conn, [{:done, ref} | new_responses]} + + true -> + conn = update_in(conn.open_server_stream_count, &(&1 + 1)) + conn = put_in(conn.streams[stream.id].state, :half_closed_local) + {conn, new_responses} + end + + end_stream? -> + conn = close_stream!(conn, stream.id, :remote_end_stream) + {conn, [{:done, ref} | new_responses]} + + true -> + {conn, new_responses} + end + + # Trailer headers. We don't care about the :status header here. + headers when received_first_headers? -> + if end_stream? do + conn = close_stream!(conn, stream.id, :remote_end_stream) + headers = headers |> Headers.remove_unallowed_trailer() |> join_cookie_headers() + {conn, [{:done, ref}, {:headers, ref, headers} | responses]} + else + # Trailer headers must set the END_STREAM flag because they're + # the last thing allowed on the stream (other than RST_STREAM and + # the usual frames). + conn = close_stream!(conn, stream.id, :protocol_error) + debug_data = "trailer headers didn't set the END_STREAM flag" + error = wrap_error({:protocol_error, debug_data}) + responses = [{:error, stream.ref, error} | responses] + {conn, responses} + end + + # Non-trailer headers need to have a :status header, otherwise + # it's a protocol error. + _headers -> + conn = close_stream!(conn, stream.id, :protocol_error) + error = wrap_error(:missing_status_header) + responses = [{:error, stream.ref, error} | responses] + {conn, responses} + end + end + + defp decode_hbf(conn, hbf) do + case Hex.Mint.HPAX.decode(hbf, conn.decode_table) do + {:ok, headers, decode_table} -> + conn = put_in(conn.decode_table, decode_table) + {conn, headers} + + {:error, reason} -> + debug_data = "unable to decode headers: #{inspect(reason)}" + send_connection_error!(conn, :compression_error, debug_data) + end + end + + defp join_cookie_headers(headers) do + # If we have 0 or 1 Cookie headers, we just use the old list of headers. + case Enum.split_with(headers, fn {name, _value} -> Headers.lower_raw(name) == "cookie" end) do + {[], _headers} -> + headers + + {[_], _headers} -> + headers + + {cookies, headers} -> + cookie = Enum.map_join(cookies, "; ", fn {_name, value} -> value end) + [{"cookie", cookie} | headers] + end + end + + # PRIORITY + + # For now we ignore all PRIORITY frames. This shouldn't cause practical trouble. + defp handle_priority(conn, frame, responses) do + log(conn, :warning, "Ignoring PRIORITY frame: #{inspect(frame)}") + {conn, responses} + end + + # RST_STREAM + + defp handle_rst_stream(conn, frame, responses) do + rst_stream(stream_id: stream_id, error_code: error_code) = frame + + # If we receive RST_STREAM on a closed stream, we ignore it. + case Map.fetch(conn.streams, stream_id) do + {:ok, stream} -> + # If we receive RST_STREAM then the stream is definitely closed. + # We won't send anything else on the stream so we can simply delete + # it, so that if we get things like DATA on that stream we error out. + conn = delete_stream(conn, stream) + + if error_code == :no_error do + {conn, [{:done, stream.ref} | responses]} + else + error = wrap_error({:server_closed_request, error_code}) + {conn, [{:error, stream.ref, error} | responses]} + end + + :error -> + {conn, responses} + end + end + + # SETTINGS + + defp handle_settings(conn, frame, responses) do + settings(flags: flags, params: params) = frame + + if flag_set?(flags, :settings, :ack) do + conn = apply_client_settings(conn) + {conn, responses} + else + conn = apply_server_settings(conn, params) + frame = settings(flags: set_flags(:settings, [:ack]), params: []) + conn = send!(conn, Frame.encode(frame)) + {conn, responses} + end + end + + defp apply_server_settings(conn, server_settings) do + Enum.reduce(server_settings, conn, fn + {:header_table_size, header_table_size}, conn -> + update_in(conn.encode_table, &Hex.Mint.HPAX.resize(&1, header_table_size)) + + {:enable_push, enable_push?}, conn -> + put_in(conn.server_settings.enable_push, enable_push?) + + {:max_concurrent_streams, max_concurrent_streams}, conn -> + put_in(conn.server_settings.max_concurrent_streams, max_concurrent_streams) + + {:initial_window_size, initial_window_size}, conn -> + if initial_window_size > @max_window_size do + debug_data = "INITIAL_WINDOW_SIZE setting of #{initial_window_size} is too big" + send_connection_error!(conn, :flow_control_error, debug_data) + end + + update_server_initial_window_size(conn, initial_window_size) + + {:max_frame_size, max_frame_size}, conn -> + if max_frame_size not in @valid_max_frame_size_range do + debug_data = "MAX_FRAME_SIZE setting parameter outside of allowed range" + send_connection_error!(conn, :protocol_error, debug_data) + end + + put_in(conn.server_settings.max_frame_size, max_frame_size) + + {:max_header_list_size, max_header_list_size}, conn -> + put_in(conn.server_settings.max_header_list_size, max_header_list_size) + + {:enable_connect_protocol, enable_connect_protocol?}, conn -> + put_in(conn.server_settings.enable_connect_protocol, enable_connect_protocol?) + end) + end + + defp apply_client_settings(conn) do + case get_and_update_in(conn.client_settings_queue, &:queue.out/1) do + {{:value, params}, conn} -> + apply_client_settings(conn, params) + + {:empty, conn} -> + log( + conn, + :warning, + "Received SETTINGS ACK but client is not waiting for ACKs; ignoring it" + ) + + conn + end + end + + defp apply_client_settings(conn, client_settings) do + Enum.reduce(client_settings, conn, fn + {setting, value}, conn when setting in @valid_client_settings -> + update_in(conn.client_settings, &%{&1 | setting => value}) + + {setting, _value}, _conn -> + raise "received ack from server for invalid client setting: #{inspect(setting)}}" + end) + end + + defp update_server_initial_window_size(conn, new_iws) do + diff = new_iws - conn.server_settings.initial_window_size + + conn = + update_in(conn.streams, fn streams -> + for {stream_id, stream} <- streams, + stream.state in [:open, :half_closed_remote], + into: streams do + send_window_size = stream.send_window_size + diff + + if send_window_size > @max_window_size do + debug_data = + "INITIAL_WINDOW_SIZE parameter of #{send_window_size} makes some window sizes too big" + + send_connection_error!(conn, :flow_control_error, debug_data) + end + + {stream_id, %{stream | send_window_size: send_window_size}} + end + end) + + put_in(conn.server_settings.initial_window_size, new_iws) + end + + # PUSH_PROMISE + + defp handle_push_promise( + %__MODULE__{client_settings: %{enable_push: false}} = conn, + push_promise(), + _responses + ) do + debug_data = "received PUSH_PROMISE frame when SETTINGS_ENABLE_PUSH was false" + send_connection_error!(conn, :protocol_error, debug_data) + end + + defp handle_push_promise(conn, push_promise() = frame, responses) do + push_promise( + stream_id: stream_id, + flags: flags, + promised_stream_id: promised_stream_id, + hbf: hbf + ) = frame + + assert_valid_promised_stream_id(conn, promised_stream_id) + + stream = fetch_stream!(conn, stream_id) + assert_stream_in_state(conn, stream, [:open, :half_closed_local]) + + if flag_set?(flags, :push_promise, :end_headers) do + decode_push_promise_headers_and_add_response( + conn, + responses, + hbf, + stream, + promised_stream_id + ) + else + callback = &decode_push_promise_headers_and_add_response(&1, &2, &3, &4, promised_stream_id) + conn = put_in(conn.headers_being_processed, {stream_id, hbf, callback}) + {conn, responses} + end + end + + defp decode_push_promise_headers_and_add_response( + conn, + responses, + hbf, + stream, + promised_stream_id + ) do + {conn, headers} = decode_hbf(conn, hbf) + + promised_stream = %{ + id: promised_stream_id, + ref: make_ref(), + state: :reserved_remote, + send_window_size: conn.server_settings.initial_window_size, + receive_window_size: conn.client_settings.initial_window_size, + receive_window: conn.client_settings.initial_window_size, + received_first_headers?: false + } + + conn = put_in(conn.streams[promised_stream.id], promised_stream) + new_response = {:push_promise, stream.ref, promised_stream.ref, headers} + {conn, [new_response | responses]} + end + + defp assert_valid_promised_stream_id(conn, promised_stream_id) do + cond do + not is_integer(promised_stream_id) or Integer.is_odd(promised_stream_id) -> + debug_data = "invalid promised stream ID: #{inspect(promised_stream_id)}" + send_connection_error!(conn, :protocol_error, debug_data) + + Map.has_key?(conn.streams, promised_stream_id) -> + debug_data = + "stream with ID #{inspect(promised_stream_id)} already exists and can't be " <> + "reserved by the server" + + send_connection_error!(conn, :protocol_error, debug_data) + + true -> + :ok + end + end + + # PING + + defp handle_ping(conn, Frame.ping() = frame, responses) do + Frame.ping(flags: flags, opaque_data: opaque_data) = frame + + if flag_set?(flags, :ping, :ack) do + handle_ping_ack(conn, opaque_data, responses) + else + ack = Frame.ping(stream_id: 0, flags: set_flags(:ping, [:ack]), opaque_data: opaque_data) + conn = send!(conn, Frame.encode(ack)) + {conn, responses} + end + end + + defp handle_ping_ack(conn, opaque_data, responses) do + case :queue.peek(conn.ping_queue) do + {:value, {ref, ^opaque_data}} -> + conn = update_in(conn.ping_queue, &:queue.drop/1) + {conn, [{:pong, ref} | responses]} + + {:value, _} -> + log(conn, :warning, "Received PING ack that doesn't match next PING request in the queue") + {conn, responses} + + :empty -> + log(conn, :warning, "Received PING ack but no PING requests are pending") + {conn, responses} + end + end + + # GOAWAY + + defp handle_goaway(conn, frame, responses) do + goaway( + last_stream_id: last_stream_id, + error_code: error_code, + debug_data: debug_data + ) = frame + + # We gather all the unprocessed requests and form {:error, _, _} tuples for each one. + # At the same time, we delete all the unprocessed requests from the stream set. + {unprocessed_request_responses, conn} = + Enum.flat_map_reduce(conn.streams, conn, fn + {stream_id, _stream}, conn_acc when stream_id <= last_stream_id -> + {[], conn_acc} + + {_stream_id, stream}, conn_acc -> + conn_acc = delete_stream(conn_acc, stream) + {[{:error, stream.ref, wrap_error(:unprocessed)}], conn_acc} + end) + + message = + case error_code do + :no_error -> "Server closed connection normally" + _other -> "Server closed connection with error #{inspect(error_code)}" + end + + log(conn, :debug, "#{message} (with debug data: #{inspect(debug_data)})") + + conn = put_in(conn.state, {:goaway, error_code, debug_data}) + {conn, unprocessed_request_responses ++ responses} + end + + # WINDOW_UPDATE + + defp handle_window_update( + conn, + window_update(stream_id: 0, window_size_increment: wsi), + responses + ) do + new_window_size = conn.send_window_size + wsi + + if new_window_size > @max_window_size do + send_connection_error!(conn, :flow_control_error, "window size too big") + else + conn = put_in(conn.send_window_size, new_window_size) + {conn, responses} + end + end + + defp handle_window_update( + conn, + window_update(stream_id: stream_id, window_size_increment: wsi), + responses + ) do + stream = fetch_stream!(conn, stream_id) + new_window_size = conn.streams[stream_id].send_window_size + wsi + + if new_window_size > @max_window_size do + conn = close_stream!(conn, stream_id, :flow_control_error) + error = wrap_error({:flow_control_error, "window size too big"}) + {conn, [{:error, stream.ref, error} | responses]} + else + conn = put_in(conn.streams[stream_id].send_window_size, new_window_size) + {conn, responses} + end + end + + # CONTINUATION + + defp handle_continuation(conn, frame, responses) do + continuation(stream_id: stream_id, flags: flags, hbf: hbf_chunk) = frame + stream = Map.get(conn.streams, stream_id) + + if stream do + assert_stream_in_state(conn, stream, [:open, :half_closed_local, :reserved_remote]) + end + + {^stream_id, hbf_acc, callback} = conn.headers_being_processed + + if flag_set?(flags, :continuation, :end_headers) do + hbf = IO.iodata_to_binary([hbf_acc, hbf_chunk]) + conn = put_in(conn.headers_being_processed, nil) + callback.(conn, responses, hbf, stream) + else + conn = put_in(conn.headers_being_processed, {stream_id, [hbf_acc, hbf_chunk], callback}) + {conn, responses} + end + end + + ## General helpers + + defp send_connection_error!(conn, error_code, debug_data) do + frame = + goaway(stream_id: 0, last_stream_id: 2, error_code: error_code, debug_data: debug_data) + + # Try to send the GOAWAY frame and close connection. + # If the frame fails to send, we still want to set the close + # the socket, set the connection state to :closed, and return an error. + _ = conn.transport.send(conn.socket, Frame.encode(frame)) + _ = conn.transport.close(conn.socket) + + throw({:hex_mint, %{conn | state: :closed}, wrap_error({error_code, debug_data})}) + end + + # Reason is either an error code or `remote_end_stream` + defp close_stream!(conn, stream_id, reason) do + stream = Map.fetch!(conn.streams, stream_id) + + conn = + cond do + # If the stream is ended on both sides, it is already deemed closed and + # there's no need to send a RST_STREAM frame + reason == :remote_end_stream and stream.state == :half_closed_local -> + conn + + # We send a RST_STREAM with the given error code so that we move the + # stream to the :closed state (that is, we remove it). + open?(conn) -> + error_code = if reason == :remote_end_stream, do: :no_error, else: reason + rst_stream_frame = rst_stream(stream_id: stream_id, error_code: error_code) + send!(conn, Frame.encode(rst_stream_frame)) + + # If the connection is already closed, no-op + true -> + conn + end + + delete_stream(conn, stream) + end + + defp delete_stream(conn, stream) do + conn = update_in(conn.streams, &Map.delete(&1, stream.id)) + conn = update_in(conn.ref_to_stream_id, &Map.delete(&1, stream.ref)) + + stream_open? = stream.state in [:open, :half_closed_local, :half_closed_remote] + + conn = + cond do + # Stream initiated by the client. + stream_open? and Integer.is_odd(stream.id) -> + update_in(conn.open_client_stream_count, &(&1 - 1)) + + # Stream initiated by the server. + stream_open? and Integer.is_even(stream.id) -> + update_in(conn.open_server_stream_count, &(&1 - 1)) + + true -> + conn + end + + conn + end + + defp fetch_stream!(conn, stream_id) do + case Map.fetch(conn.streams, stream_id) do + {:ok, stream} -> stream + :error -> throw({:hex_mint, conn, wrap_error({:stream_not_found, stream_id})}) + end + end + + defp assert_stream_in_state(conn, %{state: state}, expected_states) do + if state not in expected_states do + debug_data = + "stream was in state #{inspect(state)} and not in one of the expected states: " <> + Enum.map_join(expected_states, ", ", &inspect/1) + + send_connection_error!(conn, :protocol_error, debug_data) + end + end + + defp send!(%__MODULE__{transport: transport, socket: socket} = conn, bytes) do + case transport.send(socket, bytes) do + :ok -> + conn + + {:error, %TransportError{reason: :closed} = error} -> + throw({:hex_mint, %{conn | state: :closed}, error}) + + {:error, reason} -> + throw({:hex_mint, conn, reason}) + end + end + + defp wrap_error(reason) do + %HTTPError{reason: reason, module: __MODULE__} + end + + @doc false + def format_error(reason) + + def format_error(:closed) do + "the connection is closed" + end + + def format_error(:closed_for_writing) do + "the connection is closed for writing, which means that you cannot issue any more " <> + "requests on the connection but you can expect responses to still be delivered for " <> + "part of the requests that are in flight. If a connection is closed for writing, " <> + "it usually means that you got a :server_closed_request error already." + end + + def format_error(:too_many_concurrent_requests) do + "the number of max concurrent HTTP/2 requests supported by the server has been reached. " <> + "Use Hex.Mint.HTTP2.get_server_setting/2 with the :max_concurrent_streams setting name " <> + "to find out the maximum number of concurrent requests supported by the server." + end + + def format_error({:max_header_list_size_exceeded, size, max_size}) do + "the given header list (of size #{size}) goes over the max header list size of " <> + "#{max_size} supported by the server. In HTTP/2, the header list size is calculated " <> + "by summing up the size in bytes of each header name, value, plus 32 for each header." + end + + def format_error({:exceeds_window_size, what, window_size}) do + what = + case what do + :request -> "request" + :connection -> "connection" + end + + "the given data exceeds the #{what} window size, which is #{window_size}. " <> + "The server will refill the window size of the #{what} when ready. This will be " <> + "handled transparently by stream/2." + end + + def format_error({:stream_not_found, stream_id}) do + "request not found (with stream_id #{inspect(stream_id)})" + end + + def format_error(:unknown_request_to_stream) do + "can't stream chunk of data because the request is unknown" + end + + def format_error({:unknown_request_to_stream, ref}) do + "request with reference #{inspect(ref)} was not found" + end + + def format_error({:window_size_too_small, current, new_size}) do + "set_window_size/3 can only grow a window; new size #{new_size} is " <> + "smaller than the current size #{current}" + end + + def format_error(:request_is_not_streaming) do + "can't send more data on this request since it's not streaming" + end + + def format_error({:unallowed_trailing_header, name}) do + "header #{inspect(name)} is not allowed as a trailer header" + end + + def format_error(:missing_status_header) do + "the :status pseudo-header (which is required in HTTP/2) is missing from the response" + end + + def format_error({:server_closed_request, error_code}) do + "server closed request with error code #{inspect(error_code)}" + end + + def format_error({:server_closed_connection, error, debug_data}) do + "server closed connection with error code #{inspect(error)} and debug data: " <> debug_data + end + + def format_error(:unprocessed) do + "request was not processed by the server, which means that it's safe to retry on a " <> + "different or new connection" + end + + def format_error({:frame_size_error, frame}) do + "frame size error for #{inspect(frame)} frame" + end + + def format_error({:protocol_error, debug_data}) do + "protocol error: " <> debug_data + end + + def format_error({:compression_error, debug_data}) do + "compression error: " <> debug_data + end + + def format_error({:flow_control_error, debug_data}) do + "flow control error: " <> debug_data + end +end diff --git a/lib/hex/mint/http2/frame.ex b/lib/hex/mint/http2/frame.ex new file mode 100644 index 00000000..3bed4725 --- /dev/null +++ b/lib/hex/mint/http2/frame.ex @@ -0,0 +1,480 @@ +# Vendored from mint v1.7.1 (d30d2cf), do not edit manually + +defmodule Hex.Mint.HTTP2.Frame do + @moduledoc false + + import Bitwise, only: [band: 2, bor: 2] + import Kernel, except: [inspect: 1] + import Record + + shared_stream = [:stream_id, {:flags, 0x00}] + shared_conn = [stream_id: 0, flags: 0x00] + + defrecord :data, shared_stream ++ [:data, :padding] + defrecord :headers, shared_stream ++ [:exclusive?, :stream_dependency, :weight, :hbf, :padding] + defrecord :priority, shared_stream ++ [:exclusive?, :stream_dependency, :weight] + defrecord :rst_stream, shared_stream ++ [:error_code] + defrecord :settings, shared_conn ++ [:params] + defrecord :push_promise, shared_stream ++ [:promised_stream_id, :hbf, :padding] + defrecord :ping, shared_conn ++ [:opaque_data] + defrecord :goaway, shared_conn ++ [:last_stream_id, :error_code, :debug_data] + defrecord :window_update, shared_stream ++ [:window_size_increment] + defrecord :continuation, shared_stream ++ [:hbf] + defrecord :unknown, [] + + @types %{ + data: 0x00, + headers: 0x01, + priority: 0x02, + rst_stream: 0x03, + settings: 0x04, + push_promise: 0x05, + ping: 0x06, + goaway: 0x07, + window_update: 0x08, + continuation: 0x09 + } + + ## Inspecting + + @spec inspect(tuple()) :: String.t() + + for {type, _code} <- @types do + def inspect(frame) when is_record(frame, unquote(type)) do + unquote(String.upcase(Atom.to_string(type))) <> Kernel.inspect(unquote(type)(frame)) + end + end + + ## Flag handling + + @flags %{ + data: [end_stream: 0x01, padded: 0x08], + headers: [end_stream: 0x01, end_headers: 0x04, padded: 0x08, priority: 0x20], + settings: [ack: 0x01], + push_promise: [end_headers: 0x04, padded: 0x08], + ping: [ack: 0x01], + continuation: [end_headers: 0x04] + } + + @spec set_flags(byte(), atom(), [flag_name :: atom()]) :: byte() + def set_flags(initial_flags \\ 0x00, frame_name, flags_to_set) + when is_integer(initial_flags) and is_list(flags_to_set) do + Enum.reduce(flags_to_set, initial_flags, &set_flag(&2, frame_name, &1)) + end + + @spec flag_set?(byte(), atom(), atom()) :: boolean() + def flag_set?(flags, frame, flag_name) + + for {frame, flags} <- @flags, + {flag_name, flag_value} <- flags do + defp set_flag(flags, unquote(frame), unquote(flag_name)), do: bor(flags, unquote(flag_value)) + + def flag_set?(flags, unquote(frame), unquote(flag_name)), + do: band(flags, unquote(flag_value)) == unquote(flag_value) + end + + defmacrop is_flag_set(flags, flag) do + quote do + band(unquote(flags), unquote(flag)) == unquote(flag) + end + end + + ## Parsing + + @doc """ + Decodes the next frame of the given binary. + + Returns `{:ok, frame, rest}` if successful, `{:error, reason}` if not. + """ + @spec decode_next(binary()) :: {:ok, tuple(), binary()} | :more | {:error, reason} + when reason: + {:frame_size_error, atom()} + | {:protocol_error, binary()} + | :payload_too_big + def decode_next(bin, max_frame_size \\ 16_384) when is_binary(bin) do + case decode_next_raw(bin) do + {:ok, {_type, _flags, _stream_id, payload}, _rest} + when byte_size(payload) > max_frame_size -> + {:error, :payload_too_big} + + {:ok, {type, flags, stream_id, payload}, rest} -> + {:ok, decode_contents(type, flags, stream_id, payload), rest} + + :more -> + :more + end + catch + :throw, {:hex_mint, reason} -> {:error, reason} + end + + defp decode_next_raw(<< + length::24, + type, + flags, + _reserved::1, + stream_id::31, + payload::size(length)-binary, + rest::binary + >>) do + {:ok, {type, flags, stream_id, payload}, rest} + end + + defp decode_next_raw(_other) do + :more + end + + for {frame, type} <- @types do + function = :"decode_#{frame}" + + defp decode_contents(unquote(type), flags, stream_id, payload) do + unquote(function)(flags, stream_id, payload) + end + end + + defp decode_contents(_type, _flags, _stream_id, _payload) do + unknown() + end + + # Parsing of specific frames + + # http://httpwg.org/specs/rfc7540.html#rfc.section.6.1 + defp decode_data(flags, stream_id, payload) do + {data, padding} = decode_padding(:data, flags, payload) + data(stream_id: stream_id, flags: flags, data: data, padding: padding) + end + + # http://httpwg.org/specs/rfc7540.html#rfc.section.6.2 + defp decode_headers(flags, stream_id, payload) do + {data, padding} = decode_padding(:headers, flags, payload) + + {exclusive?, stream_dependency, weight, data} = + if flag_set?(flags, :headers, :priority) do + <> = data + {exclusive == 1, stream_dependency, weight + 1, rest} + else + {nil, nil, nil, data} + end + + headers( + stream_id: stream_id, + flags: flags, + padding: padding, + exclusive?: exclusive?, + stream_dependency: stream_dependency, + weight: weight, + hbf: data + ) + end + + # http://httpwg.org/specs/rfc7540.html#rfc.section.6.3 + defp decode_priority(_flags, _stream_id, payload) when byte_size(payload) != 5 do + throw({:hex_mint, {:frame_size_error, :priority}}) + end + + defp decode_priority(flags, stream_id, payload) do + <> = payload + + priority( + stream_id: stream_id, + flags: flags, + exclusive?: exclusive == 1, + stream_dependency: stream_dependency, + weight: weight + 1 + ) + end + + # http://httpwg.org/specs/rfc7540.html#rfc.section.6.4 + defp decode_rst_stream(_flags, _stream_id, payload) when byte_size(payload) != 4 do + throw({:hex_mint, {:frame_size_error, :rst_stream}}) + end + + defp decode_rst_stream(flags, stream_id, <>) do + rst_stream( + stream_id: stream_id, + flags: flags, + error_code: humanize_error_code(error_code) + ) + end + + # http://httpwg.org/specs/rfc7540.html#rfc.section.6.5 + defp decode_settings(_flags, _stream_id, payload) when rem(byte_size(payload), 6) != 0 do + throw({:hex_mint, {:frame_size_error, :settings}}) + end + + defp decode_settings(flags, stream_id, payload) do + settings(stream_id: stream_id, flags: flags, params: decode_settings_params(payload)) + end + + # http://httpwg.org/specs/rfc7540.html#rfc.section.6.6 + defp decode_push_promise(flags, stream_id, payload) do + {data, padding} = decode_padding(:push_promise, flags, payload) + <<_reserved::1, promised_stream_id::31, header_block_fragment::binary>> = data + + push_promise( + stream_id: stream_id, + flags: flags, + promised_stream_id: promised_stream_id, + hbf: header_block_fragment, + padding: padding + ) + end + + # http://httpwg.org/specs/rfc7540.html#rfc.section.6.7 + defp decode_ping(_flags, _stream_id, payload) when byte_size(payload) != 8 do + throw({:hex_mint, {:frame_size_error, :ping}}) + end + + defp decode_ping(flags, stream_id, payload) do + ping(stream_id: stream_id, flags: flags, opaque_data: payload) + end + + # http://httpwg.org/specs/rfc7540.html#rfc.section.6.8 + defp decode_goaway(flags, stream_id, payload) do + <<_reserved::1, last_stream_id::31, error_code::32, debug_data::binary>> = payload + + goaway( + stream_id: stream_id, + flags: flags, + last_stream_id: last_stream_id, + error_code: humanize_error_code(error_code), + debug_data: debug_data + ) + end + + # http://httpwg.org/specs/rfc7540.html#rfc.section.6.9 + defp decode_window_update(_flags, _stream_id, payload) when byte_size(payload) != 4 do + throw({:hex_mint, {:frame_size_error, :window_update}}) + end + + defp decode_window_update(_flags, _stream_id, <<_reserved::1, 0::31>>) do + throw({:hex_mint, {:protocol_error, "bad WINDOW_SIZE increment"}}) + end + + defp decode_window_update(flags, stream_id, <<_reserved::1, window_size_increment::31>>) do + window_update( + stream_id: stream_id, + flags: flags, + window_size_increment: window_size_increment + ) + end + + # http://httpwg.org/specs/rfc7540.html#rfc.section.6.10 + defp decode_continuation(flags, stream_id, payload) do + continuation(stream_id: stream_id, flags: flags, hbf: payload) + end + + defp decode_padding(frame, flags, <> = payload) + when is_flag_set(flags, unquote(@flags[:data][:padded])) do + if pad_length >= byte_size(payload) do + debug_data = + "the padding length of a #{Kernel.inspect(frame)} frame is bigger than the payload length" + + throw({:hex_mint, {:protocol_error, debug_data}}) + else + # 1 byte is for the space taken by pad_length + data_length = byte_size(payload) - pad_length - 1 + <> = rest + {data, padding} + end + end + + defp decode_padding(_frame, _flags, payload) do + {payload, nil} + end + + defp decode_settings_params(payload) do + decode_settings_params(payload, _acc = []) + end + + defp decode_settings_params(<<>>, acc) do + Enum.reverse(acc) + end + + defp decode_settings_params(<>, acc) do + # From http://httpwg.org/specs/rfc7540.html#SettingValues: + # An endpoint that receives a SETTINGS frame with any unknown or unsupported identifier MUST + # ignore that setting. + acc = + case identifier do + 0x01 -> [{:header_table_size, value} | acc] + 0x02 -> [{:enable_push, value == 1} | acc] + 0x03 -> [{:max_concurrent_streams, value} | acc] + 0x04 -> [{:initial_window_size, value} | acc] + 0x05 -> [{:max_frame_size, value} | acc] + 0x06 -> [{:max_header_list_size, value} | acc] + 0x08 -> [{:enable_connect_protocol, value == 1} | acc] + _other -> acc + end + + decode_settings_params(rest, acc) + end + + ## Encoding + + @doc """ + Encodes the given `frame`. + """ + @spec encode(tuple()) :: iodata() + def encode(frame) + + def encode(data(stream_id: stream_id, flags: flags, data: data, padding: nil)) do + encode_raw(@types[:data], flags, stream_id, data) + end + + def encode(data(stream_id: stream_id, flags: flags, data: data, padding: padding)) do + flags = set_flags(flags, :data, [:padded]) + payload = [byte_size(padding), data, padding] + encode_raw(@types[:data], flags, stream_id, payload) + end + + def encode(headers() = frame) do + headers( + flags: flags, + stream_id: stream_id, + exclusive?: exclusive?, + stream_dependency: stream_dependency, + weight: weight, + hbf: hbf, + padding: padding + ) = frame + + payload = hbf + + {payload, flags} = + if stream_dependency && weight && is_boolean(exclusive?) do + { + [<>, weight - 1, payload], + set_flags(flags, :headers, [:priority]) + } + else + {payload, flags} + end + + {payload, flags} = + if padding do + {[byte_size(padding), payload, padding], set_flags(flags, :headers, [:padded])} + else + {payload, flags} + end + + encode_raw(@types[:headers], flags, stream_id, payload) + end + + def encode(priority() = frame) do + priority( + stream_id: stream_id, + flags: flags, + exclusive?: exclusive?, + stream_dependency: stream_dependency, + weight: weight + ) = frame + + payload = [ + <>, + weight - 1 + ] + + encode_raw(@types[:priority], flags, stream_id, payload) + end + + def encode(rst_stream(stream_id: stream_id, flags: flags, error_code: error_code)) do + payload = <> + encode_raw(@types[:rst_stream], flags, stream_id, payload) + end + + def encode(settings(stream_id: stream_id, flags: flags, params: params)) do + payload = + Enum.map(params, fn + {:header_table_size, value} -> <<0x01::16, value::32>> + {:enable_push, value} -> <<0x02::16, if(value, do: 1, else: 0)::32>> + {:max_concurrent_streams, value} -> <<0x03::16, value::32>> + {:initial_window_size, value} -> <<0x04::16, value::32>> + {:max_frame_size, value} -> <<0x05::16, value::32>> + {:max_header_list_size, value} -> <<0x06::16, value::32>> + {:enable_connect_protocol, value} -> <<0x08::16, if(value, do: 1, else: 0)::32>> + end) + + encode_raw(@types[:settings], flags, stream_id, payload) + end + + def encode(push_promise() = frame) do + push_promise( + stream_id: stream_id, + flags: flags, + promised_stream_id: promised_stream_id, + hbf: hbf, + padding: padding + ) = frame + + payload = [<<0::1, promised_stream_id::31>>, hbf] + + {payload, flags} = + if padding do + { + [byte_size(padding), payload, padding], + set_flags(flags, :push_promise, [:padded]) + } + else + {payload, flags} + end + + encode_raw(@types[:push_promise], flags, stream_id, payload) + end + + def encode(ping(stream_id: 0, flags: flags, opaque_data: opaque_data)) do + encode_raw(@types[:ping], flags, 0, opaque_data) + end + + def encode(goaway() = frame) do + goaway( + stream_id: 0, + flags: flags, + last_stream_id: last_stream_id, + error_code: error_code, + debug_data: debug_data + ) = frame + + payload = [<<0::1, last_stream_id::31, dehumanize_error_code(error_code)::32>>, debug_data] + encode_raw(@types[:goaway], flags, 0, payload) + end + + def encode(window_update(stream_id: stream_id, flags: flags, window_size_increment: wsi)) do + payload = <<0::1, wsi::31>> + encode_raw(@types[:window_update], flags, stream_id, payload) + end + + def encode(continuation(stream_id: stream_id, flags: flags, hbf: hbf)) do + encode_raw(@types[:continuation], flags, stream_id, _payload = hbf) + end + + def encode_raw(type, flags, stream_id, payload) do + [<>, type, flags, <<0::1, stream_id::31>>, payload] + end + + ## Helpers + + error_codes = %{ + 0x00 => :no_error, + 0x01 => :protocol_error, + 0x02 => :internal_error, + 0x03 => :flow_control_error, + 0x04 => :settings_timeout, + 0x05 => :stream_closed, + 0x06 => :frame_size_error, + 0x07 => :refused_stream, + 0x08 => :cancel, + 0x09 => :compression_error, + 0x0A => :connect_error, + 0x0B => :enhance_your_calm, + 0x0C => :inadequate_security, + 0x0D => :http_1_1_required + } + + for {code, human_code} <- error_codes do + defp humanize_error_code(unquote(code)), do: unquote(human_code) + defp dehumanize_error_code(unquote(human_code)), do: unquote(code) + end + + defp humanize_error_code(code), do: {:custom_error, code} + defp dehumanize_error_code({:custom_error, code}), do: code +end diff --git a/lib/hex/mint/http_error.ex b/lib/hex/mint/http_error.ex new file mode 100644 index 00000000..816647c2 --- /dev/null +++ b/lib/hex/mint/http_error.ex @@ -0,0 +1,75 @@ +# Vendored from mint v1.7.1 (d30d2cf), do not edit manually + +defmodule Hex.Mint.HTTPError do + _ = """ + An HTTP error. + + This exception struct is used to represent HTTP errors of all sorts and for + both HTTP/1 and HTTP/2. + + A `Hex.Mint.HTTPError` struct is an exception, so it can be raised as any + other exception. + + ## Struct + + The `Hex.Mint.HTTPError` struct is opaque, that is, not all of its fields are public. + The list of public fields is: + + * `:reason` - the error reason. Can be one of: + + * a term of type `t:Hex.Mint.HTTP1.error_reason/0`. See its documentation for + more information. + + * a term of type `t:Hex.Mint.HTTP2.error_reason/0`. See its documentation for + more information. + + * `{:proxy, reason}`, which is used when an HTTP error happens when connecting + to a tunnel proxy. `reason` can be: + + * `:tunnel_timeout` - when the tunnel times out. + + * `{:unexpected_status, status}` - when the proxy returns an unexpected + status `status`. + + * `{:unexpected_trailing_responses, responses}` - when the proxy returns + unexpected responses (`responses`). + + ## Message representation + + If you want to convert an error reason to a human-friendly message (for example + for using in logs), you can use `Exception.message/1`: + + iex> {:error, %Hex.Mint.HTTPError{} = error} = Hex.Mint.HTTP.connect(:http, "bad-response.com", 80) + iex> Exception.message(error) + "the response contains two or more Content-Length headers" + + """ + + alias Hex.Mint.{HTTP1, HTTP2} + + @type proxy_reason() :: + {:proxy, + HTTP1.error_reason() + | HTTP2.error_reason() + | :tunnel_timeout + | {:unexpected_status, non_neg_integer()} + | {:unexpected_trailing_responses, list()}} + + @typedoc """ + The error reason. + """ + @typedoc since: "1.7.2" + @type reason :: HTTP1.error_reason() | HTTP2.error_reason() | proxy_reason() | term() + + @typedoc """ + Type for the `Hex.Mint.HTTPError` exception struct. + """ + @type t() :: %__MODULE__{reason: reason()} + + defexception [:reason, :module] + + @impl true + def message(%__MODULE__{reason: reason, module: module}) do + module.format_error(reason) + end +end diff --git a/lib/hex/mint/negotiate.ex b/lib/hex/mint/negotiate.ex new file mode 100644 index 00000000..fdc6deac --- /dev/null +++ b/lib/hex/mint/negotiate.ex @@ -0,0 +1,148 @@ +# Vendored from mint v1.7.1 (d30d2cf), do not edit manually + +defmodule Hex.Mint.Negotiate do + @moduledoc false + + alias Hex.Mint.{ + HTTP1, + HTTP2, + TransportError, + Types + } + + alias Hex.Mint.Core.Util + + @default_protocols [:http1, :http2] + @transport_opts [alpn_advertised_protocols: ["http/1.1", "h2"]] + + @spec connect(Types.scheme(), Types.address(), :inet.port_number(), keyword()) :: + {:ok, Hex.Mint.HTTP.t()} | {:error, Types.error()} + def connect(scheme, address, port, opts \\ []) do + {protocols, opts} = Keyword.pop(opts, :protocols, @default_protocols) + + case Enum.sort(protocols) do + [:http1] -> + HTTP1.connect(scheme, address, port, opts) + + [:http2] -> + HTTP2.connect(scheme, address, port, opts) + + [:http1, :http2] -> + transport_connect(scheme, address, port, opts) + end + end + + @spec upgrade( + module(), + Types.socket(), + Types.scheme(), + String.t(), + :inet.port_number(), + keyword() + ) :: {:ok, Hex.Mint.HTTP.t()} | {:error, Types.error()} + def upgrade(proxy_scheme, transport_state, scheme, hostname, port, opts) do + {protocols, opts} = Keyword.pop(opts, :protocols, @default_protocols) + + case Enum.sort(protocols) do + [:http1] -> + HTTP1.upgrade(proxy_scheme, transport_state, scheme, hostname, port, opts) + + [:http2] -> + HTTP2.upgrade(proxy_scheme, transport_state, scheme, hostname, port, opts) + + [:http1, :http2] -> + transport_upgrade(proxy_scheme, transport_state, scheme, hostname, port, opts) + end + end + + @spec initiate(module(), Types.socket(), String.t(), :inet.port_number(), keyword()) :: + {:ok, Hex.Mint.HTTP.t()} | {:error, Types.error()} + def initiate(transport, transport_state, hostname, port, opts), + do: alpn_negotiate(transport, transport_state, hostname, port, opts) + + defp transport_connect(:http, address, port, opts) do + # HTTP1 upgrade is not supported + HTTP1.connect(:http, address, port, opts) + end + + defp transport_connect(:https, address, port, opts) do + connect_negotiate(:https, address, port, opts) + end + + defp connect_negotiate(scheme, address, port, opts) do + transport = Util.scheme_to_transport(scheme) + hostname = Hex.Mint.Core.Util.hostname(opts, address) + + transport_opts = + opts + |> Keyword.get(:transport_opts, []) + |> Keyword.merge(@transport_opts) + |> Keyword.put(:hostname, hostname) + + with {:ok, transport_state} <- transport.connect(address, port, transport_opts) do + alpn_negotiate(scheme, transport_state, hostname, port, opts) + end + end + + defp transport_upgrade( + proxy_scheme, + transport_state, + :http, + hostname, + port, + opts + ) do + # HTTP1 upgrade is not supported + HTTP1.upgrade(proxy_scheme, transport_state, :http, hostname, port, opts) + end + + defp transport_upgrade( + proxy_scheme, + transport_state, + :https, + hostname, + port, + opts + ) do + connect_upgrade(proxy_scheme, transport_state, :https, hostname, port, opts) + end + + defp connect_upgrade(proxy_scheme, transport_state, new_scheme, hostname, port, opts) do + transport = Util.scheme_to_transport(new_scheme) + + transport_opts = + opts + |> Keyword.get(:transport_opts, []) + |> Keyword.merge(@transport_opts) + + case transport.upgrade(transport_state, proxy_scheme, hostname, port, transport_opts) do + {:ok, transport_state} -> + alpn_negotiate(new_scheme, transport_state, hostname, port, opts) + + {:error, reason} -> + {:error, %TransportError{reason: reason}} + end + end + + defp alpn_negotiate(scheme, socket, hostname, port, opts) do + transport = Util.scheme_to_transport(scheme) + + case transport.negotiated_protocol(socket) do + {:ok, "http/1.1"} -> + HTTP1.initiate(scheme, socket, hostname, port, opts) + + {:ok, "h2"} -> + HTTP2.initiate(scheme, socket, hostname, port, opts) + + {:error, %TransportError{reason: :protocol_not_negotiated}} -> + # Assume HTTP1 if ALPN is not supported + HTTP1.initiate(scheme, socket, hostname, port, opts) + + {:ok, protocol} -> + {:error, %TransportError{reason: {:bad_alpn_protocol, protocol}}} + + {:error, %TransportError{} = error} -> + {:error, error} + end + end +end diff --git a/lib/hex/mint/transport_error.ex b/lib/hex/mint/transport_error.ex new file mode 100644 index 00000000..84394b5c --- /dev/null +++ b/lib/hex/mint/transport_error.ex @@ -0,0 +1,104 @@ +# Vendored from mint v1.7.1 (d30d2cf), do not edit manually + +defmodule Hex.Mint.TransportError do + _ = """ + Represents an error with the transport used by an HTTP connection. + + A `Hex.Mint.TransportError` struct is an exception, so it can be raised as any + other exception. + + ## Struct fields + + This exception represents an error with the transport (TCP or SSL) used + by an HTTP connection. The exception struct itself is opaque, that is, + not all fields are public. The following are the public fields: + + * `:reason` - a term representing the error reason. The value of this field + can be: + + * `:timeout` - if there's a timeout in interacting with the socket. + + * `:closed` - if the connection has been closed. + + * `:protocol_not_negotiated` - if the ALPN protocol negotiation failed. + + * `{:bad_alpn_protocol, protocol}` - when the ALPN protocol is not + one of the supported protocols, which are `http/1.1` and `h2`. + + * `t::inet.posix/0` - if there's any other error with the socket, + such as `:econnrefused` or `:nxdomain`. + + * `t::ssl.error_alert/0` - if there's an SSL error. + + ## Message representation + + If you want to convert an error reason to a human-friendly message (for example + for using in logs), you can use `Exception.message/1`: + + iex> {:error, %Hex.Mint.TransportError{} = error} = Hex.Mint.HTTP.connect(:http, "nonexistent", 80) + iex> Exception.message(error) + "non-existing domain" + + """ + + reason_type = + quote do + :timeout + | :closed + | :protocol_not_negotiated + | {:bad_alpn_protocol, String.t()} + | :inet.posix() + end + + reason_type = + if System.otp_release() >= "21" do + quote do: unquote(reason_type) | :ssl.error_alert() + else + reason_type + end + + @typedoc """ + The error reason. + """ + @typedoc since: "1.7.2" + @type reason :: unquote(reason_type) | term() + + @typedoc """ + Type for the `Hex.Mint.TransportError` exception struct. + """ + @type t() :: %__MODULE__{reason: reason()} + + defexception [:reason] + + @impl true + def message(%__MODULE__{reason: reason}) do + format_reason(reason) + end + + ## Our reasons. + + defp format_reason(:protocol_not_negotiated) do + "ALPN protocol not negotiated" + end + + defp format_reason({:bad_alpn_protocol, protocol}) do + "bad ALPN protocol #{inspect(protocol)}, supported protocols are \"http/1.1\" and \"h2\"" + end + + defp format_reason(:closed) do + "socket closed" + end + + defp format_reason(:timeout) do + "timeout" + end + + # :ssl.format_error/1 falls back to :inet.format_error/1 when the error is not an SSL-specific + # error (at least since OTP 19+), so we can just use that. + defp format_reason(reason) do + case :ssl.format_error(reason) do + ~c"Unexpected error:" ++ _ -> inspect(reason) + message -> List.to_string(message) + end + end +end diff --git a/lib/hex/mint/tunnel_proxy.ex b/lib/hex/mint/tunnel_proxy.ex new file mode 100644 index 00000000..ee302cc8 --- /dev/null +++ b/lib/hex/mint/tunnel_proxy.ex @@ -0,0 +1,150 @@ +# Vendored from mint v1.7.1 (d30d2cf), do not edit manually + +defmodule Hex.Mint.TunnelProxy do + @moduledoc false + + alias Hex.Mint.{HTTP, HTTP1, HTTPError, Negotiate, TransportError} + + @tunnel_timeout 30_000 + + @spec connect(tuple(), tuple()) :: {:ok, Hex.Mint.HTTP.t()} | {:error, term()} + def connect(proxy, host) do + case establish_proxy(proxy, host) do + {:ok, conn} -> upgrade_connection(conn, proxy, host) + {:error, reason} -> {:error, reason} + end + end + + defp establish_proxy(proxy, host) do + {proxy_scheme, proxy_address, proxy_port, proxy_opts} = proxy + {_scheme, address, port, opts} = host + hostname = Hex.Mint.Core.Util.hostname(opts, address) + + path = "#{hostname}:#{port}" + + with {:ok, conn} <- HTTP1.connect(proxy_scheme, proxy_address, proxy_port, proxy_opts), + timeout_deadline = timeout_deadline(proxy_opts), + headers = Keyword.get(opts, :proxy_headers, []), + {:ok, conn, ref} <- HTTP1.request(conn, "CONNECT", path, headers, nil), + {:ok, proxy_headers} <- receive_response(conn, ref, timeout_deadline) do + {:ok, HTTP1.put_proxy_headers(conn, proxy_headers)} + else + {:error, reason} -> + {:error, wrap_in_proxy_error(reason)} + + {:error, conn, reason} -> + {:ok, _conn} = HTTP1.close(conn) + {:error, wrap_in_proxy_error(reason)} + end + end + + defp upgrade_connection( + conn, + {proxy_scheme, _proxy_address, _proxy_port, _proxy_opts} = _proxy, + {scheme, hostname, port, opts} = _host + ) do + proxy_headers = HTTP1.get_proxy_headers(conn) + socket = HTTP1.get_socket(conn) + + # Note that we may leak messages if the server sent data after the CONNECT response + case Negotiate.upgrade(proxy_scheme, socket, scheme, hostname, port, opts) do + {:ok, conn} -> {:ok, HTTP.put_proxy_headers(conn, proxy_headers)} + {:error, reason} -> {:error, wrap_in_proxy_error(reason)} + end + end + + defp receive_response(conn, ref, timeout_deadline) do + timeout = timeout_deadline - System.monotonic_time(:millisecond) + socket = HTTP1.get_socket(conn) + + receive do + {tag, ^socket, _data} = msg when tag in [:tcp, :ssl] -> + stream(conn, ref, timeout_deadline, msg) + + {tag, ^socket} = msg when tag in [:tcp_closed, :ssl_closed] -> + stream(conn, ref, timeout_deadline, msg) + + {tag, ^socket, _reason} = msg when tag in [:tcp_error, :ssl_error] -> + stream(conn, ref, timeout_deadline, msg) + after + timeout -> + {:error, conn, wrap_error({:proxy, :tunnel_timeout})} + end + end + + defp stream(conn, ref, timeout_deadline, msg) do + case HTTP1.stream(conn, msg) do + {:ok, conn, responses} -> + case handle_responses(ref, timeout_deadline, responses) do + {:done, proxy_headers} -> {:ok, proxy_headers} + :more -> receive_response(conn, ref, timeout_deadline) + {:error, reason} -> {:error, conn, reason} + end + + {:error, conn, reason, _responses} -> + {:error, conn, wrap_in_proxy_error(reason)} + end + end + + defp handle_responses(ref, timeout_deadline, [response | responses]) do + case response do + {:status, ^ref, status} when status in 200..299 -> + handle_responses(ref, timeout_deadline, responses) + + {:status, ^ref, status} -> + {:error, wrap_error({:proxy, {:unexpected_status, status}})} + + {:headers, ^ref, headers} when responses == [] -> + {:done, headers} + + {:headers, ^ref, _headers} -> + {:error, wrap_error({:proxy, {:unexpected_trailing_responses, responses}})} + + {:error, ^ref, reason} -> + {:error, wrap_in_proxy_error(reason)} + end + end + + defp handle_responses(_ref, _timeout_deadline, []) do + :more + end + + defp timeout_deadline(opts) do + timeout = Keyword.get(opts, :tunnel_timeout, @tunnel_timeout) + System.monotonic_time(:millisecond) + timeout + end + + defp wrap_error(reason) do + %HTTPError{module: __MODULE__, reason: reason} + end + + defp wrap_in_proxy_error(%HTTPError{reason: {:proxy, _}} = error) do + error + end + + defp wrap_in_proxy_error(%HTTPError{reason: reason}) do + %HTTPError{module: __MODULE__, reason: {:proxy, reason}} + end + + defp wrap_in_proxy_error(%TransportError{} = error) do + error + end + + @doc false + def format_error({:proxy, reason}) do + case reason do + :tunnel_timeout -> + "proxy tunnel timeout" + + {:unexpected_status, status} -> + "expected tunnel proxy to return a status between 200 and 299, got: #{inspect(status)}" + + {:unexpected_trailing_responses, responses} -> + "tunnel proxy returned unexpected trailer responses: #{inspect(responses)}" + + http_reason -> + "error when establishing the tunnel proxy connection: " <> + HTTP1.format_error(http_reason) + end + end +end diff --git a/lib/hex/mint/types.ex b/lib/hex/mint/types.ex new file mode 100644 index 00000000..8e1329eb --- /dev/null +++ b/lib/hex/mint/types.ex @@ -0,0 +1,76 @@ +# Vendored from mint v1.7.1 (d30d2cf), do not edit manually + +defmodule Hex.Mint.Types do + _ = """ + HTTP-related types. + """ + + @typedoc """ + A hostname, IP address, Unix domain socket path, `:loopback`, or any + other term representing an internet address. + """ + @type address() :: :inet.socket_address() | String.t() + + @typedoc """ + A request reference that uniquely identifies a request. + + Responses for a request are always tagged with a request reference so that you + can connect each response to the right request. Also see `Hex.Mint.HTTP.request/5`. + """ + @type request_ref() :: reference() + + @typedoc """ + An HTTP/2-specific response to a request. + + This type of response is only returned on HTTP/2 connections. See `t:response/0` for + more response types. + """ + @type http2_response() :: + {:pong, request_ref()} + | {:push_promise, request_ref(), promised_request_ref :: request_ref(), headers()} + + @typedoc """ + A response to a request. + + Terms of this type are returned as responses to requests. See `Hex.Mint.HTTP.stream/2` + for more information. + """ + @type response() :: + {:status, request_ref(), status()} + | {:headers, request_ref(), headers()} + | {:data, request_ref(), body_chunk :: binary()} + | {:done, request_ref()} + | {:error, request_ref(), reason :: term()} + | http2_response() + + @typedoc """ + An HTTP status code. + + The type for an HTTP is a generic non-negative integer since we don't formally check that + the response code is in the "common" range (`200..599`). + """ + @type status() :: non_neg_integer() + + @typedoc """ + HTTP headers. + + Headers are sent and received as lists of two-element tuples containing two strings, + the header name and header value. + """ + @type headers() :: [{header_name :: String.t(), header_value :: String.t()}] + + @typedoc """ + The scheme to use when connecting to an HTTP server. + """ + @type scheme() :: :http | :https + + @typedoc """ + An error reason. + """ + @type error() :: Hex.Mint.TransportError.t() | Hex.Mint.HTTPError.t() + + @typedoc """ + The connection socket. + """ + @type socket() :: term() +end diff --git a/lib/hex/mint/unsafe_proxy.ex b/lib/hex/mint/unsafe_proxy.ex new file mode 100644 index 00000000..e90aaf41 --- /dev/null +++ b/lib/hex/mint/unsafe_proxy.ex @@ -0,0 +1,204 @@ +# Vendored from mint v1.7.1 (d30d2cf), do not edit manually + +defmodule Hex.Mint.UnsafeProxy do + @moduledoc false + + alias Hex.Mint.{Types, UnsafeProxy} + + @behaviour Hex.Mint.Core.Conn + + defstruct [ + :hostname, + :port, + :scheme, + :module, + :proxy_headers, + :state + ] + + @opaque t() :: %UnsafeProxy{} + + @type host_triple() :: {Types.scheme(), address :: Types.address(), :inet.port_number()} + + @spec connect(host_triple(), host_triple(), opts :: keyword()) :: + {:ok, t()} | {:error, Types.error()} + def connect(proxy, host, opts \\ []) do + {proxy_scheme, proxy_address, proxy_port} = proxy + {scheme, address, port} = host + hostname = Hex.Mint.Core.Util.hostname(opts, address) + + with {:ok, state} <- Hex.Mint.HTTP1.connect(proxy_scheme, proxy_address, proxy_port, opts) do + conn = %UnsafeProxy{ + scheme: scheme, + hostname: hostname, + port: port, + module: Hex.Mint.HTTP1, + proxy_headers: Keyword.get(opts, :proxy_headers, []), + state: state + } + + {:ok, conn} + end + end + + @impl true + @spec initiate( + module(), + Hex.Mint.Types.socket(), + String.t(), + :inet.port_number(), + keyword() + ) :: no_return() + def initiate(_transport, _transport_state, _hostname, _port, _opts) do + raise "initiate/5 does not apply for #{inspect(__MODULE__)}" + end + + @impl true + @spec close(t()) :: {:ok, t()} + def close(%UnsafeProxy{module: module, state: state} = _conn) do + module.close(state) + end + + @impl true + @spec open?(t(), :read | :write) :: boolean() + def open?(%UnsafeProxy{module: module, state: state}, type \\ :write) do + module.open?(state, type) + end + + @impl true + @spec request( + t(), + method :: String.t(), + path :: String.t(), + Types.headers(), + body :: iodata() | nil | :stream + ) :: + {:ok, t(), Types.request_ref()} + | {:error, t(), Types.error()} + def request( + %UnsafeProxy{module: module, state: state} = conn, + method, + path, + headers, + body \\ nil + ) do + path = request_line(conn, path) + headers = headers ++ conn.proxy_headers + + case module.request(state, method, path, headers, body) do + {:ok, state, request} -> {:ok, %{conn | state: state}, request} + {:error, state, reason} -> {:error, %{conn | state: state}, reason} + end + end + + @impl true + @spec stream_request_body( + t(), + Types.request_ref(), + iodata() | :eof | {:eof, trailer_headers :: Types.headers()} + ) :: + {:ok, t()} | {:error, t(), Types.error()} + def stream_request_body(%UnsafeProxy{module: module, state: state} = conn, ref, body) do + case module.stream_request_body(state, ref, body) do + {:ok, state} -> {:ok, %{conn | state: state}} + {:error, state, reason} -> {:error, %{conn | state: state}, reason} + end + end + + @impl true + @spec stream(t(), term()) :: + {:ok, t(), [Types.response()]} + | {:error, t(), Types.error(), [Types.response()]} + | :unknown + def stream(%UnsafeProxy{module: module, state: state} = conn, message) do + case module.stream(state, message) do + {:ok, state, responses} -> {:ok, %{conn | state: state}, responses} + {:error, state, reason, responses} -> {:error, %{conn | state: state}, reason, responses} + :unknown -> :unknown + end + end + + @impl true + @spec open_request_count(t()) :: non_neg_integer() + def open_request_count(%UnsafeProxy{module: module, state: state} = _conn) do + module.open_request_count(state) + end + + @impl true + @spec recv(t(), non_neg_integer(), timeout()) :: + {:ok, t(), [Types.response()]} + | {:error, t(), Types.error(), [Types.response()]} + def recv(%UnsafeProxy{module: module, state: state} = conn, byte_count, timeout) do + case module.recv(state, byte_count, timeout) do + {:ok, state, responses} -> {:ok, %{conn | state: state}, responses} + {:error, state, reason, responses} -> {:error, %{conn | state: state}, reason, responses} + end + end + + @impl true + @spec set_mode(t(), :active | :passive) :: {:ok, t()} | {:error, Types.error()} + def set_mode(%UnsafeProxy{module: module, state: state} = conn, mode) do + with {:ok, state} <- module.set_mode(state, mode) do + {:ok, %{conn | state: state}} + end + end + + @impl true + @spec controlling_process(t(), pid()) :: {:ok, t()} | {:error, Types.error()} + def controlling_process(%UnsafeProxy{module: module, state: state} = conn, new_pid) do + with {:ok, _} <- module.controlling_process(state, new_pid) do + {:ok, conn} + end + end + + @impl true + @spec put_private(t(), atom(), term()) :: t() + def put_private(%UnsafeProxy{module: module, state: state} = conn, key, value) do + state = module.put_private(state, key, value) + %{conn | state: state} + end + + @impl true + @spec get_private(t(), atom(), term()) :: term() + def get_private(%UnsafeProxy{module: module, state: state}, key, default \\ nil) do + module.get_private(state, key, default) + end + + @impl true + @spec delete_private(t(), atom()) :: t() + def delete_private(%UnsafeProxy{module: module, state: state} = conn, key) do + state = module.delete_private(state, key) + %{conn | state: state} + end + + defp request_line(%UnsafeProxy{scheme: scheme, hostname: hostname, port: port}, path) do + %URI{scheme: Atom.to_string(scheme), host: hostname, port: port, path: path} + |> URI.to_string() + end + + @impl true + @spec get_socket(t()) :: Hex.Mint.Types.socket() + def get_socket(%UnsafeProxy{module: module, state: state}) do + module.get_socket(state) + end + + @impl true + @spec put_log(t(), boolean()) :: t() + def put_log(%UnsafeProxy{module: module, state: state} = conn, log) do + state = module.put_log(state, log) + %{conn | state: state} + end + + # The `%__MODULE__{proxy_headers: value}` here is the request headers, + # not the proxy response ones. Unsafe proxy mixes its headers (if any) + # with the regular response headers, so you can get them there. + @impl true + @spec get_proxy_headers(t()) :: Hex.Mint.Types.headers() + def get_proxy_headers(%__MODULE__{}), do: [] + + @impl true + @spec put_proxy_headers(t(), Hex.Mint.Types.headers()) :: t() + def put_proxy_headers(%__MODULE__{}, _headers) do + raise "invalid function for proxy unsafe proxy connections" + end +end diff --git a/lib/hex/state.ex b/lib/hex/state.ex index 2e4546d3..af2372d3 100644 --- a/lib/hex/state.ex +++ b/lib/hex/state.ex @@ -157,7 +157,6 @@ defmodule Hex.State do Map.merge(state, %{ clean_pass: {:computed, true}, - httpc_profile: {:computed, :hex}, pbkdf2_iters: {:computed, @pbkdf2_iters}, repos: {:computed, Hex.Config.read_repos(global_config)}, ssl_version: {:computed, ssl_version()}, diff --git a/mix.exs b/mix.exs index 4afb230f..1bd2f04f 100644 --- a/mix.exs +++ b/mix.exs @@ -29,7 +29,7 @@ defmodule Hex.MixProject do def application do [ - extra_applications: [:ssl, :inets, :logger], + extra_applications: [:ssl, :logger], mod: {Hex.Application, []} ] end diff --git a/scripts/vendor_mint.sh b/scripts/vendor_mint.sh new file mode 100755 index 00000000..58ced399 --- /dev/null +++ b/scripts/vendor_mint.sh @@ -0,0 +1,134 @@ +#!/bin/bash +set -e + +# Pending upstream Mint PRs that must be present in the source tree passed +# to this script. If re-vendoring from upstream `main` or a Mint release that +# does not yet include these, either wait for them to merge or point this +# script at a branch that has them applied. The easy path: use the +# integration branch below which has all of them cherry-picked. +# +# Integration branch: +# https://github.com/elixir-mint/mint/tree/ericmj/hex-vendor-integration +# +# Included (or to-be-submitted) upstream PRs: +# +# * elixir-mint/mint#478 - "Support Elixir ~> 1.12". Removes the +# `^length` binary-size pin (Elixir 1.14+) and drops `Mint.Application` +# so the persistent_term cacertfile cache works without the app starting. +# Hex supports `~> 1.12` so these changes are required. +# https://github.com/elixir-mint/mint/pull/478 +# Branch: https://github.com/elixir-mint/mint/tree/ericmj/support-elixir-1.12 +# +# * elixir-mint/mint#479 - "Fix HTTP/1 handling of 1xx informational +# responses". Without this, a 100 Continue / 103 Early Hints / any +# unsolicited 1xx causes Mint to emit `:done` prematurely and close +# the connection on the real final response. +# https://github.com/elixir-mint/mint/pull/479 +# Branch: https://github.com/elixir-mint/mint/tree/ericmj/fix-1xx-informational-response +# +# * `Mint.HTTP2.set_window_size/3`. Adds a public API to advertise a larger +# HTTP/2 receive window to the server (connection-level or per-stream) via +# a `WINDOW_UPDATE` frame. +# Branch: https://github.com/elixir-mint/mint/tree/ericmj/http2-connection-window-size +# +# * Larger default HTTP/2 receive windows (4 MB per stream, 16 MB per +# connection) plus threshold-gated `WINDOW_UPDATE` batching to mitigate +# the amplification-DoS shape of refilling on every DATA frame. +# Branch: https://github.com/elixir-mint/mint/tree/ericmj/http2-larger-default-windows + +if [[ -z "$1" ]]; then + echo "Usage: vendor_mint.sh PATH_TO_MINT" + exit 1 +fi + +dir=$1 + +pushd $dir +mix compile +version=$(mix run -e 'IO.puts(Mix.Project.config[:version])') +shortref=$(git rev-parse --short HEAD) +popd + +rm -rf lib/hex/mint + +skip_filenames="mint/application.ex" + +for filename in $(find $dir/lib -type f -name '*.ex'); do + target_filename=${filename#$dir/lib/} + target_path=lib/hex/${target_filename} + + if [[ $skip_filenames == *$target_filename* ]]; then + continue + fi + + mkdir -p $(dirname $target_path) + echo "# Vendored from mint v$version ($shortref), do not edit manually" > $target_path + echo >> $target_path + cat $filename >> $target_path + + # Suppress @moduledoc + sed -i.bak 's/@moduledoc """/_ = """/g' $target_path + rm $target_path.bak + + # Rename modules: Mint.* -> Hex.Mint.* + sed -i.bak 's/Mint\./Hex.Mint./g' $target_path + rm $target_path.bak + + # Rename internal atom tags used in throw/catch and persistent_term keys + sed -i.bak 's/{:mint,/{:hex_mint,/g' $target_path + rm $target_path.bak + + # Use vendored Mint version in user agent + sed -i.bak "s|Mix\.Project\.config()\[:version\]|\"$version\"|g" $target_path + rm $target_path.bak +done + +# Vendor Erlang shims +for filename in $(find $dir/src -type f -name '*.erl'); do + basename=$(basename $filename) + target_path=src/hex_${basename} + + echo "%% Vendored from mint v$version ($shortref), do not edit manually" > $target_path + echo >> $target_path + cat $filename >> $target_path + + sed -i.bak 's/-module(mint_/-module(hex_mint_/g' $target_path + rm $target_path.bak +done + +# Update references to Erlang shims in vendored Elixir files +for filename in $(find lib/hex/mint -type f -name '*.ex'); do + sed -i.bak 's/:mint_shims/:hex_mint_shims/g' $filename + rm $filename.bak +done + +# Vendor hpax (Mint dependency for HTTP/2 HPACK) +hpax_dir=$dir/deps/hpax +hpax_version=$(cat $hpax_dir/mix.exs | grep '@version' | head -1 | sed 's/.*"\(.*\)".*/\1/') + +for filename in $(find $hpax_dir/lib -type f -name '*.ex'); do + target_filename=${filename#$hpax_dir/lib/} + target_path=lib/hex/mint/${target_filename} + + mkdir -p $(dirname $target_path) + echo "# Vendored from hpax v$hpax_version, do not edit manually" > $target_path + echo >> $target_path + cat $filename >> $target_path + + # Suppress @moduledoc + sed -i.bak 's/@moduledoc """/_ = """/g' $target_path + rm $target_path.bak + + # Rename modules: HPAX -> Hex.Mint.HPAX + sed -i.bak 's/HPAX/Hex.Mint.HPAX/g' $target_path + rm $target_path.bak +done + +# Copy non-source files needed at compile time +cp $hpax_dir/lib/hpax/huffman_table lib/hex/mint/hpax/huffman_table + +# Also rename HPAX references in vendored Mint files +for filename in $(find lib/hex/mint -type f -name '*.ex' ! -path '*/hpax*'); do + sed -i.bak 's/HPAX/Hex.Mint.HPAX/g' $filename + rm $filename.bak +done diff --git a/src/hex_mint_shims.erl b/src/hex_mint_shims.erl new file mode 100644 index 00000000..11ff64e1 --- /dev/null +++ b/src/hex_mint_shims.erl @@ -0,0 +1,238 @@ +%% Vendored from mint v1.7.1 (d30d2cf), do not edit manually + +%% Shims for functions introduced in recent Erlang/OTP releases, +%% to enable use of Mint on older releases. The code in this module +%% was taken directly from the Erlang/OTP project. +%% +%% File: lib/public_key/src/public_key.erl +%% Tag: OTP-20.3.4 +%% Commit: f2c1d537dc28ffbde5d42aedec70bf4c6574c3ea +%% Changes from original file: +%% - extracted pkix_verify_hostname/2 and /3, and any private +%% functions they depend upon +%% - replaced local calls to other public functions in the +%% 'public_key' module with fully qualified equivalents +%% - replaced local type references with fully qualified equivalents +%% +%% The original license follows: + +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2013-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(hex_mint_shims). + +-include_lib("public_key/include/public_key.hrl"). + +-export([pkix_verify_hostname/2, pkix_verify_hostname/3]). + +%-------------------------------------------------------------------- +-spec pkix_verify_hostname(Cert :: #'OTPCertificate'{} | binary(), + ReferenceIDs :: [{uri_id | dns_id | ip | srv_id | public_key:oid(), string()}]) -> boolean(). + +-spec pkix_verify_hostname(Cert :: #'OTPCertificate'{} | binary(), + ReferenceIDs :: [{uri_id | dns_id | ip | srv_id | public_key:oid(), string()}], + Options :: proplists:proplist()) -> boolean(). + +%% Description: Validates a hostname to RFC 6125 +%%-------------------------------------------------------------------- +pkix_verify_hostname(Cert, ReferenceIDs) -> + pkix_verify_hostname(Cert, ReferenceIDs, []). + +pkix_verify_hostname(BinCert, ReferenceIDs, Options) when is_binary(BinCert) -> + pkix_verify_hostname(public_key:pkix_decode_cert(BinCert,otp), ReferenceIDs, Options); + +pkix_verify_hostname(Cert = #'OTPCertificate'{tbsCertificate = TbsCert}, ReferenceIDs0, Opts) -> + MatchFun = proplists:get_value(match_fun, Opts, undefined), + FailCB = proplists:get_value(fail_callback, Opts, fun(_Cert) -> false end), + FqdnFun = proplists:get_value(fqdn_fun, Opts, fun verify_hostname_extract_fqdn_default/1), + + ReferenceIDs = [{T,to_string(V)} || {T,V} <- ReferenceIDs0], + PresentedIDs = + try lists:keyfind(?'id-ce-subjectAltName', + #'Extension'.extnID, + TbsCert#'OTPTBSCertificate'.extensions) + of + #'Extension'{extnValue = ExtVals} -> + [{T,to_string(V)} || {T,V} <- ExtVals]; + false -> + [] + catch + _:_ -> [] + end, + %% PresentedIDs example: [{dNSName,"ewstest.ericsson.com"}, {dNSName,"www.ericsson.com"}]} + case PresentedIDs of + [] -> + %% Fallback to CN-ids [rfc6125, ch6] + case TbsCert#'OTPTBSCertificate'.subject of + {rdnSequence,RDNseq} -> + PresentedCNs = + [{cn, to_string(V)} + || ATVs <- RDNseq, % RDNseq is list-of-lists + #'AttributeTypeAndValue'{type = ?'id-at-commonName', + value = {_T,V}} <- ATVs + % _T = kind of string (teletexString etc) + ], + %% Example of PresentedCNs: [{cn,"www.ericsson.se"}] + %% match ReferenceIDs to PresentedCNs + verify_hostname_match_loop(verify_hostname_fqnds(ReferenceIDs, FqdnFun), + PresentedCNs, + MatchFun, FailCB, Cert); + + _ -> + false + end; + _ -> + %% match ReferenceIDs to PresentedIDs + case verify_hostname_match_loop(ReferenceIDs, PresentedIDs, + MatchFun, FailCB, Cert) of + false -> + %% Try to extract DNS-IDs from URIs etc + DNS_ReferenceIDs = + [{dns_id,X} || X <- verify_hostname_fqnds(ReferenceIDs, FqdnFun)], + verify_hostname_match_loop(DNS_ReferenceIDs, PresentedIDs, + MatchFun, FailCB, Cert); + true -> + true + end + end. + +%%%---------------------------------------------------------------- +%%% pkix_verify_hostname help functions +verify_hostname_extract_fqdn_default({dns_id,S}) -> + S; +verify_hostname_extract_fqdn_default({uri_id,URI}) -> + % Modified from original to remove dependency on http_uri:parse/1 from inets + #{scheme := <<"https">>, host := Host} = 'Elixir.URI':parse(list_to_binary(URI)), + binary_to_list(Host). + + +verify_hostname_fqnds(L, FqdnFun) -> + [E || E0 <- L, + E <- [try case FqdnFun(E0) of + default -> verify_hostname_extract_fqdn_default(E0); + undefined -> undefined; % will make the "is_list(E)" test fail + Other -> Other + end + catch _:_-> undefined % will make the "is_list(E)" test fail + end], + is_list(E), + E =/= "", + {error,einval} == inet:parse_address(E) + ]. + + +-define(srvName_OID, {1,3,6,1,4,1,434,2,2,1,37,0}). + +verify_hostname_match_default(Ref, Pres) -> + verify_hostname_match_default0(to_lower_ascii(Ref), to_lower_ascii(Pres)). + +verify_hostname_match_default0(FQDN=[_|_], {cn,FQDN}) -> + not lists:member($*, FQDN); +verify_hostname_match_default0(FQDN=[_|_], {cn,Name=[_|_]}) -> + [F1|Fs] = string:tokens(FQDN, "."), + [N1|Ns] = string:tokens(Name, "."), + match_wild(F1,N1) andalso Fs==Ns; +verify_hostname_match_default0({dns_id,R}, {dNSName,P}) -> + R==P; +verify_hostname_match_default0({uri_id,R}, {uniformResourceIdentifier,P}) -> + R==P; +verify_hostname_match_default0({ip,R}, {iPAddress,P}) when length(P) == 4 -> + %% IPv4 + try + list_to_tuple(P) + == if is_tuple(R), size(R)==4 -> R; + is_list(R) -> ok(inet:parse_ipv4strict_address(R)) + end + catch + _:_ -> + false + end; + +verify_hostname_match_default0({ip,R}, {iPAddress,P}) when length(P) == 16 -> + %% IPv6. The length 16 is due to the certificate specification. + try + l16_to_tup(P) + == if is_tuple(R), size(R)==8 -> R; + is_list(R) -> ok(inet:parse_ipv6strict_address(R)) + end + catch + _:_ -> + false + end; +verify_hostname_match_default0({srv_id,R}, {srvName,P}) -> + R==P; +verify_hostname_match_default0({srv_id,R}, {?srvName_OID,P}) -> + R==P; +verify_hostname_match_default0(_, _) -> + false. + +ok({ok,X}) -> X. + +l16_to_tup(L) -> list_to_tuple(l16_to_tup(L, [])). +%% +l16_to_tup([A,B|T], Acc) -> l16_to_tup(T, [(A bsl 8) bor B | Acc]); +l16_to_tup([], Acc) -> lists:reverse(Acc). + +match_wild(A, [$*|B]) -> match_wild_suffixes(A, B); +match_wild([C|A], [ C|B]) -> match_wild(A, B); +match_wild([], []) -> true; +match_wild(_, _) -> false. + +%% Match the parts after the only wildcard by comparing them from the end +match_wild_suffixes(A, B) -> match_wild_sfx(lists:reverse(A), lists:reverse(B)). + +match_wild_sfx([$*|_], _) -> false; % Bad name (no wildcards allowed) +match_wild_sfx(_, [$*|_]) -> false; % Bad pattern (no more wildcards allowed) +match_wild_sfx([A|Ar], [A|Br]) -> match_wild_sfx(Ar, Br); +match_wild_sfx(Ar, []) -> not lists:member($*, Ar); % Chk for bad name (= wildcards) +match_wild_sfx(_, _) -> false. + + +verify_hostname_match_loop(Refs0, Pres0, undefined, FailCB, Cert) -> + Pres = lists:map(fun to_lower_ascii/1, Pres0), + Refs = lists:map(fun to_lower_ascii/1, Refs0), + lists:any( + fun(R) -> + lists:any(fun(P) -> + verify_hostname_match_default(R,P) orelse FailCB(Cert) + end, Pres) + end, Refs); +verify_hostname_match_loop(Refs, Pres, MatchFun, FailCB, Cert) -> + lists:any( + fun(R) -> + lists:any(fun(P) -> + (case MatchFun(R,P) of + default -> verify_hostname_match_default(R,P); + Bool -> Bool + end) orelse FailCB(Cert) + end, + Pres) + end, + Refs). + + +to_lower_ascii({ip,_}=X) -> X; +to_lower_ascii({iPAddress,_}=X) -> X; +to_lower_ascii(S) when is_list(S) -> lists:map(fun to_lower_ascii/1, S); +to_lower_ascii({T,S}) -> {T, to_lower_ascii(S)}; +to_lower_ascii(C) when $A =< C,C =< $Z -> C + ($a-$A); +to_lower_ascii(C) -> C. + +to_string(S) when is_list(S) -> S; +to_string(B) when is_binary(B) -> binary_to_list(B); +to_string(X) -> X. diff --git a/test/hex/http/pool_test.exs b/test/hex/http/pool_test.exs new file mode 100644 index 00000000..3e1c2117 --- /dev/null +++ b/test/hex/http/pool_test.exs @@ -0,0 +1,231 @@ +defmodule Hex.HTTP.PoolTest do + use HexTest.Case, async: false + + setup do + bypass = Bypass.open() + {:ok, bypass: bypass} + end + + defp wait_for_host(key, timeout \\ 500) do + deadline = System.monotonic_time(:millisecond) + timeout + do_wait_for_host(key, deadline) + end + + defp do_wait_for_host(key, deadline) do + case Registry.lookup(Hex.HTTP.Pool.__registry__(), key) do + [{pid, _}] -> + pid + + [] -> + if System.monotonic_time(:millisecond) >= deadline do + flunk("no Host registered for #{inspect(key)}") + else + Process.sleep(10) + do_wait_for_host(key, deadline) + end + end + end + + defp wait_until(fun, timeout \\ 1_000) do + deadline = System.monotonic_time(:millisecond) + timeout + do_wait_until(fun, deadline) + end + + defp do_wait_until(fun, deadline) do + case fun.() do + {:ok, result} -> + result + + :retry -> + if System.monotonic_time(:millisecond) >= deadline do + flunk("condition not met before timeout") + else + Process.sleep(10) + do_wait_until(fun, deadline) + end + end + end + + test "different inet variants get separate Host pools", %{bypass: bypass} do + Bypass.expect(bypass, fn conn -> Plug.Conn.resp(conn, 200, "ok") end) + + url = "http://localhost:#{bypass.port}/" + + pool_opts_inet4 = [ + timeout: 5_000, + connect_opts: [transport_opts: [inet4: true, inet6: false]] + ] + + pool_opts_inet6 = [ + timeout: 5_000, + connect_opts: [transport_opts: [inet4: false, inet6: true]] + ] + + {:ok, 200, _, "ok"} = Hex.HTTP.Pool.request(url, "GET", [], nil, pool_opts_inet4) + + pid_inet4 = wait_for_host({:http, "localhost", bypass.port, :inet}) + + # IPv6 connect likely fails since bypass binds to IPv4; the request will + # error but a separate Host registration must still have been attempted. + _ = Hex.HTTP.Pool.request(url, "GET", [], nil, pool_opts_inet6) + + pid_inet6 = wait_for_host({:http, "localhost", bypass.port, :inet6}) + + assert pid_inet4 != pid_inet6 + end + + test "Host spawns a replacement Conn after one crashes", %{bypass: bypass} do + Bypass.expect(bypass, fn conn -> Plug.Conn.resp(conn, 200, "ok") end) + + url = "http://localhost:#{bypass.port}/" + pool_opts = [timeout: 5_000] + + {:ok, 200, _, "ok"} = Hex.HTTP.Pool.request(url, "GET", [], nil, pool_opts) + + host_pid = wait_for_host({:http, "localhost", bypass.port, :inet}) + + # Wait for both probe Conns to report ready. + wait_until(fn -> + state = :sys.get_state(host_pid) + ready_count = Enum.count(state.conns, fn {_, info} -> info.ready end) + if ready_count >= 2, do: {:ok, state}, else: :retry + end) + + state_before = :sys.get_state(host_pid) + conn_pids_before = Map.keys(state_before.conns) + [victim | _] = conn_pids_before + + Process.exit(victim, :kill) + + # Host traps exits and must spawn a replacement — pool size stays stable + # and the victim pid is no longer a member. + wait_until(fn -> + state = :sys.get_state(host_pid) + members = Map.keys(state.conns) + + cond do + victim in members -> + :retry + + map_size(state.conns) != map_size(state_before.conns) -> + :retry + + true -> + {:ok, state} + end + end) + + # The pool remains functional after the crash. + {:ok, 200, _, "ok"} = Hex.HTTP.Pool.request(url, "GET", [], nil, pool_opts) + end + + test "request_to_file streams response body to disk without buffering", %{bypass: bypass} do + # 5 MB body split into 64 KB chunks — bigger than any reasonable in-memory + # budget we'd want to pay twice (once in the accumulator, once in File.write). + chunk = :binary.copy(<<0xAB>>, 65_536) + chunks = 80 + expected_size = byte_size(chunk) * chunks + + Bypass.expect(bypass, fn conn -> + conn = Plug.Conn.send_chunked(conn, 200) + + Enum.reduce(1..chunks, conn, fn _, conn -> + {:ok, conn} = Plug.Conn.chunk(conn, chunk) + conn + end) + end) + + url = "http://localhost:#{bypass.port}/big" + pool_opts = [timeout: 10_000] + + tmp = + Path.join(System.tmp_dir!(), "hex-http-pool-stream-#{System.unique_integer([:positive])}") + + try do + assert {:ok, 200, _headers, nil} = + Hex.HTTP.Pool.request_to_file(url, "GET", [], nil, tmp, pool_opts) + + assert File.exists?(tmp) + %File.Stat{size: size} = File.stat!(tmp) + assert size == expected_size + + # Spot-check a few random offsets to verify bytes round-tripped intact + # without re-reading the whole file into memory. + fd = File.open!(tmp, [:read, :raw, :binary]) + + try do + for offset <- [0, div(expected_size, 2), expected_size - 1] do + {:ok, <>} = :file.pread(fd, offset, 1) + assert byte == 0xAB + end + after + File.close(fd) + end + after + _ = File.rm(tmp) + end + end + + test "request_to_file truncates prior content on redirect", %{bypass: bypass} do + # First hit the /redir path, which 302s to /final. The file should contain + # /final's body only — no trace of any body the redirect response might + # have had (and the sink must have been reset between iterations). + Bypass.expect(bypass, fn conn -> + case conn.request_path do + "/redir" -> + conn + |> Plug.Conn.put_resp_header("location", "/final") + |> Plug.Conn.resp(302, String.duplicate("X", 4096)) + + "/final" -> + Plug.Conn.resp(conn, 200, "final-body") + end + end) + + url = "http://localhost:#{bypass.port}/redir" + + tmp = + Path.join(System.tmp_dir!(), "hex-http-pool-redirect-#{System.unique_integer([:positive])}") + + try do + assert {:ok, {200, _headers}} = + Hex.HTTP.request_to_file("GET", url, %{}, nil, tmp, %{}) + + assert File.read!(tmp) == "final-body" + after + _ = File.rm(tmp) + end + end + + test "requests queue when all Conns are busy and drain as they finish", %{bypass: bypass} do + # Slow every response so we can saturate a small pool. + test_pid = self() + + Bypass.expect(bypass, fn conn -> + send(test_pid, {:bypass_got, conn.request_path}) + Process.sleep(100) + Plug.Conn.resp(conn, 200, "ok") + end) + + url_base = "http://localhost:#{bypass.port}" + pool_opts = [timeout: 5_000] + + # Warm the pool to HTTP/1 (bypass is HTTP/1 only), then spawn more + # requests than pool capacity (default 8) so the 9th+ must queue. + {:ok, 200, _, _} = Hex.HTTP.Pool.request(url_base <> "/warmup", "GET", [], nil, pool_opts) + + count = 12 + + tasks = + for i <- 1..count do + Task.async(fn -> + Hex.HTTP.Pool.request(url_base <> "/req#{i}", "GET", [], nil, pool_opts) + end) + end + + results = Task.await_many(tasks, 10_000) + + assert length(results) == count + Enum.each(results, fn {:ok, 200, _, "ok"} -> :ok end) + end +end diff --git a/test/hex/http_integration_test.exs b/test/hex/http_integration_test.exs new file mode 100644 index 00000000..b7961a1e --- /dev/null +++ b/test/hex/http_integration_test.exs @@ -0,0 +1,84 @@ +defmodule Hex.HTTPIntegrationTest do + # End-to-end HTTPS round-trips against the real hex.pm + repo.hex.pm + # servers. Exercises the full Mint pipeline: TLS handshake, certificate + # verification, ALPN protocol negotiation (HTTP/2 where the server offers + # it), the connection pool, and response decoding. + # + # These tests require working internet access and that hex.pm + + # repo.hex.pm are reachable. Tagged `:network` so they can be excluded + # with `mix test --exclude network` if the environment is offline, but + # they run by default on plain `mix test`. + + use HexTest.Case, async: false + + @moduletag :network + + @hexpm_host "hex.pm" + @repo_host "repo.hex.pm" + + test "GET https://#{@hexpm_host}/api/packages/phoenix returns JSON metadata" do + {:ok, {status, headers, body}} = + Hex.HTTP.request(:get, "https://#{@hexpm_host}/api/packages/phoenix", %{}, nil) + + assert status == 200 + assert headers["content-type"] =~ "application/" + assert byte_size(body) > 0 + assert body =~ "phoenix" + end + + test "GET https://#{@repo_host}/names returns the signed names registry" do + {:ok, {status, _headers, body}} = + Hex.HTTP.request(:get, "https://#{@repo_host}/names", %{}, nil) + + assert status == 200 + # The names registry is a signed gzipped protobuf; we only assert that + # the pipeline delivered a non-trivial payload. + assert byte_size(body) > 100 + end + + test "GET https://#{@repo_host}/packages/phoenix returns the versions record" do + {:ok, {status, _headers, body}} = + Hex.HTTP.request(:get, "https://#{@repo_host}/packages/phoenix", %{}, nil) + + assert status == 200 + assert byte_size(body) > 0 + end + + test "pool negotiates a real ALPN protocol for #{@hexpm_host}" do + {:ok, _} = + Hex.HTTP.request(:get, "https://#{@hexpm_host}/api/packages/phoenix", %{}, nil) + + host_pid = lookup_host(@hexpm_host) + %{protocol: protocol, conns: conns} = :sys.get_state(host_pid) + + assert protocol in [:http1, :http2] + assert map_size(conns) >= 1 + end + + test "pool negotiates a real ALPN protocol for #{@repo_host}" do + {:ok, _} = Hex.HTTP.request(:get, "https://#{@repo_host}/names", %{}, nil) + + host_pid = lookup_host(@repo_host) + %{protocol: protocol, conns: conns} = :sys.get_state(host_pid) + + assert protocol in [:http1, :http2] + assert map_size(conns) >= 1 + end + + test "consecutive requests to the same host reuse the same pool" do + {:ok, _} = Hex.HTTP.request(:get, "https://#{@repo_host}/names", %{}, nil) + pid1 = lookup_host(@repo_host) + + {:ok, _} = Hex.HTTP.request(:get, "https://#{@repo_host}/versions", %{}, nil) + pid2 = lookup_host(@repo_host) + + assert pid1 == pid2 + end + + defp lookup_host(host) do + case Registry.lookup(Hex.HTTP.Pool.__registry__(), {:https, host, 443, :inet}) do + [{pid, _}] -> pid + [] -> flunk("no Hex.HTTP.Pool.Host registered for #{host}") + end + end +end diff --git a/test/hex/http_test.exs b/test/hex/http_test.exs index 68ce240d..683bdf9d 100644 --- a/test/hex/http_test.exs +++ b/test/hex/http_test.exs @@ -3,12 +3,7 @@ defmodule Hex.HTTPTest do setup do on_exit(fn -> - Enum.map([:http_proxy, :https_proxy], &Hex.State.put(&1, nil)) - - Enum.map([:proxy, :https_proxy], fn opt -> - :httpc.set_options([{opt, {{~c"localhost", 80}, [~c"localhost"]}}], :hex) - end) - + Enum.each([:http_proxy, :https_proxy, :no_proxy], &Hex.State.put(&1, nil)) System.delete_env("NETRC") end) @@ -16,32 +11,50 @@ defmodule Hex.HTTPTest do {:ok, bypass: bypass} end - test "proxy_config returns no credentials when no proxy supplied" do + defp proxy_auth_header(opts) do + case Keyword.get(opts, :proxy) do + {:http, _host, _port, proxy_opts} -> + case Keyword.get(proxy_opts, :proxy_headers, []) do + [{"proxy-authorization", "Basic " <> creds}] -> creds + _ -> nil + end + + _ -> + nil + end + end + + test "proxy_config returns no proxy when none is supplied" do assert Hex.HTTP.proxy_config("http://hex.pm") == [] end - test "proxy_config returns http_proxy credentials when supplied" do + test "proxy_config encodes http_proxy credentials when supplied" do Hex.State.put(:http_proxy, "http://hex:test@example.com") - assert Hex.HTTP.proxy_config("http://hex.pm") == [proxy_auth: {~c"hex", ~c"test"}] + opts = Hex.HTTP.proxy_config("http://hex.pm") + assert proxy_auth_header(opts) == Base.encode64("hex:test") end - test "proxy_config returns http_proxy credentials when only username supplied" do + test "proxy_config encodes http_proxy credentials when only username supplied" do Hex.State.put(:http_proxy, "http://nopass@example.com") - assert Hex.HTTP.proxy_config("http://hex.pm") == [proxy_auth: {~c"nopass", ~c""}] + opts = Hex.HTTP.proxy_config("http://hex.pm") + assert proxy_auth_header(opts) == Base.encode64("nopass") end - test "proxy_config returns credentials when the protocol is https" do + test "proxy_config encodes credentials when the protocol is https" do Hex.State.put(:https_proxy, "https://test:hex@example.com") - assert Hex.HTTP.proxy_config("https://hex.pm") == [proxy_auth: {~c"test", ~c"hex"}] + opts = Hex.HTTP.proxy_config("https://hex.pm") + assert proxy_auth_header(opts) == Base.encode64("test:hex") end - test "proxy_config returns empty list when no credentials supplied" do + test "proxy_config returns proxy with no auth when no credentials supplied" do Hex.State.put(:http_proxy, "http://example.com") - assert Hex.HTTP.proxy_config("http://hex.pm") == [] + opts = Hex.HTTP.proxy_config("http://hex.pm") + assert {:http, "example.com", 80, proxy_opts} = Keyword.fetch!(opts, :proxy) + assert Keyword.get(proxy_opts, :proxy_headers, []) == [] end test "x-hex-message" do @@ -117,14 +130,11 @@ defmodule Hex.HTTPTest do end test "request with Expect 100-continue receives body after 100 response", %{bypass: bypass} do - # Test that httpc handles 100-continue flow correctly body_content = "test request body" Bypass.expect(bypass, fn conn -> - # Verify the Expect header is present assert ["100-continue"] = Plug.Conn.get_req_header(conn, "expect") - # Send 100 Continue informational response conn = Plug.Conn.inform(conn, 100, []) {:ok, body, conn} = Plug.Conn.read_body(conn) @@ -145,19 +155,119 @@ defmodule Hex.HTTPTest do assert response_body == "success" end - test "request with Expect 100-continue stops sending body on error response", %{ + test "informational (1xx) response with headers does not leak into final response", %{ bypass: bypass } do - # Test that when server responds with error before 100, body is not sent - # Note: This is handled by httpc automatically - if server responds with - # error status instead of 100 Continue, httpc won't send the body + Bypass.expect(bypass, fn conn -> + # 103 Early Hints carries real headers that must NOT appear on the final + # 200 response. + conn = Plug.Conn.inform(conn, 103, [{"link", "; rel=preload"}]) + Plug.Conn.resp(conn, 200, "body") + end) + + {:ok, {status, headers, body}} = + Hex.HTTP.request(:get, "http://localhost:#{bypass.port}", %{}, nil) + + assert status == 200 + assert body == "body" + refute Map.has_key?(headers, "link") + end + + test "routes plain HTTP requests through http_proxy when configured" do + # Bypass (Cowboy) rejects absolute-URI request lines, which is exactly + # what an HTTP/1 proxy client sends. Spin up a bare-TCP listener that + # records the raw request and returns a canned response. The pool opens + # two probe connections so we must accept in a loop and respond to each. + me = self() + {:ok, listen} = :gen_tcp.listen(0, [:binary, active: false, reuseaddr: true]) + {:ok, port} = :inet.port(listen) + + proxy = spawn(fn -> proxy_accept_loop(listen, me) end) + on_exit(fn -> Process.exit(proxy, :kill) end) + + Hex.State.put(:http_proxy, "http://user:pass@localhost:#{port}") + + # `example.invalid` never resolves, but Mint connects to the proxy (not + # the target) so the request still goes through. + {:ok, {200, _headers, body}} = + Hex.HTTP.request(:get, "http://example.invalid/some/path", %{}, nil) + + assert body == "from proxy" + assert_receive {:proxy_got, request}, 5_000 + assert request =~ ~r{^GET http://example\.invalid/some/path HTTP/1\.1\r\n} + assert request =~ "proxy-authorization: Basic " <> Base.encode64("user:pass") + end + + defp proxy_accept_loop(listen, test_pid) do + case :gen_tcp.accept(listen, 5_000) do + {:ok, socket} -> + spawn(fn -> handle_proxy_conn(socket, test_pid) end) + proxy_accept_loop(listen, test_pid) + + _ -> + :gen_tcp.close(listen) + end + end + + defp handle_proxy_conn(socket, test_pid) do + case read_until_blank_line(socket, "") do + "" -> + :gen_tcp.close(socket) + + request -> + send(test_pid, {:proxy_got, request}) + _ = :gen_tcp.send(socket, "HTTP/1.1 200 OK\r\ncontent-length: 10\r\n\r\nfrom proxy") + :gen_tcp.close(socket) + end + end + + defp read_until_blank_line(socket, acc) do + case :gen_tcp.recv(socket, 0, 1_000) do + {:ok, data} -> + acc = acc <> data + if acc =~ "\r\n\r\n", do: acc, else: read_until_blank_line(socket, acc) + + {:error, _} -> + acc + end + end + + test "streamed body with progress callback fires incrementally and delivers full body", + %{bypass: bypass} do + # 35_000 bytes = 4 full 10_000-byte chunks + one partial 5_000-byte chunk + body = String.duplicate("x", 35_000) + me = self() Bypass.expect(bypass, fn conn -> - # Verify the Expect header is present - assert ["100-continue"] = Plug.Conn.get_req_header(conn, "expect") + {:ok, received, conn} = Plug.Conn.read_body(conn, length: 64_000) + send(me, {:received, received}) + Plug.Conn.resp(conn, 200, "ok") + end) + + progress_callback = fn size -> send(me, {:progress, size}) end - # Immediately respond with 401 Unauthorized without reading body - # httpc should NOT send the body when it receives this error + {:ok, {200, _headers, "ok"}} = + Hex.HTTP.request( + :post, + "http://localhost:#{bypass.port}", + %{}, + {"application/octet-stream", body}, + %{progress_callback: progress_callback} + ) + + assert_received {:received, ^body} + # Progress callback must fire for each chunk, not just once at completion. + assert_received {:progress, 10_000} + assert_received {:progress, 20_000} + assert_received {:progress, 30_000} + assert_received {:progress, 35_000} + end + + test "request with Expect 100-continue stops sending body on error response", %{ + bypass: bypass + } do + Bypass.expect(bypass, fn conn -> + assert ["100-continue"] = Plug.Conn.get_req_header(conn, "expect") Plug.Conn.resp(conn, 401, "unauthorized") end) diff --git a/test/mix/tasks/hex.registry_test.exs b/test/mix/tasks/hex.registry_test.exs index 598f0863..b6d1189c 100644 --- a/test/mix/tasks/hex.registry_test.exs +++ b/test/mix/tasks/hex.registry_test.exs @@ -20,7 +20,7 @@ defmodule Mix.Tasks.Hex.RegistryTest do refute_received _ config = %{ - :mix_hex_core.default_config() + Hex.HTTP.config() | repo_url: "http://localhost:#{bypass.port}", repo_verify: false, repo_verify_origin: false diff --git a/test/support/hexpm.ex b/test/support/hexpm.ex index 50cd6950..ef4abaa9 100644 --- a/test/support/hexpm.ex +++ b/test/support/hexpm.ex @@ -188,8 +188,9 @@ defmodule HexTest.Hexpm do end defp wait_on_start do - case :httpc.request(:get, {~c"http://localhost:4043", []}, [], []) do - {:ok, _} -> + case :gen_tcp.connect(~c"localhost", 4043, [:binary, active: false], 1_000) do + {:ok, sock} -> + :gen_tcp.close(sock) :ok {:error, _} ->