From 93345394420413859a546bdbdb03cffeffa0d696 Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Sun, 29 Mar 2026 17:47:08 -0500
Subject: [PATCH] Retry on 429/5xx responses across all HTTP clients

Previously retry: :transient only retried connection errors, not HTTP
429 rate limits. All clients now use a custom retry function that
handles 429, 500, 502, 503, 504 with 5 retries and jittered backoff.
---
 lib/microwaveprop/terrain/elevation_client.ex | 12 ++++++++++--
 lib/microwaveprop/weather/hrrr_client.ex      | 12 ++++++++++--
 lib/microwaveprop/weather/iem_client.ex       | 12 ++++++++++--
 3 files changed, 30 insertions(+), 6 deletions(-)

diff --git a/lib/microwaveprop/terrain/elevation_client.ex b/lib/microwaveprop/terrain/elevation_client.ex
index c883fc1c..8b038682 100644
--- a/lib/microwaveprop/terrain/elevation_client.ex
+++ b/lib/microwaveprop/terrain/elevation_client.ex
@@ -101,14 +101,22 @@ defmodule Microwaveprop.Terrain.ElevationClient do
   defp deg_to_rad(deg), do: deg * :math.pi() / 180
 
   defp req_options do
-    defaults = [retry: :transient, max_retries: 2, retry_delay: &retry_delay/1]
+    defaults = [retry: &retry?/2, max_retries: 5, retry_delay: &retry_delay/1]
     overrides = Application.get_env(:microwaveprop, :elevation_req_options, [])
     Keyword.merge(defaults, overrides)
   end
 
+  defp retry?(_request, response) do
+    case response do
+      %Req.Response{status: status} when status in [429, 500, 502, 503, 504] -> true
+      %{__exception__: true} -> true
+      _ -> false
+    end
+  end
+
   defp retry_delay(n) do
     base = Integer.pow(2, n) * 1_000
-    jitter = :rand.uniform(500)
+    jitter = :rand.uniform(1_000)
     base + jitter
   end
 end
diff --git a/lib/microwaveprop/weather/hrrr_client.ex b/lib/microwaveprop/weather/hrrr_client.ex
index 133d3f28..9b7784f1 100644
--- a/lib/microwaveprop/weather/hrrr_client.ex
+++ b/lib/microwaveprop/weather/hrrr_client.ex
@@ -242,14 +242,22 @@ defmodule Microwaveprop.Weather.HrrrClient do
   end
 
   defp req_options do
-    defaults = [retry: :transient, max_retries: 3, retry_delay: &retry_delay/1]
+    defaults = [retry: &retry?/2, max_retries: 5, retry_delay: &retry_delay/1]
     overrides = Application.get_env(:microwaveprop, :hrrr_req_options, [])
     Keyword.merge(defaults, overrides)
   end
 
+  defp retry?(_request, response) do
+    case response do
+      %Req.Response{status: status} when status in [429, 500, 502, 503, 504] -> true
+      %{__exception__: true} -> true
+      _ -> false
+    end
+  end
+
   defp retry_delay(n) do
     base = Integer.pow(2, n) * 1_000
-    jitter = :rand.uniform(500)
+    jitter = :rand.uniform(1_000)
     base + jitter
   end
 end
diff --git a/lib/microwaveprop/weather/iem_client.ex b/lib/microwaveprop/weather/iem_client.ex
index 17d37859..06c86e99 100644
--- a/lib/microwaveprop/weather/iem_client.ex
+++ b/lib/microwaveprop/weather/iem_client.ex
@@ -74,14 +74,22 @@ defmodule Microwaveprop.Weather.IemClient do
   end
 
   defp req_options do
-    defaults = [retry: :transient, max_retries: 3, retry_delay: &retry_delay/1]
+    defaults = [retry: &retry?/2, max_retries: 5, retry_delay: &retry_delay/1]
     overrides = Application.get_env(:microwaveprop, :iem_req_options, [])
     Keyword.merge(defaults, overrides)
   end
 
+  defp retry?(_request, response) do
+    case response do
+      %Req.Response{status: status} when status in [429, 500, 502, 503, 504] -> true
+      %{__exception__: true} -> true
+      _ -> false
+    end
+  end
+
   defp retry_delay(n) do
     base = Integer.pow(2, n) * 1_000
-    jitter = :rand.uniform(500)
+    jitter = :rand.uniform(1_000)
     base + jitter
   end