Increase Oban retry attempts and backoff for external API workers

20 attempts with exponential backoff capped at 6 hours ensures
data is eventually fetched even during extended rate limiting or
API outages (~3 days of retrying before giving up).

lib/microwaveprop/workers/hrrr_fetch_worker.ex

@@ -1,11 +1,17 @@
 defmodule Microwaveprop.Workers.HrrrFetchWorker do
   @moduledoc false
-  use Oban.Worker, queue: :hrrr, max_attempts: 3
+  use Oban.Worker, queue: :hrrr, max_attempts: 20
 
   alias Microwaveprop.Weather
   alias Microwaveprop.Weather.HrrrClient
   alias Microwaveprop.Weather.SoundingParams
 
+  @impl Oban.Worker
+  def backoff(%Oban.Job{attempt: attempt}) do
+    # Exponential backoff: 2m, 4m, 8m, 16m, 32m, 1h, 2h, 4h, 6h, 6h, ...
+    min(120 * Integer.pow(2, attempt - 1), _six_hours = 21_600)
+  end
+
   @impl Oban.Worker
   def perform(%Oban.Job{args: args}) do
     %{"lat" => raw_lat, "lon" => raw_lon, "valid_time" => valid_time_str} = args

lib/microwaveprop/workers/terrain_profile_worker.ex

@@ -1,12 +1,17 @@
 defmodule Microwaveprop.Workers.TerrainProfileWorker do
   @moduledoc false
-  use Oban.Worker, queue: :terrain, max_attempts: 3
+  use Oban.Worker, queue: :terrain, max_attempts: 20
 
   alias Microwaveprop.Radio
   alias Microwaveprop.Terrain
   alias Microwaveprop.Terrain.ElevationClient
   alias Microwaveprop.Terrain.TerrainAnalysis
 
+  @impl Oban.Worker
+  def backoff(%Oban.Job{attempt: attempt}) do
+    min(120 * Integer.pow(2, attempt - 1), _six_hours = 21_600)
+  end
+
   @impl Oban.Worker
   def perform(%Oban.Job{args: %{"qso_id" => qso_id}}) do
     if Terrain.has_terrain_profile?(qso_id) do

lib/microwaveprop/workers/weather_fetch_worker.ex

@@ -1,6 +1,6 @@
 defmodule Microwaveprop.Workers.WeatherFetchWorker do
   @moduledoc false
-  use Oban.Worker, queue: :weather, max_attempts: 5
+  use Oban.Worker, queue: :weather, max_attempts: 20
 
   alias Microwaveprop.Repo
   alias Microwaveprop.Weather
@@ -8,6 +8,11 @@ defmodule Microwaveprop.Workers.WeatherFetchWorker do
   alias Microwaveprop.Weather.SoundingParams
   alias Microwaveprop.Weather.Station
 
+  @impl Oban.Worker
+  def backoff(%Oban.Job{attempt: attempt}) do
+    min(120 * Integer.pow(2, attempt - 1), _six_hours = 21_600)
+  end
+
   @impl Oban.Worker
   def perform(%Oban.Job{args: %{"fetch_type" => "asos"} = args}) do
     %{

lib/microwaveprop_web/live/qso_live/index.ex

@@ -64,8 +64,8 @@ defmodule MicrowavepropWeb.QsoLive.Index do
         QSOs
         <:subtitle>{@total_entries} contacts</:subtitle>
         <:actions>
-          <.link navigate={~p"/submit"} class="btn btn-sm btn-primary">
+          <.link navigate={~p"/submit"} class="btn btn-primary">
-            <.icon name="hero-plus" class="w-4 h-4" /> Submit QSO
+            <.icon name="hero-plus" class="w-5 h-5" /> Submit QSO
           </.link>
         </:actions>
       </.header>