From 4468fb3b617ea9ba208fb156e719fd8b960f4b42 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Sun, 29 Mar 2026 17:03:56 -0500 Subject: [PATCH] Increase Oban retry attempts and backoff for external API workers 20 attempts with exponential backoff capped at 6 hours ensures data is eventually fetched even during extended rate limiting or API outages (~3 days of retrying before giving up). --- lib/microwaveprop/workers/hrrr_fetch_worker.ex | 8 +++++++- lib/microwaveprop/workers/terrain_profile_worker.ex | 7 ++++++- lib/microwaveprop/workers/weather_fetch_worker.ex | 7 ++++++- lib/microwaveprop_web/live/qso_live/index.ex | 4 ++-- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/lib/microwaveprop/workers/hrrr_fetch_worker.ex b/lib/microwaveprop/workers/hrrr_fetch_worker.ex index fb5f295e..783f50f1 100644 --- a/lib/microwaveprop/workers/hrrr_fetch_worker.ex +++ b/lib/microwaveprop/workers/hrrr_fetch_worker.ex @@ -1,11 +1,17 @@ defmodule Microwaveprop.Workers.HrrrFetchWorker do @moduledoc false - use Oban.Worker, queue: :hrrr, max_attempts: 3 + use Oban.Worker, queue: :hrrr, max_attempts: 20 alias Microwaveprop.Weather alias Microwaveprop.Weather.HrrrClient alias Microwaveprop.Weather.SoundingParams + @impl Oban.Worker + def backoff(%Oban.Job{attempt: attempt}) do + # Exponential backoff: 2m, 4m, 8m, 16m, 32m, 1h, 2h, 4h, 6h, 6h, ... + min(120 * Integer.pow(2, attempt - 1), _six_hours = 21_600) + end + @impl Oban.Worker def perform(%Oban.Job{args: args}) do %{"lat" => raw_lat, "lon" => raw_lon, "valid_time" => valid_time_str} = args diff --git a/lib/microwaveprop/workers/terrain_profile_worker.ex b/lib/microwaveprop/workers/terrain_profile_worker.ex index 2978be62..d1804aef 100644 --- a/lib/microwaveprop/workers/terrain_profile_worker.ex +++ b/lib/microwaveprop/workers/terrain_profile_worker.ex @@ -1,12 +1,17 @@ defmodule Microwaveprop.Workers.TerrainProfileWorker do @moduledoc false - use Oban.Worker, queue: :terrain, max_attempts: 3 + use Oban.Worker, queue: :terrain, max_attempts: 20 alias Microwaveprop.Radio alias Microwaveprop.Terrain alias Microwaveprop.Terrain.ElevationClient alias Microwaveprop.Terrain.TerrainAnalysis + @impl Oban.Worker + def backoff(%Oban.Job{attempt: attempt}) do + min(120 * Integer.pow(2, attempt - 1), _six_hours = 21_600) + end + @impl Oban.Worker def perform(%Oban.Job{args: %{"qso_id" => qso_id}}) do if Terrain.has_terrain_profile?(qso_id) do diff --git a/lib/microwaveprop/workers/weather_fetch_worker.ex b/lib/microwaveprop/workers/weather_fetch_worker.ex index a2f197c7..ab0ae3c1 100644 --- a/lib/microwaveprop/workers/weather_fetch_worker.ex +++ b/lib/microwaveprop/workers/weather_fetch_worker.ex @@ -1,6 +1,6 @@ defmodule Microwaveprop.Workers.WeatherFetchWorker do @moduledoc false - use Oban.Worker, queue: :weather, max_attempts: 5 + use Oban.Worker, queue: :weather, max_attempts: 20 alias Microwaveprop.Repo alias Microwaveprop.Weather @@ -8,6 +8,11 @@ defmodule Microwaveprop.Workers.WeatherFetchWorker do alias Microwaveprop.Weather.SoundingParams alias Microwaveprop.Weather.Station + @impl Oban.Worker + def backoff(%Oban.Job{attempt: attempt}) do + min(120 * Integer.pow(2, attempt - 1), _six_hours = 21_600) + end + @impl Oban.Worker def perform(%Oban.Job{args: %{"fetch_type" => "asos"} = args}) do %{ diff --git a/lib/microwaveprop_web/live/qso_live/index.ex b/lib/microwaveprop_web/live/qso_live/index.ex index 7b987589..1648dff3 100644 --- a/lib/microwaveprop_web/live/qso_live/index.ex +++ b/lib/microwaveprop_web/live/qso_live/index.ex @@ -64,8 +64,8 @@ defmodule MicrowavepropWeb.QsoLive.Index do QSOs <:subtitle>{@total_entries} contacts <:actions> - <.link navigate={~p"/submit"} class="btn btn-sm btn-primary"> - <.icon name="hero-plus" class="w-4 h-4" /> Submit QSO + <.link navigate={~p"/submit"} class="btn btn-primary"> + <.icon name="hero-plus" class="w-5 h-5" /> Submit QSO