diff --git a/lib/microwaveprop/workers/iemre_fetch_worker.ex b/lib/microwaveprop/workers/iemre_fetch_worker.ex index 98b34274..75630e65 100644 --- a/lib/microwaveprop/workers/iemre_fetch_worker.ex +++ b/lib/microwaveprop/workers/iemre_fetch_worker.ex @@ -25,7 +25,9 @@ defmodule Microwaveprop.Workers.IemreFetchWorker do case IemClient.fetch_iemre(lat, lon, date) do {:ok, []} -> - Logger.info("IEMRE returned empty data for #{lat},#{lon} @ #{date_str}") + # Store stub so this lat/lon/date isn't retried on future backfills + Weather.upsert_iemre_observation(%{lat: lat, lon: lon, date: date, hourly: []}) + Logger.info("IEMRE: no data available for #{lat},#{lon} @ #{date_str}, stored stub") :ok {:ok, data} -> diff --git a/lib/microwaveprop/workers/weather_fetch_worker.ex b/lib/microwaveprop/workers/weather_fetch_worker.ex index 433ffba7..c773fbb7 100644 --- a/lib/microwaveprop/workers/weather_fetch_worker.ex +++ b/lib/microwaveprop/workers/weather_fetch_worker.ex @@ -10,6 +10,7 @@ defmodule Microwaveprop.Workers.WeatherFetchWorker do alias Microwaveprop.Weather.IemClient alias Microwaveprop.Weather.SoundingParams alias Microwaveprop.Weather.Station + alias Microwaveprop.Weather.SurfaceObservation require Logger @@ -50,6 +51,15 @@ defmodule Microwaveprop.Workers.WeatherFetchWorker do if row.observed_at, do: Weather.upsert_surface_observation(station, row) end) + # Store stub so this station/window isn't retried on future backfills + if count == 0 do + midpoint = DateTime.add(start_dt, div(DateTime.diff(end_dt, start_dt), 2)) + + %SurfaceObservation{} + |> SurfaceObservation.changeset(%{station_id: station.id, observed_at: midpoint}) + |> Repo.insert(on_conflict: :nothing, conflict_target: [:station_id, :observed_at]) + end + Logger.info("WeatherFetch ASOS: #{station_code} ingested #{count} observations") Phoenix.PubSub.broadcast( @@ -132,7 +142,14 @@ defmodule Microwaveprop.Workers.WeatherFetchWorker do :ok {:ok, []} -> - Logger.info("WeatherFetch RAOB: #{station_code} returned no data for #{sounding_time_str}") + # Store stub sounding so this station/time isn't retried on future backfills + Weather.upsert_sounding(station, %{ + observed_at: sounding_time, + profile: [], + level_count: 0 + }) + + Logger.info("WeatherFetch RAOB: #{station_code} no data for #{sounding_time_str}, stored stub") :ok {:error, reason} -> diff --git a/test/microwaveprop/workers/iemre_fetch_worker_test.exs b/test/microwaveprop/workers/iemre_fetch_worker_test.exs index 2f65d35c..0de0d747 100644 --- a/test/microwaveprop/workers/iemre_fetch_worker_test.exs +++ b/test/microwaveprop/workers/iemre_fetch_worker_test.exs @@ -60,7 +60,7 @@ defmodule Microwaveprop.Workers.IemreFetchWorkerTest do assert length(obs.hourly) == 2 end - test "returns :ok on empty data" do + test "stores stub on empty data so backfill doesn't retry" do Req.Test.stub(IemClient, fn conn -> Req.Test.json(conn, %{"data" => []}) end) @@ -74,7 +74,12 @@ defmodule Microwaveprop.Workers.IemreFetchWorkerTest do } assert :ok = IemreFetchWorker.perform(job) - assert Repo.aggregate(IemreObservation, :count) == 0 + assert Repo.aggregate(IemreObservation, :count) == 1 + + obs = Repo.one!(IemreObservation) + assert obs.hourly == [] + assert obs.lat == 32.875 + assert obs.lon == -97.0 end test "retries on transient server error" do diff --git a/test/microwaveprop/workers/weather_fetch_worker_test.exs b/test/microwaveprop/workers/weather_fetch_worker_test.exs index f6d77b9b..87adb021 100644 --- a/test/microwaveprop/workers/weather_fetch_worker_test.exs +++ b/test/microwaveprop/workers/weather_fetch_worker_test.exs @@ -122,6 +122,33 @@ defmodule Microwaveprop.Workers.WeatherFetchWorkerTest do assert {:error, "IEM ASOS HTTP 503"} = WeatherFetchWorker.perform(%Oban.Job{args: args}) end + test "stores stub observation on empty ASOS data so backfill doesn't retry", %{station: station} do + Req.Test.stub(IemClient, fn conn -> + body = """ + #DEBUG, + station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1,p01i,wxcodes + """ + + Plug.Conn.send_resp(conn, 200, body) + end) + + args = %{ + "fetch_type" => "asos", + "station_id" => station.id, + "station_code" => "KORD", + "start_dt" => "2026-03-28T16:53:00Z", + "end_dt" => "2026-03-28T20:53:00Z" + } + + assert :ok = WeatherFetchWorker.perform(%Oban.Job{args: args}) + assert Repo.aggregate(SurfaceObservation, :count) == 1 + + obs = Repo.one!(SurfaceObservation) + assert obs.station_id == station.id + # All weather fields nil — it's a stub + assert is_nil(obs.temp_f) + end + test "returns :ok when station no longer exists" do Req.Test.stub(IemClient, fn conn -> Plug.Conn.send_resp(conn, 500, "Should not be called") @@ -185,7 +212,7 @@ defmodule Microwaveprop.Workers.WeatherFetchWorkerTest do assert Repo.aggregate(Sounding, :count) == 1 end - test "handles empty profiles", %{station: station} do + test "stores stub sounding on empty profiles so backfill doesn't retry", %{station: station} do Req.Test.stub(IemClient, fn conn -> Req.Test.json(conn, %{"profiles" => []}) end) @@ -198,7 +225,12 @@ defmodule Microwaveprop.Workers.WeatherFetchWorkerTest do } assert :ok = WeatherFetchWorker.perform(%Oban.Job{args: args}) - assert Repo.aggregate(Sounding, :count) == 0 + assert Repo.aggregate(Sounding, :count) == 1 + + sounding = Repo.one!(Sounding) + assert sounding.profile == [] + assert sounding.level_count == 0 + assert sounding.station_id == station.id end test "returns error on HTTP failure", %{station: station} do