prop/test/microwaveprop/workers/weather_fetch_worker_test.exs
Graham McIntire 646467cf4d Store stub records on empty weather fetches to prevent infinite backfill retries
When ASOS/RAOB/IEMRE APIs return empty data, store a stub record so
dedup checks (has_surface_observations?, has_sounding?, has_iemre_observation?)
see coverage and don't recreate the same jobs on future backfill runs.
2026-04-06 16:10:39 -05:00

262 lines
7.8 KiB
Elixir

defmodule Microwaveprop.Workers.WeatherFetchWorkerTest do
use Microwaveprop.DataCase, async: true
alias Microwaveprop.Weather
alias Microwaveprop.Weather.IemClient
alias Microwaveprop.Weather.Sounding
alias Microwaveprop.Weather.SurfaceObservation
alias Microwaveprop.Workers.WeatherFetchWorker
defp create_asos_station(_) do
{:ok, station} =
Weather.find_or_create_station(%{
station_code: "KORD",
station_type: "asos",
name: "Chicago O'Hare",
lat: 41.98,
lon: -87.9
})
%{station: station}
end
defp create_sounding_station(_) do
{:ok, station} =
Weather.find_or_create_station(%{
station_code: "72451",
station_type: "sounding",
name: "Davenport",
lat: 41.61,
lon: -90.58
})
%{station: station}
end
defp asos_csv_body do
"""
#DEBUG,
station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1,p01i,wxcodes
KORD,2026-03-28 18:53,45.0,30.0,55.0,10.0,270,1013.5,29.92,CLR,null,null
KORD,2026-03-28 19:53,46.0,31.0,54.0,12.0,280,1013.2,29.91,FEW,0.02,RA
"""
end
defp raob_json_body do
%{
"profiles" => [
%{
"valid" => "2026-03-28 12:00:00+00:00",
"profile" => [
%{"pres" => 1000.0, "hght" => 200.0, "tmpc" => 15.0, "dwpc" => 10.0, "drct" => 180, "sknt" => 5},
%{"pres" => 925.0, "hght" => 800.0, "tmpc" => 10.0, "dwpc" => 5.0, "drct" => 200, "sknt" => 10},
%{"pres" => 850.0, "hght" => 1500.0, "tmpc" => 5.0, "dwpc" => 0.0, "drct" => 220, "sknt" => 15},
%{"pres" => 700.0, "hght" => 3000.0, "tmpc" => -5.0, "dwpc" => -10.0, "drct" => 240, "sknt" => 25},
%{"pres" => 500.0, "hght" => 5500.0, "tmpc" => -20.0, "dwpc" => -30.0, "drct" => 260, "sknt" => 40}
]
}
]
}
end
describe "ASOS perform/1" do
setup :create_asos_station
test "fetches and upserts surface observations", %{station: station} do
Req.Test.stub(IemClient, fn conn ->
Req.Test.text(conn, asos_csv_body())
end)
args = %{
"fetch_type" => "asos",
"station_id" => station.id,
"station_code" => "KORD",
"start_dt" => "2026-03-28T16:53:00Z",
"end_dt" => "2026-03-28T20:53:00Z"
}
assert :ok = WeatherFetchWorker.perform(%Oban.Job{args: args})
assert Repo.aggregate(SurfaceObservation, :count) == 2
end
test "skips fetch when observations already exist", %{station: station} do
start_dt = ~U[2026-03-28 16:53:00Z]
end_dt = ~U[2026-03-28 20:53:00Z]
Weather.upsert_surface_observation(station, %{
observed_at: ~U[2026-03-28 18:53:00Z],
temp_f: 45.0,
dewpoint_f: 30.0
})
# Stub should NOT be called — if it is, it will raise
Req.Test.stub(IemClient, fn conn ->
Plug.Conn.send_resp(conn, 500, "Should not be called")
end)
args = %{
"fetch_type" => "asos",
"station_id" => station.id,
"station_code" => "KORD",
"start_dt" => DateTime.to_iso8601(start_dt),
"end_dt" => DateTime.to_iso8601(end_dt)
}
assert :ok = WeatherFetchWorker.perform(%Oban.Job{args: args})
assert Repo.aggregate(SurfaceObservation, :count) == 1
end
test "returns error on HTTP failure to trigger Oban retry", %{station: station} do
Req.Test.stub(IemClient, fn conn ->
Plug.Conn.send_resp(conn, 503, "Service Unavailable")
end)
args = %{
"fetch_type" => "asos",
"station_id" => station.id,
"station_code" => "KORD",
"start_dt" => "2026-03-28T16:53:00Z",
"end_dt" => "2026-03-28T20:53:00Z"
}
assert {:error, "IEM ASOS HTTP 503"} = WeatherFetchWorker.perform(%Oban.Job{args: args})
end
test "stores stub observation on empty ASOS data so backfill doesn't retry", %{station: station} do
Req.Test.stub(IemClient, fn conn ->
body = """
#DEBUG,
station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1,p01i,wxcodes
"""
Plug.Conn.send_resp(conn, 200, body)
end)
args = %{
"fetch_type" => "asos",
"station_id" => station.id,
"station_code" => "KORD",
"start_dt" => "2026-03-28T16:53:00Z",
"end_dt" => "2026-03-28T20:53:00Z"
}
assert :ok = WeatherFetchWorker.perform(%Oban.Job{args: args})
assert Repo.aggregate(SurfaceObservation, :count) == 1
obs = Repo.one!(SurfaceObservation)
assert obs.station_id == station.id
# All weather fields nil — it's a stub
assert is_nil(obs.temp_f)
end
test "returns :ok when station no longer exists" do
Req.Test.stub(IemClient, fn conn ->
Plug.Conn.send_resp(conn, 500, "Should not be called")
end)
args = %{
"fetch_type" => "asos",
"station_id" => Ecto.UUID.generate(),
"station_code" => "GONE",
"start_dt" => "2026-03-28T16:53:00Z",
"end_dt" => "2026-03-28T20:53:00Z"
}
assert :ok = WeatherFetchWorker.perform(%Oban.Job{args: args})
end
end
describe "RAOB perform/1" do
setup :create_sounding_station
test "fetches and upserts sounding with derived params", %{station: station} do
Req.Test.stub(IemClient, fn conn ->
Req.Test.json(conn, raob_json_body())
end)
args = %{
"fetch_type" => "raob",
"station_id" => station.id,
"station_code" => "72451",
"sounding_time" => "2026-03-28T12:00:00Z"
}
assert :ok = WeatherFetchWorker.perform(%Oban.Job{args: args})
assert Repo.aggregate(Sounding, :count) == 1
sounding = Repo.one(Sounding)
assert sounding.station_id == station.id
assert sounding.level_count == 5
assert sounding.surface_temp_c == 15.0
end
test "skips when sounding already exists", %{station: station} do
Weather.upsert_sounding(station, %{
observed_at: ~U[2026-03-28 12:00:00Z],
profile: [],
level_count: 0
})
Req.Test.stub(IemClient, fn conn ->
Plug.Conn.send_resp(conn, 500, "Should not be called")
end)
args = %{
"fetch_type" => "raob",
"station_id" => station.id,
"station_code" => "72451",
"sounding_time" => "2026-03-28T12:00:00Z"
}
assert :ok = WeatherFetchWorker.perform(%Oban.Job{args: args})
assert Repo.aggregate(Sounding, :count) == 1
end
test "stores stub sounding on empty profiles so backfill doesn't retry", %{station: station} do
Req.Test.stub(IemClient, fn conn ->
Req.Test.json(conn, %{"profiles" => []})
end)
args = %{
"fetch_type" => "raob",
"station_id" => station.id,
"station_code" => "72451",
"sounding_time" => "2026-03-28T12:00:00Z"
}
assert :ok = WeatherFetchWorker.perform(%Oban.Job{args: args})
assert Repo.aggregate(Sounding, :count) == 1
sounding = Repo.one!(Sounding)
assert sounding.profile == []
assert sounding.level_count == 0
assert sounding.station_id == station.id
end
test "returns error on HTTP failure", %{station: station} do
Req.Test.stub(IemClient, fn conn ->
Plug.Conn.send_resp(conn, 503, "Service Unavailable")
end)
args = %{
"fetch_type" => "raob",
"station_id" => station.id,
"station_code" => "72451",
"sounding_time" => "2026-03-28T12:00:00Z"
}
assert {:error, "IEM RAOB HTTP 503"} = WeatherFetchWorker.perform(%Oban.Job{args: args})
end
test "returns :ok when station no longer exists" do
args = %{
"fetch_type" => "raob",
"station_id" => Ecto.UUID.generate(),
"station_code" => "GONE",
"sounding_time" => "2026-03-28T12:00:00Z"
}
assert :ok = WeatherFetchWorker.perform(%Oban.Job{args: args})
end
end
end