Graham McIntire
74f62834e3
Replaces per-QSO-endpoint asos_batch granularity with per-station-day granularity. Each contact in the same UTC day near the same station now produces identical unique-args and collapses to ONE Oban job, so the backlog shrinks as cross-contact duplicates dedup. Each fetch also covers a full 24h window (vs the previous 4h) so one IEM request returns ~24 hourly observations instead of ~1-2. The effective bytes-per-request is 10x higher, amortizing the IemRateLimiter gap + 429 retry tail across far more data. Changes: - Weather.station_day_covered?/2 + station_day_pairs_covered/1 — cheap UTC-day coverage checks for the enqueuer + worker skip paths. - WeatherFetchWorker gains an "asos_day" clause that fetches the full UTC day for one station and upserts. Skips if the day is already covered in the DB. - ContactWeatherEnqueueWorker.build_asos_jobs/3 now emits one asos_day job per (station_id, UTC date) pair. ±2h windows crossing midnight enumerate both dates. - Microwaveprop.Weather.RebatchAsos.to_day_jobs/1 migrates any already-queued asos/asos_batch jobs into the new shape. Idempotent; supports dry_run. - asos_batch worker clause retained for jobs already in flight at deploy time. 2835 tests + credo green.
392 lines
12 KiB
Elixir
392 lines
12 KiB
Elixir
defmodule Microwaveprop.Workers.WeatherFetchWorkerTest do
|
|
use Microwaveprop.DataCase, async: true
|
|
|
|
alias Microwaveprop.Weather
|
|
alias Microwaveprop.Weather.IemClient
|
|
alias Microwaveprop.Weather.Sounding
|
|
alias Microwaveprop.Weather.SurfaceObservation
|
|
alias Microwaveprop.Workers.WeatherFetchWorker
|
|
|
|
defp create_asos_station(_) do
|
|
{:ok, station} =
|
|
Weather.find_or_create_station(%{
|
|
station_code: "KORD",
|
|
station_type: "asos",
|
|
name: "Chicago O'Hare",
|
|
lat: 41.98,
|
|
lon: -87.9
|
|
})
|
|
|
|
%{station: station}
|
|
end
|
|
|
|
defp create_sounding_station(_) do
|
|
{:ok, station} =
|
|
Weather.find_or_create_station(%{
|
|
station_code: "72451",
|
|
station_type: "sounding",
|
|
name: "Davenport",
|
|
lat: 41.61,
|
|
lon: -90.58
|
|
})
|
|
|
|
%{station: station}
|
|
end
|
|
|
|
defp asos_csv_body do
|
|
"""
|
|
#DEBUG,
|
|
station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1,p01i,wxcodes
|
|
KORD,2026-03-28 18:53,45.0,30.0,55.0,10.0,270,1013.5,29.92,CLR,null,null
|
|
KORD,2026-03-28 19:53,46.0,31.0,54.0,12.0,280,1013.2,29.91,FEW,0.02,RA
|
|
"""
|
|
end
|
|
|
|
defp raob_json_body do
|
|
%{
|
|
"profiles" => [
|
|
%{
|
|
"valid" => "2026-03-28 12:00:00+00:00",
|
|
"profile" => [
|
|
%{"pres" => 1000.0, "hght" => 200.0, "tmpc" => 15.0, "dwpc" => 10.0, "drct" => 180, "sknt" => 5},
|
|
%{"pres" => 925.0, "hght" => 800.0, "tmpc" => 10.0, "dwpc" => 5.0, "drct" => 200, "sknt" => 10},
|
|
%{"pres" => 850.0, "hght" => 1500.0, "tmpc" => 5.0, "dwpc" => 0.0, "drct" => 220, "sknt" => 15},
|
|
%{"pres" => 700.0, "hght" => 3000.0, "tmpc" => -5.0, "dwpc" => -10.0, "drct" => 240, "sknt" => 25},
|
|
%{"pres" => 500.0, "hght" => 5500.0, "tmpc" => -20.0, "dwpc" => -30.0, "drct" => 260, "sknt" => 40}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
end
|
|
|
|
describe "ASOS day perform/1" do
|
|
test "fetches full UTC day for one station and upserts returned rows" do
|
|
{:ok, station} =
|
|
Weather.find_or_create_station(%{
|
|
station_code: "KDFW",
|
|
station_type: "asos",
|
|
name: "DFW",
|
|
lat: 32.9,
|
|
lon: -97.0
|
|
})
|
|
|
|
Req.Test.stub(IemClient, fn conn ->
|
|
# Assert the worker requested the whole UTC day 2021-08-21.
|
|
qs = URI.decode_query(conn.query_string)
|
|
assert qs["year1"] == "2021"
|
|
assert qs["month1"] == "8"
|
|
assert qs["day1"] == "21"
|
|
assert qs["year2"] == "2021"
|
|
assert qs["day2"] == "21"
|
|
|
|
body = """
|
|
station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1,p01i,wxcodes
|
|
KDFW,2021-08-21 00:53,80.0,70.0,71.0,8,180,1012.0,29.90,CLR,null,null
|
|
KDFW,2021-08-21 12:53,85.0,68.0,58.0,10,180,1013.0,29.92,FEW,null,null
|
|
KDFW,2021-08-21 23:53,78.0,69.0,73.0,6,160,1011.0,29.89,SCT,null,null
|
|
"""
|
|
|
|
Req.Test.text(conn, body)
|
|
end)
|
|
|
|
args = %{
|
|
"fetch_type" => "asos_day",
|
|
"station_id" => station.id,
|
|
"station_code" => "KDFW",
|
|
"date" => "2021-08-21"
|
|
}
|
|
|
|
assert :ok = WeatherFetchWorker.perform(%Oban.Job{args: args})
|
|
assert Repo.aggregate(SurfaceObservation, :count) == 3
|
|
end
|
|
|
|
test "skips fetch when day is already covered" do
|
|
{:ok, station} =
|
|
Weather.find_or_create_station(%{
|
|
station_code: "KDFW",
|
|
station_type: "asos",
|
|
name: "DFW",
|
|
lat: 32.9,
|
|
lon: -97.0
|
|
})
|
|
|
|
# Seed a prior observation inside the target day.
|
|
Weather.upsert_surface_observation(station, %{
|
|
observed_at: ~U[2021-08-21 12:00:00Z],
|
|
temp_f: 85.0
|
|
})
|
|
|
|
Req.Test.stub(IemClient, fn conn ->
|
|
Plug.Conn.send_resp(conn, 500, "should not be called")
|
|
end)
|
|
|
|
args = %{
|
|
"fetch_type" => "asos_day",
|
|
"station_id" => station.id,
|
|
"station_code" => "KDFW",
|
|
"date" => "2021-08-21"
|
|
}
|
|
|
|
assert :ok = WeatherFetchWorker.perform(%Oban.Job{args: args})
|
|
# Still only the seeded row — no fetch happened.
|
|
assert Repo.aggregate(SurfaceObservation, :count) == 1
|
|
end
|
|
end
|
|
|
|
describe "ASOS batch perform/1" do
|
|
test "fetches one URL covering all stations and ingests per-station rows" do
|
|
{:ok, s1} =
|
|
Weather.find_or_create_station(%{
|
|
station_code: "KDFW",
|
|
station_type: "asos",
|
|
name: "DFW",
|
|
lat: 32.9,
|
|
lon: -97.0
|
|
})
|
|
|
|
{:ok, s2} =
|
|
Weather.find_or_create_station(%{
|
|
station_code: "KFTW",
|
|
station_type: "asos",
|
|
name: "FTW",
|
|
lat: 32.82,
|
|
lon: -97.36
|
|
})
|
|
|
|
{:ok, s3} =
|
|
Weather.find_or_create_station(%{
|
|
station_code: "KAFW",
|
|
station_type: "asos",
|
|
name: "AFW",
|
|
lat: 32.98,
|
|
lon: -97.32
|
|
})
|
|
|
|
# One HTTP call returns rows for s1 + s2; s3 is absent → should stub.
|
|
Req.Test.stub(IemClient, fn conn ->
|
|
body = """
|
|
#DEBUG
|
|
station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1,p01i,wxcodes
|
|
KDFW,2026-03-28 18:53,75.0,55.0,49.0,12,180,1013.2,29.92,SCT,null,null
|
|
KDFW,2026-03-28 19:53,76.0,54.0,46.0,10,190,1013.0,29.91,FEW,null,null
|
|
KFTW,2026-03-28 18:55,73.0,53.0,49.0,10,190,1013.0,29.91,FEW,null,null
|
|
"""
|
|
|
|
Req.Test.text(conn, body)
|
|
end)
|
|
|
|
args = %{
|
|
"fetch_type" => "asos_batch",
|
|
"station_ids" => [s1.id, s2.id, s3.id],
|
|
"station_codes" => ["KDFW", "KFTW", "KAFW"],
|
|
"start_dt" => "2026-03-28T16:00:00Z",
|
|
"end_dt" => "2026-03-28T20:00:00Z"
|
|
}
|
|
|
|
assert :ok = WeatherFetchWorker.perform(%Oban.Job{args: args})
|
|
# 2 real rows for KDFW + 1 for KFTW + 1 stub for KAFW = 4
|
|
assert Repo.aggregate(SurfaceObservation, :count) == 4
|
|
end
|
|
end
|
|
|
|
describe "ASOS perform/1" do
|
|
setup :create_asos_station
|
|
|
|
test "fetches and upserts surface observations", %{station: station} do
|
|
Req.Test.stub(IemClient, fn conn ->
|
|
Req.Test.text(conn, asos_csv_body())
|
|
end)
|
|
|
|
args = %{
|
|
"fetch_type" => "asos",
|
|
"station_id" => station.id,
|
|
"station_code" => "KORD",
|
|
"start_dt" => "2026-03-28T16:53:00Z",
|
|
"end_dt" => "2026-03-28T20:53:00Z"
|
|
}
|
|
|
|
assert :ok = WeatherFetchWorker.perform(%Oban.Job{args: args})
|
|
assert Repo.aggregate(SurfaceObservation, :count) == 2
|
|
end
|
|
|
|
test "skips fetch when observations already exist", %{station: station} do
|
|
start_dt = ~U[2026-03-28 16:53:00Z]
|
|
end_dt = ~U[2026-03-28 20:53:00Z]
|
|
|
|
Weather.upsert_surface_observation(station, %{
|
|
observed_at: ~U[2026-03-28 18:53:00Z],
|
|
temp_f: 45.0,
|
|
dewpoint_f: 30.0
|
|
})
|
|
|
|
# Stub should NOT be called — if it is, it will raise
|
|
Req.Test.stub(IemClient, fn conn ->
|
|
Plug.Conn.send_resp(conn, 500, "Should not be called")
|
|
end)
|
|
|
|
args = %{
|
|
"fetch_type" => "asos",
|
|
"station_id" => station.id,
|
|
"station_code" => "KORD",
|
|
"start_dt" => DateTime.to_iso8601(start_dt),
|
|
"end_dt" => DateTime.to_iso8601(end_dt)
|
|
}
|
|
|
|
assert :ok = WeatherFetchWorker.perform(%Oban.Job{args: args})
|
|
assert Repo.aggregate(SurfaceObservation, :count) == 1
|
|
end
|
|
|
|
test "returns error on HTTP failure to trigger Oban retry", %{station: station} do
|
|
Req.Test.stub(IemClient, fn conn ->
|
|
Plug.Conn.send_resp(conn, 503, "Service Unavailable")
|
|
end)
|
|
|
|
args = %{
|
|
"fetch_type" => "asos",
|
|
"station_id" => station.id,
|
|
"station_code" => "KORD",
|
|
"start_dt" => "2026-03-28T16:53:00Z",
|
|
"end_dt" => "2026-03-28T20:53:00Z"
|
|
}
|
|
|
|
assert {:error, "IEM ASOS HTTP 503"} = WeatherFetchWorker.perform(%Oban.Job{args: args})
|
|
end
|
|
|
|
test "stores stub observation on empty ASOS data so backfill doesn't retry", %{station: station} do
|
|
Req.Test.stub(IemClient, fn conn ->
|
|
body = """
|
|
#DEBUG,
|
|
station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1,p01i,wxcodes
|
|
"""
|
|
|
|
Plug.Conn.send_resp(conn, 200, body)
|
|
end)
|
|
|
|
args = %{
|
|
"fetch_type" => "asos",
|
|
"station_id" => station.id,
|
|
"station_code" => "KORD",
|
|
"start_dt" => "2026-03-28T16:53:00Z",
|
|
"end_dt" => "2026-03-28T20:53:00Z"
|
|
}
|
|
|
|
assert :ok = WeatherFetchWorker.perform(%Oban.Job{args: args})
|
|
assert Repo.aggregate(SurfaceObservation, :count) == 1
|
|
|
|
obs = Repo.one!(SurfaceObservation)
|
|
assert obs.station_id == station.id
|
|
# All weather fields nil — it's a stub
|
|
assert is_nil(obs.temp_f)
|
|
end
|
|
|
|
test "returns :ok when station no longer exists" do
|
|
Req.Test.stub(IemClient, fn conn ->
|
|
Plug.Conn.send_resp(conn, 500, "Should not be called")
|
|
end)
|
|
|
|
args = %{
|
|
"fetch_type" => "asos",
|
|
"station_id" => Ecto.UUID.generate(),
|
|
"station_code" => "GONE",
|
|
"start_dt" => "2026-03-28T16:53:00Z",
|
|
"end_dt" => "2026-03-28T20:53:00Z"
|
|
}
|
|
|
|
assert :ok = WeatherFetchWorker.perform(%Oban.Job{args: args})
|
|
end
|
|
end
|
|
|
|
describe "RAOB perform/1" do
|
|
setup :create_sounding_station
|
|
|
|
test "fetches and upserts sounding with derived params", %{station: station} do
|
|
Req.Test.stub(IemClient, fn conn ->
|
|
Req.Test.json(conn, raob_json_body())
|
|
end)
|
|
|
|
args = %{
|
|
"fetch_type" => "raob",
|
|
"station_id" => station.id,
|
|
"station_code" => "72451",
|
|
"sounding_time" => "2026-03-28T12:00:00Z"
|
|
}
|
|
|
|
assert :ok = WeatherFetchWorker.perform(%Oban.Job{args: args})
|
|
assert Repo.aggregate(Sounding, :count) == 1
|
|
|
|
sounding = Repo.one(Sounding)
|
|
assert sounding.station_id == station.id
|
|
assert sounding.level_count == 5
|
|
assert sounding.surface_temp_c == 15.0
|
|
end
|
|
|
|
test "skips when sounding already exists", %{station: station} do
|
|
Weather.upsert_sounding(station, %{
|
|
observed_at: ~U[2026-03-28 12:00:00Z],
|
|
profile: [],
|
|
level_count: 0
|
|
})
|
|
|
|
Req.Test.stub(IemClient, fn conn ->
|
|
Plug.Conn.send_resp(conn, 500, "Should not be called")
|
|
end)
|
|
|
|
args = %{
|
|
"fetch_type" => "raob",
|
|
"station_id" => station.id,
|
|
"station_code" => "72451",
|
|
"sounding_time" => "2026-03-28T12:00:00Z"
|
|
}
|
|
|
|
assert :ok = WeatherFetchWorker.perform(%Oban.Job{args: args})
|
|
assert Repo.aggregate(Sounding, :count) == 1
|
|
end
|
|
|
|
test "stores stub sounding on empty profiles so backfill doesn't retry", %{station: station} do
|
|
Req.Test.stub(IemClient, fn conn ->
|
|
Req.Test.json(conn, %{"profiles" => []})
|
|
end)
|
|
|
|
args = %{
|
|
"fetch_type" => "raob",
|
|
"station_id" => station.id,
|
|
"station_code" => "72451",
|
|
"sounding_time" => "2026-03-28T12:00:00Z"
|
|
}
|
|
|
|
assert :ok = WeatherFetchWorker.perform(%Oban.Job{args: args})
|
|
assert Repo.aggregate(Sounding, :count) == 1
|
|
|
|
sounding = Repo.one!(Sounding)
|
|
assert sounding.profile == []
|
|
assert sounding.level_count == 0
|
|
assert sounding.station_id == station.id
|
|
end
|
|
|
|
test "returns error on HTTP failure", %{station: station} do
|
|
Req.Test.stub(IemClient, fn conn ->
|
|
Plug.Conn.send_resp(conn, 503, "Service Unavailable")
|
|
end)
|
|
|
|
args = %{
|
|
"fetch_type" => "raob",
|
|
"station_id" => station.id,
|
|
"station_code" => "72451",
|
|
"sounding_time" => "2026-03-28T12:00:00Z"
|
|
}
|
|
|
|
assert {:error, "IEM RAOB HTTP 503"} = WeatherFetchWorker.perform(%Oban.Job{args: args})
|
|
end
|
|
|
|
test "returns :ok when station no longer exists" do
|
|
args = %{
|
|
"fetch_type" => "raob",
|
|
"station_id" => Ecto.UUID.generate(),
|
|
"station_code" => "GONE",
|
|
"sounding_time" => "2026-03-28T12:00:00Z"
|
|
}
|
|
|
|
assert :ok = WeatherFetchWorker.perform(%Oban.Job{args: args})
|
|
end
|
|
end
|
|
end
|