prop/test/microwaveprop/weather/iem_client_test.exs
Graham McIntire c073c6a95a
perf(weather): batch ASOS fetches into one multi-station request
IEM's ASOS CSV endpoint accepts multiple station= params in a single
request. Prior code enqueued one WeatherFetchWorker job per nearby
station per QSO endpoint — N jobs × 1 station each — which paid the
IemRateLimiter's 1500ms gap and the IEM 429-retry tail N times per
contact.

Changes:
- IemClient.asos_url/3 accepts a list of station codes.
- IemClient.fetch_asos_batch/3 fetches N stations in one call and
  returns rows grouped by station_code (with absent codes filled as
  empty lists so callers can stub them).
- parse_asos_csv/1 now exposes the station_code column it was
  previously discarding.
- WeatherFetchWorker gains an "asos_batch" fetch_type clause that
  unpacks rows per (station_id, station_code), upserting or stubbing
  each. The single-station "asos" clause stays for already-queued
  retryable jobs.
- ContactWeatherEnqueueWorker.build_asos_jobs/3 now emits one batch
  job per (lat, lon, 4h window) covering every uncovered nearby
  station (sorted for deterministic unique-args).

Expected effect on backfill: ~10-20x fewer IEM requests per contact
enrichment cycle, matching drop in 429 retry traffic.
2026-04-24 12:44:22 -05:00

571 lines
17 KiB
Elixir

defmodule Microwaveprop.Weather.IemClientTest do
use ExUnit.Case, async: true
use ExUnitProperties
alias Microwaveprop.Weather.IemClient
describe "parse_asos_csv/1" do
test "parses valid CSV rows including precipitation" do
csv = """
station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1,p01i,wxcodes
KDFW,2026-03-28 18:53,75.0,55.0,49.12,12,180,1013.2,29.92,SCT,0.03,RA
KDFW,2026-03-28 19:53,76.0,54.0,46.00,10,190,1013.0,29.91,FEW,null,null
"""
rows = IemClient.parse_asos_csv(csv)
assert length(rows) == 2
first = hd(rows)
assert first.temp_f == 75.0
assert first.dewpoint_f == 55.0
assert first.relative_humidity == 49.12
assert first.wind_speed_kts == 12.0
assert first.wind_direction_deg == 180
assert first.sea_level_pressure_mb == 1013.2
assert first.altimeter_setting == 29.92
assert first.sky_condition == "SCT"
assert first.observed_at == ~U[2026-03-28 18:53:00Z]
assert first.precip_1h_in == 0.03
assert first.wx_codes == "RA"
second = Enum.at(rows, 1)
assert second.precip_1h_in == nil
assert second.wx_codes == nil
end
test "handles null values in CSV" do
csv = """
station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1,p01i,wxcodes
KDFW,2026-03-28 18:53,null,null,null,null,null,null,null,null,null,null
"""
rows = IemClient.parse_asos_csv(csv)
assert length(rows) == 1
first = hd(rows)
assert first.temp_f == nil
assert first.dewpoint_f == nil
assert first.sky_condition == nil
assert first.precip_1h_in == nil
assert first.wx_codes == nil
end
test "skips comment lines and empty lines" do
csv = """
#DEBUG something
station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1,p01i,wxcodes
KDFW,2026-03-28 18:53,75.0,55.0,49.0,12,180,1013.2,29.92,SCT,0.01,
"""
rows = IemClient.parse_asos_csv(csv)
assert length(rows) == 1
end
end
describe "parse_raob_json/1" do
test "parses valid RAOB JSON response" do
json = %{
"profiles" => [
%{
"station" => "KFWD",
"valid" => "2026-03-28 12:00:00+00:00",
"profile" => [
%{"pres" => 1013.0, "hght" => 171.0, "tmpc" => 25.0, "dwpc" => 15.0, "drct" => 180.0, "sknt" => 10.0},
%{"pres" => 925.0, "hght" => 800.0, "tmpc" => 20.0, "dwpc" => 12.0, "drct" => 200.0, "sknt" => 15.0}
]
}
]
}
result = IemClient.parse_raob_json(json)
assert length(result) == 1
sounding = hd(result)
assert sounding.observed_at == ~U[2026-03-28 12:00:00Z]
assert length(sounding.profile) == 2
assert hd(sounding.profile)["pres"] == 1013.0
end
test "returns empty list for empty profiles" do
json = %{"profiles" => []}
assert IemClient.parse_raob_json(json) == []
end
test "handles missing profile key gracefully" do
json = %{}
assert IemClient.parse_raob_json(json) == []
end
end
describe "asos_url/3" do
test "builds correct URL for date range" do
url =
IemClient.asos_url(
"KDFW",
~U[2026-03-28 12:00:00Z],
~U[2026-03-28 18:00:00Z]
)
assert url =~ "station=KDFW"
assert url =~ "year1=2026"
assert url =~ "month1=3"
assert url =~ "day1=28"
assert url =~ "hour1=12"
assert url =~ "year2=2026"
assert url =~ "hour2=18"
assert url =~ "format=onlycomma"
assert url =~ "data=p01i"
assert url =~ "data=wxcodes"
end
test "accepts a list of station codes and emits one station= param each" do
url =
IemClient.asos_url(
["KDFW", "KFTW", "KAFW"],
~U[2026-03-28 12:00:00Z],
~U[2026-03-28 18:00:00Z]
)
assert url =~ "station=KDFW"
assert url =~ "station=KFTW"
assert url =~ "station=KAFW"
end
end
describe "parse_asos_csv/1 — batched response" do
test "tags each row with its station_code" do
csv = """
station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1,p01i,wxcodes
KDFW,2026-03-28 18:53,75.0,55.0,49.12,12,180,1013.2,29.92,SCT,null,null
KFTW,2026-03-28 18:55,73.0,53.0,49.00,10,190,1013.0,29.91,FEW,null,null
"""
rows = IemClient.parse_asos_csv(csv)
assert length(rows) == 2
codes = Enum.map(rows, & &1.station_code)
assert "KDFW" in codes
assert "KFTW" in codes
end
end
describe "fetch_asos_batch/3" do
test "returns rows grouped by station_code" do
Req.Test.stub(IemClient, fn conn ->
body = """
station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1,p01i,wxcodes
KDFW,2026-03-28 18:53,75.0,55.0,49.12,12,180,1013.2,29.92,SCT,null,null
KDFW,2026-03-28 19:53,76.0,54.0,46.00,10,190,1013.0,29.91,FEW,null,null
KFTW,2026-03-28 18:55,73.0,53.0,49.00,10,190,1013.0,29.91,FEW,null,null
"""
conn
|> Plug.Conn.put_resp_content_type("text/plain")
|> Plug.Conn.resp(200, body)
end)
assert {:ok, grouped} =
IemClient.fetch_asos_batch(
["KDFW", "KFTW", "KSPS"],
~U[2026-03-28 18:00:00Z],
~U[2026-03-28 20:00:00Z]
)
assert is_map(grouped)
assert length(Map.fetch!(grouped, "KDFW")) == 2
assert length(Map.fetch!(grouped, "KFTW")) == 1
# Stations in the request but absent from the response get an empty list
# so callers can distinguish "no data" from "not asked about this code".
assert Map.fetch!(grouped, "KSPS") == []
end
end
describe "raob_url/2" do
test "builds correct URL for sounding fetch" do
url = IemClient.raob_url("FWD", ~U[2026-03-28 12:00:00Z])
assert url =~ "station=FWD"
assert url =~ "ts=202603281200"
end
end
describe "network_url/1" do
test "builds correct URL for state ASOS network" do
url = IemClient.network_url("NY_ASOS")
assert url == "https://mesonet.agron.iastate.edu/json/network.py?network=NY_ASOS"
end
test "builds correct URL for RAOB network" do
url = IemClient.network_url("RAOB")
assert url == "https://mesonet.agron.iastate.edu/json/network.py?network=RAOB"
end
end
describe "fetch_network/1" do
test "returns parsed stations on success" do
Req.Test.stub(IemClient, fn conn ->
Req.Test.json(conn, %{
"stations" => [
%{"id" => "KJFK", "name" => "JFK Intl", "lat" => 40.64, "lon" => -73.78}
]
})
end)
assert {:ok, [station]} = IemClient.fetch_network("NY_ASOS")
assert station.station_code == "KJFK"
assert station.lat == 40.64
end
test "returns error on non-200 status" do
Req.Test.stub(IemClient, fn conn ->
Plug.Conn.send_resp(conn, 503, "Service Unavailable")
end)
assert {:error, "IEM network HTTP 503"} = IemClient.fetch_network("NY_ASOS")
end
end
describe "fetch_asos/3" do
test "returns parsed observations on success" do
csv = """
station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1,p01i,wxcodes
KDFW,2026-03-28 18:53,75.0,55.0,49.12,12,180,1013.2,29.92,SCT,0.05,RA BR
"""
Req.Test.stub(IemClient, fn conn ->
Req.Test.text(conn, csv)
end)
assert {:ok, [obs]} =
IemClient.fetch_asos("KDFW", ~U[2026-03-28 16:00:00Z], ~U[2026-03-28 20:00:00Z])
assert obs.temp_f == 75.0
assert obs.observed_at == ~U[2026-03-28 18:53:00Z]
assert obs.precip_1h_in == 0.05
assert obs.wx_codes == "RA BR"
end
test "returns error on non-200 status" do
Req.Test.stub(IemClient, fn conn ->
Plug.Conn.send_resp(conn, 500, "error")
end)
assert {:error, "IEM ASOS HTTP 500"} =
IemClient.fetch_asos("KDFW", ~U[2026-03-28 16:00:00Z], ~U[2026-03-28 20:00:00Z])
end
end
describe "fetch_raob/2" do
test "returns parsed soundings on success" do
Req.Test.stub(IemClient, fn conn ->
Req.Test.json(conn, %{
"profiles" => [
%{
"station" => "FWD",
"valid" => "2026-03-28 12:00:00+00:00",
"profile" => [
%{"pres" => 1013.0, "hght" => 171.0, "tmpc" => 25.0, "dwpc" => 15.0, "drct" => 180.0, "sknt" => 10.0}
]
}
]
})
end)
assert {:ok, [sounding]} = IemClient.fetch_raob("FWD", ~U[2026-03-28 12:00:00Z])
assert sounding.observed_at == ~U[2026-03-28 12:00:00Z]
assert length(sounding.profile) == 1
end
test "returns error on non-200 status" do
Req.Test.stub(IemClient, fn conn ->
Plug.Conn.send_resp(conn, 404, "not found")
end)
assert {:error, "IEM RAOB HTTP 404"} = IemClient.fetch_raob("FWD", ~U[2026-03-28 12:00:00Z])
end
end
describe "fetch_current_asos/1" do
test "returns parsed observations from multiple networks" do
Req.Test.stub(IemClient, fn conn ->
Req.Test.json(conn, %{
"data" => [
%{
"station" => "KDFW",
"lat" => 32.9,
"lon" => -97.0,
"utc_valid" => "2026-03-28T18:00:00Z",
"tmpf" => 75.0,
"dwpf" => 55.0,
"sknt" => 10,
"skyc1" => "CLR",
"mslp" => 1013.2,
"alti" => 29.92,
"phour" => 0.0
}
]
})
end)
assert {:ok, observations} = IemClient.fetch_current_asos(["TX_ASOS"])
assert length(observations) == 1
assert hd(observations).station_code == "KDFW"
assert hd(observations).temp_f == 75.0
end
test "skips observations with missing core data" do
Req.Test.stub(IemClient, fn conn ->
Req.Test.json(conn, %{
"data" => [
%{"station" => "KDFW", "lat" => nil, "lon" => nil, "tmpf" => nil, "dwpf" => nil}
]
})
end)
assert {:ok, []} = IemClient.fetch_current_asos(["TX_ASOS"])
end
end
describe "parse_network_json/1" do
test "parses station list from network JSON response" do
json = %{
"stations" => [
%{
"id" => "KJFK",
"name" => "New York/JFK Intl",
"lat" => 40.6399,
"lon" => -73.7787
},
%{
"id" => "KLGA",
"name" => "New York/LaGuardia",
"lat" => 40.7772,
"lon" => -73.8726
}
]
}
stations = IemClient.parse_network_json(json)
assert length(stations) == 2
first = hd(stations)
assert first.station_code == "KJFK"
assert first.name == "New York/JFK Intl"
assert first.lat == 40.6399
assert first.lon == -73.7787
end
test "returns empty list when no stations" do
assert IemClient.parse_network_json(%{"stations" => []}) == []
end
test "returns empty list when stations key missing" do
assert IemClient.parse_network_json(%{}) == []
end
test "handles stations with extra fields gracefully" do
json = %{
"stations" => [
%{
"id" => "KDFW",
"name" => "Dallas/Fort Worth",
"lat" => 32.897,
"lon" => -97.038,
"elevation" => 171.0,
"state" => "TX",
"country" => "US",
"network" => "TX_ASOS"
}
]
}
stations = IemClient.parse_network_json(json)
assert length(stations) == 1
station = hd(stations)
assert station.station_code == "KDFW"
assert station.lat == 32.897
assert station.lon == -97.038
end
end
describe "iemre_url/3" do
test "builds correct URL for lat/lon and date" do
url = IemClient.iemre_url(32.875, -97.0, ~D[2026-03-28])
assert url == "https://mesonet.agron.iastate.edu/iemre/hourly/2026-03-28/32.875/-97.0/json"
end
end
describe "parse_iemre_json/1" do
test "extracts data array from JSON response" do
json = %{
"data" => [
%{"utc_hour" => 0, "p01m_mm" => 0.0, "skyc_pct" => 25.0},
%{"utc_hour" => 1, "p01m_mm" => 0.5, "skyc_pct" => 50.0}
]
}
result = IemClient.parse_iemre_json(json)
assert length(result) == 2
assert hd(result)["utc_hour"] == 0
end
test "returns empty list for missing data key" do
assert IemClient.parse_iemre_json(%{}) == []
end
test "returns empty list for empty data array" do
assert IemClient.parse_iemre_json(%{"data" => []}) == []
end
# IEM occasionally returns a 200 with an empty body for valid-looking
# requests (upstream ERDDAP glitch during backfill windows).
# parse_iemre_json/1 used to only match is_map, so the empty binary
# blew up with FunctionClauseError and the job exhausted retries.
# Treat "" and nil as "no data" so the worker can mark the slot and move on.
test "returns empty list for empty string body" do
assert IemClient.parse_iemre_json("") == []
end
test "returns empty list for nil body" do
assert IemClient.parse_iemre_json(nil) == []
end
end
describe "fetch_iemre/3" do
test "returns parsed hourly data on success" do
Req.Test.stub(IemClient, fn conn ->
Req.Test.json(conn, %{
"data" => [
%{"utc_hour" => 0, "p01m_mm" => 0.0},
%{"utc_hour" => 1, "p01m_mm" => 1.2}
]
})
end)
assert {:ok, data} = IemClient.fetch_iemre(32.875, -97.0, ~D[2026-03-28])
assert length(data) == 2
assert Enum.at(data, 1)["p01m_mm"] == 1.2
end
test "returns error on non-200 status" do
Req.Test.stub(IemClient, fn conn ->
Plug.Conn.send_resp(conn, 503, "Service Unavailable")
end)
assert {:error, "IEM IEMRE HTTP 503"} = IemClient.fetch_iemre(32.875, -97.0, ~D[2026-03-28])
end
test "returns error on 404" do
Req.Test.stub(IemClient, fn conn ->
Plug.Conn.send_resp(conn, 404, "not found")
end)
assert {:error, "IEM IEMRE HTTP 404"} = IemClient.fetch_iemre(32.875, -97.0, ~D[2026-03-28])
end
test "bubbles up a transport-level timeout as {:error, %TransportError{}}" do
Req.Test.stub(IemClient, fn conn ->
Req.Test.transport_error(conn, :timeout)
end)
assert {:error, reason} = IemClient.fetch_iemre(32.875, -97.0, ~D[2026-03-28])
# Req surfaces a %Req.TransportError{reason: :timeout}, not a string.
refute is_binary(reason)
end
end
describe "fetch_network/1 transport errors" do
test "bubbles a transport timeout up as {:error, _}" do
Req.Test.stub(IemClient, fn conn ->
Req.Test.transport_error(conn, :timeout)
end)
assert {:error, _reason} = IemClient.fetch_network("NY_ASOS")
end
end
describe "fetch_asos/3 transport errors" do
test "bubbles a transport timeout up as {:error, _}" do
Req.Test.stub(IemClient, fn conn ->
Req.Test.transport_error(conn, :timeout)
end)
assert {:error, _reason} =
IemClient.fetch_asos("KDFW", ~U[2026-03-28 16:00:00Z], ~U[2026-03-28 20:00:00Z])
end
end
describe "fetch_raob/2 transport errors" do
test "bubbles a transport timeout up as {:error, _}" do
Req.Test.stub(IemClient, fn conn ->
Req.Test.transport_error(conn, :timeout)
end)
assert {:error, _reason} = IemClient.fetch_raob("FWD", ~U[2026-03-28 12:00:00Z])
end
end
describe "parse_asos_csv/1 edge cases" do
test "returns [] for a CSV with only the header row" do
csv = "station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1,p01i,wxcodes\n"
assert IemClient.parse_asos_csv(csv) == []
end
test "handles a CSV with trailing newline without crashing" do
csv = """
station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1,p01i,wxcodes
KDFW,2026-03-28 18:53,75.0,55.0,49.12,12,180,1013.2,29.92,SCT,0.03,RA
"""
rows = IemClient.parse_asos_csv(csv)
assert length(rows) == 1
end
test "returns [] for an empty string body" do
assert IemClient.parse_asos_csv("") == []
end
test "row with a bogus timestamp keeps observed_at = nil" do
csv = """
station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1,p01i,wxcodes
KDFW,not-a-timestamp,75.0,55.0,49.0,12,180,1013.2,29.92,SCT,0.0,
"""
[row] = IemClient.parse_asos_csv(csv)
assert row.observed_at == nil
# Numeric columns still parse from the positional CSV.
assert row.temp_f == 75.0
end
property "never raises for any utf-8 binary input" do
check all(body <- string(:printable, max_length: 200)) do
# parse_asos_csv must be total — the ingest worker cannot tolerate
# a FunctionClauseError on a malformed upstream response.
assert is_list(IemClient.parse_asos_csv(body))
end
end
end
describe "parse_raob_json/1 edge cases" do
test "a profile with an unparseable 'valid' timestamp keeps observed_at = nil" do
json = %{
"profiles" => [
%{"station" => "FWD", "valid" => "garbage", "profile" => []}
]
}
[row] = IemClient.parse_raob_json(json)
assert row.observed_at == nil
assert row.profile == []
end
test "a profile missing the 'profile' key defaults to []" do
json = %{"profiles" => [%{"station" => "FWD", "valid" => "2026-03-28 12:00:00+00:00"}]}
[row] = IemClient.parse_raob_json(json)
assert row.profile == []
end
end
end