defmodule Microwaveprop.Weather.IemClientTest do use ExUnit.Case, async: true use ExUnitProperties alias Microwaveprop.Weather.IemClient describe "parse_asos_csv/1" do test "parses valid CSV rows including precipitation" do csv = """ station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1,p01i,wxcodes KDFW,2026-03-28 18:53,75.0,55.0,49.12,12,180,1013.2,29.92,SCT,0.03,RA KDFW,2026-03-28 19:53,76.0,54.0,46.00,10,190,1013.0,29.91,FEW,null,null """ rows = IemClient.parse_asos_csv(csv) assert length(rows) == 2 first = hd(rows) assert first.temp_f == 75.0 assert first.dewpoint_f == 55.0 assert first.relative_humidity == 49.12 assert first.wind_speed_kts == 12.0 assert first.wind_direction_deg == 180 assert first.sea_level_pressure_mb == 1013.2 assert first.altimeter_setting == 29.92 assert first.sky_condition == "SCT" assert first.observed_at == ~U[2026-03-28 18:53:00Z] assert first.precip_1h_in == 0.03 assert first.wx_codes == "RA" second = Enum.at(rows, 1) assert second.precip_1h_in == nil assert second.wx_codes == nil end test "handles null values in CSV" do csv = """ station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1,p01i,wxcodes KDFW,2026-03-28 18:53,null,null,null,null,null,null,null,null,null,null """ rows = IemClient.parse_asos_csv(csv) assert length(rows) == 1 first = hd(rows) assert first.temp_f == nil assert first.dewpoint_f == nil assert first.sky_condition == nil assert first.precip_1h_in == nil assert first.wx_codes == nil end test "skips comment lines and empty lines" do csv = """ #DEBUG something station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1,p01i,wxcodes KDFW,2026-03-28 18:53,75.0,55.0,49.0,12,180,1013.2,29.92,SCT,0.01, """ rows = IemClient.parse_asos_csv(csv) assert length(rows) == 1 end end describe "parse_raob_json/1" do test "parses valid RAOB JSON response" do json = %{ "profiles" => [ %{ "station" => "KFWD", "valid" => "2026-03-28 12:00:00+00:00", "profile" => [ %{"pres" => 1013.0, "hght" => 171.0, "tmpc" => 25.0, "dwpc" => 15.0, "drct" => 180.0, "sknt" => 10.0}, %{"pres" => 925.0, "hght" => 800.0, "tmpc" => 20.0, "dwpc" => 12.0, "drct" => 200.0, "sknt" => 15.0} ] } ] } result = IemClient.parse_raob_json(json) assert length(result) == 1 sounding = hd(result) assert sounding.observed_at == ~U[2026-03-28 12:00:00Z] assert length(sounding.profile) == 2 assert hd(sounding.profile)["pres"] == 1013.0 end test "returns empty list for empty profiles" do json = %{"profiles" => []} assert IemClient.parse_raob_json(json) == [] end test "handles missing profile key gracefully" do json = %{} assert IemClient.parse_raob_json(json) == [] end end describe "asos_url/3" do test "builds correct URL for date range" do url = IemClient.asos_url( "KDFW", ~U[2026-03-28 12:00:00Z], ~U[2026-03-28 18:00:00Z] ) assert url =~ "station=KDFW" assert url =~ "year1=2026" assert url =~ "month1=3" assert url =~ "day1=28" assert url =~ "hour1=12" assert url =~ "year2=2026" assert url =~ "hour2=18" assert url =~ "format=onlycomma" assert url =~ "data=p01i" assert url =~ "data=wxcodes" end test "accepts a list of station codes and emits one station= param each" do url = IemClient.asos_url( ["KDFW", "KFTW", "KAFW"], ~U[2026-03-28 12:00:00Z], ~U[2026-03-28 18:00:00Z] ) assert url =~ "station=KDFW" assert url =~ "station=KFTW" assert url =~ "station=KAFW" end end describe "parse_asos_csv/1 — batched response" do test "tags each row with its station_code" do csv = """ station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1,p01i,wxcodes KDFW,2026-03-28 18:53,75.0,55.0,49.12,12,180,1013.2,29.92,SCT,null,null KFTW,2026-03-28 18:55,73.0,53.0,49.00,10,190,1013.0,29.91,FEW,null,null """ rows = IemClient.parse_asos_csv(csv) assert length(rows) == 2 codes = Enum.map(rows, & &1.station_code) assert "KDFW" in codes assert "KFTW" in codes end end describe "fetch_asos_batch/3" do test "returns rows grouped by station_code" do Req.Test.stub(IemClient, fn conn -> body = """ station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1,p01i,wxcodes KDFW,2026-03-28 18:53,75.0,55.0,49.12,12,180,1013.2,29.92,SCT,null,null KDFW,2026-03-28 19:53,76.0,54.0,46.00,10,190,1013.0,29.91,FEW,null,null KFTW,2026-03-28 18:55,73.0,53.0,49.00,10,190,1013.0,29.91,FEW,null,null """ conn |> Plug.Conn.put_resp_content_type("text/plain") |> Plug.Conn.resp(200, body) end) assert {:ok, grouped} = IemClient.fetch_asos_batch( ["KDFW", "KFTW", "KSPS"], ~U[2026-03-28 18:00:00Z], ~U[2026-03-28 20:00:00Z] ) assert is_map(grouped) assert length(Map.fetch!(grouped, "KDFW")) == 2 assert length(Map.fetch!(grouped, "KFTW")) == 1 # Stations in the request but absent from the response get an empty list # so callers can distinguish "no data" from "not asked about this code". assert Map.fetch!(grouped, "KSPS") == [] end end describe "raob_url/2" do test "builds correct URL for sounding fetch" do url = IemClient.raob_url("FWD", ~U[2026-03-28 12:00:00Z]) assert url =~ "station=FWD" assert url =~ "ts=202603281200" end end describe "network_url/1" do test "builds correct URL for state ASOS network" do url = IemClient.network_url("NY_ASOS") assert url == "https://mesonet.agron.iastate.edu/json/network.py?network=NY_ASOS" end test "builds correct URL for RAOB network" do url = IemClient.network_url("RAOB") assert url == "https://mesonet.agron.iastate.edu/json/network.py?network=RAOB" end end describe "fetch_network/1" do test "returns parsed stations on success" do Req.Test.stub(IemClient, fn conn -> Req.Test.json(conn, %{ "stations" => [ %{"id" => "KJFK", "name" => "JFK Intl", "lat" => 40.64, "lon" => -73.78} ] }) end) assert {:ok, [station]} = IemClient.fetch_network("NY_ASOS") assert station.station_code == "KJFK" assert station.lat == 40.64 end test "returns error on non-200 status" do Req.Test.stub(IemClient, fn conn -> Plug.Conn.send_resp(conn, 503, "Service Unavailable") end) assert {:error, "IEM network HTTP 503"} = IemClient.fetch_network("NY_ASOS") end end describe "fetch_asos/3" do test "returns parsed observations on success" do csv = """ station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1,p01i,wxcodes KDFW,2026-03-28 18:53,75.0,55.0,49.12,12,180,1013.2,29.92,SCT,0.05,RA BR """ Req.Test.stub(IemClient, fn conn -> Req.Test.text(conn, csv) end) assert {:ok, [obs]} = IemClient.fetch_asos("KDFW", ~U[2026-03-28 16:00:00Z], ~U[2026-03-28 20:00:00Z]) assert obs.temp_f == 75.0 assert obs.observed_at == ~U[2026-03-28 18:53:00Z] assert obs.precip_1h_in == 0.05 assert obs.wx_codes == "RA BR" end test "returns error on non-200 status" do Req.Test.stub(IemClient, fn conn -> Plug.Conn.send_resp(conn, 500, "error") end) assert {:error, "IEM ASOS HTTP 500"} = IemClient.fetch_asos("KDFW", ~U[2026-03-28 16:00:00Z], ~U[2026-03-28 20:00:00Z]) end end describe "fetch_raob/2" do test "returns parsed soundings on success" do Req.Test.stub(IemClient, fn conn -> Req.Test.json(conn, %{ "profiles" => [ %{ "station" => "FWD", "valid" => "2026-03-28 12:00:00+00:00", "profile" => [ %{"pres" => 1013.0, "hght" => 171.0, "tmpc" => 25.0, "dwpc" => 15.0, "drct" => 180.0, "sknt" => 10.0} ] } ] }) end) assert {:ok, [sounding]} = IemClient.fetch_raob("FWD", ~U[2026-03-28 12:00:00Z]) assert sounding.observed_at == ~U[2026-03-28 12:00:00Z] assert length(sounding.profile) == 1 end test "returns error on non-200 status" do Req.Test.stub(IemClient, fn conn -> Plug.Conn.send_resp(conn, 404, "not found") end) assert {:error, "IEM RAOB HTTP 404"} = IemClient.fetch_raob("FWD", ~U[2026-03-28 12:00:00Z]) end end describe "fetch_current_asos/1" do test "returns parsed observations from multiple networks" do Req.Test.stub(IemClient, fn conn -> Req.Test.json(conn, %{ "data" => [ %{ "station" => "KDFW", "lat" => 32.9, "lon" => -97.0, "utc_valid" => "2026-03-28T18:00:00Z", "tmpf" => 75.0, "dwpf" => 55.0, "sknt" => 10, "skyc1" => "CLR", "mslp" => 1013.2, "alti" => 29.92, "phour" => 0.0 } ] }) end) assert {:ok, observations} = IemClient.fetch_current_asos(["TX_ASOS"]) assert length(observations) == 1 assert hd(observations).station_code == "KDFW" assert hd(observations).temp_f == 75.0 end test "skips observations with missing core data" do Req.Test.stub(IemClient, fn conn -> Req.Test.json(conn, %{ "data" => [ %{"station" => "KDFW", "lat" => nil, "lon" => nil, "tmpf" => nil, "dwpf" => nil} ] }) end) assert {:ok, []} = IemClient.fetch_current_asos(["TX_ASOS"]) end end describe "parse_network_json/1" do test "parses station list from network JSON response" do json = %{ "stations" => [ %{ "id" => "KJFK", "name" => "New York/JFK Intl", "lat" => 40.6399, "lon" => -73.7787 }, %{ "id" => "KLGA", "name" => "New York/LaGuardia", "lat" => 40.7772, "lon" => -73.8726 } ] } stations = IemClient.parse_network_json(json) assert length(stations) == 2 first = hd(stations) assert first.station_code == "KJFK" assert first.name == "New York/JFK Intl" assert first.lat == 40.6399 assert first.lon == -73.7787 end test "returns empty list when no stations" do assert IemClient.parse_network_json(%{"stations" => []}) == [] end test "returns empty list when stations key missing" do assert IemClient.parse_network_json(%{}) == [] end test "handles stations with extra fields gracefully" do json = %{ "stations" => [ %{ "id" => "KDFW", "name" => "Dallas/Fort Worth", "lat" => 32.897, "lon" => -97.038, "elevation" => 171.0, "state" => "TX", "country" => "US", "network" => "TX_ASOS" } ] } stations = IemClient.parse_network_json(json) assert length(stations) == 1 station = hd(stations) assert station.station_code == "KDFW" assert station.lat == 32.897 assert station.lon == -97.038 end end describe "iemre_url/3" do test "builds correct URL for lat/lon and date" do url = IemClient.iemre_url(32.875, -97.0, ~D[2026-03-28]) assert url == "https://mesonet.agron.iastate.edu/iemre/hourly/2026-03-28/32.875/-97.0/json" end end describe "parse_iemre_json/1" do test "extracts data array from JSON response" do json = %{ "data" => [ %{"utc_hour" => 0, "p01m_mm" => 0.0, "skyc_pct" => 25.0}, %{"utc_hour" => 1, "p01m_mm" => 0.5, "skyc_pct" => 50.0} ] } result = IemClient.parse_iemre_json(json) assert length(result) == 2 assert hd(result)["utc_hour"] == 0 end test "returns empty list for missing data key" do assert IemClient.parse_iemre_json(%{}) == [] end test "returns empty list for empty data array" do assert IemClient.parse_iemre_json(%{"data" => []}) == [] end # IEM occasionally returns a 200 with an empty body for valid-looking # requests (upstream ERDDAP glitch during backfill windows). # parse_iemre_json/1 used to only match is_map, so the empty binary # blew up with FunctionClauseError and the job exhausted retries. # Treat "" and nil as "no data" so the worker can mark the slot and move on. test "returns empty list for empty string body" do assert IemClient.parse_iemre_json("") == [] end test "returns empty list for nil body" do assert IemClient.parse_iemre_json(nil) == [] end end describe "fetch_iemre/3" do test "returns parsed hourly data on success" do Req.Test.stub(IemClient, fn conn -> Req.Test.json(conn, %{ "data" => [ %{"utc_hour" => 0, "p01m_mm" => 0.0}, %{"utc_hour" => 1, "p01m_mm" => 1.2} ] }) end) assert {:ok, data} = IemClient.fetch_iemre(32.875, -97.0, ~D[2026-03-28]) assert length(data) == 2 assert Enum.at(data, 1)["p01m_mm"] == 1.2 end test "returns error on non-200 status" do Req.Test.stub(IemClient, fn conn -> Plug.Conn.send_resp(conn, 503, "Service Unavailable") end) assert {:error, "IEM IEMRE HTTP 503"} = IemClient.fetch_iemre(32.875, -97.0, ~D[2026-03-28]) end test "returns error on 404" do Req.Test.stub(IemClient, fn conn -> Plug.Conn.send_resp(conn, 404, "not found") end) assert {:error, "IEM IEMRE HTTP 404"} = IemClient.fetch_iemre(32.875, -97.0, ~D[2026-03-28]) end test "bubbles up a transport-level timeout as {:error, %TransportError{}}" do Req.Test.stub(IemClient, fn conn -> Req.Test.transport_error(conn, :timeout) end) assert {:error, reason} = IemClient.fetch_iemre(32.875, -97.0, ~D[2026-03-28]) # Req surfaces a %Req.TransportError{reason: :timeout}, not a string. refute is_binary(reason) end end describe "fetch_network/1 transport errors" do test "bubbles a transport timeout up as {:error, _}" do Req.Test.stub(IemClient, fn conn -> Req.Test.transport_error(conn, :timeout) end) assert {:error, _reason} = IemClient.fetch_network("NY_ASOS") end end describe "fetch_asos/3 transport errors" do test "bubbles a transport timeout up as {:error, _}" do Req.Test.stub(IemClient, fn conn -> Req.Test.transport_error(conn, :timeout) end) assert {:error, _reason} = IemClient.fetch_asos("KDFW", ~U[2026-03-28 16:00:00Z], ~U[2026-03-28 20:00:00Z]) end end describe "fetch_raob/2 transport errors" do test "bubbles a transport timeout up as {:error, _}" do Req.Test.stub(IemClient, fn conn -> Req.Test.transport_error(conn, :timeout) end) assert {:error, _reason} = IemClient.fetch_raob("FWD", ~U[2026-03-28 12:00:00Z]) end end describe "parse_asos_csv/1 edge cases" do test "returns [] for a CSV with only the header row" do csv = "station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1,p01i,wxcodes\n" assert IemClient.parse_asos_csv(csv) == [] end test "handles a CSV with trailing newline without crashing" do csv = """ station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1,p01i,wxcodes KDFW,2026-03-28 18:53,75.0,55.0,49.12,12,180,1013.2,29.92,SCT,0.03,RA """ rows = IemClient.parse_asos_csv(csv) assert length(rows) == 1 end test "returns [] for an empty string body" do assert IemClient.parse_asos_csv("") == [] end test "row with a bogus timestamp keeps observed_at = nil" do csv = """ station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1,p01i,wxcodes KDFW,not-a-timestamp,75.0,55.0,49.0,12,180,1013.2,29.92,SCT,0.0, """ [row] = IemClient.parse_asos_csv(csv) assert row.observed_at == nil # Numeric columns still parse from the positional CSV. assert row.temp_f == 75.0 end property "never raises for any utf-8 binary input" do check all(body <- string(:printable, max_length: 200)) do # parse_asos_csv must be total — the ingest worker cannot tolerate # a FunctionClauseError on a malformed upstream response. assert is_list(IemClient.parse_asos_csv(body)) end end end describe "parse_raob_json/1 edge cases" do test "a profile with an unparseable 'valid' timestamp keeps observed_at = nil" do json = %{ "profiles" => [ %{"station" => "FWD", "valid" => "garbage", "profile" => []} ] } [row] = IemClient.parse_raob_json(json) assert row.observed_at == nil assert row.profile == [] end test "a profile missing the 'profile' key defaults to []" do json = %{"profiles" => [%{"station" => "FWD", "valid" => "2026-03-28 12:00:00+00:00"}]} [row] = IemClient.parse_raob_json(json) assert row.profile == [] end end end