Graham McIntire
1086f52c85
Both RTMA and METAR5 schemas + clients + workers were defined but never had a consumer. `find_nearest_rtma/3` and `recent_surface_obs/3` existed in `Microwaveprop.Weather` with zero callers cluster-wide; RtmaFetchWorker had test coverage but was never enqueued anywhere; no IEM 5-min ASOS fetcher was ever written. The tables sat empty in prod and the recalibrate audit was permanently flagging them BROKEN even though the empty state was the steady state. Drop the lot — schemas, clients, workers, queue config, Req.Test stubs, audit checks, and the per-table unique tests in the observation-changesets suite. New migration drops the now-unreferenced `rtma_observations` and `metar_5min_observations` tables (both 0 rows in prod). Trim the `:rtma` slot out of `backfill_only_queues` in runtime.exs so Oban Pro's Smart engine stops trying to manage a queue with no producer. Audit thresholds reset on the surviving NARR check: drop the "WARN < 5000 absolute rows" rule that didn't account for NarrFetchWorker's 0.13° spatial dedup, and point remediation at the existing BackfillEnqueueWorker cron instead of the dev-only `mix narr.backfill` task. scripts/recalibrate_algo.py: ROW_COUNT_SOURCES + DATA_GAP_SQL + DATA_GAP_RULES all stop referencing the dropped tables.
341 lines
10 KiB
Elixir
341 lines
10 KiB
Elixir
defmodule Microwaveprop.WeatherExtraTest do
|
|
@moduledoc """
|
|
Fills in coverage holes on lesser-used `Microwaveprop.Weather`
|
|
context helpers: has_surface_observations?/3, has_sounding?/2,
|
|
the two `station_ids_with_*` set-membership helpers,
|
|
`existing_solar_dates/0`, `sounding_times_around/1`, and
|
|
`hrrr_data_fully_present?/1`.
|
|
|
|
Each is driven with a small, deterministic fixture — we care about
|
|
the boolean / set return contracts rather than large real-world
|
|
data shapes.
|
|
"""
|
|
use Microwaveprop.DataCase, async: true
|
|
use ExUnitProperties
|
|
|
|
alias Microwaveprop.Repo
|
|
alias Microwaveprop.Weather
|
|
alias Microwaveprop.Weather.Sounding
|
|
alias Microwaveprop.Weather.SurfaceObservation
|
|
|
|
defp station!(attrs \\ %{}) do
|
|
base = %{
|
|
station_code: "TST#{System.unique_integer([:positive])}",
|
|
station_type: "asos",
|
|
name: "Test Station",
|
|
lat: 33.0,
|
|
lon: -97.0
|
|
}
|
|
|
|
{:ok, s} = Weather.find_or_create_station(Map.merge(base, attrs))
|
|
s
|
|
end
|
|
|
|
describe "has_surface_observations?/3" do
|
|
test "true when any observation falls inside the window" do
|
|
station = station!()
|
|
|
|
{:ok, _} =
|
|
%SurfaceObservation{}
|
|
|> SurfaceObservation.changeset(%{
|
|
station_id: station.id,
|
|
observed_at: ~U[2026-04-20 12:00:00Z],
|
|
temp_f: 72.0
|
|
})
|
|
|> Repo.insert()
|
|
|
|
assert Weather.has_surface_observations?(
|
|
station.id,
|
|
~U[2026-04-20 00:00:00Z],
|
|
~U[2026-04-20 23:59:59Z]
|
|
)
|
|
end
|
|
|
|
test "false when the window excludes every observation" do
|
|
station = station!()
|
|
|
|
{:ok, _} =
|
|
%SurfaceObservation{}
|
|
|> SurfaceObservation.changeset(%{
|
|
station_id: station.id,
|
|
observed_at: ~U[2026-04-20 12:00:00Z],
|
|
temp_f: 72.0
|
|
})
|
|
|> Repo.insert()
|
|
|
|
refute Weather.has_surface_observations?(
|
|
station.id,
|
|
~U[2026-04-21 00:00:00Z],
|
|
~U[2026-04-21 23:59:59Z]
|
|
)
|
|
end
|
|
|
|
test "false for a station with no observations" do
|
|
station = station!()
|
|
refute Weather.has_surface_observations?(station.id, ~U[2026-04-20 00:00:00Z], ~U[2026-04-20 23:59:59Z])
|
|
end
|
|
end
|
|
|
|
describe "has_sounding?/2" do
|
|
test "true when a sounding exists at the exact observed_at" do
|
|
station = station!(%{station_type: "sounding"})
|
|
|
|
{:ok, _} =
|
|
%Sounding{}
|
|
|> Sounding.changeset(%{
|
|
station_id: station.id,
|
|
observed_at: ~U[2026-04-20 12:00:00Z],
|
|
profile: [],
|
|
level_count: 0
|
|
})
|
|
|> Repo.insert()
|
|
|
|
assert Weather.has_sounding?(station.id, ~U[2026-04-20 12:00:00Z])
|
|
end
|
|
|
|
test "false when no sounding exists at the given time" do
|
|
station = station!(%{station_type: "sounding"})
|
|
refute Weather.has_sounding?(station.id, ~U[2026-04-20 12:00:00Z])
|
|
end
|
|
end
|
|
|
|
describe "station_ids_with_* set-membership helpers" do
|
|
test "station_ids_with_surface_observations returns only the stations that have rows in the window" do
|
|
a = station!()
|
|
b = station!()
|
|
c = station!()
|
|
|
|
{:ok, _} =
|
|
%SurfaceObservation{}
|
|
|> SurfaceObservation.changeset(%{
|
|
station_id: a.id,
|
|
observed_at: ~U[2026-04-20 12:00:00Z],
|
|
temp_f: 60.0
|
|
})
|
|
|> Repo.insert()
|
|
|
|
{:ok, _} =
|
|
%SurfaceObservation{}
|
|
|> SurfaceObservation.changeset(%{
|
|
station_id: b.id,
|
|
observed_at: ~U[2026-04-20 13:00:00Z],
|
|
temp_f: 70.0
|
|
})
|
|
|> Repo.insert()
|
|
|
|
present =
|
|
Weather.station_ids_with_surface_observations(
|
|
[a.id, b.id, c.id],
|
|
~U[2026-04-20 00:00:00Z],
|
|
~U[2026-04-20 23:59:59Z]
|
|
)
|
|
|
|
assert MapSet.size(present) == 2
|
|
assert MapSet.member?(present, a.id)
|
|
assert MapSet.member?(present, b.id)
|
|
refute MapSet.member?(present, c.id)
|
|
end
|
|
|
|
test "station_ids_with_soundings returns {station_id, observed_at} tuples" do
|
|
station = station!(%{station_type: "sounding"})
|
|
|
|
{:ok, _} =
|
|
%Sounding{}
|
|
|> Sounding.changeset(%{
|
|
station_id: station.id,
|
|
observed_at: ~U[2026-04-20 12:00:00Z],
|
|
profile: [],
|
|
level_count: 0
|
|
})
|
|
|> Repo.insert()
|
|
|
|
set =
|
|
Weather.station_ids_with_soundings(
|
|
[station.id],
|
|
[~U[2026-04-20 12:00:00Z], ~U[2026-04-20 00:00:00Z]]
|
|
)
|
|
|
|
assert MapSet.member?(set, {station.id, ~U[2026-04-20 12:00:00Z]})
|
|
refute MapSet.member?(set, {station.id, ~U[2026-04-20 00:00:00Z]})
|
|
end
|
|
end
|
|
|
|
describe "sounding_times_around/1" do
|
|
# Radiosondes fly twice a day (00Z, 12Z). The helper returns the
|
|
# two nearest bracketing times for a given wall-clock UTC.
|
|
test "early-morning times bracket prev-day 12Z and same-day 00Z" do
|
|
times = Weather.sounding_times_around(~U[2026-04-20 06:00:00Z])
|
|
|
|
assert length(times) >= 2
|
|
assert ~U[2026-04-19 12:00:00Z] in times
|
|
assert ~U[2026-04-20 00:00:00Z] in times
|
|
end
|
|
|
|
test "afternoon times bracket same-day 00Z/12Z" do
|
|
times = Weather.sounding_times_around(~U[2026-04-20 18:00:00Z])
|
|
assert ~U[2026-04-20 00:00:00Z] in times
|
|
assert ~U[2026-04-20 12:00:00Z] in times
|
|
end
|
|
|
|
test "noon UTC lands on the 12Z bracket" do
|
|
times = Weather.sounding_times_around(~U[2026-04-20 12:00:00Z])
|
|
assert ~U[2026-04-20 12:00:00Z] in times
|
|
end
|
|
end
|
|
|
|
describe "hrrr_data_fully_present?/1" do
|
|
test "false when pos1 is missing" do
|
|
refute Weather.hrrr_data_fully_present?(%{pos1: nil})
|
|
end
|
|
|
|
test "false when qso_timestamp is missing" do
|
|
refute Weather.hrrr_data_fully_present?(%{qso_timestamp: nil, pos1: %{"lat" => 32.9, "lon" => -97.0}})
|
|
end
|
|
end
|
|
|
|
describe "existing_solar_dates/0" do
|
|
test "returns a MapSet of Dates for every row in solar_indices" do
|
|
date = ~D[2026-04-20]
|
|
|
|
Weather.upsert_solar_indices_batch([
|
|
%{
|
|
date: date,
|
|
sfi: 120.0,
|
|
sfi_adjusted: 118.0,
|
|
sunspot_number: 55,
|
|
ap_index: 5,
|
|
kp_values: [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]
|
|
}
|
|
])
|
|
|
|
set = Weather.existing_solar_dates()
|
|
assert MapSet.member?(set, date)
|
|
end
|
|
end
|
|
|
|
describe "iemre_for_contact/1 / iemre_for_path/1 edge cases" do
|
|
test "iemre_for_contact returns nil with empty DB and valid pos1" do
|
|
contact = %{
|
|
pos1: %{"lat" => 32.9, "lon" => -97.0},
|
|
qso_timestamp: ~U[2026-04-20 18:00:00Z]
|
|
}
|
|
|
|
assert Weather.iemre_for_contact(contact) == nil
|
|
end
|
|
|
|
test "iemre_for_path returns [] when pos1 is nil" do
|
|
assert Weather.iemre_for_path(%{pos1: nil}) == []
|
|
end
|
|
end
|
|
|
|
describe "narr_for_contact/1 / narr_profiles_for_path/1 edge cases" do
|
|
test "narr_for_contact returns nil when pos1 is nil" do
|
|
assert Weather.narr_for_contact(%{pos1: nil}) == nil
|
|
end
|
|
|
|
test "narr_profiles_for_path returns [] when pos1 is nil" do
|
|
assert Weather.narr_profiles_for_path(%{pos1: nil}) == []
|
|
end
|
|
|
|
test "narr_for_contact returns nil when coords present but no profile exists" do
|
|
contact = %{
|
|
pos1: %{"lat" => 32.9, "lon" => -97.0},
|
|
qso_timestamp: ~U[2026-04-20 18:00:00Z]
|
|
}
|
|
|
|
assert Weather.narr_for_contact(contact) == nil
|
|
end
|
|
end
|
|
|
|
describe "best_profile_for_contact/1 fallback behavior" do
|
|
test "returns nil when neither HRRR nor NARR exists" do
|
|
contact = %{
|
|
pos1: %{"lat" => 40.0, "lon" => -80.0},
|
|
qso_timestamp: ~U[2026-04-20 18:00:00Z]
|
|
}
|
|
|
|
assert Weather.best_profile_for_contact(contact) == nil
|
|
end
|
|
end
|
|
|
|
describe "profiles_along_path/1 fallback" do
|
|
test "returns [] when no HRRR or NARR is near the path" do
|
|
contact = %{
|
|
pos1: %{"lat" => 40.0, "lon" => -80.0},
|
|
pos2: %{"lat" => 41.0, "lon" => -81.0},
|
|
qso_timestamp: ~U[2026-04-20 18:00:00Z]
|
|
}
|
|
|
|
assert Weather.profiles_along_path(contact) == []
|
|
end
|
|
end
|
|
|
|
describe "nearest_native_duct_ghz/3 and nearest_native_duct_info/3" do
|
|
test "returns nil / {:error, :not_found} with no native profile rows" do
|
|
ts = ~U[2026-04-20 18:00:00Z]
|
|
assert Weather.nearest_native_duct_ghz(32.9, -97.0, ts) == nil
|
|
assert Weather.nearest_native_duct_info(32.9, -97.0, ts) == {:error, :not_found}
|
|
end
|
|
end
|
|
|
|
describe "property: round_to_hrrr_grid/2" do
|
|
property "rounded output stays within 0.005 of input for any CONUS lat/lon" do
|
|
check all(
|
|
lat <- float(min: 20.0, max: 55.0),
|
|
lon <- float(min: -130.0, max: -60.0)
|
|
) do
|
|
{rlat, rlon} = Weather.round_to_hrrr_grid(lat, lon)
|
|
|
|
assert abs(rlat - lat) <= 0.005 + 1.0e-9
|
|
assert abs(rlon - lon) <= 0.005 + 1.0e-9
|
|
# Output is on a 0.01° grid
|
|
assert_in_delta rlat * 100, round(rlat * 100), 1.0e-6
|
|
assert_in_delta rlon * 100, round(rlon * 100), 1.0e-6
|
|
end
|
|
end
|
|
end
|
|
|
|
describe "property: round_to_iemre_grid/2" do
|
|
property "rounded output lands on the 0.125° IEMRE grid within ±0.0625° of the input" do
|
|
check all(
|
|
lat <- float(min: -60.0, max: 60.0),
|
|
lon <- float(min: -180.0, max: 180.0)
|
|
) do
|
|
{rlat, rlon} = Weather.round_to_iemre_grid(lat, lon)
|
|
|
|
# Must be on a 0.125° grid (8 per degree).
|
|
assert_in_delta rlat * 8, round(rlat * 8), 1.0e-6
|
|
assert_in_delta rlon * 8, round(rlon * 8), 1.0e-6
|
|
|
|
# And never farther than half a step from the input.
|
|
assert abs(rlat - lat) <= 0.0625 + 1.0e-9
|
|
assert abs(rlon - lon) <= 0.0625 + 1.0e-9
|
|
end
|
|
end
|
|
end
|
|
|
|
describe "property: find_or_create_station/1 insert-then-query round-trip" do
|
|
property "a newly created station is immediately returned by a duplicate call" do
|
|
check all(
|
|
suffix <- integer(1..1_000_000),
|
|
lat <- float(min: 25.0, max: 49.0),
|
|
lon <- float(min: -124.0, max: -66.0)
|
|
) do
|
|
code = "ROUND#{suffix}"
|
|
|
|
attrs = %{
|
|
station_code: code,
|
|
station_type: "asos",
|
|
name: "Round-trip station",
|
|
lat: lat,
|
|
lon: lon
|
|
}
|
|
|
|
assert {:ok, first} = Weather.find_or_create_station(attrs)
|
|
assert {:ok, second} = Weather.find_or_create_station(attrs)
|
|
assert first.id == second.id
|
|
assert second.station_code == code
|
|
end
|
|
end
|
|
end
|
|
end
|