prop/test/microwaveprop/weather/hrrr_native_client_test.exs
Graham McIntire e7a7ae073d Phase 9.3, 9.4, and Phase 3 NEXRAD pipeline
Task 9.3 - Weight recalibration via gradient descent:
- Recalibrator module fits logistic regression weights using Nx
- Trains on QSO positives vs random baseline negatives
- Cross-validates by month, normalizes weights to sum to 1.0
- Mix task: mix recalibrate_scorer --sample 5000 --epochs 2000

Task 9.4 - Side-by-side scorer comparison:
- ScorerDiff.compare/3 re-scores grid with old vs new weights
- Reports mean diff, regressions, improvements, per-band breakdown
- Mix task: mix scorer_diff --new-weights '{...}'

Phase 3 - NEXRAD ingestion pipeline:
- NexradClient fetches IEM n0q composite PNGs, extracts per-point
  box statistics (mean/max dBZ, texture variance)
- NexradObservation schema with unique (lat, lon, observed_at)
- NexradWorker on :nexrad queue for background processing
- nexrad_texture backtest feature in Features module
- mix nexrad_backfill --limit 200

All tasks added to AdminTaskWorker and Release for production use.
1116 tests, 0 failures.
2026-04-10 12:48:36 -05:00

167 lines
5.6 KiB
Elixir
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

defmodule Microwaveprop.Weather.HrrrNativeClientTest do
use ExUnit.Case, async: true
alias Microwaveprop.Weather.HrrrClient
alias Microwaveprop.Weather.HrrrNativeClient
describe "hrrr_native_url/3" do
test "builds the wrfnatf00 URL for a given date and hour" do
url = HrrrNativeClient.hrrr_native_url(~D[2026-04-09], 12)
assert url ==
"https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20260409/conus/hrrr.t12z.wrfnatf00.grib2"
end
test "pads single-digit hours" do
url = HrrrNativeClient.hrrr_native_url(~D[2026-04-09], 6)
assert url =~ "hrrr.t06z."
end
test "supports non-zero forecast hours" do
url = HrrrNativeClient.hrrr_native_url(~D[2026-04-09], 12, 3)
assert url =~ "wrfnatf03.grib2"
end
end
describe "native_messages/0" do
test "lists 350 messages (7 vars × 50 levels)" do
messages = HrrrNativeClient.native_messages()
assert length(messages) == 350
end
test "every level appears with every essential variable" do
messages = HrrrNativeClient.native_messages()
# Pick level 5 — should have all 7 vars
level5 = Enum.filter(messages, fn %{level: l} -> l == "5 hybrid level" end)
vars = level5 |> Enum.map(& &1.var) |> Enum.sort()
assert vars == ~w(HGT PRES SPFH TKE TMP UGRD VGRD)
end
test "covers levels 1 through 50" do
levels =
HrrrNativeClient.native_messages()
|> Enum.map(& &1.level)
|> Enum.uniq()
|> Enum.map(fn level_str ->
[digits, _] = String.split(level_str, " ", parts: 2)
String.to_integer(digits)
end)
|> Enum.sort()
assert levels == Enum.to_list(1..50)
end
end
describe "build_native_profile/1" do
test "assembles arrays in level order and caches surface scalars" do
parsed = %{
"HGT:1 hybrid level" => 8.0,
"TMP:1 hybrid level" => 295.0,
"SPFH:1 hybrid level" => 0.010,
"PRES:1 hybrid level" => 101_000.0,
"UGRD:1 hybrid level" => 2.0,
"VGRD:1 hybrid level" => 1.0,
"TKE:1 hybrid level" => 0.5,
"HGT:2 hybrid level" => 25.0,
"TMP:2 hybrid level" => 293.0,
"SPFH:2 hybrid level" => 0.008,
"PRES:2 hybrid level" => 99_800.0,
"UGRD:2 hybrid level" => 3.0,
"VGRD:2 hybrid level" => 1.5,
"TKE:2 hybrid level" => 0.3,
"TMP:surface" => 295.5,
"SPFH:2 m above ground" => 0.011,
"PRES:surface" => 101_325.0
}
profile = HrrrNativeClient.build_native_profile(parsed)
assert profile.level_count == 2
assert profile.heights_m == [8.0, 25.0]
assert profile.temp_k == [295.0, 293.0]
assert profile.spfh == [0.010, 0.008]
assert profile.u_wind_ms == [2.0, 3.0]
assert profile.surface_temp_k == 295.5
assert profile.surface_spfh == 0.011
assert profile.surface_pressure_pa == 101_325.0
end
test "skips levels missing either HGT or TMP" do
parsed = %{
"HGT:1 hybrid level" => 8.0,
"TMP:1 hybrid level" => 295.0,
# level 2 has HGT but no TMP → skipped
"HGT:2 hybrid level" => 25.0,
"SPFH:2 hybrid level" => 0.008
}
profile = HrrrNativeClient.build_native_profile(parsed)
assert profile.level_count == 1
assert profile.heights_m == [8.0]
end
test "orders levels by ascending height, not by hybrid level number" do
# Pathological: level 2 happens to be lower than level 1 (terrain).
parsed = %{
"HGT:1 hybrid level" => 50.0,
"TMP:1 hybrid level" => 290.0,
"HGT:2 hybrid level" => 30.0,
"TMP:2 hybrid level" => 292.0
}
profile = HrrrNativeClient.build_native_profile(parsed)
assert profile.heights_m == [30.0, 50.0]
assert profile.temp_k == [292.0, 290.0]
end
test "falls back to lowest level for surface scalars when parsed has no surface entries" do
parsed = %{
"HGT:1 hybrid level" => 8.0,
"TMP:1 hybrid level" => 295.0,
"SPFH:1 hybrid level" => 0.010,
"PRES:1 hybrid level" => 101_000.0
}
profile = HrrrNativeClient.build_native_profile(parsed)
assert profile.surface_temp_k == 295.0
assert profile.surface_spfh == 0.010
assert profile.surface_pressure_pa == 101_000.0
end
end
describe "essential_byte_ranges/1" do
test "produces one range per essential message present in the idx" do
# Minimal fake idx: level 1 TMP/SPFH/HGT with nothing else.
idx_entries = [
%{msg: 1, offset: 0, var: "TMP", level: "1 hybrid level"},
%{msg: 2, offset: 1000, var: "SPFH", level: "1 hybrid level"},
%{msg: 3, offset: 2000, var: "HGT", level: "1 hybrid level"},
%{msg: 4, offset: 3000, var: "REFC", level: "entire atmosphere"}
]
ranges = HrrrNativeClient.essential_byte_ranges(idx_entries)
# Expect 3 ranges (the three essentials we actually have)
assert length(ranges) == 3
# Each range is {start, end} tuples, all integers
Enum.each(ranges, fn {s, e} ->
assert is_integer(s) and is_integer(e)
assert s < e
end)
end
test "delegates to HrrrClient.byte_ranges_for_messages/2" do
# Just check the two stay in sync by computing the same ranges
# two different ways.
idx_entries = [
%{msg: 1, offset: 0, var: "TMP", level: "1 hybrid level"},
%{msg: 2, offset: 5000, var: "UGRD", level: "1 hybrid level"}
]
assert HrrrNativeClient.essential_byte_ranges(idx_entries) ==
HrrrClient.byte_ranges_for_messages(idx_entries, HrrrNativeClient.native_messages())
end
end
end