prop/test/microwaveprop/weather/hrrr_native_client_test.exs
Graham McIntire 900685aa06 Phase 1 tasks 1.1-1.5: HRRR native hybrid-sigma ingestion
- Spike docs at docs/research/hrrr_native_levels.md confirming files
  are on AWS S3 for 5+ years, 50 hybrid levels, and include TKE and
  SPFH needed for Phase 2 turbulence features. Architectural finding:
  per-point on-demand fetching is impractical (~530 MB/file), so
  the ingestion worker batches per (date, hour) instead.
- hrrr_native_profiles schema: arrays per level plus cached surface
  scalars and placeholder columns for Phase 2/4 derived fields.
  Strictly additive — the existing hrrr_profiles table is untouched.
- HrrrNativeClient: pure URL/message-list helpers, build_native_profile/1
  that turns a parsed wgrib2 map into the schema shape (TDD'd).
- Exposed HrrrClient.download_grib_ranges/2 so the native client
  reuses the existing parallel byte-range download + disk cache.
- HrrrNativeGridWorker: Oban worker keyed on {year, month, day, hour},
  unique at :infinity, pulls distinct (lat, lon) points from contacts
  in the ±30 min window, downloads the native grib2, extracts per
  point, bulk-upserts.
- mix hrrr_native_backfill --limit N enqueues the top-N hours by
  contact count.

Phase 1 gate still pending Task 1.6 (sanity-check backtest after
live data lands).
2026-04-09 16:23:51 -05:00

167 lines
5.6 KiB
Elixir
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

defmodule Microwaveprop.Weather.HrrrNativeClientTest do
use ExUnit.Case, async: true
alias Microwaveprop.Weather.HrrrClient
alias Microwaveprop.Weather.HrrrNativeClient
describe "hrrr_native_url/3" do
test "builds the wrfnatf00 URL for a given date and hour" do
url = HrrrNativeClient.hrrr_native_url(~D[2026-04-09], 12)
assert url ==
"https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20260409/conus/hrrr.t12z.wrfnatf00.grib2"
end
test "pads single-digit hours" do
url = HrrrNativeClient.hrrr_native_url(~D[2026-04-09], 6)
assert url =~ "hrrr.t06z."
end
test "supports non-zero forecast hours" do
url = HrrrNativeClient.hrrr_native_url(~D[2026-04-09], 12, 3)
assert url =~ "wrfnatf03.grib2"
end
end
describe "native_messages/0" do
test "lists 350 messages (7 vars × 50 levels)" do
messages = HrrrNativeClient.native_messages()
assert length(messages) == 350
end
test "every level appears with every essential variable" do
messages = HrrrNativeClient.native_messages()
# Pick level 5 — should have all 7 vars
level5 = Enum.filter(messages, fn %{level: l} -> l == "5 hybrid level" end)
vars = Enum.map(level5, & &1.var) |> Enum.sort()
assert vars == ~w(HGT PRES SPFH TKE TMP UGRD VGRD)
end
test "covers levels 1 through 50" do
levels =
HrrrNativeClient.native_messages()
|> Enum.map(& &1.level)
|> Enum.uniq()
|> Enum.map(fn level_str ->
[digits, _] = String.split(level_str, " ", parts: 2)
String.to_integer(digits)
end)
|> Enum.sort()
assert levels == Enum.to_list(1..50)
end
end
describe "build_native_profile/1" do
test "assembles arrays in level order and caches surface scalars" do
parsed = %{
"HGT:1 hybrid level" => 8.0,
"TMP:1 hybrid level" => 295.0,
"SPFH:1 hybrid level" => 0.010,
"PRES:1 hybrid level" => 101_000.0,
"UGRD:1 hybrid level" => 2.0,
"VGRD:1 hybrid level" => 1.0,
"TKE:1 hybrid level" => 0.5,
"HGT:2 hybrid level" => 25.0,
"TMP:2 hybrid level" => 293.0,
"SPFH:2 hybrid level" => 0.008,
"PRES:2 hybrid level" => 99_800.0,
"UGRD:2 hybrid level" => 3.0,
"VGRD:2 hybrid level" => 1.5,
"TKE:2 hybrid level" => 0.3,
"TMP:surface" => 295.5,
"SPFH:2 m above ground" => 0.011,
"PRES:surface" => 101_325.0
}
profile = HrrrNativeClient.build_native_profile(parsed)
assert profile.level_count == 2
assert profile.heights_m == [8.0, 25.0]
assert profile.temp_k == [295.0, 293.0]
assert profile.spfh == [0.010, 0.008]
assert profile.u_wind_ms == [2.0, 3.0]
assert profile.surface_temp_k == 295.5
assert profile.surface_spfh == 0.011
assert profile.surface_pressure_pa == 101_325.0
end
test "skips levels missing either HGT or TMP" do
parsed = %{
"HGT:1 hybrid level" => 8.0,
"TMP:1 hybrid level" => 295.0,
# level 2 has HGT but no TMP → skipped
"HGT:2 hybrid level" => 25.0,
"SPFH:2 hybrid level" => 0.008
}
profile = HrrrNativeClient.build_native_profile(parsed)
assert profile.level_count == 1
assert profile.heights_m == [8.0]
end
test "orders levels by ascending height, not by hybrid level number" do
# Pathological: level 2 happens to be lower than level 1 (terrain).
parsed = %{
"HGT:1 hybrid level" => 50.0,
"TMP:1 hybrid level" => 290.0,
"HGT:2 hybrid level" => 30.0,
"TMP:2 hybrid level" => 292.0
}
profile = HrrrNativeClient.build_native_profile(parsed)
assert profile.heights_m == [30.0, 50.0]
assert profile.temp_k == [292.0, 290.0]
end
test "falls back to lowest level for surface scalars when parsed has no surface entries" do
parsed = %{
"HGT:1 hybrid level" => 8.0,
"TMP:1 hybrid level" => 295.0,
"SPFH:1 hybrid level" => 0.010,
"PRES:1 hybrid level" => 101_000.0
}
profile = HrrrNativeClient.build_native_profile(parsed)
assert profile.surface_temp_k == 295.0
assert profile.surface_spfh == 0.010
assert profile.surface_pressure_pa == 101_000.0
end
end
describe "essential_byte_ranges/1" do
test "produces one range per essential message present in the idx" do
# Minimal fake idx: level 1 TMP/SPFH/HGT with nothing else.
idx_entries = [
%{msg: 1, offset: 0, var: "TMP", level: "1 hybrid level"},
%{msg: 2, offset: 1000, var: "SPFH", level: "1 hybrid level"},
%{msg: 3, offset: 2000, var: "HGT", level: "1 hybrid level"},
%{msg: 4, offset: 3000, var: "REFC", level: "entire atmosphere"}
]
ranges = HrrrNativeClient.essential_byte_ranges(idx_entries)
# Expect 3 ranges (the three essentials we actually have)
assert length(ranges) == 3
# Each range is {start, end} tuples, all integers
Enum.each(ranges, fn {s, e} ->
assert is_integer(s) and is_integer(e)
assert s < e
end)
end
test "delegates to HrrrClient.byte_ranges_for_messages/2" do
# Just check the two stay in sync by computing the same ranges
# two different ways.
idx_entries = [
%{msg: 1, offset: 0, var: "TMP", level: "1 hybrid level"},
%{msg: 2, offset: 5000, var: "UGRD", level: "1 hybrid level"}
]
assert HrrrNativeClient.essential_byte_ranges(idx_entries) ==
HrrrClient.byte_ranges_for_messages(idx_entries, HrrrNativeClient.native_messages())
end
end
end