- Spike docs at docs/research/hrrr_native_levels.md confirming files
are on AWS S3 for 5+ years, 50 hybrid levels, and include TKE and
SPFH needed for Phase 2 turbulence features. Architectural finding:
per-point on-demand fetching is impractical (~530 MB/file), so
the ingestion worker batches per (date, hour) instead.
- hrrr_native_profiles schema: arrays per level plus cached surface
scalars and placeholder columns for Phase 2/4 derived fields.
Strictly additive — the existing hrrr_profiles table is untouched.
- HrrrNativeClient: pure URL/message-list helpers, build_native_profile/1
that turns a parsed wgrib2 map into the schema shape (TDD'd).
- Exposed HrrrClient.download_grib_ranges/2 so the native client
reuses the existing parallel byte-range download + disk cache.
- HrrrNativeGridWorker: Oban worker keyed on {year, month, day, hour},
unique at :infinity, pulls distinct (lat, lon) points from contacts
in the ±30 min window, downloads the native grib2, extracts per
point, bulk-upserts.
- mix hrrr_native_backfill --limit N enqueues the top-N hours by
contact count.
Phase 1 gate still pending Task 1.6 (sanity-check backtest after
live data lands).
167 lines
5.6 KiB
Elixir
167 lines
5.6 KiB
Elixir
defmodule Microwaveprop.Weather.HrrrNativeClientTest do
|
||
use ExUnit.Case, async: true
|
||
|
||
alias Microwaveprop.Weather.HrrrClient
|
||
alias Microwaveprop.Weather.HrrrNativeClient
|
||
|
||
describe "hrrr_native_url/3" do
|
||
test "builds the wrfnatf00 URL for a given date and hour" do
|
||
url = HrrrNativeClient.hrrr_native_url(~D[2026-04-09], 12)
|
||
|
||
assert url ==
|
||
"https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20260409/conus/hrrr.t12z.wrfnatf00.grib2"
|
||
end
|
||
|
||
test "pads single-digit hours" do
|
||
url = HrrrNativeClient.hrrr_native_url(~D[2026-04-09], 6)
|
||
assert url =~ "hrrr.t06z."
|
||
end
|
||
|
||
test "supports non-zero forecast hours" do
|
||
url = HrrrNativeClient.hrrr_native_url(~D[2026-04-09], 12, 3)
|
||
assert url =~ "wrfnatf03.grib2"
|
||
end
|
||
end
|
||
|
||
describe "native_messages/0" do
|
||
test "lists 350 messages (7 vars × 50 levels)" do
|
||
messages = HrrrNativeClient.native_messages()
|
||
assert length(messages) == 350
|
||
end
|
||
|
||
test "every level appears with every essential variable" do
|
||
messages = HrrrNativeClient.native_messages()
|
||
|
||
# Pick level 5 — should have all 7 vars
|
||
level5 = Enum.filter(messages, fn %{level: l} -> l == "5 hybrid level" end)
|
||
vars = Enum.map(level5, & &1.var) |> Enum.sort()
|
||
|
||
assert vars == ~w(HGT PRES SPFH TKE TMP UGRD VGRD)
|
||
end
|
||
|
||
test "covers levels 1 through 50" do
|
||
levels =
|
||
HrrrNativeClient.native_messages()
|
||
|> Enum.map(& &1.level)
|
||
|> Enum.uniq()
|
||
|> Enum.map(fn level_str ->
|
||
[digits, _] = String.split(level_str, " ", parts: 2)
|
||
String.to_integer(digits)
|
||
end)
|
||
|> Enum.sort()
|
||
|
||
assert levels == Enum.to_list(1..50)
|
||
end
|
||
end
|
||
|
||
describe "build_native_profile/1" do
|
||
test "assembles arrays in level order and caches surface scalars" do
|
||
parsed = %{
|
||
"HGT:1 hybrid level" => 8.0,
|
||
"TMP:1 hybrid level" => 295.0,
|
||
"SPFH:1 hybrid level" => 0.010,
|
||
"PRES:1 hybrid level" => 101_000.0,
|
||
"UGRD:1 hybrid level" => 2.0,
|
||
"VGRD:1 hybrid level" => 1.0,
|
||
"TKE:1 hybrid level" => 0.5,
|
||
"HGT:2 hybrid level" => 25.0,
|
||
"TMP:2 hybrid level" => 293.0,
|
||
"SPFH:2 hybrid level" => 0.008,
|
||
"PRES:2 hybrid level" => 99_800.0,
|
||
"UGRD:2 hybrid level" => 3.0,
|
||
"VGRD:2 hybrid level" => 1.5,
|
||
"TKE:2 hybrid level" => 0.3,
|
||
"TMP:surface" => 295.5,
|
||
"SPFH:2 m above ground" => 0.011,
|
||
"PRES:surface" => 101_325.0
|
||
}
|
||
|
||
profile = HrrrNativeClient.build_native_profile(parsed)
|
||
|
||
assert profile.level_count == 2
|
||
assert profile.heights_m == [8.0, 25.0]
|
||
assert profile.temp_k == [295.0, 293.0]
|
||
assert profile.spfh == [0.010, 0.008]
|
||
assert profile.u_wind_ms == [2.0, 3.0]
|
||
assert profile.surface_temp_k == 295.5
|
||
assert profile.surface_spfh == 0.011
|
||
assert profile.surface_pressure_pa == 101_325.0
|
||
end
|
||
|
||
test "skips levels missing either HGT or TMP" do
|
||
parsed = %{
|
||
"HGT:1 hybrid level" => 8.0,
|
||
"TMP:1 hybrid level" => 295.0,
|
||
# level 2 has HGT but no TMP → skipped
|
||
"HGT:2 hybrid level" => 25.0,
|
||
"SPFH:2 hybrid level" => 0.008
|
||
}
|
||
|
||
profile = HrrrNativeClient.build_native_profile(parsed)
|
||
assert profile.level_count == 1
|
||
assert profile.heights_m == [8.0]
|
||
end
|
||
|
||
test "orders levels by ascending height, not by hybrid level number" do
|
||
# Pathological: level 2 happens to be lower than level 1 (terrain).
|
||
parsed = %{
|
||
"HGT:1 hybrid level" => 50.0,
|
||
"TMP:1 hybrid level" => 290.0,
|
||
"HGT:2 hybrid level" => 30.0,
|
||
"TMP:2 hybrid level" => 292.0
|
||
}
|
||
|
||
profile = HrrrNativeClient.build_native_profile(parsed)
|
||
assert profile.heights_m == [30.0, 50.0]
|
||
assert profile.temp_k == [292.0, 290.0]
|
||
end
|
||
|
||
test "falls back to lowest level for surface scalars when parsed has no surface entries" do
|
||
parsed = %{
|
||
"HGT:1 hybrid level" => 8.0,
|
||
"TMP:1 hybrid level" => 295.0,
|
||
"SPFH:1 hybrid level" => 0.010,
|
||
"PRES:1 hybrid level" => 101_000.0
|
||
}
|
||
|
||
profile = HrrrNativeClient.build_native_profile(parsed)
|
||
assert profile.surface_temp_k == 295.0
|
||
assert profile.surface_spfh == 0.010
|
||
assert profile.surface_pressure_pa == 101_000.0
|
||
end
|
||
end
|
||
|
||
describe "essential_byte_ranges/1" do
|
||
test "produces one range per essential message present in the idx" do
|
||
# Minimal fake idx: level 1 TMP/SPFH/HGT with nothing else.
|
||
idx_entries = [
|
||
%{msg: 1, offset: 0, var: "TMP", level: "1 hybrid level"},
|
||
%{msg: 2, offset: 1000, var: "SPFH", level: "1 hybrid level"},
|
||
%{msg: 3, offset: 2000, var: "HGT", level: "1 hybrid level"},
|
||
%{msg: 4, offset: 3000, var: "REFC", level: "entire atmosphere"}
|
||
]
|
||
|
||
ranges = HrrrNativeClient.essential_byte_ranges(idx_entries)
|
||
|
||
# Expect 3 ranges (the three essentials we actually have)
|
||
assert length(ranges) == 3
|
||
# Each range is {start, end} tuples, all integers
|
||
Enum.each(ranges, fn {s, e} ->
|
||
assert is_integer(s) and is_integer(e)
|
||
assert s < e
|
||
end)
|
||
end
|
||
|
||
test "delegates to HrrrClient.byte_ranges_for_messages/2" do
|
||
# Just check the two stay in sync by computing the same ranges
|
||
# two different ways.
|
||
idx_entries = [
|
||
%{msg: 1, offset: 0, var: "TMP", level: "1 hybrid level"},
|
||
%{msg: 2, offset: 5000, var: "UGRD", level: "1 hybrid level"}
|
||
]
|
||
|
||
assert HrrrNativeClient.essential_byte_ranges(idx_entries) ==
|
||
HrrrClient.byte_ranges_for_messages(idx_entries, HrrrNativeClient.native_messages())
|
||
end
|
||
end
|
||
end
|