Fix native-level GRIB2 decoding and add wgrib2 extraction path

The Elixir GRIB2 decoder didn't map level type 105 (hybrid) or
variable IDs for SPFH and TKE, so native-level messages decoded as
"unknown:105:N" keys that build_native_profile couldn't find. Add
the three missing mappings to Section.identify_level/identify_var.

Also add HrrrNativeClient.extract_native_profiles/2 which uses
wgrib2's -lola on a tight bounding-box subgrid for speed (the pure
Elixir decoder takes ~70s per point on a 395 MB file; wgrib2 handles
40 points in seconds). The worker now routes through this path.
This commit is contained in:
Graham McIntire 2026-04-10 07:54:08 -05:00
parent 900685aa06
commit f15b2aaa69
3 changed files with 94 additions and 8 deletions

View file

@ -27,11 +27,13 @@ defmodule Microwaveprop.Weather.Grib2.Section do
def identify_var(0, 3, 0), do: "PRES"
def identify_var(0, 3, 5), do: "HGT"
def identify_var(0, 3, 18), do: "HPBL"
def identify_var(0, 1, 0), do: "SPFH"
def identify_var(0, 1, 3), do: "PWAT"
def identify_var(0, 1, 8), do: "APCP"
def identify_var(0, 2, 2), do: "UGRD"
def identify_var(0, 2, 3), do: "VGRD"
def identify_var(0, 6, 1), do: "TCDC"
def identify_var(0, 19, 11), do: "TKE"
def identify_var(_discipline, cat, num), do: "#{cat}:#{num}"
@doc """
@ -42,6 +44,7 @@ defmodule Microwaveprop.Weather.Grib2.Section do
def identify_level(103, value_m), do: "#{value_m} m above ground"
def identify_level(10, _value), do: "entire atmosphere"
def identify_level(105, level), do: "#{level} hybrid level"
def identify_level(200, _value), do: "entire atmosphere (considered as a single layer)"
def identify_level(type, value), do: "unknown:#{type}:#{value}"

View file

@ -123,4 +123,90 @@ defmodule Microwaveprop.Weather.HrrrNativeClient do
def essential_byte_ranges(idx_entries) do
HrrrClient.byte_ranges_for_messages(idx_entries, native_messages())
end
@doc """
Extract native profiles for a list of `{lat, lon}` points from a
GRIB2 binary, using wgrib2 for speed.
Under the hood this uses `-lola` on a bounding-box subgrid that
covers all the requested points, then does nearest-neighbor lookup
per point. Falls back to the pure-Elixir decoder if wgrib2 is not
available (expect ~70s per point in that case).
Returns `%{{lat, lon} => native_profile_map}` where each profile
map has the shape expected by `HrrrNativeProfile.changeset/2`.
"""
def extract_native_profiles(grib_binary, points) when is_list(points) do
alias Microwaveprop.Weather.Grib2.Wgrib2
match_pattern = ":(#{Enum.join(@native_variables, "|")}):"
if Wgrib2.available?() do
grid_spec = bounding_grid(points)
case Wgrib2.extract_grid(grib_binary, match_pattern, grid_spec) do
{:ok, grid_data} ->
# Nearest-neighbor lookup: for each requested point, find
# the grid cell with the smallest (lat, lon) distance.
result =
Map.new(points, fn {lat, lon} ->
nearest = nearest_grid_cell(grid_data, lat, lon)
profile = if nearest, do: build_native_profile(nearest), else: %{level_count: 0}
{{lat, lon}, profile}
end)
{:ok, result}
error ->
error
end
else
# Fallback: pure Elixir (slow)
alias Microwaveprop.Weather.Grib2.Extractor
case Extractor.extract_grid(grib_binary, points) do
{:ok, grid_data} ->
result = Map.new(grid_data, fn {pt, parsed} -> {pt, build_native_profile(parsed)} end)
{:ok, result}
error ->
error
end
end
end
# Build a -lola grid spec that covers all points with 0.03° padding.
@grid_step 0.03
defp bounding_grid(points) do
lats = Enum.map(points, &elem(&1, 0))
lons = Enum.map(points, &elem(&1, 1))
lat_min = Enum.min(lats) - 0.1
lat_max = Enum.max(lats) + 0.1
lon_min = Enum.min(lons) - 0.1
lon_max = Enum.max(lons) + 0.1
lon_count = max(trunc(Float.ceil((lon_max - lon_min) / @grid_step)), 2)
lat_count = max(trunc(Float.ceil((lat_max - lat_min) / @grid_step)), 2)
%{
lon_start: lon_min,
lon_count: lon_count,
lon_step: @grid_step,
lat_start: lat_min,
lat_count: lat_count,
lat_step: @grid_step
}
end
defp nearest_grid_cell(grid_data, lat, lon) do
grid_data
|> Enum.min_by(fn {{glat, glon}, _} ->
:math.pow(glat - lat, 2) + :math.pow(glon - lon, 2)
end, fn -> nil end)
|> case do
nil -> nil
{_point, parsed} -> parsed
end
end
end

View file

@ -28,7 +28,6 @@ defmodule Microwaveprop.Workers.HrrrNativeGridWorker do
alias Microwaveprop.Radio.Contact
alias Microwaveprop.Repo
alias Microwaveprop.Weather.Grib2.Extractor
alias Microwaveprop.Weather.HrrrClient
alias Microwaveprop.Weather.HrrrNativeClient
alias Microwaveprop.Weather.HrrrNativeProfile
@ -110,14 +109,12 @@ defmodule Microwaveprop.Workers.HrrrNativeGridWorker do
ranges = HrrrNativeClient.essential_byte_ranges(idx_entries),
_ = Logger.info("HRRR native downloading #{length(ranges)} ranges for #{valid_time}"),
{:ok, grib_binary} <- HrrrClient.download_grib_ranges(url, ranges),
_ = Logger.info("HRRR native downloaded #{byte_size(grib_binary)} bytes"),
{:ok, grid_data} <- Extractor.extract_grid(grib_binary, points) do
_ = Logger.info("HRRR native downloaded #{byte_size(grib_binary)} bytes, extracting #{length(points)} points..."),
{:ok, profiles} <- HrrrNativeClient.extract_native_profiles(grib_binary, points) do
rows =
grid_data
|> Enum.flat_map(fn {{lat, lon}, parsed} ->
profile = HrrrNativeClient.build_native_profile(parsed)
if profile.level_count > 0 do
profiles
|> Enum.flat_map(fn {{lat, lon}, profile} ->
if profile[:level_count] && profile.level_count > 0 do
now = DateTime.truncate(DateTime.utc_now(), :second)
[