Fix native-level GRIB2 decoding and add wgrib2 extraction path
The Elixir GRIB2 decoder didn't map level type 105 (hybrid) or variable IDs for SPFH and TKE, so native-level messages decoded as "unknown:105:N" keys that build_native_profile couldn't find. Add the three missing mappings to Section.identify_level/identify_var. Also add HrrrNativeClient.extract_native_profiles/2 which uses wgrib2's -lola on a tight bounding-box subgrid for speed (the pure Elixir decoder takes ~70s per point on a 395 MB file; wgrib2 handles 40 points in seconds). The worker now routes through this path.
This commit is contained in:
parent
900685aa06
commit
f15b2aaa69
3 changed files with 94 additions and 8 deletions
|
|
@ -27,11 +27,13 @@ defmodule Microwaveprop.Weather.Grib2.Section do
|
|||
def identify_var(0, 3, 0), do: "PRES"
|
||||
def identify_var(0, 3, 5), do: "HGT"
|
||||
def identify_var(0, 3, 18), do: "HPBL"
|
||||
def identify_var(0, 1, 0), do: "SPFH"
|
||||
def identify_var(0, 1, 3), do: "PWAT"
|
||||
def identify_var(0, 1, 8), do: "APCP"
|
||||
def identify_var(0, 2, 2), do: "UGRD"
|
||||
def identify_var(0, 2, 3), do: "VGRD"
|
||||
def identify_var(0, 6, 1), do: "TCDC"
|
||||
def identify_var(0, 19, 11), do: "TKE"
|
||||
def identify_var(_discipline, cat, num), do: "#{cat}:#{num}"
|
||||
|
||||
@doc """
|
||||
|
|
@ -42,6 +44,7 @@ defmodule Microwaveprop.Weather.Grib2.Section do
|
|||
def identify_level(103, value_m), do: "#{value_m} m above ground"
|
||||
|
||||
def identify_level(10, _value), do: "entire atmosphere"
|
||||
def identify_level(105, level), do: "#{level} hybrid level"
|
||||
def identify_level(200, _value), do: "entire atmosphere (considered as a single layer)"
|
||||
|
||||
def identify_level(type, value), do: "unknown:#{type}:#{value}"
|
||||
|
|
|
|||
|
|
@ -123,4 +123,90 @@ defmodule Microwaveprop.Weather.HrrrNativeClient do
|
|||
def essential_byte_ranges(idx_entries) do
|
||||
HrrrClient.byte_ranges_for_messages(idx_entries, native_messages())
|
||||
end
|
||||
|
||||
@doc """
|
||||
Extract native profiles for a list of `{lat, lon}` points from a
|
||||
GRIB2 binary, using wgrib2 for speed.
|
||||
|
||||
Under the hood this uses `-lola` on a bounding-box subgrid that
|
||||
covers all the requested points, then does nearest-neighbor lookup
|
||||
per point. Falls back to the pure-Elixir decoder if wgrib2 is not
|
||||
available (expect ~70s per point in that case).
|
||||
|
||||
Returns `%{{lat, lon} => native_profile_map}` where each profile
|
||||
map has the shape expected by `HrrrNativeProfile.changeset/2`.
|
||||
"""
|
||||
def extract_native_profiles(grib_binary, points) when is_list(points) do
|
||||
alias Microwaveprop.Weather.Grib2.Wgrib2
|
||||
|
||||
match_pattern = ":(#{Enum.join(@native_variables, "|")}):"
|
||||
|
||||
if Wgrib2.available?() do
|
||||
grid_spec = bounding_grid(points)
|
||||
|
||||
case Wgrib2.extract_grid(grib_binary, match_pattern, grid_spec) do
|
||||
{:ok, grid_data} ->
|
||||
# Nearest-neighbor lookup: for each requested point, find
|
||||
# the grid cell with the smallest (lat, lon) distance.
|
||||
result =
|
||||
Map.new(points, fn {lat, lon} ->
|
||||
nearest = nearest_grid_cell(grid_data, lat, lon)
|
||||
profile = if nearest, do: build_native_profile(nearest), else: %{level_count: 0}
|
||||
{{lat, lon}, profile}
|
||||
end)
|
||||
|
||||
{:ok, result}
|
||||
|
||||
error ->
|
||||
error
|
||||
end
|
||||
else
|
||||
# Fallback: pure Elixir (slow)
|
||||
alias Microwaveprop.Weather.Grib2.Extractor
|
||||
|
||||
case Extractor.extract_grid(grib_binary, points) do
|
||||
{:ok, grid_data} ->
|
||||
result = Map.new(grid_data, fn {pt, parsed} -> {pt, build_native_profile(parsed)} end)
|
||||
{:ok, result}
|
||||
|
||||
error ->
|
||||
error
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Build a -lola grid spec that covers all points with 0.03° padding.
|
||||
@grid_step 0.03
|
||||
defp bounding_grid(points) do
|
||||
lats = Enum.map(points, &elem(&1, 0))
|
||||
lons = Enum.map(points, &elem(&1, 1))
|
||||
|
||||
lat_min = Enum.min(lats) - 0.1
|
||||
lat_max = Enum.max(lats) + 0.1
|
||||
lon_min = Enum.min(lons) - 0.1
|
||||
lon_max = Enum.max(lons) + 0.1
|
||||
|
||||
lon_count = max(trunc(Float.ceil((lon_max - lon_min) / @grid_step)), 2)
|
||||
lat_count = max(trunc(Float.ceil((lat_max - lat_min) / @grid_step)), 2)
|
||||
|
||||
%{
|
||||
lon_start: lon_min,
|
||||
lon_count: lon_count,
|
||||
lon_step: @grid_step,
|
||||
lat_start: lat_min,
|
||||
lat_count: lat_count,
|
||||
lat_step: @grid_step
|
||||
}
|
||||
end
|
||||
|
||||
defp nearest_grid_cell(grid_data, lat, lon) do
|
||||
grid_data
|
||||
|> Enum.min_by(fn {{glat, glon}, _} ->
|
||||
:math.pow(glat - lat, 2) + :math.pow(glon - lon, 2)
|
||||
end, fn -> nil end)
|
||||
|> case do
|
||||
nil -> nil
|
||||
{_point, parsed} -> parsed
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ defmodule Microwaveprop.Workers.HrrrNativeGridWorker do
|
|||
|
||||
alias Microwaveprop.Radio.Contact
|
||||
alias Microwaveprop.Repo
|
||||
alias Microwaveprop.Weather.Grib2.Extractor
|
||||
alias Microwaveprop.Weather.HrrrClient
|
||||
alias Microwaveprop.Weather.HrrrNativeClient
|
||||
alias Microwaveprop.Weather.HrrrNativeProfile
|
||||
|
|
@ -110,14 +109,12 @@ defmodule Microwaveprop.Workers.HrrrNativeGridWorker do
|
|||
ranges = HrrrNativeClient.essential_byte_ranges(idx_entries),
|
||||
_ = Logger.info("HRRR native downloading #{length(ranges)} ranges for #{valid_time}"),
|
||||
{:ok, grib_binary} <- HrrrClient.download_grib_ranges(url, ranges),
|
||||
_ = Logger.info("HRRR native downloaded #{byte_size(grib_binary)} bytes"),
|
||||
{:ok, grid_data} <- Extractor.extract_grid(grib_binary, points) do
|
||||
_ = Logger.info("HRRR native downloaded #{byte_size(grib_binary)} bytes, extracting #{length(points)} points..."),
|
||||
{:ok, profiles} <- HrrrNativeClient.extract_native_profiles(grib_binary, points) do
|
||||
rows =
|
||||
grid_data
|
||||
|> Enum.flat_map(fn {{lat, lon}, parsed} ->
|
||||
profile = HrrrNativeClient.build_native_profile(parsed)
|
||||
|
||||
if profile.level_count > 0 do
|
||||
profiles
|
||||
|> Enum.flat_map(fn {{lat, lon}, profile} ->
|
||||
if profile[:level_count] && profile.level_count > 0 do
|
||||
now = DateTime.truncate(DateTime.utc_now(), :second)
|
||||
|
||||
[
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue