Skip corrupt GRIB2 messages instead of failing entire fetch

Both extract_points and extract_grid now skip messages with missing
data sections instead of halting. Pressure product fetch is now
optional for both grid and single-point modes — if pressure data
has corrupt messages, surface-only profiles are still stored. Fixes
historical HRRR backfill failing on old data with incomplete files.
This commit is contained in:
Graham McIntire 2026-03-31 08:12:10 -05:00
parent c4de1dd77c
commit 81cb735d6e
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
2 changed files with 40 additions and 15 deletions

View file

@ -14,16 +14,17 @@ defmodule Microwaveprop.Weather.Grib2.Extractor do
def extract_points(binary, lat, lon) do def extract_points(binary, lat, lon) do
messages = split_messages(binary) messages = split_messages(binary)
results = Enum.reduce(messages, {:ok, %{}}, fn
Enum.reduce_while(messages, {:ok, %{}}, fn msg, {:ok, acc} -> msg, {:ok, acc} ->
case extract_single(msg, lat, lon) do case extract_single(msg, lat, lon) do
{:ok, key, value} -> {:cont, {:ok, Map.put(acc, key, value)}} {:ok, key, value} -> {:ok, Map.put(acc, key, value)}
{:error, :outside_grid} -> {:halt, {:error, :outside_grid}} {:error, :outside_grid} -> {:error, :outside_grid}
{:error, reason} -> {:halt, {:error, reason}} {:error, _reason} -> {:ok, acc}
end end
end)
results _msg, error ->
error
end)
end end
@doc """ @doc """
@ -36,7 +37,7 @@ defmodule Microwaveprop.Weather.Grib2.Extractor do
messages = split_messages(binary) messages = split_messages(binary)
result = result =
Enum.reduce_while(messages, {:ok, init_grid(points)}, fn msg, {:ok, acc} -> Enum.reduce(messages, {:ok, init_grid(points)}, fn msg, {:ok, acc} ->
case extract_single_grid(msg, points) do case extract_single_grid(msg, points) do
{:ok, point_values} -> {:ok, point_values} ->
merged = merged =
@ -44,10 +45,11 @@ defmodule Microwaveprop.Weather.Grib2.Extractor do
Map.update!(grid, point, &Map.put(&1, key, value)) Map.update!(grid, point, &Map.put(&1, key, value))
end) end)
{:cont, {:ok, merged}} {:ok, merged}
{:error, reason} -> {:error, _reason} ->
{:halt, {:error, reason}} # Skip messages with missing/corrupt data sections
{:ok, acc}
end end
end) end)

View file

@ -30,8 +30,18 @@ defmodule Microwaveprop.Weather.HrrrClient do
date = DateTime.to_date(hour_dt) date = DateTime.to_date(hour_dt)
hour = hour_dt.hour hour = hour_dt.hour
with {:ok, sfc_grid} <- fetch_product_grid(date, hour, :surface, points), with {:ok, sfc_grid} <- fetch_product_grid(date, hour, :surface, points) do
{:ok, prs_grid} <- maybe_fetch_pressure_grid(date, hour, points, opts) do # Pressure is optional — if it fails, continue with surface-only data
prs_grid =
case maybe_fetch_pressure_grid(date, hour, points, opts) do
{:ok, grid} ->
grid
{:error, reason} ->
Logger.warning("HRRR pressure fetch failed (continuing with surface only): #{inspect(reason)}")
%{}
end
merged = merge_grid_data(sfc_grid, prs_grid) merged = merge_grid_data(sfc_grid, prs_grid)
profiles = profiles =
@ -49,8 +59,21 @@ defmodule Microwaveprop.Weather.HrrrClient do
date = DateTime.to_date(hour_dt) date = DateTime.to_date(hour_dt)
hour = hour_dt.hour hour = hour_dt.hour
with {:ok, sfc_data} <- fetch_product(date, hour, :surface, lat, lon), with {:ok, sfc_data} <- fetch_product(date, hour, :surface, lat, lon) do
{:ok, prs_data} <- fetch_product(date, hour, :pressure, lat, lon) do # Pressure is optional — old HRRR data may have corrupt messages
prs_data =
case fetch_product(date, hour, :pressure, lat, lon) do
{:ok, data} ->
data
{:error, reason} ->
Logger.warning(
"HRRR pressure fetch failed for #{lat},#{lon} (continuing with surface only): #{inspect(reason)}"
)
%{}
end
merged = Map.merge(sfc_data, prs_data) merged = Map.merge(sfc_data, prs_data)
result = build_profile(merged) result = build_profile(merged)
{:ok, Map.put(result, :run_time, hour_dt)} {:ok, Map.put(result, :run_time, hour_dt)}