Skip corrupt GRIB2 messages instead of failing entire fetch

Both extract_points and extract_grid now skip messages with missing
data sections instead of halting. Pressure product fetch is now
optional for both grid and single-point modes — if pressure data
has corrupt messages, surface-only profiles are still stored. Fixes
historical HRRR backfill failing on old data with incomplete files.
This commit is contained in:
Graham McIntire 2026-03-31 08:12:10 -05:00
parent c4de1dd77c
commit 81cb735d6e
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
2 changed files with 40 additions and 15 deletions

View file

@ -14,16 +14,17 @@ defmodule Microwaveprop.Weather.Grib2.Extractor do
def extract_points(binary, lat, lon) do
messages = split_messages(binary)
results =
Enum.reduce_while(messages, {:ok, %{}}, fn msg, {:ok, acc} ->
Enum.reduce(messages, {:ok, %{}}, fn
msg, {:ok, acc} ->
case extract_single(msg, lat, lon) do
{:ok, key, value} -> {:cont, {:ok, Map.put(acc, key, value)}}
{:error, :outside_grid} -> {:halt, {:error, :outside_grid}}
{:error, reason} -> {:halt, {:error, reason}}
{:ok, key, value} -> {:ok, Map.put(acc, key, value)}
{:error, :outside_grid} -> {:error, :outside_grid}
{:error, _reason} -> {:ok, acc}
end
end)
results
_msg, error ->
error
end)
end
@doc """
@ -36,7 +37,7 @@ defmodule Microwaveprop.Weather.Grib2.Extractor do
messages = split_messages(binary)
result =
Enum.reduce_while(messages, {:ok, init_grid(points)}, fn msg, {:ok, acc} ->
Enum.reduce(messages, {:ok, init_grid(points)}, fn msg, {:ok, acc} ->
case extract_single_grid(msg, points) do
{:ok, point_values} ->
merged =
@ -44,10 +45,11 @@ defmodule Microwaveprop.Weather.Grib2.Extractor do
Map.update!(grid, point, &Map.put(&1, key, value))
end)
{:cont, {:ok, merged}}
{:ok, merged}
{:error, reason} ->
{:halt, {:error, reason}}
{:error, _reason} ->
# Skip messages with missing/corrupt data sections
{:ok, acc}
end
end)

View file

@ -30,8 +30,18 @@ defmodule Microwaveprop.Weather.HrrrClient do
date = DateTime.to_date(hour_dt)
hour = hour_dt.hour
with {:ok, sfc_grid} <- fetch_product_grid(date, hour, :surface, points),
{:ok, prs_grid} <- maybe_fetch_pressure_grid(date, hour, points, opts) do
with {:ok, sfc_grid} <- fetch_product_grid(date, hour, :surface, points) do
# Pressure is optional — if it fails, continue with surface-only data
prs_grid =
case maybe_fetch_pressure_grid(date, hour, points, opts) do
{:ok, grid} ->
grid
{:error, reason} ->
Logger.warning("HRRR pressure fetch failed (continuing with surface only): #{inspect(reason)}")
%{}
end
merged = merge_grid_data(sfc_grid, prs_grid)
profiles =
@ -49,8 +59,21 @@ defmodule Microwaveprop.Weather.HrrrClient do
date = DateTime.to_date(hour_dt)
hour = hour_dt.hour
with {:ok, sfc_data} <- fetch_product(date, hour, :surface, lat, lon),
{:ok, prs_data} <- fetch_product(date, hour, :pressure, lat, lon) do
with {:ok, sfc_data} <- fetch_product(date, hour, :surface, lat, lon) do
# Pressure is optional — old HRRR data may have corrupt messages
prs_data =
case fetch_product(date, hour, :pressure, lat, lon) do
{:ok, data} ->
data
{:error, reason} ->
Logger.warning(
"HRRR pressure fetch failed for #{lat},#{lon} (continuing with surface only): #{inspect(reason)}"
)
%{}
end
merged = Map.merge(sfc_data, prs_data)
result = build_profile(merged)
{:ok, Map.put(result, :run_time, hour_dt)}