Make GRIB2 section parser resilient to trailing padding and truncation

Instead of returning "malformed section" on trailing bytes or truncated
sections, attempt to return parsed results. This handles older HRRR
files (2019) that have padding bytes after the last section.
This commit is contained in:
Graham McIntire 2026-03-30 09:21:27 -05:00
parent 38bfd7caed
commit 5c5d398966
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
2 changed files with 133 additions and 5 deletions

View file

@ -55,18 +55,24 @@ defmodule Microwaveprop.Weather.Grib2.Section do
defp parse_sections(<<length::32-big, section_num::8, rest::binary>>, acc) when length > 5 do
body_length = length - 5
<<body::binary-size(body_length), remaining::binary>> = rest
acc = parse_section(section_num, body, acc)
parse_sections(remaining, acc)
if body_length <= byte_size(rest) do
<<body::binary-size(body_length), remaining::binary>> = rest
acc = parse_section(section_num, body, acc)
parse_sections(remaining, acc)
else
# Truncated section — return what we have
build_result(acc)
end
end
defp parse_sections(<<>>, acc) do
build_result(acc)
end
defp parse_sections(_other, _acc) do
{:error, "malformed section"}
# Trailing padding or short garbage — return what we have
defp parse_sections(_other, acc) do
build_result(acc)
end
defp parse_section(1, _body, acc) do

View file

@ -42,6 +42,20 @@ defmodule Microwaveprop.Weather.Grib2.SectionTest do
indicator = <<"GRIB", 0::16, 0::8, 1::8, 100::64-big>>
assert {:error, _} = Section.parse_message(indicator)
end
test "parses message with trailing padding bytes instead of 7777" do
msg = build_synthetic_grib2_with_trailing_padding()
assert {:ok, parsed} = Section.parse_message(msg)
assert parsed.product.var == "TMP"
assert parsed.bitmap == :none
end
test "parses message with truncated trailing section" do
msg = build_synthetic_grib2_with_truncated_section()
assert {:ok, parsed} = Section.parse_message(msg)
assert parsed.product.var == "TMP"
assert parsed.bitmap == :none
end
end
describe "product identification" do
@ -262,4 +276,112 @@ defmodule Microwaveprop.Weather.Grib2.SectionTest do
length = 5 + byte_size(body)
<<length::32-big, section_number::8, body::binary>>
end
# Like build_synthetic_grib2 but with 3 padding bytes instead of "7777"
defp build_synthetic_grib2_with_trailing_padding do
body = build_sections_body() <> <<0, 0, 0>>
total_length = 16 + byte_size(body)
sec0 = <<"GRIB", 0::16, 0::8, 2::8, total_length::64-big>>
sec0 <> body
end
# Like build_synthetic_grib2 but total_length includes extra bytes after "7777"
# that look like a truncated section header
defp build_synthetic_grib2_with_truncated_section do
# All valid sections + "7777" + a truncated next section (length says 500 but only 10 bytes)
truncated = <<500::32-big, 4::8, 0, 0, 0, 0, 0>>
body = build_sections_body() <> "7777" <> truncated
total_length = 16 + byte_size(body)
sec0 = <<"GRIB", 0::16, 0::8, 2::8, total_length::64-big>>
sec0 <> body
end
defp build_sections_body do
sec1_body = <<
0::16-big,
0::16-big,
2::8,
1::8,
1::8,
2026::16-big,
3::8,
28::8,
18::8,
0::8,
0::8,
0::8,
1::8
>>
sec3_body = <<
0::8,
1_905_141::32-big,
0::8,
0::8,
30::16-big,
6::8,
0::8,
0::32-big,
0::8,
0::32-big,
0::8,
0::32-big,
1799::32-big,
1059::32-big,
21_138_123::32-big,
237_280_472::32-big,
0b00111000::8,
38_500_000::32-big,
262_500_000::32-big,
3_000_000::32-big,
3_000_000::32-big,
0::8,
64::8,
38_500_000::32-big,
38_500_000::32-big,
0::32-big,
0::32-big
>>
sec4_body = <<
0::16-big,
0::16-big,
0::8,
0::8,
2::8,
0::8,
0::8,
0::16-big,
0::8,
1::8,
0::32-big,
103::8,
0::8,
2::32-big,
255::8,
0::8,
0::32-big
>>
ref_val = <<0.0::float-32-big>>
sec5_body = <<
100::32-big,
0::16-big,
ref_val::binary,
0::16-big,
0::16-big,
16::8
>>
sec6_body = <<255::8>>
data_bytes = :binary.copy(<<42::16-big>>, 100)
section_wrap(1, sec1_body) <>
section_wrap(3, sec3_body) <>
section_wrap(4, sec4_body) <>
section_wrap(5, sec5_body) <>
section_wrap(6, sec6_body) <>
section_wrap(7, data_bytes)
end
end