Make GRIB2 section parser resilient to trailing padding and truncation
Instead of returning "malformed section" on trailing bytes or truncated sections, attempt to return parsed results. This handles older HRRR files (2019) that have padding bytes after the last section.
This commit is contained in:
parent
38bfd7caed
commit
5c5d398966
2 changed files with 133 additions and 5 deletions
|
|
@ -55,18 +55,24 @@ defmodule Microwaveprop.Weather.Grib2.Section do
|
|||
|
||||
defp parse_sections(<<length::32-big, section_num::8, rest::binary>>, acc) when length > 5 do
|
||||
body_length = length - 5
|
||||
<<body::binary-size(body_length), remaining::binary>> = rest
|
||||
|
||||
acc = parse_section(section_num, body, acc)
|
||||
parse_sections(remaining, acc)
|
||||
if body_length <= byte_size(rest) do
|
||||
<<body::binary-size(body_length), remaining::binary>> = rest
|
||||
acc = parse_section(section_num, body, acc)
|
||||
parse_sections(remaining, acc)
|
||||
else
|
||||
# Truncated section — return what we have
|
||||
build_result(acc)
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_sections(<<>>, acc) do
|
||||
build_result(acc)
|
||||
end
|
||||
|
||||
defp parse_sections(_other, _acc) do
|
||||
{:error, "malformed section"}
|
||||
# Trailing padding or short garbage — return what we have
|
||||
defp parse_sections(_other, acc) do
|
||||
build_result(acc)
|
||||
end
|
||||
|
||||
defp parse_section(1, _body, acc) do
|
||||
|
|
|
|||
|
|
@ -42,6 +42,20 @@ defmodule Microwaveprop.Weather.Grib2.SectionTest do
|
|||
indicator = <<"GRIB", 0::16, 0::8, 1::8, 100::64-big>>
|
||||
assert {:error, _} = Section.parse_message(indicator)
|
||||
end
|
||||
|
||||
test "parses message with trailing padding bytes instead of 7777" do
|
||||
msg = build_synthetic_grib2_with_trailing_padding()
|
||||
assert {:ok, parsed} = Section.parse_message(msg)
|
||||
assert parsed.product.var == "TMP"
|
||||
assert parsed.bitmap == :none
|
||||
end
|
||||
|
||||
test "parses message with truncated trailing section" do
|
||||
msg = build_synthetic_grib2_with_truncated_section()
|
||||
assert {:ok, parsed} = Section.parse_message(msg)
|
||||
assert parsed.product.var == "TMP"
|
||||
assert parsed.bitmap == :none
|
||||
end
|
||||
end
|
||||
|
||||
describe "product identification" do
|
||||
|
|
@ -262,4 +276,112 @@ defmodule Microwaveprop.Weather.Grib2.SectionTest do
|
|||
length = 5 + byte_size(body)
|
||||
<<length::32-big, section_number::8, body::binary>>
|
||||
end
|
||||
|
||||
# Like build_synthetic_grib2 but with 3 padding bytes instead of "7777"
|
||||
defp build_synthetic_grib2_with_trailing_padding do
|
||||
body = build_sections_body() <> <<0, 0, 0>>
|
||||
total_length = 16 + byte_size(body)
|
||||
sec0 = <<"GRIB", 0::16, 0::8, 2::8, total_length::64-big>>
|
||||
sec0 <> body
|
||||
end
|
||||
|
||||
# Like build_synthetic_grib2 but total_length includes extra bytes after "7777"
|
||||
# that look like a truncated section header
|
||||
defp build_synthetic_grib2_with_truncated_section do
|
||||
# All valid sections + "7777" + a truncated next section (length says 500 but only 10 bytes)
|
||||
truncated = <<500::32-big, 4::8, 0, 0, 0, 0, 0>>
|
||||
body = build_sections_body() <> "7777" <> truncated
|
||||
total_length = 16 + byte_size(body)
|
||||
sec0 = <<"GRIB", 0::16, 0::8, 2::8, total_length::64-big>>
|
||||
sec0 <> body
|
||||
end
|
||||
|
||||
defp build_sections_body do
|
||||
sec1_body = <<
|
||||
0::16-big,
|
||||
0::16-big,
|
||||
2::8,
|
||||
1::8,
|
||||
1::8,
|
||||
2026::16-big,
|
||||
3::8,
|
||||
28::8,
|
||||
18::8,
|
||||
0::8,
|
||||
0::8,
|
||||
0::8,
|
||||
1::8
|
||||
>>
|
||||
|
||||
sec3_body = <<
|
||||
0::8,
|
||||
1_905_141::32-big,
|
||||
0::8,
|
||||
0::8,
|
||||
30::16-big,
|
||||
6::8,
|
||||
0::8,
|
||||
0::32-big,
|
||||
0::8,
|
||||
0::32-big,
|
||||
0::8,
|
||||
0::32-big,
|
||||
1799::32-big,
|
||||
1059::32-big,
|
||||
21_138_123::32-big,
|
||||
237_280_472::32-big,
|
||||
0b00111000::8,
|
||||
38_500_000::32-big,
|
||||
262_500_000::32-big,
|
||||
3_000_000::32-big,
|
||||
3_000_000::32-big,
|
||||
0::8,
|
||||
64::8,
|
||||
38_500_000::32-big,
|
||||
38_500_000::32-big,
|
||||
0::32-big,
|
||||
0::32-big
|
||||
>>
|
||||
|
||||
sec4_body = <<
|
||||
0::16-big,
|
||||
0::16-big,
|
||||
0::8,
|
||||
0::8,
|
||||
2::8,
|
||||
0::8,
|
||||
0::8,
|
||||
0::16-big,
|
||||
0::8,
|
||||
1::8,
|
||||
0::32-big,
|
||||
103::8,
|
||||
0::8,
|
||||
2::32-big,
|
||||
255::8,
|
||||
0::8,
|
||||
0::32-big
|
||||
>>
|
||||
|
||||
ref_val = <<0.0::float-32-big>>
|
||||
|
||||
sec5_body = <<
|
||||
100::32-big,
|
||||
0::16-big,
|
||||
ref_val::binary,
|
||||
0::16-big,
|
||||
0::16-big,
|
||||
16::8
|
||||
>>
|
||||
|
||||
sec6_body = <<255::8>>
|
||||
data_bytes = :binary.copy(<<42::16-big>>, 100)
|
||||
|
||||
section_wrap(1, sec1_body) <>
|
||||
section_wrap(3, sec3_body) <>
|
||||
section_wrap(4, sec4_body) <>
|
||||
section_wrap(5, sec5_body) <>
|
||||
section_wrap(6, sec6_body) <>
|
||||
section_wrap(7, data_bytes)
|
||||
end
|
||||
end
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue