Graham McIntire
5c5d398966
Instead of returning "malformed section" on trailing bytes or truncated sections, attempt to return parsed results. This handles older HRRR files (2019) that have padding bytes after the last section.
387 lines
9.9 KiB
Elixir
387 lines
9.9 KiB
Elixir
defmodule Microwaveprop.Weather.Grib2.SectionTest do
|
|
use ExUnit.Case, async: true
|
|
|
|
alias Microwaveprop.Weather.Grib2.Section
|
|
|
|
describe "parse_message/1" do
|
|
test "parses a synthetic GRIB2 message" do
|
|
msg = build_synthetic_grib2()
|
|
assert {:ok, parsed} = Section.parse_message(msg)
|
|
|
|
# Grid params from Section 3
|
|
assert parsed.grid_params.nx == 1799
|
|
assert parsed.grid_params.ny == 1059
|
|
assert_in_delta parsed.grid_params.la1, 21.138123, 0.000001
|
|
assert_in_delta parsed.grid_params.lo1, 237.280472, 0.000001
|
|
assert_in_delta parsed.grid_params.dx, 3000.0, 0.1
|
|
assert_in_delta parsed.grid_params.dy, 3000.0, 0.1
|
|
assert_in_delta parsed.grid_params.latin1, 38.5, 0.000001
|
|
assert_in_delta parsed.grid_params.latin2, 38.5, 0.000001
|
|
assert parsed.grid_params.scan_mode == 64
|
|
|
|
# Product from Section 4
|
|
assert parsed.product.var == "TMP"
|
|
assert parsed.product.level == "2 m above ground"
|
|
|
|
# Packing params from Section 5
|
|
assert parsed.packing_params.num_data_points == 100
|
|
assert parsed.packing_params.bits_per_value == 16
|
|
|
|
# Data from Section 7
|
|
assert is_binary(parsed.data)
|
|
|
|
# Bitmap indicator
|
|
assert parsed.bitmap == :none
|
|
end
|
|
|
|
test "rejects non-GRIB2 binary" do
|
|
assert {:error, _} = Section.parse_message(<<"NOT_GRIB", 0::64-big>>)
|
|
end
|
|
|
|
test "rejects GRIB edition 1" do
|
|
indicator = <<"GRIB", 0::16, 0::8, 1::8, 100::64-big>>
|
|
assert {:error, _} = Section.parse_message(indicator)
|
|
end
|
|
|
|
test "parses message with trailing padding bytes instead of 7777" do
|
|
msg = build_synthetic_grib2_with_trailing_padding()
|
|
assert {:ok, parsed} = Section.parse_message(msg)
|
|
assert parsed.product.var == "TMP"
|
|
assert parsed.bitmap == :none
|
|
end
|
|
|
|
test "parses message with truncated trailing section" do
|
|
msg = build_synthetic_grib2_with_truncated_section()
|
|
assert {:ok, parsed} = Section.parse_message(msg)
|
|
assert parsed.product.var == "TMP"
|
|
assert parsed.bitmap == :none
|
|
end
|
|
end
|
|
|
|
describe "product identification" do
|
|
test "identifies all supported variables" do
|
|
# discipline 0 (meteorological)
|
|
assert Section.identify_var(0, 0, 0) == "TMP"
|
|
assert Section.identify_var(0, 0, 6) == "DPT"
|
|
assert Section.identify_var(0, 3, 0) == "PRES"
|
|
assert Section.identify_var(0, 3, 5) == "HGT"
|
|
assert Section.identify_var(0, 3, 18) == "HPBL"
|
|
assert Section.identify_var(0, 1, 3) == "PWAT"
|
|
end
|
|
|
|
test "returns category:number for unknown variables" do
|
|
assert Section.identify_var(0, 99, 99) == "99:99"
|
|
end
|
|
end
|
|
|
|
describe "level identification" do
|
|
test "surface" do
|
|
assert Section.identify_level(1, 0) == "surface"
|
|
end
|
|
|
|
test "pressure level in mb" do
|
|
# Stored in Pa, displayed in mb
|
|
assert Section.identify_level(100, 100_000) == "1000 mb"
|
|
assert Section.identify_level(100, 85_000) == "850 mb"
|
|
end
|
|
|
|
test "meters above ground" do
|
|
assert Section.identify_level(103, 2) == "2 m above ground"
|
|
assert Section.identify_level(103, 10) == "10 m above ground"
|
|
end
|
|
|
|
test "entire atmosphere" do
|
|
assert Section.identify_level(200, 0) ==
|
|
"entire atmosphere (considered as a single layer)"
|
|
end
|
|
|
|
test "unknown level type" do
|
|
assert Section.identify_level(255, 0) == "unknown:255:0"
|
|
end
|
|
end
|
|
|
|
describe "sign_magnitude_16/1" do
|
|
test "positive value" do
|
|
assert Section.sign_magnitude_16(<<0::1, 5::15>>) == 5
|
|
end
|
|
|
|
test "negative value" do
|
|
assert Section.sign_magnitude_16(<<1::1, 5::15>>) == -5
|
|
end
|
|
|
|
test "zero" do
|
|
assert Section.sign_magnitude_16(<<0::16>>) == 0
|
|
end
|
|
end
|
|
|
|
# --- Helpers ---
|
|
|
|
defp build_synthetic_grib2 do
|
|
# Section 1 (Identification) - minimal
|
|
sec1_body = <<
|
|
# Originating center, subcenter, etc.
|
|
0::16-big,
|
|
0::16-big,
|
|
# Master/local tables
|
|
2::8,
|
|
1::8,
|
|
# Significance of reference time
|
|
1::8,
|
|
# Year, month, day, hour, minute, second
|
|
2026::16-big,
|
|
3::8,
|
|
28::8,
|
|
18::8,
|
|
0::8,
|
|
0::8,
|
|
# Production status, type
|
|
0::8,
|
|
1::8
|
|
>>
|
|
|
|
sec1 = section_wrap(1, sec1_body)
|
|
|
|
# Section 3 (Grid Definition) - Template 3.30 (Lambert Conformal)
|
|
sec3_body = <<
|
|
# Source of grid definition
|
|
0::8,
|
|
# Number of data points
|
|
1_905_141::32-big,
|
|
# Number of octets for optional list
|
|
0::8,
|
|
# Interpretation
|
|
0::8,
|
|
# Grid Definition Template Number = 30 (Lambert Conformal)
|
|
30::16-big,
|
|
# Shape of earth (6 = spherical, R=6371229)
|
|
6::8,
|
|
# Scale factor/value of radius (not used for shape 6)
|
|
0::8,
|
|
0::32-big,
|
|
# Scale factor/value of major axis
|
|
0::8,
|
|
0::32-big,
|
|
# Scale factor/value of minor axis
|
|
0::8,
|
|
0::32-big,
|
|
# Nx, Ny
|
|
1799::32-big,
|
|
1059::32-big,
|
|
# La1 in microdegrees
|
|
21_138_123::32-big,
|
|
# Lo1 in microdegrees
|
|
237_280_472::32-big,
|
|
# Resolution and component flags
|
|
0b00111000::8,
|
|
# LaD in microdegrees
|
|
38_500_000::32-big,
|
|
# LoV in microdegrees
|
|
262_500_000::32-big,
|
|
# Dx in millimetres
|
|
3_000_000::32-big,
|
|
# Dy in millimetres
|
|
3_000_000::32-big,
|
|
# Projection centre flag
|
|
0::8,
|
|
# Scanning mode
|
|
64::8,
|
|
# Latin1 in microdegrees
|
|
38_500_000::32-big,
|
|
# Latin2 in microdegrees
|
|
38_500_000::32-big,
|
|
# Latitude/longitude of southern pole
|
|
0::32-big,
|
|
0::32-big
|
|
>>
|
|
|
|
sec3 = section_wrap(3, sec3_body)
|
|
|
|
# Section 4 (Product Definition) - Template 4.0
|
|
sec4_body = <<
|
|
# Number of coordinate values
|
|
0::16-big,
|
|
# Product Definition Template Number = 0
|
|
0::16-big,
|
|
# Parameter category (0 = Temperature)
|
|
0::8,
|
|
# Parameter number (0 = Temperature)
|
|
0::8,
|
|
# Type of generating process
|
|
2::8,
|
|
# Background + forecast generating process
|
|
0::8,
|
|
0::8,
|
|
# Hours/minutes of observational data cutoff
|
|
0::16-big,
|
|
0::8,
|
|
# Indicator of unit of time range (1=hour)
|
|
1::8,
|
|
# Forecast time
|
|
0::32-big,
|
|
# Type of first fixed surface (103 = above ground)
|
|
103::8,
|
|
# Scale factor and scaled value of first fixed surface
|
|
0::8,
|
|
2::32-big,
|
|
# Type of second fixed surface (255 = missing)
|
|
255::8,
|
|
0::8,
|
|
0::32-big
|
|
>>
|
|
|
|
sec4 = section_wrap(4, sec4_body)
|
|
|
|
# Section 5 (Data Representation) - Template 5.0 (Simple packing)
|
|
# Reference value as IEEE 754 float32
|
|
ref_val = <<0.0::float-32-big>>
|
|
|
|
sec5_body = <<
|
|
# Number of data points
|
|
100::32-big,
|
|
# Data Representation Template Number = 0
|
|
0::16-big,
|
|
# Reference value (IEEE 754)
|
|
ref_val::binary,
|
|
# Binary scale factor (sign-magnitude 16-bit)
|
|
0::16-big,
|
|
# Decimal scale factor (sign-magnitude 16-bit)
|
|
0::16-big,
|
|
# Number of bits per value
|
|
16::8
|
|
>>
|
|
|
|
sec5 = section_wrap(5, sec5_body)
|
|
|
|
# Section 6 (Bitmap) - no bitmap
|
|
sec6_body = <<255::8>>
|
|
sec6 = section_wrap(6, sec6_body)
|
|
|
|
# Section 7 (Data) - 100 values of 16 bits = 200 bytes
|
|
data_bytes = :binary.copy(<<42::16-big>>, 100)
|
|
sec7 = section_wrap(7, data_bytes)
|
|
|
|
# Section 8 (End)
|
|
sec8 = "7777"
|
|
|
|
body = sec1 <> sec3 <> sec4 <> sec5 <> sec6 <> sec7 <> sec8
|
|
total_length = 16 + byte_size(body)
|
|
|
|
# Section 0 (Indicator)
|
|
sec0 = <<"GRIB", 0::16, 0::8, 2::8, total_length::64-big>>
|
|
|
|
sec0 <> body
|
|
end
|
|
|
|
defp section_wrap(section_number, body) do
|
|
length = 5 + byte_size(body)
|
|
<<length::32-big, section_number::8, body::binary>>
|
|
end
|
|
|
|
# Like build_synthetic_grib2 but with 3 padding bytes instead of "7777"
|
|
defp build_synthetic_grib2_with_trailing_padding do
|
|
body = build_sections_body() <> <<0, 0, 0>>
|
|
total_length = 16 + byte_size(body)
|
|
sec0 = <<"GRIB", 0::16, 0::8, 2::8, total_length::64-big>>
|
|
sec0 <> body
|
|
end
|
|
|
|
# Like build_synthetic_grib2 but total_length includes extra bytes after "7777"
|
|
# that look like a truncated section header
|
|
defp build_synthetic_grib2_with_truncated_section do
|
|
# All valid sections + "7777" + a truncated next section (length says 500 but only 10 bytes)
|
|
truncated = <<500::32-big, 4::8, 0, 0, 0, 0, 0>>
|
|
body = build_sections_body() <> "7777" <> truncated
|
|
total_length = 16 + byte_size(body)
|
|
sec0 = <<"GRIB", 0::16, 0::8, 2::8, total_length::64-big>>
|
|
sec0 <> body
|
|
end
|
|
|
|
defp build_sections_body do
|
|
sec1_body = <<
|
|
0::16-big,
|
|
0::16-big,
|
|
2::8,
|
|
1::8,
|
|
1::8,
|
|
2026::16-big,
|
|
3::8,
|
|
28::8,
|
|
18::8,
|
|
0::8,
|
|
0::8,
|
|
0::8,
|
|
1::8
|
|
>>
|
|
|
|
sec3_body = <<
|
|
0::8,
|
|
1_905_141::32-big,
|
|
0::8,
|
|
0::8,
|
|
30::16-big,
|
|
6::8,
|
|
0::8,
|
|
0::32-big,
|
|
0::8,
|
|
0::32-big,
|
|
0::8,
|
|
0::32-big,
|
|
1799::32-big,
|
|
1059::32-big,
|
|
21_138_123::32-big,
|
|
237_280_472::32-big,
|
|
0b00111000::8,
|
|
38_500_000::32-big,
|
|
262_500_000::32-big,
|
|
3_000_000::32-big,
|
|
3_000_000::32-big,
|
|
0::8,
|
|
64::8,
|
|
38_500_000::32-big,
|
|
38_500_000::32-big,
|
|
0::32-big,
|
|
0::32-big
|
|
>>
|
|
|
|
sec4_body = <<
|
|
0::16-big,
|
|
0::16-big,
|
|
0::8,
|
|
0::8,
|
|
2::8,
|
|
0::8,
|
|
0::8,
|
|
0::16-big,
|
|
0::8,
|
|
1::8,
|
|
0::32-big,
|
|
103::8,
|
|
0::8,
|
|
2::32-big,
|
|
255::8,
|
|
0::8,
|
|
0::32-big
|
|
>>
|
|
|
|
ref_val = <<0.0::float-32-big>>
|
|
|
|
sec5_body = <<
|
|
100::32-big,
|
|
0::16-big,
|
|
ref_val::binary,
|
|
0::16-big,
|
|
0::16-big,
|
|
16::8
|
|
>>
|
|
|
|
sec6_body = <<255::8>>
|
|
data_bytes = :binary.copy(<<42::16-big>>, 100)
|
|
|
|
section_wrap(1, sec1_body) <>
|
|
section_wrap(3, sec3_body) <>
|
|
section_wrap(4, sec4_body) <>
|
|
section_wrap(5, sec5_body) <>
|
|
section_wrap(6, sec6_body) <>
|
|
section_wrap(7, data_bytes)
|
|
end
|
|
end
|