From afbdbbc26d1c83197e7b0c664f273637af72312a Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Sun, 19 Apr 2026 17:47:04 -0500 Subject: [PATCH] feat(prop-grid-rs): port NexradClient.fetch_frame to Rust MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port of the subset of Microwaveprop.Weather.NexradClient used by f00's chain step: 5-min-rounded IEM n0q PNG fetch, palette-indexed decode via the png crate, and per-POI max_reflectivity_dbz extraction from a 25-pixel half-box (matches Elixir's DEFAULT_BOX_HALF). The palette↔dBZ math and (lat, lon)↔pixel projection are bit-for-bit copies of the Elixir formulas. 6 pure-math unit tests cover pixel mapping, box clipping, and timestamp rounding. Network-live golden fixture deferred until the full Stream A wiring lands. --- rust/prop_grid_rs/Cargo.lock | 88 ++++++++++++-- rust/prop_grid_rs/Cargo.toml | 1 + rust/prop_grid_rs/src/lib.rs | 1 + rust/prop_grid_rs/src/nexrad.rs | 209 ++++++++++++++++++++++++++++++++ 4 files changed, 290 insertions(+), 9 deletions(-) create mode 100644 rust/prop_grid_rs/src/nexrad.rs diff --git a/rust/prop_grid_rs/Cargo.lock b/rust/prop_grid_rs/Cargo.lock index fca3dc1c..1cceaf3f 100644 --- a/rust/prop_grid_rs/Cargo.lock +++ b/rust/prop_grid_rs/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aho-corasick" version = "1.1.4" @@ -135,6 +141,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.11.1" @@ -252,6 +264,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -395,12 +416,31 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "flume" version = "0.11.1" @@ -970,7 +1010,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" dependencies = [ - "bitflags", + "bitflags 2.11.1", "libc", "plain", "redox_syscall 0.7.4", @@ -1056,6 +1096,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.0" @@ -1221,6 +1271,19 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" +[[package]] +name = "png" +version = "0.17.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82151a2fc869e011c153adc57cf2789ccb8d9906ce52c0b39a6b5697749d7526" +dependencies = [ + "bitflags 1.3.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -1294,6 +1357,7 @@ dependencies = [ "chrono", "futures", "num_cpus", + "png", "pretty_assertions", "prometheus", "rayon", @@ -1470,7 +1534,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.11.1", ] [[package]] @@ -1479,7 +1543,7 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a" dependencies = [ - "bitflags", + "bitflags 2.11.1", ] [[package]] @@ -1586,7 +1650,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys", @@ -1764,6 +1828,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + [[package]] name = "slab" version = "0.4.12" @@ -1905,7 +1975,7 @@ checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" dependencies = [ "atoi", "base64", - "bitflags", + "bitflags 2.11.1", "byteorder", "bytes", "chrono", @@ -1949,7 +2019,7 @@ checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" dependencies = [ "atoi", "base64", - "bitflags", + "bitflags 2.11.1", "byteorder", "chrono", "crc", @@ -2230,7 +2300,7 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ - "bitflags", + "bitflags 2.11.1", "bytes", "futures-util", "http", @@ -2563,7 +2633,7 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags", + "bitflags 2.11.1", "hashbrown 0.15.5", "indexmap", "semver", @@ -2962,7 +3032,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags", + "bitflags 2.11.1", "indexmap", "log", "serde", diff --git a/rust/prop_grid_rs/Cargo.toml b/rust/prop_grid_rs/Cargo.toml index 26c01db9..bdcf8974 100644 --- a/rust/prop_grid_rs/Cargo.toml +++ b/rust/prop_grid_rs/Cargo.toml @@ -32,6 +32,7 @@ futures = "0.3" rayon = "1.10" axum = { version = "0.7", default-features = false, features = ["http1", "tokio"] } prometheus = { version = "0.13", default-features = false } +png = "0.17" [dev-dependencies] tokio = { version = "1", features = ["full", "test-util"] } diff --git a/rust/prop_grid_rs/src/lib.rs b/rust/prop_grid_rs/src/lib.rs index 70745d82..9c55ccb5 100644 --- a/rust/prop_grid_rs/src/lib.rs +++ b/rust/prop_grid_rs/src/lib.rs @@ -26,6 +26,7 @@ pub mod duct; pub mod fetcher; pub mod grid; pub mod metrics; +pub mod nexrad; pub mod pipeline; pub mod region; pub mod scores_file; diff --git a/rust/prop_grid_rs/src/nexrad.rs b/rust/prop_grid_rs/src/nexrad.rs new file mode 100644 index 00000000..4c2912f9 --- /dev/null +++ b/rust/prop_grid_rs/src/nexrad.rs @@ -0,0 +1,209 @@ +//! IEM n0q CONUS composite reflectivity fetch + per-POI extraction. +//! +//! Port of `Microwaveprop.Weather.NexradClient.fetch_frame/2` — the +//! subset used by f00's chain step. n0q is a palettized 8-bit PNG, +//! 12200 × 5400 pixels, 0.005°/pixel, covering (-126W..-65W, 23N..50N) +//! every 5 min. For each point of interest the extractor returns the +//! max dBZ in a ~25 km box, matching Elixir's +//! `extract_box/5` output field `max_reflectivity_dbz`. + +use bytes::Bytes; +use chrono::{DateTime, Datelike, Timelike, Utc}; + +pub const LON_MIN: f64 = -126.0; +pub const LAT_MAX: f64 = 50.0; +pub const DEG_PER_PIXEL: f64 = 0.005; +pub const DEFAULT_BOX_HALF: i32 = 25; +pub const BASE_URL: &str = "https://mesonet.agron.iastate.edu/archive/data"; + +#[derive(Debug, thiserror::Error)] +pub enum NexradError { + #[error("http: {0}")] + Http(#[from] reqwest::Error), + #[error("decode: {0}")] + Decode(String), + #[error("unexpected status {0}")] + Status(u16), +} + +#[derive(Debug, Clone, Copy)] +pub struct NexradObservation { + pub lat: f64, + pub lon: f64, + pub max_reflectivity_dbz: f64, +} + +/// Round a timestamp down to the nearest 5-min boundary (IEM archives +/// 5-min cadence). +pub fn round_to_5min(ts: DateTime) -> DateTime { + let minute = ts.minute(); + let rounded = (minute / 5) * 5; + ts.with_minute(rounded) + .and_then(|t| t.with_second(0)) + .and_then(|t| t.with_nanosecond(0)) + .expect("valid rounded timestamp") +} + +pub fn frame_url(rounded: DateTime) -> String { + format!( + "{}/{:04}/{:02}/{:02}/GIS/uscomp/n0q_{:04}{:02}{:02}{:02}{:02}.png", + BASE_URL, + rounded.year(), + rounded.month(), + rounded.day(), + rounded.year(), + rounded.month(), + rounded.day(), + rounded.hour(), + rounded.minute() + ) +} + +pub fn latlon_to_pixel(lat: f64, lon: f64) -> (i32, i32) { + let x = ((lon - LON_MIN) / DEG_PER_PIXEL).round() as i32; + let y = ((LAT_MAX - lat) / DEG_PER_PIXEL).round() as i32; + (x, y) +} + +/// Value 0 = no echo. Values 1..=255 map linearly to -30..=+95 dBZ. +pub fn pixel_to_dbz(v: u8) -> f64 { + if v == 0 { + 0.0 + } else { + -30.0 + ((v - 1) as f64) / 254.0 * 125.0 + } +} + +/// Maximum dBZ within a 2·half+1 × 2·half+1 box around (cx, cy). +/// Pixels outside the image or zero (no echo) are skipped. +pub fn max_dbz_in_box(pixels: &[u8], width: i32, cx: i32, cy: i32, half: i32) -> f64 { + let height = (pixels.len() as i32) / width; + let x_min = (cx - half).max(0); + let x_max = (cx + half).min(width - 1); + let y_min = (cy - half).max(0); + let y_max = (cy + half).min(height - 1); + + let mut max_val = 0u8; + for y in y_min..=y_max { + for x in x_min..=x_max { + let off = (y * width + x) as usize; + if off < pixels.len() { + let v = pixels[off]; + if v > max_val { + max_val = v; + } + } + } + } + pixel_to_dbz(max_val) +} + +/// Decode an IEM n0q PNG to an indexed-pixel buffer. +pub fn decode_n0q_png(bytes: &[u8]) -> Result<(Vec, i32), NexradError> { + let decoder = png::Decoder::new(std::io::Cursor::new(bytes)); + let mut reader = decoder + .read_info() + .map_err(|e| NexradError::Decode(format!("read_info: {e}")))?; + let info = reader.info(); + let width = info.width as i32; + // IEM n0q is indexed-color 8-bit. `next_frame` fills the buffer + // with raw palette indices, which is exactly what pixel_to_dbz + // expects. + let mut buf = vec![0u8; reader.output_buffer_size()]; + reader + .next_frame(&mut buf) + .map_err(|e| NexradError::Decode(format!("next_frame: {e}")))?; + Ok((buf, width)) +} + +/// Fetch + decode the n0q frame nearest `timestamp`, then return one +/// `max_reflectivity_dbz` per point of interest. +pub async fn fetch_frame( + client: &reqwest::Client, + timestamp: DateTime, + points: &[(f64, f64)], +) -> Result, NexradError> { + let rounded = round_to_5min(timestamp); + let url = frame_url(rounded); + let resp = client + .get(&url) + .timeout(std::time::Duration::from_secs(60)) + .send() + .await?; + if !resp.status().is_success() { + return Err(NexradError::Status(resp.status().as_u16())); + } + let body: Bytes = resp.bytes().await?; + let (pixels, width) = decode_n0q_png(&body)?; + + let obs = points + .iter() + .map(|&(lat, lon)| { + let (cx, cy) = latlon_to_pixel(lat, lon); + let dbz = max_dbz_in_box(&pixels, width, cx, cy, DEFAULT_BOX_HALF); + NexradObservation { lat, lon, max_reflectivity_dbz: dbz } + }) + .collect(); + Ok(obs) +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::TimeZone; + + #[test] + fn rounds_to_five_minute_boundary() { + let t = Utc.with_ymd_and_hms(2026, 4, 19, 14, 37, 45).unwrap(); + let r = round_to_5min(t); + assert_eq!(r.minute(), 35); + assert_eq!(r.second(), 0); + assert_eq!(r.hour(), 14); + } + + #[test] + fn frame_url_format() { + let t = Utc.with_ymd_and_hms(2026, 4, 19, 14, 35, 0).unwrap(); + let u = frame_url(t); + assert_eq!( + u, + "https://mesonet.agron.iastate.edu/archive/data/2026/04/19/GIS/uscomp/n0q_202604191435.png" + ); + } + + #[test] + fn pixel_to_dbz_endpoints() { + assert_eq!(pixel_to_dbz(0), 0.0); + assert!((pixel_to_dbz(1) - -30.0).abs() < 1e-9); + assert!((pixel_to_dbz(255) - 95.0).abs() < 1e-9); + } + + #[test] + fn latlon_to_pixel_reasonable() { + // Corner sanity: (50N, -126W) → (0, 0) + assert_eq!(latlon_to_pixel(50.0, -126.0), (0, 0)); + // Mid-CONUS: (32.9, -97.0) + let (x, y) = latlon_to_pixel(32.9, -97.0); + assert!(x > 5000 && x < 7000, "x={x}"); + assert!(y > 2000 && y < 4000, "y={y}"); + } + + #[test] + fn max_dbz_picks_hottest_pixel_in_box() { + // 10x10 image. Non-zero pixel at (5, 5) = 200 (→ ~68 dBZ). + let width = 10; + let mut pixels = vec![0u8; 100]; + pixels[5 * 10 + 5] = 200; + let dbz = max_dbz_in_box(&pixels, width, 5, 5, 2); + assert!((dbz - pixel_to_dbz(200)).abs() < 1e-9); + } + + #[test] + fn box_clipped_at_edges() { + let width = 10; + let pixels = vec![100u8; 100]; + // Center outside image — clipped to 0. + let dbz = max_dbz_in_box(&pixels, width, 50, 50, 5); + assert_eq!(dbz, 0.0); + } +}