prop/rust/prop_grid_rs/src/grid.rs
Graham McIntire db57b3a1c7
Some checks failed
Build prop-grid-rs / Test, build, push (push) Successful in 4m20s
Build and Push / Build and Push Docker Image (push) Failing after 4m35s
prop-pipeline: .sgrid dense scalars, HRDPS rotated-pole decode-once, memory trim
Item 1 — .sgrid dense binary scalar format replacing chunked gzip+msgpack
- New rust/prop_grid_rs/src/sgrid.rs: magic SGRD, same header layout as .pgrid,
  20-field cell-major f32 body, write_atomic (tmp+rename, NFS-safe)
- New lib/microwaveprop/weather/sgrid.ex: Elixir reader modeled on pgrid.ex
  (pread single-cell reads, bounds-filtered viewport reads, NaN→nil sentinel)
- ScalarFile updated to prefer .sgrid reads, chunked .mp.gz as fallback
- Pipeline writes .sgrid alongside existing chunked format (all three paths)

Item 2 — HRDPS decode-once + rotated-pole index, restore 0.125° resolution
- New rust/prop_grid_rs/src/rotated_pole.rs: CF-convention geographic→rotated
  transform, OnceLock-cached GDS params parsed from wgrib2 -grid, precomputed
  Vec<u32> lookup table mapping target cells to native grid indices
- Native decode path in decoder.rs: wgrib2 -no_header -order we:sn -bin
  (raw f32 dump, ~0.32 s/message) + indexing via lookup table
- HRDPS_STEP: 0.5° → 0.125° (4× finer, ~57k Canadian cells vs ~3.5k)

Item 3 — k8s memory limits: 3Gi → 1.5Gi
  (per-task grid footprint: ~200-400 MB HashMap → ~18 MB dense planes)

Item 4 — CLAUDE.md and profiles_file.ex documentation drift fixed:
  .pgrid primary, .mp.gz legacy, .sgrid added, write_atomic protocol doc,
  cleanup gaps reorganized, 'Only f00 is persisted' corrected to f00..f48
2026-08-01 09:11:59 -05:00

228 lines
8.5 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

//! Grid definitions at 0.125° resolution. 1:1 port of
//! `lib/microwaveprop/propagation/grid.ex`. Two regions, disjoint on
//! HRRR's *interior*:
//!
//! * `conus_points()` — HRRR coverage (lat 25-50, lon -125 to -66).
//! * `hrdps_only_points()` — HRDPS bbox (lat 49-60, lon -141 to -52)
//! minus HRRR's interior (lat < 50 inside HRRR's lon range). The
//! lat=50 boundary row is kept so HRDPS's halfStep painting closes
//! the seam between HRRR and HRDPS on /weather; the merge layer
//! dedupes by exact (lat, lon) for the merged read path.
//!
//! Coverage stops at 60°N for v1 (SRTM elevation cuts off there; Arctic
//! CDEM coverage is deferred to a follow-up plan).
pub const LAT_MIN: f64 = 25.0;
pub const LAT_MAX: f64 = 50.0;
pub const LON_MIN: f64 = -125.0;
pub const LON_MAX: f64 = -66.0;
pub const STEP: f64 = 0.125;
pub const HRDPS_LAT_MIN: f64 = 49.0;
pub const HRDPS_LAT_MAX: f64 = 60.0;
pub const HRDPS_LON_MIN: f64 = -141.0;
pub const HRDPS_LON_MAX: f64 = -52.0;
// HRDPS uses a rotated lat/lon grid. Before commit 63f25a96, wgrib2's
// `-lola` brute-forced per-output-point which made 0.125° resolution
// infeasible (~5 min/batch × 57 batches ≈ 5 h/chain step). The
// rotated-pole index in `rotated_pole.rs` now maps geographic cells to
// native grid indices via closed-form transform, so decode-once +
// lookup-many replaces the expensive per-point `-lon` extraction.
// 0.125° (~14 km cells) matches HRRR's resolution.
pub const HRDPS_STEP: f64 = 0.125;
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct GridSpec {
pub lon_start: f64,
pub lon_count: usize,
pub lon_step: f64,
pub lat_start: f64,
pub lat_count: usize,
pub lat_step: f64,
}
pub fn wgrib2_grid_spec() -> GridSpec {
let lon_count = ((LON_MAX - LON_MIN) / STEP).round() as usize + 1;
let lat_count = ((LAT_MAX - LAT_MIN) / STEP).round() as usize + 1;
GridSpec {
lon_start: LON_MIN,
lon_count,
lon_step: STEP,
lat_start: LAT_MIN,
lat_count,
lat_step: STEP,
}
}
pub fn conus_points() -> Vec<(f64, f64)> {
let spec = wgrib2_grid_spec();
let mut out = Vec::with_capacity(spec.lat_count * spec.lon_count);
for j in 0..spec.lat_count {
let lat = round3(spec.lat_start + j as f64 * spec.lat_step);
for i in 0..spec.lon_count {
let lon = round3(spec.lon_start + i as f64 * spec.lon_step);
out.push((lat, lon));
}
}
out
}
/// HRDPS grid spec covering the full Canadian bbox. The `hrdps_only_points()`
/// list is a subset (excludes the HRRR overlap), but the underlying wgrib2
/// extraction needs the full bbox so the score-file's lat_start/lon_start
/// align with the cells the worker actually wrote.
pub fn hrdps_grid_spec() -> GridSpec {
let lon_count = ((HRDPS_LON_MAX - HRDPS_LON_MIN) / HRDPS_STEP).round() as usize + 1;
let lat_count = ((HRDPS_LAT_MAX - HRDPS_LAT_MIN) / HRDPS_STEP).round() as usize + 1;
GridSpec {
lon_start: HRDPS_LON_MIN,
lon_count,
lon_step: HRDPS_STEP,
lat_start: HRDPS_LAT_MIN,
lat_count,
lat_step: HRDPS_STEP,
}
}
/// Canadian-only grid points: cells inside the HRDPS bbox (49-60°N,
/// -141 to -52°W) but outside HRRR's CONUS bbox. Disjoint from
/// `conus_points()` by construction so the two grids never double-write
/// the same `(lat, lon)`.
pub fn hrdps_only_points() -> Vec<(f64, f64)> {
let spec = hrdps_grid_spec();
let mut out = Vec::with_capacity(spec.lat_count * spec.lon_count);
for j in 0..spec.lat_count {
let lat = round3(spec.lat_start + j as f64 * spec.lat_step);
for i in 0..spec.lon_count {
let lon = round3(spec.lon_start + i as f64 * spec.lon_step);
// HRDPS now walks at 0.5° while HRRR walks at 0.125°, so the
// coarse cells are no longer subset-aligned with HRRR's CONUS
// grid. The disjointness rule is "if a coarse cell *center*
// falls inside HRRR's lat/lon range, drop it" — HRRR will
// cover that area at finer resolution anyway and the
// /weather merge prefers HRRR rows on collision.
if !in_conus_bbox(lat, lon) {
out.push((lat, lon));
}
}
}
out
}
#[inline]
fn in_conus_bbox(lat: f64, lon: f64) -> bool {
// Exclusive on the north edge: HRDPS keeps the lat=LAT_MAX (50.0)
// boundary row so its halfStep painting (±0.25°) overlaps HRRR's top
// edge (±0.0625°) on /weather, closing a ~0.19° blank strip the
// user would otherwise see between HRRR and HRDPS coverage. The
// merge_prefer_hrrr layer dedupes by exact (lat, lon) on the merged
// read path, and the canvas overlay paints HRRR above HRDPS, so the
// overlap is only visible north of HRRR's coverage.
(LAT_MIN..LAT_MAX).contains(&lat) && (LON_MIN..=LON_MAX).contains(&lon)
}
/// Matches Elixir's `Float.round(x, 3)` — banker's rounding isn't used,
/// half-away-from-zero is. For grid coords this matches BEAM's behavior
/// for the values we care about (no exact half cases).
pub fn round3(x: f64) -> f64 {
(x * 1000.0).round() / 1000.0
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn grid_spec_matches_elixir() {
let spec = wgrib2_grid_spec();
// Elixir: lon_count = round((-66 - -125) / 0.125) + 1 = 473
// lat_count = round((50 - 25) / 0.125) + 1 = 201
assert_eq!(spec.lon_count, 473);
assert_eq!(spec.lat_count, 201);
assert_eq!(spec.lon_start, -125.0);
assert_eq!(spec.lat_start, 25.0);
assert_eq!(spec.lon_step, 0.125);
assert_eq!(spec.lat_step, 0.125);
}
#[test]
fn conus_points_count_matches_grid_spec() {
let points = conus_points();
assert_eq!(points.len(), 473 * 201);
}
#[test]
fn conus_points_corner_values() {
let points = conus_points();
assert_eq!(points[0], (25.0, -125.0));
assert_eq!(points[points.len() - 1], (50.0, -66.0));
}
#[test]
fn hrdps_grid_spec_uses_dense_step() {
let spec = hrdps_grid_spec();
// HRDPS_STEP = 0.125 (now matching HRRR after decode-once +
// rotated-pole lookup). See rotated_pole.rs for the per-message
// decode cost that used to force a coarse 0.5° step.
// lon_count = (-52 - -141) / 0.125 + 1 = 713
// lat_count = (60 - 49) / 0.125 + 1 = 89
assert_eq!(spec.lon_count, 713);
assert_eq!(spec.lat_count, 89);
assert_eq!(spec.lon_start, -141.0);
assert_eq!(spec.lat_start, 49.0);
assert_eq!(spec.lon_step, 0.125);
assert_eq!(spec.lat_step, 0.125);
}
#[test]
fn hrdps_only_points_disjoint_from_conus_interior() {
// Disjointness now holds for CONUS *interior* (lat < LAT_MAX); the
// lat=LAT_MAX boundary row is intentionally shared so HRDPS fills
// the seam between HRRR and Canada on /weather. See
// `hrdps_keeps_lat_50_boundary_inside_conus_lons`.
let conus: std::collections::HashSet<(u64, u64)> = conus_points()
.into_iter()
.filter(|(la, _)| *la < LAT_MAX)
.map(|(la, lo)| (la.to_bits(), lo.to_bits()))
.collect();
for (la, lo) in hrdps_only_points() {
assert!(
!conus.contains(&(la.to_bits(), lo.to_bits())),
"hrdps point {la},{lo} overlaps conus interior"
);
}
}
#[test]
fn hrdps_only_points_all_within_bbox() {
for (lat, lon) in hrdps_only_points() {
assert!((49.0..=60.0).contains(&lat));
assert!((-141.0..=-52.0).contains(&lon));
}
}
#[test]
fn hrdps_only_includes_cells_above_50n() {
// At least one cell strictly north of HRRR's lat_max=50.
let any_above_50 = hrdps_only_points().iter().any(|(la, _)| *la > 50.0);
assert!(any_above_50);
}
#[test]
fn hrdps_keeps_lat_50_boundary_inside_conus_lons() {
// HRRR tops out at 50.0°N. At the same 0.125° step, HRDPS's lat=50
// row is cell-for-cell identical with HRRR's — the merge layer
// dedupes by exact (lat, lon) match and prefers HRRR. The overlap
// is invisible everywhere HRRR exists, and HRDPS fills north of 50.
let points = hrdps_only_points();
let has_50_in_conus = points
.iter()
.any(|(la, lo)| (*la - 50.0).abs() < 1e-9 && (-125.0..=-66.0).contains(lo));
assert!(
has_50_in_conus,
"expected hrdps_only_points to include the lat=50 row inside CONUS lon range"
);
}
}