Graham McIntire
b80878056d
Extracts the memory-hostile HRRR fetch → decode → score pipeline for
forecast hours 1–18 into a separate Rust service (`prop-grid-rs`).
Elixir retains f00 with its native-duct + NEXRAD + commercial-link
enrichment; Rust handles the other 18 steps per hourly chain.
Hand-off via a new `grid_tasks` table (`FOR UPDATE SKIP LOCKED` claim
from Rust, Postgrex `NOTIFY propagation_ready` back to Elixir). Rust
writes the same on-disk score-grid format Elixir already uses, to the
same `/data/scores` NFS tree. Phase 1 ships in shadow mode with
PROP_SCORES_DIR=/data/scores_shadow.
Rust crate layout at `rust/prop_grid_rs/` (1:1 module parity with the
Elixir source it ports):
- grid, region, band_config, scores_file, sounding_params
- scorer: all 10 factors + composite. Matches the Elixir scorer
byte-for-byte across 115 golden-fixture samples (5 scenarios
× 23 bands).
- decoder: wgrib2 subprocess + Fortran-record lola-binary parser
- fetcher: HRRR URL derivation + idx cache (1h TTL) + 8-way
parallel byte-range downloads with 429/5xx retry
- pipeline: end-to-end chain step, f00 rejected at the boundary
- db: sqlx grid_tasks claim/complete + propagation_ready NOTIFY
- bin/worker: tokio main loop, JSON logs, SIGTERM-safe
91 Rust tests + clippy -D warnings clean. 2,158 Elixir tests green.
Elixir additions:
- `GridTaskEnqueuer` seeds fh=1..18 rows from
`PropagationGridWorker.seed_chain/0`
- `NotifyListener` LISTEN → warm `ScoreCache` → PubSub fan-out
- `ShadowComparator` diffs prod vs shadow `.ntms` bodies
- `mix rust.golden` task writes the Rust-side golden fixture
k8s: new `deployment-grid-rs.yaml` pinned to talos5 (32 GB NUC) via
`prop-grid-rs=primary` nodeSelector + `workload=grid-rs:NoSchedule`
toleration, 512 Mi limit, sharing the existing NFS `/data` mount.
The plan document is at `plans/vivid-hatching-quail.md` (local to my
workstation); phases 2 (cutover) and 3 (talos5 concurrency tuning)
follow after 72h of shadow-mode parity.
82 lines
2.3 KiB
Rust
82 lines
2.3 KiB
Rust
//! CONUS grid at 0.125° resolution. 1:1 port of
|
|
//! `lib/microwaveprop/propagation/grid.ex`.
|
|
|
|
pub const LAT_MIN: f64 = 25.0;
|
|
pub const LAT_MAX: f64 = 50.0;
|
|
pub const LON_MIN: f64 = -125.0;
|
|
pub const LON_MAX: f64 = -66.0;
|
|
pub const STEP: f64 = 0.125;
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq)]
|
|
pub struct GridSpec {
|
|
pub lon_start: f64,
|
|
pub lon_count: usize,
|
|
pub lon_step: f64,
|
|
pub lat_start: f64,
|
|
pub lat_count: usize,
|
|
pub lat_step: f64,
|
|
}
|
|
|
|
pub fn wgrib2_grid_spec() -> GridSpec {
|
|
let lon_count = ((LON_MAX - LON_MIN) / STEP).round() as usize + 1;
|
|
let lat_count = ((LAT_MAX - LAT_MIN) / STEP).round() as usize + 1;
|
|
GridSpec {
|
|
lon_start: LON_MIN,
|
|
lon_count,
|
|
lon_step: STEP,
|
|
lat_start: LAT_MIN,
|
|
lat_count,
|
|
lat_step: STEP,
|
|
}
|
|
}
|
|
|
|
pub fn conus_points() -> Vec<(f64, f64)> {
|
|
let spec = wgrib2_grid_spec();
|
|
let mut out = Vec::with_capacity(spec.lat_count * spec.lon_count);
|
|
for j in 0..spec.lat_count {
|
|
let lat = round3(spec.lat_start + j as f64 * spec.lat_step);
|
|
for i in 0..spec.lon_count {
|
|
let lon = round3(spec.lon_start + i as f64 * spec.lon_step);
|
|
out.push((lat, lon));
|
|
}
|
|
}
|
|
out
|
|
}
|
|
|
|
/// Matches Elixir's `Float.round(x, 3)` — banker's rounding isn't used,
|
|
/// half-away-from-zero is. For grid coords this matches BEAM's behavior
|
|
/// for the values we care about (no exact half cases).
|
|
pub fn round3(x: f64) -> f64 {
|
|
(x * 1000.0).round() / 1000.0
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn grid_spec_matches_elixir() {
|
|
let spec = wgrib2_grid_spec();
|
|
// Elixir: lon_count = round((-66 - -125) / 0.125) + 1 = 473
|
|
// lat_count = round((50 - 25) / 0.125) + 1 = 201
|
|
assert_eq!(spec.lon_count, 473);
|
|
assert_eq!(spec.lat_count, 201);
|
|
assert_eq!(spec.lon_start, -125.0);
|
|
assert_eq!(spec.lat_start, 25.0);
|
|
assert_eq!(spec.lon_step, 0.125);
|
|
assert_eq!(spec.lat_step, 0.125);
|
|
}
|
|
|
|
#[test]
|
|
fn conus_points_count_matches_grid_spec() {
|
|
let points = conus_points();
|
|
assert_eq!(points.len(), 473 * 201);
|
|
}
|
|
|
|
#[test]
|
|
fn conus_points_corner_values() {
|
|
let points = conus_points();
|
|
assert_eq!(points[0], (25.0, -125.0));
|
|
assert_eq!(points[points.len() - 1], (50.0, -66.0));
|
|
}
|
|
}
|