diff --git a/rust/prop_grid_rs/src/pipeline.rs b/rust/prop_grid_rs/src/pipeline.rs index aa05cef7..de4aa05c 100644 --- a/rust/prop_grid_rs/src/pipeline.rs +++ b/rust/prop_grid_rs/src/pipeline.rs @@ -6,7 +6,6 @@ //! minus the f00-only enrichment (native duct, NEXRAD, commercial link //! boost, ProfilesFile write) — Elixir retains f00. -use std::collections::HashMap; use std::path::Path; use chrono::{DateTime, Datelike, Timelike, Utc}; @@ -108,33 +107,35 @@ pub async fn run_chain_step( let merged = merge_grids(sfc_grid, prs_grid); let point_count = merged.len() as u32; - // Score every point × every band; one ScorePoint per (cell, band). + // Step 1: consume `merged` into a compact `Vec<(lat, lon, Conditions, + // invariants)>`. This is the biggest win — `merged`'s inner HashMaps + // of `String -> f32` account for ~200 MB of heap (92k × ~60 string + // keys) and are dropped as soon as this loop finishes. + let prepared: Vec<(f64, f64, Conditions, scorer::BandInvariants)> = merged + .into_iter() + .filter_map(|(key, cell)| { + let (lat, lon) = decoder::key_to_latlon(key); + let conditions = cell_to_conditions(&cell, lat, lon, &valid_time)?; + let invariants = scorer::precompute_band_invariants(&conditions); + Some((lat, lon, conditions, invariants)) + }) + .collect(); + + // Step 2: for each band, score → write → drop. One band's score + // vector (~92k × 20 B ≈ 1.8 MB) lives at a time instead of holding + // all 23 bands at once. The write runs on the blocking pool so the + // reactor stays responsive. let bands = band_config::all_bands(); - let mut per_band: HashMap> = - HashMap::with_capacity(bands.len()); - - for (key, cell) in &merged { - let (lat, lon) = decoder::key_to_latlon(*key); - let conditions = match cell_to_conditions(cell, lat, lon, &valid_time) { - Some(c) => c, - None => continue, - }; - let invariants = scorer::precompute_band_invariants(&conditions); - for band in bands { - let r = scorer::composite_score_with(&conditions, band, Some(invariants)); - per_band - .entry(band.freq_mhz) - .or_default() - .push(ScorePoint { lat, lon, score: r.score }); - } - } - let scores_dir = scores_dir.to_path_buf(); let mut files_written = 0; - for (band_mhz, scores) in &per_band { + for band in bands { + let mut scores: Vec = Vec::with_capacity(prepared.len()); + for (lat, lon, conditions, invariants) in &prepared { + let r = scorer::composite_score_with(conditions, band, Some(*invariants)); + scores.push(ScorePoint { lat: *lat, lon: *lon, score: r.score }); + } let dir = scores_dir.clone(); - let band_mhz = *band_mhz; - let scores = scores.clone(); + let band_mhz = band.freq_mhz; tokio::task::spawn_blocking(move || { scores_file::write_atomic(&dir, band_mhz, valid_time, &scores) })