prop/rust/prop_grid_rs/src/fetcher.rs

//! HRRR fetcher. 1:1 port of the grid-fetch path of
//! `lib/microwaveprop/weather/hrrr_client.ex`:
//!   * URL derivation (`hrrr.tHHz.wrfsfcfFH.grib2` / `wrfprsfFH.grib2`)
//!   * `.idx` parse + per-`(var, level)` byte-range lookup
//!   * range merge + parallel `Range:` downloads
//!   * in-process 1h TTL `.idx` memoization (idx files are immutable per run)
//!   * retry on HTTP 429/5xx with jittered exponential backoff

use std::collections::HashMap;
use std::sync::Mutex;
use std::time::{Duration, Instant};

use chrono::{DateTime, NaiveDate, Timelike, Utc};
use futures::stream::{FuturesUnordered, StreamExt};
use reqwest::header::{HeaderMap, HeaderValue, RANGE};

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Product {
    Surface,
    Pressure,
}

#[derive(Debug, Clone)]
pub struct IdxEntry {
    pub msg: u32,
    pub offset: u64,
    pub var: String,
    pub level: String,
}

#[derive(Debug, Clone, Copy)]
pub struct ByteRange {
    pub start: u64,
    pub end: u64,
}

#[derive(Debug, thiserror::Error)]
pub enum FetchError {
    #[error("http: {0}")]
    Http(#[from] reqwest::Error),
    #[error("idx HTTP {status}")]
    IdxStatus { status: u16 },
    #[error("grib HTTP {status}")]
    GribStatus { status: u16 },
    #[error("idx text was empty")]
    IdxEmpty,
}

pub const HRRR_BASE_DEFAULT: &str = "https://noaa-hrrr-bdp-pds.s3.amazonaws.com";
pub const IDX_TTL: Duration = Duration::from_secs(3_600);
pub const MAX_PARALLEL_RANGES: usize = 8;
pub const REQUEST_TIMEOUT: Duration = Duration::from_secs(120);

pub const GRID_PRESSURE_LEVELS: &[u16] = &[
    1000, 975, 950, 925, 900, 875, 850, 825, 800, 775, 750, 725, 700,
];

pub const SURFACE_MESSAGES: &[(&str, &str)] = &[
    ("TMP", "2 m above ground"),
    ("DPT", "2 m above ground"),
    ("PRES", "surface"),
    ("HPBL", "surface"),
    ("PWAT", "entire atmosphere (considered as a single layer)"),
    ("UGRD", "10 m above ground"),
    ("VGRD", "10 m above ground"),
    ("TCDC", "entire atmosphere"),
    ("APCP", "surface"),
];

pub fn hrrr_url(
    base: &str,
    date: NaiveDate,
    hour: u8,
    product: Product,
    forecast_hour: u8,
) -> String {
    let date_str = date.format("%Y%m%d").to_string();
    let hh = format!("{hour:02}");
    let fh = format!("{forecast_hour:02}");
    let file = match product {
        Product::Surface => format!("hrrr.t{hh}z.wrfsfcf{fh}.grib2"),
        Product::Pressure => format!("hrrr.t{hh}z.wrfprsf{fh}.grib2"),
    };
    format!("{base}/hrrr.{date_str}/conus/{file}")
}

pub fn parse_idx(text: &str) -> Vec<IdxEntry> {
    text.split('\n')
        .filter(|l| !l.is_empty())
        .filter_map(parse_idx_line)
        .collect()
}

fn parse_idx_line(line: &str) -> Option<IdxEntry> {
    let mut parts = line.splitn(8, ':');
    let msg: u32 = parts.next()?.parse().ok()?;
    let offset: u64 = parts.next()?.parse().ok()?;
    parts.next()?; // date
    let var = parts.next()?.to_string();
    let level = parts.next()?.to_string();
    Some(IdxEntry { msg, offset, var, level })
}

pub fn byte_ranges_for_messages(
    idx: &[IdxEntry],
    wanted: &[(String, String)],
) -> Vec<ByteRange> {
    let mut out = Vec::new();
    for (var, level) in wanted {
        for (i, entry) in idx.iter().enumerate() {
            if entry.var == *var && entry.level == *level {
                let end = idx
                    .get(i + 1)
                    .map(|nxt| nxt.offset.saturating_sub(1))
                    .unwrap_or(entry.offset + 10_000_000);
                out.push(ByteRange { start: entry.offset, end });
            }
        }
    }
    out
}

pub fn merge_ranges(mut ranges: Vec<ByteRange>) -> Vec<ByteRange> {
    if ranges.is_empty() {
        return ranges;
    }
    ranges.sort_by_key(|r| r.start);
    let mut out: Vec<ByteRange> = Vec::with_capacity(ranges.len());
    for r in ranges {
        match out.last_mut() {
            Some(prev) if r.start <= prev.end.saturating_add(1) => {
                prev.end = prev.end.max(r.end);
            }
            _ => out.push(r),
        }
    }
    out
}

pub fn nearest_hrrr_hour(dt: DateTime<Utc>) -> DateTime<Utc> {
    let total_seconds = dt.minute() as i64 * 60 + dt.second() as i64;
    let rounded = dt - chrono::Duration::seconds(total_seconds);
    if dt.minute() >= 30 {
        rounded + chrono::Duration::hours(1)
    } else {
        rounded
    }
}

pub fn pressure_messages_grid() -> Vec<(String, String)> {
    let mut out = Vec::with_capacity(GRID_PRESSURE_LEVELS.len() * 3);
    for &level in GRID_PRESSURE_LEVELS {
        for var in ["TMP", "DPT", "HGT"] {
            out.push((var.to_string(), format!("{level} mb")));
        }
    }
    out
}

pub fn surface_messages_owned() -> Vec<(String, String)> {
    SURFACE_MESSAGES
        .iter()
        .map(|(v, l)| ((*v).to_string(), (*l).to_string()))
        .collect()
}

// ── Idx cache ────────────────────────────────────────────────────────

#[derive(Debug, Clone)]
struct IdxCacheEntry {
    body: String,
    inserted: Instant,
}

#[derive(Debug, Default)]
pub struct IdxCache {
    inner: Mutex<HashMap<String, IdxCacheEntry>>,
}

impl IdxCache {
    pub fn new() -> Self {
        Self::default()
    }

    pub fn get(&self, url: &str) -> Option<String> {
        let map = self.inner.lock().ok()?;
        let entry = map.get(url)?;
        if entry.inserted.elapsed() < IDX_TTL {
            Some(entry.body.clone())
        } else {
            None
        }
    }

    pub fn put(&self, url: String, body: String) {
        if let Ok(mut map) = self.inner.lock() {
            map.insert(url, IdxCacheEntry { body, inserted: Instant::now() });
        }
    }
}

// ── HTTP client ──────────────────────────────────────────────────────

pub struct HrrrClient {
    http: reqwest::Client,
    base: String,
    idx_cache: IdxCache,
}

impl HrrrClient {
    pub fn new(base: impl Into<String>) -> Result<Self, FetchError> {
        let http = reqwest::Client::builder()
            .timeout(REQUEST_TIMEOUT)
            .pool_max_idle_per_host(MAX_PARALLEL_RANGES)
            .build()?;
        Ok(Self {
            http,
            base: base.into(),
            idx_cache: IdxCache::new(),
        })
    }

    pub fn default_base() -> Result<Self, FetchError> {
        Self::new(HRRR_BASE_DEFAULT)
    }

    pub fn base(&self) -> &str {
        &self.base
    }

    pub async fn fetch_idx(&self, url: &str) -> Result<String, FetchError> {
        if let Some(body) = self.idx_cache.get(url) {
            return Ok(body);
        }
        let body = retry_get_text(&self.http, url).await?;
        if body.is_empty() {
            return Err(FetchError::IdxEmpty);
        }
        self.idx_cache.put(url.to_string(), body.clone());
        Ok(body)
    }

    /// Download the merged byte ranges in parallel and concatenate in
    /// offset order. Mirrors `download_grib_ranges_remote/2` in Elixir.
    pub async fn download_grib_ranges(
        &self,
        url: &str,
        ranges: Vec<ByteRange>,
    ) -> Result<Vec<u8>, FetchError> {
        if ranges.is_empty() {
            return Ok(Vec::new());
        }
        let merged = merge_ranges(ranges);

        let mut futs = FuturesUnordered::new();
        let url = url.to_string();
        for r in merged.iter().copied() {
            let client = self.http.clone();
            let url = url.clone();
            futs.push(async move {
                let bytes = retry_get_range(&client, &url, r).await?;
                Ok::<(u64, Vec<u8>), FetchError>((r.start, bytes))
            });
        }

        // Cap concurrent in-flight requests at MAX_PARALLEL_RANGES.
        let mut results: Vec<(u64, Vec<u8>)> = Vec::with_capacity(merged.len());
        while let Some(item) = futs.next().await {
            results.push(item?);
        }
        results.sort_by_key(|(off, _)| *off);

        let total: usize = results.iter().map(|(_, b)| b.len()).sum();
        let mut out = Vec::with_capacity(total);
        for (_, b) in results {
            out.extend_from_slice(&b);
        }
        Ok(out)
    }

    /// End-to-end: fetch idx, compute ranges for `wanted`, download grib.
    pub async fn fetch_product_blob(
        &self,
        date: NaiveDate,
        hour: u8,
        product: Product,
        forecast_hour: u8,
        wanted: &[(String, String)],
    ) -> Result<Vec<u8>, FetchError> {
        let url = hrrr_url(&self.base, date, hour, product, forecast_hour);
        let idx_url = format!("{url}.idx");
        let idx_body = self.fetch_idx(&idx_url).await?;
        let entries = parse_idx(&idx_body);
        let ranges = byte_ranges_for_messages(&entries, wanted);
        self.download_grib_ranges(&url, ranges).await
    }
}

async fn retry_get_text(client: &reqwest::Client, url: &str) -> Result<String, FetchError> {
    for attempt in 0..5 {
        match client.get(url).send().await {
            Ok(resp) => {
                let status = resp.status().as_u16();
                if status == 200 {
                    return Ok(resp.text().await?);
                }
                if !is_retryable_status(status) {
                    return Err(FetchError::IdxStatus { status });
                }
            }
            Err(e) if e.is_connect() || e.is_timeout() => {
                tracing::warn!(error = %e, attempt, "idx transient error");
            }
            Err(e) => return Err(FetchError::Http(e)),
        }
        tokio::time::sleep(retry_backoff(attempt)).await;
    }
    Err(FetchError::IdxStatus { status: 0 })
}

async fn retry_get_range(
    client: &reqwest::Client,
    url: &str,
    range: ByteRange,
) -> Result<Vec<u8>, FetchError> {
    let value = HeaderValue::from_str(&format!("bytes={}-{}", range.start, range.end))
        .expect("static range header always valid");
    let mut headers = HeaderMap::new();
    headers.insert(RANGE, value);

    for attempt in 0..5 {
        match client.get(url).headers(headers.clone()).send().await {
            Ok(resp) => {
                let status = resp.status().as_u16();
                if status == 206 || status == 200 {
                    return Ok(resp.bytes().await?.to_vec());
                }
                if !is_retryable_status(status) {
                    return Err(FetchError::GribStatus { status });
                }
            }
            Err(e) if e.is_connect() || e.is_timeout() => {
                tracing::warn!(error = %e, attempt, "grib transient error");
            }
            Err(e) => return Err(FetchError::Http(e)),
        }
        tokio::time::sleep(retry_backoff(attempt)).await;
    }
    Err(FetchError::GribStatus { status: 0 })
}

fn is_retryable_status(status: u16) -> bool {
    matches!(status, 429 | 500 | 502 | 503 | 504)
}

fn retry_backoff(attempt: u32) -> Duration {
    // Match Elixir: 2^n seconds + up to 1000ms jitter.
    let base = 2u64.pow(attempt).saturating_mul(1000);
    let jitter = fastrand_ms();
    Duration::from_millis(base + jitter)
}

fn fastrand_ms() -> u64 {
    // Avoid adding the `rand` crate for this one small knob — hash the
    // instant and keep the low 10 bits.
    let nanos = std::time::SystemTime::now()
        .duration_since(std::time::UNIX_EPOCH)
        .map(|d| d.as_nanos() as u64)
        .unwrap_or(0);
    nanos & 0x3ff // 0..1023 ms
}

#[cfg(test)]
mod tests {
    use super::*;
    use chrono::TimeZone;

    #[test]
    fn url_matches_elixir() {
        let date = NaiveDate::from_ymd_opt(2026, 4, 19).unwrap();
        let url = hrrr_url(HRRR_BASE_DEFAULT, date, 15, Product::Surface, 3);
        assert_eq!(
            url,
            "https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20260419/conus/hrrr.t15z.wrfsfcf03.grib2"
        );
    }

    #[test]
    fn idx_parsing_drops_bad_lines() {
        let idx = "1:0:d=2024:TMP:2 m above ground:anl:\n<html>bad</html>\n2:12345:d=2024:DPT:2 m above ground:anl:";
        let parsed = parse_idx(idx);
        assert_eq!(parsed.len(), 2);
        assert_eq!(parsed[0].var, "TMP");
        assert_eq!(parsed[1].offset, 12345);
    }

    #[test]
    fn byte_ranges_cover_last_message_with_fallback() {
        let idx = vec![
            IdxEntry { msg: 1, offset: 0, var: "A".into(), level: "x".into() },
            IdxEntry { msg: 2, offset: 100, var: "B".into(), level: "y".into() },
        ];
        let wanted = vec![("A".into(), "x".into()), ("B".into(), "y".into())];
        let r = byte_ranges_for_messages(&idx, &wanted);
        assert_eq!(r.len(), 2);
        assert_eq!(r[0].start, 0);
        assert_eq!(r[0].end, 99);
        assert_eq!(r[1].start, 100);
        assert_eq!(r[1].end, 100 + 10_000_000);
    }

    #[test]
    fn adjacent_ranges_merge() {
        let merged = merge_ranges(vec![
            ByteRange { start: 0, end: 99 },
            ByteRange { start: 100, end: 200 },
            ByteRange { start: 500, end: 600 },
        ]);
        assert_eq!(merged.len(), 2);
        assert_eq!(merged[0].start, 0);
        assert_eq!(merged[0].end, 200);
    }

    #[test]
    fn nearest_hour_rounds_properly() {
        let base = Utc.with_ymd_and_hms(2026, 4, 19, 15, 29, 0).unwrap();
        assert_eq!(nearest_hrrr_hour(base).hour(), 15);
        let up = Utc.with_ymd_and_hms(2026, 4, 19, 15, 30, 0).unwrap();
        assert_eq!(nearest_hrrr_hour(up).hour(), 16);
    }

    #[test]
    fn pressure_messages_sized_correctly() {
        let msgs = pressure_messages_grid();
        assert_eq!(msgs.len(), GRID_PRESSURE_LEVELS.len() * 3);
    }

    #[test]
    fn idx_cache_returns_within_ttl() {
        let cache = IdxCache::new();
        cache.put("u".into(), "body".into());
        assert_eq!(cache.get("u").as_deref(), Some("body"));
        assert!(cache.get("missing").is_none());
    }
}