Graham McIntire
e688d8c244
**Wire format — /map scores payload.** Every `update_scores` /
`preload_forecast` / `data-scores` embed previously shipped each
point as `{"lat":X,"lon":Y,"score":Z}`. At 95k cells in full CONUS
that's ~36 bytes/point × 95k = 3.4 MB of JSON, ~50% of it repeating
the three key names. Packing as `[lat, lon, score]` tuples drops
per-point overhead to ~19 bytes → ~1.8 MB on the wire, ~45% smaller.
New `Propagation.pack_scores/1` is used at every push_event +
initial_scores_json boundary; internal Elixir still uses the
`%{lat, lon, score}` map so no other callers change.
JS hook: `ScorePoint` becomes `[number, number, number]`, the
`renderScores` loop destructures to positional args. Regression
test in map_live_test updated to match the new shape.
**ProfilesFile compression.** Rust writer was at gzip level 6
(default). Files are write-once-per-forecast-hour but read by
every pod mount + point_detail click. Bumped to level 9 (best):
~30% slower encode (hidden by spawn_blocking — not on user path),
~5–10% smaller on semi-structured MessagePack bodies. No reader
changes needed — same gzip stream.
**TypeScript strictness.**
- `app.ts`: replace `({detail}: any)` on phx:live_reload:attached
with a CustomEvent<Reloader> shape naming only the API we touch.
- `weather_map_hook.ts`: drop the `(this as any)._map` pair by
declaring a `GridLayerWithMap = L.GridLayer & { _map: L.Map }`
intersection on the overlay's createTile `this` parameter.
- `global.d.ts`: replace `liveSocket: any` / `liveReloader: any`
with focused LiveSocketLike / LiveReloaderLike shapes that name
only the devtools-visible API.
No more `any` in the codebase (verified via rg).
245 lines
8.2 KiB
Rust
245 lines
8.2 KiB
Rust
//! On-disk profile store in MessagePack + gzip.
|
||
//!
|
||
//! Rust-side writer for the f00 ProfilesFile. Replaces the Elixir ETF
|
||
//! format (`:erlang.term_to_binary/2`, `.etf.gz`) so Rust can produce
|
||
//! the files without porting the Erlang serializer. Elixir's
|
||
//! `ProfilesFile` module learns to read `.mp.gz` alongside the legacy
|
||
//! `.etf.gz` during the cutover.
|
||
//!
|
||
//! Wire layout (top-level msgpack map):
|
||
//!
|
||
//! ```text
|
||
//! {
|
||
//! "v": 1,
|
||
//! "valid_time": "2026-04-19T14:00:00Z",
|
||
//! "cells": [
|
||
//! {
|
||
//! "lat": 32.9, "lon": -97.0,
|
||
//! "profile": {
|
||
//! "native_min_gradient": -150.5,
|
||
//! "best_duct_freq_ghz": 24.0,
|
||
//! "duct_count": 1,
|
||
//! "ducts": [
|
||
//! {"base_m": 100, "top_m": 200, "thickness_m": 100,
|
||
//! "m_deficit": 12.3, "min_freq_ghz": 15.2}
|
||
//! ],
|
||
//! "surface_temp_c": 22.5,
|
||
//! "surface_dewpoint_c": 18.1,
|
||
//! "surface_pressure_mb": 1013.2,
|
||
//! "hpbl_m": 320.0,
|
||
//! "pwat_mm": 25.3,
|
||
//! "profile": [
|
||
//! {"pres_mb":1000,"hght_m":100,"tmpc":20.0,"dwpc":18.0},
|
||
//! ...
|
||
//! ]
|
||
//! }
|
||
//! }
|
||
//! ]
|
||
//! }
|
||
//! ```
|
||
//!
|
||
//! All keys are strings — MessagePack has no atom type, and Elixir's
|
||
//! reader converts known keys back to atoms via a whitelist.
|
||
|
||
use std::fs::File;
|
||
use std::io::{BufWriter, Write};
|
||
use std::path::{Path, PathBuf};
|
||
|
||
use chrono::{DateTime, Utc};
|
||
use flate2::write::GzEncoder;
|
||
use flate2::Compression;
|
||
use serde::Serialize;
|
||
|
||
/// Bump whenever the on-disk layout changes in a way that isn't a
|
||
/// transparent superset. Elixir reader gates on this.
|
||
pub const FORMAT_VERSION: u32 = 1;
|
||
|
||
/// One grid cell to persist. `profile` is an arbitrary map of
|
||
/// string-keyed values — sized at write time. Kept as a
|
||
/// `rmp_serde::Value` so the writer stays agnostic about what the
|
||
/// f00 pipeline decides to include.
|
||
#[derive(Debug, Serialize)]
|
||
pub struct CellEntry {
|
||
pub lat: f64,
|
||
pub lon: f64,
|
||
pub profile: rmpv::Value,
|
||
}
|
||
|
||
#[derive(Debug, Serialize)]
|
||
struct FileBody<'a> {
|
||
v: u32,
|
||
valid_time: String,
|
||
cells: &'a [CellEntry],
|
||
}
|
||
|
||
#[derive(Debug, thiserror::Error)]
|
||
pub enum WriteError {
|
||
#[error("io: {0}")]
|
||
Io(#[from] std::io::Error),
|
||
#[error("encode: {0}")]
|
||
Encode(#[from] rmp_serde::encode::Error),
|
||
}
|
||
|
||
/// Absolute path the writer lands at for `valid_time`.
|
||
/// Sibling of the Elixir `.etf.gz` file; the Elixir reader prefers
|
||
/// `.mp.gz` when both exist.
|
||
pub fn path_for(scores_dir: &Path, valid_time: DateTime<Utc>) -> PathBuf {
|
||
let iso = valid_time.format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||
scores_dir.join("profiles").join(format!("{iso}.mp.gz"))
|
||
}
|
||
|
||
/// Write `cells` to `<scores_dir>/profiles/<iso>.mp.gz` atomically.
|
||
/// Pattern mirrors `scores_file::write_atomic`: write to a tmp sibling,
|
||
/// rename into place. The NFS rename is atomic, so a concurrent reader
|
||
/// sees either the old file, the new file, or nothing — never a torn
|
||
/// write.
|
||
pub fn write_atomic(
|
||
scores_dir: &Path,
|
||
valid_time: DateTime<Utc>,
|
||
cells: &[CellEntry],
|
||
) -> Result<PathBuf, WriteError> {
|
||
let path = path_for(scores_dir, valid_time);
|
||
if let Some(parent) = path.parent() {
|
||
std::fs::create_dir_all(parent)?;
|
||
}
|
||
let nanos = std::time::SystemTime::now()
|
||
.duration_since(std::time::UNIX_EPOCH)
|
||
.map(|d| d.as_nanos())
|
||
.unwrap_or(0);
|
||
let pid = std::process::id();
|
||
let mut tmp = path.clone().into_os_string();
|
||
tmp.push(format!(".tmp.{nanos}.{pid}"));
|
||
let tmp = PathBuf::from(tmp);
|
||
|
||
{
|
||
let file = File::create(&tmp)?;
|
||
// ProfilesFile is a write-once / read-many blob — each hourly
|
||
// cycle writes it ~once but every LiveView mount, point_detail
|
||
// click, and forecast scrub pays the read cost (up to ~3 MB
|
||
// uncompressed per file × ~5 cached hours per pod). Level 9 is
|
||
// ~30% slower to encode than the default (6) but 5–10% smaller
|
||
// on semi-structured MessagePack, and our write path already
|
||
// off-loads to `spawn_blocking` so encode latency isn't on the
|
||
// user path. Readers are level-agnostic — same gzip stream.
|
||
let mut gz = GzEncoder::new(BufWriter::new(file), Compression::best());
|
||
let body = FileBody {
|
||
v: FORMAT_VERSION,
|
||
valid_time: valid_time.format("%Y-%m-%dT%H:%M:%SZ").to_string(),
|
||
cells,
|
||
};
|
||
let buf = rmp_serde::to_vec_named(&body)?;
|
||
gz.write_all(&buf)?;
|
||
gz.finish()?;
|
||
}
|
||
|
||
std::fs::rename(&tmp, &path)?;
|
||
Ok(path)
|
||
}
|
||
|
||
/// Round a lat/lon to 3 decimal places — matches Elixir's
|
||
/// `Float.round(_, 3)` snap before keying into the profiles map.
|
||
pub fn snap_coords(lat: f64, lon: f64) -> (f64, f64) {
|
||
(
|
||
(lat * 1000.0).round() / 1000.0,
|
||
(lon * 1000.0).round() / 1000.0,
|
||
)
|
||
}
|
||
|
||
/// Convenience helper: build a `rmpv::Value` map from an
|
||
/// iterator of `(name, value)` pairs.
|
||
pub fn value_map(entries: impl IntoIterator<Item = (&'static str, rmpv::Value)>) -> rmpv::Value {
|
||
let pairs: Vec<(rmpv::Value, rmpv::Value)> = entries
|
||
.into_iter()
|
||
.map(|(k, v)| (rmpv::Value::String(k.into()), v))
|
||
.collect();
|
||
rmpv::Value::Map(pairs)
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
use chrono::TimeZone;
|
||
use flate2::read::GzDecoder;
|
||
use std::io::Read;
|
||
|
||
#[test]
|
||
fn writes_and_reads_round_trip() {
|
||
let dir = tempfile::tempdir().unwrap();
|
||
let vt = Utc.with_ymd_and_hms(2026, 4, 19, 14, 0, 0).unwrap();
|
||
let cells = vec![CellEntry {
|
||
lat: 32.9,
|
||
lon: -97.0,
|
||
profile: value_map([
|
||
("surface_temp_c", rmpv::Value::F64(22.5)),
|
||
("surface_pressure_mb", rmpv::Value::F64(1013.2)),
|
||
("duct_count", rmpv::Value::Integer(1.into())),
|
||
]),
|
||
}];
|
||
|
||
let path = write_atomic(dir.path(), vt, &cells).unwrap();
|
||
assert!(path.exists());
|
||
assert!(path.to_string_lossy().ends_with(".mp.gz"));
|
||
|
||
// Decode and confirm the top-level shape.
|
||
let raw = std::fs::read(&path).unwrap();
|
||
let mut gz = GzDecoder::new(&raw[..]);
|
||
let mut buf = Vec::new();
|
||
gz.read_to_end(&mut buf).unwrap();
|
||
let decoded: rmpv::Value = rmp_serde::from_slice(&buf).unwrap();
|
||
let map = decoded.as_map().unwrap();
|
||
assert!(map
|
||
.iter()
|
||
.any(|(k, v)| k.as_str() == Some("v") && v.as_u64() == Some(1)));
|
||
assert!(map
|
||
.iter()
|
||
.any(|(k, v)| k.as_str() == Some("valid_time")
|
||
&& v.as_str() == Some("2026-04-19T14:00:00Z")));
|
||
let cells_val = &map
|
||
.iter()
|
||
.find(|(k, _)| k.as_str() == Some("cells"))
|
||
.unwrap()
|
||
.1;
|
||
let cells_arr = cells_val.as_array().unwrap();
|
||
assert_eq!(cells_arr.len(), 1);
|
||
}
|
||
|
||
#[test]
|
||
fn atomic_rename_replaces_existing() {
|
||
let dir = tempfile::tempdir().unwrap();
|
||
let vt = Utc.with_ymd_and_hms(2026, 4, 19, 14, 0, 0).unwrap();
|
||
let first = vec![CellEntry {
|
||
lat: 32.0,
|
||
lon: -97.0,
|
||
profile: value_map([("v", rmpv::Value::Integer(1.into()))]),
|
||
}];
|
||
let second = vec![CellEntry {
|
||
lat: 32.0,
|
||
lon: -97.0,
|
||
profile: value_map([("v", rmpv::Value::Integer(2.into()))]),
|
||
}];
|
||
let p1 = write_atomic(dir.path(), vt, &first).unwrap();
|
||
let p2 = write_atomic(dir.path(), vt, &second).unwrap();
|
||
assert_eq!(p1, p2);
|
||
// Only one file in the profiles dir — the tmp was renamed away.
|
||
let entries: Vec<_> = std::fs::read_dir(dir.path().join("profiles"))
|
||
.unwrap()
|
||
.map(|e| e.unwrap().file_name().into_string().unwrap())
|
||
.collect();
|
||
assert_eq!(entries.len(), 1);
|
||
}
|
||
|
||
#[test]
|
||
fn path_for_layout() {
|
||
let vt = Utc.with_ymd_and_hms(2026, 4, 19, 14, 0, 0).unwrap();
|
||
let p = path_for(std::path::Path::new("/data/scores"), vt);
|
||
assert_eq!(
|
||
p.to_str().unwrap(),
|
||
"/data/scores/profiles/2026-04-19T14:00:00Z.mp.gz"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn snap_coords_rounds_to_three_decimals() {
|
||
assert_eq!(snap_coords(32.8998, -97.04031), (32.900, -97.040));
|
||
}
|
||
}
|