feat(hrdps): /weather merges HRRR with HRDPS for north-of-CONUS cells

The /weather map now shows HRRR-derived data wherever HRRR has
coverage and falls back to HRDPS-derived data for Canadian cells
outside HRRR's bbox. North of the 49°N HRRR-coverage line the map
stays populated instead of going blank.

Rust:

- weather_scalar_file::dir_for_hrdps + write_atomic_hrdps land HRDPS
  scalar chunks at `<vt>.hrdps/` so HRRR's `<vt>/` write isn't
  clobbered. The HRRR writer wipes its dir before each write — both
  sources need their own dir to coexist.
- pipeline::run_chain_step_hrdps now derives a ScalarRow for each
  scored cell (same derive_row that HRRR uses; same wire format) and
  writes them via write_atomic_hrdps alongside the .hrdps.prop score
  files.

Elixir:

- ScalarFile.dir_for_hrdps + read_bounds + read_point + exists? all
  walk both `<vt>/` (HRRR) and `<vt>.hrdps/` (HRDPS). read_bounds
  concatenates with HRRR-precedence on overlap (defensive — Rust
  pipeline doesn't write HRRR-overlap cells, but cheap to enforce).
- Weather.weather_grid_at unchanged: it routes through ScalarFile,
  which now transparently returns the union. GridCache stays
  per-valid_time so a single cache entry holds both regions.

Tests cover four scenarios: HRRR-only, HRDPS-only, both-present
union, and HRRR-precedence on the same cell.
This commit is contained in:
Graham McIntire 2026-04-29 17:36:03 -05:00
parent 1da78a80e5
commit 3607a915ba
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
4 changed files with 193 additions and 18 deletions

View file

@ -94,9 +94,24 @@ defmodule Microwaveprop.Weather.ScalarFile do
Path.join(base_dir(), iso_key(valid_time))
end
@doc """
Sibling dir for HRDPS-derived scalar chunks. The HRRR `dir_for/1`
writer wipes its destination dir of stale chunks before each write;
routing HRDPS to a parallel directory means the two sources don't
clobber each other. Read paths read both dirs and merge.
"""
@spec dir_for_hrdps(DateTime.t()) :: String.t()
def dir_for_hrdps(%DateTime{} = valid_time) do
Path.join(base_dir(), iso_key(valid_time) <> ".hrdps")
end
@spec exists?(DateTime.t()) :: boolean()
def exists?(%DateTime{} = valid_time) do
case File.ls(dir_for(valid_time)) do
has_chunks?(dir_for(valid_time)) or has_chunks?(dir_for_hrdps(valid_time))
end
defp has_chunks?(dir) do
case File.ls(dir) do
{:ok, [_ | _]} -> true
_ -> false
end
@ -147,18 +162,42 @@ defmodule Microwaveprop.Weather.ScalarFile do
"""
@spec read_bounds(DateTime.t(), bounds() | nil) :: [row()]
def read_bounds(%DateTime{} = valid_time, bounds) do
case list_chunk_files(valid_time) do
[] ->
[]
# Read HRRR + HRDPS chunks, then concat. HRRR wins where both
# sources cover (shouldn't happen in practice — Rust's HRDPS
# pipeline drops cells in HRRR's CONUS bbox before writing — but
# de-dup defensively in case mask-disagreement ever produces
# overlap). The user-visible intent: show HRRR-derived weather data
# everywhere CONUS, then fill the rest of North America with HRDPS.
hrrr = read_chunks(list_chunk_files(valid_time), bounds)
hrdps = read_chunks(list_chunk_files_hrdps(valid_time), bounds)
files ->
files
|> Enum.filter(fn {key, _path} -> chunk_intersects_bounds?(key, bounds) end)
|> Enum.flat_map(fn {_key, path} -> decode_chunk(path) end)
|> filter_bounds(bounds)
case {hrrr, hrdps} do
{[], []} -> []
{h, []} -> h
{[], c} -> c
{h, c} -> merge_prefer_hrrr(h, c)
end
end
defp read_chunks([], _bounds), do: []
defp read_chunks(files, bounds) do
files
|> Enum.filter(fn {key, _path} -> chunk_intersects_bounds?(key, bounds) end)
|> Enum.flat_map(fn {_key, path} -> decode_chunk(path) end)
|> filter_bounds(bounds)
end
defp merge_prefer_hrrr(hrrr_rows, hrdps_rows) do
hrrr_keys =
MapSet.new(hrrr_rows, fn %{lat: lat, lon: lon} -> {lat, lon} end)
extras =
Enum.reject(hrdps_rows, fn %{lat: lat, lon: lon} -> MapSet.member?(hrrr_keys, {lat, lon}) end)
hrrr_rows ++ extras
end
@doc """
Look up a single derived row at `(valid_time, lat, lon)`. Returns
`{:ok, row}` or `:miss` when either the chunk file is absent or the
@ -167,13 +206,29 @@ defmodule Microwaveprop.Weather.ScalarFile do
@spec read_point(DateTime.t(), float(), float()) :: {:ok, row()} | :miss
def read_point(%DateTime{} = valid_time, lat, lon) when is_number(lat) and is_number(lon) do
key = {chunk_band(lat * 1.0), chunk_band(lon * 1.0)}
path = Path.join(dir_for(valid_time), chunk_filename(key))
chunk_name = chunk_filename(key)
snapped_lat = snap(lat)
snapped_lon = snap(lon)
if File.exists?(path) do
find_point_in_chunk(path, snap(lat), snap(lon))
else
:miss
end
# Try HRRR first (where most points live), then HRDPS for Canadian
# cells outside HRRR coverage.
Enum.find_value(
[
Path.join(dir_for(valid_time), chunk_name),
Path.join(dir_for_hrdps(valid_time), chunk_name)
],
:miss,
fn path ->
if File.exists?(path) do
case find_point_in_chunk(path, snapped_lat, snapped_lon) do
{:ok, row} -> {:ok, row}
:miss -> false
end
else
false
end
end
)
end
defp find_point_in_chunk(path, snapped_lat, snapped_lon) do
@ -350,8 +405,14 @@ defmodule Microwaveprop.Weather.ScalarFile do
defp normalize_value(_key, v), do: v
defp list_chunk_files(%DateTime{} = valid_time) do
dir = dir_for(valid_time)
list_chunk_files_in(dir_for(valid_time))
end
defp list_chunk_files_hrdps(%DateTime{} = valid_time) do
list_chunk_files_in(dir_for_hrdps(valid_time))
end
defp list_chunk_files_in(dir) do
case File.ls(dir) do
{:ok, entries} ->
for entry <- entries,

View file

@ -340,6 +340,7 @@ pub async fn run_chain_step_hrdps(
let mut prepared: Vec<(f64, f64, Conditions, scorer::BandInvariants)> =
Vec::with_capacity(grid.len());
let mut scalar_rows: Vec<ScalarRow> = Vec::with_capacity(grid.len());
for (key, mut cell) in grid.into_iter() {
let (lat, lon) = decoder::key_to_latlon(key);
// HRDPS publishes DEPR (T-Td depression in K) as a primary
@ -350,6 +351,14 @@ pub async fn run_chain_step_hrdps(
if let Some(conditions) = cell_to_conditions(&cell, lat, lon, &valid_time) {
let invariants = scorer::precompute_band_invariants(&conditions);
prepared.push((lat, lon, conditions, invariants));
// Derive the per-cell scalar row alongside scoring. The /weather
// map reads these via Weather.weather_grid_at to render
// temperature/dewpoint-depression/refractivity layers — surfacing
// HRDPS here is what gives Canadian cells weather data on the
// map north of the HRRR coverage line.
if let Some(row) = weather_scalar_file::derive_row(lat, lon, valid_time, &cell) {
scalar_rows.push(row);
}
}
}
@ -375,6 +384,15 @@ pub async fn run_chain_step_hrdps(
};
let scores_dir_owned = scores_dir.to_path_buf();
// Write the HRDPS scalar artifact alongside the scores. Sibling dir
// (`<vt>.hrdps/`) so HRRR's `<vt>/` write isn't clobbered. Without
// this the /weather map would only show CONUS data.
let scalar_dir = scores_dir_owned.clone();
let scalar_future = tokio::task::spawn_blocking(move || {
weather_scalar_file::write_atomic_hrdps(&scalar_dir, valid_time, &scalar_rows).map(|_| ())
});
let write_handles: Vec<_> = scored
.into_iter()
.map(|(band_mhz, scores)| {
@ -387,6 +405,10 @@ pub async fn run_chain_step_hrdps(
for h in write_handles {
h.await.expect("blocking join")?;
}
scalar_future
.await
.expect("blocking join")
.map_err(PipelineError::ScalarWrite)?;
let files_written = bands.len() as u32;
Ok(ChainStepStats {

View file

@ -194,12 +194,22 @@ pub fn derive_row(
})
}
/// Absolute path the writer lands at for `valid_time`.
/// Absolute path the HRRR writer lands at for `valid_time`.
pub fn dir_for(scores_dir: &Path, valid_time: DateTime<Utc>) -> PathBuf {
let iso = valid_time.format("%Y-%m-%dT%H:%M:%SZ").to_string();
scores_dir.join(SUBDIR).join(iso)
}
/// Sibling directory for HRDPS-derived scalar chunks. Coexists with the
/// HRRR `dir_for` directory so the two sources can write 5°×5° chunks
/// independently — `write_atomic`'s "wipe dir first" sweep would
/// otherwise have whichever source ran last clobber the other's chunks.
/// Readers stitch the two sets together at request time.
pub fn dir_for_hrdps(scores_dir: &Path, valid_time: DateTime<Utc>) -> PathBuf {
let iso = valid_time.format("%Y-%m-%dT%H:%M:%SZ").to_string();
scores_dir.join(SUBDIR).join(format!("{iso}.hrdps"))
}
/// Persist `rows` for `valid_time` as 5°×5° chunked `.mp.gz` files.
/// Mirrors the Elixir writer: the destination directory is wiped of
/// any prior chunks first so a smaller follow-up write doesn't leave
@ -210,7 +220,20 @@ pub fn write_atomic(
valid_time: DateTime<Utc>,
rows: &[ScalarRow],
) -> Result<PathBuf, WriteError> {
let dir = dir_for(scores_dir, valid_time);
write_atomic_into(dir_for(scores_dir, valid_time), rows)
}
/// HRDPS sibling of `write_atomic` that lands at `dir_for_hrdps`. Wipes
/// only its own dir, leaving the HRRR `<vt>/` untouched.
pub fn write_atomic_hrdps(
scores_dir: &Path,
valid_time: DateTime<Utc>,
rows: &[ScalarRow],
) -> Result<PathBuf, WriteError> {
write_atomic_into(dir_for_hrdps(scores_dir, valid_time), rows)
}
fn write_atomic_into(dir: PathBuf, rows: &[ScalarRow]) -> Result<PathBuf, WriteError> {
std::fs::create_dir_all(&dir)?;
if let Ok(entries) = std::fs::read_dir(&dir) {

View file

@ -155,6 +155,75 @@ defmodule Microwaveprop.Weather.ScalarFileTest do
end
end
describe "HRDPS sibling read merge" do
test "read_bounds returns HRRR + HRDPS rows when both dirs exist" do
vt = ~U[2026-04-29 12:00:00Z]
ScalarFile.write!(vt, [sample_row(33.0, -97.0, 22.0)])
hand_write_chunk(ScalarFile.dir_for_hrdps(vt), 53.0, -60.0, 8.0)
rows = ScalarFile.read_bounds(vt, nil)
sorted = Enum.sort_by(rows, & &1.lat)
assert length(sorted) == 2
assert Enum.at(sorted, 0).lat == 33.0
assert Enum.at(sorted, 1).lat == 53.0
end
test "read_bounds returns HRDPS rows when only HRDPS dir exists" do
vt = ~U[2026-04-29 12:00:00Z]
hand_write_chunk(ScalarFile.dir_for_hrdps(vt), 53.0, -60.0, 8.0)
assert [%{lat: 53.0, lon: -60.0}] = ScalarFile.read_bounds(vt, nil)
end
test "read_point falls through to HRDPS when the HRRR chunk is absent" do
vt = ~U[2026-04-29 12:00:00Z]
hand_write_chunk(ScalarFile.dir_for_hrdps(vt), 53.0, -60.0, 8.0)
assert {:ok, %{lat: 53.0, lon: -60.0, temperature: 8.0}} =
ScalarFile.read_point(vt, 53.0, -60.0)
end
test "exists?/1 is true when only the HRDPS dir has chunks" do
vt = ~U[2026-04-29 12:00:00Z]
hand_write_chunk(ScalarFile.dir_for_hrdps(vt), 53.0, -60.0, 8.0)
assert ScalarFile.exists?(vt)
end
test "merge prefers HRRR row when both dirs cover the same lat/lon" do
vt = ~U[2026-04-29 12:00:00Z]
ScalarFile.write!(vt, [sample_row(33.0, -97.0, 22.0)])
hand_write_chunk(ScalarFile.dir_for_hrdps(vt), 33.0, -97.0, 99.0)
[row] = ScalarFile.read_bounds(vt, nil)
assert row.temperature == 22.0
end
end
defp hand_write_chunk(dir, lat, lon, temp) do
File.mkdir_p!(dir)
lat_band = (lat / 5) |> Float.floor() |> trunc()
lon_band = (lon / 5) |> Float.floor() |> trunc()
path = Path.join(dir, "#{lat_band}_#{lon_band}.mp.gz")
payload =
Msgpax.pack!(
[
%{
"lat" => lat,
"lon" => lon,
"valid_time" => "2026-04-29T12:00:00Z",
"temperature" => temp
}
],
iodata: false
)
File.write!(path, :zlib.gzip(payload))
end
describe "Rust-writer compatibility (the cross-language wire format)" do
test "decodes a hand-rolled gzipped MessagePack chunk and atomizes whitelisted keys" do
valid_time = ~U[2026-04-29 12:00:00Z]