From 289cfe5eef21c6d4e667bb5b5356b4cb70d4090c Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Wed, 5 Aug 2026 17:52:06 -0500 Subject: [PATCH] fix(weather): prune dense .sgrid scalar files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ScalarFile.list_valid_time_dirs/1 stripped only a `.hrdps` suffix before DateTime.from_iso8601/1, so `.sgrid` and `.hrdps.sgrid` never parsed and were skipped by both prune_older_than/1 and retain_window/2. Since .sgrid is the format the pipeline actually writes, nothing on the dense tier was ever deleted — production held 222 files spanning 8 days against a 48h window. Strip the known suffixes in order, mirroring ProfilesFile's anchored parser (which documents this exact failure mode for .hrdps.prop). Orphaned .tmp.* writes still don't parse, so they stay with the tmp sweep in Propagation.prune_old_scores/0. Also corrects the CLAUDE.md entry that recorded this as already fixed — the call site was present, the suffix parsing was not. --- CLAUDE.md | 2 +- lib/microwaveprop/weather/scalar_file.ex | 31 +++++++++++- .../weather/scalar_file_test.exs | 50 +++++++++++++++++++ 3 files changed, 80 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 86d0b448..15eb4e50 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -263,7 +263,7 @@ Both `.pgrid` (profiles) and `.sgrid` (weather scalars) use this `write_atomic` **Known cleanup gaps (confirmed):** - `.hrdps.prop` score files — ✅ **Confirmed fixed**: `scores_file.ex:516` `parse_valid_time_dt` handles `.hrdps.prop` via `~r/^(.+)\.(?:hrdps\.prop|prop|ntms)$/` -- `ScalarFile.prune_older_than` — ✅ **Confirmed fixed**: called from `propagation.ex:240` alongside `ScoresFile` and `ProfilesFile` prune calls +- `ScalarFile.prune_older_than` — ✅ **Confirmed fixed**: called from `propagation.ex:240` alongside `ScoresFile` and `ProfilesFile` prune calls. Note the call site was never the problem — until 2026-08-05 `list_valid_time_dirs/1` stripped only a `.hrdps` suffix before `DateTime.from_iso8601/1`, so `.sgrid` never parsed and both prune paths silently skipped every dense file (production held 222 files spanning 8 days against a 48 h window). Verify suffix parsing, not just the call site, when auditing a prune path. - Orphaned `.tmp.*` files — ✅ **Confirmed fixed**: `propagation.ex:254` sweeps `ScoresFile.base_dir/ProfilesFile.base_dir/ScalarFile.base_dir` for `.tmp.*` files after each prune cycle **Known cleanup gaps (unverified):** diff --git a/lib/microwaveprop/weather/scalar_file.ex b/lib/microwaveprop/weather/scalar_file.ex index 998c3093..f2ea3f96 100644 --- a/lib/microwaveprop/weather/scalar_file.ex +++ b/lib/microwaveprop/weather/scalar_file.ex @@ -553,8 +553,8 @@ defmodule Microwaveprop.Weather.ScalarFile do case File.ls(base) do {:ok, entries} -> for entry <- entries, - iso = String.replace_suffix(entry, ".hrdps", ""), - {:ok, dt, _} <- [DateTime.from_iso8601(iso)], + dt = entry_valid_time(entry), + dt != nil, do: {Path.join(base, entry), dt} _ -> @@ -562,6 +562,33 @@ defmodule Microwaveprop.Weather.ScalarFile do end end + # Entry shapes living under `weather_scalars/`: + # + # legacy HRRR chunk dir + # .hrdps legacy HRDPS chunk dir + # .sgrid dense HRRR scalars (current) + # .hrdps.sgrid dense HRDPS scalars (current) + # + # Strip the known suffixes in order, mirroring `ProfilesFile`'s + # anchored-extension parser. Handling only `.hrdps` left every + # `.sgrid` failing `DateTime.from_iso8601/1`, which made the + # dense files — the format the pipeline actually writes — invisible to + # `prune_older_than/1` and `retain_window/2`. + # + # Anything else (notably orphaned `.tmp.*` writes) returns nil and is + # left for the tmp sweep in `Propagation.prune_old_scores/0`. + defp entry_valid_time(entry) do + iso = + entry + |> String.replace_suffix(".sgrid", "") + |> String.replace_suffix(".hrdps", "") + + case DateTime.from_iso8601(iso) do + {:ok, dt, _} -> dt + _ -> nil + end + end + # Match ProfilesFile's snap step (0.125°) so a click at any lat/lon # rounds to the same key the writer used. defp snap(value) do diff --git a/test/microwaveprop/weather/scalar_file_test.exs b/test/microwaveprop/weather/scalar_file_test.exs index f58b84d8..46d969d8 100644 --- a/test/microwaveprop/weather/scalar_file_test.exs +++ b/test/microwaveprop/weather/scalar_file_test.exs @@ -461,5 +461,55 @@ defmodule Microwaveprop.Weather.ScalarFileTest do assert ScalarFile.exists?(inside) refute ScalarFile.exists?(outside) end + + # `.sgrid` is the format the pipeline actually writes. Stripping only + # a `.hrdps` suffix before `from_iso8601/1` left every dense file + # unparseable and therefore invisible to both prune paths — + # production accumulated 222 files spanning 8 days against a 48 h + # retention window. + test "prune_older_than removes dense .sgrid files" do + old = ~U[2026-04-27 12:00:00Z] + new = ~U[2026-04-28 12:00:00Z] + + File.mkdir_p!(ScalarFile.base_dir()) + File.write!(Sgrid.path_for(old), "x") + File.write!(Sgrid.path_for_hrdps(old), "x") + File.write!(Sgrid.path_for(new), "x") + + assert ScalarFile.prune_older_than(~U[2026-04-28 00:00:00Z]) == 2 + refute File.exists?(Sgrid.path_for(old)) + refute File.exists?(Sgrid.path_for_hrdps(old)) + assert File.exists?(Sgrid.path_for(new)) + end + + test "retain_window removes dense .sgrid files outside the window" do + run_time = ~U[2026-04-28 12:00:00Z] + inside = ~U[2026-04-28 14:00:00Z] + outside = ~U[2026-04-29 06:00:00Z] + + File.mkdir_p!(ScalarFile.base_dir()) + File.write!(Sgrid.path_for(run_time), "x") + File.write!(Sgrid.path_for_hrdps(inside), "x") + File.write!(Sgrid.path_for(outside), "x") + + assert ScalarFile.retain_window(run_time, 4) == 1 + assert File.exists?(Sgrid.path_for(run_time)) + assert File.exists?(Sgrid.path_for_hrdps(inside)) + refute File.exists?(Sgrid.path_for(outside)) + end + + # Orphaned atomic-write temporaries are swept separately by + # `Propagation.prune_old_scores/0`; they must not parse as a + # valid_time here or a crashed write would be pruned as if it were + # a finished artifact. + test "ignores orphaned .tmp files" do + vt = ~U[2026-04-27 12:00:00Z] + File.mkdir_p!(ScalarFile.base_dir()) + tmp = Sgrid.path_for(vt) <> ".tmp.123.4" + File.write!(tmp, "x") + + assert ScalarFile.prune_older_than(~U[2026-04-28 00:00:00Z]) == 0 + assert File.exists?(tmp) + end end end