fix(weather): prune dense .sgrid scalar files
ScalarFile.list_valid_time_dirs/1 stripped only a `.hrdps` suffix before DateTime.from_iso8601/1, so `<iso>.sgrid` and `<iso>.hrdps.sgrid` never parsed and were skipped by both prune_older_than/1 and retain_window/2. Since .sgrid is the format the pipeline actually writes, nothing on the dense tier was ever deleted — production held 222 files spanning 8 days against a 48h window. Strip the known suffixes in order, mirroring ProfilesFile's anchored parser (which documents this exact failure mode for .hrdps.prop). Orphaned .tmp.* writes still don't parse, so they stay with the tmp sweep in Propagation.prune_old_scores/0. Also corrects the CLAUDE.md entry that recorded this as already fixed — the call site was present, the suffix parsing was not.
This commit is contained in:
parent
7c37bf67fe
commit
289cfe5eef
3 changed files with 80 additions and 3 deletions
|
|
@ -263,7 +263,7 @@ Both `.pgrid` (profiles) and `.sgrid` (weather scalars) use this `write_atomic`
|
|||
|
||||
**Known cleanup gaps (confirmed):**
|
||||
- `.hrdps.prop` score files — ✅ **Confirmed fixed**: `scores_file.ex:516` `parse_valid_time_dt` handles `.hrdps.prop` via `~r/^(.+)\.(?:hrdps\.prop|prop|ntms)$/`
|
||||
- `ScalarFile.prune_older_than` — ✅ **Confirmed fixed**: called from `propagation.ex:240` alongside `ScoresFile` and `ProfilesFile` prune calls
|
||||
- `ScalarFile.prune_older_than` — ✅ **Confirmed fixed**: called from `propagation.ex:240` alongside `ScoresFile` and `ProfilesFile` prune calls. Note the call site was never the problem — until 2026-08-05 `list_valid_time_dirs/1` stripped only a `.hrdps` suffix before `DateTime.from_iso8601/1`, so `<iso>.sgrid` never parsed and both prune paths silently skipped every dense file (production held 222 files spanning 8 days against a 48 h window). Verify suffix parsing, not just the call site, when auditing a prune path.
|
||||
- Orphaned `.tmp.*` files — ✅ **Confirmed fixed**: `propagation.ex:254` sweeps `ScoresFile.base_dir/ProfilesFile.base_dir/ScalarFile.base_dir` for `.tmp.*` files after each prune cycle
|
||||
|
||||
**Known cleanup gaps (unverified):**
|
||||
|
|
|
|||
|
|
@ -553,8 +553,8 @@ defmodule Microwaveprop.Weather.ScalarFile do
|
|||
case File.ls(base) do
|
||||
{:ok, entries} ->
|
||||
for entry <- entries,
|
||||
iso = String.replace_suffix(entry, ".hrdps", ""),
|
||||
{:ok, dt, _} <- [DateTime.from_iso8601(iso)],
|
||||
dt = entry_valid_time(entry),
|
||||
dt != nil,
|
||||
do: {Path.join(base, entry), dt}
|
||||
|
||||
_ ->
|
||||
|
|
@ -562,6 +562,33 @@ defmodule Microwaveprop.Weather.ScalarFile do
|
|||
end
|
||||
end
|
||||
|
||||
# Entry shapes living under `weather_scalars/`:
|
||||
#
|
||||
# <iso> legacy HRRR chunk dir
|
||||
# <iso>.hrdps legacy HRDPS chunk dir
|
||||
# <iso>.sgrid dense HRRR scalars (current)
|
||||
# <iso>.hrdps.sgrid dense HRDPS scalars (current)
|
||||
#
|
||||
# Strip the known suffixes in order, mirroring `ProfilesFile`'s
|
||||
# anchored-extension parser. Handling only `.hrdps` left every
|
||||
# `<iso>.sgrid` failing `DateTime.from_iso8601/1`, which made the
|
||||
# dense files — the format the pipeline actually writes — invisible to
|
||||
# `prune_older_than/1` and `retain_window/2`.
|
||||
#
|
||||
# Anything else (notably orphaned `.tmp.*` writes) returns nil and is
|
||||
# left for the tmp sweep in `Propagation.prune_old_scores/0`.
|
||||
defp entry_valid_time(entry) do
|
||||
iso =
|
||||
entry
|
||||
|> String.replace_suffix(".sgrid", "")
|
||||
|> String.replace_suffix(".hrdps", "")
|
||||
|
||||
case DateTime.from_iso8601(iso) do
|
||||
{:ok, dt, _} -> dt
|
||||
_ -> nil
|
||||
end
|
||||
end
|
||||
|
||||
# Match ProfilesFile's snap step (0.125°) so a click at any lat/lon
|
||||
# rounds to the same key the writer used.
|
||||
defp snap(value) do
|
||||
|
|
|
|||
|
|
@ -461,5 +461,55 @@ defmodule Microwaveprop.Weather.ScalarFileTest do
|
|||
assert ScalarFile.exists?(inside)
|
||||
refute ScalarFile.exists?(outside)
|
||||
end
|
||||
|
||||
# `.sgrid` is the format the pipeline actually writes. Stripping only
|
||||
# a `.hrdps` suffix before `from_iso8601/1` left every dense file
|
||||
# unparseable and therefore invisible to both prune paths —
|
||||
# production accumulated 222 files spanning 8 days against a 48 h
|
||||
# retention window.
|
||||
test "prune_older_than removes dense .sgrid files" do
|
||||
old = ~U[2026-04-27 12:00:00Z]
|
||||
new = ~U[2026-04-28 12:00:00Z]
|
||||
|
||||
File.mkdir_p!(ScalarFile.base_dir())
|
||||
File.write!(Sgrid.path_for(old), "x")
|
||||
File.write!(Sgrid.path_for_hrdps(old), "x")
|
||||
File.write!(Sgrid.path_for(new), "x")
|
||||
|
||||
assert ScalarFile.prune_older_than(~U[2026-04-28 00:00:00Z]) == 2
|
||||
refute File.exists?(Sgrid.path_for(old))
|
||||
refute File.exists?(Sgrid.path_for_hrdps(old))
|
||||
assert File.exists?(Sgrid.path_for(new))
|
||||
end
|
||||
|
||||
test "retain_window removes dense .sgrid files outside the window" do
|
||||
run_time = ~U[2026-04-28 12:00:00Z]
|
||||
inside = ~U[2026-04-28 14:00:00Z]
|
||||
outside = ~U[2026-04-29 06:00:00Z]
|
||||
|
||||
File.mkdir_p!(ScalarFile.base_dir())
|
||||
File.write!(Sgrid.path_for(run_time), "x")
|
||||
File.write!(Sgrid.path_for_hrdps(inside), "x")
|
||||
File.write!(Sgrid.path_for(outside), "x")
|
||||
|
||||
assert ScalarFile.retain_window(run_time, 4) == 1
|
||||
assert File.exists?(Sgrid.path_for(run_time))
|
||||
assert File.exists?(Sgrid.path_for_hrdps(inside))
|
||||
refute File.exists?(Sgrid.path_for(outside))
|
||||
end
|
||||
|
||||
# Orphaned atomic-write temporaries are swept separately by
|
||||
# `Propagation.prune_old_scores/0`; they must not parse as a
|
||||
# valid_time here or a crashed write would be pruned as if it were
|
||||
# a finished artifact.
|
||||
test "ignores orphaned .tmp files" do
|
||||
vt = ~U[2026-04-27 12:00:00Z]
|
||||
File.mkdir_p!(ScalarFile.base_dir())
|
||||
tmp = Sgrid.path_for(vt) <> ".tmp.123.4"
|
||||
File.write!(tmp, "x")
|
||||
|
||||
assert ScalarFile.prune_older_than(~U[2026-04-28 00:00:00Z]) == 0
|
||||
assert File.exists?(tmp)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue