From 02b354f5f7e6d5afdbe7560582dc632e6e8d41ab Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Wed, 15 Apr 2026 12:14:15 -0500 Subject: [PATCH] Purge legacy is_grid_point=true rows from hrrr_profiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The grid worker stopped writing HRRR profiles to the DB on April 14 when the forecast grid moved to /data/scores binary files, but the 24/72h-windowed prune left every is_grid_point=true row older than 72 hours sitting in the table (billions of rows across every partition back to 2016, ~90% of the 146 GB table size). Nothing reads them anymore. Replace prune_old_grid_profiles/0 with purge_grid_point_profiles/0, which walks every hrrr_profiles partition directly and deletes is_grid_point=true rows regardless of age. Per-partition DELETEs avoid a parent-table full scan. QSO-linked rows (is_grid_point=false) are untouched — those still back contact enrichment and /path. Call the new purge from the grid worker's end-of-chain hook so any grid-point row that somehow slips back in gets cleaned up within the next hour. Add `mix hrrr.purge_grid_points` for a one-shot historical cleanup. --- lib/microwaveprop/weather.ex | 67 ++++++++++++------- .../workers/propagation_grid_worker.ex | 2 +- lib/mix/tasks/hrrr_purge_grid_points.ex | 28 ++++++++ test/microwaveprop/weather_test.exs | 52 ++++++++++++++ 4 files changed, 123 insertions(+), 26 deletions(-) create mode 100644 lib/mix/tasks/hrrr_purge_grid_points.ex diff --git a/lib/microwaveprop/weather.ex b/lib/microwaveprop/weather.ex index 31deabb2..14e73394 100644 --- a/lib/microwaveprop/weather.ex +++ b/lib/microwaveprop/weather.ex @@ -922,39 +922,56 @@ defmodule Microwaveprop.Weather do end @doc """ - Delete HRRR grid profiles older than 24 hours that sit on the 0.125° grid. - Preserves QSO-linked profiles which are at arbitrary lat/lon positions. - With 19 forecast hours per run, this keeps ~2 runs worth of grid profiles. + Delete every `is_grid_point = true` row from `hrrr_profiles`, regardless of + age. Grid-point profiles are historical — the propagation grid now lives in + `/data/scores` binary files, nothing reads `is_grid_point = true` rows + anymore, and forecast runs no longer write them. This purge walks each + partition directly so a single DELETE can't scan the whole parent table. + Preserves QSO-linked rows (`is_grid_point = false`), which remain the data + path for contact enrichment and the `/path` calculator. """ - @spec prune_old_grid_profiles() :: non_neg_integer() - def prune_old_grid_profiles do - cutoff = DateTime.add(DateTime.utc_now(), -24, :hour) - # Bound the scan to recent partitions only (grid profiles are at most ~48h old). - # Without a lower bound, Postgres scans every partition back to 2016. - floor = DateTime.add(DateTime.utc_now(), -72, :hour) + @spec purge_grid_point_profiles() :: non_neg_integer() + def purge_grid_point_profiles do + deleted = + Enum.reduce(hrrr_profile_partitions(), 0, fn partition, acc -> + %{num_rows: n} = + Repo.query!( + ~s(DELETE FROM "#{partition}" WHERE is_grid_point = true), + [], + timeout: 600_000 + ) - # Only delete profiles on the 0.125° propagation grid, preserving QSO-linked - # profiles which sit at arbitrary lat/lon positions (HRRR's native ~3km grid). - %{num_rows: deleted} = - Repo.query!( - """ - DELETE FROM hrrr_profiles - WHERE valid_time >= $2 AND valid_time < $1 - AND is_grid_point = true - """, - [cutoff, floor], - timeout: 300_000 - ) + if n > 0 do + require Logger - if deleted > 0 do - require Logger + Logger.info("Purged #{n} grid-point rows from #{partition}") + end - Logger.info("Pruned #{deleted} old HRRR grid profiles (valid_time < #{cutoff})") - end + acc + n + end) deleted end + @spec hrrr_profile_partitions() :: [String.t()] + defp hrrr_profile_partitions do + {:ok, %{rows: rows}} = + Repo.query( + """ + SELECT child.relname + FROM pg_inherits + JOIN pg_class parent ON pg_inherits.inhparent = parent.oid + JOIN pg_class child ON pg_inherits.inhrelid = child.oid + WHERE parent.relname = 'hrrr_profiles' + ORDER BY child.relname + """, + [], + timeout: 60_000 + ) + + Enum.map(rows, fn [name] -> name end) + end + @spec round_to_iemre_grid(float(), float()) :: {float(), float()} def round_to_iemre_grid(lat, lon) do {Float.round(lat * 8) / 8, Float.round(lon * 8) / 8} diff --git a/lib/microwaveprop/workers/propagation_grid_worker.ex b/lib/microwaveprop/workers/propagation_grid_worker.ex index eb6b95c7..22104fb1 100644 --- a/lib/microwaveprop/workers/propagation_grid_worker.ex +++ b/lib/microwaveprop/workers/propagation_grid_worker.ex @@ -133,7 +133,7 @@ defmodule Microwaveprop.Workers.PropagationGridWorker do # Split into its own function to keep `run_chain_step/2` under # credo's nesting limit. defp handle_step_transition(:final, run_time) do - Weather.prune_old_grid_profiles() + Weather.purge_grid_point_profiles() Propagation.prune_old_scores() # Drop any files left over from the previous chain that the new # chain didn't happen to overwrite (files are keyed by diff --git a/lib/mix/tasks/hrrr_purge_grid_points.ex b/lib/mix/tasks/hrrr_purge_grid_points.ex new file mode 100644 index 00000000..219d3b2a --- /dev/null +++ b/lib/mix/tasks/hrrr_purge_grid_points.ex @@ -0,0 +1,28 @@ +defmodule Mix.Tasks.Hrrr.PurgeGridPoints do + @shortdoc "Delete every `is_grid_point = true` row from hrrr_profiles" + + @moduledoc """ + Removes every `is_grid_point = true` row from `hrrr_profiles` across every + partition. The propagation grid now lives in `/data/scores` binary files, + nothing reads grid-point rows anymore, and the grid worker no longer writes + them — this task reclaims the disk space from the legacy data that + accumulated before the April 14 cutover. + + Leaves `is_grid_point = false` (QSO-linked) rows untouched. + + mix hrrr.purge_grid_points + """ + + use Mix.Task + + alias Microwaveprop.Weather + + @impl Mix.Task + def run(_argv) do + Mix.Task.run("app.start") + + Mix.shell().info("Purging grid-point rows from hrrr_profiles...") + deleted = Weather.purge_grid_point_profiles() + Mix.shell().info("Purged #{deleted} grid-point rows total.") + end +end diff --git a/test/microwaveprop/weather_test.exs b/test/microwaveprop/weather_test.exs index 3e8b495b..b2e90d65 100644 --- a/test/microwaveprop/weather_test.exs +++ b/test/microwaveprop/weather_test.exs @@ -468,6 +468,58 @@ defmodule Microwaveprop.WeatherTest do end end + describe "purge_grid_point_profiles/0" do + test "deletes all grid-point rows across partitions while preserving QSO-linked rows" do + # QSO-linked row (is_grid_point defaults to false) + {:ok, qso_linked} = Weather.upsert_hrrr_profile(@hrrr_profile_attrs) + + # Grid-point row in a historical partition + Repo.query!( + """ + INSERT INTO hrrr_profiles + (id, valid_time, lat, lon, is_grid_point, inserted_at, updated_at) + VALUES ($1, $2, $3, $4, true, $5, $5) + """, + [ + Ecto.UUID.bingenerate(), + ~N[2023-07-15 12:00:00], + 32.125, + -97.125, + NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second) + ] + ) + + # Grid-point row in the current partition + Repo.query!( + """ + INSERT INTO hrrr_profiles + (id, valid_time, lat, lon, is_grid_point, inserted_at, updated_at) + VALUES ($1, $2, $3, $4, true, $5, $5) + """, + [ + Ecto.UUID.bingenerate(), + NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second), + 33.125, + -96.125, + NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second) + ] + ) + + assert Repo.aggregate(HrrrProfile, :count) == 3 + assert Weather.purge_grid_point_profiles() == 2 + + remaining = Repo.all(HrrrProfile) + assert length(remaining) == 1 + assert hd(remaining).id == qso_linked.id + end + + test "returns 0 when there are no grid-point rows" do + {:ok, _} = Weather.upsert_hrrr_profile(@hrrr_profile_attrs) + assert Weather.purge_grid_point_profiles() == 0 + assert Repo.aggregate(HrrrProfile, :count) == 1 + end + end + describe "has_hrrr_profile?/3" do test "returns true when a profile exists at the given location and time" do Weather.upsert_hrrr_profile(@hrrr_profile_attrs)