From 23aa2786ab36dc86ecc35a4585c701305df9176e Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Sat, 4 Apr 2026 09:53:45 -0500
Subject: [PATCH] Skip updating unchanged scores to reduce dead tuples, tune
 autovacuum

ON CONFLICT now only replaces score/factors when the score actually changed,
avoiding dead tuple generation for the ~80% of grid points that don't change
between consecutive HRRR runs.

Migration sets aggressive autovacuum on propagation_scores: zero cost delay,
2000 cost limit, 1% scale factor. The table was at 72GB with 108M dead rows
because default autovacuum couldn't keep pace with 14M upserts per hour.
---
 lib/microwaveprop/propagation.ex              | 12 ++++++-
 ...255_tune_propagation_scores_autovacuum.exs | 33 +++++++++++++++++++
 2 files changed, 44 insertions(+), 1 deletion(-)
 create mode 100644 priv/repo/migrations/20260404145255_tune_propagation_scores_autovacuum.exs

diff --git a/lib/microwaveprop/propagation.ex b/lib/microwaveprop/propagation.ex
index 5f15573a..13ae877c 100644
--- a/lib/microwaveprop/propagation.ex
+++ b/lib/microwaveprop/propagation.ex
@@ -129,7 +129,17 @@ defmodule Microwaveprop.Propagation do
           |> Enum.reduce(0, fn chunk, acc ->
             {count, _} =
               Repo.insert_all(GridScore, chunk,
-                on_conflict: {:replace, [:score, :factors, :updated_at]},
+                on_conflict:
+                  from(g in GridScore,
+                    update: [
+                      set: [
+                        score: fragment("EXCLUDED.score"),
+                        factors: fragment("EXCLUDED.factors"),
+                        updated_at: fragment("EXCLUDED.updated_at")
+                      ]
+                    ],
+                    where: g.score != fragment("EXCLUDED.score")
+                  ),
                 conflict_target: [:lat, :lon, :valid_time, :band_mhz]
               )
 
diff --git a/priv/repo/migrations/20260404145255_tune_propagation_scores_autovacuum.exs b/priv/repo/migrations/20260404145255_tune_propagation_scores_autovacuum.exs
new file mode 100644
index 00000000..fa9b8cd7
--- /dev/null
+++ b/priv/repo/migrations/20260404145255_tune_propagation_scores_autovacuum.exs
@@ -0,0 +1,33 @@
+defmodule Microwaveprop.Repo.Migrations.TunePropagationScoresAutovacuum do
+  use Ecto.Migration
+
+  def up do
+    # Aggressive autovacuum for propagation_scores — high churn table with
+    # ~14M upserts per hourly run. Default settings can't keep up, causing
+    # 100M+ dead tuples and 70GB+ bloat.
+    execute """
+    ALTER TABLE propagation_scores SET (
+      autovacuum_vacuum_scale_factor = 0.01,
+      autovacuum_vacuum_threshold = 10000,
+      autovacuum_vacuum_cost_delay = 0,
+      autovacuum_vacuum_cost_limit = 2000,
+      autovacuum_analyze_scale_factor = 0.02
+    )
+    """
+
+    # Also run ANALYZE to refresh stats after this change
+    execute "ANALYZE propagation_scores"
+  end
+
+  def down do
+    execute """
+    ALTER TABLE propagation_scores RESET (
+      autovacuum_vacuum_scale_factor,
+      autovacuum_vacuum_threshold,
+      autovacuum_vacuum_cost_delay,
+      autovacuum_vacuum_cost_limit,
+      autovacuum_analyze_scale_factor
+    )
+    """
+  end
+end