From 03c91f409898357f545f497996fbb5bc70729af7 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Fri, 24 Apr 2026 15:48:42 -0500 Subject: [PATCH] perf(db): tune autovacuum per-table + one-shot ANALYZE helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two issues spotted in pg_stat_user_tables: 1. Most `hrrr_profiles` partitions have `last_autoanalyze IS NULL`. The partitions are append-only from the hourly grid worker, so autovacuum never crosses the analyze threshold — but without stats the planner falls back to estimates and picks bad joins on per-QSO lookups. 2. `contacts`, `grid_tasks`, `hrrr_fetch_tasks`, and `contact_common_volume_radar` see high UPDATE volumes (enrichment status flips, queue claim/complete cycles, mechanism classifier writes) but only accrue ~5% dead before the default 20% / 10% autovacuum triggers fire. BackfillEnqueueWorker's status-priority ORDER BY + the Rust workers' FOR UPDATE SKIP LOCKED claim scan both want fresher stats than that. Changes: - Migration `20260424204656_tune_autovacuum_for_high_churn` sets per-table `autovacuum_*_scale_factor` + `_threshold = 50` on the four tables above. Aggressive enough to keep stats current; tame enough that vacuum doesn't thrash the Turing Pi 2 Postgres node. Reversible. - `Weather.analyze_all/1` walks `pg_stat_user_tables`, runs `ANALYZE` on every table not auto-analyzed in the last 6 h, reports counts. Meant to be invoked once via `bin/microwaveprop rpc 'Microwaveprop.Weather.analyze_all()'` after deploy to seed statistics on the cold partitions; idempotent on re-run. --- lib/microwaveprop/weather.ex | 53 +++++++++++++++ ...4204656_tune_autovacuum_for_high_churn.exs | 66 +++++++++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 priv/repo/migrations/20260424204656_tune_autovacuum_for_high_churn.exs diff --git a/lib/microwaveprop/weather.ex b/lib/microwaveprop/weather.ex index 36bc4638..49eaf7ff 100644 --- a/lib/microwaveprop/weather.ex +++ b/lib/microwaveprop/weather.ex @@ -410,6 +410,59 @@ defmodule Microwaveprop.Weather do {:ok, n} end + @doc """ + Run `ANALYZE` on every public-schema table that has not been auto- + analyzed recently. Callable from a release shell: + + bin/microwaveprop rpc 'Microwaveprop.Weather.analyze_all()' + + Surfaces the HRRR partition problem: most partitions have zero + analyze stats, so the planner picks nested-loop joins that scan + 81 M rows. Running ANALYZE once gives it live_tup and column + histograms to work with. + + `skip_recent` skips anything auto-analyzed in the last N seconds + (default 6 h) so re-running is cheap. + """ + @spec analyze_all(keyword()) :: %{analyzed: non_neg_integer(), skipped: non_neg_integer()} + def analyze_all(opts \\ []) do + skip_recent = Keyword.get(opts, :skip_recent_seconds, 6 * 3600) + + q = """ + SELECT relname + FROM pg_stat_user_tables + WHERE schemaname = 'public' + AND (last_autoanalyze IS NULL + OR last_autoanalyze < now() - ($1 || ' seconds')::interval) + """ + + %{rows: rows} = Repo.query!(q, [Integer.to_string(skip_recent)]) + tables = Enum.map(rows, fn [n] -> n end) + + Enum.each(tables, fn t -> + Repo.query!("ANALYZE #{quote_ident(t)}", [], timeout: :infinity) + end) + + skipped = + Repo.query!(""" + SELECT count(*) FROM pg_stat_user_tables + WHERE schemaname = 'public' + AND last_autoanalyze >= now() - ('#{skip_recent} seconds')::interval + """).rows + |> hd() + |> hd() + + %{analyzed: length(tables), skipped: skipped} + end + + # Defensive: pg_stat_user_tables returns legitimate identifiers from + # the catalog, but we still quote just in case a partition contains a + # character the shell would interpret. + defp quote_ident(name) when is_binary(name) do + escaped = String.replace(name, ~s("), ~s("")) + ~s("#{escaped}") + end + @doc """ Derive `surface_refractivity` / `min_refractivity_gradient` / `ducting_detected` for every `hrrr_profiles` row where those columns diff --git a/priv/repo/migrations/20260424204656_tune_autovacuum_for_high_churn.exs b/priv/repo/migrations/20260424204656_tune_autovacuum_for_high_churn.exs new file mode 100644 index 00000000..a5c356f7 --- /dev/null +++ b/priv/repo/migrations/20260424204656_tune_autovacuum_for_high_churn.exs @@ -0,0 +1,66 @@ +defmodule Microwaveprop.Repo.Migrations.TuneAutovacuumForHighChurn do + use Ecto.Migration + + @moduledoc """ + Per-table autovacuum/analyze parameters for the hottest paths. + + Default PG 16 thresholds (`scale_factor 0.2` for vacuum, `0.1` for + analyze) are too lazy for a handful of tables we hit frequently: + + * `contacts` — every enrichment tick UPDATEs one or more status + columns. 81 k rows × 0.1 analyze scale = 8 k dead tuples before + stats refresh; BackfillEnqueueWorker's status-priority ORDER BY + then picks a bad plan. Drop to 0.02 / 0.01. + + * `grid_tasks` / `hrrr_fetch_tasks` — small queue tables with a + claim/complete cycle that rewrites every row each hour. Defaults + mostly keep up, but the claim query's `FOR UPDATE SKIP LOCKED` + scan benefits from fresh stats. Set thresholds so vacuum/analyze + fire at 50 modified rows instead of ~(default 50) + 20%. + + * `contact_common_volume_radar` — MechanismClassifyWorker UPDATEs + bring the analyze stale-tuple counter to ~200 k without ever + flipping the default 10 % threshold. Matches the contacts + shape at 0.02 / 0.01. + + All other tables keep PG defaults — including `hrrr_profiles` + partitions (append-heavy, analyze-fires-naturally on insert) and + the Oban-owned tables (`oban_jobs`, `oban_producers`, `oban_peers`), + which Oban's own Pruner handles. + + Reversible — `down` resets each back to the cluster-wide default. + """ + + @high_churn [ + {"contacts", 0.02, 0.01}, + {"grid_tasks", 0.05, 0.02}, + {"hrrr_fetch_tasks", 0.05, 0.02}, + {"contact_common_volume_radar", 0.02, 0.01} + ] + + def up do + for {table, vac_scale, analyze_scale} <- @high_churn do + execute(""" + ALTER TABLE #{table} SET ( + autovacuum_vacuum_scale_factor = #{vac_scale}, + autovacuum_analyze_scale_factor = #{analyze_scale}, + autovacuum_vacuum_threshold = 50, + autovacuum_analyze_threshold = 50 + ) + """) + end + end + + def down do + for {table, _, _} <- @high_churn do + execute(""" + ALTER TABLE #{table} RESET ( + autovacuum_vacuum_scale_factor, + autovacuum_analyze_scale_factor, + autovacuum_vacuum_threshold, + autovacuum_analyze_threshold + ) + """) + end + end +end