Skip updating unchanged scores to reduce dead tuples, tune autovacuum
ON CONFLICT now only replaces score/factors when the score actually changed, avoiding dead tuple generation for the ~80% of grid points that don't change between consecutive HRRR runs. Migration sets aggressive autovacuum on propagation_scores: zero cost delay, 2000 cost limit, 1% scale factor. The table was at 72GB with 108M dead rows because default autovacuum couldn't keep pace with 14M upserts per hour.
This commit is contained in:
parent
4759c65809
commit
23aa2786ab
2 changed files with 44 additions and 1 deletions
|
|
@ -129,7 +129,17 @@ defmodule Microwaveprop.Propagation do
|
|||
|> Enum.reduce(0, fn chunk, acc ->
|
||||
{count, _} =
|
||||
Repo.insert_all(GridScore, chunk,
|
||||
on_conflict: {:replace, [:score, :factors, :updated_at]},
|
||||
on_conflict:
|
||||
from(g in GridScore,
|
||||
update: [
|
||||
set: [
|
||||
score: fragment("EXCLUDED.score"),
|
||||
factors: fragment("EXCLUDED.factors"),
|
||||
updated_at: fragment("EXCLUDED.updated_at")
|
||||
]
|
||||
],
|
||||
where: g.score != fragment("EXCLUDED.score")
|
||||
),
|
||||
conflict_target: [:lat, :lon, :valid_time, :band_mhz]
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,33 @@
|
|||
defmodule Microwaveprop.Repo.Migrations.TunePropagationScoresAutovacuum do
|
||||
use Ecto.Migration
|
||||
|
||||
def up do
|
||||
# Aggressive autovacuum for propagation_scores — high churn table with
|
||||
# ~14M upserts per hourly run. Default settings can't keep up, causing
|
||||
# 100M+ dead tuples and 70GB+ bloat.
|
||||
execute """
|
||||
ALTER TABLE propagation_scores SET (
|
||||
autovacuum_vacuum_scale_factor = 0.01,
|
||||
autovacuum_vacuum_threshold = 10000,
|
||||
autovacuum_vacuum_cost_delay = 0,
|
||||
autovacuum_vacuum_cost_limit = 2000,
|
||||
autovacuum_analyze_scale_factor = 0.02
|
||||
)
|
||||
"""
|
||||
|
||||
# Also run ANALYZE to refresh stats after this change
|
||||
execute "ANALYZE propagation_scores"
|
||||
end
|
||||
|
||||
def down do
|
||||
execute """
|
||||
ALTER TABLE propagation_scores RESET (
|
||||
autovacuum_vacuum_scale_factor,
|
||||
autovacuum_vacuum_threshold,
|
||||
autovacuum_vacuum_cost_delay,
|
||||
autovacuum_vacuum_cost_limit,
|
||||
autovacuum_analyze_scale_factor
|
||||
)
|
||||
"""
|
||||
end
|
||||
end
|
||||
Loading…
Add table
Reference in a new issue