Skip updating unchanged scores to reduce dead tuples, tune autovacuum
ON CONFLICT now only replaces score/factors when the score actually changed, avoiding dead tuple generation for the ~80% of grid points that don't change between consecutive HRRR runs. Migration sets aggressive autovacuum on propagation_scores: zero cost delay, 2000 cost limit, 1% scale factor. The table was at 72GB with 108M dead rows because default autovacuum couldn't keep pace with 14M upserts per hour.
This commit is contained in:
parent
4759c65809
commit
23aa2786ab
2 changed files with 44 additions and 1 deletions
|
|
@ -129,7 +129,17 @@ defmodule Microwaveprop.Propagation do
|
||||||
|> Enum.reduce(0, fn chunk, acc ->
|
|> Enum.reduce(0, fn chunk, acc ->
|
||||||
{count, _} =
|
{count, _} =
|
||||||
Repo.insert_all(GridScore, chunk,
|
Repo.insert_all(GridScore, chunk,
|
||||||
on_conflict: {:replace, [:score, :factors, :updated_at]},
|
on_conflict:
|
||||||
|
from(g in GridScore,
|
||||||
|
update: [
|
||||||
|
set: [
|
||||||
|
score: fragment("EXCLUDED.score"),
|
||||||
|
factors: fragment("EXCLUDED.factors"),
|
||||||
|
updated_at: fragment("EXCLUDED.updated_at")
|
||||||
|
]
|
||||||
|
],
|
||||||
|
where: g.score != fragment("EXCLUDED.score")
|
||||||
|
),
|
||||||
conflict_target: [:lat, :lon, :valid_time, :band_mhz]
|
conflict_target: [:lat, :lon, :valid_time, :band_mhz]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,33 @@
|
||||||
|
defmodule Microwaveprop.Repo.Migrations.TunePropagationScoresAutovacuum do
|
||||||
|
use Ecto.Migration
|
||||||
|
|
||||||
|
def up do
|
||||||
|
# Aggressive autovacuum for propagation_scores — high churn table with
|
||||||
|
# ~14M upserts per hourly run. Default settings can't keep up, causing
|
||||||
|
# 100M+ dead tuples and 70GB+ bloat.
|
||||||
|
execute """
|
||||||
|
ALTER TABLE propagation_scores SET (
|
||||||
|
autovacuum_vacuum_scale_factor = 0.01,
|
||||||
|
autovacuum_vacuum_threshold = 10000,
|
||||||
|
autovacuum_vacuum_cost_delay = 0,
|
||||||
|
autovacuum_vacuum_cost_limit = 2000,
|
||||||
|
autovacuum_analyze_scale_factor = 0.02
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Also run ANALYZE to refresh stats after this change
|
||||||
|
execute "ANALYZE propagation_scores"
|
||||||
|
end
|
||||||
|
|
||||||
|
def down do
|
||||||
|
execute """
|
||||||
|
ALTER TABLE propagation_scores RESET (
|
||||||
|
autovacuum_vacuum_scale_factor,
|
||||||
|
autovacuum_vacuum_threshold,
|
||||||
|
autovacuum_vacuum_cost_delay,
|
||||||
|
autovacuum_vacuum_cost_limit,
|
||||||
|
autovacuum_analyze_scale_factor
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
end
|
||||||
|
end
|
||||||
Loading…
Add table
Reference in a new issue