Batch is_grid_point backfill and reduce HRRR workers to 5
- Split migration backfill into 50k-row batches to avoid container timeout - Reduce prod HRRR worker concurrency from 10 to 5
This commit is contained in:
parent
16883591b4
commit
ad1b01807c
2 changed files with 29 additions and 8 deletions
|
|
@ -117,7 +117,7 @@ if config_env() == :prod do
|
||||||
|
|
||||||
# Production Oban: live scoring, polling, and on-demand QSO enrichment (no cron backfill)
|
# Production Oban: live scoring, polling, and on-demand QSO enrichment (no cron backfill)
|
||||||
config :microwaveprop, Oban,
|
config :microwaveprop, Oban,
|
||||||
queues: [propagation: 1, commercial: 2, solar: 1, weather: 10, hrrr: 10, terrain: 5, iemre: 10, backfill_enqueue: 1],
|
queues: [propagation: 1, commercial: 2, solar: 1, weather: 10, hrrr: 5, terrain: 5, iemre: 10, backfill_enqueue: 1],
|
||||||
plugins: [
|
plugins: [
|
||||||
{Oban.Plugins.Pruner, max_age: 3600 * 24},
|
{Oban.Plugins.Pruner, max_age: 3600 * 24},
|
||||||
{Oban.Plugins.Lifeline, rescue_after: 30 * 60 * 1000},
|
{Oban.Plugins.Lifeline, rescue_after: 30 * 60 * 1000},
|
||||||
|
|
|
||||||
|
|
@ -6,16 +6,37 @@ defmodule Microwaveprop.Repo.Migrations.AddIsGridPointToHrrrProfiles do
|
||||||
add :is_grid_point, :boolean, default: false, null: false
|
add :is_grid_point, :boolean, default: false, null: false
|
||||||
end
|
end
|
||||||
|
|
||||||
flush()
|
|
||||||
|
|
||||||
execute(
|
|
||||||
"UPDATE hrrr_profiles SET is_grid_point = true WHERE (lat * 1000)::int % 125 = 0 AND (lon * 1000)::int % 125 = 0",
|
|
||||||
"SELECT 1"
|
|
||||||
)
|
|
||||||
|
|
||||||
create index(:hrrr_profiles, [:valid_time],
|
create index(:hrrr_profiles, [:valid_time],
|
||||||
where: "is_grid_point = true",
|
where: "is_grid_point = true",
|
||||||
name: :hrrr_profiles_grid_point_valid_time_idx
|
name: :hrrr_profiles_grid_point_valid_time_idx
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Backfill is_grid_point in batches to avoid long-running single UPDATE
|
||||||
|
flush()
|
||||||
|
|
||||||
|
execute(
|
||||||
|
"""
|
||||||
|
DO $$
|
||||||
|
DECLARE
|
||||||
|
batch_size INT := 50000;
|
||||||
|
rows_updated INT := 1;
|
||||||
|
BEGIN
|
||||||
|
WHILE rows_updated > 0 LOOP
|
||||||
|
UPDATE hrrr_profiles
|
||||||
|
SET is_grid_point = true
|
||||||
|
WHERE id IN (
|
||||||
|
SELECT id FROM hrrr_profiles
|
||||||
|
WHERE is_grid_point = false
|
||||||
|
AND (lat * 1000)::int % 125 = 0
|
||||||
|
AND (lon * 1000)::int % 125 = 0
|
||||||
|
LIMIT batch_size
|
||||||
|
);
|
||||||
|
GET DIAGNOSTICS rows_updated = ROW_COUNT;
|
||||||
|
RAISE NOTICE 'Updated % rows', rows_updated;
|
||||||
|
END LOOP;
|
||||||
|
END $$;
|
||||||
|
""",
|
||||||
|
"SELECT 1"
|
||||||
|
)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue