diff --git a/config/runtime.exs b/config/runtime.exs index 08ef519a..c96f5d6e 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -117,7 +117,7 @@ if config_env() == :prod do # Production Oban: live scoring, polling, and on-demand QSO enrichment (no cron backfill) config :microwaveprop, Oban, - queues: [propagation: 1, commercial: 2, solar: 1, weather: 10, hrrr: 10, terrain: 5, iemre: 10, backfill_enqueue: 1], + queues: [propagation: 1, commercial: 2, solar: 1, weather: 10, hrrr: 5, terrain: 5, iemre: 10, backfill_enqueue: 1], plugins: [ {Oban.Plugins.Pruner, max_age: 3600 * 24}, {Oban.Plugins.Lifeline, rescue_after: 30 * 60 * 1000}, diff --git a/priv/repo/migrations/20260405001416_add_is_grid_point_to_hrrr_profiles.exs b/priv/repo/migrations/20260405001416_add_is_grid_point_to_hrrr_profiles.exs index 8bc61d42..af3e10c4 100644 --- a/priv/repo/migrations/20260405001416_add_is_grid_point_to_hrrr_profiles.exs +++ b/priv/repo/migrations/20260405001416_add_is_grid_point_to_hrrr_profiles.exs @@ -6,16 +6,37 @@ defmodule Microwaveprop.Repo.Migrations.AddIsGridPointToHrrrProfiles do add :is_grid_point, :boolean, default: false, null: false end - flush() - - execute( - "UPDATE hrrr_profiles SET is_grid_point = true WHERE (lat * 1000)::int % 125 = 0 AND (lon * 1000)::int % 125 = 0", - "SELECT 1" - ) - create index(:hrrr_profiles, [:valid_time], where: "is_grid_point = true", name: :hrrr_profiles_grid_point_valid_time_idx ) + + # Backfill is_grid_point in batches to avoid long-running single UPDATE + flush() + + execute( + """ + DO $$ + DECLARE + batch_size INT := 50000; + rows_updated INT := 1; + BEGIN + WHILE rows_updated > 0 LOOP + UPDATE hrrr_profiles + SET is_grid_point = true + WHERE id IN ( + SELECT id FROM hrrr_profiles + WHERE is_grid_point = false + AND (lat * 1000)::int % 125 = 0 + AND (lon * 1000)::int % 125 = 0 + LIMIT batch_size + ); + GET DIAGNOSTICS rows_updated = ROW_COUNT; + RAISE NOTICE 'Updated % rows', rows_updated; + END LOOP; + END $$; + """, + "SELECT 1" + ) end end