graham/prop
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use ON CONFLICT DO NOTHING for HRRR backfill to prevent table bloat

Browse source
This commit is contained in:
Graham McIntire 2026-04-03 13:29:15 -05:00
parent 1c7c980cf4
commit 411951147a
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
2 changed files with 4 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
lib/microwaveprop/weather.ex
View file

@ -208,8 +208,9 @@ defmodule Microwaveprop.Weather do
)
end
def upsert_hrrr_profiles_batch(profiles) do
def upsert_hrrr_profiles_batch(profiles, opts \\ []) do
now = DateTime.truncate(DateTime.utc_now(), :second)
on_conflict = if Keyword.get(opts, :skip_existing, false), do: :nothing, else: {:replace_all_except, [:id, :inserted_at]}
profiles
|> Enum.chunk_every(500)
@ -225,7 +226,7 @@ defmodule Microwaveprop.Weather do
{count, rows} =
Repo.insert_all(HrrrProfile, entries,
on_conflict: {:replace_all_except, [:id, :inserted_at]},
on_conflict: on_conflict,
conflict_target: [:lat, :lon, :valid_time]
)

2
lib/microwaveprop/workers/hrrr_fetch_worker.ex
View file

@ -37,7 +37,7 @@ defmodule Microwaveprop.Workers.HrrrFetchWorker do
build_profile_attrs(lat, lon, valid_time, data)
end)
Weather.upsert_hrrr_profiles_batch(profiles)
Weather.upsert_hrrr_profiles_batch(profiles, skip_existing: true)
Enum.each(grid_data, fn {{lat, lon}, _data} ->
broadcast_enrichment(lat, lon, valid_time)