From 411951147a201c18054081a7ecc24719a67c4ef7 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Fri, 3 Apr 2026 13:29:15 -0500 Subject: [PATCH] Use ON CONFLICT DO NOTHING for HRRR backfill to prevent table bloat --- lib/microwaveprop/weather.ex | 5 +++-- lib/microwaveprop/workers/hrrr_fetch_worker.ex | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/microwaveprop/weather.ex b/lib/microwaveprop/weather.ex index f87fb2ea..0bc8adc0 100644 --- a/lib/microwaveprop/weather.ex +++ b/lib/microwaveprop/weather.ex @@ -208,8 +208,9 @@ defmodule Microwaveprop.Weather do ) end - def upsert_hrrr_profiles_batch(profiles) do + def upsert_hrrr_profiles_batch(profiles, opts \\ []) do now = DateTime.truncate(DateTime.utc_now(), :second) + on_conflict = if Keyword.get(opts, :skip_existing, false), do: :nothing, else: {:replace_all_except, [:id, :inserted_at]} profiles |> Enum.chunk_every(500) @@ -225,7 +226,7 @@ defmodule Microwaveprop.Weather do {count, rows} = Repo.insert_all(HrrrProfile, entries, - on_conflict: {:replace_all_except, [:id, :inserted_at]}, + on_conflict: on_conflict, conflict_target: [:lat, :lon, :valid_time] ) diff --git a/lib/microwaveprop/workers/hrrr_fetch_worker.ex b/lib/microwaveprop/workers/hrrr_fetch_worker.ex index 3020d0c9..d13f6081 100644 --- a/lib/microwaveprop/workers/hrrr_fetch_worker.ex +++ b/lib/microwaveprop/workers/hrrr_fetch_worker.ex @@ -37,7 +37,7 @@ defmodule Microwaveprop.Workers.HrrrFetchWorker do build_profile_attrs(lat, lon, valid_time, data) end) - Weather.upsert_hrrr_profiles_batch(profiles) + Weather.upsert_hrrr_profiles_batch(profiles, skip_existing: true) Enum.each(grid_data, fn {{lat, lon}, _data} -> broadcast_enrichment(lat, lon, valid_time)