diff --git a/lib/microwaveprop/weather.ex b/lib/microwaveprop/weather.ex index f87fb2ea..0bc8adc0 100644 --- a/lib/microwaveprop/weather.ex +++ b/lib/microwaveprop/weather.ex @@ -208,8 +208,9 @@ defmodule Microwaveprop.Weather do ) end - def upsert_hrrr_profiles_batch(profiles) do + def upsert_hrrr_profiles_batch(profiles, opts \\ []) do now = DateTime.truncate(DateTime.utc_now(), :second) + on_conflict = if Keyword.get(opts, :skip_existing, false), do: :nothing, else: {:replace_all_except, [:id, :inserted_at]} profiles |> Enum.chunk_every(500) @@ -225,7 +226,7 @@ defmodule Microwaveprop.Weather do {count, rows} = Repo.insert_all(HrrrProfile, entries, - on_conflict: {:replace_all_except, [:id, :inserted_at]}, + on_conflict: on_conflict, conflict_target: [:lat, :lon, :valid_time] ) diff --git a/lib/microwaveprop/workers/hrrr_fetch_worker.ex b/lib/microwaveprop/workers/hrrr_fetch_worker.ex index 3020d0c9..d13f6081 100644 --- a/lib/microwaveprop/workers/hrrr_fetch_worker.ex +++ b/lib/microwaveprop/workers/hrrr_fetch_worker.ex @@ -37,7 +37,7 @@ defmodule Microwaveprop.Workers.HrrrFetchWorker do build_profile_attrs(lat, lon, valid_time, data) end) - Weather.upsert_hrrr_profiles_batch(profiles) + Weather.upsert_hrrr_profiles_batch(profiles, skip_existing: true) Enum.each(grid_data, fn {{lat, lon}, _data} -> broadcast_enrichment(lat, lon, valid_time)