Chunk batch inserts to avoid Postgres parameter limit, add ANALYZE migration

This commit is contained in:
Graham McIntire 2026-04-03 09:34:02 -05:00
parent b4012baf89
commit 33490d37d0
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
2 changed files with 28 additions and 12 deletions

View file

@ -211,19 +211,26 @@ defmodule Microwaveprop.Weather do
def upsert_hrrr_profiles_batch(profiles) do def upsert_hrrr_profiles_batch(profiles) do
now = DateTime.truncate(DateTime.utc_now(), :second) now = DateTime.truncate(DateTime.utc_now(), :second)
entries = profiles
Enum.map(profiles, fn attrs -> |> Enum.chunk_every(500)
Map.merge(attrs, %{ |> Enum.reduce({0, nil}, fn chunk, {total_count, _} ->
id: Ecto.UUID.generate(), entries =
inserted_at: now, Enum.map(chunk, fn attrs ->
updated_at: now Map.merge(attrs, %{
}) id: Ecto.UUID.generate(),
end) inserted_at: now,
updated_at: now
})
end)
Repo.insert_all(HrrrProfile, entries, {count, rows} =
on_conflict: {:replace_all_except, [:id, :inserted_at]}, Repo.insert_all(HrrrProfile, entries,
conflict_target: [:lat, :lon, :valid_time] on_conflict: {:replace_all_except, [:id, :inserted_at]},
) conflict_target: [:lat, :lon, :valid_time]
)
{total_count + count, rows}
end)
end end
def has_hrrr_profile?(lat, lon, valid_time) do def has_hrrr_profile?(lat, lon, valid_time) do

View file

@ -0,0 +1,9 @@
defmodule Microwaveprop.Repo.Migrations.AnalyzeAllTables do
use Ecto.Migration
def up do
execute "ANALYZE"
end
def down, do: :ok
end