Chunk batch inserts to avoid Postgres parameter limit, add ANALYZE migration

This commit is contained in:
Graham McIntire 2026-04-03 09:34:02 -05:00
parent b4012baf89
commit 33490d37d0
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
2 changed files with 28 additions and 12 deletions

View file

@ -211,19 +211,26 @@ defmodule Microwaveprop.Weather do
def upsert_hrrr_profiles_batch(profiles) do
now = DateTime.truncate(DateTime.utc_now(), :second)
entries =
Enum.map(profiles, fn attrs ->
Map.merge(attrs, %{
id: Ecto.UUID.generate(),
inserted_at: now,
updated_at: now
})
end)
profiles
|> Enum.chunk_every(500)
|> Enum.reduce({0, nil}, fn chunk, {total_count, _} ->
entries =
Enum.map(chunk, fn attrs ->
Map.merge(attrs, %{
id: Ecto.UUID.generate(),
inserted_at: now,
updated_at: now
})
end)
Repo.insert_all(HrrrProfile, entries,
on_conflict: {:replace_all_except, [:id, :inserted_at]},
conflict_target: [:lat, :lon, :valid_time]
)
{count, rows} =
Repo.insert_all(HrrrProfile, entries,
on_conflict: {:replace_all_except, [:id, :inserted_at]},
conflict_target: [:lat, :lon, :valid_time]
)
{total_count + count, rows}
end)
end
def has_hrrr_profile?(lat, lon, valid_time) do

View file

@ -0,0 +1,9 @@
defmodule Microwaveprop.Repo.Migrations.AnalyzeAllTables do
use Ecto.Migration
def up do
execute "ANALYZE"
end
def down, do: :ok
end