fix: batch pskr calibration upsert into 1,000-row chunks to avoid Postgres 65,535 param limit

This commit is contained in:
Graham McInitre 2026-07-15 14:18:43 -05:00
parent 38ee285833
commit a2e2776e7c

View file

@ -267,11 +267,19 @@ defmodule Microwaveprop.Pskr.CalibrationSampler do
# ── Upsert ──────────────────────────────────────────────────────
# Each sample has ~20 fields. PostgreSQL's parameter limit is 65,535,
# so a single insert_all with > ~3,200 rows will fail. Batch at 1,000
# to stay well under the limit even at peak hours.
@upsert_batch_size 1_000
defp upsert([]), do: 0
defp upsert(samples) do
samples
|> Enum.chunk_every(@upsert_batch_size)
|> Enum.reduce(0, fn batch, total ->
{count, _} =
Repo.insert_all(CalibrationSample, samples,
Repo.insert_all(CalibrationSample, batch,
on_conflict:
from(c in CalibrationSample,
update: [
@ -295,7 +303,8 @@ defmodule Microwaveprop.Pskr.CalibrationSampler do
conflict_target: [:hour_utc, :band, :midpoint_lat, :midpoint_lon]
)
count
total + count
end)
end
defp align_to_hour(%DateTime{} = dt) do