fix: batch pskr calibration upsert into 1,000-row chunks to avoid Postgres 65,535 param limit
This commit is contained in:
parent
38ee285833
commit
a2e2776e7c
1 changed files with 33 additions and 24 deletions
|
|
@ -267,11 +267,19 @@ defmodule Microwaveprop.Pskr.CalibrationSampler do
|
|||
|
||||
# ── Upsert ──────────────────────────────────────────────────────
|
||||
|
||||
# Each sample has ~20 fields. PostgreSQL's parameter limit is 65,535,
|
||||
# so a single insert_all with > ~3,200 rows will fail. Batch at 1,000
|
||||
# to stay well under the limit even at peak hours.
|
||||
@upsert_batch_size 1_000
|
||||
|
||||
defp upsert([]), do: 0
|
||||
|
||||
defp upsert(samples) do
|
||||
samples
|
||||
|> Enum.chunk_every(@upsert_batch_size)
|
||||
|> Enum.reduce(0, fn batch, total ->
|
||||
{count, _} =
|
||||
Repo.insert_all(CalibrationSample, samples,
|
||||
Repo.insert_all(CalibrationSample, batch,
|
||||
on_conflict:
|
||||
from(c in CalibrationSample,
|
||||
update: [
|
||||
|
|
@ -295,7 +303,8 @@ defmodule Microwaveprop.Pskr.CalibrationSampler do
|
|||
conflict_target: [:hour_utc, :band, :midpoint_lat, :midpoint_lon]
|
||||
)
|
||||
|
||||
count
|
||||
total + count
|
||||
end)
|
||||
end
|
||||
|
||||
defp align_to_hour(%DateTime{} = dt) do
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue