fix: batch pskr calibration upsert into 1,000-row chunks to avoid Postgres 65,535 param limit
This commit is contained in:
parent
38ee285833
commit
a2e2776e7c
1 changed files with 33 additions and 24 deletions
|
|
@ -267,11 +267,19 @@ defmodule Microwaveprop.Pskr.CalibrationSampler do
|
||||||
|
|
||||||
# ── Upsert ──────────────────────────────────────────────────────
|
# ── Upsert ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
# Each sample has ~20 fields. PostgreSQL's parameter limit is 65,535,
|
||||||
|
# so a single insert_all with > ~3,200 rows will fail. Batch at 1,000
|
||||||
|
# to stay well under the limit even at peak hours.
|
||||||
|
@upsert_batch_size 1_000
|
||||||
|
|
||||||
defp upsert([]), do: 0
|
defp upsert([]), do: 0
|
||||||
|
|
||||||
defp upsert(samples) do
|
defp upsert(samples) do
|
||||||
|
samples
|
||||||
|
|> Enum.chunk_every(@upsert_batch_size)
|
||||||
|
|> Enum.reduce(0, fn batch, total ->
|
||||||
{count, _} =
|
{count, _} =
|
||||||
Repo.insert_all(CalibrationSample, samples,
|
Repo.insert_all(CalibrationSample, batch,
|
||||||
on_conflict:
|
on_conflict:
|
||||||
from(c in CalibrationSample,
|
from(c in CalibrationSample,
|
||||||
update: [
|
update: [
|
||||||
|
|
@ -295,7 +303,8 @@ defmodule Microwaveprop.Pskr.CalibrationSampler do
|
||||||
conflict_target: [:hour_utc, :band, :midpoint_lat, :midpoint_lon]
|
conflict_target: [:hour_utc, :band, :midpoint_lat, :midpoint_lon]
|
||||||
)
|
)
|
||||||
|
|
||||||
count
|
total + count
|
||||||
|
end)
|
||||||
end
|
end
|
||||||
|
|
||||||
defp align_to_hour(%DateTime{} = dt) do
|
defp align_to_hour(%DateTime{} = dt) do
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue