Fix ASOS upsert timeout: infinite tx timeout, bump prod pool to 20

458K-record upsert held a connection for the entire transaction,
exceeding the 15s Postgrex timeout on prod. Set transaction timeout
to infinity and increase prod pool from 10 to 20.
This commit is contained in:
Graham McIntire 2026-03-31 15:54:19 -05:00
parent ffae4b52e7
commit ee6975bc39
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
2 changed files with 16 additions and 13 deletions

View file

@ -49,7 +49,7 @@ if config_env() == :prod do
config :microwaveprop, Microwaveprop.Repo,
# ssl: true,
url: database_url,
pool_size: String.to_integer(System.get_env("POOL_SIZE") || "10"),
pool_size: String.to_integer(System.get_env("POOL_SIZE") || "20"),
# For machines with several cores, consider starting multiple pools of `pool_size`
# pool_count: 4,
socket_options: maybe_ipv6

View file

@ -77,19 +77,22 @@ defmodule Microwaveprop.Propagation do
end)
result =
Repo.transaction(fn ->
entries
|> Enum.chunk_every(500)
|> Enum.reduce(0, fn chunk, acc ->
{count, _} =
Repo.insert_all(GridScore, chunk,
on_conflict: {:replace, [:score, :factors, :updated_at]},
conflict_target: [:lat, :lon, :valid_time, :band_mhz]
)
Repo.transaction(
fn ->
entries
|> Enum.chunk_every(500)
|> Enum.reduce(0, fn chunk, acc ->
{count, _} =
Repo.insert_all(GridScore, chunk,
on_conflict: {:replace, [:score, :factors, :updated_at]},
conflict_target: [:lat, :lon, :valid_time, :band_mhz]
)
acc + count
end)
end)
acc + count
end)
end,
timeout: :infinity
)
case result do
{:ok, _count} -> prune_old_scores()