Fix ASOS upsert timeout: infinite tx timeout, bump prod pool to 20

458K-record upsert held a connection for the entire transaction,
exceeding the 15s Postgrex timeout on prod. Set transaction timeout
to infinity and increase prod pool from 10 to 20.
This commit is contained in:
Graham McIntire 2026-03-31 15:54:19 -05:00
parent ffae4b52e7
commit ee6975bc39
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
2 changed files with 16 additions and 13 deletions

View file

@ -49,7 +49,7 @@ if config_env() == :prod do
config :microwaveprop, Microwaveprop.Repo, config :microwaveprop, Microwaveprop.Repo,
# ssl: true, # ssl: true,
url: database_url, url: database_url,
pool_size: String.to_integer(System.get_env("POOL_SIZE") || "10"), pool_size: String.to_integer(System.get_env("POOL_SIZE") || "20"),
# For machines with several cores, consider starting multiple pools of `pool_size` # For machines with several cores, consider starting multiple pools of `pool_size`
# pool_count: 4, # pool_count: 4,
socket_options: maybe_ipv6 socket_options: maybe_ipv6

View file

@ -77,19 +77,22 @@ defmodule Microwaveprop.Propagation do
end) end)
result = result =
Repo.transaction(fn -> Repo.transaction(
entries fn ->
|> Enum.chunk_every(500) entries
|> Enum.reduce(0, fn chunk, acc -> |> Enum.chunk_every(500)
{count, _} = |> Enum.reduce(0, fn chunk, acc ->
Repo.insert_all(GridScore, chunk, {count, _} =
on_conflict: {:replace, [:score, :factors, :updated_at]}, Repo.insert_all(GridScore, chunk,
conflict_target: [:lat, :lon, :valid_time, :band_mhz] on_conflict: {:replace, [:score, :factors, :updated_at]},
) conflict_target: [:lat, :lon, :valid_time, :band_mhz]
)
acc + count acc + count
end) end)
end) end,
timeout: :infinity
)
case result do case result do
{:ok, _count} -> prune_old_scores() {:ok, _count} -> prune_old_scores()