Fix ASOS upsert timeout: infinite tx timeout, bump prod pool to 20
458K-record upsert held a connection for the entire transaction, exceeding the 15s Postgrex timeout on prod. Set transaction timeout to infinity and increase prod pool from 10 to 20.
This commit is contained in:
parent
ffae4b52e7
commit
ee6975bc39
2 changed files with 16 additions and 13 deletions
|
|
@ -49,7 +49,7 @@ if config_env() == :prod do
|
|||
config :microwaveprop, Microwaveprop.Repo,
|
||||
# ssl: true,
|
||||
url: database_url,
|
||||
pool_size: String.to_integer(System.get_env("POOL_SIZE") || "10"),
|
||||
pool_size: String.to_integer(System.get_env("POOL_SIZE") || "20"),
|
||||
# For machines with several cores, consider starting multiple pools of `pool_size`
|
||||
# pool_count: 4,
|
||||
socket_options: maybe_ipv6
|
||||
|
|
|
|||
|
|
@ -77,19 +77,22 @@ defmodule Microwaveprop.Propagation do
|
|||
end)
|
||||
|
||||
result =
|
||||
Repo.transaction(fn ->
|
||||
entries
|
||||
|> Enum.chunk_every(500)
|
||||
|> Enum.reduce(0, fn chunk, acc ->
|
||||
{count, _} =
|
||||
Repo.insert_all(GridScore, chunk,
|
||||
on_conflict: {:replace, [:score, :factors, :updated_at]},
|
||||
conflict_target: [:lat, :lon, :valid_time, :band_mhz]
|
||||
)
|
||||
Repo.transaction(
|
||||
fn ->
|
||||
entries
|
||||
|> Enum.chunk_every(500)
|
||||
|> Enum.reduce(0, fn chunk, acc ->
|
||||
{count, _} =
|
||||
Repo.insert_all(GridScore, chunk,
|
||||
on_conflict: {:replace, [:score, :factors, :updated_at]},
|
||||
conflict_target: [:lat, :lon, :valid_time, :band_mhz]
|
||||
)
|
||||
|
||||
acc + count
|
||||
end)
|
||||
end)
|
||||
acc + count
|
||||
end)
|
||||
end,
|
||||
timeout: :infinity
|
||||
)
|
||||
|
||||
case result do
|
||||
{:ok, _count} -> prune_old_scores()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue