Fix ASOS upsert timeout: infinite tx timeout, bump prod pool to 20
458K-record upsert held a connection for the entire transaction, exceeding the 15s Postgrex timeout on prod. Set transaction timeout to infinity and increase prod pool from 10 to 20.
This commit is contained in:
parent
ffae4b52e7
commit
ee6975bc39
2 changed files with 16 additions and 13 deletions
|
|
@ -49,7 +49,7 @@ if config_env() == :prod do
|
||||||
config :microwaveprop, Microwaveprop.Repo,
|
config :microwaveprop, Microwaveprop.Repo,
|
||||||
# ssl: true,
|
# ssl: true,
|
||||||
url: database_url,
|
url: database_url,
|
||||||
pool_size: String.to_integer(System.get_env("POOL_SIZE") || "10"),
|
pool_size: String.to_integer(System.get_env("POOL_SIZE") || "20"),
|
||||||
# For machines with several cores, consider starting multiple pools of `pool_size`
|
# For machines with several cores, consider starting multiple pools of `pool_size`
|
||||||
# pool_count: 4,
|
# pool_count: 4,
|
||||||
socket_options: maybe_ipv6
|
socket_options: maybe_ipv6
|
||||||
|
|
|
||||||
|
|
@ -77,19 +77,22 @@ defmodule Microwaveprop.Propagation do
|
||||||
end)
|
end)
|
||||||
|
|
||||||
result =
|
result =
|
||||||
Repo.transaction(fn ->
|
Repo.transaction(
|
||||||
entries
|
fn ->
|
||||||
|> Enum.chunk_every(500)
|
entries
|
||||||
|> Enum.reduce(0, fn chunk, acc ->
|
|> Enum.chunk_every(500)
|
||||||
{count, _} =
|
|> Enum.reduce(0, fn chunk, acc ->
|
||||||
Repo.insert_all(GridScore, chunk,
|
{count, _} =
|
||||||
on_conflict: {:replace, [:score, :factors, :updated_at]},
|
Repo.insert_all(GridScore, chunk,
|
||||||
conflict_target: [:lat, :lon, :valid_time, :band_mhz]
|
on_conflict: {:replace, [:score, :factors, :updated_at]},
|
||||||
)
|
conflict_target: [:lat, :lon, :valid_time, :band_mhz]
|
||||||
|
)
|
||||||
|
|
||||||
acc + count
|
acc + count
|
||||||
end)
|
end)
|
||||||
end)
|
end,
|
||||||
|
timeout: :infinity
|
||||||
|
)
|
||||||
|
|
||||||
case result do
|
case result do
|
||||||
{:ok, _count} -> prune_old_scores()
|
{:ok, _count} -> prune_old_scores()
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue