graham/prop
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix backfill page 500 by using pg_stat estimates instead of count(*)

Browse source 
The 6 sequential count(*) queries on large tables were timing out
at 15s when concurrent HRRR batch inserts held locks. Reuse the
n_live_tup estimates already fetched from pg_stat_user_tables.
This commit is contained in:
Graham McIntire 2026-04-02 14:55:53 -05:00
parent 66de041886
commit bb1c46fd57
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
1 changed files with 28 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
lib/microwaveprop_web/live/backfill_live.ex
View file

@ -81,15 +81,16 @@ defmodule MicrowavepropWeb.BackfillLive do
defp enqueue_batch(limit) do defp enqueue_batch(limit) do
contacts = contacts =
from(c in Contact, Repo.all(
where: from(c in Contact,
(not is_nil(c.pos1) or not is_nil(c.grid1)) and where:
(c.hrrr_status in ^@enrichable or c.weather_status in ^@enrichable or (not is_nil(c.pos1) or not is_nil(c.grid1)) and
c.terrain_status in ^@enrichable or c.iemre_status in ^@enrichable), (c.hrrr_status in ^@enrichable or c.weather_status in ^@enrichable or c.terrain_status in ^@enrichable or
order_by: [desc: c.qso_timestamp], c.iemre_status in ^@enrichable),
limit: ^limit order_by: [desc: c.qso_timestamp],
limit: ^limit
)
) )
|> Repo.all()
# ensure_positions! is called inside enqueue_for_contact # ensure_positions! is called inside enqueue_for_contact
Enum.each(contacts, &ContactWeatherEnqueueWorker.enqueue_for_contact/1) Enum.each(contacts, &ContactWeatherEnqueueWorker.enqueue_for_contact/1)
@ -127,7 +128,8 @@ defmodule MicrowavepropWeb.BackfillLive do
) )
by_worker = by_worker =
Enum.group_by(jobs, &elem(&1, 0), fn {_, state, count} -> {state, count} end) jobs
|> Enum.group_by(&elem(&1, 0), fn {_, state, count} -> {state, count} end)
|> Enum.map(fn {worker, states} -> |> Enum.map(fn {worker, states} ->
short_name = worker |> String.split(".") |> List.last() short_name = worker |> String.split(".") |> List.last()
total = Enum.reduce(states, 0, fn {_, c}, acc -> acc + c end) total = Enum.reduce(states, 0, fn {_, c}, acc -> acc + c end)
@ -207,21 +209,15 @@ defmodule MicrowavepropWeb.BackfillLive do
{status, count} {status, count}
end) end)
# Per-contact averages # Per-table row counts from pg_stat estimates (already fetched above, avoids slow count(*))
%{rows: avg_rows} = table_counts = Map.new(tables, fn %{table: t, rows: r} -> {t, r} end)
Repo.query!("""
SELECT
(SELECT count(*) FROM hrrr_profiles) as hrrr_profiles,
(SELECT count(*) FROM terrain_profiles) as terrain_profiles,
(SELECT count(*) FROM surface_observations) as surface_obs,
(SELECT count(*) FROM soundings) as soundings,
(SELECT count(*) FROM iemre_observations) as iemre_obs,
(SELECT count(*) FROM qsos WHERE pos1 IS NOT NULL) as contacts
""")
[[hrrr_count, terrain_count, obs_count, sounding_count, iemre_count, contact_count]] = avg_rows hrrr_count = Map.get(table_counts, "hrrr_profiles", 0)
terrain_count = Map.get(table_counts, "terrain_profiles", 0)
contact_count = max(contact_count, 1) obs_count = Map.get(table_counts, "surface_observations", 0)
sounding_count = Map.get(table_counts, "soundings", 0)
iemre_count = Map.get(table_counts, "iemre_observations", 0)
contact_count = max(Map.get(table_counts, "qsos", 0), 1)
%{ %{
db_size: format_bytes(db_bytes), db_size: format_bytes(db_bytes),
@ -265,14 +261,11 @@ defmodule MicrowavepropWeb.BackfillLive do
defp status_class("unavailable"), do: "opacity-40" defp status_class("unavailable"), do: "opacity-40"
defp status_class(_), do: "" defp status_class(_), do: ""
defp format_bytes(bytes) when bytes >= 1_073_741_824, defp format_bytes(bytes) when bytes >= 1_073_741_824, do: "#{Float.round(bytes / 1_073_741_824, 1)} GB"
do: "#{Float.round(bytes / 1_073_741_824, 1)} GB"
defp format_bytes(bytes) when bytes >= 1_048_576, defp format_bytes(bytes) when bytes >= 1_048_576, do: "#{Float.round(bytes / 1_048_576, 1)} MB"
do: "#{Float.round(bytes / 1_048_576, 1)} MB"
defp format_bytes(bytes) when bytes >= 1024, defp format_bytes(bytes) when bytes >= 1024, do: "#{Float.round(bytes / 1024, 1)} KB"
do: "#{Float.round(bytes / 1024, 1)} KB"
defp format_bytes(bytes), do: "#{bytes} B" defp format_bytes(bytes), do: "#{bytes} B"
@ -412,12 +405,16 @@ defmodule MicrowavepropWeb.BackfillLive do
</div> </div>
<div class="bg-base-200 rounded-box p-3"> <div class="bg-base-200 rounded-box p-3">
<div class="text-xs opacity-60">HRRR Profiles</div> <div class="text-xs opacity-60">HRRR Profiles</div>
<div class="text-lg font-bold font-mono">{format_number(@db_stats.totals.hrrr_profiles)}</div> <div class="text-lg font-bold font-mono">
{format_number(@db_stats.totals.hrrr_profiles)}
</div>
<div class="text-xs opacity-60">{@db_stats.averages.hrrr_per_contact} per contact</div> <div class="text-xs opacity-60">{@db_stats.averages.hrrr_per_contact} per contact</div>
</div> </div>
<div class="bg-base-200 rounded-box p-3"> <div class="bg-base-200 rounded-box p-3">
<div class="text-xs opacity-60">Surface Observations</div> <div class="text-xs opacity-60">Surface Observations</div>
<div class="text-lg font-bold font-mono">{format_number(@db_stats.totals.surface_observations)}</div> <div class="text-lg font-bold font-mono">
{format_number(@db_stats.totals.surface_observations)}
</div>
<div class="text-xs opacity-60">{@db_stats.averages.obs_per_contact} per contact</div> <div class="text-xs opacity-60">{@db_stats.averages.obs_per_contact} per contact</div>
</div> </div>
<div class="bg-base-200 rounded-box p-3"> <div class="bg-base-200 rounded-box p-3">