graham/prop
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix backfill page 500 by using pg_stat estimates instead of count(*)

Browse source 
The 6 sequential count(*) queries on large tables were timing out
at 15s when concurrent HRRR batch inserts held locks. Reuse the
n_live_tup estimates already fetched from pg_stat_user_tables.
This commit is contained in:
Graham McIntire 2026-04-02 14:55:53 -05:00
parent 66de041886
commit bb1c46fd57
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
1 changed files with 28 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
lib/microwaveprop_web/live/backfill_live.ex
View file

@ -81,15 +81,16 @@ defmodule MicrowavepropWeb.BackfillLive do
defp enqueue_batch(limit) do
contacts =
from(c in Contact,
where:
(not is_nil(c.pos1) or not is_nil(c.grid1)) and
(c.hrrr_status in ^@enrichable or c.weather_status in ^@enrichable or
c.terrain_status in ^@enrichable or c.iemre_status in ^@enrichable),
order_by: [desc: c.qso_timestamp],
limit: ^limit
Repo.all(
from(c in Contact,
where:
(not is_nil(c.pos1) or not is_nil(c.grid1)) and
(c.hrrr_status in ^@enrichable or c.weather_status in ^@enrichable or c.terrain_status in ^@enrichable or
c.iemre_status in ^@enrichable),
order_by: [desc: c.qso_timestamp],
limit: ^limit
)
)
|> Repo.all()
# ensure_positions! is called inside enqueue_for_contact
Enum.each(contacts, &ContactWeatherEnqueueWorker.enqueue_for_contact/1)
@ -127,7 +128,8 @@ defmodule MicrowavepropWeb.BackfillLive do
)
by_worker =
Enum.group_by(jobs, &elem(&1, 0), fn {_, state, count} -> {state, count} end)
jobs
|> Enum.group_by(&elem(&1, 0), fn {_, state, count} -> {state, count} end)
|> Enum.map(fn {worker, states} ->
short_name = worker |> String.split(".") |> List.last()
total = Enum.reduce(states, 0, fn {_, c}, acc -> acc + c end)
@ -207,21 +209,15 @@ defmodule MicrowavepropWeb.BackfillLive do
{status, count}
end)
# Per-contact averages
%{rows: avg_rows} =
Repo.query!("""
SELECT
(SELECT count(*) FROM hrrr_profiles) as hrrr_profiles,
(SELECT count(*) FROM terrain_profiles) as terrain_profiles,
(SELECT count(*) FROM surface_observations) as surface_obs,
(SELECT count(*) FROM soundings) as soundings,
(SELECT count(*) FROM iemre_observations) as iemre_obs,
(SELECT count(*) FROM qsos WHERE pos1 IS NOT NULL) as contacts
""")
# Per-table row counts from pg_stat estimates (already fetched above, avoids slow count(*))
table_counts = Map.new(tables, fn %{table: t, rows: r} -> {t, r} end)
[[hrrr_count, terrain_count, obs_count, sounding_count, iemre_count, contact_count]] = avg_rows
contact_count = max(contact_count, 1)
hrrr_count = Map.get(table_counts, "hrrr_profiles", 0)
terrain_count = Map.get(table_counts, "terrain_profiles", 0)
obs_count = Map.get(table_counts, "surface_observations", 0)
sounding_count = Map.get(table_counts, "soundings", 0)
iemre_count = Map.get(table_counts, "iemre_observations", 0)
contact_count = max(Map.get(table_counts, "qsos", 0), 1)
%{
db_size: format_bytes(db_bytes),
@ -265,14 +261,11 @@ defmodule MicrowavepropWeb.BackfillLive do
defp status_class("unavailable"), do: "opacity-40"
defp status_class(_), do: ""
defp format_bytes(bytes) when bytes >= 1_073_741_824,
do: "#{Float.round(bytes / 1_073_741_824, 1)} GB"
defp format_bytes(bytes) when bytes >= 1_073_741_824, do: "#{Float.round(bytes / 1_073_741_824, 1)} GB"
defp format_bytes(bytes) when bytes >= 1_048_576,
do: "#{Float.round(bytes / 1_048_576, 1)} MB"
defp format_bytes(bytes) when bytes >= 1_048_576, do: "#{Float.round(bytes / 1_048_576, 1)} MB"
defp format_bytes(bytes) when bytes >= 1024,
do: "#{Float.round(bytes / 1024, 1)} KB"
defp format_bytes(bytes) when bytes >= 1024, do: "#{Float.round(bytes / 1024, 1)} KB"
defp format_bytes(bytes), do: "#{bytes} B"
@ -412,12 +405,16 @@ defmodule MicrowavepropWeb.BackfillLive do
</div>
<div class="bg-base-200 rounded-box p-3">
<div class="text-xs opacity-60">HRRR Profiles</div>
<div class="text-lg font-bold font-mono">{format_number(@db_stats.totals.hrrr_profiles)}</div>
<div class="text-lg font-bold font-mono">
{format_number(@db_stats.totals.hrrr_profiles)}
</div>
<div class="text-xs opacity-60">{@db_stats.averages.hrrr_per_contact} per contact</div>
</div>
<div class="bg-base-200 rounded-box p-3">
<div class="text-xs opacity-60">Surface Observations</div>
<div class="text-lg font-bold font-mono">{format_number(@db_stats.totals.surface_observations)}</div>
<div class="text-lg font-bold font-mono">
{format_number(@db_stats.totals.surface_observations)}
</div>
<div class="text-xs opacity-60">{@db_stats.averages.obs_per_contact} per contact</div>
</div>
<div class="bg-base-200 rounded-box p-3">