graham/prop
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

perf(status): collapse 7 contact-count queries to a single FILTER scan

Browse source 
count_unprocessed_raw fired seven separate COUNT(*) scans on the
contacts table every 2 s refresh (total, terrain, hrrr, weather, iemre,
radar, all-done). Each one went through the 58k+ row table
independently, so the status page was hitting Postgres with ~3.5 full
scans per second of connected viewer time.

A single query with COUNT(*) FILTER clauses pulls every count in one
sequential scan. Same result, roughly 1/7 the CPU on Postgres.
This commit is contained in:
Graham McIntire 2026-04-20 16:56:14 -05:00
parent ff241afa6e
commit 30525a18f3
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
1 changed files with 25 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
lib/microwaveprop_web/live/status_live.ex
View file

@ -83,21 +83,35 @@ defmodule MicrowavepropWeb.StatusLive do
end end
defp count_unprocessed_raw do defp count_unprocessed_raw do
incomplete = [:pending, :queued, :processing, :failed] # Previously issued 7 separate `SELECT COUNT(*) FROM contacts WHERE
done = [:complete, :unavailable] # <field> IN (...)` queries. Postgres ran 7 index scans over the
# same 58k+ row table on every 2s status refresh; collapse into a
total = Repo.one(from(c in Contact, where: not is_nil(c.pos1), select: count())) # single scan that returns every needed count via FILTER clauses.
terrain = count_incomplete(:terrain_status, incomplete) %{rows: [[total, terrain, hrrr, weather, iemre, radar, all_done]]} =
hrrr = count_incomplete(:hrrr_status, incomplete) Repo.query!("""
weather = count_incomplete(:weather_status, incomplete) SELECT
iemre = count_incomplete(:iemre_status, incomplete) COUNT(*) FILTER (WHERE pos1 IS NOT NULL),
radar = count_incomplete(:radar_status, incomplete) COUNT(*) FILTER (WHERE pos1 IS NOT NULL
AND terrain_status::text IN ('pending','queued','processing','failed')),
COUNT(*) FILTER (WHERE pos1 IS NOT NULL
AND hrrr_status::text IN ('pending','queued','processing','failed')),
COUNT(*) FILTER (WHERE pos1 IS NOT NULL
AND weather_status::text IN ('pending','queued','processing','failed')),
COUNT(*) FILTER (WHERE pos1 IS NOT NULL
AND iemre_status::text IN ('pending','queued','processing','failed')),
COUNT(*) FILTER (WHERE pos1 IS NOT NULL
AND radar_status::text IN ('pending','queued','processing','failed')),
COUNT(*) FILTER (WHERE pos1 IS NOT NULL
AND hrrr_status::text IN ('complete','unavailable')
AND weather_status::text IN ('complete','unavailable')
AND terrain_status::text IN ('complete','unavailable')
AND iemre_status::text IN ('complete','unavailable'))
FROM contacts
""")
narr_candidates = count_narr_candidates() narr_candidates = count_narr_candidates()
narr_done = count_narr_done() narr_done = count_narr_done()
all_done = count_all_done(done)
%{ %{
terrain: terrain, terrain: terrain,
hrrr: hrrr, hrrr: hrrr,
@ -159,29 +173,6 @@ defmodule MicrowavepropWeb.StatusLive do
done done
end end
defp count_incomplete(status_field, statuses) do
Repo.one(
from(c in Contact,
where: field(c, ^status_field) in ^statuses and not is_nil(c.pos1),
select: count()
)
)
end
defp count_all_done(done) do
Repo.one(
from(c in Contact,
where:
not is_nil(c.pos1) and
c.hrrr_status in ^done and
c.weather_status in ^done and
c.terrain_status in ^done and
c.iemre_status in ^done,
select: count()
)
)
end
defp fetch_grid_tasks_stats do defp fetch_grid_tasks_stats do
Cache.fetch_or_store({__MODULE__, :grid_tasks_stats}, 2_000, fn -> fetch_grid_tasks_stats_raw() end) Cache.fetch_or_store({__MODULE__, :grid_tasks_stats}, 2_000, fn -> fetch_grid_tasks_stats_raw() end)
end end