refactor(format): consolidate number/bytes formatters + handle flaky IEM transports
StatusLive had a hand-rolled thousands-separator format_number/1 and a
binary-prefix format_bytes/1 — duplication that was already part of the
review punch-list. Move both to Microwaveprop.Format (which previously
held only distance_km) and delete the local defps. Fourteen call sites
switched to Format.number / Format.bytes.
Also extend the IEM error classifier to snooze on Req.TransportError
{:closed, :timeout, :econnrefused, :nxdomain}. Same rationale as the
429 snooze in batch 1: upstream TCP resets mid-fetch aren't a job
failure, they're transient — returning {:error, _} blows them up into
Oban.PerformError logspam with full stacktraces.
This commit is contained in:
parent
e9a38623d8
commit
49d0594d28
3 changed files with 58 additions and 42 deletions
|
|
@ -27,4 +27,34 @@ defmodule Microwaveprop.Format do
|
|||
end
|
||||
|
||||
defp one_decimal(n), do: :erlang.float_to_binary(n * 1.0, decimals: 1)
|
||||
|
||||
@doc """
|
||||
Thousand-grouped integer for UI counters (row counts, job totals, etc).
|
||||
Floats round to nearest integer first; non-numerics are passed through
|
||||
to `to_string/1` so a `nil` or `:unknown` renders without crashing.
|
||||
"""
|
||||
@spec number(integer() | float() | any()) :: String.t()
|
||||
def number(n) when is_integer(n) do
|
||||
n
|
||||
|> Integer.to_string()
|
||||
|> String.graphemes()
|
||||
|> Enum.reverse()
|
||||
|> Enum.chunk_every(3)
|
||||
|> Enum.map_join(",", &Enum.join/1)
|
||||
|> String.reverse()
|
||||
|> String.replace(~r/^,/, "")
|
||||
end
|
||||
|
||||
def number(n) when is_float(n), do: number(round(n))
|
||||
def number(n), do: to_string(n)
|
||||
|
||||
@doc """
|
||||
Binary-prefix byte count (1024-based) with one decimal. Intended for
|
||||
admin / status surfaces where "1.4 MB" is more useful than "1468006".
|
||||
"""
|
||||
@spec bytes(non_neg_integer()) :: String.t()
|
||||
def bytes(b) when b >= 1_073_741_824, do: "#{Float.round(b / 1_073_741_824, 1)} GB"
|
||||
def bytes(b) when b >= 1_048_576, do: "#{Float.round(b / 1_048_576, 1)} MB"
|
||||
def bytes(b) when b >= 1024, do: "#{Float.round(b / 1024, 1)} KB"
|
||||
def bytes(b), do: "#{b} B"
|
||||
end
|
||||
|
|
|
|||
|
|
@ -84,12 +84,12 @@ defmodule Microwaveprop.Workers.WeatherFetchWorker do
|
|||
end
|
||||
end
|
||||
|
||||
# IEM rate-limits heavy ASOS pulls with HTTP 429. That's routine
|
||||
# during a backfill sweep, not a job failure — snooze the job
|
||||
# instead of surfacing Oban.PerformError, which spams the logger
|
||||
# with a full exception per 429 even though the retry is working
|
||||
# as intended. Other statuses (5xx, connection errors) stay on the
|
||||
# {:error, _} path so they count against max_attempts.
|
||||
# IEM upstream is routinely flaky — HTTP 429 rate-limits during a
|
||||
# backfill sweep, and transient connection resets (Req.TransportError
|
||||
# :closed / :timeout) on long-running fetches. Neither is a job
|
||||
# failure worth Oban.PerformError's stack-trace; snooze instead so
|
||||
# the retry is silent. Permanent errors (4xx ≠ 429, 5xx) fall through
|
||||
# to the warning + {:error, _} path so they count toward max_attempts.
|
||||
defp handle_iem_error(kind, station_code, "IEM " <> _ = reason) do
|
||||
if String.contains?(reason, "429") do
|
||||
Logger.debug("WeatherFetch #{kind}: #{station_code} rate-limited, snoozing")
|
||||
|
|
@ -100,8 +100,15 @@ defmodule Microwaveprop.Workers.WeatherFetchWorker do
|
|||
end
|
||||
end
|
||||
|
||||
defp handle_iem_error(kind, station_code, %Req.TransportError{reason: tr_reason})
|
||||
when tr_reason in [:closed, :timeout, :econnrefused, :nxdomain] do
|
||||
Logger.debug("WeatherFetch #{kind}: #{station_code} transport #{tr_reason}, snoozing")
|
||||
{:snooze, 60}
|
||||
end
|
||||
|
||||
defp handle_iem_error(kind, station_code, reason) do
|
||||
Logger.warning("WeatherFetch #{kind}: #{station_code} failed: #{inspect(reason, printable_limit: 200)}")
|
||||
|
||||
{:error, reason}
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ defmodule MicrowavepropWeb.StatusLive do
|
|||
import Ecto.Query
|
||||
|
||||
alias Microwaveprop.Cache
|
||||
alias Microwaveprop.Format
|
||||
alias Microwaveprop.Radio.Contact
|
||||
alias Microwaveprop.Repo
|
||||
alias Microwaveprop.Weather.NarrClient
|
||||
|
|
@ -352,7 +353,7 @@ defmodule MicrowavepropWeb.StatusLive do
|
|||
|
||||
tables =
|
||||
Enum.map(rows, fn [table, row_count, dead, bytes] ->
|
||||
%{table: table, rows: row_count, dead: dead, size: format_bytes(bytes)}
|
||||
%{table: table, rows: row_count, dead: dead, size: Format.bytes(bytes)}
|
||||
end)
|
||||
|
||||
%{rows: total_rows} =
|
||||
|
|
@ -409,7 +410,7 @@ defmodule MicrowavepropWeb.StatusLive do
|
|||
contact_count = max(Map.get(table_counts, "contacts", 0), 1)
|
||||
|
||||
%{
|
||||
db_size: format_bytes(db_bytes),
|
||||
db_size: Format.bytes(db_bytes),
|
||||
tables: tables,
|
||||
statuses: statuses,
|
||||
totals: %{
|
||||
|
|
@ -429,20 +430,6 @@ defmodule MicrowavepropWeb.StatusLive do
|
|||
}
|
||||
end
|
||||
|
||||
defp format_number(n) when is_integer(n) do
|
||||
n
|
||||
|> Integer.to_string()
|
||||
|> String.graphemes()
|
||||
|> Enum.reverse()
|
||||
|> Enum.chunk_every(3)
|
||||
|> Enum.map_join(",", &Enum.join/1)
|
||||
|> String.reverse()
|
||||
|> String.replace(~r/^,/, "")
|
||||
end
|
||||
|
||||
defp format_number(n) when is_float(n), do: format_number(round(n))
|
||||
defp format_number(n), do: to_string(n)
|
||||
|
||||
defp status_class("complete"), do: "text-success"
|
||||
defp status_class("pending"), do: "text-warning"
|
||||
defp status_class("queued"), do: "text-info"
|
||||
|
|
@ -451,14 +438,6 @@ defmodule MicrowavepropWeb.StatusLive do
|
|||
defp status_class("unavailable"), do: "opacity-40"
|
||||
defp status_class(_), do: ""
|
||||
|
||||
defp format_bytes(bytes) when bytes >= 1_073_741_824, do: "#{Float.round(bytes / 1_073_741_824, 1)} GB"
|
||||
|
||||
defp format_bytes(bytes) when bytes >= 1_048_576, do: "#{Float.round(bytes / 1_048_576, 1)} MB"
|
||||
|
||||
defp format_bytes(bytes) when bytes >= 1024, do: "#{Float.round(bytes / 1024, 1)} KB"
|
||||
|
||||
defp format_bytes(bytes), do: "#{bytes} B"
|
||||
|
||||
@impl true
|
||||
def render(assigns) do
|
||||
~H"""
|
||||
|
|
@ -472,7 +451,7 @@ defmodule MicrowavepropWeb.StatusLive do
|
|||
<div class="stat bg-base-200 rounded-box overflow-hidden md:col-span-2">
|
||||
<div class="stat-title">Enrichment Progress</div>
|
||||
<div class="stat-value text-2xl">
|
||||
{format_number(@unprocessed.complete)} / {format_number(@unprocessed.total)}
|
||||
{Format.number(@unprocessed.complete)} / {Format.number(@unprocessed.total)}
|
||||
</div>
|
||||
<div class="stat-desc">fully enriched</div>
|
||||
<div class="flex flex-col gap-1.5 mt-3">
|
||||
|
|
@ -493,7 +472,7 @@ defmodule MicrowavepropWeb.StatusLive do
|
|||
max={100}
|
||||
/>
|
||||
<span class="shrink-0 font-mono opacity-60 whitespace-nowrap">
|
||||
{format_number(complete)} / {format_number(denom)}
|
||||
{Format.number(complete)} / {Format.number(denom)}
|
||||
</span>
|
||||
</div>
|
||||
<% end %>
|
||||
|
|
@ -508,7 +487,7 @@ defmodule MicrowavepropWeb.StatusLive do
|
|||
</div>
|
||||
<div class="stat">
|
||||
<div class="stat-title">Completed (last hour)</div>
|
||||
<div class="stat-value text-2xl">{format_number(@stats.completed_1h)}</div>
|
||||
<div class="stat-value text-2xl">{Format.number(@stats.completed_1h)}</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -599,7 +578,7 @@ defmodule MicrowavepropWeb.StatusLive do
|
|||
0
|
||||
<% end %>
|
||||
</td>
|
||||
<td>{format_number(@grid_tasks.done_1h)}</td>
|
||||
<td>{Format.number(@grid_tasks.done_1h)}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="font-semibold">hrrr-point-rs</td>
|
||||
|
|
@ -614,7 +593,7 @@ defmodule MicrowavepropWeb.StatusLive do
|
|||
</span>
|
||||
<% end %>
|
||||
</td>
|
||||
<td>{format_number(@hrrr_point_tasks.queued)}</td>
|
||||
<td>{Format.number(@hrrr_point_tasks.queued)}</td>
|
||||
<td>
|
||||
<%= if @hrrr_point_tasks.failed > 0 do %>
|
||||
<span class="text-warning">{@hrrr_point_tasks.failed}</span>
|
||||
|
|
@ -622,7 +601,7 @@ defmodule MicrowavepropWeb.StatusLive do
|
|||
0
|
||||
<% end %>
|
||||
</td>
|
||||
<td>{format_number(@hrrr_point_tasks.done_1h)}</td>
|
||||
<td>{Format.number(@hrrr_point_tasks.done_1h)}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
|
@ -636,7 +615,7 @@ defmodule MicrowavepropWeb.StatusLive do
|
|||
<%= for {status, count} <- Map.get(@db_stats.statuses, type, []) do %>
|
||||
<div class="flex justify-between">
|
||||
<span class={status_class(status)}>{status}</span>
|
||||
<span class="font-mono">{format_number(count)}</span>
|
||||
<span class="font-mono">{Format.number(count)}</span>
|
||||
</div>
|
||||
<% end %>
|
||||
</div>
|
||||
|
|
@ -652,21 +631,21 @@ defmodule MicrowavepropWeb.StatusLive do
|
|||
<div class="stat bg-base-200 rounded-box overflow-hidden">
|
||||
<div class="stat-title">HRRR Profiles</div>
|
||||
<div class="stat-value text-xl font-mono">
|
||||
{format_number(@db_stats.totals.hrrr_profiles)}
|
||||
{Format.number(@db_stats.totals.hrrr_profiles)}
|
||||
</div>
|
||||
<div class="stat-desc">{@db_stats.averages.hrrr_per_contact} per contact</div>
|
||||
</div>
|
||||
<div class="stat bg-base-200 rounded-box overflow-hidden">
|
||||
<div class="stat-title">Surface Observations</div>
|
||||
<div class="stat-value text-xl font-mono">
|
||||
{format_number(@db_stats.totals.surface_observations)}
|
||||
{Format.number(@db_stats.totals.surface_observations)}
|
||||
</div>
|
||||
<div class="stat-desc">{@db_stats.averages.obs_per_contact} per contact</div>
|
||||
</div>
|
||||
<div class="stat bg-base-200 rounded-box overflow-hidden">
|
||||
<div class="stat-title">Soundings</div>
|
||||
<div class="stat-value text-xl font-mono">
|
||||
{format_number(@db_stats.totals.soundings)}
|
||||
{Format.number(@db_stats.totals.soundings)}
|
||||
</div>
|
||||
<div class="stat-desc">{@db_stats.averages.soundings_per_contact} per contact</div>
|
||||
</div>
|
||||
|
|
@ -687,12 +666,12 @@ defmodule MicrowavepropWeb.StatusLive do
|
|||
<%= for t <- @db_stats.tables do %>
|
||||
<tr>
|
||||
<td class="font-mono text-xs">{t.table}</td>
|
||||
<td class="text-right font-mono text-xs">{format_number(t.rows)}</td>
|
||||
<td class="text-right font-mono text-xs">{Format.number(t.rows)}</td>
|
||||
<td class={[
|
||||
"text-right font-mono text-xs",
|
||||
t.dead > 0 && "text-warning"
|
||||
]}>
|
||||
{format_number(t.dead)}
|
||||
{Format.number(t.dead)}
|
||||
</td>
|
||||
<td class="text-right font-mono text-xs">{t.size}</td>
|
||||
</tr>
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue