refactor(format): consolidate number/bytes formatters + handle flaky IEM transports

StatusLive had a hand-rolled thousands-separator format_number/1 and a
binary-prefix format_bytes/1 — duplication that was already part of the
review punch-list. Move both to Microwaveprop.Format (which previously
held only distance_km) and delete the local defps. Fourteen call sites
switched to Format.number / Format.bytes.

Also extend the IEM error classifier to snooze on Req.TransportError
{:closed, :timeout, :econnrefused, :nxdomain}. Same rationale as the
429 snooze in batch 1: upstream TCP resets mid-fetch aren't a job
failure, they're transient — returning {:error, _} blows them up into
Oban.PerformError logspam with full stacktraces.

lib/microwaveprop/format.ex

@@ -27,4 +27,34 @@ defmodule Microwaveprop.Format do
   end
 
   defp one_decimal(n), do: :erlang.float_to_binary(n * 1.0, decimals: 1)
+
+  @doc """
+  Thousand-grouped integer for UI counters (row counts, job totals, etc).
+  Floats round to nearest integer first; non-numerics are passed through
+  to `to_string/1` so a `nil` or `:unknown` renders without crashing.
+  """
+  @spec number(integer() | float() | any()) :: String.t()
+  def number(n) when is_integer(n) do
+    n
+    |> Integer.to_string()
+    |> String.graphemes()
+    |> Enum.reverse()
+    |> Enum.chunk_every(3)
+    |> Enum.map_join(",", &Enum.join/1)
+    |> String.reverse()
+    |> String.replace(~r/^,/, "")
+  end
+
+  def number(n) when is_float(n), do: number(round(n))
+  def number(n), do: to_string(n)
+
+  @doc """
+  Binary-prefix byte count (1024-based) with one decimal. Intended for
+  admin / status surfaces where "1.4 MB" is more useful than "1468006".
+  """
+  @spec bytes(non_neg_integer()) :: String.t()
+  def bytes(b) when b >= 1_073_741_824, do: "#{Float.round(b / 1_073_741_824, 1)} GB"
+  def bytes(b) when b >= 1_048_576, do: "#{Float.round(b / 1_048_576, 1)} MB"
+  def bytes(b) when b >= 1024, do: "#{Float.round(b / 1024, 1)} KB"
+  def bytes(b), do: "#{b} B"
 end

lib/microwaveprop/workers/weather_fetch_worker.ex

@@ -84,12 +84,12 @@ defmodule Microwaveprop.Workers.WeatherFetchWorker do
     end
   end
 
-  # IEM rate-limits heavy ASOS pulls with HTTP 429. That's routine
+  # IEM upstream is routinely flaky — HTTP 429 rate-limits during a
-  # during a backfill sweep, not a job failure — snooze the job
+  # backfill sweep, and transient connection resets (Req.TransportError
-  # instead of surfacing Oban.PerformError, which spams the logger
+  # :closed / :timeout) on long-running fetches. Neither is a job
-  # with a full exception per 429 even though the retry is working
+  # failure worth Oban.PerformError's stack-trace; snooze instead so
-  # as intended. Other statuses (5xx, connection errors) stay on the
+  # the retry is silent. Permanent errors (4xx ≠ 429, 5xx) fall through
-  # {:error, _} path so they count against max_attempts.
+  # to the warning + {:error, _} path so they count toward max_attempts.
   defp handle_iem_error(kind, station_code, "IEM " <> _ = reason) do
     if String.contains?(reason, "429") do
       Logger.debug("WeatherFetch #{kind}: #{station_code} rate-limited, snoozing")
@@ -100,8 +100,15 @@ defmodule Microwaveprop.Workers.WeatherFetchWorker do
     end
   end
 
+  defp handle_iem_error(kind, station_code, %Req.TransportError{reason: tr_reason})
+       when tr_reason in [:closed, :timeout, :econnrefused, :nxdomain] do
+    Logger.debug("WeatherFetch #{kind}: #{station_code} transport #{tr_reason}, snoozing")
+    {:snooze, 60}
+  end
+
   defp handle_iem_error(kind, station_code, reason) do
     Logger.warning("WeatherFetch #{kind}: #{station_code} failed: #{inspect(reason, printable_limit: 200)}")
+
     {:error, reason}
   end
 

lib/microwaveprop_web/live/status_live.ex

@@ -5,6 +5,7 @@ defmodule MicrowavepropWeb.StatusLive do
   import Ecto.Query
 
   alias Microwaveprop.Cache
+  alias Microwaveprop.Format
   alias Microwaveprop.Radio.Contact
   alias Microwaveprop.Repo
   alias Microwaveprop.Weather.NarrClient
@@ -352,7 +353,7 @@ defmodule MicrowavepropWeb.StatusLive do
 
     tables =
       Enum.map(rows, fn [table, row_count, dead, bytes] ->
-        %{table: table, rows: row_count, dead: dead, size: format_bytes(bytes)}
+        %{table: table, rows: row_count, dead: dead, size: Format.bytes(bytes)}
       end)
 
     %{rows: total_rows} =
@@ -409,7 +410,7 @@ defmodule MicrowavepropWeb.StatusLive do
     contact_count = max(Map.get(table_counts, "contacts", 0), 1)
 
     %{
-      db_size: format_bytes(db_bytes),
+      db_size: Format.bytes(db_bytes),
       tables: tables,
       statuses: statuses,
       totals: %{
@@ -429,20 +430,6 @@ defmodule MicrowavepropWeb.StatusLive do
     }
   end
 
-  defp format_number(n) when is_integer(n) do
-    n
-    |> Integer.to_string()
-    |> String.graphemes()
-    |> Enum.reverse()
-    |> Enum.chunk_every(3)
-    |> Enum.map_join(",", &Enum.join/1)
-    |> String.reverse()
-    |> String.replace(~r/^,/, "")
-  end
-
-  defp format_number(n) when is_float(n), do: format_number(round(n))
-  defp format_number(n), do: to_string(n)
-
   defp status_class("complete"), do: "text-success"
   defp status_class("pending"), do: "text-warning"
   defp status_class("queued"), do: "text-info"
@@ -451,14 +438,6 @@ defmodule MicrowavepropWeb.StatusLive do
   defp status_class("unavailable"), do: "opacity-40"
   defp status_class(_), do: ""
 
-  defp format_bytes(bytes) when bytes >= 1_073_741_824, do: "#{Float.round(bytes / 1_073_741_824, 1)} GB"
-
-  defp format_bytes(bytes) when bytes >= 1_048_576, do: "#{Float.round(bytes / 1_048_576, 1)} MB"
-
-  defp format_bytes(bytes) when bytes >= 1024, do: "#{Float.round(bytes / 1024, 1)} KB"
-
-  defp format_bytes(bytes), do: "#{bytes} B"
-
   @impl true
   def render(assigns) do
     ~H"""
@@ -472,7 +451,7 @@ defmodule MicrowavepropWeb.StatusLive do
         <div class="stat bg-base-200 rounded-box overflow-hidden md:col-span-2">
           <div class="stat-title">Enrichment Progress</div>
           <div class="stat-value text-2xl">
-            {format_number(@unprocessed.complete)} / {format_number(@unprocessed.total)}
+            {Format.number(@unprocessed.complete)} / {Format.number(@unprocessed.total)}
           </div>
           <div class="stat-desc">fully enriched</div>
           <div class="flex flex-col gap-1.5 mt-3">
@@ -493,7 +472,7 @@ defmodule MicrowavepropWeb.StatusLive do
                   max={100}
                 />
                 <span class="shrink-0 font-mono opacity-60 whitespace-nowrap">
-                  {format_number(complete)} / {format_number(denom)}
+                  {Format.number(complete)} / {Format.number(denom)}
                 </span>
               </div>
             <% end %>
@@ -508,7 +487,7 @@ defmodule MicrowavepropWeb.StatusLive do
           </div>
           <div class="stat">
             <div class="stat-title">Completed (last hour)</div>
-            <div class="stat-value text-2xl">{format_number(@stats.completed_1h)}</div>
+            <div class="stat-value text-2xl">{Format.number(@stats.completed_1h)}</div>
           </div>
         </div>
       </div>
@@ -599,7 +578,7 @@ defmodule MicrowavepropWeb.StatusLive do
                   0
                 <% end %>
               </td>
-              <td>{format_number(@grid_tasks.done_1h)}</td>
+              <td>{Format.number(@grid_tasks.done_1h)}</td>
             </tr>
             <tr>
               <td class="font-semibold">hrrr-point-rs</td>
@@ -614,7 +593,7 @@ defmodule MicrowavepropWeb.StatusLive do
                   </span>
                 <% end %>
               </td>
-              <td>{format_number(@hrrr_point_tasks.queued)}</td>
+              <td>{Format.number(@hrrr_point_tasks.queued)}</td>
               <td>
                 <%= if @hrrr_point_tasks.failed > 0 do %>
                   <span class="text-warning">{@hrrr_point_tasks.failed}</span>
@@ -622,7 +601,7 @@ defmodule MicrowavepropWeb.StatusLive do
                   0
                 <% end %>
               </td>
-              <td>{format_number(@hrrr_point_tasks.done_1h)}</td>
+              <td>{Format.number(@hrrr_point_tasks.done_1h)}</td>
             </tr>
           </tbody>
         </table>
@@ -636,7 +615,7 @@ defmodule MicrowavepropWeb.StatusLive do
             <%= for {status, count} <- Map.get(@db_stats.statuses, type, []) do %>
               <div class="flex justify-between">
                 <span class={status_class(status)}>{status}</span>
-                <span class="font-mono">{format_number(count)}</span>
+                <span class="font-mono">{Format.number(count)}</span>
               </div>
             <% end %>
           </div>
@@ -652,21 +631,21 @@ defmodule MicrowavepropWeb.StatusLive do
         <div class="stat bg-base-200 rounded-box overflow-hidden">
           <div class="stat-title">HRRR Profiles</div>
           <div class="stat-value text-xl font-mono">
-            {format_number(@db_stats.totals.hrrr_profiles)}
+            {Format.number(@db_stats.totals.hrrr_profiles)}
           </div>
           <div class="stat-desc">{@db_stats.averages.hrrr_per_contact} per contact</div>
         </div>
         <div class="stat bg-base-200 rounded-box overflow-hidden">
           <div class="stat-title">Surface Observations</div>
           <div class="stat-value text-xl font-mono">
-            {format_number(@db_stats.totals.surface_observations)}
+            {Format.number(@db_stats.totals.surface_observations)}
           </div>
           <div class="stat-desc">{@db_stats.averages.obs_per_contact} per contact</div>
         </div>
         <div class="stat bg-base-200 rounded-box overflow-hidden">
           <div class="stat-title">Soundings</div>
           <div class="stat-value text-xl font-mono">
-            {format_number(@db_stats.totals.soundings)}
+            {Format.number(@db_stats.totals.soundings)}
           </div>
           <div class="stat-desc">{@db_stats.averages.soundings_per_contact} per contact</div>
         </div>
@@ -687,12 +666,12 @@ defmodule MicrowavepropWeb.StatusLive do
             <%= for t <- @db_stats.tables do %>
               <tr>
                 <td class="font-mono text-xs">{t.table}</td>
-                <td class="text-right font-mono text-xs">{format_number(t.rows)}</td>
+                <td class="text-right font-mono text-xs">{Format.number(t.rows)}</td>
                 <td class={[
                   "text-right font-mono text-xs",
                   t.dead > 0 && "text-warning"
                 ]}>
-                  {format_number(t.dead)}
+                  {Format.number(t.dead)}
                 </td>
                 <td class="text-right font-mono text-xs">{t.size}</td>
               </tr>