Graham McIntire
764643bc62
Changes under lib/ (source):
- iemre_fetch_worker: replace `if transient_failure?/1` boolean predicate
+ `if/else` with a classifier that pattern-matches the error reason
directly, returning `:transient | :permanent`, and dispatch via
`case`. Three-clause head is clearer than the boolean predicate.
- scores_file.extract_points and nexrad_client.extract_box: force `//1`
step on the row/col comprehensions so an image box whose centre
lands outside the raster collapses to an empty iteration instead
of firing Elixir 1.19's ambiguous-range warning. Adds a regression
test for the right-edge box.
Test coverage added:
- Feature wrappers in Backtest.Features (theta_e_jump, shear_at_top,
duct_thickness, best_duct_freq, duct_usable_for_band,
distance_to_front / parallel_to_front stubs, all_features/0).
- NotifyListener handle_info tolerance for malformed payloads +
unknown messages, plus the PubSub broadcast side effect.
- Viewshed.effective_reach_km across every verdict / diffraction
tier / ducting-score tier combination.
- SnmpClient parse_af60_output unit conversions + unknown-column
handling; parse_snmpget_output OID normalisation and junk-line
tolerance.
- HrrrNativeClient native_level_count, native_variables,
native_messages shape, duct_messages / duct_byte_ranges.
- Changeset coverage for GeomagneticObservation, SolarFluxObservation,
SolarXrayObservation, Ionosphere.Observation, HrrrClimatology, and
Metar5minObservation — lifted each from ~50% / 0% to 100%.
- Property tests: GefsClient.dewpoint_from_rh invariants (Td ≤ T,
monotonic in RH, finite across the operating envelope) + idempotent
nearest_run. NexradClient.pixel_to_dbz monotonicity + bounded
output; latlon_to_pixel round-trip for every grid cell.
Also cleared several unrelated compiler/linter warnings:
- Three private test helpers (create_contact, insert_contact,
current_hour) had `\\ %{}` / `\\ 0` defaults that no caller used.
- profiles_file_test bound `dir` in a pattern match without using it.
Suite: 2,359 tests + 146 properties pass (was 2,300 / 139); coverage
70.38% → 70.89%; credo strict clean.
91 lines
3 KiB
Elixir
91 lines
3 KiB
Elixir
defmodule Microwaveprop.Workers.IemreFetchWorker do
|
|
@moduledoc false
|
|
use Oban.Worker,
|
|
queue: :iemre,
|
|
max_attempts: 20,
|
|
unique: [period: 300, fields: [:args], states: [:scheduled, :available]]
|
|
|
|
alias Microwaveprop.Weather
|
|
alias Microwaveprop.Weather.IemClient
|
|
|
|
require Logger
|
|
|
|
@impl Oban.Worker
|
|
def backoff(%Oban.Job{attempt: attempt}) do
|
|
min(120 * Integer.pow(2, attempt - 1), _six_hours = 21_600)
|
|
end
|
|
|
|
@impl Oban.Worker
|
|
def perform(%Oban.Job{args: args}) do
|
|
%{"lat" => lat, "lon" => lon, "date" => date_str} = args
|
|
date = Date.from_iso8601!(date_str)
|
|
|
|
if Weather.has_iemre_observation?(lat, lon, date) do
|
|
Logger.info("IEMRE observation already exists for #{lat},#{lon} @ #{date_str}")
|
|
:ok
|
|
else
|
|
Logger.info("Fetching IEMRE data for #{lat},#{lon} @ #{date_str}")
|
|
fetch_and_store_iemre(lat, lon, date, date_str)
|
|
end
|
|
end
|
|
|
|
defp fetch_and_store_iemre(lat, lon, date, date_str) do
|
|
case IemClient.fetch_iemre(lat, lon, date) do
|
|
{:ok, []} ->
|
|
# Store stub so this lat/lon/date isn't retried on future backfills
|
|
_ = Weather.upsert_iemre_observation(%{lat: lat, lon: lon, date: date, hourly: []})
|
|
Logger.info("IEMRE: no data available for #{lat},#{lon} @ #{date_str}, stored stub")
|
|
:ok
|
|
|
|
{:ok, data} ->
|
|
_ =
|
|
Weather.upsert_iemre_observation(%{
|
|
lat: lat,
|
|
lon: lon,
|
|
date: date,
|
|
hourly: data
|
|
})
|
|
|
|
Logger.info("IEMRE observation saved for #{lat},#{lon} @ #{date_str} (#{length(data)} hours)")
|
|
:ok
|
|
|
|
{:error, reason} ->
|
|
handle_error(reason, lat, lon, date, date_str)
|
|
end
|
|
end
|
|
|
|
defp handle_error(reason, lat, lon, date, date_str) do
|
|
case classify_failure(reason) do
|
|
:transient ->
|
|
Logger.error("IEMRE transient error for #{lat},#{lon} @ #{date_str}: #{inspect(reason)}")
|
|
{:error, reason}
|
|
|
|
:permanent ->
|
|
# Permanent upstream failure (e.g. 404, 422 out-of-grid). Stub
|
|
# the bucket so future backfill enqueues skip the fetch and
|
|
# let ContactWeatherEnqueueWorker reconcile the contacts'
|
|
# :queued → :complete transition via the empty-jobs branch.
|
|
_ = Weather.upsert_iemre_observation(%{lat: lat, lon: lon, date: date, hourly: []})
|
|
|
|
Logger.warning("IEMRE permanent failure for #{lat},#{lon} @ #{date_str}: #{inspect(reason)}, stored stub")
|
|
|
|
{:cancel, reason}
|
|
end
|
|
end
|
|
|
|
# Network/transport exceptions are always transient; retry.
|
|
defp classify_failure(%{__exception__: true}), do: :transient
|
|
|
|
# IEM surfaces the HTTP status in the reason string. Split the integer
|
|
# back out and pattern-match rate-limit / 5xx codes as transient.
|
|
defp classify_failure("IEM IEMRE HTTP " <> status) do
|
|
case Integer.parse(status) do
|
|
{code, _} when code in [429, 500, 502, 503, 504] -> :transient
|
|
_ -> :permanent
|
|
end
|
|
end
|
|
|
|
# Anything else (404, 422, unexpected shapes) is permanent — cancel
|
|
# the job and stub the bucket.
|
|
defp classify_failure(_), do: :permanent
|
|
end
|