Graham McIntire
47ba8d3b6d
IemreFetchWorker now writes an empty stub observation when IEM returns
a permanent-failure status (404, 422), mirroring the existing {:ok, []}
path. The backfill cron's next tick sees the stub via
has_iemre_observation?/3, generates no job, and mark_status!/3 flips
the contact's iemre_status from :queued to :complete — draining the
51 contacts that have been stuck behind cancelled out-of-grid IEMRE
jobs.
Raise hot pod CPU limit 2 → 3 so BEAM gets 3 schedulers online.
Observed run queues of [2, 10, 0, 0, 0, 0] on the 2-scheduler
configuration during the :05 propagation chain, starving /live and
/health probe handlers and tripping intermittent liveness timeouts.
153 lines
4.2 KiB
Elixir
153 lines
4.2 KiB
Elixir
defmodule Microwaveprop.Workers.IemreFetchWorkerTest do
|
|
use Microwaveprop.DataCase, async: true
|
|
|
|
alias Microwaveprop.Weather
|
|
alias Microwaveprop.Weather.IemClient
|
|
alias Microwaveprop.Weather.IemreObservation
|
|
alias Microwaveprop.Workers.IemreFetchWorker
|
|
|
|
@sample_iemre_data [
|
|
%{"utc_hour" => 0, "p01m_mm" => 0.0, "skyc_pct" => 25.0},
|
|
%{"utc_hour" => 1, "p01m_mm" => 0.5, "skyc_pct" => 50.0}
|
|
]
|
|
|
|
describe "perform/1" do
|
|
test "skips if IEMRE observation already exists" do
|
|
Weather.upsert_iemre_observation(%{
|
|
lat: 32.875,
|
|
lon: -97.0,
|
|
date: ~D[2026-03-28],
|
|
hourly: @sample_iemre_data
|
|
})
|
|
|
|
# Stub should NOT be called
|
|
Req.Test.stub(IemClient, fn conn ->
|
|
Plug.Conn.send_resp(conn, 500, "Should not be called")
|
|
end)
|
|
|
|
job = %Oban.Job{
|
|
args: %{
|
|
"lat" => 32.875,
|
|
"lon" => -97.0,
|
|
"date" => "2026-03-28"
|
|
}
|
|
}
|
|
|
|
assert :ok = IemreFetchWorker.perform(job)
|
|
assert Repo.aggregate(IemreObservation, :count) == 1
|
|
end
|
|
|
|
test "fetches and stores IEMRE data on success" do
|
|
Req.Test.stub(IemClient, fn conn ->
|
|
Req.Test.json(conn, %{"data" => @sample_iemre_data})
|
|
end)
|
|
|
|
job = %Oban.Job{
|
|
args: %{
|
|
"lat" => 32.875,
|
|
"lon" => -97.0,
|
|
"date" => "2026-03-28"
|
|
}
|
|
}
|
|
|
|
assert :ok = IemreFetchWorker.perform(job)
|
|
assert Repo.aggregate(IemreObservation, :count) == 1
|
|
|
|
obs = Repo.one(IemreObservation)
|
|
assert obs.lat == 32.875
|
|
assert obs.lon == -97.0
|
|
assert obs.date == ~D[2026-03-28]
|
|
assert length(obs.hourly) == 2
|
|
end
|
|
|
|
test "stores stub on empty data so backfill doesn't retry" do
|
|
Req.Test.stub(IemClient, fn conn ->
|
|
Req.Test.json(conn, %{"data" => []})
|
|
end)
|
|
|
|
job = %Oban.Job{
|
|
args: %{
|
|
"lat" => 32.875,
|
|
"lon" => -97.0,
|
|
"date" => "2026-03-28"
|
|
}
|
|
}
|
|
|
|
assert :ok = IemreFetchWorker.perform(job)
|
|
assert Repo.aggregate(IemreObservation, :count) == 1
|
|
|
|
obs = Repo.one!(IemreObservation)
|
|
assert obs.hourly == []
|
|
assert obs.lat == 32.875
|
|
assert obs.lon == -97.0
|
|
end
|
|
|
|
test "retries on transient server error" do
|
|
Req.Test.stub(IemClient, fn conn ->
|
|
Plug.Conn.send_resp(conn, 503, "Service Unavailable")
|
|
end)
|
|
|
|
job = %Oban.Job{
|
|
args: %{
|
|
"lat" => 32.875,
|
|
"lon" => -97.0,
|
|
"date" => "2026-03-28"
|
|
}
|
|
}
|
|
|
|
assert {:error, "IEM IEMRE HTTP 503"} = IemreFetchWorker.perform(job)
|
|
end
|
|
|
|
test "cancels on permanent failure (404) and stores stub so backfill reconciles" do
|
|
Req.Test.stub(IemClient, fn conn ->
|
|
Plug.Conn.send_resp(conn, 404, "not found")
|
|
end)
|
|
|
|
job = %Oban.Job{
|
|
args: %{
|
|
"lat" => 32.875,
|
|
"lon" => -97.0,
|
|
"date" => "2026-03-28"
|
|
}
|
|
}
|
|
|
|
assert {:cancel, "IEM IEMRE HTTP 404"} = IemreFetchWorker.perform(job)
|
|
assert Repo.aggregate(IemreObservation, :count) == 1
|
|
obs = Repo.one!(IemreObservation)
|
|
assert obs.hourly == []
|
|
assert obs.lat == 32.875
|
|
assert obs.lon == -97.0
|
|
end
|
|
|
|
test "cancels on permanent failure (422 out-of-grid) and stores stub" do
|
|
Req.Test.stub(IemClient, fn conn ->
|
|
Plug.Conn.send_resp(conn, 422, "out of grid")
|
|
end)
|
|
|
|
job = %Oban.Job{
|
|
args: %{
|
|
"lat" => 60.75,
|
|
"lon" => -151.25,
|
|
"date" => "2019-08-03"
|
|
}
|
|
}
|
|
|
|
assert {:cancel, "IEM IEMRE HTTP 422"} = IemreFetchWorker.perform(job)
|
|
assert Repo.aggregate(IemreObservation, :count) == 1
|
|
obs = Repo.one!(IemreObservation)
|
|
assert obs.hourly == []
|
|
assert obs.lat == 60.75
|
|
assert obs.lon == -151.25
|
|
assert obs.date == ~D[2019-08-03]
|
|
end
|
|
end
|
|
|
|
describe "backoff/1" do
|
|
test "uses exponential backoff capped at 6 hours" do
|
|
assert IemreFetchWorker.backoff(%Oban.Job{attempt: 1}) == 120
|
|
assert IemreFetchWorker.backoff(%Oban.Job{attempt: 2}) == 240
|
|
assert IemreFetchWorker.backoff(%Oban.Job{attempt: 10}) == 21_600
|
|
assert IemreFetchWorker.backoff(%Oban.Job{attempt: 20}) == 21_600
|
|
end
|
|
end
|
|
end
|