Era5SubmitWorker was counting era5_cds_jobs rows 1:1 against the CDS
150-job ceiling, but each row holds TWO CDS job IDs (single-level +
pressure-level). Prod hit 74 rows = 148 in-flight jobs and got stuck
in a reject→resubmit spiral. The cap guard now multiplies row count
by 2 to match what CDS sees.
Era5PollWorker @stuck_after_seconds goes from 4h to 18h. CDS
single-level routinely sits at 'accepted' for 12+h under load while
pressure finishes in ~90 min; the 4h threshold was firing on normal
slow runs and feeding the same spiral.
PipelineStatus now returns `label` + `detail` separately so the map
chip can render "Updating propagation" on one line and the
sub-detail ("HRRR run" / "ASOS nudge" / "+3h" / "now") on a second
line below it. running_label/1 is renamed to running_detail/1 and
returns just the sub-part.
161 lines
4.4 KiB
Elixir
161 lines
4.4 KiB
Elixir
defmodule Microwaveprop.Propagation.PipelineStatusTest do
|
|
use Microwaveprop.DataCase, async: false
|
|
|
|
alias Microwaveprop.Propagation.PipelineStatus
|
|
alias Microwaveprop.Repo
|
|
|
|
@grid_worker "Microwaveprop.Workers.PropagationGridWorker"
|
|
@asos_worker "Microwaveprop.Workers.AsosAdjustmentWorker"
|
|
|
|
# Bypass Oban's normal insert pipeline (which executes inline in test)
|
|
# by writing directly to the Oban.Job schema. The SQL sandbox keeps
|
|
# Oban's executor from seeing these rows.
|
|
defp insert_oban_job(overrides) do
|
|
now = DateTime.utc_now()
|
|
|
|
defaults = %{
|
|
state: "available",
|
|
queue: "propagation",
|
|
worker: @grid_worker,
|
|
args: %{},
|
|
attempt: 0,
|
|
max_attempts: 3,
|
|
inserted_at: now,
|
|
scheduled_at: now
|
|
}
|
|
|
|
attrs = Map.merge(defaults, Map.new(overrides))
|
|
Repo.insert!(struct!(Oban.Job, attrs))
|
|
end
|
|
|
|
defp minutes_ago(n) do
|
|
DateTime.add(DateTime.utc_now(), -n * 60, :second)
|
|
end
|
|
|
|
describe "current/0" do
|
|
test "returns :unknown when no pipeline jobs have ever run" do
|
|
status = PipelineStatus.current()
|
|
|
|
assert status.state == :unknown
|
|
assert status.last_update_at == nil
|
|
assert is_binary(status.label)
|
|
end
|
|
|
|
test "returns :running with grid worker detail when PropagationGridWorker is executing" do
|
|
insert_oban_job(%{
|
|
state: "executing",
|
|
worker: @grid_worker,
|
|
attempted_at: minutes_ago(5)
|
|
})
|
|
|
|
status = PipelineStatus.current()
|
|
|
|
assert status.state == :running
|
|
assert status.label == "Updating propagation"
|
|
assert status.detail =~ "HRRR"
|
|
end
|
|
|
|
test "returns :running with ASOS detail when AsosAdjustmentWorker is executing" do
|
|
insert_oban_job(%{
|
|
state: "executing",
|
|
worker: @asos_worker,
|
|
attempted_at: minutes_ago(1)
|
|
})
|
|
|
|
status = PipelineStatus.current()
|
|
|
|
assert status.state == :running
|
|
assert status.label == "Updating propagation"
|
|
assert status.detail =~ "ASOS"
|
|
end
|
|
|
|
test "returns :idle with Up to date label when last grid run completed recently" do
|
|
completed = minutes_ago(15)
|
|
|
|
insert_oban_job(%{
|
|
state: "completed",
|
|
worker: @grid_worker,
|
|
attempted_at: completed,
|
|
completed_at: completed
|
|
})
|
|
|
|
status = PipelineStatus.current()
|
|
|
|
assert status.state == :idle
|
|
assert status.last_update_at
|
|
assert status.label =~ "Up to date"
|
|
end
|
|
|
|
test "returns :stale when last completed was more than 120 minutes ago" do
|
|
stale = minutes_ago(180)
|
|
|
|
insert_oban_job(%{
|
|
state: "completed",
|
|
worker: @grid_worker,
|
|
attempted_at: stale,
|
|
completed_at: stale
|
|
})
|
|
|
|
status = PipelineStatus.current()
|
|
|
|
assert status.state == :stale
|
|
assert status.label =~ "stale"
|
|
end
|
|
|
|
test "prefers :running over :idle when a job is executing even if a recent job completed" do
|
|
recent = minutes_ago(5)
|
|
|
|
insert_oban_job(%{
|
|
state: "completed",
|
|
worker: @grid_worker,
|
|
attempted_at: recent,
|
|
completed_at: recent
|
|
})
|
|
|
|
insert_oban_job(%{
|
|
state: "executing",
|
|
worker: @grid_worker,
|
|
attempted_at: minutes_ago(1)
|
|
})
|
|
|
|
status = PipelineStatus.current()
|
|
|
|
assert status.state == :running
|
|
end
|
|
|
|
test "ignores completed jobs from unrelated workers" do
|
|
# WeatherFetchWorker enriches individual QSOs — it's not part of
|
|
# the CONUS grid update pipeline, so its recent success should
|
|
# not make the map page show 'Up to date'.
|
|
completed = minutes_ago(5)
|
|
|
|
insert_oban_job(%{
|
|
state: "completed",
|
|
worker: "Microwaveprop.Workers.WeatherFetchWorker",
|
|
queue: "weather",
|
|
attempted_at: completed,
|
|
completed_at: completed
|
|
})
|
|
|
|
status = PipelineStatus.current()
|
|
|
|
assert status.state == :unknown
|
|
end
|
|
end
|
|
|
|
describe "running_detail/1" do
|
|
test "forecast hour 0 renders as 'now'" do
|
|
assert PipelineStatus.running_detail(%{forecast_hour: 0}) == "now"
|
|
end
|
|
|
|
test "positive forecast hours render as +Nh" do
|
|
assert PipelineStatus.running_detail(%{forecast_hour: 1}) == "+1h"
|
|
assert PipelineStatus.running_detail(%{forecast_hour: 18}) == "+18h"
|
|
end
|
|
|
|
test "returns nil when no forecast hour is provided" do
|
|
assert PipelineStatus.running_detail(nil) == nil
|
|
assert PipelineStatus.running_detail(%{}) == nil
|
|
end
|
|
end
|
|
end
|