fix: stabilize flaky tests + silence PromEx Oban poller in test
- HrrrClient idx-cache test only invalidated the surface idx URL, but fetch_profile also fetches a pressure idx. Previous runs' state for the pressure key decided whether the counter landed at 1 (prior run cached it, pass) or 2 (cold, fail). Invalidate both + assert 2 total fetches to reflect the actual code path. - CsvImportTest deadlocked against other async DataCase tests when inline Oban child jobs upserted iemre_observations/terrain_profiles with a shared conflict target. Flip to async: false — same fix as ContactWeatherEnqueueWorkerTest earlier this session. - PromEx.Plugins.Oban runs a 5s telemetry_poller that queries the DB, but its poller PID has no sandbox connection in test and crashed with DBConnection.OwnershipError on every tick, spamming the log. Gate the plugin on a config flag and skip it in config/test.exs; prod behaviour unchanged.
This commit is contained in:
parent
7a7b30f7bf
commit
71e8f53142
4 changed files with 33 additions and 6 deletions
|
|
@ -48,6 +48,14 @@ config :microwaveprop, MicrowavepropWeb.Endpoint,
|
||||||
# Run Oban jobs inline during tests
|
# Run Oban jobs inline during tests
|
||||||
config :microwaveprop, Oban, testing: :inline
|
config :microwaveprop, Oban, testing: :inline
|
||||||
|
|
||||||
|
# MetricsLogSuppressionTest hits GET /metrics and asserts no "Sent 200"
|
||||||
|
# log line; that path needs PromEx alive to return a 200. We can't
|
||||||
|
# blanket-disable PromEx. Instead, toggle off the Oban-queue poller
|
||||||
|
# below — its 5 s telemetry measurement issues an Ecto query from a
|
||||||
|
# process that doesn't hold a sandbox connection and crashes with
|
||||||
|
# DBConnection.OwnershipError, spamming the test log.
|
||||||
|
config :microwaveprop, :prom_ex_include_oban_plugin, false
|
||||||
|
|
||||||
# Score files land under a per-run tmp directory so replace_scores tests
|
# Score files land under a per-run tmp directory so replace_scores tests
|
||||||
# don't write into a shared location (and we can assert on the file
|
# don't write into a shared location (and we can assert on the file
|
||||||
# contents from inside tests).
|
# contents from inside tests).
|
||||||
|
|
|
||||||
|
|
@ -14,6 +14,15 @@ defmodule Microwaveprop.PromEx do
|
||||||
|
|
||||||
@impl true
|
@impl true
|
||||||
def plugins do
|
def plugins do
|
||||||
|
# Oban plugin's queue-depth metric is a telemetry_poller that runs
|
||||||
|
# an Ecto query every 5 s; in the test env that poller has no
|
||||||
|
# sandbox connection and crashes with DBConnection.OwnershipError,
|
||||||
|
# so it's skipped there.
|
||||||
|
oban_plugin =
|
||||||
|
if Application.get_env(:microwaveprop, :prom_ex_include_oban_plugin, true),
|
||||||
|
do: [Plugins.Oban],
|
||||||
|
else: []
|
||||||
|
|
||||||
[
|
[
|
||||||
# Defaults: VM memory, process counts, scheduler util, app version.
|
# Defaults: VM memory, process counts, scheduler util, app version.
|
||||||
Plugins.Application,
|
Plugins.Application,
|
||||||
|
|
@ -25,13 +34,10 @@ defmodule Microwaveprop.PromEx do
|
||||||
# Ecto: query duration, queue time, pool size.
|
# Ecto: query duration, queue time, pool size.
|
||||||
{Plugins.Ecto, otp_app: :microwaveprop, repos: [Microwaveprop.Repo]},
|
{Plugins.Ecto, otp_app: :microwaveprop, repos: [Microwaveprop.Repo]},
|
||||||
|
|
||||||
# Oban: job duration, queue time, state counts, queue depth.
|
|
||||||
Plugins.Oban,
|
|
||||||
|
|
||||||
# Our own spans: Instrument.span/3 emits [:microwaveprop | suffix]
|
# Our own spans: Instrument.span/3 emits [:microwaveprop | suffix]
|
||||||
# events; this plugin converts them to Prometheus histograms.
|
# events; this plugin converts them to Prometheus histograms.
|
||||||
Microwaveprop.PromEx.InstrumentPlugin
|
Microwaveprop.PromEx.InstrumentPlugin
|
||||||
]
|
] ++ oban_plugin
|
||||||
end
|
end
|
||||||
|
|
||||||
@impl true
|
@impl true
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,9 @@
|
||||||
defmodule Microwaveprop.Radio.CsvImportTest do
|
defmodule Microwaveprop.Radio.CsvImportTest do
|
||||||
use Microwaveprop.DataCase, async: true
|
# async: false — CsvImport.commit/1 inserts contacts that trigger inline
|
||||||
|
# Oban enrichment jobs (IEMRE, weather, HRRR, terrain) which upsert rows
|
||||||
|
# with a shared conflict target. Running concurrently with sibling
|
||||||
|
# DataCase tests produced intermittent "ShareLock deadlock_detected".
|
||||||
|
use Microwaveprop.DataCase, async: false
|
||||||
use Oban.Testing, repo: Microwaveprop.Repo
|
use Oban.Testing, repo: Microwaveprop.Repo
|
||||||
|
|
||||||
alias Microwaveprop.Radio.Contact
|
alias Microwaveprop.Radio.Contact
|
||||||
|
|
|
||||||
|
|
@ -189,6 +189,13 @@ defmodule Microwaveprop.Weather.HrrrClientTest do
|
||||||
{:hrrr_idx, "https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20260328/conus/hrrr.t18z.wrfsfcf00.grib2.idx"}
|
{:hrrr_idx, "https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20260328/conus/hrrr.t18z.wrfsfcf00.grib2.idx"}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# `do_fetch_profile` fetches TWO idx URLs (surface + pressure). Also drop
|
||||||
|
# the pressure cache so the test starts from a deterministic state —
|
||||||
|
# without this, whether this assertion passes depends on seed ordering.
|
||||||
|
Microwaveprop.Cache.invalidate(
|
||||||
|
{:hrrr_idx, "https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20260328/conus/hrrr.t18z.wrfprsf00.grib2.idx"}
|
||||||
|
)
|
||||||
|
|
||||||
{:ok, counter} = Agent.start_link(fn -> %{idx: 0, grib: 0} end)
|
{:ok, counter} = Agent.start_link(fn -> %{idx: 0, grib: 0} end)
|
||||||
|
|
||||||
Req.Test.stub(HrrrClient, fn conn ->
|
Req.Test.stub(HrrrClient, fn conn ->
|
||||||
|
|
@ -214,7 +221,9 @@ defmodule Microwaveprop.Weather.HrrrClientTest do
|
||||||
|
|
||||||
%{idx: idx_count, grib: grib_count} = Agent.get(counter, & &1)
|
%{idx: idx_count, grib: grib_count} = Agent.get(counter, & &1)
|
||||||
|
|
||||||
assert idx_count == 1, "idx should be cached — expected 1 fetch, got #{idx_count}"
|
# fetch_profile pulls 2 idx URLs (surface + pressure) on the first call;
|
||||||
|
# the second call must be a full cache hit, so the total stays at 2.
|
||||||
|
assert idx_count == 2, "idx should be cached — expected 2 total fetches, got #{idx_count}"
|
||||||
assert grib_count > 0, "grib fetches should still happen"
|
assert grib_count > 0, "grib fetches should still happen"
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue