From 71e8f5314206c1c4dc5a391e794cbc76e5d6b2e1 Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Tue, 21 Apr 2026 13:49:07 -0500
Subject: [PATCH] fix: stabilize flaky tests + silence PromEx Oban poller in
 test
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- HrrrClient idx-cache test only invalidated the surface idx URL, but
  fetch_profile also fetches a pressure idx. Previous runs' state for
  the pressure key decided whether the counter landed at 1 (prior run
  cached it, pass) or 2 (cold, fail). Invalidate both + assert 2 total
  fetches to reflect the actual code path.
- CsvImportTest deadlocked against other async DataCase tests when
  inline Oban child jobs upserted iemre_observations/terrain_profiles
  with a shared conflict target. Flip to async: false — same fix as
  ContactWeatherEnqueueWorkerTest earlier this session.
- PromEx.Plugins.Oban runs a 5s telemetry_poller that queries the DB,
  but its poller PID has no sandbox connection in test and crashed
  with DBConnection.OwnershipError on every tick, spamming the log.
  Gate the plugin on a config flag and skip it in config/test.exs;
  prod behaviour unchanged.
---
 config/test.exs                                 |  8 ++++++++
 lib/microwaveprop/prom_ex.ex                    | 14 ++++++++++----
 test/microwaveprop/radio/csv_import_test.exs    |  6 +++++-
 test/microwaveprop/weather/hrrr_client_test.exs | 11 ++++++++++-
 4 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/config/test.exs b/config/test.exs
index 34223a3a..6c27f380 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -48,6 +48,14 @@ config :microwaveprop, MicrowavepropWeb.Endpoint,
 # Run Oban jobs inline during tests
 config :microwaveprop, Oban, testing: :inline
 
+# MetricsLogSuppressionTest hits GET /metrics and asserts no "Sent 200"
+# log line; that path needs PromEx alive to return a 200. We can't
+# blanket-disable PromEx. Instead, toggle off the Oban-queue poller
+# below — its 5 s telemetry measurement issues an Ecto query from a
+# process that doesn't hold a sandbox connection and crashes with
+# DBConnection.OwnershipError, spamming the test log.
+config :microwaveprop, :prom_ex_include_oban_plugin, false
+
 # Score files land under a per-run tmp directory so replace_scores tests
 # don't write into a shared location (and we can assert on the file
 # contents from inside tests).
diff --git a/lib/microwaveprop/prom_ex.ex b/lib/microwaveprop/prom_ex.ex
index f83ccc8e..84d874aa 100644
--- a/lib/microwaveprop/prom_ex.ex
+++ b/lib/microwaveprop/prom_ex.ex
@@ -14,6 +14,15 @@ defmodule Microwaveprop.PromEx do
 
   @impl true
   def plugins do
+    # Oban plugin's queue-depth metric is a telemetry_poller that runs
+    # an Ecto query every 5 s; in the test env that poller has no
+    # sandbox connection and crashes with DBConnection.OwnershipError,
+    # so it's skipped there.
+    oban_plugin =
+      if Application.get_env(:microwaveprop, :prom_ex_include_oban_plugin, true),
+        do: [Plugins.Oban],
+        else: []
+
     [
       # Defaults: VM memory, process counts, scheduler util, app version.
       Plugins.Application,
@@ -25,13 +34,10 @@ defmodule Microwaveprop.PromEx do
       # Ecto: query duration, queue time, pool size.
       {Plugins.Ecto, otp_app: :microwaveprop, repos: [Microwaveprop.Repo]},
 
-      # Oban: job duration, queue time, state counts, queue depth.
-      Plugins.Oban,
-
       # Our own spans: Instrument.span/3 emits [:microwaveprop | suffix]
       # events; this plugin converts them to Prometheus histograms.
       Microwaveprop.PromEx.InstrumentPlugin
-    ]
+    ] ++ oban_plugin
   end
 
   @impl true
diff --git a/test/microwaveprop/radio/csv_import_test.exs b/test/microwaveprop/radio/csv_import_test.exs
index fc6508ac..c616b338 100644
--- a/test/microwaveprop/radio/csv_import_test.exs
+++ b/test/microwaveprop/radio/csv_import_test.exs
@@ -1,5 +1,9 @@
 defmodule Microwaveprop.Radio.CsvImportTest do
-  use Microwaveprop.DataCase, async: true
+  # async: false — CsvImport.commit/1 inserts contacts that trigger inline
+  # Oban enrichment jobs (IEMRE, weather, HRRR, terrain) which upsert rows
+  # with a shared conflict target. Running concurrently with sibling
+  # DataCase tests produced intermittent "ShareLock deadlock_detected".
+  use Microwaveprop.DataCase, async: false
   use Oban.Testing, repo: Microwaveprop.Repo
 
   alias Microwaveprop.Radio.Contact
diff --git a/test/microwaveprop/weather/hrrr_client_test.exs b/test/microwaveprop/weather/hrrr_client_test.exs
index 15057bec..29996419 100644
--- a/test/microwaveprop/weather/hrrr_client_test.exs
+++ b/test/microwaveprop/weather/hrrr_client_test.exs
@@ -189,6 +189,13 @@ defmodule Microwaveprop.Weather.HrrrClientTest do
         {:hrrr_idx, "https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20260328/conus/hrrr.t18z.wrfsfcf00.grib2.idx"}
       )
 
+      # `do_fetch_profile` fetches TWO idx URLs (surface + pressure). Also drop
+      # the pressure cache so the test starts from a deterministic state —
+      # without this, whether this assertion passes depends on seed ordering.
+      Microwaveprop.Cache.invalidate(
+        {:hrrr_idx, "https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20260328/conus/hrrr.t18z.wrfprsf00.grib2.idx"}
+      )
+
       {:ok, counter} = Agent.start_link(fn -> %{idx: 0, grib: 0} end)
 
       Req.Test.stub(HrrrClient, fn conn ->
@@ -214,7 +221,9 @@ defmodule Microwaveprop.Weather.HrrrClientTest do
 
       %{idx: idx_count, grib: grib_count} = Agent.get(counter, & &1)
 
-      assert idx_count == 1, "idx should be cached — expected 1 fetch, got #{idx_count}"
+      # fetch_profile pulls 2 idx URLs (surface + pressure) on the first call;
+      # the second call must be a full cache hit, so the total stays at 2.
+      assert idx_count == 2, "idx should be cached — expected 2 total fetches, got #{idx_count}"
       assert grib_count > 0, "grib fetches should still happen"
     end
   end