Rename ERA5 → NARR across the codebase

- Schema: Era5Profile → NarrProfile; table renamed via migration
  rename_era5_profiles_to_narr_profiles (ALTER TABLE + rename indexes,
  atomic and instant, no row rewrite)
- Weather helpers: find_nearest_era5/era5_for_contact/era5_profiles_for_path
  → find_nearest_narr/narr_for_contact/narr_profiles_for_path
- BackfillEnqueueWorker: :era5 → :narr type (virtual, no narr_status column);
  reconcile_stale_queued_to_unavailable and status_priority_order now skip
  virtual types via @virtual_types. Fixes the prod crash where the cron tried
  to update a non-existent era5_status column.
- ContactWeatherEnqueueWorker: build_era5_jobs → build_narr_jobs,
  era5_jobs_for_contact → narr_jobs_for_contact
- ContactLive: @era5 → @narr, maybe_enqueue_era5 → maybe_enqueue_narr;
  UI label "ERA5 (0.25°)" → "NARR (32 km)"
- Cron (runtime.exs): args types list now "narr" instead of "era5"
- /status: progress row, status-by-type card, totals.narr_profiles, and
  table-count lookup all target narr_profiles
- Drop obsolete Era5CdsJob schema + era5_cds_jobs table (inspection
  artifact from the retired CDS pipeline; 34 orphan rows in prod)
- Misc docstring/comment cleanups (skew_t, about, wgrib2, propagation_train)

Includes a regression test for the virtual-type crash.
This commit is contained in:
Graham McIntire 2026-04-16 09:22:23 -05:00
parent 9e94759f7a
commit 3604896726
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
22 changed files with 235 additions and 258 deletions

View file

@ -218,11 +218,11 @@ if config_env() == :prod do
# full grid-side JSONB write for.
# {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker},
{"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
# The :era5 type dispatches NarrFetchWorker for pre-2014 contacts
# (see ContactWeatherEnqueueWorker.era5_jobs_for_contact/1). The
# type key is a historical artifact — the data source is NARR.
# The :narr type is virtual: it targets contacts with hrrr_status =
# :unavailable (pre-2014, missing from the HRRR archive) and dispatches
# NarrFetchWorker against NCEI. See narr_jobs_for_contact/1.
{"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker,
args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre", "era5"]}},
args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre", "narr"]}},
# UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch
{"30 1 * * *", CanadianSoundingFetchWorker},
{"30 13 * * *", CanadianSoundingFetchWorker},

View file

@ -5,13 +5,13 @@ defmodule Microwaveprop.Weather do
alias Microwaveprop.Propagation.ProfilesFile
alias Microwaveprop.Repo
alias Microwaveprop.Weather.Era5Profile
alias Microwaveprop.Weather.GridCache
alias Microwaveprop.Weather.HrrrNativeProfile
alias Microwaveprop.Weather.HrrrProfile
alias Microwaveprop.Weather.IemClient
alias Microwaveprop.Weather.IemreObservation
alias Microwaveprop.Weather.Metar5minObservation
alias Microwaveprop.Weather.NarrProfile
alias Microwaveprop.Weather.RtmaObservation
alias Microwaveprop.Weather.SolarIndex
alias Microwaveprop.Weather.Sounding
@ -868,56 +868,56 @@ defmodule Microwaveprop.Weather do
@doc """
Find the best available atmospheric profile for a contact.
Tries HRRR first (3 km, hourly), falls back to ERA5 (0.25°, hourly).
Tries HRRR first (3 km, hourly), falls back to NARR (32 km, 3-hourly).
Returns the profile struct or nil.
"""
@spec best_profile_for_contact(map()) :: HrrrProfile.t() | Era5Profile.t() | nil
@spec best_profile_for_contact(map()) :: HrrrProfile.t() | NarrProfile.t() | nil
def best_profile_for_contact(contact) do
hrrr_for_contact(contact) || era5_for_contact(contact)
hrrr_for_contact(contact) || narr_for_contact(contact)
end
@doc "Find all atmospheric profiles along a contact's path, from any source."
@spec profiles_along_path(map()) :: [HrrrProfile.t() | Era5Profile.t()]
@spec profiles_along_path(map()) :: [HrrrProfile.t() | NarrProfile.t()]
def profiles_along_path(contact) do
hrrr_path = hrrr_profiles_for_path(contact)
if hrrr_path == [] do
era5_profiles_for_path(contact)
narr_profiles_for_path(contact)
else
hrrr_path
end
end
@spec era5_for_contact(map()) :: Era5Profile.t() | nil
def era5_for_contact(%{pos1: nil}), do: nil
@spec narr_for_contact(map()) :: NarrProfile.t() | nil
def narr_for_contact(%{pos1: nil}), do: nil
def era5_for_contact(contact) do
def narr_for_contact(contact) do
lat = contact.pos1["lat"]
lon = contact.pos1["lon"] || contact.pos1["lng"]
if lat && lon do
find_nearest_era5(lat, lon, contact.qso_timestamp)
find_nearest_narr(lat, lon, contact.qso_timestamp)
end
end
@spec era5_profiles_for_path(map()) :: [Era5Profile.t()]
def era5_profiles_for_path(%{pos1: nil}), do: []
@spec narr_profiles_for_path(map()) :: [NarrProfile.t()]
def narr_profiles_for_path(%{pos1: nil}), do: []
def era5_profiles_for_path(contact) do
def narr_profiles_for_path(contact) do
contact
|> Microwaveprop.Radio.contact_path_points()
|> Enum.map(fn {lat, lon} -> find_nearest_era5(lat, lon, contact.qso_timestamp) end)
|> Enum.map(fn {lat, lon} -> find_nearest_narr(lat, lon, contact.qso_timestamp) end)
|> Enum.reject(&is_nil/1)
end
@spec find_nearest_era5(float(), float(), DateTime.t()) :: Era5Profile.t() | nil
def find_nearest_era5(lat, lon, timestamp) do
@spec find_nearest_narr(float(), float(), DateTime.t()) :: NarrProfile.t() | nil
def find_nearest_narr(lat, lon, timestamp) do
dlat = 0.15
dlon = 0.15
time_start = DateTime.add(timestamp, -1800, :second)
time_end = DateTime.add(timestamp, 1800, :second)
Era5Profile
NarrProfile
|> where(
[p],
p.lat >= ^(lat - dlat) and p.lat <= ^(lat + dlat) and

View file

@ -1,44 +0,0 @@
defmodule Microwaveprop.Weather.Era5CdsJob do
@moduledoc """
Tracks an in-flight ERA5 CDS request submitted by `Era5SubmitWorker` and
awaiting download+decode by `Era5PollWorker`. Persisting the CDS job IDs
decouples the Oban worker lifecycle from the 30+ minute CDS queue time,
so a pod restart resumes polling instead of re-submitting.
"""
use Ecto.Schema
import Ecto.Changeset
@primary_key {:id, :binary_id, autogenerate: true}
@foreign_key_type :binary_id
schema "era5_cds_jobs" do
field :year, :integer
field :month, :integer
field :tile_lat, :integer
field :tile_lon, :integer
field :single_job_id, :string
field :pressure_job_id, :string
field :submitted_at, :utc_datetime
field :last_polled_at, :utc_datetime
field :poll_count, :integer, default: 0
timestamps(type: :utc_datetime)
end
@type t :: %__MODULE__{}
@required_fields ~w(year month tile_lat tile_lon single_job_id pressure_job_id submitted_at)a
@optional_fields ~w(last_polled_at poll_count)a
@spec changeset(t() | Ecto.Changeset.t(), map()) :: Ecto.Changeset.t()
def changeset(job, attrs) do
job
|> cast(attrs, @required_fields ++ @optional_fields)
|> validate_required(@required_fields)
|> unique_constraint([:year, :month, :tile_lat, :tile_lon],
name: :era5_cds_jobs_year_month_tile_lat_tile_lon_index,
error_key: :tile_month
)
end
end

View file

@ -336,7 +336,7 @@ defmodule Microwaveprop.Weather.Grib2.Wgrib2 do
@doc """
Like `extract_grid/3`, but returns a flat list of per-message results so
the time dimension is preserved. Used when a single GRIB2 blob carries
many timesteps (e.g. a month of ERA5 data fetched in one CDS request).
many timesteps.
Each entry is `%{datetime: DateTime.t() | nil, var: String.t(),
level: String.t(), values: %{{lat, lon} => float}}`.
@ -354,7 +354,7 @@ defmodule Microwaveprop.Weather.Grib2.Wgrib2 do
| {:error, term()}
def extract_grid_messages(grib_binary, match_pattern, grid_spec) do
if available?() do
tmp_grib = Path.join(System.tmp_dir!(), "era5_#{System.unique_integer([:positive])}.grib2")
tmp_grib = Path.join(System.tmp_dir!(), "wgrib2_#{System.unique_integer([:positive])}.grib2")
try do
File.write!(tmp_grib, grib_binary)

View file

@ -141,13 +141,13 @@ defmodule Microwaveprop.Weather.NarrClient do
end
@doc """
Extracts an `era5_profiles`-shaped attribute map from a composite NARR
Extracts a NarrProfile-shaped attribute map from a composite NARR
GRIB1 file at the given lat/lon point.
Pure-ish: shells out to `cdo -outputtab,name,lev,value -remapnn,lon=_lat=`
on the local file. No network.
Returns `{:ok, attrs}` a map with the `era5_profiles` fields
Returns `{:ok, attrs}` a map with the NarrProfile fields
(`:surface_temp_c`, `:surface_dewpoint_c`, `:surface_pressure_mb`,
`:hpbl_m`, `:pwat_mm`, `:profile` list plus the derived fields from
`SoundingParams.derive/1`) or `{:error, reason}` on missing file,
@ -170,8 +170,8 @@ defmodule Microwaveprop.Weather.NarrClient do
using `extract_profile_from_file/3`.
Returns `{:ok, profile_attrs}` shaped for direct `Repo.insert_all/3`
into the `era5_profiles` table, or `{:error, reason}`. Temp files are
always cleaned up via `try/after`.
against `NarrProfile`, or `{:error, reason}`. Temp files are always
cleaned up via `try/after`.
"""
@spec fetch_profile_at(DateTime.t(), {float(), float()}) ::
{:ok, map()} | {:error, term()}

View file

@ -1,5 +1,13 @@
defmodule Microwaveprop.Weather.Era5Profile do
@moduledoc false
defmodule Microwaveprop.Weather.NarrProfile do
@moduledoc """
Profile rows fetched from NCEP NARR (North American Regional Reanalysis)
for pre-2014 contacts where the HRRR archive doesn't reach.
The table was originally created as `era5_profiles` for the retired
ERA5/CDS pipeline and reused 1:1 by NARR; the
`rename_era5_profiles_to_narr_profiles` migration renames it to
`narr_profiles`.
"""
use Ecto.Schema
import Ecto.Changeset
@ -7,7 +15,7 @@ defmodule Microwaveprop.Weather.Era5Profile do
@primary_key {:id, :binary_id, autogenerate: true}
@foreign_key_type :binary_id
schema "era5_profiles" do
schema "narr_profiles" do
field :valid_time, :utc_datetime
field :lat, :float
field :lon, :float

View file

@ -14,7 +14,12 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
require Logger
@enrichable [:pending, :queued, :failed]
@valid_types ~w(hrrr weather terrain iemre era5)
@valid_types ~w(hrrr weather terrain iemre narr)
# Virtual types have no <type>_status column on contacts; they're synthesized
# from other status fields (e.g. :narr candidates are `hrrr_status = :unavailable`).
# Skip these in any reconcile / ordering pass that reads a <type>_status field.
@virtual_types [:narr]
# A :queued contact whose updated_at is older than this is one the cron has
# already tried ~144 times over 3 days without the underlying worker landing
@ -67,30 +72,45 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
types |> Enum.filter(&(&1 in @valid_types)) |> Enum.map(&String.to_existing_atom/1)
end
defp parse_types(_), do: [:hrrr, :weather, :terrain, :iemre, :era5]
defp parse_types(_), do: [:hrrr, :weather, :terrain, :iemre, :narr]
defp status_priority_order(types) do
# Prioritize pending/failed over queued so real work happens first
field = :"#{hd(types)}_status"
# Prioritize pending/failed over queued so real work happens first.
# Virtual types (e.g. :narr) have no *_status column — fall back to
# time-only ordering when every requested type is virtual.
case Enum.find(types, &(&1 not in @virtual_types)) do
nil ->
[desc: dynamic([c], c.qso_timestamp)]
[
asc: dynamic([c], fragment("CASE ? WHEN 'pending' THEN 0 WHEN 'failed' THEN 1 ELSE 2 END", field(c, ^field))),
desc: dynamic([c], c.qso_timestamp)
]
type ->
field = :"#{type}_status"
[
asc:
dynamic(
[c],
fragment("CASE ? WHEN 'pending' THEN 0 WHEN 'failed' THEN 1 ELSE 2 END", field(c, ^field))
),
desc: dynamic([c], c.qso_timestamp)
]
end
end
# Flip contacts that have been stuck in :queued for longer than
# @stale_queued_cutoff_seconds to :unavailable. One Repo.update_all per type,
# scoped to only the types passed in args so a partial cron (e.g. hrrr-only)
# doesn't touch other statuses. Fast: hits the (<field>, updated_at) combo
# with a simple range scan.
# doesn't touch other statuses. Virtual types (e.g. :narr) have no *_status
# column and are skipped. Fast: hits the (<field>, updated_at) combo with a
# simple range scan.
defp reconcile_stale_queued_to_unavailable(types) do
cutoff =
DateTime.utc_now()
|> DateTime.add(-@stale_queued_cutoff_seconds, :second)
|> DateTime.truncate(:second)
Enum.reduce(types, 0, fn type, acc ->
types
|> Enum.reject(&(&1 in @virtual_types))
|> Enum.reduce(0, fn type, acc ->
field = :"#{type}_status"
{n, _} =
@ -126,8 +146,8 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
defp type_filter(types) do
Enum.reduce(types, dynamic(false), fn
:era5, acc ->
# ERA5 targets contacts where HRRR is unavailable (pre-2014, missing from archive)
:narr, acc ->
# NARR targets contacts where HRRR is unavailable (pre-2014, missing from archive)
dynamic([c], ^acc or c.hrrr_status == :unavailable)
type, acc ->

View file

@ -33,9 +33,9 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
insert_all_chunked(bulk_jobs)
end
# ERA5 jobs must go through Oban.insert/1 so the worker's unique
# NARR jobs must go through Oban.insert/1 so the worker's unique
# constraint is honored; Oban OSS insert_all does not check uniqueness.
insert_unique(jobs_by_type[:era5])
insert_unique(jobs_by_type[:narr])
mark_enrichment_statuses(contact, types, jobs_by_type)
@ -48,7 +48,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
hrrr: if(:hrrr in types, do: build_hrrr_jobs([contact]), else: []),
terrain: if(:terrain in types, do: build_terrain_jobs([contact]), else: []),
iemre: if(:iemre in types, do: build_iemre_jobs([contact]), else: []),
era5: if(:era5 in types, do: build_era5_jobs([contact]), else: [])
narr: if(:narr in types, do: build_narr_jobs([contact]), else: [])
}
end
@ -173,18 +173,15 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
|> Enum.uniq_by(fn changeset -> changeset.changes.args end)
end
def build_era5_jobs(contacts) do
def build_narr_jobs(contacts) do
contacts
|> Enum.flat_map(&era5_jobs_for_contact/1)
|> Enum.flat_map(&narr_jobs_for_contact/1)
|> Enum.uniq_by(fn changeset -> changeset.changes.args end)
end
# The function name is a historical artifact: the `:era5` type key still
# refers to the `era5_profiles` table (reused 1:1), but this now dispatches
# `NarrFetchWorker` against NCEP NARR — the CDS/ERA5 pipeline is gone.
defp era5_jobs_for_contact(%{pos1: nil}), do: []
defp narr_jobs_for_contact(%{pos1: nil}), do: []
defp era5_jobs_for_contact(contact) do
defp narr_jobs_for_contact(contact) do
# NARR analyses are 3-hourly (00/03/06/09/12/15/18/21 UTC). NarrFetchWorker
# rejects anything else, so snap here in the caller.
valid_time = NarrClient.snap_to_analysis_hour(contact.qso_timestamp)
@ -195,7 +192,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
rlat = Float.round(lat * 4) / 4
rlon = Float.round(lon * 4) / 4
if Weather.find_nearest_era5(rlat, rlon, valid_time) do
if Weather.find_nearest_narr(rlat, rlon, valid_time) do
[]
else
[

View file

@ -1,12 +1,12 @@
defmodule Microwaveprop.Workers.NarrFetchWorker do
@moduledoc """
Fetches one NCEP NARR profile from NCEI via `NarrClient.fetch_profile_at/2`
and inserts it into the `era5_profiles` table.
and inserts it as a `NarrProfile` row.
Replaces `Era5FetchWorker` for pre-2014 contacts. Unlike the ERA5 router,
NARR fetches are per-point (one analysis hour, one location), so this
worker does the fetch + insert directly no downstream batch worker,
no routing.
Covers pre-2014 contacts where the HRRR archive doesn't reach. Replaces
the retired ERA5/CDS pipeline. NARR fetches are per-point (one analysis
hour, one location), so this worker does the fetch + insert directly
no downstream batch worker, no routing.
Jobs are unique on `{lat, lon, valid_time}` so duplicate enrichment
requests collapse. `valid_time` MUST already be snapped to a NARR
@ -15,8 +15,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorker do
`NarrClient.url_for/1`.
See `docs/plans/2026-04-15-merra2-historical-backfill.md` for the full
architecture. The `era5_profiles` table is reused 1:1 (rename is a
follow-up).
architecture.
"""
use Oban.Worker,
queue: :narr,
@ -29,8 +28,8 @@ defmodule Microwaveprop.Workers.NarrFetchWorker do
import Ecto.Query
alias Microwaveprop.Repo
alias Microwaveprop.Weather.Era5Profile
alias Microwaveprop.Weather.NarrClient
alias Microwaveprop.Weather.NarrProfile
require Logger
@ -41,7 +40,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorker do
rlat = Float.round(lat * 4) / 4
rlon = Float.round(lon * 4) / 4
if has_era5_profile?(rlat, rlon, valid_time) do
if has_narr_profile?(rlat, rlon, valid_time) do
:ok
else
fetch_and_insert(rlat, rlon, valid_time)
@ -74,22 +73,21 @@ defmodule Microwaveprop.Workers.NarrFetchWorker do
|> Map.put(:inserted_at, now)
|> Map.put(:updated_at, now)
Repo.insert_all(Era5Profile, [row],
Repo.insert_all(NarrProfile, [row],
on_conflict: :nothing,
conflict_target: [:lat, :lon, :valid_time]
)
end
defp has_era5_profile?(lat, lon, valid_time) do
# NARR is exact 3-hourly, so we tighten the time window to ±15 minutes
# compared to Era5FetchWorker's ±30-minute check. Lat/lon tolerance
# matches the 0.25° dedup granularity.
defp has_narr_profile?(lat, lon, valid_time) do
# NARR is exact 3-hourly, so the time window can be tight (±15 min).
# Lat/lon tolerance matches the 0.25° dedup granularity.
dlat = 0.13
dlon = 0.13
time_start = DateTime.add(valid_time, -900, :second)
time_end = DateTime.add(valid_time, 900, :second)
Era5Profile
NarrProfile
|> where(
[p],
p.lat >= ^(lat - dlat) and p.lat <= ^(lat + dlat) and

View file

@ -172,7 +172,7 @@ defmodule MicrowavepropWeb.AboutLive do
forecasts), Iowa Environmental Mesonet (ASOS and upper-air
soundings), IEMRE gridded reanalysis, SRTM 90 m terrain, SNMP
polling on seven commercial microwave links near DFW at 5-minute
intervals, and Copernicus ERA5 for pre-2014 contact enrichment.
intervals, and NCEP NARR for pre-2014 contact enrichment.
</li>
<li>
There's Nx/Axon/EXLA scaffolding for a small feed-forward model

View file

@ -46,8 +46,8 @@ defmodule MicrowavepropWeb.ContactLive.Show do
hrrr: nil,
hrrr_path: [],
native_profile: nil,
era5: nil,
era5_path: [],
narr: nil,
narr_path: [],
iemre: nil,
terrain: nil,
elevation_profile: nil,
@ -91,7 +91,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
|> start_async(:weather, fn -> load_weather(contact) end)
|> start_async(:solar, fn -> load_solar(contact) end)
|> start_async(:hrrr_path, fn -> Weather.hrrr_profiles_for_path(contact) end)
|> start_async(:era5_path, fn -> Weather.era5_profiles_for_path(contact) end)
|> start_async(:narr_path, fn -> Weather.narr_profiles_for_path(contact) end)
|> start_async(:native_profile, fn -> load_native_profile(contact) end)
|> start_async(:terrain, fn -> Terrain.get_terrain_profile(contact.id) end)
|> start_async(:iemre, fn -> load_iemre(contact) end)
@ -328,14 +328,14 @@ defmodule MicrowavepropWeb.ContactLive.Show do
{:noreply, socket}
end
def handle_async(:era5_path, {:ok, era5_path}, socket) do
era5 = List.first(era5_path)
def handle_async(:narr_path, {:ok, narr_path}, socket) do
narr = List.first(narr_path)
contact = socket.assigns.contact
contact =
if socket.assigns.can_enqueue, do: maybe_enqueue_era5(era5, contact), else: contact
if socket.assigns.can_enqueue, do: maybe_enqueue_narr(narr, contact), else: contact
{:noreply, assign(socket, contact: contact, era5: era5, era5_path: era5_path)}
{:noreply, assign(socket, contact: contact, narr: narr, narr_path: narr_path)}
end
def handle_async(:terrain, {:ok, terrain}, socket) do
@ -361,7 +361,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
end
def handle_async(slot, {:exit, reason}, socket)
when slot in [:weather, :solar, :hrrr_path, :era5_path, :native_profile, :terrain, :iemre] do
when slot in [:weather, :solar, :hrrr_path, :narr_path, :native_profile, :terrain, :iemre] do
Logger.warning("contact hydrate task #{slot} exited: #{inspect(reason)}")
{:noreply, socket}
end
@ -661,13 +661,12 @@ defmodule MicrowavepropWeb.ContactLive.Show do
end
end
# ERA5 is a fallback for contacts where HRRR is unavailable (pre-2014 or
# NARR is the fallback for contacts where HRRR is unavailable (pre-2014 or
# missing from the archive). Only enqueue when HRRR is known-unavailable AND
# we don't already have an ERA5 profile — otherwise we'd waste CDS quota on
# contacts the HRRR path will cover.
defp maybe_enqueue_era5(era5, contact) when not is_nil(era5), do: contact
# we don't already have a NARR profile.
defp maybe_enqueue_narr(narr, contact) when not is_nil(narr), do: contact
defp maybe_enqueue_era5(nil, %{hrrr_status: :unavailable} = contact) do
defp maybe_enqueue_narr(nil, %{hrrr_status: :unavailable} = contact) do
lat = contact.pos1 && contact.pos1["lat"]
lon = contact.pos1 && (contact.pos1["lon"] || contact.pos1["lng"])
@ -686,7 +685,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
contact
end
defp maybe_enqueue_era5(_nil, contact), do: contact
defp maybe_enqueue_narr(_nil, contact), do: contact
defp queue_info(counts, queue) do
case Map.get(counts, queue, 0) do
@ -1140,11 +1139,11 @@ defmodule MicrowavepropWeb.ContactLive.Show do
<div class="divider" />
<h2 class="text-base font-semibold mb-2">Atmospheric Profile</h2>
<% profile = @hrrr || @era5 %>
<% profile = @hrrr || @narr %>
<% profile_source =
cond do
@hrrr -> "HRRR (3 km)"
@era5 -> "ERA5 (0.25°)"
@narr -> "NARR (32 km)"
true -> nil
end %>
<% skew_t_levels =
@ -1253,7 +1252,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
<% :unavailable -> %>
<span class="flex items-center gap-2">
<span class="loading loading-spinner loading-sm"></span>
HRRR unavailable fetching ERA5 reanalysis {queue_info(@queue_counts, "era5_batch")}
HRRR unavailable fetching NARR reanalysis {queue_info(@queue_counts, "narr")}
</span>
<% _ -> %>
No atmospheric profile available.

View file

@ -97,9 +97,8 @@ defmodule MicrowavepropWeb.StatusLive do
end
# NARR is the fallback for pre-2014 contacts (HRRR archive starts 2014).
# Profiles land in the era5_profiles table (kept 1:1 after the ERA5/CDS
# pipeline was replaced). "Done" means a row within 0.15° + 30 min of the
# contact, matching `Weather.find_nearest_era5/3` lookup tolerance.
# "Done" means a narr_profiles row within 0.15° + 30 min of the contact,
# matching `Weather.find_nearest_narr/3` lookup tolerance.
defp count_narr_done do
%{rows: [[done]]} =
Repo.query!("""
@ -109,7 +108,7 @@ defmodule MicrowavepropWeb.StatusLive do
AND c.pos1 IS NOT NULL
AND EXISTS (
SELECT 1
FROM era5_profiles p
FROM narr_profiles p
WHERE p.lat BETWEEN ((c.pos1->>'lat')::float - 0.15) AND ((c.pos1->>'lat')::float + 0.15)
AND p.lon BETWEEN ((c.pos1->>'lon')::float - 0.15) AND ((c.pos1->>'lon')::float + 0.15)
AND p.valid_time BETWEEN (c.qso_timestamp - INTERVAL '30 minutes') AND (c.qso_timestamp + INTERVAL '30 minutes')
@ -264,8 +263,8 @@ defmodule MicrowavepropWeb.StatusLive do
table_counts = Map.new(tables, fn %{table: t, rows: r} -> {t, r} end)
# NARR doesn't have its own status column — synthesize from HRRR unavailable
# + era5_profiles row count (the table is reused as the NARR landing spot).
narr_count = Map.get(table_counts, "era5_profiles", 0)
# + narr_profiles row count.
narr_count = Map.get(table_counts, "narr_profiles", 0)
hrrr_unavailable =
statuses |> Map.get("hrrr", []) |> Enum.find_value(0, fn {s, c} -> if s == "unavailable", do: c end)
@ -289,7 +288,7 @@ defmodule MicrowavepropWeb.StatusLive do
statuses: statuses,
totals: %{
hrrr_profiles: hrrr_count,
narr_profiles: Map.get(table_counts, "era5_profiles", 0),
narr_profiles: Map.get(table_counts, "narr_profiles", 0),
terrain_profiles: terrain_count,
surface_observations: obs_count,
soundings: sounding_count,

View file

@ -3,7 +3,7 @@ defmodule MicrowavepropWeb.SkewT do
Skew-T log-P diagram renderer. Builds the full set of SVG
primitives (isobars, isotherms, dry adiabats, saturation mixing
ratio lines, and temperature/dewpoint traces) for a single HRRR
or ERA5 pressure-level profile so the contact detail template can
or NARR pressure-level profile so the contact detail template can
draw it declaratively.
## Meteorology

View file

@ -59,7 +59,7 @@ defmodule Mix.Tasks.PropagationTrain do
)
# Boot the app without starting Oban queues so training doesn't have
# the HRRR / ERA5 / weather cron firing in the background against the
# the HRRR / NARR / weather cron firing in the background against the
# local DB we're reading from.
Application.put_env(
:microwaveprop,

View file

@ -0,0 +1,27 @@
defmodule Microwaveprop.Repo.Migrations.RenameEra5ProfilesToNarrProfiles do
use Ecto.Migration
# ALTER TABLE ... RENAME is atomic and instant (no row rewrite). User-created
# indexes keep their old names after the table rename, so we rename them too
# for consistency. Implicit PK index / sequence are renamed automatically by
# PostgreSQL.
def up do
rename table(:era5_profiles), to: table(:narr_profiles)
execute "ALTER INDEX era5_profiles_lat_lon_valid_time_index RENAME TO narr_profiles_lat_lon_valid_time_index"
execute "ALTER INDEX era5_profiles_valid_time_index RENAME TO narr_profiles_valid_time_index"
execute "ALTER INDEX era5_profiles_valid_time_lat_lon_index RENAME TO narr_profiles_valid_time_lat_lon_index"
end
def down do
execute "ALTER INDEX narr_profiles_valid_time_lat_lon_index RENAME TO era5_profiles_valid_time_lat_lon_index"
execute "ALTER INDEX narr_profiles_valid_time_index RENAME TO era5_profiles_valid_time_index"
execute "ALTER INDEX narr_profiles_lat_lon_valid_time_index RENAME TO era5_profiles_lat_lon_valid_time_index"
rename table(:narr_profiles), to: table(:era5_profiles)
end
end

View file

@ -0,0 +1,29 @@
defmodule Microwaveprop.Repo.Migrations.DropEra5CdsJobs do
use Ecto.Migration
# The ERA5/CDS pipeline was retired in favor of NARR (see
# rename_era5_profiles_to_narr_profiles). Orphan rows in this table are
# useless without the workers that wrote them.
def up do
drop table(:era5_cds_jobs)
end
def down do
create table(:era5_cds_jobs, primary_key: false) do
add :id, :binary_id, primary_key: true
add :year, :integer, null: false
add :month, :integer, null: false
add :tile_lat, :integer, null: false
add :tile_lon, :integer, null: false
add :single_job_id, :string, null: false
add :pressure_job_id, :string, null: false
add :submitted_at, :utc_datetime, null: false
add :last_polled_at, :utc_datetime
add :poll_count, :integer, default: 0, null: false
timestamps(type: :utc_datetime)
end
create unique_index(:era5_cds_jobs, [:year, :month, :tile_lat, :tile_lon])
end
end

View file

@ -1,82 +0,0 @@
defmodule Microwaveprop.Weather.Era5CdsJobTest do
use Microwaveprop.DataCase, async: true
alias Microwaveprop.Weather.Era5CdsJob
describe "changeset/2" do
@valid_attrs %{
year: 2014,
month: 3,
tile_lat: 32,
tile_lon: -98,
single_job_id: "abc-single",
pressure_job_id: "abc-pressure",
submitted_at: ~U[2026-04-13 21:00:00Z]
}
test "accepts a complete attrs map" do
changeset = Era5CdsJob.changeset(%Era5CdsJob{}, @valid_attrs)
assert changeset.valid?
end
test "requires year, month, tile coordinates, and both CDS job IDs" do
changeset = Era5CdsJob.changeset(%Era5CdsJob{}, %{})
refute changeset.valid?
required = ~w(year month tile_lat tile_lon single_job_id pressure_job_id submitted_at)a
errors = Keyword.keys(changeset.errors)
for field <- required do
assert field in errors, "expected #{inspect(field)} to be required"
end
end
test "last_polled_at and poll_count default to nil/0 and can be updated" do
{:ok, row} =
%Era5CdsJob{}
|> Era5CdsJob.changeset(@valid_attrs)
|> Repo.insert()
assert row.poll_count == 0
assert is_nil(row.last_polled_at)
now = DateTime.truncate(DateTime.utc_now(), :second)
{:ok, bumped} =
row
|> Era5CdsJob.changeset(%{last_polled_at: now, poll_count: 1})
|> Repo.update()
assert bumped.poll_count == 1
assert bumped.last_polled_at == now
end
end
describe "uniqueness" do
test "collapses duplicate (year, month, tile_lat, tile_lon) inserts" do
attrs = %{
year: 2014,
month: 3,
tile_lat: 32,
tile_lon: -98,
single_job_id: "first-single",
pressure_job_id: "first-pressure",
submitted_at: ~U[2026-04-13 21:00:00Z]
}
{:ok, _} =
%Era5CdsJob{}
|> Era5CdsJob.changeset(attrs)
|> Repo.insert()
{:error, changeset} =
%Era5CdsJob{}
|> Era5CdsJob.changeset(%{attrs | single_job_id: "second-single", pressure_job_id: "second-pressure"})
|> Repo.insert()
refute changeset.valid?
assert {:tile_month, _} = List.keyfind(changeset.errors, :tile_month, 0) || {:tile_month, nil}
end
end
end

View file

@ -5,6 +5,7 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorkerTest do
alias Microwaveprop.Terrain.ElevationClient
alias Microwaveprop.Weather.HrrrClient
alias Microwaveprop.Weather.IemClient
alias Microwaveprop.Weather.NarrClient
alias Microwaveprop.Workers.BackfillEnqueueWorker
defp create_contact(attrs \\ %{}) do
@ -53,12 +54,19 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorkerTest do
Req.Test.json(conn, %{"elevation" => List.duplicate(200.0, lat_count)})
end)
# NARR fetches land in era5_profiles; stub returns 500 so inline jobs fail
# gracefully instead of hitting NCEI.
Req.Test.stub(NarrClient, fn conn ->
Plug.Conn.send_resp(conn, 500, "stub")
end)
:ok
end
# ERA5 goes through Copernicus CDS API which is async (submit → poll → download)
# and has no Req.Test plug hook, so inline Oban would block on Process.sleep.
# These tests exercise the non-ERA5 enrichment path; ERA5 has its own coverage.
# NARR fetches (the :era5 type) run inline via testing: :inline — the
# NarrClient Req.Test stub above keeps them from hitting NCEI. Most tests
# below target the other pipelines and use this narrowed type list to keep
# the test surface tight.
@non_era5_types ["hrrr", "weather", "terrain", "iemre"]
describe "perform/1" do
@ -113,6 +121,24 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorkerTest do
assert_receive {:enqueue_complete, 1}
end
# :narr is a virtual type (no narr_status column) — it targets contacts
# whose hrrr_status is :unavailable. BackfillEnqueueWorker must not try
# to reconcile a non-existent narr_status column or use it for priority
# ordering.
test "handles virtual :narr type without referencing non-existent narr_status column" do
# Pre-2014 contact (HRRR unavailable) — NARR candidate.
contact = create_contact(%{qso_timestamp: ~U[2010-01-01 00:00:00Z]})
contact
|> Ecto.Changeset.change(%{hrrr_status: :unavailable})
|> Repo.update!()
assert :ok =
BackfillEnqueueWorker.perform(%Oban.Job{
args: %{"limit" => 10, "types" => ["narr"]}
})
end
end
describe "reconcile stuck queued → unavailable" do

View file

@ -4,9 +4,9 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
alias Microwaveprop.Radio.Contact
alias Microwaveprop.Terrain.ElevationClient
alias Microwaveprop.Weather
alias Microwaveprop.Weather.Era5Profile
alias Microwaveprop.Weather.HrrrClient
alias Microwaveprop.Weather.IemClient
alias Microwaveprop.Weather.NarrProfile
alias Microwaveprop.Workers.ContactWeatherEnqueueWorker
@qso_attrs %{
@ -562,7 +562,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
end
end
describe "build_era5_jobs/1" do
describe "build_narr_jobs/1" do
test "builds NarrFetchWorker jobs for each rounded path point" do
contact =
create_contact(%{
@ -571,7 +571,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
pos2: nil
})
jobs = ContactWeatherEnqueueWorker.build_era5_jobs([contact])
jobs = ContactWeatherEnqueueWorker.build_narr_jobs([contact])
assert length(jobs) == 1
job = hd(jobs)
@ -589,7 +589,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
pos2: nil
})
jobs = ContactWeatherEnqueueWorker.build_era5_jobs([contact])
jobs = ContactWeatherEnqueueWorker.build_narr_jobs([contact])
job = hd(jobs)
# 13:42 UTC snaps down to the 12:00 UTC NARR analysis slot.
@ -604,14 +604,14 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
pos2: %{"lat" => 30.3, "lon" => -97.7}
})
jobs = ContactWeatherEnqueueWorker.build_era5_jobs([contact])
jobs = ContactWeatherEnqueueWorker.build_narr_jobs([contact])
# pos1, midpoint, pos2 round to three distinct 0.25° grid cells.
assert length(jobs) == 3
assert Enum.all?(jobs, &(&1.changes.worker == "Microwaveprop.Workers.NarrFetchWorker"))
end
test "skips path points that already have an era5_profile row" do
test "skips path points that already have a narr_profile row" do
contact =
create_contact(%{
qso_timestamp: ~U[2010-06-15 13:42:00Z],
@ -620,8 +620,8 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
})
# pos1 (32.907, -97.038) snaps to the (33.0, -97.0) 0.25° grid cell
%Era5Profile{}
|> Era5Profile.changeset(%{
%NarrProfile{}
|> NarrProfile.changeset(%{
lat: 33.0,
lon: -97.0,
valid_time: ~U[2010-06-15 12:00:00Z],
@ -629,13 +629,13 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
})
|> Repo.insert!()
jobs = ContactWeatherEnqueueWorker.build_era5_jobs([contact])
jobs = ContactWeatherEnqueueWorker.build_narr_jobs([contact])
assert jobs == []
end
test "skips QSOs without pos1" do
contact = create_contact(%{pos1: nil})
assert ContactWeatherEnqueueWorker.build_era5_jobs([contact]) == []
assert ContactWeatherEnqueueWorker.build_narr_jobs([contact]) == []
end
test "deduplicates jobs with identical args across contacts" do
@ -655,7 +655,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
pos2: nil
})
jobs = ContactWeatherEnqueueWorker.build_era5_jobs([q1, q2])
jobs = ContactWeatherEnqueueWorker.build_narr_jobs([q1, q2])
# Both snap to 12:00Z and round to the same 0.25° grid cell → 1 job.
assert length(jobs) == 1

View file

@ -2,8 +2,8 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
use Microwaveprop.DataCase, async: false
use Oban.Testing, repo: Microwaveprop.Repo
alias Microwaveprop.Weather.Era5Profile
alias Microwaveprop.Weather.NarrClient
alias Microwaveprop.Weather.NarrProfile
alias Microwaveprop.Workers.NarrFetchWorker
@inv_fixture "test/fixtures/narr/narr-a_221_20100615_1200_000.inv"
@ -36,7 +36,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
end
describe "perform/1" do
test "fetches a profile via NarrClient and inserts it into era5_profiles" do
test "fetches a profile via NarrClient and inserts it as a NarrProfile" do
stub_narr_success()
valid_time = ~U[2010-06-15 12:00:00Z]
@ -51,7 +51,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
# Row snapped to 0.25° grid (32.9 → 33.0, -97.0 → -97.0) and stored at
# the exact 3-hourly valid_time.
assert [row] = Repo.all(Era5Profile)
assert [row] = Repo.all(NarrProfile)
assert row.lat == 33.0
assert row.lon == -97.0
assert DateTime.compare(row.valid_time, valid_time) == :eq
@ -62,7 +62,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
assert_in_delta row.pwat_mm, 45.1, 2.0
end
test "is a no-op when an era5_profiles row already exists for the snapped lat/lon/time" do
test "is a no-op when a NarrProfile row already exists for the snapped lat/lon/time" do
valid_time = ~U[2010-06-15 12:00:00Z]
# Stub that raises if called — this test asserts no HTTP happens.
@ -70,9 +70,9 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
raise "NarrClient should not be contacted when cache hit"
end)
# Pre-populate era5_profiles with a row at the snapped coordinates.
%Era5Profile{}
|> Era5Profile.changeset(%{
# Pre-populate with a row at the snapped coordinates.
%NarrProfile{}
|> NarrProfile.changeset(%{
lat: 33.0,
lon: -97.0,
valid_time: valid_time,
@ -89,7 +89,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
assert :ok = NarrFetchWorker.perform(%Oban.Job{args: args})
# Only the pre-seeded row exists — no new insert.
assert Repo.aggregate(Era5Profile, :count, :id) == 1
assert Repo.aggregate(NarrProfile, :count, :id) == 1
end
test "returns {:error, _} when NarrClient.fetch_profile_at fails" do
@ -104,7 +104,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
}
assert {:error, _reason} = NarrFetchWorker.perform(%Oban.Job{args: args})
assert Repo.aggregate(Era5Profile, :count, :id) == 0
assert Repo.aggregate(NarrProfile, :count, :id) == 0
end
test "raises ArgumentError when valid_time is not on a 3-hourly NARR slot" do

View file

@ -5,8 +5,8 @@ defmodule MicrowavepropWeb.ContactLiveTest do
alias Microwaveprop.Radio.Contact
alias Microwaveprop.Repo
alias Microwaveprop.Weather.Era5Profile
alias Microwaveprop.Weather.HrrrNativeProfile
alias Microwaveprop.Weather.NarrProfile
setup do
Req.Test.stub(Microwaveprop.Terrain.ElevationClient, fn conn ->
@ -156,8 +156,8 @@ defmodule MicrowavepropWeb.ContactLiveTest do
end
end
test "shows atmospheric profile section with ERA5 source for pre-HRRR contact", %{conn: conn} do
# Pre-2014 contact — HRRR is unavailable, ERA5 is the fallback source.
test "shows atmospheric profile section with NARR source for pre-HRRR contact", %{conn: conn} do
# Pre-2014 contact — HRRR is unavailable, NARR is the fallback source.
contact =
create_contact(%{
qso_timestamp: ~U[2010-06-15 18:00:00Z],
@ -166,7 +166,7 @@ defmodule MicrowavepropWeb.ContactLiveTest do
})
{:ok, _} =
Repo.insert(%Era5Profile{
Repo.insert(%NarrProfile{
valid_time: ~U[2010-06-15 18:00:00Z],
lat: 33.0,
lon: -97.0,
@ -187,8 +187,8 @@ defmodule MicrowavepropWeb.ContactLiveTest do
html = render_async(lv)
assert html =~ "Atmospheric Profile"
assert html =~ "ERA5"
# Collapsed summary shows HPBL / PWAT / lat,lon from the injected ERA5 row.
assert html =~ "NARR"
# Collapsed summary shows HPBL / PWAT / lat,lon from the injected NARR row.
assert html =~ "1200"
assert html =~ "33.0"
end
@ -203,10 +203,10 @@ defmodule MicrowavepropWeb.ContactLiveTest do
pos2: %{"lat" => 30.3, "lon" => -97.7}
})
# ERA5 row keeps the atmospheric profile section rendering with its
# NARR row keeps the atmospheric profile section rendering with its
# aggregate metadata (HPBL / PWAT / surface scalars).
{:ok, _} =
Repo.insert(%Era5Profile{
Repo.insert(%NarrProfile{
valid_time: ~U[2024-06-15 18:00:00Z],
lat: 32.9,
lon: -97.0,
@ -223,7 +223,7 @@ defmodule MicrowavepropWeb.ContactLiveTest do
# Native profile at the same point/time. `pressure_pa: [..., 30_000.0]`
# corresponds to 300 mb — a level the pressure-level profile cannot
# cover because HRRR/ERA5 stopped at 700 mb historically.
# cover because HRRR/NARR pressure-level data stops at 700 mb historically.
{:ok, _} =
%HrrrNativeProfile{}
|> HrrrNativeProfile.changeset(%{

View file

@ -7,7 +7,7 @@ defmodule MicrowavepropWeb.StatusLiveTest do
alias Microwaveprop.Cache
alias Microwaveprop.Radio.Contact
alias Microwaveprop.Repo
alias Microwaveprop.Weather.Era5Profile
alias Microwaveprop.Weather.NarrProfile
setup %{conn: conn} do
Cache.invalidate({MicrowavepropWeb.StatusLive, :count_unprocessed})
@ -41,8 +41,8 @@ defmodule MicrowavepropWeb.StatusLiveTest do
contact
end
# NARR fetches land in era5_profiles (table name kept as historical artifact);
# the status UI labels this "NARR" now.
# NARR fetches land in era5_profiles (table rename is a follow-up migration).
# The status UI labels this "NARR" now.
test "NARR progress bar numerator is actual profile matches not total-minus-candidates", %{conn: conn} do
create_narr_candidate()
@ -54,7 +54,7 @@ defmodule MicrowavepropWeb.StatusLiveTest do
# Insert a matching profile row — done should become 1
{:ok, _} =
Repo.insert(%Era5Profile{
Repo.insert(%NarrProfile{
valid_time: ~U[2010-06-15 18:00:00Z],
lat: 33.0,
lon: -97.0,