Rename ERA5 → NARR across the codebase

- Schema: Era5Profile → NarrProfile; table renamed via migration
  rename_era5_profiles_to_narr_profiles (ALTER TABLE + rename indexes,
  atomic and instant, no row rewrite)
- Weather helpers: find_nearest_era5/era5_for_contact/era5_profiles_for_path
  → find_nearest_narr/narr_for_contact/narr_profiles_for_path
- BackfillEnqueueWorker: :era5 → :narr type (virtual, no narr_status column);
  reconcile_stale_queued_to_unavailable and status_priority_order now skip
  virtual types via @virtual_types. Fixes the prod crash where the cron tried
  to update a non-existent era5_status column.
- ContactWeatherEnqueueWorker: build_era5_jobs → build_narr_jobs,
  era5_jobs_for_contact → narr_jobs_for_contact
- ContactLive: @era5 → @narr, maybe_enqueue_era5 → maybe_enqueue_narr;
  UI label "ERA5 (0.25°)" → "NARR (32 km)"
- Cron (runtime.exs): args types list now "narr" instead of "era5"
- /status: progress row, status-by-type card, totals.narr_profiles, and
  table-count lookup all target narr_profiles
- Drop obsolete Era5CdsJob schema + era5_cds_jobs table (inspection
  artifact from the retired CDS pipeline; 34 orphan rows in prod)
- Misc docstring/comment cleanups (skew_t, about, wgrib2, propagation_train)

Includes a regression test for the virtual-type crash.
This commit is contained in:
Graham McIntire 2026-04-16 09:22:23 -05:00
parent 9e94759f7a
commit 3604896726
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
22 changed files with 235 additions and 258 deletions

View file

@ -218,11 +218,11 @@ if config_env() == :prod do
# full grid-side JSONB write for. # full grid-side JSONB write for.
# {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker}, # {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker},
{"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker}, {"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
# The :era5 type dispatches NarrFetchWorker for pre-2014 contacts # The :narr type is virtual: it targets contacts with hrrr_status =
# (see ContactWeatherEnqueueWorker.era5_jobs_for_contact/1). The # :unavailable (pre-2014, missing from the HRRR archive) and dispatches
# type key is a historical artifact — the data source is NARR. # NarrFetchWorker against NCEI. See narr_jobs_for_contact/1.
{"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker, {"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker,
args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre", "era5"]}}, args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre", "narr"]}},
# UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch # UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch
{"30 1 * * *", CanadianSoundingFetchWorker}, {"30 1 * * *", CanadianSoundingFetchWorker},
{"30 13 * * *", CanadianSoundingFetchWorker}, {"30 13 * * *", CanadianSoundingFetchWorker},

View file

@ -5,13 +5,13 @@ defmodule Microwaveprop.Weather do
alias Microwaveprop.Propagation.ProfilesFile alias Microwaveprop.Propagation.ProfilesFile
alias Microwaveprop.Repo alias Microwaveprop.Repo
alias Microwaveprop.Weather.Era5Profile
alias Microwaveprop.Weather.GridCache alias Microwaveprop.Weather.GridCache
alias Microwaveprop.Weather.HrrrNativeProfile alias Microwaveprop.Weather.HrrrNativeProfile
alias Microwaveprop.Weather.HrrrProfile alias Microwaveprop.Weather.HrrrProfile
alias Microwaveprop.Weather.IemClient alias Microwaveprop.Weather.IemClient
alias Microwaveprop.Weather.IemreObservation alias Microwaveprop.Weather.IemreObservation
alias Microwaveprop.Weather.Metar5minObservation alias Microwaveprop.Weather.Metar5minObservation
alias Microwaveprop.Weather.NarrProfile
alias Microwaveprop.Weather.RtmaObservation alias Microwaveprop.Weather.RtmaObservation
alias Microwaveprop.Weather.SolarIndex alias Microwaveprop.Weather.SolarIndex
alias Microwaveprop.Weather.Sounding alias Microwaveprop.Weather.Sounding
@ -868,56 +868,56 @@ defmodule Microwaveprop.Weather do
@doc """ @doc """
Find the best available atmospheric profile for a contact. Find the best available atmospheric profile for a contact.
Tries HRRR first (3 km, hourly), falls back to ERA5 (0.25°, hourly). Tries HRRR first (3 km, hourly), falls back to NARR (32 km, 3-hourly).
Returns the profile struct or nil. Returns the profile struct or nil.
""" """
@spec best_profile_for_contact(map()) :: HrrrProfile.t() | Era5Profile.t() | nil @spec best_profile_for_contact(map()) :: HrrrProfile.t() | NarrProfile.t() | nil
def best_profile_for_contact(contact) do def best_profile_for_contact(contact) do
hrrr_for_contact(contact) || era5_for_contact(contact) hrrr_for_contact(contact) || narr_for_contact(contact)
end end
@doc "Find all atmospheric profiles along a contact's path, from any source." @doc "Find all atmospheric profiles along a contact's path, from any source."
@spec profiles_along_path(map()) :: [HrrrProfile.t() | Era5Profile.t()] @spec profiles_along_path(map()) :: [HrrrProfile.t() | NarrProfile.t()]
def profiles_along_path(contact) do def profiles_along_path(contact) do
hrrr_path = hrrr_profiles_for_path(contact) hrrr_path = hrrr_profiles_for_path(contact)
if hrrr_path == [] do if hrrr_path == [] do
era5_profiles_for_path(contact) narr_profiles_for_path(contact)
else else
hrrr_path hrrr_path
end end
end end
@spec era5_for_contact(map()) :: Era5Profile.t() | nil @spec narr_for_contact(map()) :: NarrProfile.t() | nil
def era5_for_contact(%{pos1: nil}), do: nil def narr_for_contact(%{pos1: nil}), do: nil
def era5_for_contact(contact) do def narr_for_contact(contact) do
lat = contact.pos1["lat"] lat = contact.pos1["lat"]
lon = contact.pos1["lon"] || contact.pos1["lng"] lon = contact.pos1["lon"] || contact.pos1["lng"]
if lat && lon do if lat && lon do
find_nearest_era5(lat, lon, contact.qso_timestamp) find_nearest_narr(lat, lon, contact.qso_timestamp)
end end
end end
@spec era5_profiles_for_path(map()) :: [Era5Profile.t()] @spec narr_profiles_for_path(map()) :: [NarrProfile.t()]
def era5_profiles_for_path(%{pos1: nil}), do: [] def narr_profiles_for_path(%{pos1: nil}), do: []
def era5_profiles_for_path(contact) do def narr_profiles_for_path(contact) do
contact contact
|> Microwaveprop.Radio.contact_path_points() |> Microwaveprop.Radio.contact_path_points()
|> Enum.map(fn {lat, lon} -> find_nearest_era5(lat, lon, contact.qso_timestamp) end) |> Enum.map(fn {lat, lon} -> find_nearest_narr(lat, lon, contact.qso_timestamp) end)
|> Enum.reject(&is_nil/1) |> Enum.reject(&is_nil/1)
end end
@spec find_nearest_era5(float(), float(), DateTime.t()) :: Era5Profile.t() | nil @spec find_nearest_narr(float(), float(), DateTime.t()) :: NarrProfile.t() | nil
def find_nearest_era5(lat, lon, timestamp) do def find_nearest_narr(lat, lon, timestamp) do
dlat = 0.15 dlat = 0.15
dlon = 0.15 dlon = 0.15
time_start = DateTime.add(timestamp, -1800, :second) time_start = DateTime.add(timestamp, -1800, :second)
time_end = DateTime.add(timestamp, 1800, :second) time_end = DateTime.add(timestamp, 1800, :second)
Era5Profile NarrProfile
|> where( |> where(
[p], [p],
p.lat >= ^(lat - dlat) and p.lat <= ^(lat + dlat) and p.lat >= ^(lat - dlat) and p.lat <= ^(lat + dlat) and

View file

@ -1,44 +0,0 @@
defmodule Microwaveprop.Weather.Era5CdsJob do
@moduledoc """
Tracks an in-flight ERA5 CDS request submitted by `Era5SubmitWorker` and
awaiting download+decode by `Era5PollWorker`. Persisting the CDS job IDs
decouples the Oban worker lifecycle from the 30+ minute CDS queue time,
so a pod restart resumes polling instead of re-submitting.
"""
use Ecto.Schema
import Ecto.Changeset
@primary_key {:id, :binary_id, autogenerate: true}
@foreign_key_type :binary_id
schema "era5_cds_jobs" do
field :year, :integer
field :month, :integer
field :tile_lat, :integer
field :tile_lon, :integer
field :single_job_id, :string
field :pressure_job_id, :string
field :submitted_at, :utc_datetime
field :last_polled_at, :utc_datetime
field :poll_count, :integer, default: 0
timestamps(type: :utc_datetime)
end
@type t :: %__MODULE__{}
@required_fields ~w(year month tile_lat tile_lon single_job_id pressure_job_id submitted_at)a
@optional_fields ~w(last_polled_at poll_count)a
@spec changeset(t() | Ecto.Changeset.t(), map()) :: Ecto.Changeset.t()
def changeset(job, attrs) do
job
|> cast(attrs, @required_fields ++ @optional_fields)
|> validate_required(@required_fields)
|> unique_constraint([:year, :month, :tile_lat, :tile_lon],
name: :era5_cds_jobs_year_month_tile_lat_tile_lon_index,
error_key: :tile_month
)
end
end

View file

@ -336,7 +336,7 @@ defmodule Microwaveprop.Weather.Grib2.Wgrib2 do
@doc """ @doc """
Like `extract_grid/3`, but returns a flat list of per-message results so Like `extract_grid/3`, but returns a flat list of per-message results so
the time dimension is preserved. Used when a single GRIB2 blob carries the time dimension is preserved. Used when a single GRIB2 blob carries
many timesteps (e.g. a month of ERA5 data fetched in one CDS request). many timesteps.
Each entry is `%{datetime: DateTime.t() | nil, var: String.t(), Each entry is `%{datetime: DateTime.t() | nil, var: String.t(),
level: String.t(), values: %{{lat, lon} => float}}`. level: String.t(), values: %{{lat, lon} => float}}`.
@ -354,7 +354,7 @@ defmodule Microwaveprop.Weather.Grib2.Wgrib2 do
| {:error, term()} | {:error, term()}
def extract_grid_messages(grib_binary, match_pattern, grid_spec) do def extract_grid_messages(grib_binary, match_pattern, grid_spec) do
if available?() do if available?() do
tmp_grib = Path.join(System.tmp_dir!(), "era5_#{System.unique_integer([:positive])}.grib2") tmp_grib = Path.join(System.tmp_dir!(), "wgrib2_#{System.unique_integer([:positive])}.grib2")
try do try do
File.write!(tmp_grib, grib_binary) File.write!(tmp_grib, grib_binary)

View file

@ -141,13 +141,13 @@ defmodule Microwaveprop.Weather.NarrClient do
end end
@doc """ @doc """
Extracts an `era5_profiles`-shaped attribute map from a composite NARR Extracts a NarrProfile-shaped attribute map from a composite NARR
GRIB1 file at the given lat/lon point. GRIB1 file at the given lat/lon point.
Pure-ish: shells out to `cdo -outputtab,name,lev,value -remapnn,lon=_lat=` Pure-ish: shells out to `cdo -outputtab,name,lev,value -remapnn,lon=_lat=`
on the local file. No network. on the local file. No network.
Returns `{:ok, attrs}` a map with the `era5_profiles` fields Returns `{:ok, attrs}` a map with the NarrProfile fields
(`:surface_temp_c`, `:surface_dewpoint_c`, `:surface_pressure_mb`, (`:surface_temp_c`, `:surface_dewpoint_c`, `:surface_pressure_mb`,
`:hpbl_m`, `:pwat_mm`, `:profile` list plus the derived fields from `:hpbl_m`, `:pwat_mm`, `:profile` list plus the derived fields from
`SoundingParams.derive/1`) or `{:error, reason}` on missing file, `SoundingParams.derive/1`) or `{:error, reason}` on missing file,
@ -170,8 +170,8 @@ defmodule Microwaveprop.Weather.NarrClient do
using `extract_profile_from_file/3`. using `extract_profile_from_file/3`.
Returns `{:ok, profile_attrs}` shaped for direct `Repo.insert_all/3` Returns `{:ok, profile_attrs}` shaped for direct `Repo.insert_all/3`
into the `era5_profiles` table, or `{:error, reason}`. Temp files are against `NarrProfile`, or `{:error, reason}`. Temp files are always
always cleaned up via `try/after`. cleaned up via `try/after`.
""" """
@spec fetch_profile_at(DateTime.t(), {float(), float()}) :: @spec fetch_profile_at(DateTime.t(), {float(), float()}) ::
{:ok, map()} | {:error, term()} {:ok, map()} | {:error, term()}

View file

@ -1,5 +1,13 @@
defmodule Microwaveprop.Weather.Era5Profile do defmodule Microwaveprop.Weather.NarrProfile do
@moduledoc false @moduledoc """
Profile rows fetched from NCEP NARR (North American Regional Reanalysis)
for pre-2014 contacts where the HRRR archive doesn't reach.
The table was originally created as `era5_profiles` for the retired
ERA5/CDS pipeline and reused 1:1 by NARR; the
`rename_era5_profiles_to_narr_profiles` migration renames it to
`narr_profiles`.
"""
use Ecto.Schema use Ecto.Schema
import Ecto.Changeset import Ecto.Changeset
@ -7,7 +15,7 @@ defmodule Microwaveprop.Weather.Era5Profile do
@primary_key {:id, :binary_id, autogenerate: true} @primary_key {:id, :binary_id, autogenerate: true}
@foreign_key_type :binary_id @foreign_key_type :binary_id
schema "era5_profiles" do schema "narr_profiles" do
field :valid_time, :utc_datetime field :valid_time, :utc_datetime
field :lat, :float field :lat, :float
field :lon, :float field :lon, :float

View file

@ -14,7 +14,12 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
require Logger require Logger
@enrichable [:pending, :queued, :failed] @enrichable [:pending, :queued, :failed]
@valid_types ~w(hrrr weather terrain iemre era5) @valid_types ~w(hrrr weather terrain iemre narr)
# Virtual types have no <type>_status column on contacts; they're synthesized
# from other status fields (e.g. :narr candidates are `hrrr_status = :unavailable`).
# Skip these in any reconcile / ordering pass that reads a <type>_status field.
@virtual_types [:narr]
# A :queued contact whose updated_at is older than this is one the cron has # A :queued contact whose updated_at is older than this is one the cron has
# already tried ~144 times over 3 days without the underlying worker landing # already tried ~144 times over 3 days without the underlying worker landing
@ -67,30 +72,45 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
types |> Enum.filter(&(&1 in @valid_types)) |> Enum.map(&String.to_existing_atom/1) types |> Enum.filter(&(&1 in @valid_types)) |> Enum.map(&String.to_existing_atom/1)
end end
defp parse_types(_), do: [:hrrr, :weather, :terrain, :iemre, :era5] defp parse_types(_), do: [:hrrr, :weather, :terrain, :iemre, :narr]
defp status_priority_order(types) do defp status_priority_order(types) do
# Prioritize pending/failed over queued so real work happens first # Prioritize pending/failed over queued so real work happens first.
field = :"#{hd(types)}_status" # Virtual types (e.g. :narr) have no *_status column — fall back to
# time-only ordering when every requested type is virtual.
case Enum.find(types, &(&1 not in @virtual_types)) do
nil ->
[desc: dynamic([c], c.qso_timestamp)]
[ type ->
asc: dynamic([c], fragment("CASE ? WHEN 'pending' THEN 0 WHEN 'failed' THEN 1 ELSE 2 END", field(c, ^field))), field = :"#{type}_status"
desc: dynamic([c], c.qso_timestamp)
] [
asc:
dynamic(
[c],
fragment("CASE ? WHEN 'pending' THEN 0 WHEN 'failed' THEN 1 ELSE 2 END", field(c, ^field))
),
desc: dynamic([c], c.qso_timestamp)
]
end
end end
# Flip contacts that have been stuck in :queued for longer than # Flip contacts that have been stuck in :queued for longer than
# @stale_queued_cutoff_seconds to :unavailable. One Repo.update_all per type, # @stale_queued_cutoff_seconds to :unavailable. One Repo.update_all per type,
# scoped to only the types passed in args so a partial cron (e.g. hrrr-only) # scoped to only the types passed in args so a partial cron (e.g. hrrr-only)
# doesn't touch other statuses. Fast: hits the (<field>, updated_at) combo # doesn't touch other statuses. Virtual types (e.g. :narr) have no *_status
# with a simple range scan. # column and are skipped. Fast: hits the (<field>, updated_at) combo with a
# simple range scan.
defp reconcile_stale_queued_to_unavailable(types) do defp reconcile_stale_queued_to_unavailable(types) do
cutoff = cutoff =
DateTime.utc_now() DateTime.utc_now()
|> DateTime.add(-@stale_queued_cutoff_seconds, :second) |> DateTime.add(-@stale_queued_cutoff_seconds, :second)
|> DateTime.truncate(:second) |> DateTime.truncate(:second)
Enum.reduce(types, 0, fn type, acc -> types
|> Enum.reject(&(&1 in @virtual_types))
|> Enum.reduce(0, fn type, acc ->
field = :"#{type}_status" field = :"#{type}_status"
{n, _} = {n, _} =
@ -126,8 +146,8 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
defp type_filter(types) do defp type_filter(types) do
Enum.reduce(types, dynamic(false), fn Enum.reduce(types, dynamic(false), fn
:era5, acc -> :narr, acc ->
# ERA5 targets contacts where HRRR is unavailable (pre-2014, missing from archive) # NARR targets contacts where HRRR is unavailable (pre-2014, missing from archive)
dynamic([c], ^acc or c.hrrr_status == :unavailable) dynamic([c], ^acc or c.hrrr_status == :unavailable)
type, acc -> type, acc ->

View file

@ -33,9 +33,9 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
insert_all_chunked(bulk_jobs) insert_all_chunked(bulk_jobs)
end end
# ERA5 jobs must go through Oban.insert/1 so the worker's unique # NARR jobs must go through Oban.insert/1 so the worker's unique
# constraint is honored; Oban OSS insert_all does not check uniqueness. # constraint is honored; Oban OSS insert_all does not check uniqueness.
insert_unique(jobs_by_type[:era5]) insert_unique(jobs_by_type[:narr])
mark_enrichment_statuses(contact, types, jobs_by_type) mark_enrichment_statuses(contact, types, jobs_by_type)
@ -48,7 +48,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
hrrr: if(:hrrr in types, do: build_hrrr_jobs([contact]), else: []), hrrr: if(:hrrr in types, do: build_hrrr_jobs([contact]), else: []),
terrain: if(:terrain in types, do: build_terrain_jobs([contact]), else: []), terrain: if(:terrain in types, do: build_terrain_jobs([contact]), else: []),
iemre: if(:iemre in types, do: build_iemre_jobs([contact]), else: []), iemre: if(:iemre in types, do: build_iemre_jobs([contact]), else: []),
era5: if(:era5 in types, do: build_era5_jobs([contact]), else: []) narr: if(:narr in types, do: build_narr_jobs([contact]), else: [])
} }
end end
@ -173,18 +173,15 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
|> Enum.uniq_by(fn changeset -> changeset.changes.args end) |> Enum.uniq_by(fn changeset -> changeset.changes.args end)
end end
def build_era5_jobs(contacts) do def build_narr_jobs(contacts) do
contacts contacts
|> Enum.flat_map(&era5_jobs_for_contact/1) |> Enum.flat_map(&narr_jobs_for_contact/1)
|> Enum.uniq_by(fn changeset -> changeset.changes.args end) |> Enum.uniq_by(fn changeset -> changeset.changes.args end)
end end
# The function name is a historical artifact: the `:era5` type key still defp narr_jobs_for_contact(%{pos1: nil}), do: []
# refers to the `era5_profiles` table (reused 1:1), but this now dispatches
# `NarrFetchWorker` against NCEP NARR — the CDS/ERA5 pipeline is gone.
defp era5_jobs_for_contact(%{pos1: nil}), do: []
defp era5_jobs_for_contact(contact) do defp narr_jobs_for_contact(contact) do
# NARR analyses are 3-hourly (00/03/06/09/12/15/18/21 UTC). NarrFetchWorker # NARR analyses are 3-hourly (00/03/06/09/12/15/18/21 UTC). NarrFetchWorker
# rejects anything else, so snap here in the caller. # rejects anything else, so snap here in the caller.
valid_time = NarrClient.snap_to_analysis_hour(contact.qso_timestamp) valid_time = NarrClient.snap_to_analysis_hour(contact.qso_timestamp)
@ -195,7 +192,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
rlat = Float.round(lat * 4) / 4 rlat = Float.round(lat * 4) / 4
rlon = Float.round(lon * 4) / 4 rlon = Float.round(lon * 4) / 4
if Weather.find_nearest_era5(rlat, rlon, valid_time) do if Weather.find_nearest_narr(rlat, rlon, valid_time) do
[] []
else else
[ [

View file

@ -1,12 +1,12 @@
defmodule Microwaveprop.Workers.NarrFetchWorker do defmodule Microwaveprop.Workers.NarrFetchWorker do
@moduledoc """ @moduledoc """
Fetches one NCEP NARR profile from NCEI via `NarrClient.fetch_profile_at/2` Fetches one NCEP NARR profile from NCEI via `NarrClient.fetch_profile_at/2`
and inserts it into the `era5_profiles` table. and inserts it as a `NarrProfile` row.
Replaces `Era5FetchWorker` for pre-2014 contacts. Unlike the ERA5 router, Covers pre-2014 contacts where the HRRR archive doesn't reach. Replaces
NARR fetches are per-point (one analysis hour, one location), so this the retired ERA5/CDS pipeline. NARR fetches are per-point (one analysis
worker does the fetch + insert directly no downstream batch worker, hour, one location), so this worker does the fetch + insert directly
no routing. no downstream batch worker, no routing.
Jobs are unique on `{lat, lon, valid_time}` so duplicate enrichment Jobs are unique on `{lat, lon, valid_time}` so duplicate enrichment
requests collapse. `valid_time` MUST already be snapped to a NARR requests collapse. `valid_time` MUST already be snapped to a NARR
@ -15,8 +15,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorker do
`NarrClient.url_for/1`. `NarrClient.url_for/1`.
See `docs/plans/2026-04-15-merra2-historical-backfill.md` for the full See `docs/plans/2026-04-15-merra2-historical-backfill.md` for the full
architecture. The `era5_profiles` table is reused 1:1 (rename is a architecture.
follow-up).
""" """
use Oban.Worker, use Oban.Worker,
queue: :narr, queue: :narr,
@ -29,8 +28,8 @@ defmodule Microwaveprop.Workers.NarrFetchWorker do
import Ecto.Query import Ecto.Query
alias Microwaveprop.Repo alias Microwaveprop.Repo
alias Microwaveprop.Weather.Era5Profile
alias Microwaveprop.Weather.NarrClient alias Microwaveprop.Weather.NarrClient
alias Microwaveprop.Weather.NarrProfile
require Logger require Logger
@ -41,7 +40,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorker do
rlat = Float.round(lat * 4) / 4 rlat = Float.round(lat * 4) / 4
rlon = Float.round(lon * 4) / 4 rlon = Float.round(lon * 4) / 4
if has_era5_profile?(rlat, rlon, valid_time) do if has_narr_profile?(rlat, rlon, valid_time) do
:ok :ok
else else
fetch_and_insert(rlat, rlon, valid_time) fetch_and_insert(rlat, rlon, valid_time)
@ -74,22 +73,21 @@ defmodule Microwaveprop.Workers.NarrFetchWorker do
|> Map.put(:inserted_at, now) |> Map.put(:inserted_at, now)
|> Map.put(:updated_at, now) |> Map.put(:updated_at, now)
Repo.insert_all(Era5Profile, [row], Repo.insert_all(NarrProfile, [row],
on_conflict: :nothing, on_conflict: :nothing,
conflict_target: [:lat, :lon, :valid_time] conflict_target: [:lat, :lon, :valid_time]
) )
end end
defp has_era5_profile?(lat, lon, valid_time) do defp has_narr_profile?(lat, lon, valid_time) do
# NARR is exact 3-hourly, so we tighten the time window to ±15 minutes # NARR is exact 3-hourly, so the time window can be tight (±15 min).
# compared to Era5FetchWorker's ±30-minute check. Lat/lon tolerance # Lat/lon tolerance matches the 0.25° dedup granularity.
# matches the 0.25° dedup granularity.
dlat = 0.13 dlat = 0.13
dlon = 0.13 dlon = 0.13
time_start = DateTime.add(valid_time, -900, :second) time_start = DateTime.add(valid_time, -900, :second)
time_end = DateTime.add(valid_time, 900, :second) time_end = DateTime.add(valid_time, 900, :second)
Era5Profile NarrProfile
|> where( |> where(
[p], [p],
p.lat >= ^(lat - dlat) and p.lat <= ^(lat + dlat) and p.lat >= ^(lat - dlat) and p.lat <= ^(lat + dlat) and

View file

@ -172,7 +172,7 @@ defmodule MicrowavepropWeb.AboutLive do
forecasts), Iowa Environmental Mesonet (ASOS and upper-air forecasts), Iowa Environmental Mesonet (ASOS and upper-air
soundings), IEMRE gridded reanalysis, SRTM 90 m terrain, SNMP soundings), IEMRE gridded reanalysis, SRTM 90 m terrain, SNMP
polling on seven commercial microwave links near DFW at 5-minute polling on seven commercial microwave links near DFW at 5-minute
intervals, and Copernicus ERA5 for pre-2014 contact enrichment. intervals, and NCEP NARR for pre-2014 contact enrichment.
</li> </li>
<li> <li>
There's Nx/Axon/EXLA scaffolding for a small feed-forward model There's Nx/Axon/EXLA scaffolding for a small feed-forward model

View file

@ -46,8 +46,8 @@ defmodule MicrowavepropWeb.ContactLive.Show do
hrrr: nil, hrrr: nil,
hrrr_path: [], hrrr_path: [],
native_profile: nil, native_profile: nil,
era5: nil, narr: nil,
era5_path: [], narr_path: [],
iemre: nil, iemre: nil,
terrain: nil, terrain: nil,
elevation_profile: nil, elevation_profile: nil,
@ -91,7 +91,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
|> start_async(:weather, fn -> load_weather(contact) end) |> start_async(:weather, fn -> load_weather(contact) end)
|> start_async(:solar, fn -> load_solar(contact) end) |> start_async(:solar, fn -> load_solar(contact) end)
|> start_async(:hrrr_path, fn -> Weather.hrrr_profiles_for_path(contact) end) |> start_async(:hrrr_path, fn -> Weather.hrrr_profiles_for_path(contact) end)
|> start_async(:era5_path, fn -> Weather.era5_profiles_for_path(contact) end) |> start_async(:narr_path, fn -> Weather.narr_profiles_for_path(contact) end)
|> start_async(:native_profile, fn -> load_native_profile(contact) end) |> start_async(:native_profile, fn -> load_native_profile(contact) end)
|> start_async(:terrain, fn -> Terrain.get_terrain_profile(contact.id) end) |> start_async(:terrain, fn -> Terrain.get_terrain_profile(contact.id) end)
|> start_async(:iemre, fn -> load_iemre(contact) end) |> start_async(:iemre, fn -> load_iemre(contact) end)
@ -328,14 +328,14 @@ defmodule MicrowavepropWeb.ContactLive.Show do
{:noreply, socket} {:noreply, socket}
end end
def handle_async(:era5_path, {:ok, era5_path}, socket) do def handle_async(:narr_path, {:ok, narr_path}, socket) do
era5 = List.first(era5_path) narr = List.first(narr_path)
contact = socket.assigns.contact contact = socket.assigns.contact
contact = contact =
if socket.assigns.can_enqueue, do: maybe_enqueue_era5(era5, contact), else: contact if socket.assigns.can_enqueue, do: maybe_enqueue_narr(narr, contact), else: contact
{:noreply, assign(socket, contact: contact, era5: era5, era5_path: era5_path)} {:noreply, assign(socket, contact: contact, narr: narr, narr_path: narr_path)}
end end
def handle_async(:terrain, {:ok, terrain}, socket) do def handle_async(:terrain, {:ok, terrain}, socket) do
@ -361,7 +361,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
end end
def handle_async(slot, {:exit, reason}, socket) def handle_async(slot, {:exit, reason}, socket)
when slot in [:weather, :solar, :hrrr_path, :era5_path, :native_profile, :terrain, :iemre] do when slot in [:weather, :solar, :hrrr_path, :narr_path, :native_profile, :terrain, :iemre] do
Logger.warning("contact hydrate task #{slot} exited: #{inspect(reason)}") Logger.warning("contact hydrate task #{slot} exited: #{inspect(reason)}")
{:noreply, socket} {:noreply, socket}
end end
@ -661,13 +661,12 @@ defmodule MicrowavepropWeb.ContactLive.Show do
end end
end end
# ERA5 is a fallback for contacts where HRRR is unavailable (pre-2014 or # NARR is the fallback for contacts where HRRR is unavailable (pre-2014 or
# missing from the archive). Only enqueue when HRRR is known-unavailable AND # missing from the archive). Only enqueue when HRRR is known-unavailable AND
# we don't already have an ERA5 profile — otherwise we'd waste CDS quota on # we don't already have a NARR profile.
# contacts the HRRR path will cover. defp maybe_enqueue_narr(narr, contact) when not is_nil(narr), do: contact
defp maybe_enqueue_era5(era5, contact) when not is_nil(era5), do: contact
defp maybe_enqueue_era5(nil, %{hrrr_status: :unavailable} = contact) do defp maybe_enqueue_narr(nil, %{hrrr_status: :unavailable} = contact) do
lat = contact.pos1 && contact.pos1["lat"] lat = contact.pos1 && contact.pos1["lat"]
lon = contact.pos1 && (contact.pos1["lon"] || contact.pos1["lng"]) lon = contact.pos1 && (contact.pos1["lon"] || contact.pos1["lng"])
@ -686,7 +685,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
contact contact
end end
defp maybe_enqueue_era5(_nil, contact), do: contact defp maybe_enqueue_narr(_nil, contact), do: contact
defp queue_info(counts, queue) do defp queue_info(counts, queue) do
case Map.get(counts, queue, 0) do case Map.get(counts, queue, 0) do
@ -1140,11 +1139,11 @@ defmodule MicrowavepropWeb.ContactLive.Show do
<div class="divider" /> <div class="divider" />
<h2 class="text-base font-semibold mb-2">Atmospheric Profile</h2> <h2 class="text-base font-semibold mb-2">Atmospheric Profile</h2>
<% profile = @hrrr || @era5 %> <% profile = @hrrr || @narr %>
<% profile_source = <% profile_source =
cond do cond do
@hrrr -> "HRRR (3 km)" @hrrr -> "HRRR (3 km)"
@era5 -> "ERA5 (0.25°)" @narr -> "NARR (32 km)"
true -> nil true -> nil
end %> end %>
<% skew_t_levels = <% skew_t_levels =
@ -1253,7 +1252,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
<% :unavailable -> %> <% :unavailable -> %>
<span class="flex items-center gap-2"> <span class="flex items-center gap-2">
<span class="loading loading-spinner loading-sm"></span> <span class="loading loading-spinner loading-sm"></span>
HRRR unavailable fetching ERA5 reanalysis {queue_info(@queue_counts, "era5_batch")} HRRR unavailable fetching NARR reanalysis {queue_info(@queue_counts, "narr")}
</span> </span>
<% _ -> %> <% _ -> %>
No atmospheric profile available. No atmospheric profile available.

View file

@ -97,9 +97,8 @@ defmodule MicrowavepropWeb.StatusLive do
end end
# NARR is the fallback for pre-2014 contacts (HRRR archive starts 2014). # NARR is the fallback for pre-2014 contacts (HRRR archive starts 2014).
# Profiles land in the era5_profiles table (kept 1:1 after the ERA5/CDS # "Done" means a narr_profiles row within 0.15° + 30 min of the contact,
# pipeline was replaced). "Done" means a row within 0.15° + 30 min of the # matching `Weather.find_nearest_narr/3` lookup tolerance.
# contact, matching `Weather.find_nearest_era5/3` lookup tolerance.
defp count_narr_done do defp count_narr_done do
%{rows: [[done]]} = %{rows: [[done]]} =
Repo.query!(""" Repo.query!("""
@ -109,7 +108,7 @@ defmodule MicrowavepropWeb.StatusLive do
AND c.pos1 IS NOT NULL AND c.pos1 IS NOT NULL
AND EXISTS ( AND EXISTS (
SELECT 1 SELECT 1
FROM era5_profiles p FROM narr_profiles p
WHERE p.lat BETWEEN ((c.pos1->>'lat')::float - 0.15) AND ((c.pos1->>'lat')::float + 0.15) WHERE p.lat BETWEEN ((c.pos1->>'lat')::float - 0.15) AND ((c.pos1->>'lat')::float + 0.15)
AND p.lon BETWEEN ((c.pos1->>'lon')::float - 0.15) AND ((c.pos1->>'lon')::float + 0.15) AND p.lon BETWEEN ((c.pos1->>'lon')::float - 0.15) AND ((c.pos1->>'lon')::float + 0.15)
AND p.valid_time BETWEEN (c.qso_timestamp - INTERVAL '30 minutes') AND (c.qso_timestamp + INTERVAL '30 minutes') AND p.valid_time BETWEEN (c.qso_timestamp - INTERVAL '30 minutes') AND (c.qso_timestamp + INTERVAL '30 minutes')
@ -264,8 +263,8 @@ defmodule MicrowavepropWeb.StatusLive do
table_counts = Map.new(tables, fn %{table: t, rows: r} -> {t, r} end) table_counts = Map.new(tables, fn %{table: t, rows: r} -> {t, r} end)
# NARR doesn't have its own status column — synthesize from HRRR unavailable # NARR doesn't have its own status column — synthesize from HRRR unavailable
# + era5_profiles row count (the table is reused as the NARR landing spot). # + narr_profiles row count.
narr_count = Map.get(table_counts, "era5_profiles", 0) narr_count = Map.get(table_counts, "narr_profiles", 0)
hrrr_unavailable = hrrr_unavailable =
statuses |> Map.get("hrrr", []) |> Enum.find_value(0, fn {s, c} -> if s == "unavailable", do: c end) statuses |> Map.get("hrrr", []) |> Enum.find_value(0, fn {s, c} -> if s == "unavailable", do: c end)
@ -289,7 +288,7 @@ defmodule MicrowavepropWeb.StatusLive do
statuses: statuses, statuses: statuses,
totals: %{ totals: %{
hrrr_profiles: hrrr_count, hrrr_profiles: hrrr_count,
narr_profiles: Map.get(table_counts, "era5_profiles", 0), narr_profiles: Map.get(table_counts, "narr_profiles", 0),
terrain_profiles: terrain_count, terrain_profiles: terrain_count,
surface_observations: obs_count, surface_observations: obs_count,
soundings: sounding_count, soundings: sounding_count,

View file

@ -3,7 +3,7 @@ defmodule MicrowavepropWeb.SkewT do
Skew-T log-P diagram renderer. Builds the full set of SVG Skew-T log-P diagram renderer. Builds the full set of SVG
primitives (isobars, isotherms, dry adiabats, saturation mixing primitives (isobars, isotherms, dry adiabats, saturation mixing
ratio lines, and temperature/dewpoint traces) for a single HRRR ratio lines, and temperature/dewpoint traces) for a single HRRR
or ERA5 pressure-level profile so the contact detail template can or NARR pressure-level profile so the contact detail template can
draw it declaratively. draw it declaratively.
## Meteorology ## Meteorology

View file

@ -59,7 +59,7 @@ defmodule Mix.Tasks.PropagationTrain do
) )
# Boot the app without starting Oban queues so training doesn't have # Boot the app without starting Oban queues so training doesn't have
# the HRRR / ERA5 / weather cron firing in the background against the # the HRRR / NARR / weather cron firing in the background against the
# local DB we're reading from. # local DB we're reading from.
Application.put_env( Application.put_env(
:microwaveprop, :microwaveprop,

View file

@ -0,0 +1,27 @@
defmodule Microwaveprop.Repo.Migrations.RenameEra5ProfilesToNarrProfiles do
use Ecto.Migration
# ALTER TABLE ... RENAME is atomic and instant (no row rewrite). User-created
# indexes keep their old names after the table rename, so we rename them too
# for consistency. Implicit PK index / sequence are renamed automatically by
# PostgreSQL.
def up do
rename table(:era5_profiles), to: table(:narr_profiles)
execute "ALTER INDEX era5_profiles_lat_lon_valid_time_index RENAME TO narr_profiles_lat_lon_valid_time_index"
execute "ALTER INDEX era5_profiles_valid_time_index RENAME TO narr_profiles_valid_time_index"
execute "ALTER INDEX era5_profiles_valid_time_lat_lon_index RENAME TO narr_profiles_valid_time_lat_lon_index"
end
def down do
execute "ALTER INDEX narr_profiles_valid_time_lat_lon_index RENAME TO era5_profiles_valid_time_lat_lon_index"
execute "ALTER INDEX narr_profiles_valid_time_index RENAME TO era5_profiles_valid_time_index"
execute "ALTER INDEX narr_profiles_lat_lon_valid_time_index RENAME TO era5_profiles_lat_lon_valid_time_index"
rename table(:narr_profiles), to: table(:era5_profiles)
end
end

View file

@ -0,0 +1,29 @@
defmodule Microwaveprop.Repo.Migrations.DropEra5CdsJobs do
use Ecto.Migration
# The ERA5/CDS pipeline was retired in favor of NARR (see
# rename_era5_profiles_to_narr_profiles). Orphan rows in this table are
# useless without the workers that wrote them.
def up do
drop table(:era5_cds_jobs)
end
def down do
create table(:era5_cds_jobs, primary_key: false) do
add :id, :binary_id, primary_key: true
add :year, :integer, null: false
add :month, :integer, null: false
add :tile_lat, :integer, null: false
add :tile_lon, :integer, null: false
add :single_job_id, :string, null: false
add :pressure_job_id, :string, null: false
add :submitted_at, :utc_datetime, null: false
add :last_polled_at, :utc_datetime
add :poll_count, :integer, default: 0, null: false
timestamps(type: :utc_datetime)
end
create unique_index(:era5_cds_jobs, [:year, :month, :tile_lat, :tile_lon])
end
end

View file

@ -1,82 +0,0 @@
defmodule Microwaveprop.Weather.Era5CdsJobTest do
use Microwaveprop.DataCase, async: true
alias Microwaveprop.Weather.Era5CdsJob
describe "changeset/2" do
@valid_attrs %{
year: 2014,
month: 3,
tile_lat: 32,
tile_lon: -98,
single_job_id: "abc-single",
pressure_job_id: "abc-pressure",
submitted_at: ~U[2026-04-13 21:00:00Z]
}
test "accepts a complete attrs map" do
changeset = Era5CdsJob.changeset(%Era5CdsJob{}, @valid_attrs)
assert changeset.valid?
end
test "requires year, month, tile coordinates, and both CDS job IDs" do
changeset = Era5CdsJob.changeset(%Era5CdsJob{}, %{})
refute changeset.valid?
required = ~w(year month tile_lat tile_lon single_job_id pressure_job_id submitted_at)a
errors = Keyword.keys(changeset.errors)
for field <- required do
assert field in errors, "expected #{inspect(field)} to be required"
end
end
test "last_polled_at and poll_count default to nil/0 and can be updated" do
{:ok, row} =
%Era5CdsJob{}
|> Era5CdsJob.changeset(@valid_attrs)
|> Repo.insert()
assert row.poll_count == 0
assert is_nil(row.last_polled_at)
now = DateTime.truncate(DateTime.utc_now(), :second)
{:ok, bumped} =
row
|> Era5CdsJob.changeset(%{last_polled_at: now, poll_count: 1})
|> Repo.update()
assert bumped.poll_count == 1
assert bumped.last_polled_at == now
end
end
describe "uniqueness" do
test "collapses duplicate (year, month, tile_lat, tile_lon) inserts" do
attrs = %{
year: 2014,
month: 3,
tile_lat: 32,
tile_lon: -98,
single_job_id: "first-single",
pressure_job_id: "first-pressure",
submitted_at: ~U[2026-04-13 21:00:00Z]
}
{:ok, _} =
%Era5CdsJob{}
|> Era5CdsJob.changeset(attrs)
|> Repo.insert()
{:error, changeset} =
%Era5CdsJob{}
|> Era5CdsJob.changeset(%{attrs | single_job_id: "second-single", pressure_job_id: "second-pressure"})
|> Repo.insert()
refute changeset.valid?
assert {:tile_month, _} = List.keyfind(changeset.errors, :tile_month, 0) || {:tile_month, nil}
end
end
end

View file

@ -5,6 +5,7 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorkerTest do
alias Microwaveprop.Terrain.ElevationClient alias Microwaveprop.Terrain.ElevationClient
alias Microwaveprop.Weather.HrrrClient alias Microwaveprop.Weather.HrrrClient
alias Microwaveprop.Weather.IemClient alias Microwaveprop.Weather.IemClient
alias Microwaveprop.Weather.NarrClient
alias Microwaveprop.Workers.BackfillEnqueueWorker alias Microwaveprop.Workers.BackfillEnqueueWorker
defp create_contact(attrs \\ %{}) do defp create_contact(attrs \\ %{}) do
@ -53,12 +54,19 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorkerTest do
Req.Test.json(conn, %{"elevation" => List.duplicate(200.0, lat_count)}) Req.Test.json(conn, %{"elevation" => List.duplicate(200.0, lat_count)})
end) end)
# NARR fetches land in era5_profiles; stub returns 500 so inline jobs fail
# gracefully instead of hitting NCEI.
Req.Test.stub(NarrClient, fn conn ->
Plug.Conn.send_resp(conn, 500, "stub")
end)
:ok :ok
end end
# ERA5 goes through Copernicus CDS API which is async (submit → poll → download) # NARR fetches (the :era5 type) run inline via testing: :inline — the
# and has no Req.Test plug hook, so inline Oban would block on Process.sleep. # NarrClient Req.Test stub above keeps them from hitting NCEI. Most tests
# These tests exercise the non-ERA5 enrichment path; ERA5 has its own coverage. # below target the other pipelines and use this narrowed type list to keep
# the test surface tight.
@non_era5_types ["hrrr", "weather", "terrain", "iemre"] @non_era5_types ["hrrr", "weather", "terrain", "iemre"]
describe "perform/1" do describe "perform/1" do
@ -113,6 +121,24 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorkerTest do
assert_receive {:enqueue_complete, 1} assert_receive {:enqueue_complete, 1}
end end
# :narr is a virtual type (no narr_status column) — it targets contacts
# whose hrrr_status is :unavailable. BackfillEnqueueWorker must not try
# to reconcile a non-existent narr_status column or use it for priority
# ordering.
test "handles virtual :narr type without referencing non-existent narr_status column" do
# Pre-2014 contact (HRRR unavailable) — NARR candidate.
contact = create_contact(%{qso_timestamp: ~U[2010-01-01 00:00:00Z]})
contact
|> Ecto.Changeset.change(%{hrrr_status: :unavailable})
|> Repo.update!()
assert :ok =
BackfillEnqueueWorker.perform(%Oban.Job{
args: %{"limit" => 10, "types" => ["narr"]}
})
end
end end
describe "reconcile stuck queued → unavailable" do describe "reconcile stuck queued → unavailable" do

View file

@ -4,9 +4,9 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
alias Microwaveprop.Radio.Contact alias Microwaveprop.Radio.Contact
alias Microwaveprop.Terrain.ElevationClient alias Microwaveprop.Terrain.ElevationClient
alias Microwaveprop.Weather alias Microwaveprop.Weather
alias Microwaveprop.Weather.Era5Profile
alias Microwaveprop.Weather.HrrrClient alias Microwaveprop.Weather.HrrrClient
alias Microwaveprop.Weather.IemClient alias Microwaveprop.Weather.IemClient
alias Microwaveprop.Weather.NarrProfile
alias Microwaveprop.Workers.ContactWeatherEnqueueWorker alias Microwaveprop.Workers.ContactWeatherEnqueueWorker
@qso_attrs %{ @qso_attrs %{
@ -562,7 +562,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
end end
end end
describe "build_era5_jobs/1" do describe "build_narr_jobs/1" do
test "builds NarrFetchWorker jobs for each rounded path point" do test "builds NarrFetchWorker jobs for each rounded path point" do
contact = contact =
create_contact(%{ create_contact(%{
@ -571,7 +571,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
pos2: nil pos2: nil
}) })
jobs = ContactWeatherEnqueueWorker.build_era5_jobs([contact]) jobs = ContactWeatherEnqueueWorker.build_narr_jobs([contact])
assert length(jobs) == 1 assert length(jobs) == 1
job = hd(jobs) job = hd(jobs)
@ -589,7 +589,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
pos2: nil pos2: nil
}) })
jobs = ContactWeatherEnqueueWorker.build_era5_jobs([contact]) jobs = ContactWeatherEnqueueWorker.build_narr_jobs([contact])
job = hd(jobs) job = hd(jobs)
# 13:42 UTC snaps down to the 12:00 UTC NARR analysis slot. # 13:42 UTC snaps down to the 12:00 UTC NARR analysis slot.
@ -604,14 +604,14 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
pos2: %{"lat" => 30.3, "lon" => -97.7} pos2: %{"lat" => 30.3, "lon" => -97.7}
}) })
jobs = ContactWeatherEnqueueWorker.build_era5_jobs([contact]) jobs = ContactWeatherEnqueueWorker.build_narr_jobs([contact])
# pos1, midpoint, pos2 round to three distinct 0.25° grid cells. # pos1, midpoint, pos2 round to three distinct 0.25° grid cells.
assert length(jobs) == 3 assert length(jobs) == 3
assert Enum.all?(jobs, &(&1.changes.worker == "Microwaveprop.Workers.NarrFetchWorker")) assert Enum.all?(jobs, &(&1.changes.worker == "Microwaveprop.Workers.NarrFetchWorker"))
end end
test "skips path points that already have an era5_profile row" do test "skips path points that already have a narr_profile row" do
contact = contact =
create_contact(%{ create_contact(%{
qso_timestamp: ~U[2010-06-15 13:42:00Z], qso_timestamp: ~U[2010-06-15 13:42:00Z],
@ -620,8 +620,8 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
}) })
# pos1 (32.907, -97.038) snaps to the (33.0, -97.0) 0.25° grid cell # pos1 (32.907, -97.038) snaps to the (33.0, -97.0) 0.25° grid cell
%Era5Profile{} %NarrProfile{}
|> Era5Profile.changeset(%{ |> NarrProfile.changeset(%{
lat: 33.0, lat: 33.0,
lon: -97.0, lon: -97.0,
valid_time: ~U[2010-06-15 12:00:00Z], valid_time: ~U[2010-06-15 12:00:00Z],
@ -629,13 +629,13 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
}) })
|> Repo.insert!() |> Repo.insert!()
jobs = ContactWeatherEnqueueWorker.build_era5_jobs([contact]) jobs = ContactWeatherEnqueueWorker.build_narr_jobs([contact])
assert jobs == [] assert jobs == []
end end
test "skips QSOs without pos1" do test "skips QSOs without pos1" do
contact = create_contact(%{pos1: nil}) contact = create_contact(%{pos1: nil})
assert ContactWeatherEnqueueWorker.build_era5_jobs([contact]) == [] assert ContactWeatherEnqueueWorker.build_narr_jobs([contact]) == []
end end
test "deduplicates jobs with identical args across contacts" do test "deduplicates jobs with identical args across contacts" do
@ -655,7 +655,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
pos2: nil pos2: nil
}) })
jobs = ContactWeatherEnqueueWorker.build_era5_jobs([q1, q2]) jobs = ContactWeatherEnqueueWorker.build_narr_jobs([q1, q2])
# Both snap to 12:00Z and round to the same 0.25° grid cell → 1 job. # Both snap to 12:00Z and round to the same 0.25° grid cell → 1 job.
assert length(jobs) == 1 assert length(jobs) == 1

View file

@ -2,8 +2,8 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
use Microwaveprop.DataCase, async: false use Microwaveprop.DataCase, async: false
use Oban.Testing, repo: Microwaveprop.Repo use Oban.Testing, repo: Microwaveprop.Repo
alias Microwaveprop.Weather.Era5Profile
alias Microwaveprop.Weather.NarrClient alias Microwaveprop.Weather.NarrClient
alias Microwaveprop.Weather.NarrProfile
alias Microwaveprop.Workers.NarrFetchWorker alias Microwaveprop.Workers.NarrFetchWorker
@inv_fixture "test/fixtures/narr/narr-a_221_20100615_1200_000.inv" @inv_fixture "test/fixtures/narr/narr-a_221_20100615_1200_000.inv"
@ -36,7 +36,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
end end
describe "perform/1" do describe "perform/1" do
test "fetches a profile via NarrClient and inserts it into era5_profiles" do test "fetches a profile via NarrClient and inserts it as a NarrProfile" do
stub_narr_success() stub_narr_success()
valid_time = ~U[2010-06-15 12:00:00Z] valid_time = ~U[2010-06-15 12:00:00Z]
@ -51,7 +51,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
# Row snapped to 0.25° grid (32.9 → 33.0, -97.0 → -97.0) and stored at # Row snapped to 0.25° grid (32.9 → 33.0, -97.0 → -97.0) and stored at
# the exact 3-hourly valid_time. # the exact 3-hourly valid_time.
assert [row] = Repo.all(Era5Profile) assert [row] = Repo.all(NarrProfile)
assert row.lat == 33.0 assert row.lat == 33.0
assert row.lon == -97.0 assert row.lon == -97.0
assert DateTime.compare(row.valid_time, valid_time) == :eq assert DateTime.compare(row.valid_time, valid_time) == :eq
@ -62,7 +62,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
assert_in_delta row.pwat_mm, 45.1, 2.0 assert_in_delta row.pwat_mm, 45.1, 2.0
end end
test "is a no-op when an era5_profiles row already exists for the snapped lat/lon/time" do test "is a no-op when a NarrProfile row already exists for the snapped lat/lon/time" do
valid_time = ~U[2010-06-15 12:00:00Z] valid_time = ~U[2010-06-15 12:00:00Z]
# Stub that raises if called — this test asserts no HTTP happens. # Stub that raises if called — this test asserts no HTTP happens.
@ -70,9 +70,9 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
raise "NarrClient should not be contacted when cache hit" raise "NarrClient should not be contacted when cache hit"
end) end)
# Pre-populate era5_profiles with a row at the snapped coordinates. # Pre-populate with a row at the snapped coordinates.
%Era5Profile{} %NarrProfile{}
|> Era5Profile.changeset(%{ |> NarrProfile.changeset(%{
lat: 33.0, lat: 33.0,
lon: -97.0, lon: -97.0,
valid_time: valid_time, valid_time: valid_time,
@ -89,7 +89,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
assert :ok = NarrFetchWorker.perform(%Oban.Job{args: args}) assert :ok = NarrFetchWorker.perform(%Oban.Job{args: args})
# Only the pre-seeded row exists — no new insert. # Only the pre-seeded row exists — no new insert.
assert Repo.aggregate(Era5Profile, :count, :id) == 1 assert Repo.aggregate(NarrProfile, :count, :id) == 1
end end
test "returns {:error, _} when NarrClient.fetch_profile_at fails" do test "returns {:error, _} when NarrClient.fetch_profile_at fails" do
@ -104,7 +104,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
} }
assert {:error, _reason} = NarrFetchWorker.perform(%Oban.Job{args: args}) assert {:error, _reason} = NarrFetchWorker.perform(%Oban.Job{args: args})
assert Repo.aggregate(Era5Profile, :count, :id) == 0 assert Repo.aggregate(NarrProfile, :count, :id) == 0
end end
test "raises ArgumentError when valid_time is not on a 3-hourly NARR slot" do test "raises ArgumentError when valid_time is not on a 3-hourly NARR slot" do

View file

@ -5,8 +5,8 @@ defmodule MicrowavepropWeb.ContactLiveTest do
alias Microwaveprop.Radio.Contact alias Microwaveprop.Radio.Contact
alias Microwaveprop.Repo alias Microwaveprop.Repo
alias Microwaveprop.Weather.Era5Profile
alias Microwaveprop.Weather.HrrrNativeProfile alias Microwaveprop.Weather.HrrrNativeProfile
alias Microwaveprop.Weather.NarrProfile
setup do setup do
Req.Test.stub(Microwaveprop.Terrain.ElevationClient, fn conn -> Req.Test.stub(Microwaveprop.Terrain.ElevationClient, fn conn ->
@ -156,8 +156,8 @@ defmodule MicrowavepropWeb.ContactLiveTest do
end end
end end
test "shows atmospheric profile section with ERA5 source for pre-HRRR contact", %{conn: conn} do test "shows atmospheric profile section with NARR source for pre-HRRR contact", %{conn: conn} do
# Pre-2014 contact — HRRR is unavailable, ERA5 is the fallback source. # Pre-2014 contact — HRRR is unavailable, NARR is the fallback source.
contact = contact =
create_contact(%{ create_contact(%{
qso_timestamp: ~U[2010-06-15 18:00:00Z], qso_timestamp: ~U[2010-06-15 18:00:00Z],
@ -166,7 +166,7 @@ defmodule MicrowavepropWeb.ContactLiveTest do
}) })
{:ok, _} = {:ok, _} =
Repo.insert(%Era5Profile{ Repo.insert(%NarrProfile{
valid_time: ~U[2010-06-15 18:00:00Z], valid_time: ~U[2010-06-15 18:00:00Z],
lat: 33.0, lat: 33.0,
lon: -97.0, lon: -97.0,
@ -187,8 +187,8 @@ defmodule MicrowavepropWeb.ContactLiveTest do
html = render_async(lv) html = render_async(lv)
assert html =~ "Atmospheric Profile" assert html =~ "Atmospheric Profile"
assert html =~ "ERA5" assert html =~ "NARR"
# Collapsed summary shows HPBL / PWAT / lat,lon from the injected ERA5 row. # Collapsed summary shows HPBL / PWAT / lat,lon from the injected NARR row.
assert html =~ "1200" assert html =~ "1200"
assert html =~ "33.0" assert html =~ "33.0"
end end
@ -203,10 +203,10 @@ defmodule MicrowavepropWeb.ContactLiveTest do
pos2: %{"lat" => 30.3, "lon" => -97.7} pos2: %{"lat" => 30.3, "lon" => -97.7}
}) })
# ERA5 row keeps the atmospheric profile section rendering with its # NARR row keeps the atmospheric profile section rendering with its
# aggregate metadata (HPBL / PWAT / surface scalars). # aggregate metadata (HPBL / PWAT / surface scalars).
{:ok, _} = {:ok, _} =
Repo.insert(%Era5Profile{ Repo.insert(%NarrProfile{
valid_time: ~U[2024-06-15 18:00:00Z], valid_time: ~U[2024-06-15 18:00:00Z],
lat: 32.9, lat: 32.9,
lon: -97.0, lon: -97.0,
@ -223,7 +223,7 @@ defmodule MicrowavepropWeb.ContactLiveTest do
# Native profile at the same point/time. `pressure_pa: [..., 30_000.0]` # Native profile at the same point/time. `pressure_pa: [..., 30_000.0]`
# corresponds to 300 mb — a level the pressure-level profile cannot # corresponds to 300 mb — a level the pressure-level profile cannot
# cover because HRRR/ERA5 stopped at 700 mb historically. # cover because HRRR/NARR pressure-level data stops at 700 mb historically.
{:ok, _} = {:ok, _} =
%HrrrNativeProfile{} %HrrrNativeProfile{}
|> HrrrNativeProfile.changeset(%{ |> HrrrNativeProfile.changeset(%{

View file

@ -7,7 +7,7 @@ defmodule MicrowavepropWeb.StatusLiveTest do
alias Microwaveprop.Cache alias Microwaveprop.Cache
alias Microwaveprop.Radio.Contact alias Microwaveprop.Radio.Contact
alias Microwaveprop.Repo alias Microwaveprop.Repo
alias Microwaveprop.Weather.Era5Profile alias Microwaveprop.Weather.NarrProfile
setup %{conn: conn} do setup %{conn: conn} do
Cache.invalidate({MicrowavepropWeb.StatusLive, :count_unprocessed}) Cache.invalidate({MicrowavepropWeb.StatusLive, :count_unprocessed})
@ -41,8 +41,8 @@ defmodule MicrowavepropWeb.StatusLiveTest do
contact contact
end end
# NARR fetches land in era5_profiles (table name kept as historical artifact); # NARR fetches land in era5_profiles (table rename is a follow-up migration).
# the status UI labels this "NARR" now. # The status UI labels this "NARR" now.
test "NARR progress bar numerator is actual profile matches not total-minus-candidates", %{conn: conn} do test "NARR progress bar numerator is actual profile matches not total-minus-candidates", %{conn: conn} do
create_narr_candidate() create_narr_candidate()
@ -54,7 +54,7 @@ defmodule MicrowavepropWeb.StatusLiveTest do
# Insert a matching profile row — done should become 1 # Insert a matching profile row — done should become 1
{:ok, _} = {:ok, _} =
Repo.insert(%Era5Profile{ Repo.insert(%NarrProfile{
valid_time: ~U[2010-06-15 18:00:00Z], valid_time: ~U[2010-06-15 18:00:00Z],
lat: 33.0, lat: 33.0,
lon: -97.0, lon: -97.0,