Rename ERA5 → NARR across the codebase
- Schema: Era5Profile → NarrProfile; table renamed via migration rename_era5_profiles_to_narr_profiles (ALTER TABLE + rename indexes, atomic and instant, no row rewrite) - Weather helpers: find_nearest_era5/era5_for_contact/era5_profiles_for_path → find_nearest_narr/narr_for_contact/narr_profiles_for_path - BackfillEnqueueWorker: :era5 → :narr type (virtual, no narr_status column); reconcile_stale_queued_to_unavailable and status_priority_order now skip virtual types via @virtual_types. Fixes the prod crash where the cron tried to update a non-existent era5_status column. - ContactWeatherEnqueueWorker: build_era5_jobs → build_narr_jobs, era5_jobs_for_contact → narr_jobs_for_contact - ContactLive: @era5 → @narr, maybe_enqueue_era5 → maybe_enqueue_narr; UI label "ERA5 (0.25°)" → "NARR (32 km)" - Cron (runtime.exs): args types list now "narr" instead of "era5" - /status: progress row, status-by-type card, totals.narr_profiles, and table-count lookup all target narr_profiles - Drop obsolete Era5CdsJob schema + era5_cds_jobs table (inspection artifact from the retired CDS pipeline; 34 orphan rows in prod) - Misc docstring/comment cleanups (skew_t, about, wgrib2, propagation_train) Includes a regression test for the virtual-type crash.
This commit is contained in:
parent
9e94759f7a
commit
3604896726
22 changed files with 235 additions and 258 deletions
|
|
@ -218,11 +218,11 @@ if config_env() == :prod do
|
|||
# full grid-side JSONB write for.
|
||||
# {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker},
|
||||
{"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
|
||||
# The :era5 type dispatches NarrFetchWorker for pre-2014 contacts
|
||||
# (see ContactWeatherEnqueueWorker.era5_jobs_for_contact/1). The
|
||||
# type key is a historical artifact — the data source is NARR.
|
||||
# The :narr type is virtual: it targets contacts with hrrr_status =
|
||||
# :unavailable (pre-2014, missing from the HRRR archive) and dispatches
|
||||
# NarrFetchWorker against NCEI. See narr_jobs_for_contact/1.
|
||||
{"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker,
|
||||
args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre", "era5"]}},
|
||||
args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre", "narr"]}},
|
||||
# UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch
|
||||
{"30 1 * * *", CanadianSoundingFetchWorker},
|
||||
{"30 13 * * *", CanadianSoundingFetchWorker},
|
||||
|
|
|
|||
|
|
@ -5,13 +5,13 @@ defmodule Microwaveprop.Weather do
|
|||
|
||||
alias Microwaveprop.Propagation.ProfilesFile
|
||||
alias Microwaveprop.Repo
|
||||
alias Microwaveprop.Weather.Era5Profile
|
||||
alias Microwaveprop.Weather.GridCache
|
||||
alias Microwaveprop.Weather.HrrrNativeProfile
|
||||
alias Microwaveprop.Weather.HrrrProfile
|
||||
alias Microwaveprop.Weather.IemClient
|
||||
alias Microwaveprop.Weather.IemreObservation
|
||||
alias Microwaveprop.Weather.Metar5minObservation
|
||||
alias Microwaveprop.Weather.NarrProfile
|
||||
alias Microwaveprop.Weather.RtmaObservation
|
||||
alias Microwaveprop.Weather.SolarIndex
|
||||
alias Microwaveprop.Weather.Sounding
|
||||
|
|
@ -868,56 +868,56 @@ defmodule Microwaveprop.Weather do
|
|||
|
||||
@doc """
|
||||
Find the best available atmospheric profile for a contact.
|
||||
Tries HRRR first (3 km, hourly), falls back to ERA5 (0.25°, hourly).
|
||||
Tries HRRR first (3 km, hourly), falls back to NARR (32 km, 3-hourly).
|
||||
Returns the profile struct or nil.
|
||||
"""
|
||||
@spec best_profile_for_contact(map()) :: HrrrProfile.t() | Era5Profile.t() | nil
|
||||
@spec best_profile_for_contact(map()) :: HrrrProfile.t() | NarrProfile.t() | nil
|
||||
def best_profile_for_contact(contact) do
|
||||
hrrr_for_contact(contact) || era5_for_contact(contact)
|
||||
hrrr_for_contact(contact) || narr_for_contact(contact)
|
||||
end
|
||||
|
||||
@doc "Find all atmospheric profiles along a contact's path, from any source."
|
||||
@spec profiles_along_path(map()) :: [HrrrProfile.t() | Era5Profile.t()]
|
||||
@spec profiles_along_path(map()) :: [HrrrProfile.t() | NarrProfile.t()]
|
||||
def profiles_along_path(contact) do
|
||||
hrrr_path = hrrr_profiles_for_path(contact)
|
||||
|
||||
if hrrr_path == [] do
|
||||
era5_profiles_for_path(contact)
|
||||
narr_profiles_for_path(contact)
|
||||
else
|
||||
hrrr_path
|
||||
end
|
||||
end
|
||||
|
||||
@spec era5_for_contact(map()) :: Era5Profile.t() | nil
|
||||
def era5_for_contact(%{pos1: nil}), do: nil
|
||||
@spec narr_for_contact(map()) :: NarrProfile.t() | nil
|
||||
def narr_for_contact(%{pos1: nil}), do: nil
|
||||
|
||||
def era5_for_contact(contact) do
|
||||
def narr_for_contact(contact) do
|
||||
lat = contact.pos1["lat"]
|
||||
lon = contact.pos1["lon"] || contact.pos1["lng"]
|
||||
|
||||
if lat && lon do
|
||||
find_nearest_era5(lat, lon, contact.qso_timestamp)
|
||||
find_nearest_narr(lat, lon, contact.qso_timestamp)
|
||||
end
|
||||
end
|
||||
|
||||
@spec era5_profiles_for_path(map()) :: [Era5Profile.t()]
|
||||
def era5_profiles_for_path(%{pos1: nil}), do: []
|
||||
@spec narr_profiles_for_path(map()) :: [NarrProfile.t()]
|
||||
def narr_profiles_for_path(%{pos1: nil}), do: []
|
||||
|
||||
def era5_profiles_for_path(contact) do
|
||||
def narr_profiles_for_path(contact) do
|
||||
contact
|
||||
|> Microwaveprop.Radio.contact_path_points()
|
||||
|> Enum.map(fn {lat, lon} -> find_nearest_era5(lat, lon, contact.qso_timestamp) end)
|
||||
|> Enum.map(fn {lat, lon} -> find_nearest_narr(lat, lon, contact.qso_timestamp) end)
|
||||
|> Enum.reject(&is_nil/1)
|
||||
end
|
||||
|
||||
@spec find_nearest_era5(float(), float(), DateTime.t()) :: Era5Profile.t() | nil
|
||||
def find_nearest_era5(lat, lon, timestamp) do
|
||||
@spec find_nearest_narr(float(), float(), DateTime.t()) :: NarrProfile.t() | nil
|
||||
def find_nearest_narr(lat, lon, timestamp) do
|
||||
dlat = 0.15
|
||||
dlon = 0.15
|
||||
time_start = DateTime.add(timestamp, -1800, :second)
|
||||
time_end = DateTime.add(timestamp, 1800, :second)
|
||||
|
||||
Era5Profile
|
||||
NarrProfile
|
||||
|> where(
|
||||
[p],
|
||||
p.lat >= ^(lat - dlat) and p.lat <= ^(lat + dlat) and
|
||||
|
|
|
|||
|
|
@ -1,44 +0,0 @@
|
|||
defmodule Microwaveprop.Weather.Era5CdsJob do
|
||||
@moduledoc """
|
||||
Tracks an in-flight ERA5 CDS request submitted by `Era5SubmitWorker` and
|
||||
awaiting download+decode by `Era5PollWorker`. Persisting the CDS job IDs
|
||||
decouples the Oban worker lifecycle from the 30+ minute CDS queue time,
|
||||
so a pod restart resumes polling instead of re-submitting.
|
||||
"""
|
||||
use Ecto.Schema
|
||||
|
||||
import Ecto.Changeset
|
||||
|
||||
@primary_key {:id, :binary_id, autogenerate: true}
|
||||
@foreign_key_type :binary_id
|
||||
|
||||
schema "era5_cds_jobs" do
|
||||
field :year, :integer
|
||||
field :month, :integer
|
||||
field :tile_lat, :integer
|
||||
field :tile_lon, :integer
|
||||
field :single_job_id, :string
|
||||
field :pressure_job_id, :string
|
||||
field :submitted_at, :utc_datetime
|
||||
field :last_polled_at, :utc_datetime
|
||||
field :poll_count, :integer, default: 0
|
||||
|
||||
timestamps(type: :utc_datetime)
|
||||
end
|
||||
|
||||
@type t :: %__MODULE__{}
|
||||
|
||||
@required_fields ~w(year month tile_lat tile_lon single_job_id pressure_job_id submitted_at)a
|
||||
@optional_fields ~w(last_polled_at poll_count)a
|
||||
|
||||
@spec changeset(t() | Ecto.Changeset.t(), map()) :: Ecto.Changeset.t()
|
||||
def changeset(job, attrs) do
|
||||
job
|
||||
|> cast(attrs, @required_fields ++ @optional_fields)
|
||||
|> validate_required(@required_fields)
|
||||
|> unique_constraint([:year, :month, :tile_lat, :tile_lon],
|
||||
name: :era5_cds_jobs_year_month_tile_lat_tile_lon_index,
|
||||
error_key: :tile_month
|
||||
)
|
||||
end
|
||||
end
|
||||
|
|
@ -336,7 +336,7 @@ defmodule Microwaveprop.Weather.Grib2.Wgrib2 do
|
|||
@doc """
|
||||
Like `extract_grid/3`, but returns a flat list of per-message results so
|
||||
the time dimension is preserved. Used when a single GRIB2 blob carries
|
||||
many timesteps (e.g. a month of ERA5 data fetched in one CDS request).
|
||||
many timesteps.
|
||||
|
||||
Each entry is `%{datetime: DateTime.t() | nil, var: String.t(),
|
||||
level: String.t(), values: %{{lat, lon} => float}}`.
|
||||
|
|
@ -354,7 +354,7 @@ defmodule Microwaveprop.Weather.Grib2.Wgrib2 do
|
|||
| {:error, term()}
|
||||
def extract_grid_messages(grib_binary, match_pattern, grid_spec) do
|
||||
if available?() do
|
||||
tmp_grib = Path.join(System.tmp_dir!(), "era5_#{System.unique_integer([:positive])}.grib2")
|
||||
tmp_grib = Path.join(System.tmp_dir!(), "wgrib2_#{System.unique_integer([:positive])}.grib2")
|
||||
|
||||
try do
|
||||
File.write!(tmp_grib, grib_binary)
|
||||
|
|
|
|||
|
|
@ -141,13 +141,13 @@ defmodule Microwaveprop.Weather.NarrClient do
|
|||
end
|
||||
|
||||
@doc """
|
||||
Extracts an `era5_profiles`-shaped attribute map from a composite NARR
|
||||
Extracts a NarrProfile-shaped attribute map from a composite NARR
|
||||
GRIB1 file at the given lat/lon point.
|
||||
|
||||
Pure-ish: shells out to `cdo -outputtab,name,lev,value -remapnn,lon=…_lat=…`
|
||||
on the local file. No network.
|
||||
|
||||
Returns `{:ok, attrs}` — a map with the `era5_profiles` fields
|
||||
Returns `{:ok, attrs}` — a map with the NarrProfile fields
|
||||
(`:surface_temp_c`, `:surface_dewpoint_c`, `:surface_pressure_mb`,
|
||||
`:hpbl_m`, `:pwat_mm`, `:profile` list plus the derived fields from
|
||||
`SoundingParams.derive/1`) — or `{:error, reason}` on missing file,
|
||||
|
|
@ -170,8 +170,8 @@ defmodule Microwaveprop.Weather.NarrClient do
|
|||
using `extract_profile_from_file/3`.
|
||||
|
||||
Returns `{:ok, profile_attrs}` shaped for direct `Repo.insert_all/3`
|
||||
into the `era5_profiles` table, or `{:error, reason}`. Temp files are
|
||||
always cleaned up via `try/after`.
|
||||
against `NarrProfile`, or `{:error, reason}`. Temp files are always
|
||||
cleaned up via `try/after`.
|
||||
"""
|
||||
@spec fetch_profile_at(DateTime.t(), {float(), float()}) ::
|
||||
{:ok, map()} | {:error, term()}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,13 @@
|
|||
defmodule Microwaveprop.Weather.Era5Profile do
|
||||
@moduledoc false
|
||||
defmodule Microwaveprop.Weather.NarrProfile do
|
||||
@moduledoc """
|
||||
Profile rows fetched from NCEP NARR (North American Regional Reanalysis)
|
||||
for pre-2014 contacts where the HRRR archive doesn't reach.
|
||||
|
||||
The table was originally created as `era5_profiles` for the retired
|
||||
ERA5/CDS pipeline and reused 1:1 by NARR; the
|
||||
`rename_era5_profiles_to_narr_profiles` migration renames it to
|
||||
`narr_profiles`.
|
||||
"""
|
||||
use Ecto.Schema
|
||||
|
||||
import Ecto.Changeset
|
||||
|
|
@ -7,7 +15,7 @@ defmodule Microwaveprop.Weather.Era5Profile do
|
|||
@primary_key {:id, :binary_id, autogenerate: true}
|
||||
@foreign_key_type :binary_id
|
||||
|
||||
schema "era5_profiles" do
|
||||
schema "narr_profiles" do
|
||||
field :valid_time, :utc_datetime
|
||||
field :lat, :float
|
||||
field :lon, :float
|
||||
|
|
@ -14,7 +14,12 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
|
|||
require Logger
|
||||
|
||||
@enrichable [:pending, :queued, :failed]
|
||||
@valid_types ~w(hrrr weather terrain iemre era5)
|
||||
@valid_types ~w(hrrr weather terrain iemre narr)
|
||||
|
||||
# Virtual types have no <type>_status column on contacts; they're synthesized
|
||||
# from other status fields (e.g. :narr candidates are `hrrr_status = :unavailable`).
|
||||
# Skip these in any reconcile / ordering pass that reads a <type>_status field.
|
||||
@virtual_types [:narr]
|
||||
|
||||
# A :queued contact whose updated_at is older than this is one the cron has
|
||||
# already tried ~144 times over 3 days without the underlying worker landing
|
||||
|
|
@ -67,30 +72,45 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
|
|||
types |> Enum.filter(&(&1 in @valid_types)) |> Enum.map(&String.to_existing_atom/1)
|
||||
end
|
||||
|
||||
defp parse_types(_), do: [:hrrr, :weather, :terrain, :iemre, :era5]
|
||||
defp parse_types(_), do: [:hrrr, :weather, :terrain, :iemre, :narr]
|
||||
|
||||
defp status_priority_order(types) do
|
||||
# Prioritize pending/failed over queued so real work happens first
|
||||
field = :"#{hd(types)}_status"
|
||||
# Prioritize pending/failed over queued so real work happens first.
|
||||
# Virtual types (e.g. :narr) have no *_status column — fall back to
|
||||
# time-only ordering when every requested type is virtual.
|
||||
case Enum.find(types, &(&1 not in @virtual_types)) do
|
||||
nil ->
|
||||
[desc: dynamic([c], c.qso_timestamp)]
|
||||
|
||||
[
|
||||
asc: dynamic([c], fragment("CASE ? WHEN 'pending' THEN 0 WHEN 'failed' THEN 1 ELSE 2 END", field(c, ^field))),
|
||||
desc: dynamic([c], c.qso_timestamp)
|
||||
]
|
||||
type ->
|
||||
field = :"#{type}_status"
|
||||
|
||||
[
|
||||
asc:
|
||||
dynamic(
|
||||
[c],
|
||||
fragment("CASE ? WHEN 'pending' THEN 0 WHEN 'failed' THEN 1 ELSE 2 END", field(c, ^field))
|
||||
),
|
||||
desc: dynamic([c], c.qso_timestamp)
|
||||
]
|
||||
end
|
||||
end
|
||||
|
||||
# Flip contacts that have been stuck in :queued for longer than
|
||||
# @stale_queued_cutoff_seconds to :unavailable. One Repo.update_all per type,
|
||||
# scoped to only the types passed in args so a partial cron (e.g. hrrr-only)
|
||||
# doesn't touch other statuses. Fast: hits the (<field>, updated_at) combo
|
||||
# with a simple range scan.
|
||||
# doesn't touch other statuses. Virtual types (e.g. :narr) have no *_status
|
||||
# column and are skipped. Fast: hits the (<field>, updated_at) combo with a
|
||||
# simple range scan.
|
||||
defp reconcile_stale_queued_to_unavailable(types) do
|
||||
cutoff =
|
||||
DateTime.utc_now()
|
||||
|> DateTime.add(-@stale_queued_cutoff_seconds, :second)
|
||||
|> DateTime.truncate(:second)
|
||||
|
||||
Enum.reduce(types, 0, fn type, acc ->
|
||||
types
|
||||
|> Enum.reject(&(&1 in @virtual_types))
|
||||
|> Enum.reduce(0, fn type, acc ->
|
||||
field = :"#{type}_status"
|
||||
|
||||
{n, _} =
|
||||
|
|
@ -126,8 +146,8 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
|
|||
|
||||
defp type_filter(types) do
|
||||
Enum.reduce(types, dynamic(false), fn
|
||||
:era5, acc ->
|
||||
# ERA5 targets contacts where HRRR is unavailable (pre-2014, missing from archive)
|
||||
:narr, acc ->
|
||||
# NARR targets contacts where HRRR is unavailable (pre-2014, missing from archive)
|
||||
dynamic([c], ^acc or c.hrrr_status == :unavailable)
|
||||
|
||||
type, acc ->
|
||||
|
|
|
|||
|
|
@ -33,9 +33,9 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
|
|||
insert_all_chunked(bulk_jobs)
|
||||
end
|
||||
|
||||
# ERA5 jobs must go through Oban.insert/1 so the worker's unique
|
||||
# NARR jobs must go through Oban.insert/1 so the worker's unique
|
||||
# constraint is honored; Oban OSS insert_all does not check uniqueness.
|
||||
insert_unique(jobs_by_type[:era5])
|
||||
insert_unique(jobs_by_type[:narr])
|
||||
|
||||
mark_enrichment_statuses(contact, types, jobs_by_type)
|
||||
|
||||
|
|
@ -48,7 +48,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
|
|||
hrrr: if(:hrrr in types, do: build_hrrr_jobs([contact]), else: []),
|
||||
terrain: if(:terrain in types, do: build_terrain_jobs([contact]), else: []),
|
||||
iemre: if(:iemre in types, do: build_iemre_jobs([contact]), else: []),
|
||||
era5: if(:era5 in types, do: build_era5_jobs([contact]), else: [])
|
||||
narr: if(:narr in types, do: build_narr_jobs([contact]), else: [])
|
||||
}
|
||||
end
|
||||
|
||||
|
|
@ -173,18 +173,15 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
|
|||
|> Enum.uniq_by(fn changeset -> changeset.changes.args end)
|
||||
end
|
||||
|
||||
def build_era5_jobs(contacts) do
|
||||
def build_narr_jobs(contacts) do
|
||||
contacts
|
||||
|> Enum.flat_map(&era5_jobs_for_contact/1)
|
||||
|> Enum.flat_map(&narr_jobs_for_contact/1)
|
||||
|> Enum.uniq_by(fn changeset -> changeset.changes.args end)
|
||||
end
|
||||
|
||||
# The function name is a historical artifact: the `:era5` type key still
|
||||
# refers to the `era5_profiles` table (reused 1:1), but this now dispatches
|
||||
# `NarrFetchWorker` against NCEP NARR — the CDS/ERA5 pipeline is gone.
|
||||
defp era5_jobs_for_contact(%{pos1: nil}), do: []
|
||||
defp narr_jobs_for_contact(%{pos1: nil}), do: []
|
||||
|
||||
defp era5_jobs_for_contact(contact) do
|
||||
defp narr_jobs_for_contact(contact) do
|
||||
# NARR analyses are 3-hourly (00/03/06/09/12/15/18/21 UTC). NarrFetchWorker
|
||||
# rejects anything else, so snap here in the caller.
|
||||
valid_time = NarrClient.snap_to_analysis_hour(contact.qso_timestamp)
|
||||
|
|
@ -195,7 +192,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
|
|||
rlat = Float.round(lat * 4) / 4
|
||||
rlon = Float.round(lon * 4) / 4
|
||||
|
||||
if Weather.find_nearest_era5(rlat, rlon, valid_time) do
|
||||
if Weather.find_nearest_narr(rlat, rlon, valid_time) do
|
||||
[]
|
||||
else
|
||||
[
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
defmodule Microwaveprop.Workers.NarrFetchWorker do
|
||||
@moduledoc """
|
||||
Fetches one NCEP NARR profile from NCEI via `NarrClient.fetch_profile_at/2`
|
||||
and inserts it into the `era5_profiles` table.
|
||||
and inserts it as a `NarrProfile` row.
|
||||
|
||||
Replaces `Era5FetchWorker` for pre-2014 contacts. Unlike the ERA5 router,
|
||||
NARR fetches are per-point (one analysis hour, one location), so this
|
||||
worker does the fetch + insert directly — no downstream batch worker,
|
||||
no routing.
|
||||
Covers pre-2014 contacts where the HRRR archive doesn't reach. Replaces
|
||||
the retired ERA5/CDS pipeline. NARR fetches are per-point (one analysis
|
||||
hour, one location), so this worker does the fetch + insert directly —
|
||||
no downstream batch worker, no routing.
|
||||
|
||||
Jobs are unique on `{lat, lon, valid_time}` so duplicate enrichment
|
||||
requests collapse. `valid_time` MUST already be snapped to a NARR
|
||||
|
|
@ -15,8 +15,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorker do
|
|||
`NarrClient.url_for/1`.
|
||||
|
||||
See `docs/plans/2026-04-15-merra2-historical-backfill.md` for the full
|
||||
architecture. The `era5_profiles` table is reused 1:1 (rename is a
|
||||
follow-up).
|
||||
architecture.
|
||||
"""
|
||||
use Oban.Worker,
|
||||
queue: :narr,
|
||||
|
|
@ -29,8 +28,8 @@ defmodule Microwaveprop.Workers.NarrFetchWorker do
|
|||
import Ecto.Query
|
||||
|
||||
alias Microwaveprop.Repo
|
||||
alias Microwaveprop.Weather.Era5Profile
|
||||
alias Microwaveprop.Weather.NarrClient
|
||||
alias Microwaveprop.Weather.NarrProfile
|
||||
|
||||
require Logger
|
||||
|
||||
|
|
@ -41,7 +40,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorker do
|
|||
rlat = Float.round(lat * 4) / 4
|
||||
rlon = Float.round(lon * 4) / 4
|
||||
|
||||
if has_era5_profile?(rlat, rlon, valid_time) do
|
||||
if has_narr_profile?(rlat, rlon, valid_time) do
|
||||
:ok
|
||||
else
|
||||
fetch_and_insert(rlat, rlon, valid_time)
|
||||
|
|
@ -74,22 +73,21 @@ defmodule Microwaveprop.Workers.NarrFetchWorker do
|
|||
|> Map.put(:inserted_at, now)
|
||||
|> Map.put(:updated_at, now)
|
||||
|
||||
Repo.insert_all(Era5Profile, [row],
|
||||
Repo.insert_all(NarrProfile, [row],
|
||||
on_conflict: :nothing,
|
||||
conflict_target: [:lat, :lon, :valid_time]
|
||||
)
|
||||
end
|
||||
|
||||
defp has_era5_profile?(lat, lon, valid_time) do
|
||||
# NARR is exact 3-hourly, so we tighten the time window to ±15 minutes
|
||||
# compared to Era5FetchWorker's ±30-minute check. Lat/lon tolerance
|
||||
# matches the 0.25° dedup granularity.
|
||||
defp has_narr_profile?(lat, lon, valid_time) do
|
||||
# NARR is exact 3-hourly, so the time window can be tight (±15 min).
|
||||
# Lat/lon tolerance matches the 0.25° dedup granularity.
|
||||
dlat = 0.13
|
||||
dlon = 0.13
|
||||
time_start = DateTime.add(valid_time, -900, :second)
|
||||
time_end = DateTime.add(valid_time, 900, :second)
|
||||
|
||||
Era5Profile
|
||||
NarrProfile
|
||||
|> where(
|
||||
[p],
|
||||
p.lat >= ^(lat - dlat) and p.lat <= ^(lat + dlat) and
|
||||
|
|
|
|||
|
|
@ -172,7 +172,7 @@ defmodule MicrowavepropWeb.AboutLive do
|
|||
forecasts), Iowa Environmental Mesonet (ASOS and upper-air
|
||||
soundings), IEMRE gridded reanalysis, SRTM 90 m terrain, SNMP
|
||||
polling on seven commercial microwave links near DFW at 5-minute
|
||||
intervals, and Copernicus ERA5 for pre-2014 contact enrichment.
|
||||
intervals, and NCEP NARR for pre-2014 contact enrichment.
|
||||
</li>
|
||||
<li>
|
||||
There's Nx/Axon/EXLA scaffolding for a small feed-forward model
|
||||
|
|
|
|||
|
|
@ -46,8 +46,8 @@ defmodule MicrowavepropWeb.ContactLive.Show do
|
|||
hrrr: nil,
|
||||
hrrr_path: [],
|
||||
native_profile: nil,
|
||||
era5: nil,
|
||||
era5_path: [],
|
||||
narr: nil,
|
||||
narr_path: [],
|
||||
iemre: nil,
|
||||
terrain: nil,
|
||||
elevation_profile: nil,
|
||||
|
|
@ -91,7 +91,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
|
|||
|> start_async(:weather, fn -> load_weather(contact) end)
|
||||
|> start_async(:solar, fn -> load_solar(contact) end)
|
||||
|> start_async(:hrrr_path, fn -> Weather.hrrr_profiles_for_path(contact) end)
|
||||
|> start_async(:era5_path, fn -> Weather.era5_profiles_for_path(contact) end)
|
||||
|> start_async(:narr_path, fn -> Weather.narr_profiles_for_path(contact) end)
|
||||
|> start_async(:native_profile, fn -> load_native_profile(contact) end)
|
||||
|> start_async(:terrain, fn -> Terrain.get_terrain_profile(contact.id) end)
|
||||
|> start_async(:iemre, fn -> load_iemre(contact) end)
|
||||
|
|
@ -328,14 +328,14 @@ defmodule MicrowavepropWeb.ContactLive.Show do
|
|||
{:noreply, socket}
|
||||
end
|
||||
|
||||
def handle_async(:era5_path, {:ok, era5_path}, socket) do
|
||||
era5 = List.first(era5_path)
|
||||
def handle_async(:narr_path, {:ok, narr_path}, socket) do
|
||||
narr = List.first(narr_path)
|
||||
contact = socket.assigns.contact
|
||||
|
||||
contact =
|
||||
if socket.assigns.can_enqueue, do: maybe_enqueue_era5(era5, contact), else: contact
|
||||
if socket.assigns.can_enqueue, do: maybe_enqueue_narr(narr, contact), else: contact
|
||||
|
||||
{:noreply, assign(socket, contact: contact, era5: era5, era5_path: era5_path)}
|
||||
{:noreply, assign(socket, contact: contact, narr: narr, narr_path: narr_path)}
|
||||
end
|
||||
|
||||
def handle_async(:terrain, {:ok, terrain}, socket) do
|
||||
|
|
@ -361,7 +361,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
|
|||
end
|
||||
|
||||
def handle_async(slot, {:exit, reason}, socket)
|
||||
when slot in [:weather, :solar, :hrrr_path, :era5_path, :native_profile, :terrain, :iemre] do
|
||||
when slot in [:weather, :solar, :hrrr_path, :narr_path, :native_profile, :terrain, :iemre] do
|
||||
Logger.warning("contact hydrate task #{slot} exited: #{inspect(reason)}")
|
||||
{:noreply, socket}
|
||||
end
|
||||
|
|
@ -661,13 +661,12 @@ defmodule MicrowavepropWeb.ContactLive.Show do
|
|||
end
|
||||
end
|
||||
|
||||
# ERA5 is a fallback for contacts where HRRR is unavailable (pre-2014 or
|
||||
# NARR is the fallback for contacts where HRRR is unavailable (pre-2014 or
|
||||
# missing from the archive). Only enqueue when HRRR is known-unavailable AND
|
||||
# we don't already have an ERA5 profile — otherwise we'd waste CDS quota on
|
||||
# contacts the HRRR path will cover.
|
||||
defp maybe_enqueue_era5(era5, contact) when not is_nil(era5), do: contact
|
||||
# we don't already have a NARR profile.
|
||||
defp maybe_enqueue_narr(narr, contact) when not is_nil(narr), do: contact
|
||||
|
||||
defp maybe_enqueue_era5(nil, %{hrrr_status: :unavailable} = contact) do
|
||||
defp maybe_enqueue_narr(nil, %{hrrr_status: :unavailable} = contact) do
|
||||
lat = contact.pos1 && contact.pos1["lat"]
|
||||
lon = contact.pos1 && (contact.pos1["lon"] || contact.pos1["lng"])
|
||||
|
||||
|
|
@ -686,7 +685,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
|
|||
contact
|
||||
end
|
||||
|
||||
defp maybe_enqueue_era5(_nil, contact), do: contact
|
||||
defp maybe_enqueue_narr(_nil, contact), do: contact
|
||||
|
||||
defp queue_info(counts, queue) do
|
||||
case Map.get(counts, queue, 0) do
|
||||
|
|
@ -1140,11 +1139,11 @@ defmodule MicrowavepropWeb.ContactLive.Show do
|
|||
<div class="divider" />
|
||||
|
||||
<h2 class="text-base font-semibold mb-2">Atmospheric Profile</h2>
|
||||
<% profile = @hrrr || @era5 %>
|
||||
<% profile = @hrrr || @narr %>
|
||||
<% profile_source =
|
||||
cond do
|
||||
@hrrr -> "HRRR (3 km)"
|
||||
@era5 -> "ERA5 (0.25°)"
|
||||
@narr -> "NARR (32 km)"
|
||||
true -> nil
|
||||
end %>
|
||||
<% skew_t_levels =
|
||||
|
|
@ -1253,7 +1252,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
|
|||
<% :unavailable -> %>
|
||||
<span class="flex items-center gap-2">
|
||||
<span class="loading loading-spinner loading-sm"></span>
|
||||
HRRR unavailable — fetching ERA5 reanalysis {queue_info(@queue_counts, "era5_batch")}
|
||||
HRRR unavailable — fetching NARR reanalysis {queue_info(@queue_counts, "narr")}
|
||||
</span>
|
||||
<% _ -> %>
|
||||
No atmospheric profile available.
|
||||
|
|
|
|||
|
|
@ -97,9 +97,8 @@ defmodule MicrowavepropWeb.StatusLive do
|
|||
end
|
||||
|
||||
# NARR is the fallback for pre-2014 contacts (HRRR archive starts 2014).
|
||||
# Profiles land in the era5_profiles table (kept 1:1 after the ERA5/CDS
|
||||
# pipeline was replaced). "Done" means a row within 0.15° + 30 min of the
|
||||
# contact, matching `Weather.find_nearest_era5/3` lookup tolerance.
|
||||
# "Done" means a narr_profiles row within 0.15° + 30 min of the contact,
|
||||
# matching `Weather.find_nearest_narr/3` lookup tolerance.
|
||||
defp count_narr_done do
|
||||
%{rows: [[done]]} =
|
||||
Repo.query!("""
|
||||
|
|
@ -109,7 +108,7 @@ defmodule MicrowavepropWeb.StatusLive do
|
|||
AND c.pos1 IS NOT NULL
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM era5_profiles p
|
||||
FROM narr_profiles p
|
||||
WHERE p.lat BETWEEN ((c.pos1->>'lat')::float - 0.15) AND ((c.pos1->>'lat')::float + 0.15)
|
||||
AND p.lon BETWEEN ((c.pos1->>'lon')::float - 0.15) AND ((c.pos1->>'lon')::float + 0.15)
|
||||
AND p.valid_time BETWEEN (c.qso_timestamp - INTERVAL '30 minutes') AND (c.qso_timestamp + INTERVAL '30 minutes')
|
||||
|
|
@ -264,8 +263,8 @@ defmodule MicrowavepropWeb.StatusLive do
|
|||
table_counts = Map.new(tables, fn %{table: t, rows: r} -> {t, r} end)
|
||||
|
||||
# NARR doesn't have its own status column — synthesize from HRRR unavailable
|
||||
# + era5_profiles row count (the table is reused as the NARR landing spot).
|
||||
narr_count = Map.get(table_counts, "era5_profiles", 0)
|
||||
# + narr_profiles row count.
|
||||
narr_count = Map.get(table_counts, "narr_profiles", 0)
|
||||
|
||||
hrrr_unavailable =
|
||||
statuses |> Map.get("hrrr", []) |> Enum.find_value(0, fn {s, c} -> if s == "unavailable", do: c end)
|
||||
|
|
@ -289,7 +288,7 @@ defmodule MicrowavepropWeb.StatusLive do
|
|||
statuses: statuses,
|
||||
totals: %{
|
||||
hrrr_profiles: hrrr_count,
|
||||
narr_profiles: Map.get(table_counts, "era5_profiles", 0),
|
||||
narr_profiles: Map.get(table_counts, "narr_profiles", 0),
|
||||
terrain_profiles: terrain_count,
|
||||
surface_observations: obs_count,
|
||||
soundings: sounding_count,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ defmodule MicrowavepropWeb.SkewT do
|
|||
Skew-T log-P diagram renderer. Builds the full set of SVG
|
||||
primitives (isobars, isotherms, dry adiabats, saturation mixing
|
||||
ratio lines, and temperature/dewpoint traces) for a single HRRR
|
||||
or ERA5 pressure-level profile so the contact detail template can
|
||||
or NARR pressure-level profile so the contact detail template can
|
||||
draw it declaratively.
|
||||
|
||||
## Meteorology
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@ defmodule Mix.Tasks.PropagationTrain do
|
|||
)
|
||||
|
||||
# Boot the app without starting Oban queues so training doesn't have
|
||||
# the HRRR / ERA5 / weather cron firing in the background against the
|
||||
# the HRRR / NARR / weather cron firing in the background against the
|
||||
# local DB we're reading from.
|
||||
Application.put_env(
|
||||
:microwaveprop,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
defmodule Microwaveprop.Repo.Migrations.RenameEra5ProfilesToNarrProfiles do
|
||||
use Ecto.Migration
|
||||
|
||||
# ALTER TABLE ... RENAME is atomic and instant (no row rewrite). User-created
|
||||
# indexes keep their old names after the table rename, so we rename them too
|
||||
# for consistency. Implicit PK index / sequence are renamed automatically by
|
||||
# PostgreSQL.
|
||||
def up do
|
||||
rename table(:era5_profiles), to: table(:narr_profiles)
|
||||
|
||||
execute "ALTER INDEX era5_profiles_lat_lon_valid_time_index RENAME TO narr_profiles_lat_lon_valid_time_index"
|
||||
|
||||
execute "ALTER INDEX era5_profiles_valid_time_index RENAME TO narr_profiles_valid_time_index"
|
||||
|
||||
execute "ALTER INDEX era5_profiles_valid_time_lat_lon_index RENAME TO narr_profiles_valid_time_lat_lon_index"
|
||||
end
|
||||
|
||||
def down do
|
||||
execute "ALTER INDEX narr_profiles_valid_time_lat_lon_index RENAME TO era5_profiles_valid_time_lat_lon_index"
|
||||
|
||||
execute "ALTER INDEX narr_profiles_valid_time_index RENAME TO era5_profiles_valid_time_index"
|
||||
|
||||
execute "ALTER INDEX narr_profiles_lat_lon_valid_time_index RENAME TO era5_profiles_lat_lon_valid_time_index"
|
||||
|
||||
rename table(:narr_profiles), to: table(:era5_profiles)
|
||||
end
|
||||
end
|
||||
29
priv/repo/migrations/20260416142116_drop_era5_cds_jobs.exs
Normal file
29
priv/repo/migrations/20260416142116_drop_era5_cds_jobs.exs
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
defmodule Microwaveprop.Repo.Migrations.DropEra5CdsJobs do
|
||||
use Ecto.Migration
|
||||
|
||||
# The ERA5/CDS pipeline was retired in favor of NARR (see
|
||||
# rename_era5_profiles_to_narr_profiles). Orphan rows in this table are
|
||||
# useless without the workers that wrote them.
|
||||
def up do
|
||||
drop table(:era5_cds_jobs)
|
||||
end
|
||||
|
||||
def down do
|
||||
create table(:era5_cds_jobs, primary_key: false) do
|
||||
add :id, :binary_id, primary_key: true
|
||||
add :year, :integer, null: false
|
||||
add :month, :integer, null: false
|
||||
add :tile_lat, :integer, null: false
|
||||
add :tile_lon, :integer, null: false
|
||||
add :single_job_id, :string, null: false
|
||||
add :pressure_job_id, :string, null: false
|
||||
add :submitted_at, :utc_datetime, null: false
|
||||
add :last_polled_at, :utc_datetime
|
||||
add :poll_count, :integer, default: 0, null: false
|
||||
|
||||
timestamps(type: :utc_datetime)
|
||||
end
|
||||
|
||||
create unique_index(:era5_cds_jobs, [:year, :month, :tile_lat, :tile_lon])
|
||||
end
|
||||
end
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
defmodule Microwaveprop.Weather.Era5CdsJobTest do
|
||||
use Microwaveprop.DataCase, async: true
|
||||
|
||||
alias Microwaveprop.Weather.Era5CdsJob
|
||||
|
||||
describe "changeset/2" do
|
||||
@valid_attrs %{
|
||||
year: 2014,
|
||||
month: 3,
|
||||
tile_lat: 32,
|
||||
tile_lon: -98,
|
||||
single_job_id: "abc-single",
|
||||
pressure_job_id: "abc-pressure",
|
||||
submitted_at: ~U[2026-04-13 21:00:00Z]
|
||||
}
|
||||
|
||||
test "accepts a complete attrs map" do
|
||||
changeset = Era5CdsJob.changeset(%Era5CdsJob{}, @valid_attrs)
|
||||
assert changeset.valid?
|
||||
end
|
||||
|
||||
test "requires year, month, tile coordinates, and both CDS job IDs" do
|
||||
changeset = Era5CdsJob.changeset(%Era5CdsJob{}, %{})
|
||||
|
||||
refute changeset.valid?
|
||||
|
||||
required = ~w(year month tile_lat tile_lon single_job_id pressure_job_id submitted_at)a
|
||||
errors = Keyword.keys(changeset.errors)
|
||||
|
||||
for field <- required do
|
||||
assert field in errors, "expected #{inspect(field)} to be required"
|
||||
end
|
||||
end
|
||||
|
||||
test "last_polled_at and poll_count default to nil/0 and can be updated" do
|
||||
{:ok, row} =
|
||||
%Era5CdsJob{}
|
||||
|> Era5CdsJob.changeset(@valid_attrs)
|
||||
|> Repo.insert()
|
||||
|
||||
assert row.poll_count == 0
|
||||
assert is_nil(row.last_polled_at)
|
||||
|
||||
now = DateTime.truncate(DateTime.utc_now(), :second)
|
||||
|
||||
{:ok, bumped} =
|
||||
row
|
||||
|> Era5CdsJob.changeset(%{last_polled_at: now, poll_count: 1})
|
||||
|> Repo.update()
|
||||
|
||||
assert bumped.poll_count == 1
|
||||
assert bumped.last_polled_at == now
|
||||
end
|
||||
end
|
||||
|
||||
describe "uniqueness" do
|
||||
test "collapses duplicate (year, month, tile_lat, tile_lon) inserts" do
|
||||
attrs = %{
|
||||
year: 2014,
|
||||
month: 3,
|
||||
tile_lat: 32,
|
||||
tile_lon: -98,
|
||||
single_job_id: "first-single",
|
||||
pressure_job_id: "first-pressure",
|
||||
submitted_at: ~U[2026-04-13 21:00:00Z]
|
||||
}
|
||||
|
||||
{:ok, _} =
|
||||
%Era5CdsJob{}
|
||||
|> Era5CdsJob.changeset(attrs)
|
||||
|> Repo.insert()
|
||||
|
||||
{:error, changeset} =
|
||||
%Era5CdsJob{}
|
||||
|> Era5CdsJob.changeset(%{attrs | single_job_id: "second-single", pressure_job_id: "second-pressure"})
|
||||
|> Repo.insert()
|
||||
|
||||
refute changeset.valid?
|
||||
assert {:tile_month, _} = List.keyfind(changeset.errors, :tile_month, 0) || {:tile_month, nil}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -5,6 +5,7 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorkerTest do
|
|||
alias Microwaveprop.Terrain.ElevationClient
|
||||
alias Microwaveprop.Weather.HrrrClient
|
||||
alias Microwaveprop.Weather.IemClient
|
||||
alias Microwaveprop.Weather.NarrClient
|
||||
alias Microwaveprop.Workers.BackfillEnqueueWorker
|
||||
|
||||
defp create_contact(attrs \\ %{}) do
|
||||
|
|
@ -53,12 +54,19 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorkerTest do
|
|||
Req.Test.json(conn, %{"elevation" => List.duplicate(200.0, lat_count)})
|
||||
end)
|
||||
|
||||
# NARR fetches land in era5_profiles; stub returns 500 so inline jobs fail
|
||||
# gracefully instead of hitting NCEI.
|
||||
Req.Test.stub(NarrClient, fn conn ->
|
||||
Plug.Conn.send_resp(conn, 500, "stub")
|
||||
end)
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
# ERA5 goes through Copernicus CDS API which is async (submit → poll → download)
|
||||
# and has no Req.Test plug hook, so inline Oban would block on Process.sleep.
|
||||
# These tests exercise the non-ERA5 enrichment path; ERA5 has its own coverage.
|
||||
# NARR fetches (the :era5 type) run inline via testing: :inline — the
|
||||
# NarrClient Req.Test stub above keeps them from hitting NCEI. Most tests
|
||||
# below target the other pipelines and use this narrowed type list to keep
|
||||
# the test surface tight.
|
||||
@non_era5_types ["hrrr", "weather", "terrain", "iemre"]
|
||||
|
||||
describe "perform/1" do
|
||||
|
|
@ -113,6 +121,24 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorkerTest do
|
|||
|
||||
assert_receive {:enqueue_complete, 1}
|
||||
end
|
||||
|
||||
# :narr is a virtual type (no narr_status column) — it targets contacts
|
||||
# whose hrrr_status is :unavailable. BackfillEnqueueWorker must not try
|
||||
# to reconcile a non-existent narr_status column or use it for priority
|
||||
# ordering.
|
||||
test "handles virtual :narr type without referencing non-existent narr_status column" do
|
||||
# Pre-2014 contact (HRRR unavailable) — NARR candidate.
|
||||
contact = create_contact(%{qso_timestamp: ~U[2010-01-01 00:00:00Z]})
|
||||
|
||||
contact
|
||||
|> Ecto.Changeset.change(%{hrrr_status: :unavailable})
|
||||
|> Repo.update!()
|
||||
|
||||
assert :ok =
|
||||
BackfillEnqueueWorker.perform(%Oban.Job{
|
||||
args: %{"limit" => 10, "types" => ["narr"]}
|
||||
})
|
||||
end
|
||||
end
|
||||
|
||||
describe "reconcile stuck queued → unavailable" do
|
||||
|
|
|
|||
|
|
@ -4,9 +4,9 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
|
|||
alias Microwaveprop.Radio.Contact
|
||||
alias Microwaveprop.Terrain.ElevationClient
|
||||
alias Microwaveprop.Weather
|
||||
alias Microwaveprop.Weather.Era5Profile
|
||||
alias Microwaveprop.Weather.HrrrClient
|
||||
alias Microwaveprop.Weather.IemClient
|
||||
alias Microwaveprop.Weather.NarrProfile
|
||||
alias Microwaveprop.Workers.ContactWeatherEnqueueWorker
|
||||
|
||||
@qso_attrs %{
|
||||
|
|
@ -562,7 +562,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
|
|||
end
|
||||
end
|
||||
|
||||
describe "build_era5_jobs/1" do
|
||||
describe "build_narr_jobs/1" do
|
||||
test "builds NarrFetchWorker jobs for each rounded path point" do
|
||||
contact =
|
||||
create_contact(%{
|
||||
|
|
@ -571,7 +571,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
|
|||
pos2: nil
|
||||
})
|
||||
|
||||
jobs = ContactWeatherEnqueueWorker.build_era5_jobs([contact])
|
||||
jobs = ContactWeatherEnqueueWorker.build_narr_jobs([contact])
|
||||
|
||||
assert length(jobs) == 1
|
||||
job = hd(jobs)
|
||||
|
|
@ -589,7 +589,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
|
|||
pos2: nil
|
||||
})
|
||||
|
||||
jobs = ContactWeatherEnqueueWorker.build_era5_jobs([contact])
|
||||
jobs = ContactWeatherEnqueueWorker.build_narr_jobs([contact])
|
||||
|
||||
job = hd(jobs)
|
||||
# 13:42 UTC snaps down to the 12:00 UTC NARR analysis slot.
|
||||
|
|
@ -604,14 +604,14 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
|
|||
pos2: %{"lat" => 30.3, "lon" => -97.7}
|
||||
})
|
||||
|
||||
jobs = ContactWeatherEnqueueWorker.build_era5_jobs([contact])
|
||||
jobs = ContactWeatherEnqueueWorker.build_narr_jobs([contact])
|
||||
|
||||
# pos1, midpoint, pos2 round to three distinct 0.25° grid cells.
|
||||
assert length(jobs) == 3
|
||||
assert Enum.all?(jobs, &(&1.changes.worker == "Microwaveprop.Workers.NarrFetchWorker"))
|
||||
end
|
||||
|
||||
test "skips path points that already have an era5_profile row" do
|
||||
test "skips path points that already have a narr_profile row" do
|
||||
contact =
|
||||
create_contact(%{
|
||||
qso_timestamp: ~U[2010-06-15 13:42:00Z],
|
||||
|
|
@ -620,8 +620,8 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
|
|||
})
|
||||
|
||||
# pos1 (32.907, -97.038) snaps to the (33.0, -97.0) 0.25° grid cell
|
||||
%Era5Profile{}
|
||||
|> Era5Profile.changeset(%{
|
||||
%NarrProfile{}
|
||||
|> NarrProfile.changeset(%{
|
||||
lat: 33.0,
|
||||
lon: -97.0,
|
||||
valid_time: ~U[2010-06-15 12:00:00Z],
|
||||
|
|
@ -629,13 +629,13 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
|
|||
})
|
||||
|> Repo.insert!()
|
||||
|
||||
jobs = ContactWeatherEnqueueWorker.build_era5_jobs([contact])
|
||||
jobs = ContactWeatherEnqueueWorker.build_narr_jobs([contact])
|
||||
assert jobs == []
|
||||
end
|
||||
|
||||
test "skips QSOs without pos1" do
|
||||
contact = create_contact(%{pos1: nil})
|
||||
assert ContactWeatherEnqueueWorker.build_era5_jobs([contact]) == []
|
||||
assert ContactWeatherEnqueueWorker.build_narr_jobs([contact]) == []
|
||||
end
|
||||
|
||||
test "deduplicates jobs with identical args across contacts" do
|
||||
|
|
@ -655,7 +655,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
|
|||
pos2: nil
|
||||
})
|
||||
|
||||
jobs = ContactWeatherEnqueueWorker.build_era5_jobs([q1, q2])
|
||||
jobs = ContactWeatherEnqueueWorker.build_narr_jobs([q1, q2])
|
||||
|
||||
# Both snap to 12:00Z and round to the same 0.25° grid cell → 1 job.
|
||||
assert length(jobs) == 1
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
|
|||
use Microwaveprop.DataCase, async: false
|
||||
use Oban.Testing, repo: Microwaveprop.Repo
|
||||
|
||||
alias Microwaveprop.Weather.Era5Profile
|
||||
alias Microwaveprop.Weather.NarrClient
|
||||
alias Microwaveprop.Weather.NarrProfile
|
||||
alias Microwaveprop.Workers.NarrFetchWorker
|
||||
|
||||
@inv_fixture "test/fixtures/narr/narr-a_221_20100615_1200_000.inv"
|
||||
|
|
@ -36,7 +36,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
|
|||
end
|
||||
|
||||
describe "perform/1" do
|
||||
test "fetches a profile via NarrClient and inserts it into era5_profiles" do
|
||||
test "fetches a profile via NarrClient and inserts it as a NarrProfile" do
|
||||
stub_narr_success()
|
||||
|
||||
valid_time = ~U[2010-06-15 12:00:00Z]
|
||||
|
|
@ -51,7 +51,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
|
|||
|
||||
# Row snapped to 0.25° grid (32.9 → 33.0, -97.0 → -97.0) and stored at
|
||||
# the exact 3-hourly valid_time.
|
||||
assert [row] = Repo.all(Era5Profile)
|
||||
assert [row] = Repo.all(NarrProfile)
|
||||
assert row.lat == 33.0
|
||||
assert row.lon == -97.0
|
||||
assert DateTime.compare(row.valid_time, valid_time) == :eq
|
||||
|
|
@ -62,7 +62,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
|
|||
assert_in_delta row.pwat_mm, 45.1, 2.0
|
||||
end
|
||||
|
||||
test "is a no-op when an era5_profiles row already exists for the snapped lat/lon/time" do
|
||||
test "is a no-op when a NarrProfile row already exists for the snapped lat/lon/time" do
|
||||
valid_time = ~U[2010-06-15 12:00:00Z]
|
||||
|
||||
# Stub that raises if called — this test asserts no HTTP happens.
|
||||
|
|
@ -70,9 +70,9 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
|
|||
raise "NarrClient should not be contacted when cache hit"
|
||||
end)
|
||||
|
||||
# Pre-populate era5_profiles with a row at the snapped coordinates.
|
||||
%Era5Profile{}
|
||||
|> Era5Profile.changeset(%{
|
||||
# Pre-populate with a row at the snapped coordinates.
|
||||
%NarrProfile{}
|
||||
|> NarrProfile.changeset(%{
|
||||
lat: 33.0,
|
||||
lon: -97.0,
|
||||
valid_time: valid_time,
|
||||
|
|
@ -89,7 +89,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
|
|||
assert :ok = NarrFetchWorker.perform(%Oban.Job{args: args})
|
||||
|
||||
# Only the pre-seeded row exists — no new insert.
|
||||
assert Repo.aggregate(Era5Profile, :count, :id) == 1
|
||||
assert Repo.aggregate(NarrProfile, :count, :id) == 1
|
||||
end
|
||||
|
||||
test "returns {:error, _} when NarrClient.fetch_profile_at fails" do
|
||||
|
|
@ -104,7 +104,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
|
|||
}
|
||||
|
||||
assert {:error, _reason} = NarrFetchWorker.perform(%Oban.Job{args: args})
|
||||
assert Repo.aggregate(Era5Profile, :count, :id) == 0
|
||||
assert Repo.aggregate(NarrProfile, :count, :id) == 0
|
||||
end
|
||||
|
||||
test "raises ArgumentError when valid_time is not on a 3-hourly NARR slot" do
|
||||
|
|
|
|||
|
|
@ -5,8 +5,8 @@ defmodule MicrowavepropWeb.ContactLiveTest do
|
|||
|
||||
alias Microwaveprop.Radio.Contact
|
||||
alias Microwaveprop.Repo
|
||||
alias Microwaveprop.Weather.Era5Profile
|
||||
alias Microwaveprop.Weather.HrrrNativeProfile
|
||||
alias Microwaveprop.Weather.NarrProfile
|
||||
|
||||
setup do
|
||||
Req.Test.stub(Microwaveprop.Terrain.ElevationClient, fn conn ->
|
||||
|
|
@ -156,8 +156,8 @@ defmodule MicrowavepropWeb.ContactLiveTest do
|
|||
end
|
||||
end
|
||||
|
||||
test "shows atmospheric profile section with ERA5 source for pre-HRRR contact", %{conn: conn} do
|
||||
# Pre-2014 contact — HRRR is unavailable, ERA5 is the fallback source.
|
||||
test "shows atmospheric profile section with NARR source for pre-HRRR contact", %{conn: conn} do
|
||||
# Pre-2014 contact — HRRR is unavailable, NARR is the fallback source.
|
||||
contact =
|
||||
create_contact(%{
|
||||
qso_timestamp: ~U[2010-06-15 18:00:00Z],
|
||||
|
|
@ -166,7 +166,7 @@ defmodule MicrowavepropWeb.ContactLiveTest do
|
|||
})
|
||||
|
||||
{:ok, _} =
|
||||
Repo.insert(%Era5Profile{
|
||||
Repo.insert(%NarrProfile{
|
||||
valid_time: ~U[2010-06-15 18:00:00Z],
|
||||
lat: 33.0,
|
||||
lon: -97.0,
|
||||
|
|
@ -187,8 +187,8 @@ defmodule MicrowavepropWeb.ContactLiveTest do
|
|||
|
||||
html = render_async(lv)
|
||||
assert html =~ "Atmospheric Profile"
|
||||
assert html =~ "ERA5"
|
||||
# Collapsed summary shows HPBL / PWAT / lat,lon from the injected ERA5 row.
|
||||
assert html =~ "NARR"
|
||||
# Collapsed summary shows HPBL / PWAT / lat,lon from the injected NARR row.
|
||||
assert html =~ "1200"
|
||||
assert html =~ "33.0"
|
||||
end
|
||||
|
|
@ -203,10 +203,10 @@ defmodule MicrowavepropWeb.ContactLiveTest do
|
|||
pos2: %{"lat" => 30.3, "lon" => -97.7}
|
||||
})
|
||||
|
||||
# ERA5 row keeps the atmospheric profile section rendering with its
|
||||
# NARR row keeps the atmospheric profile section rendering with its
|
||||
# aggregate metadata (HPBL / PWAT / surface scalars).
|
||||
{:ok, _} =
|
||||
Repo.insert(%Era5Profile{
|
||||
Repo.insert(%NarrProfile{
|
||||
valid_time: ~U[2024-06-15 18:00:00Z],
|
||||
lat: 32.9,
|
||||
lon: -97.0,
|
||||
|
|
@ -223,7 +223,7 @@ defmodule MicrowavepropWeb.ContactLiveTest do
|
|||
|
||||
# Native profile at the same point/time. `pressure_pa: [..., 30_000.0]`
|
||||
# corresponds to 300 mb — a level the pressure-level profile cannot
|
||||
# cover because HRRR/ERA5 stopped at 700 mb historically.
|
||||
# cover because HRRR/NARR pressure-level data stops at 700 mb historically.
|
||||
{:ok, _} =
|
||||
%HrrrNativeProfile{}
|
||||
|> HrrrNativeProfile.changeset(%{
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ defmodule MicrowavepropWeb.StatusLiveTest do
|
|||
alias Microwaveprop.Cache
|
||||
alias Microwaveprop.Radio.Contact
|
||||
alias Microwaveprop.Repo
|
||||
alias Microwaveprop.Weather.Era5Profile
|
||||
alias Microwaveprop.Weather.NarrProfile
|
||||
|
||||
setup %{conn: conn} do
|
||||
Cache.invalidate({MicrowavepropWeb.StatusLive, :count_unprocessed})
|
||||
|
|
@ -41,8 +41,8 @@ defmodule MicrowavepropWeb.StatusLiveTest do
|
|||
contact
|
||||
end
|
||||
|
||||
# NARR fetches land in era5_profiles (table name kept as historical artifact);
|
||||
# the status UI labels this "NARR" now.
|
||||
# NARR fetches land in era5_profiles (table rename is a follow-up migration).
|
||||
# The status UI labels this "NARR" now.
|
||||
test "NARR progress bar numerator is actual profile matches not total-minus-candidates", %{conn: conn} do
|
||||
create_narr_candidate()
|
||||
|
||||
|
|
@ -54,7 +54,7 @@ defmodule MicrowavepropWeb.StatusLiveTest do
|
|||
|
||||
# Insert a matching profile row — done should become 1
|
||||
{:ok, _} =
|
||||
Repo.insert(%Era5Profile{
|
||||
Repo.insert(%NarrProfile{
|
||||
valid_time: ~U[2010-06-15 18:00:00Z],
|
||||
lat: 33.0,
|
||||
lon: -97.0,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue