Rename ERA5 → NARR across the codebase
- Schema: Era5Profile → NarrProfile; table renamed via migration rename_era5_profiles_to_narr_profiles (ALTER TABLE + rename indexes, atomic and instant, no row rewrite) - Weather helpers: find_nearest_era5/era5_for_contact/era5_profiles_for_path → find_nearest_narr/narr_for_contact/narr_profiles_for_path - BackfillEnqueueWorker: :era5 → :narr type (virtual, no narr_status column); reconcile_stale_queued_to_unavailable and status_priority_order now skip virtual types via @virtual_types. Fixes the prod crash where the cron tried to update a non-existent era5_status column. - ContactWeatherEnqueueWorker: build_era5_jobs → build_narr_jobs, era5_jobs_for_contact → narr_jobs_for_contact - ContactLive: @era5 → @narr, maybe_enqueue_era5 → maybe_enqueue_narr; UI label "ERA5 (0.25°)" → "NARR (32 km)" - Cron (runtime.exs): args types list now "narr" instead of "era5" - /status: progress row, status-by-type card, totals.narr_profiles, and table-count lookup all target narr_profiles - Drop obsolete Era5CdsJob schema + era5_cds_jobs table (inspection artifact from the retired CDS pipeline; 34 orphan rows in prod) - Misc docstring/comment cleanups (skew_t, about, wgrib2, propagation_train) Includes a regression test for the virtual-type crash.
This commit is contained in:
parent
9e94759f7a
commit
3604896726
22 changed files with 235 additions and 258 deletions
|
|
@ -218,11 +218,11 @@ if config_env() == :prod do
|
||||||
# full grid-side JSONB write for.
|
# full grid-side JSONB write for.
|
||||||
# {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker},
|
# {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker},
|
||||||
{"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
|
{"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
|
||||||
# The :era5 type dispatches NarrFetchWorker for pre-2014 contacts
|
# The :narr type is virtual: it targets contacts with hrrr_status =
|
||||||
# (see ContactWeatherEnqueueWorker.era5_jobs_for_contact/1). The
|
# :unavailable (pre-2014, missing from the HRRR archive) and dispatches
|
||||||
# type key is a historical artifact — the data source is NARR.
|
# NarrFetchWorker against NCEI. See narr_jobs_for_contact/1.
|
||||||
{"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker,
|
{"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker,
|
||||||
args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre", "era5"]}},
|
args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre", "narr"]}},
|
||||||
# UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch
|
# UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch
|
||||||
{"30 1 * * *", CanadianSoundingFetchWorker},
|
{"30 1 * * *", CanadianSoundingFetchWorker},
|
||||||
{"30 13 * * *", CanadianSoundingFetchWorker},
|
{"30 13 * * *", CanadianSoundingFetchWorker},
|
||||||
|
|
|
||||||
|
|
@ -5,13 +5,13 @@ defmodule Microwaveprop.Weather do
|
||||||
|
|
||||||
alias Microwaveprop.Propagation.ProfilesFile
|
alias Microwaveprop.Propagation.ProfilesFile
|
||||||
alias Microwaveprop.Repo
|
alias Microwaveprop.Repo
|
||||||
alias Microwaveprop.Weather.Era5Profile
|
|
||||||
alias Microwaveprop.Weather.GridCache
|
alias Microwaveprop.Weather.GridCache
|
||||||
alias Microwaveprop.Weather.HrrrNativeProfile
|
alias Microwaveprop.Weather.HrrrNativeProfile
|
||||||
alias Microwaveprop.Weather.HrrrProfile
|
alias Microwaveprop.Weather.HrrrProfile
|
||||||
alias Microwaveprop.Weather.IemClient
|
alias Microwaveprop.Weather.IemClient
|
||||||
alias Microwaveprop.Weather.IemreObservation
|
alias Microwaveprop.Weather.IemreObservation
|
||||||
alias Microwaveprop.Weather.Metar5minObservation
|
alias Microwaveprop.Weather.Metar5minObservation
|
||||||
|
alias Microwaveprop.Weather.NarrProfile
|
||||||
alias Microwaveprop.Weather.RtmaObservation
|
alias Microwaveprop.Weather.RtmaObservation
|
||||||
alias Microwaveprop.Weather.SolarIndex
|
alias Microwaveprop.Weather.SolarIndex
|
||||||
alias Microwaveprop.Weather.Sounding
|
alias Microwaveprop.Weather.Sounding
|
||||||
|
|
@ -868,56 +868,56 @@ defmodule Microwaveprop.Weather do
|
||||||
|
|
||||||
@doc """
|
@doc """
|
||||||
Find the best available atmospheric profile for a contact.
|
Find the best available atmospheric profile for a contact.
|
||||||
Tries HRRR first (3 km, hourly), falls back to ERA5 (0.25°, hourly).
|
Tries HRRR first (3 km, hourly), falls back to NARR (32 km, 3-hourly).
|
||||||
Returns the profile struct or nil.
|
Returns the profile struct or nil.
|
||||||
"""
|
"""
|
||||||
@spec best_profile_for_contact(map()) :: HrrrProfile.t() | Era5Profile.t() | nil
|
@spec best_profile_for_contact(map()) :: HrrrProfile.t() | NarrProfile.t() | nil
|
||||||
def best_profile_for_contact(contact) do
|
def best_profile_for_contact(contact) do
|
||||||
hrrr_for_contact(contact) || era5_for_contact(contact)
|
hrrr_for_contact(contact) || narr_for_contact(contact)
|
||||||
end
|
end
|
||||||
|
|
||||||
@doc "Find all atmospheric profiles along a contact's path, from any source."
|
@doc "Find all atmospheric profiles along a contact's path, from any source."
|
||||||
@spec profiles_along_path(map()) :: [HrrrProfile.t() | Era5Profile.t()]
|
@spec profiles_along_path(map()) :: [HrrrProfile.t() | NarrProfile.t()]
|
||||||
def profiles_along_path(contact) do
|
def profiles_along_path(contact) do
|
||||||
hrrr_path = hrrr_profiles_for_path(contact)
|
hrrr_path = hrrr_profiles_for_path(contact)
|
||||||
|
|
||||||
if hrrr_path == [] do
|
if hrrr_path == [] do
|
||||||
era5_profiles_for_path(contact)
|
narr_profiles_for_path(contact)
|
||||||
else
|
else
|
||||||
hrrr_path
|
hrrr_path
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
@spec era5_for_contact(map()) :: Era5Profile.t() | nil
|
@spec narr_for_contact(map()) :: NarrProfile.t() | nil
|
||||||
def era5_for_contact(%{pos1: nil}), do: nil
|
def narr_for_contact(%{pos1: nil}), do: nil
|
||||||
|
|
||||||
def era5_for_contact(contact) do
|
def narr_for_contact(contact) do
|
||||||
lat = contact.pos1["lat"]
|
lat = contact.pos1["lat"]
|
||||||
lon = contact.pos1["lon"] || contact.pos1["lng"]
|
lon = contact.pos1["lon"] || contact.pos1["lng"]
|
||||||
|
|
||||||
if lat && lon do
|
if lat && lon do
|
||||||
find_nearest_era5(lat, lon, contact.qso_timestamp)
|
find_nearest_narr(lat, lon, contact.qso_timestamp)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
@spec era5_profiles_for_path(map()) :: [Era5Profile.t()]
|
@spec narr_profiles_for_path(map()) :: [NarrProfile.t()]
|
||||||
def era5_profiles_for_path(%{pos1: nil}), do: []
|
def narr_profiles_for_path(%{pos1: nil}), do: []
|
||||||
|
|
||||||
def era5_profiles_for_path(contact) do
|
def narr_profiles_for_path(contact) do
|
||||||
contact
|
contact
|
||||||
|> Microwaveprop.Radio.contact_path_points()
|
|> Microwaveprop.Radio.contact_path_points()
|
||||||
|> Enum.map(fn {lat, lon} -> find_nearest_era5(lat, lon, contact.qso_timestamp) end)
|
|> Enum.map(fn {lat, lon} -> find_nearest_narr(lat, lon, contact.qso_timestamp) end)
|
||||||
|> Enum.reject(&is_nil/1)
|
|> Enum.reject(&is_nil/1)
|
||||||
end
|
end
|
||||||
|
|
||||||
@spec find_nearest_era5(float(), float(), DateTime.t()) :: Era5Profile.t() | nil
|
@spec find_nearest_narr(float(), float(), DateTime.t()) :: NarrProfile.t() | nil
|
||||||
def find_nearest_era5(lat, lon, timestamp) do
|
def find_nearest_narr(lat, lon, timestamp) do
|
||||||
dlat = 0.15
|
dlat = 0.15
|
||||||
dlon = 0.15
|
dlon = 0.15
|
||||||
time_start = DateTime.add(timestamp, -1800, :second)
|
time_start = DateTime.add(timestamp, -1800, :second)
|
||||||
time_end = DateTime.add(timestamp, 1800, :second)
|
time_end = DateTime.add(timestamp, 1800, :second)
|
||||||
|
|
||||||
Era5Profile
|
NarrProfile
|
||||||
|> where(
|
|> where(
|
||||||
[p],
|
[p],
|
||||||
p.lat >= ^(lat - dlat) and p.lat <= ^(lat + dlat) and
|
p.lat >= ^(lat - dlat) and p.lat <= ^(lat + dlat) and
|
||||||
|
|
|
||||||
|
|
@ -1,44 +0,0 @@
|
||||||
defmodule Microwaveprop.Weather.Era5CdsJob do
|
|
||||||
@moduledoc """
|
|
||||||
Tracks an in-flight ERA5 CDS request submitted by `Era5SubmitWorker` and
|
|
||||||
awaiting download+decode by `Era5PollWorker`. Persisting the CDS job IDs
|
|
||||||
decouples the Oban worker lifecycle from the 30+ minute CDS queue time,
|
|
||||||
so a pod restart resumes polling instead of re-submitting.
|
|
||||||
"""
|
|
||||||
use Ecto.Schema
|
|
||||||
|
|
||||||
import Ecto.Changeset
|
|
||||||
|
|
||||||
@primary_key {:id, :binary_id, autogenerate: true}
|
|
||||||
@foreign_key_type :binary_id
|
|
||||||
|
|
||||||
schema "era5_cds_jobs" do
|
|
||||||
field :year, :integer
|
|
||||||
field :month, :integer
|
|
||||||
field :tile_lat, :integer
|
|
||||||
field :tile_lon, :integer
|
|
||||||
field :single_job_id, :string
|
|
||||||
field :pressure_job_id, :string
|
|
||||||
field :submitted_at, :utc_datetime
|
|
||||||
field :last_polled_at, :utc_datetime
|
|
||||||
field :poll_count, :integer, default: 0
|
|
||||||
|
|
||||||
timestamps(type: :utc_datetime)
|
|
||||||
end
|
|
||||||
|
|
||||||
@type t :: %__MODULE__{}
|
|
||||||
|
|
||||||
@required_fields ~w(year month tile_lat tile_lon single_job_id pressure_job_id submitted_at)a
|
|
||||||
@optional_fields ~w(last_polled_at poll_count)a
|
|
||||||
|
|
||||||
@spec changeset(t() | Ecto.Changeset.t(), map()) :: Ecto.Changeset.t()
|
|
||||||
def changeset(job, attrs) do
|
|
||||||
job
|
|
||||||
|> cast(attrs, @required_fields ++ @optional_fields)
|
|
||||||
|> validate_required(@required_fields)
|
|
||||||
|> unique_constraint([:year, :month, :tile_lat, :tile_lon],
|
|
||||||
name: :era5_cds_jobs_year_month_tile_lat_tile_lon_index,
|
|
||||||
error_key: :tile_month
|
|
||||||
)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
@ -336,7 +336,7 @@ defmodule Microwaveprop.Weather.Grib2.Wgrib2 do
|
||||||
@doc """
|
@doc """
|
||||||
Like `extract_grid/3`, but returns a flat list of per-message results so
|
Like `extract_grid/3`, but returns a flat list of per-message results so
|
||||||
the time dimension is preserved. Used when a single GRIB2 blob carries
|
the time dimension is preserved. Used when a single GRIB2 blob carries
|
||||||
many timesteps (e.g. a month of ERA5 data fetched in one CDS request).
|
many timesteps.
|
||||||
|
|
||||||
Each entry is `%{datetime: DateTime.t() | nil, var: String.t(),
|
Each entry is `%{datetime: DateTime.t() | nil, var: String.t(),
|
||||||
level: String.t(), values: %{{lat, lon} => float}}`.
|
level: String.t(), values: %{{lat, lon} => float}}`.
|
||||||
|
|
@ -354,7 +354,7 @@ defmodule Microwaveprop.Weather.Grib2.Wgrib2 do
|
||||||
| {:error, term()}
|
| {:error, term()}
|
||||||
def extract_grid_messages(grib_binary, match_pattern, grid_spec) do
|
def extract_grid_messages(grib_binary, match_pattern, grid_spec) do
|
||||||
if available?() do
|
if available?() do
|
||||||
tmp_grib = Path.join(System.tmp_dir!(), "era5_#{System.unique_integer([:positive])}.grib2")
|
tmp_grib = Path.join(System.tmp_dir!(), "wgrib2_#{System.unique_integer([:positive])}.grib2")
|
||||||
|
|
||||||
try do
|
try do
|
||||||
File.write!(tmp_grib, grib_binary)
|
File.write!(tmp_grib, grib_binary)
|
||||||
|
|
|
||||||
|
|
@ -141,13 +141,13 @@ defmodule Microwaveprop.Weather.NarrClient do
|
||||||
end
|
end
|
||||||
|
|
||||||
@doc """
|
@doc """
|
||||||
Extracts an `era5_profiles`-shaped attribute map from a composite NARR
|
Extracts a NarrProfile-shaped attribute map from a composite NARR
|
||||||
GRIB1 file at the given lat/lon point.
|
GRIB1 file at the given lat/lon point.
|
||||||
|
|
||||||
Pure-ish: shells out to `cdo -outputtab,name,lev,value -remapnn,lon=…_lat=…`
|
Pure-ish: shells out to `cdo -outputtab,name,lev,value -remapnn,lon=…_lat=…`
|
||||||
on the local file. No network.
|
on the local file. No network.
|
||||||
|
|
||||||
Returns `{:ok, attrs}` — a map with the `era5_profiles` fields
|
Returns `{:ok, attrs}` — a map with the NarrProfile fields
|
||||||
(`:surface_temp_c`, `:surface_dewpoint_c`, `:surface_pressure_mb`,
|
(`:surface_temp_c`, `:surface_dewpoint_c`, `:surface_pressure_mb`,
|
||||||
`:hpbl_m`, `:pwat_mm`, `:profile` list plus the derived fields from
|
`:hpbl_m`, `:pwat_mm`, `:profile` list plus the derived fields from
|
||||||
`SoundingParams.derive/1`) — or `{:error, reason}` on missing file,
|
`SoundingParams.derive/1`) — or `{:error, reason}` on missing file,
|
||||||
|
|
@ -170,8 +170,8 @@ defmodule Microwaveprop.Weather.NarrClient do
|
||||||
using `extract_profile_from_file/3`.
|
using `extract_profile_from_file/3`.
|
||||||
|
|
||||||
Returns `{:ok, profile_attrs}` shaped for direct `Repo.insert_all/3`
|
Returns `{:ok, profile_attrs}` shaped for direct `Repo.insert_all/3`
|
||||||
into the `era5_profiles` table, or `{:error, reason}`. Temp files are
|
against `NarrProfile`, or `{:error, reason}`. Temp files are always
|
||||||
always cleaned up via `try/after`.
|
cleaned up via `try/after`.
|
||||||
"""
|
"""
|
||||||
@spec fetch_profile_at(DateTime.t(), {float(), float()}) ::
|
@spec fetch_profile_at(DateTime.t(), {float(), float()}) ::
|
||||||
{:ok, map()} | {:error, term()}
|
{:ok, map()} | {:error, term()}
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,13 @@
|
||||||
defmodule Microwaveprop.Weather.Era5Profile do
|
defmodule Microwaveprop.Weather.NarrProfile do
|
||||||
@moduledoc false
|
@moduledoc """
|
||||||
|
Profile rows fetched from NCEP NARR (North American Regional Reanalysis)
|
||||||
|
for pre-2014 contacts where the HRRR archive doesn't reach.
|
||||||
|
|
||||||
|
The table was originally created as `era5_profiles` for the retired
|
||||||
|
ERA5/CDS pipeline and reused 1:1 by NARR; the
|
||||||
|
`rename_era5_profiles_to_narr_profiles` migration renames it to
|
||||||
|
`narr_profiles`.
|
||||||
|
"""
|
||||||
use Ecto.Schema
|
use Ecto.Schema
|
||||||
|
|
||||||
import Ecto.Changeset
|
import Ecto.Changeset
|
||||||
|
|
@ -7,7 +15,7 @@ defmodule Microwaveprop.Weather.Era5Profile do
|
||||||
@primary_key {:id, :binary_id, autogenerate: true}
|
@primary_key {:id, :binary_id, autogenerate: true}
|
||||||
@foreign_key_type :binary_id
|
@foreign_key_type :binary_id
|
||||||
|
|
||||||
schema "era5_profiles" do
|
schema "narr_profiles" do
|
||||||
field :valid_time, :utc_datetime
|
field :valid_time, :utc_datetime
|
||||||
field :lat, :float
|
field :lat, :float
|
||||||
field :lon, :float
|
field :lon, :float
|
||||||
|
|
@ -14,7 +14,12 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
|
||||||
require Logger
|
require Logger
|
||||||
|
|
||||||
@enrichable [:pending, :queued, :failed]
|
@enrichable [:pending, :queued, :failed]
|
||||||
@valid_types ~w(hrrr weather terrain iemre era5)
|
@valid_types ~w(hrrr weather terrain iemre narr)
|
||||||
|
|
||||||
|
# Virtual types have no <type>_status column on contacts; they're synthesized
|
||||||
|
# from other status fields (e.g. :narr candidates are `hrrr_status = :unavailable`).
|
||||||
|
# Skip these in any reconcile / ordering pass that reads a <type>_status field.
|
||||||
|
@virtual_types [:narr]
|
||||||
|
|
||||||
# A :queued contact whose updated_at is older than this is one the cron has
|
# A :queued contact whose updated_at is older than this is one the cron has
|
||||||
# already tried ~144 times over 3 days without the underlying worker landing
|
# already tried ~144 times over 3 days without the underlying worker landing
|
||||||
|
|
@ -67,30 +72,45 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
|
||||||
types |> Enum.filter(&(&1 in @valid_types)) |> Enum.map(&String.to_existing_atom/1)
|
types |> Enum.filter(&(&1 in @valid_types)) |> Enum.map(&String.to_existing_atom/1)
|
||||||
end
|
end
|
||||||
|
|
||||||
defp parse_types(_), do: [:hrrr, :weather, :terrain, :iemre, :era5]
|
defp parse_types(_), do: [:hrrr, :weather, :terrain, :iemre, :narr]
|
||||||
|
|
||||||
defp status_priority_order(types) do
|
defp status_priority_order(types) do
|
||||||
# Prioritize pending/failed over queued so real work happens first
|
# Prioritize pending/failed over queued so real work happens first.
|
||||||
field = :"#{hd(types)}_status"
|
# Virtual types (e.g. :narr) have no *_status column — fall back to
|
||||||
|
# time-only ordering when every requested type is virtual.
|
||||||
|
case Enum.find(types, &(&1 not in @virtual_types)) do
|
||||||
|
nil ->
|
||||||
|
[desc: dynamic([c], c.qso_timestamp)]
|
||||||
|
|
||||||
[
|
type ->
|
||||||
asc: dynamic([c], fragment("CASE ? WHEN 'pending' THEN 0 WHEN 'failed' THEN 1 ELSE 2 END", field(c, ^field))),
|
field = :"#{type}_status"
|
||||||
desc: dynamic([c], c.qso_timestamp)
|
|
||||||
]
|
[
|
||||||
|
asc:
|
||||||
|
dynamic(
|
||||||
|
[c],
|
||||||
|
fragment("CASE ? WHEN 'pending' THEN 0 WHEN 'failed' THEN 1 ELSE 2 END", field(c, ^field))
|
||||||
|
),
|
||||||
|
desc: dynamic([c], c.qso_timestamp)
|
||||||
|
]
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
# Flip contacts that have been stuck in :queued for longer than
|
# Flip contacts that have been stuck in :queued for longer than
|
||||||
# @stale_queued_cutoff_seconds to :unavailable. One Repo.update_all per type,
|
# @stale_queued_cutoff_seconds to :unavailable. One Repo.update_all per type,
|
||||||
# scoped to only the types passed in args so a partial cron (e.g. hrrr-only)
|
# scoped to only the types passed in args so a partial cron (e.g. hrrr-only)
|
||||||
# doesn't touch other statuses. Fast: hits the (<field>, updated_at) combo
|
# doesn't touch other statuses. Virtual types (e.g. :narr) have no *_status
|
||||||
# with a simple range scan.
|
# column and are skipped. Fast: hits the (<field>, updated_at) combo with a
|
||||||
|
# simple range scan.
|
||||||
defp reconcile_stale_queued_to_unavailable(types) do
|
defp reconcile_stale_queued_to_unavailable(types) do
|
||||||
cutoff =
|
cutoff =
|
||||||
DateTime.utc_now()
|
DateTime.utc_now()
|
||||||
|> DateTime.add(-@stale_queued_cutoff_seconds, :second)
|
|> DateTime.add(-@stale_queued_cutoff_seconds, :second)
|
||||||
|> DateTime.truncate(:second)
|
|> DateTime.truncate(:second)
|
||||||
|
|
||||||
Enum.reduce(types, 0, fn type, acc ->
|
types
|
||||||
|
|> Enum.reject(&(&1 in @virtual_types))
|
||||||
|
|> Enum.reduce(0, fn type, acc ->
|
||||||
field = :"#{type}_status"
|
field = :"#{type}_status"
|
||||||
|
|
||||||
{n, _} =
|
{n, _} =
|
||||||
|
|
@ -126,8 +146,8 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
|
||||||
|
|
||||||
defp type_filter(types) do
|
defp type_filter(types) do
|
||||||
Enum.reduce(types, dynamic(false), fn
|
Enum.reduce(types, dynamic(false), fn
|
||||||
:era5, acc ->
|
:narr, acc ->
|
||||||
# ERA5 targets contacts where HRRR is unavailable (pre-2014, missing from archive)
|
# NARR targets contacts where HRRR is unavailable (pre-2014, missing from archive)
|
||||||
dynamic([c], ^acc or c.hrrr_status == :unavailable)
|
dynamic([c], ^acc or c.hrrr_status == :unavailable)
|
||||||
|
|
||||||
type, acc ->
|
type, acc ->
|
||||||
|
|
|
||||||
|
|
@ -33,9 +33,9 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
|
||||||
insert_all_chunked(bulk_jobs)
|
insert_all_chunked(bulk_jobs)
|
||||||
end
|
end
|
||||||
|
|
||||||
# ERA5 jobs must go through Oban.insert/1 so the worker's unique
|
# NARR jobs must go through Oban.insert/1 so the worker's unique
|
||||||
# constraint is honored; Oban OSS insert_all does not check uniqueness.
|
# constraint is honored; Oban OSS insert_all does not check uniqueness.
|
||||||
insert_unique(jobs_by_type[:era5])
|
insert_unique(jobs_by_type[:narr])
|
||||||
|
|
||||||
mark_enrichment_statuses(contact, types, jobs_by_type)
|
mark_enrichment_statuses(contact, types, jobs_by_type)
|
||||||
|
|
||||||
|
|
@ -48,7 +48,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
|
||||||
hrrr: if(:hrrr in types, do: build_hrrr_jobs([contact]), else: []),
|
hrrr: if(:hrrr in types, do: build_hrrr_jobs([contact]), else: []),
|
||||||
terrain: if(:terrain in types, do: build_terrain_jobs([contact]), else: []),
|
terrain: if(:terrain in types, do: build_terrain_jobs([contact]), else: []),
|
||||||
iemre: if(:iemre in types, do: build_iemre_jobs([contact]), else: []),
|
iemre: if(:iemre in types, do: build_iemre_jobs([contact]), else: []),
|
||||||
era5: if(:era5 in types, do: build_era5_jobs([contact]), else: [])
|
narr: if(:narr in types, do: build_narr_jobs([contact]), else: [])
|
||||||
}
|
}
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
@ -173,18 +173,15 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
|
||||||
|> Enum.uniq_by(fn changeset -> changeset.changes.args end)
|
|> Enum.uniq_by(fn changeset -> changeset.changes.args end)
|
||||||
end
|
end
|
||||||
|
|
||||||
def build_era5_jobs(contacts) do
|
def build_narr_jobs(contacts) do
|
||||||
contacts
|
contacts
|
||||||
|> Enum.flat_map(&era5_jobs_for_contact/1)
|
|> Enum.flat_map(&narr_jobs_for_contact/1)
|
||||||
|> Enum.uniq_by(fn changeset -> changeset.changes.args end)
|
|> Enum.uniq_by(fn changeset -> changeset.changes.args end)
|
||||||
end
|
end
|
||||||
|
|
||||||
# The function name is a historical artifact: the `:era5` type key still
|
defp narr_jobs_for_contact(%{pos1: nil}), do: []
|
||||||
# refers to the `era5_profiles` table (reused 1:1), but this now dispatches
|
|
||||||
# `NarrFetchWorker` against NCEP NARR — the CDS/ERA5 pipeline is gone.
|
|
||||||
defp era5_jobs_for_contact(%{pos1: nil}), do: []
|
|
||||||
|
|
||||||
defp era5_jobs_for_contact(contact) do
|
defp narr_jobs_for_contact(contact) do
|
||||||
# NARR analyses are 3-hourly (00/03/06/09/12/15/18/21 UTC). NarrFetchWorker
|
# NARR analyses are 3-hourly (00/03/06/09/12/15/18/21 UTC). NarrFetchWorker
|
||||||
# rejects anything else, so snap here in the caller.
|
# rejects anything else, so snap here in the caller.
|
||||||
valid_time = NarrClient.snap_to_analysis_hour(contact.qso_timestamp)
|
valid_time = NarrClient.snap_to_analysis_hour(contact.qso_timestamp)
|
||||||
|
|
@ -195,7 +192,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
|
||||||
rlat = Float.round(lat * 4) / 4
|
rlat = Float.round(lat * 4) / 4
|
||||||
rlon = Float.round(lon * 4) / 4
|
rlon = Float.round(lon * 4) / 4
|
||||||
|
|
||||||
if Weather.find_nearest_era5(rlat, rlon, valid_time) do
|
if Weather.find_nearest_narr(rlat, rlon, valid_time) do
|
||||||
[]
|
[]
|
||||||
else
|
else
|
||||||
[
|
[
|
||||||
|
|
|
||||||
|
|
@ -1,12 +1,12 @@
|
||||||
defmodule Microwaveprop.Workers.NarrFetchWorker do
|
defmodule Microwaveprop.Workers.NarrFetchWorker do
|
||||||
@moduledoc """
|
@moduledoc """
|
||||||
Fetches one NCEP NARR profile from NCEI via `NarrClient.fetch_profile_at/2`
|
Fetches one NCEP NARR profile from NCEI via `NarrClient.fetch_profile_at/2`
|
||||||
and inserts it into the `era5_profiles` table.
|
and inserts it as a `NarrProfile` row.
|
||||||
|
|
||||||
Replaces `Era5FetchWorker` for pre-2014 contacts. Unlike the ERA5 router,
|
Covers pre-2014 contacts where the HRRR archive doesn't reach. Replaces
|
||||||
NARR fetches are per-point (one analysis hour, one location), so this
|
the retired ERA5/CDS pipeline. NARR fetches are per-point (one analysis
|
||||||
worker does the fetch + insert directly — no downstream batch worker,
|
hour, one location), so this worker does the fetch + insert directly —
|
||||||
no routing.
|
no downstream batch worker, no routing.
|
||||||
|
|
||||||
Jobs are unique on `{lat, lon, valid_time}` so duplicate enrichment
|
Jobs are unique on `{lat, lon, valid_time}` so duplicate enrichment
|
||||||
requests collapse. `valid_time` MUST already be snapped to a NARR
|
requests collapse. `valid_time` MUST already be snapped to a NARR
|
||||||
|
|
@ -15,8 +15,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorker do
|
||||||
`NarrClient.url_for/1`.
|
`NarrClient.url_for/1`.
|
||||||
|
|
||||||
See `docs/plans/2026-04-15-merra2-historical-backfill.md` for the full
|
See `docs/plans/2026-04-15-merra2-historical-backfill.md` for the full
|
||||||
architecture. The `era5_profiles` table is reused 1:1 (rename is a
|
architecture.
|
||||||
follow-up).
|
|
||||||
"""
|
"""
|
||||||
use Oban.Worker,
|
use Oban.Worker,
|
||||||
queue: :narr,
|
queue: :narr,
|
||||||
|
|
@ -29,8 +28,8 @@ defmodule Microwaveprop.Workers.NarrFetchWorker do
|
||||||
import Ecto.Query
|
import Ecto.Query
|
||||||
|
|
||||||
alias Microwaveprop.Repo
|
alias Microwaveprop.Repo
|
||||||
alias Microwaveprop.Weather.Era5Profile
|
|
||||||
alias Microwaveprop.Weather.NarrClient
|
alias Microwaveprop.Weather.NarrClient
|
||||||
|
alias Microwaveprop.Weather.NarrProfile
|
||||||
|
|
||||||
require Logger
|
require Logger
|
||||||
|
|
||||||
|
|
@ -41,7 +40,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorker do
|
||||||
rlat = Float.round(lat * 4) / 4
|
rlat = Float.round(lat * 4) / 4
|
||||||
rlon = Float.round(lon * 4) / 4
|
rlon = Float.round(lon * 4) / 4
|
||||||
|
|
||||||
if has_era5_profile?(rlat, rlon, valid_time) do
|
if has_narr_profile?(rlat, rlon, valid_time) do
|
||||||
:ok
|
:ok
|
||||||
else
|
else
|
||||||
fetch_and_insert(rlat, rlon, valid_time)
|
fetch_and_insert(rlat, rlon, valid_time)
|
||||||
|
|
@ -74,22 +73,21 @@ defmodule Microwaveprop.Workers.NarrFetchWorker do
|
||||||
|> Map.put(:inserted_at, now)
|
|> Map.put(:inserted_at, now)
|
||||||
|> Map.put(:updated_at, now)
|
|> Map.put(:updated_at, now)
|
||||||
|
|
||||||
Repo.insert_all(Era5Profile, [row],
|
Repo.insert_all(NarrProfile, [row],
|
||||||
on_conflict: :nothing,
|
on_conflict: :nothing,
|
||||||
conflict_target: [:lat, :lon, :valid_time]
|
conflict_target: [:lat, :lon, :valid_time]
|
||||||
)
|
)
|
||||||
end
|
end
|
||||||
|
|
||||||
defp has_era5_profile?(lat, lon, valid_time) do
|
defp has_narr_profile?(lat, lon, valid_time) do
|
||||||
# NARR is exact 3-hourly, so we tighten the time window to ±15 minutes
|
# NARR is exact 3-hourly, so the time window can be tight (±15 min).
|
||||||
# compared to Era5FetchWorker's ±30-minute check. Lat/lon tolerance
|
# Lat/lon tolerance matches the 0.25° dedup granularity.
|
||||||
# matches the 0.25° dedup granularity.
|
|
||||||
dlat = 0.13
|
dlat = 0.13
|
||||||
dlon = 0.13
|
dlon = 0.13
|
||||||
time_start = DateTime.add(valid_time, -900, :second)
|
time_start = DateTime.add(valid_time, -900, :second)
|
||||||
time_end = DateTime.add(valid_time, 900, :second)
|
time_end = DateTime.add(valid_time, 900, :second)
|
||||||
|
|
||||||
Era5Profile
|
NarrProfile
|
||||||
|> where(
|
|> where(
|
||||||
[p],
|
[p],
|
||||||
p.lat >= ^(lat - dlat) and p.lat <= ^(lat + dlat) and
|
p.lat >= ^(lat - dlat) and p.lat <= ^(lat + dlat) and
|
||||||
|
|
|
||||||
|
|
@ -172,7 +172,7 @@ defmodule MicrowavepropWeb.AboutLive do
|
||||||
forecasts), Iowa Environmental Mesonet (ASOS and upper-air
|
forecasts), Iowa Environmental Mesonet (ASOS and upper-air
|
||||||
soundings), IEMRE gridded reanalysis, SRTM 90 m terrain, SNMP
|
soundings), IEMRE gridded reanalysis, SRTM 90 m terrain, SNMP
|
||||||
polling on seven commercial microwave links near DFW at 5-minute
|
polling on seven commercial microwave links near DFW at 5-minute
|
||||||
intervals, and Copernicus ERA5 for pre-2014 contact enrichment.
|
intervals, and NCEP NARR for pre-2014 contact enrichment.
|
||||||
</li>
|
</li>
|
||||||
<li>
|
<li>
|
||||||
There's Nx/Axon/EXLA scaffolding for a small feed-forward model
|
There's Nx/Axon/EXLA scaffolding for a small feed-forward model
|
||||||
|
|
|
||||||
|
|
@ -46,8 +46,8 @@ defmodule MicrowavepropWeb.ContactLive.Show do
|
||||||
hrrr: nil,
|
hrrr: nil,
|
||||||
hrrr_path: [],
|
hrrr_path: [],
|
||||||
native_profile: nil,
|
native_profile: nil,
|
||||||
era5: nil,
|
narr: nil,
|
||||||
era5_path: [],
|
narr_path: [],
|
||||||
iemre: nil,
|
iemre: nil,
|
||||||
terrain: nil,
|
terrain: nil,
|
||||||
elevation_profile: nil,
|
elevation_profile: nil,
|
||||||
|
|
@ -91,7 +91,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
|
||||||
|> start_async(:weather, fn -> load_weather(contact) end)
|
|> start_async(:weather, fn -> load_weather(contact) end)
|
||||||
|> start_async(:solar, fn -> load_solar(contact) end)
|
|> start_async(:solar, fn -> load_solar(contact) end)
|
||||||
|> start_async(:hrrr_path, fn -> Weather.hrrr_profiles_for_path(contact) end)
|
|> start_async(:hrrr_path, fn -> Weather.hrrr_profiles_for_path(contact) end)
|
||||||
|> start_async(:era5_path, fn -> Weather.era5_profiles_for_path(contact) end)
|
|> start_async(:narr_path, fn -> Weather.narr_profiles_for_path(contact) end)
|
||||||
|> start_async(:native_profile, fn -> load_native_profile(contact) end)
|
|> start_async(:native_profile, fn -> load_native_profile(contact) end)
|
||||||
|> start_async(:terrain, fn -> Terrain.get_terrain_profile(contact.id) end)
|
|> start_async(:terrain, fn -> Terrain.get_terrain_profile(contact.id) end)
|
||||||
|> start_async(:iemre, fn -> load_iemre(contact) end)
|
|> start_async(:iemre, fn -> load_iemre(contact) end)
|
||||||
|
|
@ -328,14 +328,14 @@ defmodule MicrowavepropWeb.ContactLive.Show do
|
||||||
{:noreply, socket}
|
{:noreply, socket}
|
||||||
end
|
end
|
||||||
|
|
||||||
def handle_async(:era5_path, {:ok, era5_path}, socket) do
|
def handle_async(:narr_path, {:ok, narr_path}, socket) do
|
||||||
era5 = List.first(era5_path)
|
narr = List.first(narr_path)
|
||||||
contact = socket.assigns.contact
|
contact = socket.assigns.contact
|
||||||
|
|
||||||
contact =
|
contact =
|
||||||
if socket.assigns.can_enqueue, do: maybe_enqueue_era5(era5, contact), else: contact
|
if socket.assigns.can_enqueue, do: maybe_enqueue_narr(narr, contact), else: contact
|
||||||
|
|
||||||
{:noreply, assign(socket, contact: contact, era5: era5, era5_path: era5_path)}
|
{:noreply, assign(socket, contact: contact, narr: narr, narr_path: narr_path)}
|
||||||
end
|
end
|
||||||
|
|
||||||
def handle_async(:terrain, {:ok, terrain}, socket) do
|
def handle_async(:terrain, {:ok, terrain}, socket) do
|
||||||
|
|
@ -361,7 +361,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
|
||||||
end
|
end
|
||||||
|
|
||||||
def handle_async(slot, {:exit, reason}, socket)
|
def handle_async(slot, {:exit, reason}, socket)
|
||||||
when slot in [:weather, :solar, :hrrr_path, :era5_path, :native_profile, :terrain, :iemre] do
|
when slot in [:weather, :solar, :hrrr_path, :narr_path, :native_profile, :terrain, :iemre] do
|
||||||
Logger.warning("contact hydrate task #{slot} exited: #{inspect(reason)}")
|
Logger.warning("contact hydrate task #{slot} exited: #{inspect(reason)}")
|
||||||
{:noreply, socket}
|
{:noreply, socket}
|
||||||
end
|
end
|
||||||
|
|
@ -661,13 +661,12 @@ defmodule MicrowavepropWeb.ContactLive.Show do
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
# ERA5 is a fallback for contacts where HRRR is unavailable (pre-2014 or
|
# NARR is the fallback for contacts where HRRR is unavailable (pre-2014 or
|
||||||
# missing from the archive). Only enqueue when HRRR is known-unavailable AND
|
# missing from the archive). Only enqueue when HRRR is known-unavailable AND
|
||||||
# we don't already have an ERA5 profile — otherwise we'd waste CDS quota on
|
# we don't already have a NARR profile.
|
||||||
# contacts the HRRR path will cover.
|
defp maybe_enqueue_narr(narr, contact) when not is_nil(narr), do: contact
|
||||||
defp maybe_enqueue_era5(era5, contact) when not is_nil(era5), do: contact
|
|
||||||
|
|
||||||
defp maybe_enqueue_era5(nil, %{hrrr_status: :unavailable} = contact) do
|
defp maybe_enqueue_narr(nil, %{hrrr_status: :unavailable} = contact) do
|
||||||
lat = contact.pos1 && contact.pos1["lat"]
|
lat = contact.pos1 && contact.pos1["lat"]
|
||||||
lon = contact.pos1 && (contact.pos1["lon"] || contact.pos1["lng"])
|
lon = contact.pos1 && (contact.pos1["lon"] || contact.pos1["lng"])
|
||||||
|
|
||||||
|
|
@ -686,7 +685,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
|
||||||
contact
|
contact
|
||||||
end
|
end
|
||||||
|
|
||||||
defp maybe_enqueue_era5(_nil, contact), do: contact
|
defp maybe_enqueue_narr(_nil, contact), do: contact
|
||||||
|
|
||||||
defp queue_info(counts, queue) do
|
defp queue_info(counts, queue) do
|
||||||
case Map.get(counts, queue, 0) do
|
case Map.get(counts, queue, 0) do
|
||||||
|
|
@ -1140,11 +1139,11 @@ defmodule MicrowavepropWeb.ContactLive.Show do
|
||||||
<div class="divider" />
|
<div class="divider" />
|
||||||
|
|
||||||
<h2 class="text-base font-semibold mb-2">Atmospheric Profile</h2>
|
<h2 class="text-base font-semibold mb-2">Atmospheric Profile</h2>
|
||||||
<% profile = @hrrr || @era5 %>
|
<% profile = @hrrr || @narr %>
|
||||||
<% profile_source =
|
<% profile_source =
|
||||||
cond do
|
cond do
|
||||||
@hrrr -> "HRRR (3 km)"
|
@hrrr -> "HRRR (3 km)"
|
||||||
@era5 -> "ERA5 (0.25°)"
|
@narr -> "NARR (32 km)"
|
||||||
true -> nil
|
true -> nil
|
||||||
end %>
|
end %>
|
||||||
<% skew_t_levels =
|
<% skew_t_levels =
|
||||||
|
|
@ -1253,7 +1252,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
|
||||||
<% :unavailable -> %>
|
<% :unavailable -> %>
|
||||||
<span class="flex items-center gap-2">
|
<span class="flex items-center gap-2">
|
||||||
<span class="loading loading-spinner loading-sm"></span>
|
<span class="loading loading-spinner loading-sm"></span>
|
||||||
HRRR unavailable — fetching ERA5 reanalysis {queue_info(@queue_counts, "era5_batch")}
|
HRRR unavailable — fetching NARR reanalysis {queue_info(@queue_counts, "narr")}
|
||||||
</span>
|
</span>
|
||||||
<% _ -> %>
|
<% _ -> %>
|
||||||
No atmospheric profile available.
|
No atmospheric profile available.
|
||||||
|
|
|
||||||
|
|
@ -97,9 +97,8 @@ defmodule MicrowavepropWeb.StatusLive do
|
||||||
end
|
end
|
||||||
|
|
||||||
# NARR is the fallback for pre-2014 contacts (HRRR archive starts 2014).
|
# NARR is the fallback for pre-2014 contacts (HRRR archive starts 2014).
|
||||||
# Profiles land in the era5_profiles table (kept 1:1 after the ERA5/CDS
|
# "Done" means a narr_profiles row within 0.15° + 30 min of the contact,
|
||||||
# pipeline was replaced). "Done" means a row within 0.15° + 30 min of the
|
# matching `Weather.find_nearest_narr/3` lookup tolerance.
|
||||||
# contact, matching `Weather.find_nearest_era5/3` lookup tolerance.
|
|
||||||
defp count_narr_done do
|
defp count_narr_done do
|
||||||
%{rows: [[done]]} =
|
%{rows: [[done]]} =
|
||||||
Repo.query!("""
|
Repo.query!("""
|
||||||
|
|
@ -109,7 +108,7 @@ defmodule MicrowavepropWeb.StatusLive do
|
||||||
AND c.pos1 IS NOT NULL
|
AND c.pos1 IS NOT NULL
|
||||||
AND EXISTS (
|
AND EXISTS (
|
||||||
SELECT 1
|
SELECT 1
|
||||||
FROM era5_profiles p
|
FROM narr_profiles p
|
||||||
WHERE p.lat BETWEEN ((c.pos1->>'lat')::float - 0.15) AND ((c.pos1->>'lat')::float + 0.15)
|
WHERE p.lat BETWEEN ((c.pos1->>'lat')::float - 0.15) AND ((c.pos1->>'lat')::float + 0.15)
|
||||||
AND p.lon BETWEEN ((c.pos1->>'lon')::float - 0.15) AND ((c.pos1->>'lon')::float + 0.15)
|
AND p.lon BETWEEN ((c.pos1->>'lon')::float - 0.15) AND ((c.pos1->>'lon')::float + 0.15)
|
||||||
AND p.valid_time BETWEEN (c.qso_timestamp - INTERVAL '30 minutes') AND (c.qso_timestamp + INTERVAL '30 minutes')
|
AND p.valid_time BETWEEN (c.qso_timestamp - INTERVAL '30 minutes') AND (c.qso_timestamp + INTERVAL '30 minutes')
|
||||||
|
|
@ -264,8 +263,8 @@ defmodule MicrowavepropWeb.StatusLive do
|
||||||
table_counts = Map.new(tables, fn %{table: t, rows: r} -> {t, r} end)
|
table_counts = Map.new(tables, fn %{table: t, rows: r} -> {t, r} end)
|
||||||
|
|
||||||
# NARR doesn't have its own status column — synthesize from HRRR unavailable
|
# NARR doesn't have its own status column — synthesize from HRRR unavailable
|
||||||
# + era5_profiles row count (the table is reused as the NARR landing spot).
|
# + narr_profiles row count.
|
||||||
narr_count = Map.get(table_counts, "era5_profiles", 0)
|
narr_count = Map.get(table_counts, "narr_profiles", 0)
|
||||||
|
|
||||||
hrrr_unavailable =
|
hrrr_unavailable =
|
||||||
statuses |> Map.get("hrrr", []) |> Enum.find_value(0, fn {s, c} -> if s == "unavailable", do: c end)
|
statuses |> Map.get("hrrr", []) |> Enum.find_value(0, fn {s, c} -> if s == "unavailable", do: c end)
|
||||||
|
|
@ -289,7 +288,7 @@ defmodule MicrowavepropWeb.StatusLive do
|
||||||
statuses: statuses,
|
statuses: statuses,
|
||||||
totals: %{
|
totals: %{
|
||||||
hrrr_profiles: hrrr_count,
|
hrrr_profiles: hrrr_count,
|
||||||
narr_profiles: Map.get(table_counts, "era5_profiles", 0),
|
narr_profiles: Map.get(table_counts, "narr_profiles", 0),
|
||||||
terrain_profiles: terrain_count,
|
terrain_profiles: terrain_count,
|
||||||
surface_observations: obs_count,
|
surface_observations: obs_count,
|
||||||
soundings: sounding_count,
|
soundings: sounding_count,
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ defmodule MicrowavepropWeb.SkewT do
|
||||||
Skew-T log-P diagram renderer. Builds the full set of SVG
|
Skew-T log-P diagram renderer. Builds the full set of SVG
|
||||||
primitives (isobars, isotherms, dry adiabats, saturation mixing
|
primitives (isobars, isotherms, dry adiabats, saturation mixing
|
||||||
ratio lines, and temperature/dewpoint traces) for a single HRRR
|
ratio lines, and temperature/dewpoint traces) for a single HRRR
|
||||||
or ERA5 pressure-level profile so the contact detail template can
|
or NARR pressure-level profile so the contact detail template can
|
||||||
draw it declaratively.
|
draw it declaratively.
|
||||||
|
|
||||||
## Meteorology
|
## Meteorology
|
||||||
|
|
|
||||||
|
|
@ -59,7 +59,7 @@ defmodule Mix.Tasks.PropagationTrain do
|
||||||
)
|
)
|
||||||
|
|
||||||
# Boot the app without starting Oban queues so training doesn't have
|
# Boot the app without starting Oban queues so training doesn't have
|
||||||
# the HRRR / ERA5 / weather cron firing in the background against the
|
# the HRRR / NARR / weather cron firing in the background against the
|
||||||
# local DB we're reading from.
|
# local DB we're reading from.
|
||||||
Application.put_env(
|
Application.put_env(
|
||||||
:microwaveprop,
|
:microwaveprop,
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,27 @@
|
||||||
|
defmodule Microwaveprop.Repo.Migrations.RenameEra5ProfilesToNarrProfiles do
|
||||||
|
use Ecto.Migration
|
||||||
|
|
||||||
|
# ALTER TABLE ... RENAME is atomic and instant (no row rewrite). User-created
|
||||||
|
# indexes keep their old names after the table rename, so we rename them too
|
||||||
|
# for consistency. Implicit PK index / sequence are renamed automatically by
|
||||||
|
# PostgreSQL.
|
||||||
|
def up do
|
||||||
|
rename table(:era5_profiles), to: table(:narr_profiles)
|
||||||
|
|
||||||
|
execute "ALTER INDEX era5_profiles_lat_lon_valid_time_index RENAME TO narr_profiles_lat_lon_valid_time_index"
|
||||||
|
|
||||||
|
execute "ALTER INDEX era5_profiles_valid_time_index RENAME TO narr_profiles_valid_time_index"
|
||||||
|
|
||||||
|
execute "ALTER INDEX era5_profiles_valid_time_lat_lon_index RENAME TO narr_profiles_valid_time_lat_lon_index"
|
||||||
|
end
|
||||||
|
|
||||||
|
def down do
|
||||||
|
execute "ALTER INDEX narr_profiles_valid_time_lat_lon_index RENAME TO era5_profiles_valid_time_lat_lon_index"
|
||||||
|
|
||||||
|
execute "ALTER INDEX narr_profiles_valid_time_index RENAME TO era5_profiles_valid_time_index"
|
||||||
|
|
||||||
|
execute "ALTER INDEX narr_profiles_lat_lon_valid_time_index RENAME TO era5_profiles_lat_lon_valid_time_index"
|
||||||
|
|
||||||
|
rename table(:narr_profiles), to: table(:era5_profiles)
|
||||||
|
end
|
||||||
|
end
|
||||||
29
priv/repo/migrations/20260416142116_drop_era5_cds_jobs.exs
Normal file
29
priv/repo/migrations/20260416142116_drop_era5_cds_jobs.exs
Normal file
|
|
@ -0,0 +1,29 @@
|
||||||
|
defmodule Microwaveprop.Repo.Migrations.DropEra5CdsJobs do
|
||||||
|
use Ecto.Migration
|
||||||
|
|
||||||
|
# The ERA5/CDS pipeline was retired in favor of NARR (see
|
||||||
|
# rename_era5_profiles_to_narr_profiles). Orphan rows in this table are
|
||||||
|
# useless without the workers that wrote them.
|
||||||
|
def up do
|
||||||
|
drop table(:era5_cds_jobs)
|
||||||
|
end
|
||||||
|
|
||||||
|
def down do
|
||||||
|
create table(:era5_cds_jobs, primary_key: false) do
|
||||||
|
add :id, :binary_id, primary_key: true
|
||||||
|
add :year, :integer, null: false
|
||||||
|
add :month, :integer, null: false
|
||||||
|
add :tile_lat, :integer, null: false
|
||||||
|
add :tile_lon, :integer, null: false
|
||||||
|
add :single_job_id, :string, null: false
|
||||||
|
add :pressure_job_id, :string, null: false
|
||||||
|
add :submitted_at, :utc_datetime, null: false
|
||||||
|
add :last_polled_at, :utc_datetime
|
||||||
|
add :poll_count, :integer, default: 0, null: false
|
||||||
|
|
||||||
|
timestamps(type: :utc_datetime)
|
||||||
|
end
|
||||||
|
|
||||||
|
create unique_index(:era5_cds_jobs, [:year, :month, :tile_lat, :tile_lon])
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
@ -1,82 +0,0 @@
|
||||||
defmodule Microwaveprop.Weather.Era5CdsJobTest do
|
|
||||||
use Microwaveprop.DataCase, async: true
|
|
||||||
|
|
||||||
alias Microwaveprop.Weather.Era5CdsJob
|
|
||||||
|
|
||||||
describe "changeset/2" do
|
|
||||||
@valid_attrs %{
|
|
||||||
year: 2014,
|
|
||||||
month: 3,
|
|
||||||
tile_lat: 32,
|
|
||||||
tile_lon: -98,
|
|
||||||
single_job_id: "abc-single",
|
|
||||||
pressure_job_id: "abc-pressure",
|
|
||||||
submitted_at: ~U[2026-04-13 21:00:00Z]
|
|
||||||
}
|
|
||||||
|
|
||||||
test "accepts a complete attrs map" do
|
|
||||||
changeset = Era5CdsJob.changeset(%Era5CdsJob{}, @valid_attrs)
|
|
||||||
assert changeset.valid?
|
|
||||||
end
|
|
||||||
|
|
||||||
test "requires year, month, tile coordinates, and both CDS job IDs" do
|
|
||||||
changeset = Era5CdsJob.changeset(%Era5CdsJob{}, %{})
|
|
||||||
|
|
||||||
refute changeset.valid?
|
|
||||||
|
|
||||||
required = ~w(year month tile_lat tile_lon single_job_id pressure_job_id submitted_at)a
|
|
||||||
errors = Keyword.keys(changeset.errors)
|
|
||||||
|
|
||||||
for field <- required do
|
|
||||||
assert field in errors, "expected #{inspect(field)} to be required"
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
test "last_polled_at and poll_count default to nil/0 and can be updated" do
|
|
||||||
{:ok, row} =
|
|
||||||
%Era5CdsJob{}
|
|
||||||
|> Era5CdsJob.changeset(@valid_attrs)
|
|
||||||
|> Repo.insert()
|
|
||||||
|
|
||||||
assert row.poll_count == 0
|
|
||||||
assert is_nil(row.last_polled_at)
|
|
||||||
|
|
||||||
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
||||||
|
|
||||||
{:ok, bumped} =
|
|
||||||
row
|
|
||||||
|> Era5CdsJob.changeset(%{last_polled_at: now, poll_count: 1})
|
|
||||||
|> Repo.update()
|
|
||||||
|
|
||||||
assert bumped.poll_count == 1
|
|
||||||
assert bumped.last_polled_at == now
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
describe "uniqueness" do
|
|
||||||
test "collapses duplicate (year, month, tile_lat, tile_lon) inserts" do
|
|
||||||
attrs = %{
|
|
||||||
year: 2014,
|
|
||||||
month: 3,
|
|
||||||
tile_lat: 32,
|
|
||||||
tile_lon: -98,
|
|
||||||
single_job_id: "first-single",
|
|
||||||
pressure_job_id: "first-pressure",
|
|
||||||
submitted_at: ~U[2026-04-13 21:00:00Z]
|
|
||||||
}
|
|
||||||
|
|
||||||
{:ok, _} =
|
|
||||||
%Era5CdsJob{}
|
|
||||||
|> Era5CdsJob.changeset(attrs)
|
|
||||||
|> Repo.insert()
|
|
||||||
|
|
||||||
{:error, changeset} =
|
|
||||||
%Era5CdsJob{}
|
|
||||||
|> Era5CdsJob.changeset(%{attrs | single_job_id: "second-single", pressure_job_id: "second-pressure"})
|
|
||||||
|> Repo.insert()
|
|
||||||
|
|
||||||
refute changeset.valid?
|
|
||||||
assert {:tile_month, _} = List.keyfind(changeset.errors, :tile_month, 0) || {:tile_month, nil}
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
@ -5,6 +5,7 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorkerTest do
|
||||||
alias Microwaveprop.Terrain.ElevationClient
|
alias Microwaveprop.Terrain.ElevationClient
|
||||||
alias Microwaveprop.Weather.HrrrClient
|
alias Microwaveprop.Weather.HrrrClient
|
||||||
alias Microwaveprop.Weather.IemClient
|
alias Microwaveprop.Weather.IemClient
|
||||||
|
alias Microwaveprop.Weather.NarrClient
|
||||||
alias Microwaveprop.Workers.BackfillEnqueueWorker
|
alias Microwaveprop.Workers.BackfillEnqueueWorker
|
||||||
|
|
||||||
defp create_contact(attrs \\ %{}) do
|
defp create_contact(attrs \\ %{}) do
|
||||||
|
|
@ -53,12 +54,19 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorkerTest do
|
||||||
Req.Test.json(conn, %{"elevation" => List.duplicate(200.0, lat_count)})
|
Req.Test.json(conn, %{"elevation" => List.duplicate(200.0, lat_count)})
|
||||||
end)
|
end)
|
||||||
|
|
||||||
|
# NARR fetches land in era5_profiles; stub returns 500 so inline jobs fail
|
||||||
|
# gracefully instead of hitting NCEI.
|
||||||
|
Req.Test.stub(NarrClient, fn conn ->
|
||||||
|
Plug.Conn.send_resp(conn, 500, "stub")
|
||||||
|
end)
|
||||||
|
|
||||||
:ok
|
:ok
|
||||||
end
|
end
|
||||||
|
|
||||||
# ERA5 goes through Copernicus CDS API which is async (submit → poll → download)
|
# NARR fetches (the :era5 type) run inline via testing: :inline — the
|
||||||
# and has no Req.Test plug hook, so inline Oban would block on Process.sleep.
|
# NarrClient Req.Test stub above keeps them from hitting NCEI. Most tests
|
||||||
# These tests exercise the non-ERA5 enrichment path; ERA5 has its own coverage.
|
# below target the other pipelines and use this narrowed type list to keep
|
||||||
|
# the test surface tight.
|
||||||
@non_era5_types ["hrrr", "weather", "terrain", "iemre"]
|
@non_era5_types ["hrrr", "weather", "terrain", "iemre"]
|
||||||
|
|
||||||
describe "perform/1" do
|
describe "perform/1" do
|
||||||
|
|
@ -113,6 +121,24 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorkerTest do
|
||||||
|
|
||||||
assert_receive {:enqueue_complete, 1}
|
assert_receive {:enqueue_complete, 1}
|
||||||
end
|
end
|
||||||
|
|
||||||
|
# :narr is a virtual type (no narr_status column) — it targets contacts
|
||||||
|
# whose hrrr_status is :unavailable. BackfillEnqueueWorker must not try
|
||||||
|
# to reconcile a non-existent narr_status column or use it for priority
|
||||||
|
# ordering.
|
||||||
|
test "handles virtual :narr type without referencing non-existent narr_status column" do
|
||||||
|
# Pre-2014 contact (HRRR unavailable) — NARR candidate.
|
||||||
|
contact = create_contact(%{qso_timestamp: ~U[2010-01-01 00:00:00Z]})
|
||||||
|
|
||||||
|
contact
|
||||||
|
|> Ecto.Changeset.change(%{hrrr_status: :unavailable})
|
||||||
|
|> Repo.update!()
|
||||||
|
|
||||||
|
assert :ok =
|
||||||
|
BackfillEnqueueWorker.perform(%Oban.Job{
|
||||||
|
args: %{"limit" => 10, "types" => ["narr"]}
|
||||||
|
})
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
describe "reconcile stuck queued → unavailable" do
|
describe "reconcile stuck queued → unavailable" do
|
||||||
|
|
|
||||||
|
|
@ -4,9 +4,9 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
|
||||||
alias Microwaveprop.Radio.Contact
|
alias Microwaveprop.Radio.Contact
|
||||||
alias Microwaveprop.Terrain.ElevationClient
|
alias Microwaveprop.Terrain.ElevationClient
|
||||||
alias Microwaveprop.Weather
|
alias Microwaveprop.Weather
|
||||||
alias Microwaveprop.Weather.Era5Profile
|
|
||||||
alias Microwaveprop.Weather.HrrrClient
|
alias Microwaveprop.Weather.HrrrClient
|
||||||
alias Microwaveprop.Weather.IemClient
|
alias Microwaveprop.Weather.IemClient
|
||||||
|
alias Microwaveprop.Weather.NarrProfile
|
||||||
alias Microwaveprop.Workers.ContactWeatherEnqueueWorker
|
alias Microwaveprop.Workers.ContactWeatherEnqueueWorker
|
||||||
|
|
||||||
@qso_attrs %{
|
@qso_attrs %{
|
||||||
|
|
@ -562,7 +562,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
describe "build_era5_jobs/1" do
|
describe "build_narr_jobs/1" do
|
||||||
test "builds NarrFetchWorker jobs for each rounded path point" do
|
test "builds NarrFetchWorker jobs for each rounded path point" do
|
||||||
contact =
|
contact =
|
||||||
create_contact(%{
|
create_contact(%{
|
||||||
|
|
@ -571,7 +571,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
|
||||||
pos2: nil
|
pos2: nil
|
||||||
})
|
})
|
||||||
|
|
||||||
jobs = ContactWeatherEnqueueWorker.build_era5_jobs([contact])
|
jobs = ContactWeatherEnqueueWorker.build_narr_jobs([contact])
|
||||||
|
|
||||||
assert length(jobs) == 1
|
assert length(jobs) == 1
|
||||||
job = hd(jobs)
|
job = hd(jobs)
|
||||||
|
|
@ -589,7 +589,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
|
||||||
pos2: nil
|
pos2: nil
|
||||||
})
|
})
|
||||||
|
|
||||||
jobs = ContactWeatherEnqueueWorker.build_era5_jobs([contact])
|
jobs = ContactWeatherEnqueueWorker.build_narr_jobs([contact])
|
||||||
|
|
||||||
job = hd(jobs)
|
job = hd(jobs)
|
||||||
# 13:42 UTC snaps down to the 12:00 UTC NARR analysis slot.
|
# 13:42 UTC snaps down to the 12:00 UTC NARR analysis slot.
|
||||||
|
|
@ -604,14 +604,14 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
|
||||||
pos2: %{"lat" => 30.3, "lon" => -97.7}
|
pos2: %{"lat" => 30.3, "lon" => -97.7}
|
||||||
})
|
})
|
||||||
|
|
||||||
jobs = ContactWeatherEnqueueWorker.build_era5_jobs([contact])
|
jobs = ContactWeatherEnqueueWorker.build_narr_jobs([contact])
|
||||||
|
|
||||||
# pos1, midpoint, pos2 round to three distinct 0.25° grid cells.
|
# pos1, midpoint, pos2 round to three distinct 0.25° grid cells.
|
||||||
assert length(jobs) == 3
|
assert length(jobs) == 3
|
||||||
assert Enum.all?(jobs, &(&1.changes.worker == "Microwaveprop.Workers.NarrFetchWorker"))
|
assert Enum.all?(jobs, &(&1.changes.worker == "Microwaveprop.Workers.NarrFetchWorker"))
|
||||||
end
|
end
|
||||||
|
|
||||||
test "skips path points that already have an era5_profile row" do
|
test "skips path points that already have a narr_profile row" do
|
||||||
contact =
|
contact =
|
||||||
create_contact(%{
|
create_contact(%{
|
||||||
qso_timestamp: ~U[2010-06-15 13:42:00Z],
|
qso_timestamp: ~U[2010-06-15 13:42:00Z],
|
||||||
|
|
@ -620,8 +620,8 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
|
||||||
})
|
})
|
||||||
|
|
||||||
# pos1 (32.907, -97.038) snaps to the (33.0, -97.0) 0.25° grid cell
|
# pos1 (32.907, -97.038) snaps to the (33.0, -97.0) 0.25° grid cell
|
||||||
%Era5Profile{}
|
%NarrProfile{}
|
||||||
|> Era5Profile.changeset(%{
|
|> NarrProfile.changeset(%{
|
||||||
lat: 33.0,
|
lat: 33.0,
|
||||||
lon: -97.0,
|
lon: -97.0,
|
||||||
valid_time: ~U[2010-06-15 12:00:00Z],
|
valid_time: ~U[2010-06-15 12:00:00Z],
|
||||||
|
|
@ -629,13 +629,13 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
|
||||||
})
|
})
|
||||||
|> Repo.insert!()
|
|> Repo.insert!()
|
||||||
|
|
||||||
jobs = ContactWeatherEnqueueWorker.build_era5_jobs([contact])
|
jobs = ContactWeatherEnqueueWorker.build_narr_jobs([contact])
|
||||||
assert jobs == []
|
assert jobs == []
|
||||||
end
|
end
|
||||||
|
|
||||||
test "skips QSOs without pos1" do
|
test "skips QSOs without pos1" do
|
||||||
contact = create_contact(%{pos1: nil})
|
contact = create_contact(%{pos1: nil})
|
||||||
assert ContactWeatherEnqueueWorker.build_era5_jobs([contact]) == []
|
assert ContactWeatherEnqueueWorker.build_narr_jobs([contact]) == []
|
||||||
end
|
end
|
||||||
|
|
||||||
test "deduplicates jobs with identical args across contacts" do
|
test "deduplicates jobs with identical args across contacts" do
|
||||||
|
|
@ -655,7 +655,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
|
||||||
pos2: nil
|
pos2: nil
|
||||||
})
|
})
|
||||||
|
|
||||||
jobs = ContactWeatherEnqueueWorker.build_era5_jobs([q1, q2])
|
jobs = ContactWeatherEnqueueWorker.build_narr_jobs([q1, q2])
|
||||||
|
|
||||||
# Both snap to 12:00Z and round to the same 0.25° grid cell → 1 job.
|
# Both snap to 12:00Z and round to the same 0.25° grid cell → 1 job.
|
||||||
assert length(jobs) == 1
|
assert length(jobs) == 1
|
||||||
|
|
|
||||||
|
|
@ -2,8 +2,8 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
|
||||||
use Microwaveprop.DataCase, async: false
|
use Microwaveprop.DataCase, async: false
|
||||||
use Oban.Testing, repo: Microwaveprop.Repo
|
use Oban.Testing, repo: Microwaveprop.Repo
|
||||||
|
|
||||||
alias Microwaveprop.Weather.Era5Profile
|
|
||||||
alias Microwaveprop.Weather.NarrClient
|
alias Microwaveprop.Weather.NarrClient
|
||||||
|
alias Microwaveprop.Weather.NarrProfile
|
||||||
alias Microwaveprop.Workers.NarrFetchWorker
|
alias Microwaveprop.Workers.NarrFetchWorker
|
||||||
|
|
||||||
@inv_fixture "test/fixtures/narr/narr-a_221_20100615_1200_000.inv"
|
@inv_fixture "test/fixtures/narr/narr-a_221_20100615_1200_000.inv"
|
||||||
|
|
@ -36,7 +36,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
|
||||||
end
|
end
|
||||||
|
|
||||||
describe "perform/1" do
|
describe "perform/1" do
|
||||||
test "fetches a profile via NarrClient and inserts it into era5_profiles" do
|
test "fetches a profile via NarrClient and inserts it as a NarrProfile" do
|
||||||
stub_narr_success()
|
stub_narr_success()
|
||||||
|
|
||||||
valid_time = ~U[2010-06-15 12:00:00Z]
|
valid_time = ~U[2010-06-15 12:00:00Z]
|
||||||
|
|
@ -51,7 +51,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
|
||||||
|
|
||||||
# Row snapped to 0.25° grid (32.9 → 33.0, -97.0 → -97.0) and stored at
|
# Row snapped to 0.25° grid (32.9 → 33.0, -97.0 → -97.0) and stored at
|
||||||
# the exact 3-hourly valid_time.
|
# the exact 3-hourly valid_time.
|
||||||
assert [row] = Repo.all(Era5Profile)
|
assert [row] = Repo.all(NarrProfile)
|
||||||
assert row.lat == 33.0
|
assert row.lat == 33.0
|
||||||
assert row.lon == -97.0
|
assert row.lon == -97.0
|
||||||
assert DateTime.compare(row.valid_time, valid_time) == :eq
|
assert DateTime.compare(row.valid_time, valid_time) == :eq
|
||||||
|
|
@ -62,7 +62,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
|
||||||
assert_in_delta row.pwat_mm, 45.1, 2.0
|
assert_in_delta row.pwat_mm, 45.1, 2.0
|
||||||
end
|
end
|
||||||
|
|
||||||
test "is a no-op when an era5_profiles row already exists for the snapped lat/lon/time" do
|
test "is a no-op when a NarrProfile row already exists for the snapped lat/lon/time" do
|
||||||
valid_time = ~U[2010-06-15 12:00:00Z]
|
valid_time = ~U[2010-06-15 12:00:00Z]
|
||||||
|
|
||||||
# Stub that raises if called — this test asserts no HTTP happens.
|
# Stub that raises if called — this test asserts no HTTP happens.
|
||||||
|
|
@ -70,9 +70,9 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
|
||||||
raise "NarrClient should not be contacted when cache hit"
|
raise "NarrClient should not be contacted when cache hit"
|
||||||
end)
|
end)
|
||||||
|
|
||||||
# Pre-populate era5_profiles with a row at the snapped coordinates.
|
# Pre-populate with a row at the snapped coordinates.
|
||||||
%Era5Profile{}
|
%NarrProfile{}
|
||||||
|> Era5Profile.changeset(%{
|
|> NarrProfile.changeset(%{
|
||||||
lat: 33.0,
|
lat: 33.0,
|
||||||
lon: -97.0,
|
lon: -97.0,
|
||||||
valid_time: valid_time,
|
valid_time: valid_time,
|
||||||
|
|
@ -89,7 +89,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
|
||||||
assert :ok = NarrFetchWorker.perform(%Oban.Job{args: args})
|
assert :ok = NarrFetchWorker.perform(%Oban.Job{args: args})
|
||||||
|
|
||||||
# Only the pre-seeded row exists — no new insert.
|
# Only the pre-seeded row exists — no new insert.
|
||||||
assert Repo.aggregate(Era5Profile, :count, :id) == 1
|
assert Repo.aggregate(NarrProfile, :count, :id) == 1
|
||||||
end
|
end
|
||||||
|
|
||||||
test "returns {:error, _} when NarrClient.fetch_profile_at fails" do
|
test "returns {:error, _} when NarrClient.fetch_profile_at fails" do
|
||||||
|
|
@ -104,7 +104,7 @@ defmodule Microwaveprop.Workers.NarrFetchWorkerTest do
|
||||||
}
|
}
|
||||||
|
|
||||||
assert {:error, _reason} = NarrFetchWorker.perform(%Oban.Job{args: args})
|
assert {:error, _reason} = NarrFetchWorker.perform(%Oban.Job{args: args})
|
||||||
assert Repo.aggregate(Era5Profile, :count, :id) == 0
|
assert Repo.aggregate(NarrProfile, :count, :id) == 0
|
||||||
end
|
end
|
||||||
|
|
||||||
test "raises ArgumentError when valid_time is not on a 3-hourly NARR slot" do
|
test "raises ArgumentError when valid_time is not on a 3-hourly NARR slot" do
|
||||||
|
|
|
||||||
|
|
@ -5,8 +5,8 @@ defmodule MicrowavepropWeb.ContactLiveTest do
|
||||||
|
|
||||||
alias Microwaveprop.Radio.Contact
|
alias Microwaveprop.Radio.Contact
|
||||||
alias Microwaveprop.Repo
|
alias Microwaveprop.Repo
|
||||||
alias Microwaveprop.Weather.Era5Profile
|
|
||||||
alias Microwaveprop.Weather.HrrrNativeProfile
|
alias Microwaveprop.Weather.HrrrNativeProfile
|
||||||
|
alias Microwaveprop.Weather.NarrProfile
|
||||||
|
|
||||||
setup do
|
setup do
|
||||||
Req.Test.stub(Microwaveprop.Terrain.ElevationClient, fn conn ->
|
Req.Test.stub(Microwaveprop.Terrain.ElevationClient, fn conn ->
|
||||||
|
|
@ -156,8 +156,8 @@ defmodule MicrowavepropWeb.ContactLiveTest do
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
test "shows atmospheric profile section with ERA5 source for pre-HRRR contact", %{conn: conn} do
|
test "shows atmospheric profile section with NARR source for pre-HRRR contact", %{conn: conn} do
|
||||||
# Pre-2014 contact — HRRR is unavailable, ERA5 is the fallback source.
|
# Pre-2014 contact — HRRR is unavailable, NARR is the fallback source.
|
||||||
contact =
|
contact =
|
||||||
create_contact(%{
|
create_contact(%{
|
||||||
qso_timestamp: ~U[2010-06-15 18:00:00Z],
|
qso_timestamp: ~U[2010-06-15 18:00:00Z],
|
||||||
|
|
@ -166,7 +166,7 @@ defmodule MicrowavepropWeb.ContactLiveTest do
|
||||||
})
|
})
|
||||||
|
|
||||||
{:ok, _} =
|
{:ok, _} =
|
||||||
Repo.insert(%Era5Profile{
|
Repo.insert(%NarrProfile{
|
||||||
valid_time: ~U[2010-06-15 18:00:00Z],
|
valid_time: ~U[2010-06-15 18:00:00Z],
|
||||||
lat: 33.0,
|
lat: 33.0,
|
||||||
lon: -97.0,
|
lon: -97.0,
|
||||||
|
|
@ -187,8 +187,8 @@ defmodule MicrowavepropWeb.ContactLiveTest do
|
||||||
|
|
||||||
html = render_async(lv)
|
html = render_async(lv)
|
||||||
assert html =~ "Atmospheric Profile"
|
assert html =~ "Atmospheric Profile"
|
||||||
assert html =~ "ERA5"
|
assert html =~ "NARR"
|
||||||
# Collapsed summary shows HPBL / PWAT / lat,lon from the injected ERA5 row.
|
# Collapsed summary shows HPBL / PWAT / lat,lon from the injected NARR row.
|
||||||
assert html =~ "1200"
|
assert html =~ "1200"
|
||||||
assert html =~ "33.0"
|
assert html =~ "33.0"
|
||||||
end
|
end
|
||||||
|
|
@ -203,10 +203,10 @@ defmodule MicrowavepropWeb.ContactLiveTest do
|
||||||
pos2: %{"lat" => 30.3, "lon" => -97.7}
|
pos2: %{"lat" => 30.3, "lon" => -97.7}
|
||||||
})
|
})
|
||||||
|
|
||||||
# ERA5 row keeps the atmospheric profile section rendering with its
|
# NARR row keeps the atmospheric profile section rendering with its
|
||||||
# aggregate metadata (HPBL / PWAT / surface scalars).
|
# aggregate metadata (HPBL / PWAT / surface scalars).
|
||||||
{:ok, _} =
|
{:ok, _} =
|
||||||
Repo.insert(%Era5Profile{
|
Repo.insert(%NarrProfile{
|
||||||
valid_time: ~U[2024-06-15 18:00:00Z],
|
valid_time: ~U[2024-06-15 18:00:00Z],
|
||||||
lat: 32.9,
|
lat: 32.9,
|
||||||
lon: -97.0,
|
lon: -97.0,
|
||||||
|
|
@ -223,7 +223,7 @@ defmodule MicrowavepropWeb.ContactLiveTest do
|
||||||
|
|
||||||
# Native profile at the same point/time. `pressure_pa: [..., 30_000.0]`
|
# Native profile at the same point/time. `pressure_pa: [..., 30_000.0]`
|
||||||
# corresponds to 300 mb — a level the pressure-level profile cannot
|
# corresponds to 300 mb — a level the pressure-level profile cannot
|
||||||
# cover because HRRR/ERA5 stopped at 700 mb historically.
|
# cover because HRRR/NARR pressure-level data stops at 700 mb historically.
|
||||||
{:ok, _} =
|
{:ok, _} =
|
||||||
%HrrrNativeProfile{}
|
%HrrrNativeProfile{}
|
||||||
|> HrrrNativeProfile.changeset(%{
|
|> HrrrNativeProfile.changeset(%{
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ defmodule MicrowavepropWeb.StatusLiveTest do
|
||||||
alias Microwaveprop.Cache
|
alias Microwaveprop.Cache
|
||||||
alias Microwaveprop.Radio.Contact
|
alias Microwaveprop.Radio.Contact
|
||||||
alias Microwaveprop.Repo
|
alias Microwaveprop.Repo
|
||||||
alias Microwaveprop.Weather.Era5Profile
|
alias Microwaveprop.Weather.NarrProfile
|
||||||
|
|
||||||
setup %{conn: conn} do
|
setup %{conn: conn} do
|
||||||
Cache.invalidate({MicrowavepropWeb.StatusLive, :count_unprocessed})
|
Cache.invalidate({MicrowavepropWeb.StatusLive, :count_unprocessed})
|
||||||
|
|
@ -41,8 +41,8 @@ defmodule MicrowavepropWeb.StatusLiveTest do
|
||||||
contact
|
contact
|
||||||
end
|
end
|
||||||
|
|
||||||
# NARR fetches land in era5_profiles (table name kept as historical artifact);
|
# NARR fetches land in era5_profiles (table rename is a follow-up migration).
|
||||||
# the status UI labels this "NARR" now.
|
# The status UI labels this "NARR" now.
|
||||||
test "NARR progress bar numerator is actual profile matches not total-minus-candidates", %{conn: conn} do
|
test "NARR progress bar numerator is actual profile matches not total-minus-candidates", %{conn: conn} do
|
||||||
create_narr_candidate()
|
create_narr_candidate()
|
||||||
|
|
||||||
|
|
@ -54,7 +54,7 @@ defmodule MicrowavepropWeb.StatusLiveTest do
|
||||||
|
|
||||||
# Insert a matching profile row — done should become 1
|
# Insert a matching profile row — done should become 1
|
||||||
{:ok, _} =
|
{:ok, _} =
|
||||||
Repo.insert(%Era5Profile{
|
Repo.insert(%NarrProfile{
|
||||||
valid_time: ~U[2010-06-15 18:00:00Z],
|
valid_time: ~U[2010-06-15 18:00:00Z],
|
||||||
lat: 33.0,
|
lat: 33.0,
|
||||||
lon: -97.0,
|
lon: -97.0,
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue