Graham McIntire
cea2507fac
- MsFootprints (51% → 93%): http_get injection for dataset_index and download_tile, with stubbed CSV parsing, empty CSV, caching, and disk-cache-skip tests - HrdpsClient (47% → 71%): http_get/http_head injection for fetch_grid and cycle_available, with stubbed probe, success/failure/transport- error tests, plus fetch_grid error path - NexradClient (57% → 58%): http_get injection, fetch_frame success path with valid PNG stub, process_frame coverage - HrrrNativeClient: http_get injection for fetch_idx (no direct test since function is private) Coverage: 79.43% → 79.68% (need 0.32% more)
392 lines
14 KiB
Elixir
392 lines
14 KiB
Elixir
defmodule Microwaveprop.Weather.HrdpsClient do
|
||
@moduledoc """
|
||
Client for ECCC's HRDPS (High Resolution Deterministic Prediction System) —
|
||
the Canadian analog to HRRR. Mirrors `Microwaveprop.Weather.HrrrClient`'s
|
||
shape (`fetch_grid/3` returning `{:ok, %{{lat, lon} => profile}}`) so the
|
||
propagation chain can call either polymorphically.
|
||
|
||
## Architectural differences from HRRR
|
||
|
||
* **URL structure** — date-prefixed datamart paths
|
||
(`https://dd.weather.gc.ca/{YYYYMMDD}/WXO-DD/model_hrdps/continental/2.5km/{HH}/{FFF}/...`).
|
||
No flat `/model_hrdps/` root; the old structure 404s as of the 2026-04-29
|
||
re-verification.
|
||
* **One variable per file** — HRDPS publishes each variable as its own
|
||
GRIB2 file (~3.5 MB each) instead of HRRR's bundled `wrfsfcf`/`wrfprsf`.
|
||
No idx files, no byte-range partials. We fetch each needed variable's
|
||
file concurrently and **byte-concatenate** them into a single multi-record
|
||
binary — wgrib2 reads the result as if it were a normal multi-message
|
||
GRIB2 file. No `wgrib2 -merge` step needed.
|
||
* **Rotated lat/lon projection** — handled transparently by `wgrib2 -lon`
|
||
inside `Microwaveprop.Weather.Grib2.Wgrib2.extract_points_from_file/3`.
|
||
* **Cycle cadence** — 4×/day at 00/06/12/18Z (vs HRRR's hourly), forecasts
|
||
out to 48h. Publish latency is ~3-4h after each cycle.
|
||
"""
|
||
|
||
alias Microwaveprop.Weather.Grib2.Wgrib2
|
||
|
||
require Logger
|
||
|
||
@datamart_base_default "https://dd.weather.gc.ca"
|
||
|
||
defp datamart_base, do: Application.get_env(:microwaveprop, :hrdps_datamart_base, @datamart_base_default)
|
||
|
||
# Surface variables required by the propagation scorer. HRRR's equivalent
|
||
# set is in `HrrrClient.@surface_messages`; HRDPS doesn't publish the
|
||
# bundled APCP / PWAT inventory at f000, so the scorer's PWAT factor falls
|
||
# back to its default for HRDPS-derived points (see plan stage 1.2).
|
||
@surface_vars [
|
||
{:tmp_2m, "TMP", "AGL-2m"},
|
||
{:depr_2m, "DEPR", "AGL-2m"},
|
||
{:dpt_2m, "DPT", "AGL-2m"},
|
||
{:pres_sfc, "PRES", "Sfc"},
|
||
{:hpbl_sfc, "HPBL", "Sfc"},
|
||
{:ugrd_10m, "UGRD", "AGL-10m"},
|
||
{:vgrd_10m, "VGRD", "AGL-10m"},
|
||
{:tcdc_sfc, "TCDC", "Sfc"}
|
||
]
|
||
|
||
# Pressure-level set covering the lower troposphere where the refractivity
|
||
# gradient is computed. Matches HRRR's `@grid_pressure_levels` (1000→700 mb)
|
||
# so SoundingParams.derive sees a comparable profile shape.
|
||
@grid_pressure_levels [1000, 950, 900, 850, 800, 750, 700]
|
||
|
||
@pressure_var_kinds [{:tmp, "TMP"}, {:depr, "DEPR"}, {:hgt, "HGT"}]
|
||
|
||
@pressure_vars (for level <- @grid_pressure_levels,
|
||
{kind, msc} <- @pressure_var_kinds do
|
||
atom = String.to_atom("#{kind}_#{level}mb")
|
||
msc_level = "ISBL_#{level |> Integer.to_string() |> String.pad_leading(4, "0")}"
|
||
{atom, msc, msc_level}
|
||
end)
|
||
|
||
@all_vars @surface_vars ++ @pressure_vars
|
||
|
||
@var_lookup Map.new(@all_vars, fn {atom, msc_var, msc_level} -> {atom, {msc_var, msc_level}} end)
|
||
|
||
# --- Public API ---
|
||
|
||
@doc "List of all variable atoms this client knows how to fetch."
|
||
@spec variables() :: [atom()]
|
||
def variables, do: Enum.map(@all_vars, fn {atom, _, _} -> atom end)
|
||
|
||
@doc """
|
||
Build the MSC Datamart URL for a single HRDPS variable file.
|
||
|
||
ECCC reorganized to a date-prefixed structure as of 2026; the old
|
||
`/model_hrdps/...` flat root 404s. URLs always look like:
|
||
|
||
https://dd.weather.gc.ca/{YYYYMMDD}/WXO-DD/model_hrdps/continental/2.5km/{HH}/{FFF}/
|
||
{YYYYMMDD}T{HH}Z_MSC_HRDPS_{VAR}_{LEVEL}_RLatLon0.0225_PT{FFF}H.grib2
|
||
"""
|
||
@spec grib_url(atom(), DateTime.t(), non_neg_integer()) :: String.t()
|
||
def grib_url(variable, %DateTime{} = run_time, forecast_hour) do
|
||
{msc_var, msc_level} =
|
||
Map.get(@var_lookup, variable) ||
|
||
raise ArgumentError, "unknown HRDPS variable #{inspect(variable)}"
|
||
|
||
cycle = nearest_hrdps_cycle(run_time)
|
||
date_str = Calendar.strftime(cycle, "%Y%m%d")
|
||
hour_str = cycle.hour |> Integer.to_string() |> String.pad_leading(2, "0")
|
||
fff_str = forecast_hour |> Integer.to_string() |> String.pad_leading(3, "0")
|
||
|
||
filename =
|
||
"#{date_str}T#{hour_str}Z_MSC_HRDPS_#{msc_var}_#{msc_level}_RLatLon0.0225_PT#{fff_str}H.grib2"
|
||
|
||
"#{datamart_base()}/#{date_str}/WXO-DD/model_hrdps/continental/2.5km/#{hour_str}/#{fff_str}/#{filename}"
|
||
end
|
||
|
||
@doc """
|
||
Snap a `DateTime` to the start of the most recent HRDPS cycle hour
|
||
(00/06/12/18Z). Always rounds DOWN — never points at a future cycle that
|
||
may not have published yet. Callers needing a later valid_time should
|
||
reach it via `forecast_hour` instead.
|
||
"""
|
||
@spec nearest_hrdps_cycle(DateTime.t()) :: DateTime.t()
|
||
def nearest_hrdps_cycle(%DateTime{} = dt) do
|
||
cycle_hour = div(dt.hour, 6) * 6
|
||
seconds_into_cycle = (dt.hour - cycle_hour) * 3600 + dt.minute * 60 + dt.second
|
||
DateTime.add(dt, -seconds_into_cycle, :second)
|
||
end
|
||
|
||
@doc """
|
||
Returns true if the HRDPS cycle for `run_time` is published. Probes the
|
||
cycle's f000 directory existence; the directory only appears once the
|
||
first forecast hour's files are uploaded.
|
||
|
||
Override the probe in tests via the `:microwaveprop, :hrdps_cycle_available_fn`
|
||
Application env (a 1-arity function from `DateTime` to `boolean`).
|
||
"""
|
||
@spec cycle_available?(DateTime.t()) :: boolean()
|
||
def cycle_available?(run_time) do
|
||
rounded = nearest_hrdps_cycle(run_time)
|
||
|
||
case Application.get_env(:microwaveprop, :hrdps_cycle_available_fn) do
|
||
fun when is_function(fun, 1) -> fun.(rounded)
|
||
_ -> probe_cycle(rounded)
|
||
end
|
||
end
|
||
|
||
defp probe_cycle(%DateTime{} = run_time) do
|
||
date_str = Calendar.strftime(run_time, "%Y%m%d")
|
||
hour_str = run_time.hour |> Integer.to_string() |> String.pad_leading(2, "0")
|
||
url = "#{datamart_base()}/#{date_str}/WXO-DD/model_hrdps/continental/2.5km/#{hour_str}/000/"
|
||
|
||
case http_head(url, req_options()) do
|
||
{:ok, %{status: 200}} -> true
|
||
_ -> false
|
||
end
|
||
end
|
||
|
||
@doc """
|
||
Fetch HRDPS profiles for the given list of `{lat, lon}` points at the
|
||
cycle's `forecast_hour`. Returns `{:ok, %{{lat, lon} => profile}}` matching
|
||
the shape `HrrrClient.fetch_grid/3` produces, so downstream code is
|
||
source-agnostic.
|
||
|
||
Internally:
|
||
|
||
1. Build URLs for every variable in `variables/0`.
|
||
2. Concurrently download each file as a binary.
|
||
3. Byte-concatenate into a single multi-record GRIB2.
|
||
4. Stage to a temp file; call `Wgrib2.extract_points_from_file/3` with all
|
||
points in one pass — wgrib2 handles the rotated-lat/lon reprojection.
|
||
5. Build per-point profiles via `build_profile_from_extracted/1`.
|
||
"""
|
||
@spec fetch_grid([{float(), float()}], DateTime.t(), keyword()) ::
|
||
{:ok, %{{float(), float()} => map()}} | {:error, term()}
|
||
def fetch_grid(points, run_time, opts \\ []) do
|
||
Microwaveprop.Instrument.span(
|
||
[:hrdps, :fetch_grid],
|
||
%{point_count: length(points), forecast_hour: Keyword.get(opts, :forecast_hour, 0)},
|
||
fn -> do_fetch_grid(points, run_time, opts) end
|
||
)
|
||
end
|
||
|
||
defp do_fetch_grid(points, run_time, opts) do
|
||
cycle = nearest_hrdps_cycle(run_time)
|
||
forecast_hour = Keyword.get(opts, :forecast_hour, 0)
|
||
|
||
with {:ok, binaries} <- fetch_all_variables(cycle, forecast_hour),
|
||
combined = concat_grib_binaries(binaries),
|
||
{:ok, path} <- write_temp_grib(combined),
|
||
{:ok, extracted} <- extract_points(path, points) do
|
||
_ = File.rm(path)
|
||
|
||
profiles =
|
||
Map.new(extracted, fn {point, raw} ->
|
||
profile =
|
||
raw
|
||
|> build_profile_from_extracted()
|
||
|> Map.put(:run_time, cycle)
|
||
|> Map.put(:forecast_hour, forecast_hour)
|
||
|
||
{point, profile}
|
||
end)
|
||
|
||
{:ok, profiles}
|
||
end
|
||
end
|
||
|
||
@doc """
|
||
Concatenate a list of GRIB2 binaries. wgrib2 reads multi-record GRIB2 files
|
||
natively, so byte-concatenation is a valid stand-in for the `wgrib2 -merge`
|
||
workflow — and skips the temp-file shuffle.
|
||
"""
|
||
@spec concat_grib_binaries([binary()]) :: binary()
|
||
def concat_grib_binaries(binaries), do: IO.iodata_to_binary(binaries)
|
||
|
||
@doc """
|
||
Translate the `Wgrib2.extract_points_from_file/3` output (`%{"VAR:LEVEL" => float}`)
|
||
into the same profile shape `HrrrClient.build_profile/1` produces, so
|
||
downstream code can consume HRDPS-derived rows interchangeably with
|
||
HRRR-derived rows.
|
||
|
||
HRDPS publishes DEPR (T-Td depression in K) as a primary surface variable
|
||
rather than DPT. When DPT is absent this builder derives dewpoint as
|
||
`temp - depression`.
|
||
"""
|
||
@spec build_profile_from_extracted(%{String.t() => number()}) :: map()
|
||
def build_profile_from_extracted(extracted) do
|
||
sfc_temp_k = extracted["TMP:2 m above ground"]
|
||
sfc_dpt_k = surface_dewpoint_k(extracted, sfc_temp_k)
|
||
sfc_pres_pa = extracted["PRES:surface"]
|
||
|
||
profile_levels = build_pressure_profile(extracted)
|
||
lowest = List.first(profile_levels)
|
||
|
||
sfc_temp_c =
|
||
cond do
|
||
sfc_temp_k -> sfc_temp_k - 273.15
|
||
lowest -> lowest["tmpc"]
|
||
true -> nil
|
||
end
|
||
|
||
sfc_dpt_c =
|
||
cond do
|
||
sfc_dpt_k -> sfc_dpt_k - 273.15
|
||
lowest -> lowest["dwpc"]
|
||
true -> nil
|
||
end
|
||
|
||
%{
|
||
surface_temp_c: sfc_temp_c,
|
||
surface_dewpoint_c: sfc_dpt_c,
|
||
surface_pressure_mb: if(sfc_pres_pa, do: sfc_pres_pa / 100.0),
|
||
hpbl_m: extracted["HPBL:surface"],
|
||
pwat_mm: extracted["PWAT:entire atmosphere (considered as a single layer)"],
|
||
wind_u: extracted["UGRD:10 m above ground"],
|
||
wind_v: extracted["VGRD:10 m above ground"],
|
||
cloud_cover_pct: extracted["TCDC:surface"] || extracted["TCDC:entire atmosphere"],
|
||
precip_mm: extracted["APCP:surface"],
|
||
profile: profile_levels
|
||
}
|
||
end
|
||
|
||
defp surface_dewpoint_k(extracted, sfc_temp_k) do
|
||
dpt = extracted["DPT:2 m above ground"]
|
||
depr = extracted["DEPR:2 m above ground"]
|
||
|
||
cond do
|
||
not is_nil(dpt) -> dpt
|
||
not is_nil(sfc_temp_k) and not is_nil(depr) -> sfc_temp_k - depr
|
||
true -> nil
|
||
end
|
||
end
|
||
|
||
defp build_pressure_profile(extracted) do
|
||
Enum.flat_map(@grid_pressure_levels, fn level ->
|
||
level_str = "#{level} mb"
|
||
tmp = extracted["TMP:#{level_str}"]
|
||
dpt = pressure_level_dewpoint(extracted, level_str, tmp)
|
||
hgt = extracted["HGT:#{level_str}"]
|
||
|
||
if tmp && dpt && hgt do
|
||
[
|
||
%{
|
||
"pres" => level * 1.0,
|
||
"tmpc" => tmp - 273.15,
|
||
"dwpc" => dpt - 273.15,
|
||
"hght" => hgt
|
||
}
|
||
]
|
||
else
|
||
[]
|
||
end
|
||
end)
|
||
end
|
||
|
||
defp pressure_level_dewpoint(extracted, level_str, tmp) do
|
||
dpt = extracted["DPT:#{level_str}"]
|
||
depr = extracted["DEPR:#{level_str}"]
|
||
|
||
cond do
|
||
not is_nil(dpt) -> dpt
|
||
not is_nil(tmp) and not is_nil(depr) -> tmp - depr
|
||
true -> nil
|
||
end
|
||
end
|
||
|
||
# --- Internal: file fetch + concat ---
|
||
|
||
# Fetch every variable's file concurrently. Datamart files are small
|
||
# (~3.5 MB) and individually quick, but ~14 files done serially still adds
|
||
# up to several seconds. 4-way concurrency stays well under any sensible
|
||
# rate limit.
|
||
defp fetch_all_variables(%DateTime{} = cycle, forecast_hour) do
|
||
supervisor = {:via, PartitionSupervisor, {Microwaveprop.TaskSupervisor, self()}}
|
||
|
||
results =
|
||
supervisor
|
||
|> Task.Supervisor.async_stream_nolink(
|
||
variables(),
|
||
fn variable -> fetch_variable(variable, cycle, forecast_hour) end,
|
||
max_concurrency: 4,
|
||
timeout: 120_000,
|
||
ordered: true
|
||
)
|
||
|> Enum.map(fn
|
||
{:ok, result} ->
|
||
result
|
||
|
||
{:exit, reason} ->
|
||
# Per CLAUDE.md: never silently drop async failures. A swallowed
|
||
# download here means a Canadian dead-zone we won't notice for
|
||
# hours.
|
||
Logger.error(
|
||
"HRDPS variable fetch crashed: cycle=#{inspect(cycle)} f=#{forecast_hour} reason=#{inspect(reason)}"
|
||
)
|
||
|
||
{:error, reason}
|
||
end)
|
||
|
||
case Enum.find(results, &match?({:error, _}, &1)) do
|
||
nil ->
|
||
binaries = Enum.map(results, fn {:ok, _var, body} -> body end)
|
||
{:ok, binaries}
|
||
|
||
{:error, _} = error ->
|
||
error
|
||
end
|
||
end
|
||
|
||
defp fetch_variable(variable, cycle, forecast_hour) do
|
||
url = grib_url(variable, cycle, forecast_hour)
|
||
|
||
case http_get(url, req_options()) do
|
||
{:ok, %{status: 200, body: body}} ->
|
||
{:ok, variable, body}
|
||
|
||
{:ok, %{status: status}} ->
|
||
Logger.error("HRDPS fetch HTTP #{status} for variable=#{variable} url=#{url}")
|
||
{:error, "HRDPS fetch HTTP #{status}"}
|
||
|
||
{:error, reason} ->
|
||
Logger.error("HRDPS fetch crashed: variable=#{variable} url=#{url} reason=#{inspect(reason)}")
|
||
{:error, reason}
|
||
end
|
||
end
|
||
|
||
defp write_temp_grib(binary) do
|
||
path = Path.join(System.tmp_dir!(), "hrdps_#{System.unique_integer([:positive])}.grib2")
|
||
|
||
case File.write(path, binary) do
|
||
:ok -> {:ok, path}
|
||
{:error, reason} -> {:error, {:tmp_write, reason}}
|
||
end
|
||
end
|
||
|
||
defp extract_points(path, points) do
|
||
Wgrib2.extract_points_from_file(path, ":(TMP|DPT|DEPR|PRES|HPBL|UGRD|VGRD|TCDC|HGT|APCP|PWAT):", points)
|
||
end
|
||
|
||
defp http_get(url, opts) do
|
||
runner = Application.get_env(:microwaveprop, :hrdps_http_get, &Req.get/2)
|
||
runner.(url, opts)
|
||
end
|
||
|
||
defp http_head(url, opts) do
|
||
runner = Application.get_env(:microwaveprop, :hrdps_http_head, &Req.head/2)
|
||
runner.(url, opts)
|
||
end
|
||
|
||
defp req_options do
|
||
defaults = [receive_timeout: 120_000, retry: &retry?/2, max_retries: 5, retry_delay: &retry_delay/1]
|
||
overrides = Application.get_env(:microwaveprop, :hrdps_req_options, [])
|
||
Keyword.merge(defaults, overrides)
|
||
end
|
||
|
||
defp retry?(_request, response) do
|
||
case response do
|
||
%Req.Response{status: status} when status in [429, 500, 502, 503, 504] -> true
|
||
%{__exception__: true} -> true
|
||
_ -> false
|
||
end
|
||
end
|
||
|
||
defp retry_delay(n) do
|
||
base = Integer.pow(2, n) * 1_000
|
||
jitter = :rand.uniform(1_000)
|
||
base + jitter
|
||
end
|
||
end
|