Adds mix hrdps.probe — throwaway wedge that retires URL/projection/decode
risks against MSC Datamart. Verified against five Canadian cities with
plausible 2m temperatures via wgrib2's native rotated-lat/lon support.
Findings update what we knew:
- ECCC reorganized to date-prefixed paths; the old /model_hrdps/ root
404s now. Real URL is dd.weather.gc.ca/{YYYYMMDD}/WXO-DD/model_hrdps/...
- HRDPS publishes one variable per GRIB2 file (~3.5 MB each), not
HRRR's bundled wrfsfcf format. ~14 files per cycle/forecast hour.
- wgrib2 -lon LON LAT handles the rotated grid natively — no manual
reprojection math needed.
New plan at 2026-04-29-hrdps-canadian-prop-grid.md supersedes the
2026-04-13 one (which assumed HRRR-style idx + byte-range fetches that
don't apply). Probe deletion is a Stage 10 step in the new plan.
165 lines
5.7 KiB
Elixir
165 lines
5.7 KiB
Elixir
defmodule Mix.Tasks.Hrdps.Probe do
|
|
@shortdoc "Throwaway probe: fetch one HRDPS variable, decode at sample cities"
|
|
@moduledoc """
|
|
One-shot probe for the Canadian HRDPS model. Fetches a single GRIB2
|
|
file from MSC Datamart and uses `wgrib2 -lon` to extract values at
|
|
five Canadian cities, printing a table for visual sanity-checking.
|
|
|
|
Designed as a wedge to retire risk before committing to a real
|
|
HrdpsClient: validates URL pattern, GRIB2 decode, and rotated-lat/lon
|
|
reprojection without touching the DB, Oban, or the score grid.
|
|
|
|
Throwaway code. Will be deleted once the real client lands.
|
|
|
|
mix hrdps.probe # latest cycle, f000, 2m temperature
|
|
mix hrdps.probe --hour 6 # f006 from latest cycle
|
|
mix hrdps.probe --cycle 20260429T12Z
|
|
|
|
URL pattern (current as of 2026-04-29; ECCC reorganized away from the
|
|
old `/model_hrdps/` root to a date-prefixed structure):
|
|
|
|
https://dd.weather.gc.ca/{YYYYMMDD}/WXO-DD/model_hrdps/continental/2.5km/
|
|
{HH}/{FFF}/{YYYYMMDD}T{HH}Z_MSC_HRDPS_{VAR}_{LEVEL}_RLatLon0.0225_PT{FFF}H.grib2
|
|
"""
|
|
use Mix.Task
|
|
|
|
@datamart_base "https://dd.weather.gc.ca"
|
|
|
|
# Variable / level slug for 2m temperature in HRDPS naming. HRRR-side
|
|
# equivalent is `TMP:2 m above ground`; HRDPS uses MSC's per-variable
|
|
# filename convention.
|
|
@variable_slug "TMP_AGL-2m"
|
|
|
|
@cities [
|
|
{"YYZ Toronto", 43.68, -79.63},
|
|
{"YVR Vancouver", 49.19, -123.18},
|
|
{"YYC Calgary", 51.11, -114.02},
|
|
{"YOW Ottawa", 45.32, -75.67},
|
|
{"YHZ Halifax", 44.88, -63.51}
|
|
]
|
|
|
|
@impl Mix.Task
|
|
def run(argv) do
|
|
{opts, _, _} =
|
|
OptionParser.parse(argv,
|
|
strict: [cycle: :string, hour: :integer],
|
|
aliases: [c: :cycle, h: :hour]
|
|
)
|
|
|
|
{:ok, _} = Application.ensure_all_started(:req)
|
|
|
|
forecast_hour = Keyword.get(opts, :hour, 0)
|
|
cycle = opts |> Keyword.get(:cycle) |> resolve_cycle()
|
|
|
|
url = build_url(cycle, forecast_hour)
|
|
IO.puts("Fetching: #{url}")
|
|
|
|
{:ok, path} = download(url)
|
|
IO.puts("Downloaded #{path} (#{File.stat!(path).size} bytes)")
|
|
|
|
print_inventory(path)
|
|
print_city_table(path, cycle, forecast_hour)
|
|
end
|
|
|
|
# ----- Cycle resolution -----
|
|
|
|
defp resolve_cycle(nil), do: latest_published_cycle()
|
|
|
|
defp resolve_cycle(<<year::binary-size(4), month::binary-size(2), day::binary-size(2), "T", hour::binary-size(2), "Z">>) do
|
|
%{date: "#{year}#{month}#{day}", hour: hour}
|
|
end
|
|
|
|
defp resolve_cycle(other) do
|
|
Mix.raise("Invalid --cycle #{inspect(other)} — expected YYYYMMDDTHHZ")
|
|
end
|
|
|
|
# HRDPS publishes ~3-4h after each cycle (00/06/12/18Z). Walk back from
|
|
# "now - 4h" until we find a cycle whose root directory exists.
|
|
defp latest_published_cycle do
|
|
now = DateTime.utc_now()
|
|
candidates = for h <- 0..3, do: DateTime.add(now, -((4 + h * 6) * 3600), :second)
|
|
|
|
candidates
|
|
|> Enum.map(&snap_to_cycle/1)
|
|
|> Enum.uniq()
|
|
|> Enum.find(&cycle_published?/1)
|
|
|> case do
|
|
nil -> Mix.raise("No published HRDPS cycle found in the last 24h")
|
|
cycle -> cycle
|
|
end
|
|
end
|
|
|
|
defp snap_to_cycle(%DateTime{} = dt) do
|
|
cycle_hour = div(dt.hour, 6) * 6
|
|
%{date: Calendar.strftime(dt, "%Y%m%d"), hour: String.pad_leading("#{cycle_hour}", 2, "0")}
|
|
end
|
|
|
|
defp cycle_published?(%{date: date, hour: hour}) do
|
|
url = "#{@datamart_base}/#{date}/WXO-DD/model_hrdps/continental/2.5km/#{hour}/000/"
|
|
|
|
case Req.head(url, retry: false, connect_options: [timeout: 5_000]) do
|
|
{:ok, %{status: 200}} -> true
|
|
_ -> false
|
|
end
|
|
end
|
|
|
|
# ----- URL + download -----
|
|
|
|
defp build_url(%{date: date, hour: hour}, forecast_hour) do
|
|
fff = String.pad_leading("#{forecast_hour}", 3, "0")
|
|
filename = "#{date}T#{hour}Z_MSC_HRDPS_#{@variable_slug}_RLatLon0.0225_PT#{fff}H.grib2"
|
|
|
|
"#{@datamart_base}/#{date}/WXO-DD/model_hrdps/continental/2.5km/#{hour}/#{fff}/#{filename}"
|
|
end
|
|
|
|
defp download(url) do
|
|
path = Path.join(System.tmp_dir!(), "hrdps_probe_#{System.unique_integer([:positive])}.grib2")
|
|
|
|
case Req.get(url, into: File.stream!(path), retry: false) do
|
|
{:ok, %{status: 200}} -> {:ok, path}
|
|
{:ok, %{status: status}} -> Mix.raise("Datamart returned HTTP #{status} for #{url}")
|
|
{:error, reason} -> Mix.raise("Fetch failed: #{inspect(reason)}")
|
|
end
|
|
end
|
|
|
|
# ----- wgrib2 inspection -----
|
|
|
|
defp print_inventory(path) do
|
|
{output, 0} = System.cmd("wgrib2", [path], stderr_to_stdout: true)
|
|
IO.puts("\nGRIB2 inventory:")
|
|
IO.puts(" " <> String.trim(output))
|
|
end
|
|
|
|
defp print_city_table(path, cycle, forecast_hour) do
|
|
args = [path] ++ Enum.flat_map(@cities, fn {_, lat, lon} -> ["-lon", "#{lon}", "#{lat}"] end)
|
|
{output, 0} = System.cmd("wgrib2", args, stderr_to_stdout: true)
|
|
|
|
values = parse_wgrib2_lon_output(output)
|
|
|
|
IO.puts("\nHRDPS 2m temperature, cycle #{cycle.date}T#{cycle.hour}Z f#{pad3(forecast_hour)}:\n")
|
|
IO.puts(" city lat,lon temp")
|
|
IO.puts(" -------------------- -------------------- ------")
|
|
|
|
@cities
|
|
|> Enum.zip(values)
|
|
|> Enum.each(fn {{name, lat, lon}, kelvin} ->
|
|
celsius = kelvin - 273.15
|
|
|
|
IO.puts(" #{String.pad_trailing(name, 20)} #{format_coord(lat, lon)} #{format_temp(celsius)}")
|
|
end)
|
|
end
|
|
|
|
# wgrib2 -lon emits one line for the GRIB record with all `lon=…lat=…val=…`
|
|
# entries colon-separated. For five cities we get five `val=` matches.
|
|
defp parse_wgrib2_lon_output(output) do
|
|
~r/val=([\-\d\.]+)/
|
|
|> Regex.scan(output)
|
|
|> Enum.map(fn [_, v] -> String.to_float(v) end)
|
|
end
|
|
|
|
defp format_coord(lat, lon), do: "#{:io_lib.format("~6.2f", [lat])},#{:io_lib.format("~7.2f", [lon])}"
|
|
|
|
defp format_temp(celsius), do: "#{:io_lib.format("~5.1f", [celsius])}°C"
|
|
|
|
defp pad3(n), do: String.pad_leading("#{n}", 3, "0")
|
|
end
|