defmodule Microwaveprop.Weather.NarrClient do @moduledoc """ Client for NCEP North American Regional Reanalysis (NARR) data served anonymously from NCEI at `https://www.ncei.noaa.gov/data/north-american-regional-reanalysis/`. Replaces the broken ERA5/CDS historical backfill with anonymous NCEI fetches for pre-HRRR (1979-01-01 → 2014-10-02) contact enrichment. See `docs/plans/2026-04-15-merra2-historical-backfill.md` for the full architecture — the filename is a historical artifact from an earlier MERRA-2 plan. Currently provides URL/time helpers, the inventory parser, and the inventory HTTP fetch. The byte-range `fetch_profile_at/2` pipeline lands in a follow-up task of the same plan. """ @base_url "https://www.ncei.noaa.gov/data/north-american-regional-reanalysis/access/3-hourly" @valid_hours [0, 3, 6, 9, 12, 15, 18, 21] @doc """ Builds the NCEI HTTPS URL for the NARR analysis GRIB1 file at `valid_time`. Raises `ArgumentError` if `valid_time` is not on a 3-hourly analysis boundary — NARR analyses exist only at 00, 03, 06, 09, 12, 15, 18, and 21 UTC. """ @spec url_for(DateTime.t()) :: String.t() def url_for(valid_time) do validate_analysis_time!(valid_time) yyyymm = Calendar.strftime(valid_time, "%Y%m") yyyymmdd = Calendar.strftime(valid_time, "%Y%m%d") hhmm = Calendar.strftime(valid_time, "%H%M") "#{@base_url}/#{yyyymm}/#{yyyymmdd}/narr-a_221_#{yyyymmdd}_#{hhmm}_000.grb" end @doc """ Builds the NCEI HTTPS URL for the NARR analysis inventory (`.inv`) file at `valid_time`. Same validation rules as `url_for/1`. """ @spec url_for_inventory(DateTime.t()) :: String.t() def url_for_inventory(valid_time) do valid_time |> url_for() |> String.replace_suffix(".grb", ".inv") end @doc """ Rounds a `DateTime` *down* to the nearest 3-hourly NARR analysis slot. NARR analyses are produced at 00, 03, 06, 09, 12, 15, 18, and 21 UTC. Minutes, seconds, and microseconds are zeroed out; the hour becomes `(hour div 3) * 3`. """ @spec snap_to_analysis_hour(DateTime.t()) :: DateTime.t() def snap_to_analysis_hour(%DateTime{} = datetime) do snapped_hour = div(datetime.hour, 3) * 3 %{datetime | hour: snapped_hour, minute: 0, second: 0, microsecond: {0, 0}} end @doc """ Parses the raw text of a NARR `.inv` file into a map keyed by `{var, level}` with values of `{byte_offset, length}`. Each record's length is computed as `next_offset - this_offset`. The final record's length uses `file_size` as its "next offset". Blank lines and trailing whitespace are ignored. `var` and `level` are returned as-is (no trimming) so levels like `"2 m above gnd"` round-trip unchanged. """ @spec parse_inventory(String.t(), non_neg_integer()) :: {:ok, %{{String.t(), String.t()} => {non_neg_integer(), non_neg_integer()}}} def parse_inventory(inv_text, file_size) when is_binary(inv_text) and is_integer(file_size) do records = inv_text |> String.split("\n", trim: true) |> Enum.map(&parse_inventory_line/1) |> Enum.reject(&is_nil/1) index = records |> Enum.zip(Enum.drop(records, 1) ++ [{nil, file_size, nil, nil}]) |> Map.new(fn {{_num, offset, var, level}, {_next_num, next_offset, _next_var, _next_level}} -> {{var, level}, {offset, next_offset - offset}} end) {:ok, index} end @doc """ Fetches the `.inv` text and the `.grb` `Content-Length` for `valid_time`, then delegates to `parse_inventory/2`. Does one `GET` against the inventory URL and one `HEAD` against the GRIB URL. Returns `{:ok, index}` on success; `{:error, reason}` if either HTTP call is non-200, transport-errored, or missing a `content-length` header. """ @spec fetch_inventory(DateTime.t()) :: {:ok, %{{String.t(), String.t()} => {non_neg_integer(), non_neg_integer()}}} | {:error, term()} def fetch_inventory(%DateTime{} = valid_time) do inv_url = url_for_inventory(valid_time) grb_url = url_for(valid_time) with {:ok, inv_body} <- do_get_inventory(inv_url), {:ok, file_size} <- do_head_grib_size(grb_url) do parse_inventory(inv_body, file_size) end end defp do_get_inventory(url) do case Req.get(url, [receive_timeout: 30_000] ++ req_options()) do {:ok, %{status: 200, body: body}} when is_binary(body) -> {:ok, body} {:ok, %{status: status, body: body}} -> {:error, "NARR inventory HTTP #{status}: #{inspect(body)}"} {:error, reason} -> {:error, "NARR inventory request failed: #{inspect(reason)}"} end end defp do_head_grib_size(url) do case Req.head(url, [receive_timeout: 30_000] ++ req_options()) do {:ok, %{status: 200} = response} -> case content_length(response) do {:ok, size} -> {:ok, size} :error -> {:error, "NARR grib HEAD missing content-length"} end {:ok, %{status: status}} -> {:error, "NARR grib HEAD HTTP #{status}"} {:error, reason} -> {:error, "NARR grib HEAD failed: #{inspect(reason)}"} end end defp content_length(%Req.Response{} = response) do case response |> Req.Response.get_header("content-length") |> List.first() do nil -> :error value when is_binary(value) -> case Integer.parse(value) do {size, ""} when size >= 0 -> {:ok, size} _ -> :error end end end defp parse_inventory_line(line) do case String.split(line, ":") do [num, offset, _date, var, level | _rest] -> with {num_int, ""} <- Integer.parse(num), {offset_int, ""} <- Integer.parse(offset) do {num_int, offset_int, var, level} else _ -> nil end _ -> nil end end defp req_options do Application.get_env(:microwaveprop, :narr_req_options, []) end defp validate_analysis_time!(%DateTime{hour: hour, minute: 0, second: 0} = _vt) when hour in @valid_hours, do: :ok defp validate_analysis_time!(vt) do raise ArgumentError, "NARR analyses are only produced at 00/03/06/09/12/15/18/21 UTC on the hour, " <> "got #{inspect(vt)}" end end