defmodule Microwaveprop.Weather.NceiMetarClient do @moduledoc """ Fetches 5-minute ASOS observations from NCEI's C00418 dataset. Data URL pattern: ``` https://www.ncei.noaa.gov/data/automated-surface-observing-system-five-minute/access/YYYY/MM/asos-5min-KXXX-YYYYMM.dat ``` Each file is one station-month of 5-minute observations in a fixed-width METAR-like format. See the meteorologist's recommendation in the April 2026 review. """ @base_url "https://www.ncei.noaa.gov/data/automated-surface-observing-system-five-minute/access" @doc """ Fetch and parse 5-minute observations for a given station, year, and month. Returns `{:ok, [parsed_obs]}` or `{:error, reason}`. """ @spec fetch(String.t(), pos_integer(), pos_integer()) :: {:ok, [map()]} | {:error, term()} def fetch(icao, year, month) do Microwaveprop.Instrument.span([:ncei, :fetch_metar], %{station: icao}, fn -> month_str = month |> Integer.to_string() |> String.pad_leading(2, "0") url = "#{@base_url}/#{year}/#{month_str}/asos-5min-#{icao}-#{year}#{month_str}.dat" case Req.get(url, receive_timeout: 60_000) do {:ok, %{status: 200, body: body}} -> {:ok, parse(body)} {:ok, %{status: status}} -> {:error, "NCEI 5-min HTTP #{status}"} {:error, reason} -> {:error, reason} end end) end @doc """ Parse the fixed-width NCEI 5-minute ASOS format into maps. Each line has a METAR embedded in a fixed-width wrapper. We extract the key fields without a full METAR parser — this covers the data the scorer and backtest need (temp, dewpoint, wind, pressure, sky). """ @spec parse(String.t()) :: [map()] def parse(text) when is_binary(text) do text |> String.split("\n") |> Enum.flat_map(&parse_line/1) end defp parse_line(line) when byte_size(line) < 60, do: [] defp parse_line(line) do # Fixed-width header: WBAN(5) ICAO(4) FAA(4) DATE(8) HHMM(4) ... # Then METAR text after the "5-MIN" marker with <<_wban::binary-5, icao::binary-4, _faa::binary-4, date::binary-8, hhmm::binary-4, _rest::binary>> <- line, {:ok, observed_at} <- parse_datetime(date, hhmm) do metar_text = extract_metar(line) obs = parse_metar_fields(metar_text) [Map.merge(obs, %{icao: String.trim(icao), observed_at: observed_at})] else _ -> [] end end defp parse_datetime(<>, <>) do with {year, ""} <- Integer.parse(y), {month, ""} <- Integer.parse(m), {day, ""} <- Integer.parse(d), {hour, ""} <- Integer.parse(hh), {minute, ""} <- Integer.parse(mm), {:ok, date} <- Date.new(year, month, day), {:ok, time} <- Time.new(hour, minute, 0), {:ok, dt} <- DateTime.new(date, time, "Etc/UTC") do {:ok, DateTime.truncate(dt, :second)} else _ -> :error end end defp extract_metar(line) do case String.split(line, "5-MIN ", parts: 2) do [_, metar] -> metar _ -> "" end end defp parse_metar_fields(metar) do parts = String.split(metar) %{ temp_f: extract_temp_f(parts), dewpoint_f: extract_dewpoint_f(parts), wind_speed_kts: extract_wind_speed(parts), wind_direction_deg: extract_wind_dir(parts), altimeter_setting: extract_altimeter(parts), sky_condition: extract_sky(parts), relative_humidity: nil, sea_level_pressure_mb: nil, precip_1h_in: nil, wx_codes: nil } end # Extract precise temp/dew from RMK T-group: T02110094 → 21.1°C / 9.4°C defp extract_temp_f(parts) do case Enum.find(parts, &String.starts_with?(&1, "T0")) do t_group when is_binary(t_group) -> parse_t_group_temp(t_group) _ -> # Fall back to temp/dew field: "21/09" case Enum.find(parts, &Regex.match?(~r/^M?\d+\/M?\d+$/, &1)) do td when is_binary(td) -> [t, _d] = String.split(td, "/") t |> parse_metar_temp() |> c_to_f() _ -> nil end end end defp extract_dewpoint_f(parts) do case Enum.find(parts, &String.starts_with?(&1, "T0")) do t_group when is_binary(t_group) -> parse_t_group_dew(t_group) _ -> case Enum.find(parts, &Regex.match?(~r/^M?\d+\/M?\d+$/, &1)) do td when is_binary(td) -> [_t, d] = String.split(td, "/") d |> parse_metar_temp() |> c_to_f() _ -> nil end end end # T02110094 → temp = +21.1°C, dew = +9.4°C (leading 0=+, 1=-) defp parse_t_group_temp(<<"T", sign::binary-1, d1::binary-2, d2::binary-1, _rest::binary>>) do val = String.to_integer(d1) + String.to_integer(d2) / 10.0 if_result = if sign == "1", do: -val, else: val c_to_f(if_result) end defp parse_t_group_temp(_), do: nil defp parse_t_group_dew(<<"T", _::binary-4, sign::binary-1, d1::binary-2, d2::binary-1, _rest::binary>>) do val = String.to_integer(d1) + String.to_integer(d2) / 10.0 if_result = if sign == "1", do: -val, else: val c_to_f(if_result) end defp parse_t_group_dew(_), do: nil defp parse_metar_temp("M" <> rest), do: -String.to_integer(rest) * 1.0 defp parse_metar_temp(s), do: String.to_integer(s) * 1.0 defp c_to_f(c) when is_number(c), do: c * 9.0 / 5.0 + 32.0 defp extract_wind_speed(parts) do with wind when is_binary(wind) <- Enum.find(parts, &Regex.match?(~r/^\d{3}\d{2,3}KT/, &1)), {speed, _} <- Integer.parse(String.slice(wind, 3..4)) do speed * 1.0 else _ -> nil end end defp extract_wind_dir(parts) do with wind when is_binary(wind) <- Enum.find(parts, &Regex.match?(~r/^\d{3}\d{2,3}KT/, &1)), {dir, _} <- Integer.parse(String.slice(wind, 0..2)) do dir else _ -> nil end end defp extract_altimeter(parts) do case Enum.find(parts, &String.starts_with?(&1, "A2")) do alt when is_binary(alt) -> case Float.parse(String.slice(alt, 1..-1//1)) do {val, _} -> val / 100.0 :error -> nil end _ -> nil end end defp extract_sky(parts) do sky_tokens = Enum.filter(parts, &Regex.match?(~r/^(CLR|FEW|SCT|BKN|OVC|VV)\d*/, &1)) if sky_tokens == [], do: nil, else: Enum.join(sky_tokens, " ") end end