prop/lib/microwaveprop/space_weather/swpc_client.ex
Graham McIntire 923c8a468d
SpaceWeather: SWPC JSON ingestion (Kp, F10.7, GOES X-ray)
Adds the second ionospheric data source layer. Polls NOAA SWPC's free
public JSON services every 5 minutes for three products:

* Planetary K-index (1-min cadence) → geomagnetic_observations
* 10.7 cm solar flux (hourly) → solar_flux_observations
* GOES X-ray flux (1-min, 0.1-0.8nm long-wave band only) →
  solar_xray_observations

All three products feed HF / Es scoring inputs that the propagation
engine will start using in a follow-up commit:

* Kp drives HF absorption and Es damping (memory notes Kp is inversely
  correlated with tropo/Es stability).
* F10.7 is the SSN proxy for ITU-R P.533 HF MUF prediction.
* GOES X-ray long-wave flux is the short-wave fade (SWF) / D-layer
  absorption proxy — a C-class flare starts attenuating HF within
  seconds of the X-ray peak.

New context `Microwaveprop.SpaceWeather` exposes `upsert_*/1` and
`latest_*/0` per product; `SpaceWeatherFetchWorker` pulls all three
independently so one endpoint's outage doesn't block the others.
Fixtures captured from live SWPC endpoints on 2026-04-15 for the
parser tests.

Not yet wired into scoring — that's a separate commit.
2026-04-15 14:55:24 -05:00

227 lines
6.4 KiB
Elixir
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

defmodule Microwaveprop.SpaceWeather.SwpcClient do
@moduledoc """
Client for NOAA SWPC's public JSON services (https://services.swpc.noaa.gov/).
All three products we care about for HF / Es scoring are free, no-auth
JSON endpoints:
- `/json/planetary_k_index_1m.json` — planetary K 1-min cadence
- `/json/f107_cm_flux.json` — 10.7 cm solar flux (hourly)
- `/json/goes/primary/xrays-1-day.json` — GOES X-ray flux 1-min, 2 bands
Each product has its own wire format so each has its own `fetch_*` +
`parse_*` pair. Parsed rows are shaped for direct insert into the
corresponding schemas via `Microwaveprop.SpaceWeather` context.
"""
require Logger
@base_url "https://services.swpc.noaa.gov"
@default_timeout_ms 15_000
# ---------- Kp ----------
@doc """
Fetch and parse the planetary K 1-min feed. Returns `{:ok, [row]}` where
each row is `%{valid_time, kp_index, estimated_kp}`.
"""
@spec fetch_kp() :: {:ok, [map()]} | {:error, term()}
def fetch_kp, do: fetch_and_parse("/json/planetary_k_index_1m.json", &parse_kp/1)
@doc false
@spec parse_kp(String.t()) :: {:ok, [map()]} | {:error, term()}
def parse_kp(body) do
with {:ok, decoded} <- decode_array(body) do
rows =
decoded
|> Enum.map(&parse_kp_row/1)
|> Enum.reject(&is_nil/1)
{:ok, rows}
end
end
defp parse_kp_row(%{"time_tag" => t, "kp_index" => kp_index} = row) do
case parse_naive_iso(t) do
{:ok, valid_time} ->
%{
valid_time: valid_time,
kp_index: to_integer(kp_index),
estimated_kp: to_float(Map.get(row, "estimated_kp"))
}
:error ->
nil
end
end
defp parse_kp_row(_), do: nil
# ---------- F10.7 ----------
@doc """
Fetch and parse the F10.7 cm flux feed. Returns `{:ok, [row]}` where
each row is `%{valid_time, frequency_mhz, flux_sfu, reporting_schedule,
ninety_day_mean}`.
"""
@spec fetch_f107() :: {:ok, [map()]} | {:error, term()}
def fetch_f107, do: fetch_and_parse("/json/f107_cm_flux.json", &parse_f107/1)
@doc false
@spec parse_f107(String.t()) :: {:ok, [map()]} | {:error, term()}
def parse_f107(body) do
with {:ok, decoded} <- decode_array(body) do
rows =
decoded
|> Enum.map(&parse_f107_row/1)
|> Enum.reject(&is_nil/1)
{:ok, rows}
end
end
defp parse_f107_row(%{"time_tag" => t, "flux" => flux} = row) do
case parse_naive_iso(t) do
{:ok, valid_time} ->
%{
valid_time: valid_time,
frequency_mhz: to_integer(Map.get(row, "frequency")),
flux_sfu: to_float(flux),
reporting_schedule: Map.get(row, "reporting_schedule"),
ninety_day_mean: to_float(Map.get(row, "ninety_day_mean"))
}
:error ->
nil
end
end
defp parse_f107_row(_), do: nil
# ---------- GOES X-ray ----------
@long_band "0.1-0.8nm"
@doc """
Fetch and parse the GOES primary 1-day X-ray feed. Only keeps the
0.10.8nm long-wavelength band (the standard SWF / D-layer absorption
proxy). Returns `{:ok, [row]}` where each row is `%{valid_time,
satellite, flux_wm2, energy_band, electron_contaminated}`.
"""
@spec fetch_xrays() :: {:ok, [map()]} | {:error, term()}
def fetch_xrays, do: fetch_and_parse("/json/goes/primary/xrays-1-day.json", &parse_xrays/1)
@doc false
@spec parse_xrays(String.t()) :: {:ok, [map()]} | {:error, term()}
def parse_xrays(body) do
with {:ok, decoded} <- decode_array(body) do
rows =
decoded
|> Enum.filter(&long_band?/1)
|> Enum.map(&parse_xray_row/1)
|> Enum.reject(&is_nil/1)
{:ok, rows}
end
end
defp long_band?(%{"energy" => @long_band}), do: true
defp long_band?(_), do: false
defp parse_xray_row(%{"time_tag" => t, "flux" => flux} = row) do
case parse_iso_z(t) do
{:ok, valid_time} ->
%{
valid_time: valid_time,
satellite: to_integer(Map.get(row, "satellite")),
flux_wm2: to_float(flux),
energy_band: Map.get(row, "energy"),
# SWPC's field name is misspelled "electron_contaminaton" upstream.
electron_contaminated: !!Map.get(row, "electron_contaminaton")
}
:error ->
nil
end
end
defp parse_xray_row(_), do: nil
# ---------- Shared HTTP / parsing helpers ----------
defp fetch_and_parse(path, parser) do
url = @base_url <> path
case Req.get(url, [receive_timeout: @default_timeout_ms] ++ req_options()) do
{:ok, %{status: 200, body: body}} ->
parser.(body)
{:ok, %{status: status, body: body}} ->
{:error, "SWPC HTTP #{status}: #{inspect(body)}"}
{:error, reason} ->
{:error, "SWPC request failed: #{inspect(reason)}"}
end
end
defp decode_array(body) when is_binary(body) do
case Jason.decode(body) do
{:ok, list} when is_list(list) -> {:ok, list}
{:ok, _other} -> {:error, :not_a_list}
{:error, reason} -> {:error, reason}
end
end
# Req's JSON auto-decode may already have parsed the body if the
# upstream set Content-Type: application/json. In that case body is a
# list, not a binary.
defp decode_array(list) when is_list(list), do: {:ok, list}
defp decode_array(_), do: {:error, :not_a_list}
# SWPC's Kp/F10.7 time_tag is naive ISO ("2026-04-15T13:49:00") —
# documented UTC but without a Z suffix. Add the Z so DateTime.from_iso8601
# parses it as UTC.
defp parse_naive_iso(s) when is_binary(s) do
case DateTime.from_iso8601(s <> "Z") do
{:ok, dt, _} -> {:ok, DateTime.truncate(dt, :second)}
_ -> :error
end
end
defp parse_naive_iso(_), do: :error
defp parse_iso_z(s) when is_binary(s) do
case DateTime.from_iso8601(s) do
{:ok, dt, _} -> {:ok, DateTime.truncate(dt, :second)}
_ -> :error
end
end
defp parse_iso_z(_), do: :error
defp to_integer(nil), do: nil
defp to_integer(n) when is_integer(n), do: n
defp to_integer(n) when is_float(n), do: trunc(n)
defp to_integer(s) when is_binary(s) do
case Integer.parse(s) do
{n, ""} -> n
_ -> nil
end
end
defp to_float(nil), do: nil
defp to_float(n) when is_float(n), do: n
defp to_float(n) when is_integer(n), do: n * 1.0
defp to_float(s) when is_binary(s) do
case Float.parse(s) do
{f, ""} -> f
_ -> nil
end
end
defp req_options do
Application.get_env(:microwaveprop, :swpc_req_options, [])
end
end