Graham McIntire
4e6c87eca2
Adds spans to 15 previously-unmeasured hot paths so every question we might ask while tuning has a histogram to answer it: External I/O: - iem.fetch_iemre (gridded weather reanalysis) - mrms.list_latest / mrms.download (precip radar) - rtma.fetch_observation - ncei.fetch_metar (historical 5-min METAR backfill) - solar.fetch_indices (GFZ solar indices) - swpc.fetch (SWPC Kp/F10.7/X-ray) - giro.fetch (ionosonde) - qrz.request, geocoder.geocode (callsign enrichment) - srtm.download_tile (terrain tile download + gunzip) - hrrr.download_grib_ranges (parallel byte-range fetch phase) Subprocess: - wgrib2.extract_grid / extract_grid_from_file / extract_grid_from_file_mapped LiveView hot paths: - propagation.scores_at (map score fetch + cache hit/miss counter) - propagation.point_forecast (sparkline) - propagation.point_detail (click-to-inspect) - propagation.daily_outlook_at (/map outlook strip) Worker-level end-to-end: - worker.terrain_profile - worker.mechanism_classify - worker.mrms_fetch Each event is registered in Microwaveprop.PromEx.InstrumentPlugin as a Prometheus histogram (default / long buckets as appropriate) plus a counter for the scores_at cache hit/miss ratio. Prometheus at 10.0.15.25 will start seeing the new series on the next scrape after deploy.
101 lines
2.7 KiB
Elixir
101 lines
2.7 KiB
Elixir
defmodule Microwaveprop.Weather.SolarClient do
|
|
@moduledoc false
|
|
|
|
@gfz_url "https://kp.gfz.de/app/files/Kp_ap_Ap_SN_F107_since_1932.txt"
|
|
|
|
@spec data_url() :: String.t()
|
|
def data_url, do: @gfz_url
|
|
|
|
@spec fetch_solar_indices() :: {:ok, [map()]} | {:error, term()}
|
|
def fetch_solar_indices do
|
|
Microwaveprop.Instrument.span([:solar, :fetch_indices], %{}, fn ->
|
|
req_options = Application.get_env(:microwaveprop, :solar_req_options, [])
|
|
|
|
case Req.get(@gfz_url, [receive_timeout: 120_000, retry: false] ++ req_options) do
|
|
{:ok, %{status: 200, body: body}} ->
|
|
{:ok, parse_gfz_file(body)}
|
|
|
|
{:ok, %{status: status}} ->
|
|
{:error, "HTTP #{status}"}
|
|
|
|
{:error, reason} ->
|
|
{:error, reason}
|
|
end
|
|
end)
|
|
end
|
|
|
|
@spec filter_since([map()], Date.t()) :: [map()]
|
|
def filter_since(records, since_date) do
|
|
Enum.filter(records, fn r -> Date.compare(r.date, since_date) != :lt end)
|
|
end
|
|
|
|
@spec parse_gfz_file(String.t()) :: [map()]
|
|
def parse_gfz_file(text) do
|
|
text
|
|
|> String.split("\n")
|
|
|> Enum.reject(fn line ->
|
|
trimmed = String.trim(line)
|
|
trimmed == "" or String.starts_with?(trimmed, "#")
|
|
end)
|
|
|> Enum.flat_map(fn line ->
|
|
case parse_gfz_row(line) do
|
|
{:ok, record} -> [record]
|
|
:error -> []
|
|
end
|
|
end)
|
|
end
|
|
|
|
@spec parse_gfz_row(String.t()) :: {:ok, map()} | :error
|
|
def parse_gfz_row(line) do
|
|
fields = String.split(line)
|
|
|
|
if length(fields) >= 28 do
|
|
[yyyy, mm, dd | rest] = fields
|
|
|
|
# Skip days, days_m, Bsr, dB (indices 0-3 of rest)
|
|
# Kp1-Kp8 at rest indices 4-11
|
|
# ap1-ap8 at rest indices 12-19
|
|
# Ap at rest index 20
|
|
# SN at rest index 21
|
|
# F10.7obs at rest index 22
|
|
# F10.7adj at rest index 23
|
|
kp_raw = Enum.slice(rest, 4, 8)
|
|
ap_daily = Enum.at(rest, 20)
|
|
sn = Enum.at(rest, 21)
|
|
f107_obs = Enum.at(rest, 22)
|
|
f107_adj = Enum.at(rest, 23)
|
|
|
|
{:ok,
|
|
%{
|
|
date: Date.new!(String.to_integer(yyyy), String.to_integer(mm), String.to_integer(dd)),
|
|
sfi: parse_float_or_nil(f107_obs),
|
|
sfi_adjusted: parse_float_or_nil(f107_adj),
|
|
sunspot_number: parse_int_or_nil(sn),
|
|
ap_index: parse_int_or_nil(ap_daily),
|
|
kp_values: Enum.map(kp_raw, &parse_float_or_nil/1)
|
|
}}
|
|
else
|
|
:error
|
|
end
|
|
end
|
|
|
|
defp parse_float_or_nil(str) do
|
|
val = String.to_float(str)
|
|
if val < 0, do: nil, else: val
|
|
rescue
|
|
ArgumentError ->
|
|
case Integer.parse(str) do
|
|
{n, _} when n < 0 -> nil
|
|
{n, _} -> n / 1
|
|
:error -> nil
|
|
end
|
|
end
|
|
|
|
defp parse_int_or_nil(str) do
|
|
case Integer.parse(str) do
|
|
{n, _} when n < 0 -> nil
|
|
{n, _} -> n
|
|
:error -> nil
|
|
end
|
|
end
|
|
end
|