Flip map read path to ScoresFile, stop writing grid HRRR profiles

Read-side cutover for the binary scores store and a companion
cleanup that removes the biggest remaining DB write from the hot
path.

Propagation.scores_at/3, available_valid_times/1, latest_valid_time/0,
latest_valid_time/1, earliest_valid_time/1, point_detail/4, and
point_forecast/3 all now prefer ScoresFile and fall back to the
propagation_scores table when a file is missing. The map render
path reads from /data/scores first; Postgres stays as a safety
net while dual-write is on. point_detail still pulls factors from
Postgres (analysis-hour rows only) and coalesces nil to an empty
map so the JS popup iterates cleanly.

replace_scores/2 is now gated by a postgres_writes_enabled? flag
(runtime env MICROWAVEPROP_SCORES_POSTGRES=false, or the
:propagation_scores_postgres app env key) so the binary-only path
can be benchmarked locally without the DB insert. Default stays
true.

PropagationGridWorker no longer calls store_hrrr_profiles —
persisting 92k grid rows × 19 forecast hours of JSONB profiles
was ~12 min of wall time per chain for a table only
AsosAdjustmentWorker read from. Per-contact HRRR enrichment
through HrrrFetchWorker still writes its own (is_grid_point:
false) rows. AsosAdjustmentWorker is disabled in all three cron
configs since its data source is gone.

DataCase resets the scores tree between tests so per-test
ScoresFile writes don't leak across cases, and ScoresFileTest
switches to async: false because it mutates the global
:propagation_scores_dir env.
This commit is contained in:
Graham McIntire 2026-04-14 14:50:43 -05:00
parent 690a3523e2
commit 07ffcf52d7
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
8 changed files with 439 additions and 121 deletions

View file

@ -79,7 +79,8 @@ config :microwaveprop, Oban,
{"*/5 * * * *", Microwaveprop.Commercial.PollWorker}, {"*/5 * * * *", Microwaveprop.Commercial.PollWorker},
{"5 */3 * * *", Microwaveprop.Workers.PropagationGridWorker}, {"5 */3 * * *", Microwaveprop.Workers.PropagationGridWorker},
{"*/2 * * * *", Microwaveprop.Workers.MrmsFetchWorker}, {"*/2 * * * *", Microwaveprop.Workers.MrmsFetchWorker},
{"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker}, # AsosAdjustmentWorker disabled — see runtime.exs for rationale.
# {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker},
{"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker}, {"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
# UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch # UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch
{"30 1 * * *", CanadianSoundingFetchWorker}, {"30 1 * * *", CanadianSoundingFetchWorker},

View file

@ -92,7 +92,8 @@ config :microwaveprop, Oban,
crontab: [ crontab: [
{"5 */3 * * *", Microwaveprop.Workers.PropagationGridWorker}, {"5 */3 * * *", Microwaveprop.Workers.PropagationGridWorker},
{"*/2 * * * *", Microwaveprop.Workers.MrmsFetchWorker}, {"*/2 * * * *", Microwaveprop.Workers.MrmsFetchWorker},
{"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker}, # AsosAdjustmentWorker disabled — see runtime.exs for rationale.
# {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker},
{"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker}, {"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
{"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker} {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker}
]} ]}

View file

@ -221,7 +221,11 @@ if config_env() == :prod do
# rain. Requires wgrib2 built with PNG decoder support (DRT # rain. Requires wgrib2 built with PNG decoder support (DRT
# 5.41), which the Dockerfile now enables via NCEPLIBS-g2c. # 5.41), which the Dockerfile now enables via NCEPLIBS-g2c.
{"*/2 * * * *", Microwaveprop.Workers.MrmsFetchWorker}, {"*/2 * * * *", Microwaveprop.Workers.MrmsFetchWorker},
{"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker}, # AsosAdjustmentWorker disabled: depends on hrrr_profiles rows
# that PropagationGridWorker no longer persists. The 10-min
# ASOS nudge is a nice-to-have refinement we're not paying the
# full grid-side JSONB write for.
# {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker},
{"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker}, {"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
{"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker, {"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker,
args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre"]}}, args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre"]}},

View file

@ -167,13 +167,41 @@ defmodule Microwaveprop.Propagation do
""" """
@spec replace_scores(Enumerable.t(), DateTime.t()) :: {:ok, non_neg_integer()} | {:error, term()} @spec replace_scores(Enumerable.t(), DateTime.t()) :: {:ok, non_neg_integer()} | {:error, term()}
def replace_scores(scores, %DateTime{} = valid_time) do def replace_scores(scores, %DateTime{} = valid_time) do
now = DateTime.truncate(DateTime.utc_now(), :second) # Materialize once — the DB insert and the per-band ScoresFile
# Materialize once — we need to traverse twice: once for the DB # writer both traverse the stream. 475k maps × ~150 B each is
# insert, once for the per-band ScoresFile writer. 475k maps × # ~70 MB, well within the pod memory budget.
# ~150 B each is ~70 MB, well within the pod memory budget.
scores_list = Enum.to_list(scores) scores_list = Enum.to_list(scores)
result = result =
if postgres_writes_enabled?() do
replace_postgres_scores(scores_list, valid_time)
else
{:ok, length(scores_list)}
end
case result do
{:ok, _count} -> write_score_files(scores_list, valid_time)
_error -> :ok
end
result
end
# Toggle for benchmarking the binary-file-only path. Flip to false
# via `config :microwaveprop, :propagation_scores_postgres: false`
# (or set the env var `MICROWAVEPROP_SCORES_POSTGRES=false`) in dev
# to skip the DB insert entirely.
defp postgres_writes_enabled? do
case System.get_env("MICROWAVEPROP_SCORES_POSTGRES") do
"false" -> false
"0" -> false
_ -> Application.get_env(:microwaveprop, :propagation_scores_postgres, true)
end
end
defp replace_postgres_scores(scores_list, valid_time) do
now = DateTime.truncate(DateTime.utc_now(), :second)
Repo.transaction( Repo.transaction(
fn -> fn ->
Repo.delete_all(from(gs in GridScore, where: gs.valid_time == ^valid_time), timeout: 300_000) Repo.delete_all(from(gs in GridScore, where: gs.valid_time == ^valid_time), timeout: 300_000)
@ -200,13 +228,6 @@ defmodule Microwaveprop.Propagation do
end, end,
timeout: 600_000 timeout: 600_000
) )
case result do
{:ok, _count} -> write_score_files(scores_list, valid_time)
_error -> :ok
end
result
end end
# Dual-write the grid to disk-backed ScoresFile binaries so the map # Dual-write the grid to disk-backed ScoresFile binaries so the map
@ -324,7 +345,7 @@ defmodule Microwaveprop.Propagation do
cutoff = DateTime.add(DateTime.utc_now(), -3600, :second) cutoff = DateTime.add(DateTime.utc_now(), -3600, :second)
case ScoreCache.valid_times(band_mhz) do case ScoreCache.valid_times(band_mhz) do
[] -> available_valid_times_from_db(band_mhz, cutoff) [] -> available_valid_times_from_store(band_mhz, cutoff)
cached -> filter_fresh(cached, cutoff) cached -> filter_fresh(cached, cutoff)
end end
end end
@ -333,6 +354,23 @@ defmodule Microwaveprop.Propagation do
Enum.filter(times, fn t -> DateTime.compare(t, cutoff) != :lt end) Enum.filter(times, fn t -> DateTime.compare(t, cutoff) != :lt end)
end end
defp available_valid_times_from_store(band_mhz, cutoff) do
case ScoresFile.list_valid_times(band_mhz) do
[] -> available_valid_times_from_db(band_mhz, cutoff)
times -> filter_or_latest(times, cutoff)
end
end
defp filter_or_latest(times, cutoff) do
fresh = Enum.filter(times, fn t -> DateTime.compare(t, cutoff) != :lt end)
if fresh == [] do
[Enum.max(times, DateTime)]
else
fresh
end
end
defp available_valid_times_from_db(band_mhz, cutoff) do defp available_valid_times_from_db(band_mhz, cutoff) do
times = times =
Repo.all( Repo.all(
@ -380,7 +418,7 @@ defmodule Microwaveprop.Propagation do
Enum.map(scores, &Map.put(&1, :valid_time, time)) Enum.map(scores, &Map.put(&1, :valid_time, time))
:miss -> :miss ->
full = load_scores_from_db(band_mhz, time) full = load_scores_from_store(band_mhz, time)
ScoreCache.put(band_mhz, time, full) ScoreCache.put(band_mhz, time, full)
full full
@ -389,6 +427,13 @@ defmodule Microwaveprop.Propagation do
end end
end end
defp load_scores_from_store(band_mhz, time) do
case ScoresFile.read_bounds(band_mhz, time) do
[] -> load_scores_from_db(band_mhz, time)
scores -> scores
end
end
defp load_scores_from_db(band_mhz, time) do defp load_scores_from_db(band_mhz, time) do
Repo.all( Repo.all(
from(gs in GridScore, from(gs in GridScore,
@ -427,8 +472,14 @@ defmodule Microwaveprop.Propagation do
end end
defp earliest_valid_time(band_mhz) do defp earliest_valid_time(band_mhz) do
case ScoresFile.list_valid_times(band_mhz) do
[earliest | _] ->
earliest
[] ->
Repo.one(from(gs in GridScore, where: gs.band_mhz == ^band_mhz, select: min(gs.valid_time))) Repo.one(from(gs in GridScore, where: gs.band_mhz == ^band_mhz, select: min(gs.valid_time)))
end end
end
@doc "Get scores across all forecast hours for a single grid point (for sparkline)." @doc "Get scores across all forecast hours for a single grid point (for sparkline)."
@spec point_forecast(non_neg_integer(), float(), float()) :: @spec point_forecast(non_neg_integer(), float(), float()) ::
@ -438,11 +489,31 @@ defmodule Microwaveprop.Propagation do
now = DateTime.utc_now() now = DateTime.utc_now()
case ScoreCache.valid_times(band_mhz) do case ScoreCache.valid_times(band_mhz) do
[] -> point_forecast_from_db(band_mhz, snapped_lat, snapped_lon, now) [] -> point_forecast_from_store(band_mhz, snapped_lat, snapped_lon, now)
cached -> point_forecast_from_cache(band_mhz, snapped_lat, snapped_lon, now, cached) cached -> point_forecast_from_cache(band_mhz, snapped_lat, snapped_lon, now, cached)
end end
end end
defp point_forecast_from_store(band_mhz, lat, lon, now) do
case ScoresFile.list_valid_times(band_mhz) do
[] ->
point_forecast_from_db(band_mhz, lat, lon, now)
times ->
times
|> Enum.filter(fn t -> DateTime.compare(t, now) != :lt end)
|> Enum.map(&point_forecast_entry(&1, band_mhz, lat, lon))
|> Enum.reject(&is_nil/1)
end
end
defp point_forecast_entry(valid_time, band_mhz, lat, lon) do
case ScoresFile.read_point(band_mhz, valid_time, lat, lon) do
nil -> nil
score -> %{valid_time: valid_time, score: score}
end
end
defp point_forecast_from_cache(band_mhz, lat, lon, now, cached_times) do defp point_forecast_from_cache(band_mhz, lat, lon, now, cached_times) do
cached_times cached_times
|> Enum.filter(fn t -> DateTime.compare(t, now) != :lt end) |> Enum.filter(fn t -> DateTime.compare(t, now) != :lt end)
@ -485,23 +556,38 @@ defmodule Microwaveprop.Propagation do
} }
| nil | nil
def point_detail(band_mhz, lat, lon, valid_time \\ nil) do def point_detail(band_mhz, lat, lon, valid_time \\ nil) do
step = Grid.step() {snapped_lat, snapped_lon} = snap_to_grid(lat, lon)
snapped_lat = Float.round(Float.round(lat / step) * step, 3)
snapped_lon = Float.round(Float.round(lon / step) * step, 3)
time = valid_time || latest_valid_time(band_mhz) time = valid_time || latest_valid_time(band_mhz)
case time do case time do
nil -> nil -> nil
nil _ -> build_point_detail(band_mhz, time, snapped_lat, snapped_lon)
end
end
_ -> defp build_point_detail(band_mhz, time, lat, lon) do
case ScoresFile.read_point(band_mhz, time, lat, lon) do
nil ->
point_detail_from_db(band_mhz, time, lat, lon)
score ->
%{
lat: lat,
lon: lon,
score: score,
factors: fetch_factors(band_mhz, time, lat, lon) || %{},
valid_time: time
}
end
end
defp point_detail_from_db(band_mhz, time, lat, lon) do
from(gs in GridScore, from(gs in GridScore,
where: where:
gs.band_mhz == ^band_mhz and gs.band_mhz == ^band_mhz and
gs.valid_time == ^time and gs.valid_time == ^time and
gs.lat == ^snapped_lat and gs.lat == ^lat and
gs.lon == ^snapped_lon, gs.lon == ^lon,
select: %{ select: %{
lat: gs.lat, lat: gs.lat,
lon: gs.lon, lon: gs.lon,
@ -513,6 +599,22 @@ defmodule Microwaveprop.Propagation do
|> Repo.one() |> Repo.one()
|> coalesce_factors() |> coalesce_factors()
end end
# Factors live only in Postgres (only for f00 analysis rows, since
# forecast hours skip the JSONB write entirely). Returns nil if
# the row isn't there — the caller falls back to an empty map so
# the JS popup iterates cleanly.
defp fetch_factors(band_mhz, time, lat, lon) do
Repo.one(
from(gs in GridScore,
where:
gs.band_mhz == ^band_mhz and
gs.valid_time == ^time and
gs.lat == ^lat and
gs.lon == ^lon,
select: gs.factors
)
)
end end
# Forecast-hour rows skip the factors JSONB to save write time, so # Forecast-hour rows skip the factors JSONB to save write time, so
@ -526,13 +628,19 @@ defmodule Microwaveprop.Propagation do
@doc "Get the latest valid_time across all scores." @doc "Get the latest valid_time across all scores."
@spec latest_valid_time() :: DateTime.t() | nil @spec latest_valid_time() :: DateTime.t() | nil
def latest_valid_time do def latest_valid_time do
Repo.one(from(gs in GridScore, select: max(gs.valid_time))) case ScoresFile.latest_valid_time() do
nil -> Repo.one(from(gs in GridScore, select: max(gs.valid_time)))
dt -> dt
end
end end
@doc "Get the latest valid_time for a specific band." @doc "Get the latest valid_time for a specific band."
@spec latest_valid_time(non_neg_integer()) :: DateTime.t() | nil @spec latest_valid_time(non_neg_integer()) :: DateTime.t() | nil
def latest_valid_time(band_mhz) do def latest_valid_time(band_mhz) do
Repo.one(from(gs in GridScore, where: gs.band_mhz == ^band_mhz, select: max(gs.valid_time))) case ScoresFile.list_valid_times(band_mhz) do
[] -> Repo.one(from(gs in GridScore, where: gs.band_mhz == ^band_mhz, select: max(gs.valid_time)))
times -> Enum.max(times, DateTime)
end
end end
# Prefer the persisted scalar — `hrrr_profiles` already stored this at # Prefer the persisted scalar — `hrrr_profiles` already stored this at

View file

@ -110,6 +110,84 @@ defmodule Microwaveprop.Propagation.ScoresFile do
end) end)
end end
@doc """
List every `valid_time` that has a score file on disk for
`band_mhz`, ordered ascending. Empty list if the band directory
doesn't exist yet.
"""
@spec list_valid_times(non_neg_integer()) :: [DateTime.t()]
def list_valid_times(band_mhz) when is_integer(band_mhz) do
dir = Path.join(base_dir(), Integer.to_string(band_mhz))
case File.ls(dir) do
{:ok, files} ->
files
|> Enum.map(&parse_valid_time_dt/1)
|> Enum.reject(&is_nil/1)
|> Enum.sort(DateTime)
_ ->
[]
end
end
@doc """
Return the most recent `valid_time` across all bands on disk, or
`nil` if the scores tree is empty.
"""
@spec latest_valid_time() :: DateTime.t() | nil
def latest_valid_time do
case File.ls(base_dir()) do
{:ok, band_dirs} ->
band_dirs
|> Enum.flat_map(&band_times_or_empty/1)
|> max_datetime()
_ ->
nil
end
end
defp band_times_or_empty(band_dir) do
case Integer.parse(band_dir) do
{band_mhz, ""} -> list_valid_times(band_mhz)
_ -> []
end
end
@doc """
Read the file for `(band_mhz, valid_time)` and return every
scored cell as a list of `%{lat, lon, score}` maps. Optional
`bounds` map (`%{"north", "south", "east", "west"}` or
`%{north:, south:, east:, west:}`) filters the return to cells
inside that box. Cells with the no-data sentinel are excluded.
Returns `[]` when the file doesn't exist or is unreadable —
callers should treat this as "no scores available for this
band/time yet" rather than an error.
"""
@spec read_bounds(non_neg_integer(), DateTime.t(), map() | nil) ::
[%{lat: float(), lon: float(), score: non_neg_integer()}]
def read_bounds(band_mhz, %DateTime{} = valid_time, bounds \\ nil) do
case read(band_mhz, valid_time) do
{:ok, payload} -> extract_points(payload, bounds)
_ -> []
end
end
@doc """
Fetch the score for a single grid cell from the file for
`(band_mhz, valid_time)`. Returns `nil` if the file is missing or
if the cell is outside the grid / carries the no-data sentinel.
"""
@spec read_point(non_neg_integer(), DateTime.t(), float(), float()) :: non_neg_integer() | nil
def read_point(band_mhz, %DateTime{} = valid_time, lat, lon) do
case read(band_mhz, valid_time) do
{:ok, payload} -> point_score(payload, lat, lon)
_ -> nil
end
end
# ── Encoding ──────────────────────────────────────────────────── # ── Encoding ────────────────────────────────────────────────────
defp encode(band_mhz, valid_time, scores) do defp encode(band_mhz, valid_time, scores) do
@ -190,6 +268,81 @@ defmodule Microwaveprop.Propagation.ScoresFile do
defp decode(_), do: {:error, :invalid_format} defp decode(_), do: {:error, :invalid_format}
# ── Read helpers ────────────────────────────────────────────────
defp extract_points(%{scores: body} = payload, bounds) do
%{lat_min: lat_min, lon_min: lon_min, step: step, n_rows: n_rows, n_cols: n_cols} = payload
{row_lo, row_hi, col_lo, col_hi} = bounds_to_index_range(payload, bounds)
for row <- row_lo..row_hi,
col <- col_lo..col_hi,
byte = :binary.at(body, row * n_cols + col),
byte != @no_data do
# guard against the ignored variables warning
_ = n_rows
lat = Float.round(lat_min + row * step, 3)
lon = Float.round(lon_min + col * step, 3)
%{lat: lat, lon: lon, score: byte}
end
end
defp bounds_to_index_range(payload, nil) do
{0, payload.n_rows - 1, 0, payload.n_cols - 1}
end
defp bounds_to_index_range(payload, bounds) do
%{lat_min: lat_min, lon_min: lon_min, step: step, n_rows: n_rows, n_cols: n_cols} = payload
south = fetch_bound(bounds, :south, "south", lat_min)
north = fetch_bound(bounds, :north, "north", lat_min + (n_rows - 1) * step)
west = fetch_bound(bounds, :west, "west", lon_min)
east = fetch_bound(bounds, :east, "east", lon_min + (n_cols - 1) * step)
row_lo = clamp(trunc((south - lat_min) / step), 0, n_rows - 1)
row_hi = clamp(ceil((north - lat_min) / step), 0, n_rows - 1)
col_lo = clamp(trunc((west - lon_min) / step), 0, n_cols - 1)
col_hi = clamp(ceil((east - lon_min) / step), 0, n_cols - 1)
{min(row_lo, row_hi), max(row_lo, row_hi), min(col_lo, col_hi), max(col_lo, col_hi)}
end
defp fetch_bound(bounds, atom_key, string_key, default) do
case Map.get(bounds, atom_key) || Map.get(bounds, string_key) do
nil -> default
v when is_number(v) -> v
v when is_binary(v) -> String.to_float(v)
end
end
defp clamp(n, lo, _hi) when n < lo, do: lo
defp clamp(n, _lo, hi) when n > hi, do: hi
defp clamp(n, _lo, _hi), do: n
defp point_score(payload, lat, lon) do
%{lat_min: lat_min, lon_min: lon_min, step: step, n_rows: n_rows, n_cols: n_cols, scores: body} = payload
row = round((lat - lat_min) / step)
col = round((lon - lon_min) / step)
if row in 0..(n_rows - 1) and col in 0..(n_cols - 1) do
case :binary.at(body, row * n_cols + col) do
@no_data -> nil
score -> score
end
end
end
defp max_datetime([]), do: nil
defp max_datetime(times), do: Enum.max(times, DateTime)
defp parse_valid_time_dt(filename) do
with [_, iso] <- Regex.run(~r/^(.+)\.ntms$/, filename),
{:ok, dt, _} <- DateTime.from_iso8601(iso) do
dt
else
_ -> nil
end
end
# ── Prune helpers ─────────────────────────────────────────────── # ── Prune helpers ───────────────────────────────────────────────
defp list_score_files(root) do defp list_score_files(root) do

View file

@ -32,7 +32,6 @@ defmodule Microwaveprop.Workers.PropagationGridWorker do
alias Microwaveprop.Weather.HrrrClient alias Microwaveprop.Weather.HrrrClient
alias Microwaveprop.Weather.HrrrNativeClient alias Microwaveprop.Weather.HrrrNativeClient
alias Microwaveprop.Weather.NexradClient alias Microwaveprop.Weather.NexradClient
alias Microwaveprop.Weather.SoundingParams
require Logger require Logger
@ -144,8 +143,13 @@ defmodule Microwaveprop.Workers.PropagationGridWorker do
HrrrClient.fetch_grid(points, run_time, forecast_hour: forecast_hour) HrrrClient.fetch_grid(points, run_time, forecast_hour: forecast_hour)
end) do end) do
{:ok, grid_data} -> {:ok, grid_data} ->
store_hrrr_profiles(grid_data, valid_time) # HRRR profiles used to be persisted to the hrrr_profiles table
:erlang.garbage_collect() # here for AsosAdjustmentWorker to re-score from. That was ~12
# minutes of JSONB inserts per chain (92k rows × 19 forecast
# hours) with no user-visible benefit — the scores live in the
# /data/scores files now, and AsosAdjustmentWorker is disabled.
# Per-contact HRRR enrichment still uses HrrrFetchWorker, which
# writes its own `is_grid_point: false` rows.
# Fetch native duct metrics and merge into grid_data for scoring # Fetch native duct metrics and merge into grid_data for scoring
grid_data = merge_native_duct_data(grid_data, run_time, forecast_hour) grid_data = merge_native_duct_data(grid_data, run_time, forecast_hour)
@ -221,60 +225,6 @@ defmodule Microwaveprop.Workers.PropagationGridWorker do
defp format_duration(ms) when ms < 1000, do: "#{ms}ms" defp format_duration(ms) when ms < 1000, do: "#{ms}ms"
defp format_duration(ms), do: "#{Float.round(ms / 1000, 1)}s" defp format_duration(ms), do: "#{Float.round(ms / 1000, 1)}s"
defp store_hrrr_profiles(grid_data, valid_time) do
count =
grid_data
|> Stream.flat_map(fn {{lat, lon}, profile} ->
build_profile_attrs_for_grid(lat, lon, valid_time, profile)
end)
|> Stream.chunk_every(500)
|> Enum.reduce(0, fn chunk, acc ->
Weather.upsert_hrrr_profiles_batch(chunk)
acc + length(chunk)
end)
Logger.info("PropagationGrid: stored #{count} HRRR profiles")
end
defp build_profile_attrs_for_grid(lat, lon, valid_time, profile) do
temp = profile.surface_temp_c
if temp && temp > -80 && temp < 60 do
params =
if is_list(profile.profile) and length(profile.profile) >= 3,
do: SoundingParams.derive(profile.profile)
attrs = %{
valid_time: valid_time,
lat: lat,
lon: lon,
run_time: profile.run_time,
profile: profile.profile || [],
hpbl_m: profile.hpbl_m,
pwat_mm: profile.pwat_mm,
surface_temp_c: profile.surface_temp_c,
surface_dewpoint_c: profile.surface_dewpoint_c,
surface_pressure_mb: profile.surface_pressure_mb
}
attrs =
if params do
Map.merge(attrs, %{
surface_refractivity: params.surface_refractivity,
min_refractivity_gradient: params.min_refractivity_gradient,
ducting_detected: params.ducting_detected,
duct_characteristics: params.duct_characteristics
})
else
attrs
end
[attrs]
else
[]
end
end
defp merge_native_duct_data(grid_data, run_time, forecast_hour) do defp merge_native_duct_data(grid_data, run_time, forecast_hour) do
hour_dt = HrrrClient.nearest_hrrr_hour(run_time) hour_dt = HrrrClient.nearest_hrrr_hour(run_time)
date = DateTime.to_date(hour_dt) date = DateTime.to_date(hour_dt)

View file

@ -1,5 +1,8 @@
defmodule Microwaveprop.Propagation.ScoresFileTest do defmodule Microwaveprop.Propagation.ScoresFileTest do
use ExUnit.Case, async: true # async: false — tests mutate the global Application env to redirect
# the scores dir to a private tmp tree. Running these in parallel
# would have concurrent tests trample each other's directory setting.
use ExUnit.Case, async: false
alias Microwaveprop.Propagation.Grid alias Microwaveprop.Propagation.Grid
alias Microwaveprop.Propagation.ScoresFile alias Microwaveprop.Propagation.ScoresFile
@ -94,6 +97,94 @@ defmodule Microwaveprop.Propagation.ScoresFileTest do
end end
end end
describe "list_valid_times/1" do
test "returns files sorted ascending for a band" do
ScoresFile.write!(10_000, ~U[2026-04-14 18:00:00Z], [])
ScoresFile.write!(10_000, ~U[2026-04-14 16:00:00Z], [])
ScoresFile.write!(10_000, ~U[2026-04-14 17:00:00Z], [])
# Unrelated band's files shouldn't leak in.
ScoresFile.write!(24_000, ~U[2026-04-14 20:00:00Z], [])
assert ScoresFile.list_valid_times(10_000) == [
~U[2026-04-14 16:00:00Z],
~U[2026-04-14 17:00:00Z],
~U[2026-04-14 18:00:00Z]
]
end
test "returns an empty list when the band dir doesn't exist" do
assert ScoresFile.list_valid_times(75_000) == []
end
end
describe "latest_valid_time/0" do
test "returns the max valid_time across every band on disk" do
ScoresFile.write!(10_000, ~U[2026-04-14 16:00:00Z], [])
ScoresFile.write!(24_000, ~U[2026-04-14 18:00:00Z], [])
ScoresFile.write!(10_000, ~U[2026-04-14 17:00:00Z], [])
assert ScoresFile.latest_valid_time() == ~U[2026-04-14 18:00:00Z]
end
test "returns nil on an empty tree" do
assert ScoresFile.latest_valid_time() == nil
end
end
describe "read_bounds/3" do
test "returns every scored cell (no-data excluded) when bounds is nil" do
ScoresFile.write!(10_000, ~U[2026-04-14 18:00:00Z], [
%{lat: 25.0, lon: -125.0, score: 10},
%{lat: 25.125, lon: -125.0, score: 20}
])
scores = ScoresFile.read_bounds(10_000, ~U[2026-04-14 18:00:00Z])
assert [
%{lat: 25.0, lon: -125.0, score: 10},
%{lat: 25.125, lon: -125.0, score: 20}
] = Enum.sort_by(scores, & &1.lat)
end
test "restricts results to the bounding box when provided" do
ScoresFile.write!(10_000, ~U[2026-04-14 18:00:00Z], [
%{lat: 25.0, lon: -125.0, score: 10},
%{lat: 49.875, lon: -66.0, score: 90}
])
bounds = %{"south" => 24.0, "north" => 26.0, "west" => -126.0, "east" => -124.0}
scores = ScoresFile.read_bounds(10_000, ~U[2026-04-14 18:00:00Z], bounds)
assert scores == [%{lat: 25.0, lon: -125.0, score: 10}]
end
test "returns an empty list when the file doesn't exist" do
assert ScoresFile.read_bounds(10_000, ~U[2026-04-14 18:00:00Z]) == []
end
end
describe "read_point/4" do
test "returns the score for a grid cell by (lat, lon) lookup" do
ScoresFile.write!(10_000, ~U[2026-04-14 18:00:00Z], [
%{lat: 25.0, lon: -125.0, score: 42}
])
assert ScoresFile.read_point(10_000, ~U[2026-04-14 18:00:00Z], 25.0, -125.0) == 42
end
test "returns nil for cells that weren't scored" do
ScoresFile.write!(10_000, ~U[2026-04-14 18:00:00Z], [
%{lat: 25.0, lon: -125.0, score: 42}
])
assert ScoresFile.read_point(10_000, ~U[2026-04-14 18:00:00Z], 40.0, -100.0) == nil
end
test "returns nil when the file is missing" do
assert ScoresFile.read_point(10_000, ~U[2026-04-14 18:00:00Z], 25.0, -125.0) == nil
end
end
describe "prune_older_than/1" do describe "prune_older_than/1" do
test "deletes files whose valid_time is strictly before the cutoff", %{dir: _dir} do test "deletes files whose valid_time is strictly before the cutoff", %{dir: _dir} do
old = ~U[2026-04-14 10:00:00Z] old = ~U[2026-04-14 10:00:00Z]

View file

@ -31,6 +31,7 @@ defmodule Microwaveprop.DataCase do
setup tags do setup tags do
Microwaveprop.DataCase.setup_sandbox(tags) Microwaveprop.DataCase.setup_sandbox(tags)
Microwaveprop.DataCase.reset_score_files()
:ok :ok
end end
@ -42,6 +43,15 @@ defmodule Microwaveprop.DataCase do
on_exit(fn -> Sandbox.stop_owner(pid) end) on_exit(fn -> Sandbox.stop_owner(pid) end)
end end
@doc """
Wipe the propagation ScoresFile tree between tests so files don't
leak between cases sharing the same tmp dir.
"""
def reset_score_files do
dir = Application.get_env(:microwaveprop, :propagation_scores_dir)
if is_binary(dir), do: File.rm_rf!(dir)
end
@doc """ @doc """
A helper that transforms changeset errors into a map of messages. A helper that transforms changeset errors into a map of messages.