Graham McIntire
d41ced5850
Phase 3 Stream A: Rust's prop-grid-rs writes f00 profile files as
MessagePack (c4f309c). Elixir needs to read both formats during the
cutover window and indefinitely afterward — .mp.gz wins when both
exist.
Reader:
- path_for/1 stays on .etf.gz (Elixir legacy writer)
- mp_path_for/1 returns the .mp.gz sibling Rust lands
- read/1 prefers .mp.gz, falls back to .etf.gz, :enoent if neither
- decode_mp_body handles the top-level {v, valid_time, cells} shape
- normalize_profile atomizes a whitelist of known keys (surface_*,
native_min_gradient, ducts[], profile[] levels, ...) and leaves
everything else as strings so no String.to_atom bomb is possible
- Directory scan picks up both extensions so retain_window and
prune_older_than also work uniformly; list_valid_times dedupes
by timestamp
Adds msgpax ~> 2.4 dependency. 4 new tests cover format preference,
key atomization, :enoent path, and dedupe.
319 lines
10 KiB
Elixir
319 lines
10 KiB
Elixir
defmodule Microwaveprop.Propagation.ProfilesFile do
|
|
@moduledoc """
|
|
On-disk store for the raw, enriched HRRR grid_data that
|
|
`PropagationGridWorker` produces for the f00 analysis hour. One
|
|
compressed ETF file per `valid_time` lands at
|
|
`{base_dir}/profiles/{iso}.etf.gz`. Used by `Propagation.point_detail/4`
|
|
to rebuild the factor breakdown for a clicked cell without a database
|
|
round trip.
|
|
|
|
Only f00 is persisted — forecast hours intentionally skip the factor
|
|
breakdown on the map to keep the scoring+upsert phase under a minute
|
|
(see `Propagation.compute_scores/3`).
|
|
|
|
## Atomic writes
|
|
|
|
Files are written through `rename(2)`: `path.tmp.<uniq>` → rename to
|
|
the final path. On the shared NFS mount the rename is atomic, so a
|
|
concurrent reader sees either the old file, the new file, or nothing —
|
|
never a partial write.
|
|
"""
|
|
|
|
alias Microwaveprop.Propagation.Grid
|
|
|
|
require Logger
|
|
|
|
@doc "Base directory the profile store lives under."
|
|
@spec base_dir() :: String.t()
|
|
def base_dir do
|
|
Path.join(
|
|
Application.get_env(:microwaveprop, :propagation_scores_dir, "/data/scores"),
|
|
"profiles"
|
|
)
|
|
end
|
|
|
|
@doc "Absolute path for the Elixir-written ETF profile file covering `valid_time`."
|
|
@spec path_for(DateTime.t()) :: String.t()
|
|
def path_for(%DateTime{} = valid_time) do
|
|
Path.join(base_dir(), "#{iso_key(valid_time)}.etf.gz")
|
|
end
|
|
|
|
@doc """
|
|
Absolute path for the Rust-written MessagePack profile file covering
|
|
`valid_time`. Rust's `prop_grid_rs::profiles_file::write_atomic`
|
|
lands here. Reader prefers `.mp.gz` when both formats coexist during
|
|
Phase 3 Stream A cutover.
|
|
"""
|
|
@spec mp_path_for(DateTime.t()) :: String.t()
|
|
def mp_path_for(%DateTime{} = valid_time) do
|
|
Path.join(base_dir(), "#{iso_key(valid_time)}.mp.gz")
|
|
end
|
|
|
|
defp iso_key(%DateTime{} = valid_time) do
|
|
valid_time |> DateTime.truncate(:second) |> DateTime.to_iso8601()
|
|
end
|
|
|
|
# Keys Rust writes as strings; Elixir callers read them as atoms.
|
|
# Anything outside this whitelist stays a string and is ignored by
|
|
# downstream consumers (weather.ex uses the atom form throughout).
|
|
@mp_atom_keys MapSet.new([
|
|
"native_min_gradient",
|
|
"best_duct_freq_ghz",
|
|
"max_duct_thickness_m",
|
|
"duct_count",
|
|
"ducts",
|
|
"base_m",
|
|
"top_m",
|
|
"thickness_m",
|
|
"m_deficit",
|
|
"min_freq_ghz",
|
|
"surface_temp_c",
|
|
"surface_dewpoint_c",
|
|
"surface_pressure_mb",
|
|
"surface_refractivity",
|
|
"hpbl_m",
|
|
"pwat_mm",
|
|
"min_refractivity_gradient",
|
|
"ducting_detected",
|
|
"duct_characteristics",
|
|
"nexrad_max_reflectivity_dbz",
|
|
"commercial_link_degradation",
|
|
"degradation_db",
|
|
"baseline_dbm",
|
|
"current_dbm",
|
|
"n_links",
|
|
"profile",
|
|
"pres_mb",
|
|
"hght_m",
|
|
"tmpc",
|
|
"dwpc"
|
|
])
|
|
|
|
@doc """
|
|
Persist the enriched grid_data (`%{{lat, lon} => profile}`) for
|
|
`valid_time`. Writes compressed ETF; lat/lon keys are rounded to the
|
|
grid step so later point lookups can snap user-provided coordinates.
|
|
"""
|
|
@spec write!(DateTime.t(), %{{float(), float()} => map()}) :: :ok
|
|
def write!(%DateTime{} = valid_time, grid_data) when is_map(grid_data) do
|
|
path = path_for(valid_time)
|
|
File.mkdir_p!(Path.dirname(path))
|
|
|
|
snapped =
|
|
Map.new(grid_data, fn {{lat, lon}, profile} ->
|
|
{{Float.round(lat, 3), Float.round(lon, 3)}, profile}
|
|
end)
|
|
|
|
binary = :erlang.term_to_binary(snapped, [:compressed])
|
|
|
|
tmp = path <> ".tmp." <> unique_suffix()
|
|
File.write!(tmp, binary, [:binary])
|
|
File.rename!(tmp, path)
|
|
:ok
|
|
end
|
|
|
|
@doc """
|
|
Read the full persisted grid_data for `valid_time`. Returns
|
|
`{:ok, %{{lat, lon} => profile}}` or `{:error, :enoent}` if the file
|
|
doesn't exist. Used by `Microwaveprop.Weather` to warm `GridCache`
|
|
on pod startup without touching the database.
|
|
|
|
Does *not* pass `[:safe]` to `binary_to_term/1`. The file was written
|
|
by `write!/2` in the same code base, so we trust it. The previous
|
|
`[:safe]` flag was rejecting our own files at app-startup warm time
|
|
because some atoms in the persisted profile (`:base_m`, `:top_m`,
|
|
`:thickness_m`, `:min_freq_ghz`, `:native_min_gradient`, etc.) live
|
|
in modules that aren't eagerly loaded during boot, so they weren't
|
|
in the runtime atom table when `warm_grid_cache_from_latest_profile`
|
|
ran. The fix is not "load more modules eagerly" — it's "trust files
|
|
we wrote ourselves."
|
|
"""
|
|
@spec read(DateTime.t()) :: {:ok, %{{float(), float()} => map()}} | {:error, :enoent}
|
|
def read(%DateTime{} = valid_time) do
|
|
# Rust-written MessagePack wins over Elixir-written ETF if both
|
|
# exist. After Phase 3 Stream A cutover only `.mp.gz` is produced;
|
|
# the ETF branch remains so pods picking up an older in-flight run
|
|
# during the rollout window still render correctly.
|
|
cond do
|
|
File.exists?(mp_path_for(valid_time)) -> read_mp(valid_time)
|
|
File.exists?(path_for(valid_time)) -> read_etf(valid_time)
|
|
true -> {:error, :enoent}
|
|
end
|
|
end
|
|
|
|
defp read_etf(valid_time) do
|
|
case File.read(path_for(valid_time)) do
|
|
{:ok, binary} -> {:ok, :erlang.binary_to_term(binary)}
|
|
{:error, _} -> {:error, :enoent}
|
|
end
|
|
end
|
|
|
|
defp read_mp(valid_time) do
|
|
with {:ok, gz} <- File.read(mp_path_for(valid_time)),
|
|
binary = :zlib.gunzip(gz),
|
|
{:ok, body} <- Msgpax.unpack(binary) do
|
|
{:ok, decode_mp_body(body)}
|
|
else
|
|
{:error, _} -> {:error, :enoent}
|
|
end
|
|
end
|
|
|
|
defp decode_mp_body(%{"cells" => cells}) when is_list(cells) do
|
|
Map.new(cells, fn cell ->
|
|
lat = cell |> Map.get("lat") |> to_float()
|
|
lon = cell |> Map.get("lon") |> to_float()
|
|
profile = cell |> Map.get("profile", %{}) |> normalize_profile()
|
|
{{lat, lon}, profile}
|
|
end)
|
|
end
|
|
|
|
defp decode_mp_body(_), do: %{}
|
|
|
|
defp normalize_profile(value) when is_map(value) do
|
|
Map.new(value, fn {k, v} ->
|
|
key = if is_binary(k) and MapSet.member?(@mp_atom_keys, k), do: String.to_atom(k), else: k
|
|
{key, normalize_profile(v)}
|
|
end)
|
|
end
|
|
|
|
defp normalize_profile(value) when is_list(value) do
|
|
Enum.map(value, &normalize_profile/1)
|
|
end
|
|
|
|
defp normalize_profile(value), do: value
|
|
|
|
defp to_float(v) when is_float(v), do: v
|
|
defp to_float(v) when is_integer(v), do: v * 1.0
|
|
defp to_float(_), do: 0.0
|
|
|
|
@doc """
|
|
Read a single profile for `(valid_time, lat, lon)`. Returns `nil` if
|
|
the file is missing or the point has no profile. Input lat/lon are
|
|
snapped to the nearest grid cell before lookup.
|
|
"""
|
|
@spec read_point(DateTime.t(), float(), float()) :: map() | nil
|
|
def read_point(%DateTime{} = valid_time, lat, lon) do
|
|
case read(valid_time) do
|
|
{:ok, grid_data} ->
|
|
{snapped_lat, snapped_lon} = snap(lat, lon)
|
|
Map.get(grid_data, {snapped_lat, snapped_lon})
|
|
|
|
{:error, _} ->
|
|
nil
|
|
end
|
|
end
|
|
|
|
@doc """
|
|
Delete profile files whose valid_time is strictly before `cutoff`.
|
|
Returns the number of files removed.
|
|
"""
|
|
@spec prune_older_than(DateTime.t()) :: non_neg_integer()
|
|
def prune_older_than(%DateTime{} = cutoff) do
|
|
cutoff_unix = DateTime.to_unix(cutoff)
|
|
|
|
base_dir()
|
|
|> list_profile_files()
|
|
|> Enum.reduce(0, fn {path, valid_time_unix}, acc ->
|
|
if valid_time_unix < cutoff_unix do
|
|
_ = File.rm(path)
|
|
acc + 1
|
|
else
|
|
acc
|
|
end
|
|
end)
|
|
end
|
|
|
|
@doc """
|
|
Keep only profile files whose valid_time is inside the closed window
|
|
`[run_time, run_time + max_forecast_hour * 3600]`. Mirrors
|
|
`ScoresFile.retain_window/2` and is called at the end of a
|
|
`PropagationGridWorker` chain.
|
|
"""
|
|
@spec retain_window(DateTime.t(), non_neg_integer()) :: non_neg_integer()
|
|
def retain_window(%DateTime{} = run_time, max_forecast_hour) when max_forecast_hour >= 0 do
|
|
lo = DateTime.to_unix(run_time)
|
|
hi = lo + max_forecast_hour * 3600
|
|
|
|
base_dir()
|
|
|> list_profile_files()
|
|
|> Enum.reduce(0, fn {path, valid_time_unix}, acc ->
|
|
if valid_time_unix < lo or valid_time_unix > hi do
|
|
_ = File.rm(path)
|
|
acc + 1
|
|
else
|
|
acc
|
|
end
|
|
end)
|
|
end
|
|
|
|
@doc """
|
|
Latest valid_time of any persisted profile, or nil if the store is
|
|
empty.
|
|
"""
|
|
@spec latest_valid_time() :: DateTime.t() | nil
|
|
def latest_valid_time do
|
|
case list_profile_files(base_dir()) do
|
|
[] ->
|
|
nil
|
|
|
|
files ->
|
|
files
|
|
|> Enum.map(fn {_path, unix} -> unix end)
|
|
|> Enum.max()
|
|
|> DateTime.from_unix!()
|
|
end
|
|
end
|
|
|
|
@doc """
|
|
Every persisted valid_time, sorted ascending. Used to build the
|
|
/weather forecast timeline from the on-disk f00..f18 profile files.
|
|
"""
|
|
@spec list_valid_times() :: [DateTime.t()]
|
|
def list_valid_times do
|
|
base_dir()
|
|
|> list_profile_files()
|
|
|> Enum.map(fn {_path, unix} -> unix end)
|
|
|> Enum.uniq()
|
|
|> Enum.map(&DateTime.from_unix!/1)
|
|
|> Enum.sort(DateTime)
|
|
end
|
|
|
|
defp snap(lat, lon) do
|
|
step = Grid.step()
|
|
snapped_lat = Float.round(Float.round(lat / step) * step, 3)
|
|
snapped_lon = Float.round(Float.round(lon / step) * step, 3)
|
|
{snapped_lat, snapped_lon}
|
|
end
|
|
|
|
defp list_profile_files(dir) do
|
|
case File.ls(dir) do
|
|
{:ok, entries} ->
|
|
for entry <- entries,
|
|
valid_time_unix = parse_valid_time(entry),
|
|
valid_time_unix != nil do
|
|
{Path.join(dir, entry), valid_time_unix}
|
|
end
|
|
|
|
_ ->
|
|
[]
|
|
end
|
|
end
|
|
|
|
defp parse_valid_time(filename) do
|
|
# Matches both `.etf.gz` (Elixir legacy) and `.mp.gz` (Rust Phase 3
|
|
# Stream A). If both exist for the same valid_time the directory
|
|
# listing yields two entries with the same timestamp; the pipeline
|
|
# downstream of list_valid_times/0 uniq-sorts, and read/1 prefers
|
|
# the mp.gz variant via mp_path_for/1.
|
|
with [_, iso, _fmt] <- Regex.run(~r/^(.+)\.(etf|mp)\.gz$/, filename),
|
|
{:ok, dt, _} <- DateTime.from_iso8601(iso) do
|
|
DateTime.to_unix(dt)
|
|
else
|
|
_ -> nil
|
|
end
|
|
end
|
|
|
|
defp unique_suffix do
|
|
"#{System.system_time(:nanosecond)}.#{:erlang.unique_integer([:positive])}"
|
|
end
|
|
end
|