prop/lib/microwaveprop/propagation/profiles_file.ex

defmodule Microwaveprop.Propagation.ProfilesFile do
  @moduledoc """
  On-disk store for the raw, enriched HRRR grid_data that the Rust
  pipeline (`prop-grid-rs`) produces for every forecast hour (f00..f18
  hourly, f21..f48 3-hourly). One compressed file per `valid_time` lands at
  `{base_dir}/profiles/{iso}.pgrid` (dense cell-major f32 binary, the
  primary format since Rust Phase 4). Legacy `.mp.gz` and `.etf.gz`
  formats are still readable during the drain/transition window. Used by
  `Propagation.point_detail/4` to rebuild the factor breakdown for a
  clicked cell without a database round trip, and by the weather scalar
  pipeline to derive the `.sgrid` per-forecast-hour files.

  ## Atomic writes

  Files are written through `rename(2)`: `path.tmp.<uniq>` → rename to
  the final path. On the shared NFS mount the rename is atomic, so a
  concurrent reader sees either the old file, the new file, or nothing —
  never a partial write.
  """

  alias Microwaveprop.Propagation.Grid
  alias Microwaveprop.Propagation.Pgrid

  @doc "Base directory the profile store lives under."
  @spec base_dir() :: String.t()
  def base_dir do
    Path.join(
      Application.get_env(:microwaveprop, :propagation_scores_dir, "/data/scores"),
      "profiles"
    )
  end

  @doc "Absolute path for the Elixir-written ETF profile file covering `valid_time`."
  @spec path_for(DateTime.t()) :: String.t()
  def path_for(%DateTime{} = valid_time) do
    Path.join(base_dir(), "#{iso_key(valid_time)}.etf.gz")
  end

  @doc """
  Absolute path for the Rust-written MessagePack profile file covering
  `valid_time`. Rust's `prop_grid_rs::profiles_file::write_atomic`
  lands here. Reader prefers `.mp.gz` when both formats coexist during
  Phase 3 Stream A cutover.
  """
  @spec mp_path_for(DateTime.t()) :: String.t()
  def mp_path_for(%DateTime{} = valid_time) do
    Path.join(base_dir(), "#{iso_key(valid_time)}.mp.gz")
  end

  defp iso_key(%DateTime{} = valid_time) do
    valid_time |> DateTime.truncate(:second) |> DateTime.to_iso8601()
  end

  # Keys Rust writes as strings; Elixir callers read them as atoms.
  # Anything outside this whitelist stays a string and is ignored by
  # downstream consumers (weather.ex uses the atom form throughout).
  @mp_atom_keys MapSet.new([
                  "native_min_gradient",
                  "best_duct_freq_ghz",
                  "max_duct_thickness_m",
                  "duct_count",
                  "wind_u",
                  "wind_v",
                  "cloud_cover_pct",
                  "precip_mm",
                  "ducts",
                  "base_m",
                  "top_m",
                  "thickness_m",
                  "m_deficit",
                  "min_freq_ghz",
                  "surface_temp_c",
                  "surface_dewpoint_c",
                  "surface_pressure_mb",
                  "surface_refractivity",
                  "hpbl_m",
                  "pwat_mm",
                  "min_refractivity_gradient",
                  "ducting_detected",
                  "duct_characteristics",
                  "nexrad_max_reflectivity_dbz",
                  "commercial_link_degradation",
                  "degradation_db",
                  "baseline_dbm",
                  "current_dbm",
                  "n_links",
                  "profile",
                  "pres_mb",
                  "hght_m",
                  "tmpc",
                  "dwpc"
                ])

  @doc """
  Persist the enriched grid_data (`%{{lat, lon} => profile}`) for
  `valid_time`. Writes compressed ETF; lat/lon keys are rounded to the
  grid step so later point lookups can snap user-provided coordinates.
  """
  @spec write!(DateTime.t(), %{{float(), float()} => map()}) :: :ok
  def write!(%DateTime{} = valid_time, grid_data) when is_map(grid_data) do
    path = path_for(valid_time)
    File.mkdir_p!(Path.dirname(path))

    snapped =
      Map.new(grid_data, fn {{lat, lon}, profile} ->
        {{Float.round(lat, 3), Float.round(lon, 3)}, profile}
      end)

    binary = :erlang.term_to_binary(snapped, [:compressed])

    tmp = path <> ".tmp." <> unique_suffix()
    File.write!(tmp, binary, [:binary])
    File.rename!(tmp, path)
    invalidate_caches_for(valid_time)
    :ok
  end

  @doc """
  Read the full persisted grid_data for `valid_time`. Returns
  `{:ok, %{{lat, lon} => profile}}` or `{:error, :enoent}` if the file
  doesn't exist. Used by `Microwaveprop.Weather` to warm `GridCache`
  on pod startup without touching the database.

  Does *not* pass `[:safe]` to `binary_to_term/1`. The file was written
  by `write!/2` in the same code base, so we trust it. The previous
  `[:safe]` flag was rejecting our own files at app-startup warm time
  because some atoms in the persisted profile (`:base_m`, `:top_m`,
  `:thickness_m`, `:min_freq_ghz`, `:native_min_gradient`, etc.) live
  in modules that aren't eagerly loaded during boot, so they weren't
  in the runtime atom table when `warm_grid_cache_from_latest_profile`
  ran. The fix is not "load more modules eagerly" — it's "trust files
  we wrote ourselves."
  """
  @spec read(DateTime.t()) :: {:ok, %{{float(), float()} => map()}} | {:error, :enoent}
  def read(%DateTime{} = valid_time) do
    # Decoded profile maps are ~10 MB each (92k cells × atmospheric
    # profile). A map click fires `point_detail` → `factors_for` which
    # calls this, and the user scrubbing the forecast timeline hits
    # the same valid_time repeatedly within seconds. Caching the full
    # decoded map for 5s turns scrub clicks into ETS lookups.
    Microwaveprop.Cache.fetch_or_store({__MODULE__, :read, base_dir(), valid_time}, 5_000, fn ->
      do_read(valid_time)
    end)
  end

  defp do_read(valid_time) do
    # Preference order is newest-format-first. Rust now writes only
    # `.pgrid` (dense cell-major f32, see `Microwaveprop.Propagation.Pgrid`);
    # the `.mp.gz` and `.etf.gz` branches remain so pods reading files
    # written before the cutover still render while those drain out of
    # the 48 h retention window.
    cond do
      Pgrid.exists?(valid_time) -> Pgrid.read(valid_time)
      File.exists?(mp_path_for(valid_time)) -> read_mp(valid_time)
      File.exists?(path_for(valid_time)) -> read_etf(valid_time)
      true -> {:error, :enoent}
    end
  end

  defp read_etf(valid_time) do
    case File.read(path_for(valid_time)) do
      # `:safe` rejects unknown atoms and external function refs in the
      # decoded term — defense-in-depth in case the on-disk file is ever
      # tampered with (it is owned by the prop pipeline, but the cost is
      # zero and the failure mode without :safe is atom-table exhaustion).
      {:ok, binary} -> {:ok, :erlang.binary_to_term(binary, [:safe])}
      {:error, _} -> {:error, :enoent}
    end
  end

  defp read_mp(valid_time) do
    with {:ok, gz} <- File.read(mp_path_for(valid_time)),
         {:ok, binary} <- gunzip_safe(gz),
         {:ok, body} <- Msgpax.unpack(binary) do
      {:ok, decode_mp_body(body)}
    else
      {:error, _} -> {:error, :enoent}
    end
  end

  defp gunzip_safe(data) do
    {:ok, :zlib.gunzip(data)}
  rescue
    _ -> {:error, :corrupt}
  end

  defp decode_mp_body(%{"cells" => cells}) when is_list(cells) do
    Map.new(cells, fn cell ->
      lat = cell |> Map.get("lat") |> to_float()
      lon = cell |> Map.get("lon") |> to_float()
      profile = cell |> Map.get("profile", %{}) |> normalize_profile()
      {{lat, lon}, profile}
    end)
  end

  defp decode_mp_body(_), do: %{}

  defp normalize_profile(value) when is_map(value) do
    Map.new(value, fn {k, v} ->
      key = if is_binary(k) and MapSet.member?(@mp_atom_keys, k), do: String.to_existing_atom(k), else: k
      {key, normalize_profile(v)}
    end)
  end

  defp normalize_profile(value) when is_list(value) do
    Enum.map(value, &normalize_profile/1)
  end

  defp normalize_profile(value), do: value

  defp to_float(v) when is_float(v), do: v
  defp to_float(v) when is_integer(v), do: v * 1.0
  defp to_float(_), do: 0.0

  @doc """
  Read a single profile for `(valid_time, lat, lon)`. Returns `nil` if
  the file is missing or the point has no profile. Input lat/lon are
  snapped to the nearest grid cell before lookup.
  """
  @spec read_point(DateTime.t(), float(), float()) :: map() | nil
  def read_point(%DateTime{} = valid_time, lat, lon) do
    {snapped_lat, snapped_lon} = snap(lat, lon)

    # `.pgrid` supports true random access: one `pread` of the cell's
    # record. Do NOT route this through `read/1` — that would decode the
    # whole grid to answer a single point lookup, which is exactly the
    # cost this format exists to remove. The legacy branches have no
    # random access, so they still go through the cached full read.
    if Pgrid.exists?(valid_time) do
      Pgrid.read_point(valid_time, snapped_lat, snapped_lon)
    else
      case read(valid_time) do
        {:ok, grid_data} -> Map.get(grid_data, {snapped_lat, snapped_lon})
        {:error, _} -> nil
      end
    end
  end

  @doc """
  Delete profile files whose valid_time is strictly before `cutoff`.
  Returns the number of files removed.
  """
  @spec prune_older_than(DateTime.t()) :: non_neg_integer()
  def prune_older_than(%DateTime{} = cutoff) do
    cutoff_unix = DateTime.to_unix(cutoff)

    deleted =
      base_dir()
      |> list_profile_files()
      |> Enum.reduce(0, fn {path, valid_time_unix}, acc ->
        if valid_time_unix < cutoff_unix do
          _ = File.rm(path)
          acc + 1
        else
          acc
        end
      end)

    if deleted > 0, do: invalidate_all_caches()
    deleted
  end

  @doc """
  Keep only profile files whose valid_time is inside the closed window
  `[run_time, run_time + max_forecast_hour * 3600]`. Mirrors
  `ScoresFile.retain_window/2` and is called at the end of a
  `PropagationGridWorker` chain.
  """
  @spec retain_window(DateTime.t(), non_neg_integer()) :: non_neg_integer()
  def retain_window(%DateTime{} = run_time, max_forecast_hour) when max_forecast_hour >= 0 do
    lo = DateTime.to_unix(run_time)
    hi = lo + max_forecast_hour * 3600

    deleted =
      base_dir()
      |> list_profile_files()
      |> Enum.reduce(0, fn {path, valid_time_unix}, acc ->
        if valid_time_unix < lo or valid_time_unix > hi do
          _ = File.rm(path)
          acc + 1
        else
          acc
        end
      end)

    if deleted > 0, do: invalidate_all_caches()
    deleted
  end

  @doc """
  Latest valid_time of any persisted profile, or nil if the store is
  empty.
  """
  @spec latest_valid_time() :: DateTime.t() | nil
  def latest_valid_time do
    case list_profile_files(base_dir()) do
      [] ->
        nil

      files ->
        files
        |> Enum.map(fn {_path, unix} -> unix end)
        |> Enum.max()
        |> DateTime.from_unix!()
    end
  end

  @doc """
  Every persisted valid_time, sorted ascending. Used to build the
  /weather forecast timeline from the on-disk f00..f48 profile files.
  """
  @spec list_valid_times() :: [DateTime.t()]
  def list_valid_times do
    # Hit on every point_detail click when the clicked valid_time
    # lacks its own profile and fallback scan searches for the
    # nearest past analysis. NFS dir-listing on every click was a
    # visible chunk of click latency.
    Microwaveprop.Cache.fetch_or_store({__MODULE__, :list_valid_times, base_dir()}, 5_000, fn ->
      base_dir()
      |> list_profile_files()
      |> Enum.map(fn {_path, unix} -> unix end)
      |> Enum.uniq()
      |> Enum.map(&DateTime.from_unix!/1)
      |> Enum.sort(DateTime)
    end)
  end

  defp invalidate_caches_for(valid_time) do
    dir = base_dir()
    Microwaveprop.Cache.invalidate({__MODULE__, :read, dir, valid_time})
    Microwaveprop.Cache.invalidate({__MODULE__, :list_valid_times, dir})
    :ok
  end

  defp invalidate_all_caches do
    Microwaveprop.Cache.match_delete({{__MODULE__, :read, :_, :_}, :_, :_})
    Microwaveprop.Cache.invalidate({__MODULE__, :list_valid_times, base_dir()})
  end

  @doc """
  Snap a lat/lon pair to the nearest grid cell key used in the decoded
  profile map. Public so callers that have already loaded the full grid
  via `read/1` can do their own keyed lookups without re-fetching.
  """
  @spec snap(float(), float()) :: {float(), float()}
  def snap(lat, lon) do
    step = Grid.step()
    snapped_lat = Float.round(Float.round(lat / step) * step, 3)
    snapped_lon = Float.round(Float.round(lon / step) * step, 3)
    {snapped_lat, snapped_lon}
  end

  defp list_profile_files(dir) do
    case File.ls(dir) do
      {:ok, entries} ->
        for entry <- entries,
            valid_time_unix = parse_valid_time(entry),
            valid_time_unix != nil do
          {Path.join(dir, entry), valid_time_unix}
        end

      _ ->
        []
    end
  end

  defp parse_valid_time(filename) do
    # Matches `.pgrid` (current), `.mp.gz` (Rust Phase 3 Stream A) and
    # `.etf.gz` (Elixir legacy). If several exist for the same valid_time
    # the directory listing yields multiple entries with the same
    # timestamp; the pipeline downstream of list_valid_times/0 uniq-sorts,
    # and read/1 prefers the newest format.
    #
    # Anchoring on the *known* extension set (rather than a permissive
    # `\.(.+)$`) is what keeps sibling-suffixed files such as
    # `<iso>.hrdps.prop` from parsing as `<iso>.hrdps` and then failing
    # `DateTime.from_iso8601/1` — the failure mode that made HRDPS score
    # files invisible to every prune and list operation.
    with [_, iso] <- Regex.run(~r/^(.+)\.(?:pgrid|etf\.gz|mp\.gz)$/, filename),
         {:ok, dt, _} <- DateTime.from_iso8601(iso) do
      DateTime.to_unix(dt)
    else
      _ -> nil
    end
  end

  defp unique_suffix do
    "#{System.system_time(:nanosecond)}.#{:erlang.unique_integer([:positive])}"
  end
end