defmodule Microwaveprop.Propagation.ScoresFile do @moduledoc """ Binary on-disk storage for propagation score grids. One file per `(band_mhz, valid_time)` tuple, ~93 KB each. Written in the forecast-hour hot path as a companion to the Postgres upsert, and eventually replacing it. ## Layout magic : 4 bytes "NTMS" version : 1 byte 0x01 band_mhz : 4 bytes uint32 little-endian valid_time : 8 bytes int64 unix seconds little-endian lat_min : 4 bytes float32 little-endian lon_min : 4 bytes float32 little-endian step_deg : 4 bytes float32 little-endian n_rows : 2 bytes uint16 little-endian (lat dimension) n_cols : 2 bytes uint16 little-endian (lon dimension) scores : n_rows × n_cols bytes (uint8, 0-100 or 255 = no-data) A cell at `(lat, lon)` is at byte offset `row * n_cols + col` in the scores body, where `row = (lat - lat_min) / step` and `col = (lon - lon_min) / step` (both rounded to the nearest integer). ## Atomic writes Files are written through `rename(2)`: `path.tmp.` → fsync → rename to the final path. On the same NFSv4 filesystem the rename is atomic, so a concurrent reader either sees the old file, the new file, or the absence of either — never a partial file. """ alias Microwaveprop.Propagation.Grid require Logger @magic "NTMS" @version 1 @no_data 255 @doc "Returns the root directory score files are written to." @spec base_dir() :: String.t() def base_dir do Application.get_env(:microwaveprop, :propagation_scores_dir, "/data/scores") end @doc "Returns the full path for a `(band_mhz, valid_time)` score file." @spec path_for(non_neg_integer(), DateTime.t()) :: String.t() def path_for(band_mhz, %DateTime{} = valid_time) when is_integer(band_mhz) do iso = valid_time |> DateTime.truncate(:second) |> DateTime.to_iso8601() Path.join([base_dir(), Integer.to_string(band_mhz), "#{iso}.ntms"]) end @doc """ Write a score grid to disk for `(band_mhz, valid_time)`. `scores` must be a list of `%{lat, lon, score}` maps. Missing cells are stored as the no-data sentinel (255). """ @spec write!(non_neg_integer(), DateTime.t(), [map()]) :: :ok def write!(band_mhz, %DateTime{} = valid_time, scores) when is_integer(band_mhz) and is_list(scores) do path = path_for(band_mhz, valid_time) File.mkdir_p!(Path.dirname(path)) binary = encode(band_mhz, valid_time, scores) tmp = path <> ".tmp." <> unique_suffix() File.write!(tmp, binary, [:binary]) File.rename!(tmp, path) :ok end @doc """ Read a score grid from disk. Returns: * `{:ok, %{band_mhz, valid_time, lat_min, lon_min, step, n_rows, n_cols, scores}}` where `scores` is the raw binary (row-major, uint8 per cell) * `{:error, :enoent}` when the file doesn't exist * `{:error, :invalid_format}` when the file isn't a score grid """ @spec read(non_neg_integer(), DateTime.t()) :: {:ok, map()} | {:error, :enoent | :invalid_format} def read(band_mhz, %DateTime{} = valid_time) when is_integer(band_mhz) do case File.read(path_for(band_mhz, valid_time)) do {:ok, binary} -> decode(binary) {:error, :enoent} -> {:error, :enoent} {:error, other} -> {:error, other} end end @doc """ Delete every score file whose `valid_time` is strictly before `cutoff`. Returns the number of files deleted. Foreign files in the scores tree (anything that doesn't parse as `.ntms`) are left alone. """ @spec prune_older_than(DateTime.t()) :: non_neg_integer() def prune_older_than(%DateTime{} = cutoff) do cutoff_unix = DateTime.to_unix(cutoff) base_dir() |> list_score_files() |> Enum.reduce(0, fn {path, valid_time_unix}, acc -> if valid_time_unix < cutoff_unix do _ = File.rm(path) acc + 1 else acc end end) end # ── Encoding ──────────────────────────────────────────────────── defp encode(band_mhz, valid_time, scores) do %{lat_min: lat_min, lat_max: lat_max, lon_min: lon_min, lon_max: lon_max} = Grid.bounds() step = Grid.step() n_rows = round((lat_max - lat_min) / step) + 1 n_cols = round((lon_max - lon_min) / step) + 1 lookup = Map.new(scores, fn s -> {{Float.round(s.lat, 3), Float.round(s.lon, 3)}, s.score} end) body = build_body(lookup, lat_min, lon_min, step, n_rows, n_cols) valid_time_unix = valid_time |> DateTime.truncate(:second) |> DateTime.to_unix() << @magic::binary, @version::unsigned-8, band_mhz::unsigned-little-32, valid_time_unix::signed-little-64, lat_min::float-little-32, lon_min::float-little-32, step::float-little-32, n_rows::unsigned-little-16, n_cols::unsigned-little-16, body::binary >> end defp build_body(lookup, lat_min, lon_min, step, n_rows, n_cols) do for row <- 0..(n_rows - 1), col <- 0..(n_cols - 1), into: <<>> do lat = Float.round(lat_min + row * step, 3) lon = Float.round(lon_min + col * step, 3) <> end end defp score_byte(nil), do: @no_data defp score_byte(s) when is_integer(s) and s >= 0 and s <= 100, do: s defp score_byte(s) when is_integer(s) and s > 100, do: 100 defp score_byte(s) when is_integer(s) and s < 0, do: 0 defp score_byte(_), do: @no_data # ── Decoding ──────────────────────────────────────────────────── defp decode( <<@magic, @version::unsigned-8, band_mhz::unsigned-little-32, valid_time_unix::signed-little-64, lat_min::float-little-32, lon_min::float-little-32, step::float-little-32, n_rows::unsigned-little-16, n_cols::unsigned-little-16, body::binary>> ) do expected = n_rows * n_cols if byte_size(body) == expected do case DateTime.from_unix(valid_time_unix) do {:ok, valid_time} -> {:ok, %{ band_mhz: band_mhz, valid_time: valid_time, lat_min: lat_min, lon_min: lon_min, step: step, n_rows: n_rows, n_cols: n_cols, scores: body }} _ -> {:error, :invalid_format} end else {:error, :invalid_format} end end defp decode(_), do: {:error, :invalid_format} # ── Prune helpers ─────────────────────────────────────────────── defp list_score_files(root) do case File.ls(root) do {:ok, entries} -> for entry <- entries, band_path = Path.join(root, entry), File.dir?(band_path), file <- files_in(band_path), valid_time_unix = parse_valid_time(file), valid_time_unix != nil do {Path.join(band_path, file), valid_time_unix} end _ -> [] end end defp files_in(dir) do case File.ls(dir) do {:ok, entries} -> entries _ -> [] end end defp parse_valid_time(filename) do with [_, iso] <- Regex.run(~r/^(.+)\.ntms$/, filename), {:ok, dt, _} <- DateTime.from_iso8601(iso) do DateTime.to_unix(dt) else _ -> nil end end defp unique_suffix do "#{System.system_time(:nanosecond)}.#{:erlang.unique_integer([:positive])}" end end