Graham McIntire
690a3523e2
First step of the disk-backed scores migration from the DuckDB plan doc. Ended up shipping the raw-binary variant instead of Parquet because the data is disposable after ~2h — the ecosystem benefits of Parquet only pay off for long-lived datasets, and the binary path has zero new dependencies. Microwaveprop.Propagation.ScoresFile writes one file per (band_mhz, valid_time) tuple under the configured scores dir, default /data/scores in prod and priv/dev_scores in dev. Layout is a 33-byte header plus a dense n_rows × n_cols uint8 array (255 is the no-data sentinel). The whole CONUS grid serializes to ~93 KB per band, and writes use the temp-then-rename pattern so NFSv4 concurrent readers never see a partial file. Propagation.replace_scores/2 now materializes the score stream once and dual-writes: Postgres on the primary path, then one ScoresFile per band as a best-effort follow-up (any file error is logged but doesn't fail the DB write, so we can verify the file path in prod before cutting readers over). Propagation.prune_old_scores/0 also clears expired score files so the existing 15-minute prune cron covers both storage layers. Dev configuration points at priv/dev_scores/, added to .gitignore. Test configuration points at a per-run tmp directory.
231 lines
7.6 KiB
Elixir
231 lines
7.6 KiB
Elixir
defmodule Microwaveprop.Propagation.ScoresFile do
|
||
@moduledoc """
|
||
Binary on-disk storage for propagation score grids. One file per
|
||
`(band_mhz, valid_time)` tuple, ~93 KB each. Written in the
|
||
forecast-hour hot path as a companion to the Postgres upsert, and
|
||
eventually replacing it.
|
||
|
||
## Layout
|
||
|
||
magic : 4 bytes "NTMS"
|
||
version : 1 byte 0x01
|
||
band_mhz : 4 bytes uint32 little-endian
|
||
valid_time : 8 bytes int64 unix seconds little-endian
|
||
lat_min : 4 bytes float32 little-endian
|
||
lon_min : 4 bytes float32 little-endian
|
||
step_deg : 4 bytes float32 little-endian
|
||
n_rows : 2 bytes uint16 little-endian (lat dimension)
|
||
n_cols : 2 bytes uint16 little-endian (lon dimension)
|
||
scores : n_rows × n_cols bytes (uint8, 0-100 or 255 = no-data)
|
||
|
||
A cell at `(lat, lon)` is at byte offset
|
||
`row * n_cols + col` in the scores body, where
|
||
`row = (lat - lat_min) / step` and `col = (lon - lon_min) / step`
|
||
(both rounded to the nearest integer).
|
||
|
||
## Atomic writes
|
||
|
||
Files are written through `rename(2)`: `path.tmp.<uniq>` → fsync →
|
||
rename to the final path. On the same NFSv4 filesystem the rename
|
||
is atomic, so a concurrent reader either sees the old file, the
|
||
new file, or the absence of either — never a partial file.
|
||
"""
|
||
|
||
alias Microwaveprop.Propagation.Grid
|
||
|
||
require Logger
|
||
|
||
@magic "NTMS"
|
||
@version 1
|
||
@no_data 255
|
||
|
||
@doc "Returns the root directory score files are written to."
|
||
@spec base_dir() :: String.t()
|
||
def base_dir do
|
||
Application.get_env(:microwaveprop, :propagation_scores_dir, "/data/scores")
|
||
end
|
||
|
||
@doc "Returns the full path for a `(band_mhz, valid_time)` score file."
|
||
@spec path_for(non_neg_integer(), DateTime.t()) :: String.t()
|
||
def path_for(band_mhz, %DateTime{} = valid_time) when is_integer(band_mhz) do
|
||
iso = valid_time |> DateTime.truncate(:second) |> DateTime.to_iso8601()
|
||
Path.join([base_dir(), Integer.to_string(band_mhz), "#{iso}.ntms"])
|
||
end
|
||
|
||
@doc """
|
||
Write a score grid to disk for `(band_mhz, valid_time)`. `scores`
|
||
must be a list of `%{lat, lon, score}` maps. Missing cells are
|
||
stored as the no-data sentinel (255).
|
||
"""
|
||
@spec write!(non_neg_integer(), DateTime.t(), [map()]) :: :ok
|
||
def write!(band_mhz, %DateTime{} = valid_time, scores) when is_integer(band_mhz) and is_list(scores) do
|
||
path = path_for(band_mhz, valid_time)
|
||
File.mkdir_p!(Path.dirname(path))
|
||
|
||
binary = encode(band_mhz, valid_time, scores)
|
||
|
||
tmp = path <> ".tmp." <> unique_suffix()
|
||
File.write!(tmp, binary, [:binary])
|
||
File.rename!(tmp, path)
|
||
:ok
|
||
end
|
||
|
||
@doc """
|
||
Read a score grid from disk. Returns:
|
||
|
||
* `{:ok, %{band_mhz, valid_time, lat_min, lon_min, step, n_rows, n_cols, scores}}`
|
||
where `scores` is the raw binary (row-major, uint8 per cell)
|
||
* `{:error, :enoent}` when the file doesn't exist
|
||
* `{:error, :invalid_format}` when the file isn't a score grid
|
||
"""
|
||
@spec read(non_neg_integer(), DateTime.t()) ::
|
||
{:ok, map()} | {:error, :enoent | :invalid_format}
|
||
def read(band_mhz, %DateTime{} = valid_time) when is_integer(band_mhz) do
|
||
case File.read(path_for(band_mhz, valid_time)) do
|
||
{:ok, binary} -> decode(binary)
|
||
{:error, :enoent} -> {:error, :enoent}
|
||
{:error, other} -> {:error, other}
|
||
end
|
||
end
|
||
|
||
@doc """
|
||
Delete every score file whose `valid_time` is strictly before
|
||
`cutoff`. Returns the number of files deleted. Foreign files in the
|
||
scores tree (anything that doesn't parse as `<iso>.ntms`) are left
|
||
alone.
|
||
"""
|
||
@spec prune_older_than(DateTime.t()) :: non_neg_integer()
|
||
def prune_older_than(%DateTime{} = cutoff) do
|
||
cutoff_unix = DateTime.to_unix(cutoff)
|
||
|
||
base_dir()
|
||
|> list_score_files()
|
||
|> Enum.reduce(0, fn {path, valid_time_unix}, acc ->
|
||
if valid_time_unix < cutoff_unix do
|
||
_ = File.rm(path)
|
||
acc + 1
|
||
else
|
||
acc
|
||
end
|
||
end)
|
||
end
|
||
|
||
# ── Encoding ────────────────────────────────────────────────────
|
||
|
||
defp encode(band_mhz, valid_time, scores) do
|
||
%{lat_min: lat_min, lat_max: lat_max, lon_min: lon_min, lon_max: lon_max} = Grid.bounds()
|
||
step = Grid.step()
|
||
n_rows = round((lat_max - lat_min) / step) + 1
|
||
n_cols = round((lon_max - lon_min) / step) + 1
|
||
|
||
lookup =
|
||
Map.new(scores, fn s ->
|
||
{{Float.round(s.lat, 3), Float.round(s.lon, 3)}, s.score}
|
||
end)
|
||
|
||
body = build_body(lookup, lat_min, lon_min, step, n_rows, n_cols)
|
||
valid_time_unix = valid_time |> DateTime.truncate(:second) |> DateTime.to_unix()
|
||
|
||
<<
|
||
@magic::binary,
|
||
@version::unsigned-8,
|
||
band_mhz::unsigned-little-32,
|
||
valid_time_unix::signed-little-64,
|
||
lat_min::float-little-32,
|
||
lon_min::float-little-32,
|
||
step::float-little-32,
|
||
n_rows::unsigned-little-16,
|
||
n_cols::unsigned-little-16,
|
||
body::binary
|
||
>>
|
||
end
|
||
|
||
defp build_body(lookup, lat_min, lon_min, step, n_rows, n_cols) do
|
||
for row <- 0..(n_rows - 1),
|
||
col <- 0..(n_cols - 1),
|
||
into: <<>> do
|
||
lat = Float.round(lat_min + row * step, 3)
|
||
lon = Float.round(lon_min + col * step, 3)
|
||
<<score_byte(Map.get(lookup, {lat, lon}))::unsigned-8>>
|
||
end
|
||
end
|
||
|
||
defp score_byte(nil), do: @no_data
|
||
defp score_byte(s) when is_integer(s) and s >= 0 and s <= 100, do: s
|
||
defp score_byte(s) when is_integer(s) and s > 100, do: 100
|
||
defp score_byte(s) when is_integer(s) and s < 0, do: 0
|
||
defp score_byte(_), do: @no_data
|
||
|
||
# ── Decoding ────────────────────────────────────────────────────
|
||
|
||
defp decode(
|
||
<<@magic, @version::unsigned-8, band_mhz::unsigned-little-32, valid_time_unix::signed-little-64,
|
||
lat_min::float-little-32, lon_min::float-little-32, step::float-little-32, n_rows::unsigned-little-16,
|
||
n_cols::unsigned-little-16, body::binary>>
|
||
) do
|
||
expected = n_rows * n_cols
|
||
|
||
if byte_size(body) == expected do
|
||
case DateTime.from_unix(valid_time_unix) do
|
||
{:ok, valid_time} ->
|
||
{:ok,
|
||
%{
|
||
band_mhz: band_mhz,
|
||
valid_time: valid_time,
|
||
lat_min: lat_min,
|
||
lon_min: lon_min,
|
||
step: step,
|
||
n_rows: n_rows,
|
||
n_cols: n_cols,
|
||
scores: body
|
||
}}
|
||
|
||
_ ->
|
||
{:error, :invalid_format}
|
||
end
|
||
else
|
||
{:error, :invalid_format}
|
||
end
|
||
end
|
||
|
||
defp decode(_), do: {:error, :invalid_format}
|
||
|
||
# ── Prune helpers ───────────────────────────────────────────────
|
||
|
||
defp list_score_files(root) do
|
||
case File.ls(root) do
|
||
{:ok, entries} ->
|
||
for entry <- entries,
|
||
band_path = Path.join(root, entry),
|
||
File.dir?(band_path),
|
||
file <- files_in(band_path),
|
||
valid_time_unix = parse_valid_time(file),
|
||
valid_time_unix != nil do
|
||
{Path.join(band_path, file), valid_time_unix}
|
||
end
|
||
|
||
_ ->
|
||
[]
|
||
end
|
||
end
|
||
|
||
defp files_in(dir) do
|
||
case File.ls(dir) do
|
||
{:ok, entries} -> entries
|
||
_ -> []
|
||
end
|
||
end
|
||
|
||
defp parse_valid_time(filename) do
|
||
with [_, iso] <- Regex.run(~r/^(.+)\.ntms$/, filename),
|
||
{:ok, dt, _} <- DateTime.from_iso8601(iso) do
|
||
DateTime.to_unix(dt)
|
||
else
|
||
_ -> nil
|
||
end
|
||
end
|
||
|
||
defp unique_suffix do
|
||
"#{System.system_time(:nanosecond)}.#{:erlang.unique_integer([:positive])}"
|
||
end
|
||
end
|