prop/lib/microwaveprop/canopy.ex
Graham McInitre 4ac606e682 fix: ensure all unused historical NFS data gets cleaned up
Close 6 cleanup gaps on the shared /data NFS mount:

- HRDPS .hrdps.prop score files: parse_valid_time regex now matches
  the .hrdps.prop extension, so these files are found and pruned
- HRDPS scalar dirs (weather_scalars/{iso}.hrdps/): strip .hrdps suffix
  before DateTime.from_iso8601 in list_valid_time_dirs
- ScalarFile: prune_older_than was defined but never called from
  Propagation.prune_old_scores - now called alongside Scores/Profiles
- Orphan .tmp.* files: sweep all three base dirs (scores/profiles/scalars)
  for .tmp.* files left by crashed atomic-write processes
- Building cache (/data/buildings/): add MsFootprints.prune_older_than
  with 30-day mtime cutoff
- Canopy cache (/data/canopy/ + staging/): add Canopy.prune_older_than
  with 30-day cutoff, cleanup empty staging dir

All cleanup is cutoff-based (older than X time) so if the prune worker
is missed for any period it catches up fully on the next run.

PropagationPruneWorker updated to call all new cleanup functions.
2026-07-20 09:27:36 -05:00

157 lines
4.6 KiB
Elixir
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

defmodule Microwaveprop.Canopy do
@moduledoc """
Tree-canopy heights from a 30 m global canopy-height grid (Potapov et al.
2019, Lang et al. 2022, or any source able to produce 1° × 1° uint8
per-pixel rasters).
Tile format: `@samples × @samples` bytes, row 0 at the north edge,
one byte per pixel = canopy height in meters (0255). Filename
`N{lat}W{lon}.canopy` mirrors the SRTM `.hgt` convention.
Returns 0 m for points without a tile on disk so the lookup can be
threaded through the path-clearance pipeline without raising on
un-fetched areas (treat unknown as treeless).
"""
@samples 3600
@spec tile_filename(float(), float()) :: String.t()
def tile_filename(lat, lon) do
lat_floor = floor(lat)
lon_floor = floor(lon)
lat_prefix = if lat_floor >= 0, do: "N", else: "S"
lon_prefix = if lon_floor >= 0, do: "E", else: "W"
lat_str = lat_floor |> abs() |> Integer.to_string() |> String.pad_leading(2, "0")
lon_str = lon_floor |> abs() |> Integer.to_string() |> String.pad_leading(3, "0")
"#{lat_prefix}#{lat_str}#{lon_prefix}#{lon_str}.canopy"
end
@doc """
Returns the canopy height in meters at `lat, lon`, or 0 if no tile is
on disk for that 1° square.
"""
@spec lookup(float(), float(), String.t(), keyword()) :: non_neg_integer()
def lookup(lat, lon, tiles_dir \\ default_dir(), opts \\ []) do
samples = Keyword.get(opts, :samples, @samples)
path = Path.join(tiles_dir, tile_filename(lat, lon))
case :file.open(path, [:read, :binary, :raw]) do
{:ok, fd} ->
result = read_height(fd, lat, lon, samples)
_ = :file.close(fd)
result
{:error, _} ->
0
end
end
@doc """
Batch lookup. Groups points by tile so each tile is opened once.
Returns `%{ {lat, lon} => height_m }`. Missing tiles yield 0.
"""
@spec lookup_many([{float(), float()}], String.t(), keyword()) ::
%{{float(), float()} => non_neg_integer()}
def lookup_many(points, tiles_dir \\ default_dir(), opts \\ []) do
samples = Keyword.get(opts, :samples, @samples)
points
|> Enum.group_by(fn {lat, lon} -> {floor(lat), floor(lon)} end)
|> Enum.flat_map(fn {_tile_key, pts} -> read_tile_points(pts, tiles_dir, samples) end)
|> Map.new()
end
defp read_tile_points([{lat0, lon0} | _] = pts, tiles_dir, samples) do
path = Path.join(tiles_dir, tile_filename(lat0, lon0))
case :file.open(path, [:read, :binary, :raw]) do
{:ok, fd} ->
result =
Enum.map(pts, fn {lat, lon} = pt -> {pt, read_height(fd, lat, lon, samples)} end)
_ = :file.close(fd)
result
{:error, _} ->
Enum.map(pts, fn pt -> {pt, 0} end)
end
end
defp read_height(fd, lat, lon, samples) do
row = round((floor(lat) + 1 - lat) * (samples - 1))
col = round((lon - floor(lon)) * (samples - 1))
offset = row * samples + col
case :file.pread(fd, offset, 1) do
{:ok, <<h::unsigned-integer-size(8)>>} -> h
_ -> 0
end
end
defp default_dir do
Application.get_env(:microwaveprop, :canopy_tiles_dir, "/data/canopy")
end
@doc """
Delete canopy tile cache files older than `cutoff` (by mtime).
Cleans up both tile cache and staging COG files.
Returns the number of files deleted.
"""
@spec prune_older_than(DateTime.t()) :: number()
def prune_older_than(%DateTime{} = cutoff) do
cutoff_unix = DateTime.to_unix(cutoff)
canopy_dir = Application.get_env(:microwaveprop, :canopy_tiles_dir, "/data/canopy")
staging_dir = Application.get_env(:microwaveprop, :canopy_staging_dir, "/data/canopy/staging")
tiles_deleted = prune_dir(canopy_dir, cutoff_unix)
staging_deleted = prune_dir(staging_dir, cutoff_unix)
_ = maybe_rmdir_empty(staging_dir, staging_deleted)
tiles_deleted + staging_deleted
end
defp prune_dir(dir, cutoff_unix) do
case File.ls(dir) do
{:ok, files} -> Enum.reduce(files, 0, &prune_file(dir, &1, &2, cutoff_unix))
_ -> 0
end
end
defp prune_file(dir, file, acc, cutoff_unix) do
path = Path.join(dir, file)
case File.stat(path) do
{:ok, stat} ->
mtime_unix =
stat.mtime |> NaiveDateTime.from_erl!() |> DateTime.from_naive!("Etc/UTC") |> DateTime.to_unix()
if mtime_unix < cutoff_unix do
_ = do_delete(path)
acc + 1
else
acc
end
_ ->
acc
end
end
defp do_delete(path) do
_ = if File.dir?(path), do: File.rm_rf(path), else: File.rm(path)
end
defp maybe_rmdir_empty(_dir, 0), do: :ok
defp maybe_rmdir_empty(dir, _deleted) do
case File.ls(dir) do
{:ok, []} -> File.rmdir(dir)
_ -> :ok
end
end
end