Close 6 cleanup gaps on the shared /data NFS mount:
- HRDPS .hrdps.prop score files: parse_valid_time regex now matches
the .hrdps.prop extension, so these files are found and pruned
- HRDPS scalar dirs (weather_scalars/{iso}.hrdps/): strip .hrdps suffix
before DateTime.from_iso8601 in list_valid_time_dirs
- ScalarFile: prune_older_than was defined but never called from
Propagation.prune_old_scores - now called alongside Scores/Profiles
- Orphan .tmp.* files: sweep all three base dirs (scores/profiles/scalars)
for .tmp.* files left by crashed atomic-write processes
- Building cache (/data/buildings/): add MsFootprints.prune_older_than
with 30-day mtime cutoff
- Canopy cache (/data/canopy/ + staging/): add Canopy.prune_older_than
with 30-day cutoff, cleanup empty staging dir
All cleanup is cutoff-based (older than X time) so if the prune worker
is missed for any period it catches up fully on the next run.
PropagationPruneWorker updated to call all new cleanup functions.
157 lines
4.6 KiB
Elixir
157 lines
4.6 KiB
Elixir
defmodule Microwaveprop.Canopy do
|
||
@moduledoc """
|
||
Tree-canopy heights from a 30 m global canopy-height grid (Potapov et al.
|
||
2019, Lang et al. 2022, or any source able to produce 1° × 1° uint8
|
||
per-pixel rasters).
|
||
|
||
Tile format: `@samples × @samples` bytes, row 0 at the north edge,
|
||
one byte per pixel = canopy height in meters (0–255). Filename
|
||
`N{lat}W{lon}.canopy` mirrors the SRTM `.hgt` convention.
|
||
|
||
Returns 0 m for points without a tile on disk so the lookup can be
|
||
threaded through the path-clearance pipeline without raising on
|
||
un-fetched areas (treat unknown as treeless).
|
||
"""
|
||
|
||
@samples 3600
|
||
|
||
@spec tile_filename(float(), float()) :: String.t()
|
||
def tile_filename(lat, lon) do
|
||
lat_floor = floor(lat)
|
||
lon_floor = floor(lon)
|
||
|
||
lat_prefix = if lat_floor >= 0, do: "N", else: "S"
|
||
lon_prefix = if lon_floor >= 0, do: "E", else: "W"
|
||
|
||
lat_str = lat_floor |> abs() |> Integer.to_string() |> String.pad_leading(2, "0")
|
||
lon_str = lon_floor |> abs() |> Integer.to_string() |> String.pad_leading(3, "0")
|
||
|
||
"#{lat_prefix}#{lat_str}#{lon_prefix}#{lon_str}.canopy"
|
||
end
|
||
|
||
@doc """
|
||
Returns the canopy height in meters at `lat, lon`, or 0 if no tile is
|
||
on disk for that 1° square.
|
||
"""
|
||
@spec lookup(float(), float(), String.t(), keyword()) :: non_neg_integer()
|
||
def lookup(lat, lon, tiles_dir \\ default_dir(), opts \\ []) do
|
||
samples = Keyword.get(opts, :samples, @samples)
|
||
path = Path.join(tiles_dir, tile_filename(lat, lon))
|
||
|
||
case :file.open(path, [:read, :binary, :raw]) do
|
||
{:ok, fd} ->
|
||
result = read_height(fd, lat, lon, samples)
|
||
_ = :file.close(fd)
|
||
result
|
||
|
||
{:error, _} ->
|
||
0
|
||
end
|
||
end
|
||
|
||
@doc """
|
||
Batch lookup. Groups points by tile so each tile is opened once.
|
||
Returns `%{ {lat, lon} => height_m }`. Missing tiles yield 0.
|
||
"""
|
||
@spec lookup_many([{float(), float()}], String.t(), keyword()) ::
|
||
%{{float(), float()} => non_neg_integer()}
|
||
def lookup_many(points, tiles_dir \\ default_dir(), opts \\ []) do
|
||
samples = Keyword.get(opts, :samples, @samples)
|
||
|
||
points
|
||
|> Enum.group_by(fn {lat, lon} -> {floor(lat), floor(lon)} end)
|
||
|> Enum.flat_map(fn {_tile_key, pts} -> read_tile_points(pts, tiles_dir, samples) end)
|
||
|> Map.new()
|
||
end
|
||
|
||
defp read_tile_points([{lat0, lon0} | _] = pts, tiles_dir, samples) do
|
||
path = Path.join(tiles_dir, tile_filename(lat0, lon0))
|
||
|
||
case :file.open(path, [:read, :binary, :raw]) do
|
||
{:ok, fd} ->
|
||
result =
|
||
Enum.map(pts, fn {lat, lon} = pt -> {pt, read_height(fd, lat, lon, samples)} end)
|
||
|
||
_ = :file.close(fd)
|
||
result
|
||
|
||
{:error, _} ->
|
||
Enum.map(pts, fn pt -> {pt, 0} end)
|
||
end
|
||
end
|
||
|
||
defp read_height(fd, lat, lon, samples) do
|
||
row = round((floor(lat) + 1 - lat) * (samples - 1))
|
||
col = round((lon - floor(lon)) * (samples - 1))
|
||
offset = row * samples + col
|
||
|
||
case :file.pread(fd, offset, 1) do
|
||
{:ok, <<h::unsigned-integer-size(8)>>} -> h
|
||
_ -> 0
|
||
end
|
||
end
|
||
|
||
defp default_dir do
|
||
Application.get_env(:microwaveprop, :canopy_tiles_dir, "/data/canopy")
|
||
end
|
||
|
||
@doc """
|
||
Delete canopy tile cache files older than `cutoff` (by mtime).
|
||
Cleans up both tile cache and staging COG files.
|
||
Returns the number of files deleted.
|
||
"""
|
||
@spec prune_older_than(DateTime.t()) :: number()
|
||
def prune_older_than(%DateTime{} = cutoff) do
|
||
cutoff_unix = DateTime.to_unix(cutoff)
|
||
|
||
canopy_dir = Application.get_env(:microwaveprop, :canopy_tiles_dir, "/data/canopy")
|
||
staging_dir = Application.get_env(:microwaveprop, :canopy_staging_dir, "/data/canopy/staging")
|
||
|
||
tiles_deleted = prune_dir(canopy_dir, cutoff_unix)
|
||
staging_deleted = prune_dir(staging_dir, cutoff_unix)
|
||
|
||
_ = maybe_rmdir_empty(staging_dir, staging_deleted)
|
||
|
||
tiles_deleted + staging_deleted
|
||
end
|
||
|
||
defp prune_dir(dir, cutoff_unix) do
|
||
case File.ls(dir) do
|
||
{:ok, files} -> Enum.reduce(files, 0, &prune_file(dir, &1, &2, cutoff_unix))
|
||
_ -> 0
|
||
end
|
||
end
|
||
|
||
defp prune_file(dir, file, acc, cutoff_unix) do
|
||
path = Path.join(dir, file)
|
||
|
||
case File.stat(path) do
|
||
{:ok, stat} ->
|
||
mtime_unix =
|
||
stat.mtime |> NaiveDateTime.from_erl!() |> DateTime.from_naive!("Etc/UTC") |> DateTime.to_unix()
|
||
|
||
if mtime_unix < cutoff_unix do
|
||
_ = do_delete(path)
|
||
acc + 1
|
||
else
|
||
acc
|
||
end
|
||
|
||
_ ->
|
||
acc
|
||
end
|
||
end
|
||
|
||
defp do_delete(path) do
|
||
_ = if File.dir?(path), do: File.rm_rf(path), else: File.rm(path)
|
||
end
|
||
|
||
defp maybe_rmdir_empty(_dir, 0), do: :ok
|
||
|
||
defp maybe_rmdir_empty(dir, _deleted) do
|
||
case File.ls(dir) do
|
||
{:ok, []} -> File.rmdir(dir)
|
||
_ -> :ok
|
||
end
|
||
end
|
||
end
|