graham/towerops
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: add Texas LIDAR DEM catalog with on-demand elevation reads

Browse source 
Catalog-only system for LIDAR-derived 1m DEMs covering Texas. We do not
mirror raster data — the DB stores tile metadata (URL + footprint
geometry) and elevation values are streamed on-demand from public USGS
3DEP / TNRIS Cloud-Optimized GeoTIFFs via GDAL /vsicurl/ HTTP byte-range
reads. Used for future tower coverage line-of-sight calculations.

- Enable PostGIS extension; tile footprints indexed via GiST
- Catalog sync from USGS 3DEP (primary) and TNRIS (fallback for gaps)
- Tile dedup by ST_Contains so TNRIS doesn't duplicate 3DEP coverage
- Single-point reads via gdallocationinfo with tile fallthrough on nodata
- Grid retrieval via gdal_translate AAIGrid with multi-tile mosaicing
- Monthly Oban cron worker keeps catalog fresh
- gdal-bin added to runtime image; geo_postgis dep added
This commit is contained in:
Graham McIntire 2026-05-04 13:03:28 -05:00
parent 20c464137f
commit 158ee9d323
28 changed files with 2659 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
CLAUDE.md
View file

@ -192,6 +192,26 @@ Pure C NIF calling libnetsnmp directly for fast in-process MIB resolution
Note: SnmpKit still used for SNMP protocol operations (get, walk, etc.).
### LIDAR Elevation Catalog
Catalog-only system for accessing LIDAR-derived DEMs covering Texas. We do
not mirror raster data — the DB stores tile metadata (URL + footprint
geometry) and elevation values are streamed on-demand from public USGS
3DEP / TNRIS COGs via GDAL `/vsicurl/` HTTP byte-range reads.
- **Catalog**: `lib/towerops/lidar/catalog.ex`, schemas in `lib/towerops/lidar/`
- **Sources**: `lib/towerops/lidar/sources/three_dep.ex` (primary),
`lib/towerops/lidar/sources/tnris.ex` (fallback for areas not in 3DEP)
- **Reader**: `lib/towerops/lidar/reader.ex` shells to `gdallocationinfo` /
`gdal_translate` against `/vsicurl/<url>` — only fetches the bytes needed.
- **Sync**: `Towerops.Workers.LidarCatalogSyncWorker` runs monthly via Oban cron
- **Dependencies**: `brew install gdal postgis` (macOS), `apt-get install
gdal-bin postgis` (Docker — `gdal-bin` is in `k8s/Dockerfile`)
- **PostGIS**: enabled by migration `20260504123340_enable_postgis`, used for
spatial indexes on tile footprints (GiST) and `ST_Contains` dedup checks
- **Geometry types**: registered via `lib/towerops/postgrex_types.ex`, wired
into the Repo via `config :towerops, Towerops.Repo, types: ...`
## Admin Features
### User Impersonation

3
config/config.exs
View file

@ -109,7 +109,8 @@ config :towerops, Towerops.Mailer, adapter: Swoosh.Adapters.Local
# This allows CI to load the schema instantly instead of running 172+ migrations
config :towerops, Towerops.Repo,
dump_path: "priv/repo/structure.sql",
migration_timestamps: [type: :naive_datetime_usec]
migration_timestamps: [type: :naive_datetime_usec],
types: Towerops.PostgrexTypes
# Configure the endpoint
config :towerops, ToweropsWeb.Endpoint,

7
config/runtime.exs
View file

@ -187,7 +187,8 @@ if config_env() == :prod do
# Alert notifications (PagerDuty, email, etc.)
notifications: max(1, div(25, oban_scale)),
maintenance: max(1, div(5, oban_scale)),
weather: max(1, div(2, oban_scale))
weather: max(1, div(2, oban_scale)),
lidar: max(1, div(2, oban_scale))
],
plugins: [
# Cron jobs for periodic maintenance tasks
@ -254,7 +255,9 @@ if config_env() == :prod do
# Sync device usage to Stripe daily at 3 AM UTC
{"0 3 * * *", Towerops.Workers.BillingSyncWorker},
# Data retention cleanup nightly at 1 AM
{"0 1 * * *", Towerops.Workers.DataRetentionWorker}
{"0 1 * * *", Towerops.Workers.DataRetentionWorker},
# Refresh LIDAR tile catalog from public 3DEP/TNRIS sources monthly
{"0 6 1 * *", Towerops.Workers.LidarCatalogSyncWorker}
]},
# Automatically delete completed jobs after 60 seconds.
# Default limit of 10k/30s (333/s) can't keep up with job throughput (~600+/s),

2
k8s/Dockerfile
View file

@ -82,7 +82,7 @@ FROM ${RUNNER_IMAGE} AS final
RUN apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends libstdc++6 openssl libncurses6 locales ca-certificates iputils-ping snmp libsnmp40 \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends libstdc++6 openssl libncurses6 locales ca-certificates iputils-ping snmp libsnmp40 gdal-bin \
&& rm -rf /var/lib/apt/lists/*
# Set the locale

162
lib/towerops/lidar.ex Normal file
View file

@ -0,0 +1,162 @@
defmodule Towerops.Lidar do
@moduledoc """
Catalog and on-demand retrieval of LIDAR-derived DEMs covering Texas.
We do not mirror raster data. The catalog stores tile metadata and the
public HTTPS URL of each Cloud-Optimized GeoTIFF (USGS 3DEP, with TNRIS
as fallback). Elevation reads stream byte ranges directly from those
public buckets via GDAL's `/vsicurl/` driver.
"""
import Ecto.Query
alias Towerops.Lidar.Reader
alias Towerops.Lidar.Tile
alias Towerops.Repo
@typedoc "{west, south, east, north} in WGS84 degrees"
@type bbox :: {number(), number(), number(), number()}
@doc """
Lists catalog tiles whose footprint contains the given point, ordered
best-first (highest resolution, then newest year).
Filters out tiles flagged as unavailable.
"""
@spec list_tiles_for_point(number(), number()) :: [Tile.t()]
def list_tiles_for_point(lat, lon) do
point = %Geo.Point{coordinates: {lon, lat}, srid: 4326}
Tile
|> where([t], t.availability == "available")
|> where([t], fragment("ST_Intersects(?, ?)", t.footprint, ^point))
|> order_by([t], asc: t.resolution_m, desc: t.year)
|> Repo.all()
end
@doc """
Lists catalog tiles whose footprint intersects the given bbox, ordered
best-first (highest resolution, then newest year).
"""
@spec list_tiles_for_area(bbox()) :: [Tile.t()]
def list_tiles_for_area({west, south, east, north}) do
envelope =
%Geo.Polygon{
coordinates: [
[
{west, south},
{east, south},
{east, north},
{west, north},
{west, south}
]
],
srid: 4326
}
Tile
|> where([t], t.availability == "available")
|> where([t], fragment("ST_Intersects(?, ?)", t.footprint, ^envelope))
|> order_by([t], asc: t.resolution_m, desc: t.year)
|> Repo.all()
end
@doc """
Returns the elevation in meters at the given (lat, lon).
Picks the best (highest-resolution, newest) tile that covers the point.
Falls through to the next-best tile if the chosen one returns nodata.
Returns `{:error, :no_tile}` if no catalog tile covers the point, or
`{:error, :nodata}` if every covering tile lacks data at that pixel.
"""
@spec get_elevation(number(), number()) :: {:ok, float()} | {:error, term()}
def get_elevation(lat, lon) do
case list_tiles_for_point(lat, lon) do
[] -> {:error, :no_tile}
tiles -> read_first_value(tiles, lat, lon)
end
end
defp read_first_value([], _lat, _lon), do: {:error, :nodata}
defp read_first_value([tile | rest], lat, lon) do
case Reader.elevation_at(tile, lat, lon) do
{:ok, value} -> {:ok, value}
{:error, :nodata} -> read_first_value(rest, lat, lon)
{:error, _} = err when rest == [] -> err
{:error, _} -> read_first_value(rest, lat, lon)
end
end
@max_grid_cells 4_000_000
@doc """
Returns a 2D elevation grid covering the bbox at the given cell size
(in degrees, EPSG:4326).
Mosaics across all intersecting catalog tiles, filling nodata cells from
next-best tiles. Caps at #{@max_grid_cells} cells to bound memory.
"""
@spec get_elevation_grid(bbox(), number()) ::
{:ok, map()} | {:error, term()}
def get_elevation_grid({west, south, east, north} = bbox, cell_size_deg) do
ncols = ceil((east - west) / cell_size_deg)
nrows = ceil((north - south) / cell_size_deg)
if ncols * nrows > @max_grid_cells do
{:error, :grid_too_large}
else
case list_tiles_for_area(bbox) do
[] -> {:error, :no_tile}
tiles -> mosaic(tiles, bbox, cell_size_deg)
end
end
end
defp mosaic(tiles, bbox, cell_size_deg) do
{grids, last_error} =
Enum.reduce(tiles, {[], nil}, fn tile, {acc, err} ->
case Reader.elevation_grid(tile, bbox, cell_size_deg) do
{:ok, grid} -> {[grid | acc], err}
{:error, reason} -> {acc, reason}
end
end)
case Enum.reverse(grids) do
[] -> {:error, last_error || :no_data}
[first | rest] -> finalize_mosaic(first, rest)
end
end
defp finalize_mosaic(base, additional) do
nodata = base.nodata_value
merged_cells =
Enum.reduce(additional, base.cells, fn next, current ->
merge_grid_cells(current, next.cells, nodata, next.nodata_value)
end)
if all_nodata?(merged_cells, nodata) do
{:error, :nodata}
else
{:ok, %{base | cells: merged_cells}}
end
end
defp merge_grid_cells(current_rows, next_rows, base_nodata, next_nodata) do
current_rows
|> Enum.zip(next_rows)
|> Enum.map(fn {current_row, next_row} ->
current_row
|> Enum.zip(next_row)
|> Enum.map(fn
{cur, next} when cur == base_nodata and next != next_nodata -> next
{cur, _next} -> cur
end)
end)
end
defp all_nodata?(rows, nodata) do
Enum.all?(rows, fn row -> Enum.all?(row, &(&1 == nodata)) end)
end
end

278
lib/towerops/lidar/catalog.ex Normal file
View file

@ -0,0 +1,278 @@
defmodule Towerops.Lidar.Catalog do
@moduledoc """
Orchestrates LIDAR tile catalog discovery + verification.
Sync runs do NOT download raster data — they only enumerate which tiles
exist publicly and store HTTPS URLs + footprint metadata in our DB.
"""
import Ecto.Query
alias Towerops.HTTP
alias Towerops.Lidar.Sources.ThreeDep
alias Towerops.Lidar.Sources.Tnris
alias Towerops.Lidar.SyncLog
alias Towerops.Lidar.Tile
alias Towerops.Repo
require Logger
@batch_size 5_000
@type sync_summary :: %{
projects_synced: non_neg_integer(),
tiles_upserted: non_neg_integer(),
errors: map(),
duration_ms: integer()
}
@doc """
Syncs the 3DEP catalog: lists Texas-prefixed projects, enumerates tiles
per project, batches upserts into `lidar_tiles`, and records a `SyncLog`.
Returns `{:ok, summary}` on success/partial, `{:error, reason}` on total
failure.
"""
@spec sync_3dep() :: {:ok, sync_summary()} | {:error, term()}
def sync_3dep do
started_at = System.monotonic_time(:millisecond)
case list_projects() do
{:ok, projects} ->
run_sync(projects, started_at)
{:error, reason} ->
record_failed_log(started_at)
{:error, reason}
end
end
defp list_projects do
case Application.get_env(:towerops, :lidar_3dep_lister) do
nil -> ThreeDep.list_texas_projects()
fun when is_function(fun, 0) -> fun.()
end
end
defp run_sync(projects, started_at) do
{tiles_upserted, errors} =
Enum.reduce(projects, {0, %{}}, fn project, {count, errs} ->
case ThreeDep.enumerate_tiles(project) do
{:ok, tiles} ->
inserted = upsert_tiles(project, tiles)
{count + inserted, errs}
{:error, reason} ->
Logger.warning("Lidar sync: enumerate_tiles failed for #{project}: #{inspect(reason)}")
{count, Map.put(errs, project, inspect(reason))}
end
end)
duration_ms = System.monotonic_time(:millisecond) - started_at
summary = %{
projects_synced: length(projects) - map_size(errors),
tiles_upserted: tiles_upserted,
errors: errors,
duration_ms: duration_ms
}
status = if map_size(errors) == 0, do: "success", else: "partial"
record_log(status, summary)
{:ok, summary}
end
defp upsert_tiles(project, tiles) when is_list(tiles) do
now = DateTime.truncate(DateTime.utc_now(), :second)
rows =
Enum.map(tiles, fn t ->
%{
id: Ecto.UUID.generate(),
source: "3dep",
project_name: project,
tile_name: t.tile_name,
year: t.year,
resolution_m: t.resolution_m,
url: t.url,
crs: t.crs,
file_size_bytes: t.file_size_bytes,
footprint: wkt_to_geometry(t.footprint_wkt),
metadata: %{},
availability: "available",
inserted_at: now,
updated_at: now
}
end)
rows
|> Enum.chunk_every(@batch_size)
|> Enum.reduce(0, fn batch, acc ->
{count, _} =
Repo.insert_all(Tile, batch,
on_conflict: {:replace_all_except, [:id, :inserted_at]},
conflict_target: [:source, :project_name, :tile_name]
)
acc + count
end)
end
defp wkt_to_geometry(wkt) when is_binary(wkt) do
geo = Geo.WKT.decode!(wkt)
%{geo | srid: 4326}
end
@doc """
Syncs the TNRIS catalog as a fallback. Skips any tile whose footprint
centroid is already covered by a 3DEP tile.
Records a `SyncLog` with `source: "tnris"`.
"""
@spec sync_tnris() :: {:ok, sync_summary()} | {:error, term()}
def sync_tnris do
started_at = System.monotonic_time(:millisecond)
case Tnris.list_dem_resources() do
{:ok, resources} ->
{kept, skipped} =
Enum.split_with(resources, fn r -> not covered_by_3dep?(r.footprint_wkt) end)
upserted =
if kept == [] do
0
else
upsert_tnris_tiles(kept)
end
duration_ms = System.monotonic_time(:millisecond) - started_at
summary = %{
projects_synced: 1,
tiles_upserted: upserted,
errors: %{"skipped_covered" => length(skipped)},
duration_ms: duration_ms
}
record_log("tnris", "success", summary)
{:ok, summary}
{:error, reason} ->
record_log("tnris", "failed", %{
projects_synced: 0,
tiles_upserted: 0,
errors: %{"list" => inspect(reason)},
duration_ms: System.monotonic_time(:millisecond) - started_at
})
{:error, reason}
end
end
defp covered_by_3dep?(wkt) when is_binary(wkt) do
geom = wkt_to_geometry(wkt)
query =
from t in Tile,
where: t.source == "3dep",
where:
fragment(
"ST_Contains(?, ST_Centroid(?))",
t.footprint,
type(^geom, Geo.PostGIS.Geometry)
),
select: count(t.id)
Repo.one(query) > 0
end
defp upsert_tnris_tiles(resources) do
now = DateTime.truncate(DateTime.utc_now(), :second)
rows =
Enum.map(resources, fn r ->
%{
id: Ecto.UUID.generate(),
source: "tnris",
project_name: "tnris",
tile_name: r.tile_name,
year: r.year,
resolution_m: r.resolution_m,
url: r.url,
crs: r.crs,
file_size_bytes: r.file_size_bytes,
footprint: wkt_to_geometry(r.footprint_wkt),
metadata: %{},
availability: "available",
inserted_at: now,
updated_at: now
}
end)
rows
|> Enum.chunk_every(@batch_size)
|> Enum.reduce(0, fn batch, acc ->
{count, _} =
Repo.insert_all(Tile, batch,
on_conflict: {:replace_all_except, [:id, :inserted_at]},
conflict_target: [:source, :project_name, :tile_name]
)
acc + count
end)
end
@doc """
HEAD-checks a tile's URL and updates `last_verified_at` + `availability`.
"""
@spec verify_tile(Tile.t()) :: {:ok, Tile.t()} | {:error, term()}
def verify_tile(%Tile{} = tile) do
availability =
case HTTP.request(__MODULE__, method: :head, url: tile.url) do
{:ok, %Req.Response{status: 200}} -> "available"
{:ok, %Req.Response{status: 404}} -> "missing"
{:ok, %Req.Response{}} -> "error"
{:error, _} -> "error"
end
tile
|> Tile.changeset(%{
availability: availability,
last_verified_at: DateTime.truncate(DateTime.utc_now(), :second)
})
|> Repo.update()
end
defp record_log(status, summary), do: record_log("3dep", status, summary)
defp record_log(source, status, summary) do
%SyncLog{}
|> SyncLog.changeset(%{
source: source,
status: status,
projects_synced: summary.projects_synced,
tiles_upserted: summary.tiles_upserted,
errors: summary.errors,
duration_ms: summary.duration_ms,
inserted_at: DateTime.truncate(DateTime.utc_now(), :second)
})
|> Repo.insert()
end
defp record_failed_log(started_at) do
duration_ms = System.monotonic_time(:millisecond) - started_at
%SyncLog{}
|> SyncLog.changeset(%{
source: "3dep",
status: "failed",
projects_synced: 0,
tiles_upserted: 0,
duration_ms: duration_ms,
inserted_at: DateTime.truncate(DateTime.utc_now(), :second)
})
|> Repo.insert()
end
end

214
lib/towerops/lidar/reader.ex Normal file
View file

@ -0,0 +1,214 @@
defmodule Towerops.Lidar.Reader do
@moduledoc """
Reads elevation values from public Cloud-Optimized GeoTIFFs via GDAL.
Uses `gdallocationinfo` with `/vsicurl/` to perform HTTP byte-range
reads — only the bytes needed for a single pixel are fetched. Tiles
are never downloaded in full and never persisted on our side.
"""
alias Towerops.Lidar.Tile
require Logger
@gdal_bin "gdallocationinfo"
# GDAL emits this sentinel for nodata in -valonly mode
@nodata_sentinels ["-3.4028235e+38", "-3.402823466e+38", "-9999"]
@doc """
Returns `{:ok, elevation_m}` for the given (lat, lon) within the tile,
or `{:error, :nodata}` if no value at that point.
"""
@spec elevation_at(Tile.t(), float(), float()) ::
{:ok, float()} | {:error, :nodata | term()}
def elevation_at(%Tile{url: url}, lat, lon) when is_binary(url) and is_number(lat) and is_number(lon) do
args = [
"-wgs84",
"-valonly",
vsicurl(url),
to_string(lon),
to_string(lat)
]
case run(@gdal_bin, args) do
{output, 0} -> parse_value(output)
{output, code} -> {:error, {:gdal, code, output}}
end
end
@doc """
Verifies GDAL is callable. Intended to be invoked at boot in non-test envs.
"""
@spec healthcheck() :: :ok | {:error, :gdal_missing}
def healthcheck do
case run(@gdal_bin, ["--version"]) do
{_output, 0} -> :ok
_ -> {:error, :gdal_missing}
end
end
@doc """
Reads an elevation grid for the given bbox at the requested cell size
(degrees), reprojecting to EPSG:4326 on the fly. Returns a parsed AAIGrid.
Returns `{:ok, grid}` or `{:error, reason}`. If the underlying COG has no
data over the bbox, the cells field will be all-nodata; the caller is
responsible for treating that as `:nodata` if every tile produces nodata.
"""
@spec elevation_grid(Tile.t(), {number(), number(), number(), number()}, number()) ::
{:ok, map()} | {:error, term()}
def elevation_grid(%Tile{url: url}, {west, south, east, north}, cell_size_deg) do
args = [
"-of",
"AAIGrid",
"-projwin_srs",
"EPSG:4326",
"-projwin",
to_string(west),
to_string(north),
to_string(east),
to_string(south),
"-tr",
to_string(cell_size_deg),
to_string(cell_size_deg),
"-t_srs",
"EPSG:4326",
vsicurl(url),
"/vsistdout/"
]
case run("gdal_translate", args) do
{output, 0} -> parse_aaigrid(output)
{output, code} -> {:error, {:gdal, code, output}}
end
end
@doc """
Parses an Esri ASCII Grid (AAIGrid) text into a structured map.
Public for testability.
"""
@spec parse_aaigrid(String.t()) :: {:ok, map()} | {:error, term()}
def parse_aaigrid(text) when is_binary(text) do
{header, body_lines} = split_header(text)
with {:ok, ncols} <- header_int(header, "ncols"),
{:ok, nrows} <- header_int(header, "nrows"),
{:ok, xll} <- header_float(header, "xllcorner"),
{:ok, yll} <- header_float(header, "yllcorner"),
{:ok, cellsize} <- header_float(header, "cellsize"),
{:ok, nodata} <- header_nodata(header) do
cells =
body_lines
|> Enum.take(nrows)
|> Enum.map(&parse_row/1)
{:ok,
%{
ncols: ncols,
nrows: nrows,
xllcorner: xll,
yllcorner: yll,
cellsize: cellsize,
nodata_value: nodata,
cells: cells
}}
end
end
defp split_header(text) do
lines = text |> String.split(~r/\r?\n/, trim: true) |> Enum.map(&String.trim/1)
{header, body} = Enum.split_while(lines, &header_line?/1)
{Map.new(header, &parse_header_line/1), body}
end
defp header_line?(line) do
case Regex.run(~r/^([A-Za-z_]+)\s+/, line) do
[_, key] ->
String.downcase(key) in ~w(ncols nrows xllcorner yllcorner xllcenter yllcenter cellsize dx dy nodata_value)
_ ->
false
end
end
defp parse_header_line(line) do
[k, v] = String.split(line, ~r/\s+/, parts: 2)
{String.downcase(k), v}
end
defp header_int(header, key) do
case Map.fetch(header, key) do
{:ok, val} ->
case Integer.parse(val) do
{n, _} -> {:ok, n}
_ -> {:error, {:bad_header, key}}
end
:error ->
{:error, {:missing_header, key}}
end
end
defp header_float(header, key) do
case Map.fetch(header, key) do
{:ok, val} ->
case Float.parse(val) do
{n, _} -> {:ok, n}
_ -> {:error, {:bad_header, key}}
end
:error ->
{:error, {:missing_header, key}}
end
end
defp header_nodata(header) do
case Map.get(header, "nodata_value") do
nil ->
{:ok, -9999.0}
val ->
case Float.parse(val) do
{n, _} -> {:ok, n}
_ -> {:error, {:bad_header, "nodata_value"}}
end
end
end
defp parse_row(line) do
line
|> String.split(~r/\s+/, trim: true)
|> Enum.map(fn token ->
case Float.parse(token) do
{n, _} -> n
:error -> nil
end
end)
end
defp vsicurl("/vsicurl/" <> _rest = path), do: path
defp vsicurl(url), do: "/vsicurl/#{url}"
defp parse_value(output) do
trimmed = String.trim(output)
cond do
trimmed == "" ->
{:error, :nodata}
Enum.any?(@nodata_sentinels, &String.starts_with?(trimmed, &1)) ->
{:error, :nodata}
true ->
case Float.parse(trimmed) do
{value, _rest} -> {:ok, value}
:error -> {:error, :nodata}
end
end
end
defp run(cmd, args, opts \\ [stderr_to_stdout: true]) do
runner = Application.get_env(:towerops, :lidar_runner, &System.cmd/3)
runner.(cmd, args, opts)
end
end

104
lib/towerops/lidar/sources/three_dep.ex Normal file
View file

@ -0,0 +1,104 @@
defmodule Towerops.Lidar.Sources.ThreeDep do
@moduledoc """
USGS 3DEP catalog discovery.
Lists Texas-prefixed LIDAR projects on the public `prd-tnm` S3 bucket and
enumerates per-project tiles via an injectable manifest parser.
"""
import SweetXml
alias Towerops.HTTP
@s3_host "https://prd-tnm.s3.amazonaws.com"
@projects_prefix "StagedProducts/Elevation/1m/Projects/"
@texas_prefix "TX_"
@type project_name :: String.t()
@type tile :: %{
tile_name: String.t(),
url: String.t(),
footprint_wkt: String.t(),
resolution_m: Decimal.t(),
year: integer() | nil,
crs: String.t() | nil,
file_size_bytes: integer() | nil
}
@doc """
Lists Texas-prefixed project names by paginating through S3 ListObjectsV2.
"""
@spec list_texas_projects() :: {:ok, [project_name()]} | {:error, term()}
def list_texas_projects do
do_list_projects(nil, [])
end
defp do_list_projects(continuation_token, acc) do
base_query = [
{"list-type", "2"},
{"prefix", @projects_prefix <> @texas_prefix},
{"delimiter", "/"}
]
query =
if continuation_token do
base_query ++ [{"continuation-token", continuation_token}]
else
base_query
end
case HTTP.get(__MODULE__, url: @s3_host <> "/", params: query) do
{:ok, %Req.Response{status: 200, body: body}} ->
{projects, next} = parse_list_response(body)
merged = acc ++ projects
if next do
do_list_projects(next, merged)
else
{:ok, merged}
end
{:ok, %Req.Response{status: status, body: body}} ->
{:error, {:http, status, body}}
{:error, reason} ->
{:error, reason}
end
end
@doc false
# Public for tests: parses the S3 ListObjectsV2 XML response.
def parse_list_response(xml) when is_binary(xml) do
prefixes = xpath(xml, ~x"//CommonPrefixes/Prefix/text()"sl)
truncated = xpath(xml, ~x"//IsTruncated/text()"s)
next_token = xpath(xml, ~x"//NextContinuationToken/text()"s)
projects =
prefixes
|> Enum.map(&strip_project_path/1)
|> Enum.reject(&(&1 == ""))
next = if truncated == "true" and next_token != "", do: next_token
{projects, next}
end
defp strip_project_path(prefix) do
prefix
|> String.replace_prefix(@projects_prefix, "")
|> String.trim_trailing("/")
end
@doc """
Enumerates tiles for a project using the configured parser module.
Tests inject a fake parser via `Application.put_env(:towerops, :lidar_3dep_parser, ...)`.
Production uses `Towerops.Lidar.Sources.ThreeDep.GdalParser` (default).
"""
@spec enumerate_tiles(project_name(), keyword()) :: {:ok, [tile()]} | {:error, term()}
def enumerate_tiles(project_name, opts \\ []) do
parser = Application.get_env(:towerops, :lidar_3dep_parser, __MODULE__.GdalParser)
parser.enumerate(project_name, opts)
end
end

150
lib/towerops/lidar/sources/three_dep/gdal_parser.ex Normal file
View file

@ -0,0 +1,150 @@
defmodule Towerops.Lidar.Sources.ThreeDep.GdalParser do
@moduledoc """
Default 3DEP manifest parser.
Each Texas 3DEP project ships with a tile index — typically a GeoPackage
(`<project>.gpkg`) under the `metadata/` prefix. This parser shells out to
`ogr2ogr` to convert the GeoPackage to GeoJSON in stdout, then maps each
feature to a tile row.
The implementation depends on a runtime `ogr2ogr` binary. Tests inject a
fake parser via `Application.put_env(:towerops, :lidar_3dep_parser, ...)`
to avoid this dependency.
"""
require Logger
@s3_host "https://prd-tnm.s3.amazonaws.com"
@doc """
Returns `{:ok, [tile_map]}` for the project, or `{:error, reason}`.
Tile map shape:
%{
tile_name: String.t(),
url: String.t(),
footprint_wkt: String.t(),
resolution_m: Decimal.t(),
year: integer() | nil,
crs: String.t() | nil,
file_size_bytes: integer() | nil
}
"""
def enumerate(project_name, opts \\ []) do
manifest_url = manifest_url(project_name)
ogr_bin = Keyword.get(opts, :ogr_bin, Application.get_env(:towerops, :ogr2ogr_bin, "ogr2ogr"))
if executable_available?(ogr_bin) do
run_ogr2ogr(ogr_bin, manifest_url, project_name)
else
{:error, :ogr2ogr_not_found}
end
end
defp executable_available?(bin) do
if Application.get_env(:towerops, :lidar_ogr_runner) do
true
else
System.find_executable(bin) != nil
end
end
defp manifest_url(project_name) do
# Convention: metadata GeoPackage lives at metadata/<project>/<project>.gpkg.
# Some older projects use spatial_metadata/ — the parser falls back if the
# primary URL 404s (handled at the catalog layer, not here).
"/vsicurl/#{@s3_host}/StagedProducts/Elevation/metadata/#{project_name}/#{project_name}.gpkg"
end
defp run_ogr2ogr(ogr_bin, manifest_url, project_name) do
args = ["-f", "GeoJSON", "-t_srs", "EPSG:4326", "/vsistdout/", manifest_url]
runner = Application.get_env(:towerops, :lidar_ogr_runner, &System.cmd/3)
case runner.(ogr_bin, args, stderr_to_stdout: true) do
{output, 0} ->
decode_geojson(output, project_name)
{output, code} ->
Logger.warning("ogr2ogr failed (#{code}) for #{project_name}: #{output}")
{:error, {:ogr2ogr, code, output}}
end
end
@doc """
Decodes a GeoJSON FeatureCollection (as text) into tile maps. Public for
testability — exercised directly without invoking `ogr2ogr`.
"""
@spec decode_geojson(String.t(), String.t()) :: {:ok, list(map())} | {:error, term()}
def decode_geojson(json, project_name) do
case Jason.decode(json) do
{:ok, %{"features" => features}} ->
tiles =
features
|> Enum.map(&feature_to_tile(&1, project_name))
|> Enum.reject(&is_nil/1)
{:ok, tiles}
{:ok, _other} ->
{:error, :no_features}
{:error, reason} ->
{:error, {:json_decode, reason}}
end
end
defp feature_to_tile(%{"properties" => props, "geometry" => geom}, project_name) when is_map(props) and is_map(geom) do
tile_name = first_present(props, ["tile_id", "TILE_ID", "filename"])
url = first_present(props, ["download_url", "DOWNLOAD_URL"]) || derive_tile_url(project_name, tile_name)
build_tile(tile_name, url, geom, props, project_name)
end
defp feature_to_tile(_, _), do: nil
defp first_present(_props, []), do: nil
defp first_present(props, [key | rest]) do
case Map.get(props, key) do
nil -> first_present(props, rest)
"" -> first_present(props, rest)
value -> value
end
end
defp build_tile(nil, _url, _geom, _props, _project_name), do: nil
defp build_tile(_tile_name, nil, _geom, _props, _project_name), do: nil
defp build_tile(tile_name, url, geom, props, project_name) do
%{
tile_name: to_string(tile_name),
url: to_string(url),
footprint_wkt: geometry_to_wkt(geom),
resolution_m: Decimal.new("1.0"),
year: extract_year(project_name),
crs: props["crs"],
file_size_bytes: props["file_size"]
}
end
defp derive_tile_url(project_name, tile_name) when is_binary(tile_name) do
"#{@s3_host}/StagedProducts/Elevation/1m/Projects/#{project_name}/TIFF/#{tile_name}"
end
defp derive_tile_url(_, _), do: nil
defp geometry_to_wkt(%{"type" => "Polygon", "coordinates" => [ring | _]}) do
points = Enum.map_join(ring, ", ", fn [lon, lat | _] -> "#{lon} #{lat}" end)
"POLYGON((#{points}))"
end
defp geometry_to_wkt(_), do: nil
defp extract_year(project_name) do
case Regex.run(~r/(\d{4})/, project_name) do
[_, year] -> String.to_integer(year)
_ -> nil
end
end
end

74
lib/towerops/lidar/sources/tnris.ex Normal file
View file

@ -0,0 +1,74 @@
defmodule Towerops.Lidar.Sources.Tnris do
@moduledoc """
TNRIS (TxGIO) catalog discovery — used as a fallback for areas not yet
covered by USGS 3DEP.
Only DEM/COG-like resources are surfaced; raw LAZ point clouds are
filtered out.
"""
alias Towerops.HTTP
@api_base "https://api.tnris.org/api/v1/resources"
@type tile :: %{
tile_name: String.t(),
url: String.t(),
footprint_wkt: String.t(),
resolution_m: Decimal.t(),
year: integer() | nil,
crs: String.t() | nil,
file_size_bytes: integer() | nil
}
@doc """
Fetches the LIDAR collection resources from TNRIS, filters to DEM-style
entries, and converts each to a tile map.
"""
@spec list_dem_resources() :: {:ok, [tile()]} | {:error, term()}
def list_dem_resources do
case HTTP.get(__MODULE__, url: @api_base, params: [{"collection", "lidar"}]) do
{:ok, %Req.Response{status: 200, body: body}} ->
{:ok, parse_resources(body)}
{:ok, %Req.Response{status: status, body: body}} ->
{:error, {:http, status, body}}
{:error, reason} ->
{:error, reason}
end
end
defp parse_resources(%{"results" => results}) when is_list(results) do
results
|> Enum.filter(&dem_resource?/1)
|> Enum.map(&resource_to_tile/1)
|> Enum.reject(&is_nil/1)
end
defp parse_resources(_), do: []
defp dem_resource?(%{"resource_type" => type}) when is_binary(type) do
String.upcase(type) =~ "DEM"
end
defp dem_resource?(_), do: false
defp resource_to_tile(%{"name" => name, "download_url" => url, "bbox" => [w, s, e, n]} = r) do
%{
tile_name: to_string(name),
url: to_string(url),
footprint_wkt: bbox_to_wkt(w, s, e, n),
resolution_m: Decimal.new("1.0"),
year: r["year"],
crs: r["crs"],
file_size_bytes: r["file_size"]
}
end
defp resource_to_tile(_), do: nil
defp bbox_to_wkt(w, s, e, n) do
"POLYGON((#{w} #{s}, #{e} #{s}, #{e} #{n}, #{w} #{n}, #{w} #{s}))"
end
end

39
lib/towerops/lidar/sync_log.ex Normal file
View file

@ -0,0 +1,39 @@
defmodule Towerops.Lidar.SyncLog do
@moduledoc """
Audit log for LIDAR catalog sync operations.
"""
use Ecto.Schema
import Ecto.Changeset
@primary_key {:id, :binary_id, autogenerate: true}
@valid_statuses ~w(success partial failed)
@valid_sources ~w(3dep tnris)
schema "lidar_sync_logs" do
field :source, :string
field :status, :string
field :projects_synced, :integer, default: 0
field :tiles_upserted, :integer, default: 0
field :errors, :map
field :duration_ms, :integer
field :inserted_at, :utc_datetime
end
def changeset(sync_log, attrs) do
sync_log
|> cast(attrs, [
:source,
:status,
:projects_synced,
:tiles_upserted,
:errors,
:duration_ms,
:inserted_at
])
|> validate_required([:source, :status, :inserted_at])
|> validate_inclusion(:source, @valid_sources)
|> validate_inclusion(:status, @valid_statuses)
end
end

60
lib/towerops/lidar/tile.ex Normal file
View file

@ -0,0 +1,60 @@
defmodule Towerops.Lidar.Tile do
@moduledoc """
A single LIDAR-derived DEM tile in the catalog.
We do not store raster bytes — only the public HTTPS URL and footprint
metadata. Reads happen on-demand via GDAL's `/vsicurl/` driver.
"""
use Ecto.Schema
import Ecto.Changeset
@primary_key {:id, :binary_id, autogenerate: true}
@foreign_key_type :binary_id
@valid_sources ~w(3dep tnris)
@valid_availability ~w(available missing error no_cog)
schema "lidar_tiles" do
field :source, :string
field :project_name, :string
field :tile_name, :string
field :year, :integer
field :resolution_m, :decimal
field :url, :string
field :crs, :string
field :file_size_bytes, :integer
field :metadata, :map, default: %{}
field :last_verified_at, :utc_datetime
field :availability, :string, default: "available"
field :footprint, Geo.PostGIS.Geometry
timestamps(type: :utc_datetime)
end
@castable [
:source,
:project_name,
:tile_name,
:year,
:resolution_m,
:url,
:crs,
:file_size_bytes,
:metadata,
:last_verified_at,
:availability,
:footprint,
:inserted_at,
:updated_at
]
def changeset(tile, attrs) do
tile
|> cast(attrs, @castable)
|> validate_required([:source, :project_name, :tile_name, :url, :footprint])
|> validate_inclusion(:source, @valid_sources)
|> validate_inclusion(:availability, @valid_availability)
|> unique_constraint([:source, :project_name, :tile_name])
end
end

5
lib/towerops/postgrex_types.ex Normal file
View file

@ -0,0 +1,5 @@
Postgrex.Types.define(
Towerops.PostgrexTypes,
[Geo.PostGIS.Extension] ++ Ecto.Adapters.Postgres.extensions(),
json: Jason
)

39
lib/towerops/workers/lidar_catalog_sync_worker.ex Normal file
View file

@ -0,0 +1,39 @@
defmodule Towerops.Workers.LidarCatalogSyncWorker do
@moduledoc """
Oban worker that syncs the LIDAR tile catalog from public sources.
Runs monthly via cron. Discovers Texas LIDAR projects on USGS 3DEP and
upserts tile metadata into `lidar_tiles`. Does not download raster data.
"""
use Oban.Worker,
queue: :lidar,
unique: [period: 86_400, states: [:available, :scheduled, :executing]]
alias Towerops.Lidar.Catalog
require Logger
@impl Oban.Worker
def perform(%Oban.Job{}) do
case run_sync() do
{:ok, summary} ->
Logger.info(
"Lidar catalog sync: #{summary.tiles_upserted} tiles across " <>
"#{summary.projects_synced} projects in #{summary.duration_ms}ms"
)
:ok
{:error, reason} = error ->
Logger.error("Lidar catalog sync failed: #{inspect(reason)}")
error
end
end
defp run_sync do
case Application.get_env(:towerops, :lidar_catalog_runner) do
nil -> Catalog.sync_3dep()
fun when is_function(fun, 0) -> fun.()
end
end
end

1
mix.exs
View file

@ -107,6 +107,7 @@ defmodule Towerops.MixProject do
{:mix_audit, "~> 2.1", only: [:dev, :test], runtime: false},
{:inet_cidr, "~> 1.0"},
{:cloak_ecto, "~> 1.3"},
{:geo_postgis, "~> 3.7"},
{:logger_file_backend, "~> 0.0.13", only: :dev},
{:logger_backends, "~> 1.0", only: :dev},
{:tidewave, "~> 0.5", only: :dev}

2
mix.lock
View file

@ -32,6 +32,8 @@
"finch": {:hex, :finch, "0.21.0", "b1c3b2d48af02d0c66d2a9ebfb5622be5c5ecd62937cf79a88a7f98d48a8290c", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "87dc6e169794cb2570f75841a19da99cfde834249568f2a5b121b809588a4377"},
"fine": {:hex, :fine, "0.1.6", "4bf7151493443c454aac9f2fa2f34f5fefd0346a83fb5586a016c4a135c63247", [:mix], [], "hexpm", "5638eb4495488e885ebec167fa57973e5c35e1a50c344eb7666c90ec1c4e3b12"},
"gen_smtp": {:hex, :gen_smtp, "1.3.0", "62c3d91f0dcf6ce9db71bcb6881d7ad0d1d834c7f38c13fa8e952f4104a8442e", [:rebar3], [{:ranch, ">= 1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "0b73fbf069864ecbce02fe653b16d3f35fd889d0fdd4e14527675565c39d84e6"},
"geo": {:hex, :geo, "4.1.0", "64ba89a64cc400b5b16dd2f5bd644cb141776eb8c2ac5a983332c8d944936c12", [:mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "19edb2b3398ca9f701b573b1fb11bc90951ebd64f18b06bd1bf35abe509a2934"},
"geo_postgis": {:hex, :geo_postgis, "3.7.1", "614f25b42334a615bd54bb09c22030b1aac7bac8f829bd823ab1faccf093a324", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:geo, "~> 3.6 or ~> 4.0", [hex: :geo, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0 or ~> 4.0 or ~> 5.0 or ~> 6.0", [hex: :poison, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: false]}], "hexpm", "c20d823c600d35b7fe9ddd5be03052bb7136c57d6f1775dbd46871545e405280"},
"gettext": {:hex, :gettext, "1.0.2", "5457e1fd3f4abe47b0e13ff85086aabae760497a3497909b8473e0acee57673b", [:mix], [{:expo, "~> 0.5.1 or ~> 1.0", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "eab805501886802071ad290714515c8c4a17196ea76e5afc9d06ca85fb1bfeb3"},
"hackney": {:hex, :hackney, "1.25.0", "390e9b83f31e5b325b9f43b76e1a785cbdb69b5b6cd4e079aa67835ded046867", [:rebar3], [{:certifi, "~> 2.15.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.4", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "7209bfd75fd1f42467211ff8f59ea74d6f2a9e81cbcee95a56711ee79fd6b1d4"},
"heroicons": {:git, "https://github.com/tailwindlabs/heroicons.git", "0435d4ca364a608cc75e2f8683d374e55abbae26", [tag: "v2.2.0", sparse: "optimized", depth: 1]},

11
priv/repo/migrations/20260504123340_enable_postgis.exs Normal file
View file

@ -0,0 +1,11 @@
defmodule Towerops.Repo.Migrations.EnablePostgis do
use Ecto.Migration
def up do
execute "CREATE EXTENSION IF NOT EXISTS postgis"
end
def down do
execute "DROP EXTENSION IF EXISTS postgis"
end
end

46
priv/repo/migrations/20260504123341_create_lidar_tiles.exs Normal file
View file

@ -0,0 +1,46 @@
defmodule Towerops.Repo.Migrations.CreateLidarTiles do
use Ecto.Migration
def change do
create table(:lidar_tiles, primary_key: false) do
add :id, :binary_id, primary_key: true
add :source, :string, null: false
add :project_name, :string, null: false
add :tile_name, :string, null: false
add :year, :integer
add :resolution_m, :decimal, precision: 6, scale: 3
add :url, :text, null: false
add :crs, :string
add :file_size_bytes, :bigint
add :metadata, :map, default: %{}
add :last_verified_at, :utc_datetime
add :availability, :string, null: false, default: "available"
timestamps(type: :utc_datetime)
end
# PostGIS geometry column. AddGeometryColumn registers the column in
# geometry_columns metadata and applies SRID/type constraints.
execute(
"""
SELECT AddGeometryColumn('lidar_tiles', 'footprint', 4326, 'POLYGON', 2)
""",
"SELECT DropGeometryColumn('lidar_tiles', 'footprint')"
)
execute(
"ALTER TABLE lidar_tiles ALTER COLUMN footprint SET NOT NULL",
"ALTER TABLE lidar_tiles ALTER COLUMN footprint DROP NOT NULL"
)
create unique_index(:lidar_tiles, [:source, :project_name, :tile_name])
create index(:lidar_tiles, [:year])
create index(:lidar_tiles, [:availability])
# GiST index on footprint for fast spatial intersection queries.
execute(
"CREATE INDEX lidar_tiles_footprint_gist ON lidar_tiles USING gist (footprint)",
"DROP INDEX lidar_tiles_footprint_gist"
)
end
end

18
priv/repo/migrations/20260504123342_create_lidar_sync_logs.exs Normal file
View file

@ -0,0 +1,18 @@
defmodule Towerops.Repo.Migrations.CreateLidarSyncLogs do
use Ecto.Migration
def change do
create table(:lidar_sync_logs, primary_key: false) do
add :id, :binary_id, primary_key: true
add :source, :string, null: false
add :status, :string, null: false
add :projects_synced, :integer, default: 0
add :tiles_upserted, :integer, default: 0
add :errors, :map
add :duration_ms, :integer
add :inserted_at, :utc_datetime, null: false
end
create index(:lidar_sync_logs, [:source, :inserted_at])
end
end

39
test/support/lidar_fake_parser.ex Normal file
View file

@ -0,0 +1,39 @@
defmodule Towerops.Lidar.Test.FakeParser do
@moduledoc """
Test-only stand-in for `Towerops.Lidar.Sources.ThreeDep.GdalParser`.
Tests configure canned tile responses per project name via `put_state/1`,
which are stored in an ETS table so the parser process can be the test
process or any Oban-spawned worker.
"""
@table :lidar_catalog_fake_parser
def put_state(map) do
ensure_table()
:ets.insert(@table, {:state, map})
end
def enumerate(project_name, _opts) do
ensure_table()
case :ets.lookup(@table, :state) do
[{:state, map}] ->
case Map.fetch(map, project_name) do
{:ok, {:error, _} = err} -> err
{:ok, tiles} -> {:ok, tiles}
:error -> {:error, :unknown_project}
end
_ ->
{:error, :no_state}
end
end
defp ensure_table do
case :ets.info(@table) do
:undefined -> :ets.new(@table, [:named_table, :public, :set])
_ -> @table
end
end
end

293
test/towerops/lidar/catalog_test.exs Normal file
View file

@ -0,0 +1,293 @@
defmodule Towerops.Lidar.CatalogTest do
use Towerops.DataCase
alias Towerops.Lidar.Catalog
alias Towerops.Lidar.Sources.ThreeDep
alias Towerops.Lidar.Sources.Tnris
alias Towerops.Lidar.SyncLog
alias Towerops.Lidar.Test.FakeParser
alias Towerops.Lidar.Tile
alias Towerops.Repo
setup do
on_exit(fn ->
Application.delete_env(:towerops, :lidar_3dep_parser)
end)
:ok
end
describe "sync_3dep/0" do
test "upserts tiles for each project and records a SyncLog with status :success" do
stub_projects(["TX_Foo_2024"])
stub_parser(%{"TX_Foo_2024" => sample_tiles(2, "TX_Foo_2024")})
assert {:ok, summary} = Catalog.sync_3dep()
assert summary.projects_synced == 1
assert summary.tiles_upserted == 2
assert summary.errors == %{}
tiles = Repo.all(Tile)
assert length(tiles) == 2
assert Enum.all?(tiles, &(&1.source == "3dep"))
assert Enum.all?(tiles, &(&1.project_name == "TX_Foo_2024"))
[log] = Repo.all(SyncLog)
assert log.source == "3dep"
assert log.status == "success"
assert log.projects_synced == 1
assert log.tiles_upserted == 2
assert is_integer(log.duration_ms)
end
test "is idempotent: re-running with the same data does not duplicate rows" do
stub_projects(["TX_Foo_2024"])
stub_parser(%{"TX_Foo_2024" => sample_tiles(3, "TX_Foo_2024")})
assert {:ok, _} = Catalog.sync_3dep()
assert Repo.aggregate(Tile, :count) == 3
assert {:ok, _} = Catalog.sync_3dep()
assert Repo.aggregate(Tile, :count) == 3
end
test "records SyncLog with status :partial when one project fails" do
stub_projects(["TX_Good_2024", "TX_Bad_2018"])
stub_parser(%{
"TX_Good_2024" => sample_tiles(1, "TX_Good_2024"),
"TX_Bad_2018" => {:error, :ogr2ogr_not_found}
})
assert {:ok, summary} = Catalog.sync_3dep()
assert summary.projects_synced == 1
assert summary.tiles_upserted == 1
assert Map.has_key?(summary.errors, "TX_Bad_2018")
[log] = Repo.all(SyncLog)
assert log.status == "partial"
end
test "records SyncLog with status :failed when project listing fails" do
stub_projects({:error, {:http, 500, "boom"}})
assert {:error, _} = Catalog.sync_3dep()
[log] = Repo.all(SyncLog)
assert log.status == "failed"
assert log.projects_synced == 0
assert log.tiles_upserted == 0
end
end
describe "sync_tnris/0" do
test "upserts TNRIS tiles not already covered by 3DEP" do
Req.Test.stub(Tnris, fn conn ->
conn
|> Plug.Conn.put_resp_content_type("application/json")
|> Plug.Conn.send_resp(
200,
Jason.encode!(%{
"results" => [
%{
"resource_type" => "DEM",
"name" => "tnris_tile_1.tif",
"year" => 2018,
"download_url" => "https://tnris.example.com/t1.tif",
"bbox" => [-100.0, 32.0, -99.9, 32.1]
}
]
})
)
end)
assert {:ok, summary} = Catalog.sync_tnris()
assert summary.tiles_upserted == 1
[tile] = Repo.all(Tile)
assert tile.source == "tnris"
assert tile.tile_name == "tnris_tile_1.tif"
end
test "skips TNRIS tiles whose centroid is contained by an existing 3DEP tile" do
_existing = insert_tile!(%{source: "3dep"})
Req.Test.stub(Tnris, fn conn ->
# bbox centroid is well inside the 3DEP tile fixture footprint
# (the helper insert_tile! places 3DEP at -97.7..-97.8, 30.2..30.3)
conn
|> Plug.Conn.put_resp_content_type("application/json")
|> Plug.Conn.send_resp(
200,
Jason.encode!(%{
"results" => [
%{
"resource_type" => "DEM",
"name" => "tnris_overlap.tif",
"year" => 2018,
"download_url" => "https://tnris.example.com/t1.tif",
"bbox" => [-97.78, 30.22, -97.72, 30.28]
}
]
})
)
end)
assert {:ok, summary} = Catalog.sync_tnris()
assert summary.tiles_upserted == 0
assert summary.errors["skipped_covered"] == 1
end
test "records SyncLog with status :failed when listing fails" do
Req.Test.stub(Tnris, fn conn ->
Plug.Conn.send_resp(conn, 503, "down")
end)
assert {:error, _} = Catalog.sync_tnris()
[log] = Repo.all(SyncLog)
assert log.source == "tnris"
assert log.status == "failed"
end
end
describe "verify_tile/1" do
test "sets last_verified_at on a 200 response" do
tile = insert_tile!()
Req.Test.stub(Catalog, fn conn ->
assert conn.method == "HEAD"
Plug.Conn.send_resp(conn, 200, "")
end)
assert {:ok, %Tile{} = updated} = Catalog.verify_tile(tile)
assert updated.availability == "available"
assert updated.last_verified_at
end
test "flips availability to \"missing\" on 404" do
tile = insert_tile!()
Req.Test.stub(Catalog, fn conn ->
Plug.Conn.send_resp(conn, 404, "")
end)
assert {:ok, %Tile{} = updated} = Catalog.verify_tile(tile)
assert updated.availability == "missing"
assert updated.last_verified_at
end
test "flips availability to \"error\" on transport failures" do
tile = insert_tile!()
Req.Test.stub(Catalog, fn conn ->
Plug.Conn.send_resp(conn, 503, "")
end)
assert {:ok, %Tile{} = updated} = Catalog.verify_tile(tile)
assert updated.availability == "error"
end
test "flips availability to \"error\" when no stub is configured (transport-failure path)" do
tile = insert_tile!()
# No Req.Test.stub — HTTP returns {:error, :no_test_stub}
assert {:ok, %Tile{availability: "error"}} = Catalog.verify_tile(tile)
end
end
describe "sync_3dep/0 with default lister (no app config override)" do
test "calls ThreeDep.list_texas_projects via Req.Test stub" do
Req.Test.stub(ThreeDep, fn conn ->
Plug.Conn.send_resp(
conn,
200,
"""
<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult>
<IsTruncated>false</IsTruncated>
<CommonPrefixes>
<Prefix>StagedProducts/Elevation/1m/Projects/TX_Default_2024/</Prefix>
</CommonPrefixes>
</ListBucketResult>
"""
)
end)
stub_parser(%{"TX_Default_2024" => sample_tiles(1, "TX_Default_2024")})
assert {:ok, summary} = Catalog.sync_3dep()
assert summary.tiles_upserted == 1
end
end
# ---------- helpers ----------
defp insert_tile!(attrs \\ %{}) do
poly = %Geo.Polygon{
coordinates: [
[
{-97.8, 30.2},
{-97.7, 30.2},
{-97.7, 30.3},
{-97.8, 30.3},
{-97.8, 30.2}
]
],
srid: 4326
}
defaults = %{
source: "3dep",
project_name: "TX_Test_2024",
tile_name: "tile_#{System.unique_integer([:positive])}",
year: 2024,
resolution_m: Decimal.new("1.0"),
url: "https://prd-tnm.s3.amazonaws.com/example.tif",
crs: "EPSG:6343",
footprint: poly,
availability: "available",
metadata: %{},
inserted_at: DateTime.truncate(DateTime.utc_now(), :second),
updated_at: DateTime.truncate(DateTime.utc_now(), :second)
}
%Tile{}
|> Tile.changeset(Map.merge(defaults, attrs))
|> Repo.insert!()
end
defp sample_tiles(count, project) do
Enum.map(1..count, fn idx ->
%{
tile_name: "#{project}_tile_#{idx}",
url: "https://prd-tnm.s3.amazonaws.com/#{project}/tile_#{idx}.tif",
footprint_wkt:
"POLYGON((-97.#{idx} 30.2, -97.#{idx + 1} 30.2, -97.#{idx + 1} 30.3, -97.#{idx} 30.3, -97.#{idx} 30.2))",
resolution_m: Decimal.new("1.0"),
year: 2024,
crs: "EPSG:6343",
file_size_bytes: 1_000_000
}
end)
end
defp stub_projects({:error, _} = err) do
test_pid = self()
Application.put_env(:towerops, :lidar_3dep_lister, fn -> send(test_pid, :listed) && err end)
on_exit(fn -> Application.delete_env(:towerops, :lidar_3dep_lister) end)
end
defp stub_projects(projects) when is_list(projects) do
Application.put_env(:towerops, :lidar_3dep_lister, fn -> {:ok, projects} end)
on_exit(fn -> Application.delete_env(:towerops, :lidar_3dep_lister) end)
# Some tests may exercise the real ThreeDep.list_texas_projects path through
# Req.Test stubs; we don't need that here, but defensively no-op.
Req.Test.stub(ThreeDep, fn conn -> Plug.Conn.send_resp(conn, 500, "") end)
end
defp stub_parser(map) do
FakeParser.put_state(map)
Application.put_env(:towerops, :lidar_3dep_parser, FakeParser)
end
end

152
test/towerops/lidar/grid_test.exs Normal file
View file

@ -0,0 +1,152 @@
defmodule Towerops.Lidar.GridTest do
use Towerops.DataCase, async: false
alias Towerops.Lidar
alias Towerops.Lidar.Tile
alias Towerops.Repo
setup do
on_exit(fn -> Application.delete_env(:towerops, :lidar_runner) end)
:ok
end
describe "get_elevation_grid/3" do
test "returns :no_tile when no catalog tile intersects the bbox" do
assert {:error, :no_tile} =
Lidar.get_elevation_grid({-97.8, 30.2, -97.7, 30.3}, 0.01)
end
test "returns :grid_too_large when requested cells exceed the cap" do
_ = insert_tile!()
# 1 degree x 1 degree at 0.0001 = ~10000x10000 = 100M cells
bbox = {-98.0, 30.0, -97.0, 31.0}
assert {:error, :grid_too_large} = Lidar.get_elevation_grid(bbox, 0.0001)
end
test "parses an AAIGrid response from a single tile" do
_ = insert_tile!()
stub_grid_runner("""
ncols 3
nrows 2
xllcorner -97.8
yllcorner 30.2
cellsize 0.05
NODATA_value -9999
100.0 101.0 102.0
103.0 104.0 105.0
""")
assert {:ok, grid} =
Lidar.get_elevation_grid({-97.8, 30.2, -97.65, 30.3}, 0.05)
assert grid.nrows == 2
assert grid.ncols == 3
assert grid.cells == [[100.0, 101.0, 102.0], [103.0, 104.0, 105.0]]
end
test "mosaics across two tiles by filling nodata cells from the next-best tile" do
_ = insert_tile!(%{tile_name: "best", resolution_m: Decimal.new("1.0")})
_ = insert_tile!(%{tile_name: "fallback", resolution_m: Decimal.new("3.0")})
1 |> :counters.new([:atomics]) |> tap(&Process.put(:c, &1))
Application.put_env(:towerops, :lidar_runner, fn _cmd, args, _opts ->
c = Process.get(:c)
:counters.add(c, 1, 1)
n = :counters.get(c, 1)
# Detect which tile is being read by the URL position in args
if String.contains?(Enum.join(args, " "), "best") or n == 1 do
{grid_aai([[100.0, -9999], [-9999, 105.0]]), 0}
else
{grid_aai([[200.0, 201.0], [202.0, 203.0]]), 0}
end
end)
assert {:ok, grid} =
Lidar.get_elevation_grid({-97.8, 30.2, -97.7, 30.3}, 0.05)
# Best tile contributes (0,0) and (1,1); fallback fills (0,1) and (1,0)
assert grid.cells == [[100.0, 201.0], [202.0, 105.0]]
end
test "returns :nodata when all tiles return only nodata" do
_ = insert_tile!()
stub_grid_runner(grid_aai([[-9999.0, -9999.0], [-9999.0, -9999.0]]))
assert {:error, :nodata} =
Lidar.get_elevation_grid({-97.8, 30.2, -97.7, 30.3}, 0.05)
end
test "propagates GDAL errors" do
_ = insert_tile!()
Application.put_env(:towerops, :lidar_runner, fn _cmd, _args, _opts ->
{"network unreachable", 1}
end)
assert {:error, {:gdal, 1, "network unreachable"}} =
Lidar.get_elevation_grid({-97.8, 30.2, -97.7, 30.3}, 0.05)
end
end
defp insert_tile!(attrs \\ %{}) do
poly = %Geo.Polygon{
coordinates: [
[
{-97.9, 30.1},
{-97.6, 30.1},
{-97.6, 30.4},
{-97.9, 30.4},
{-97.9, 30.1}
]
],
srid: 4326
}
defaults = %{
source: "3dep",
project_name: "TX_Test_2024",
tile_name: "tile_#{System.unique_integer([:positive])}",
year: 2024,
resolution_m: Decimal.new("1.0"),
url: "https://prd-tnm.s3.amazonaws.com/example.tif",
crs: "EPSG:6343",
footprint: poly,
availability: "available",
metadata: %{},
inserted_at: DateTime.truncate(DateTime.utc_now(), :second),
updated_at: DateTime.truncate(DateTime.utc_now(), :second)
}
%Tile{}
|> Tile.changeset(Map.merge(defaults, attrs))
|> Repo.insert!()
end
defp stub_grid_runner(output) do
Application.put_env(:towerops, :lidar_runner, fn _cmd, _args, _opts ->
{output, 0}
end)
end
defp grid_aai(rows) do
ncols = length(hd(rows))
nrows = length(rows)
body =
Enum.map_join(rows, "\n", fn row -> Enum.map_join(row, " ", &to_string/1) end)
"""
ncols #{ncols}
nrows #{nrows}
xllcorner -97.8
yllcorner 30.2
cellsize 0.05
NODATA_value -9999
#{body}
"""
end
end

218
test/towerops/lidar/reader_test.exs Normal file
View file

@ -0,0 +1,218 @@
defmodule Towerops.Lidar.ReaderTest do
use ExUnit.Case, async: false
alias Towerops.Lidar.Reader
alias Towerops.Lidar.Tile
setup do
on_exit(fn -> Application.delete_env(:towerops, :lidar_runner) end)
:ok
end
describe "elevation_at/3" do
test "returns the parsed float on success" do
stub_runner(fn cmd, args, _opts ->
assert cmd == "gdallocationinfo"
assert "-wgs84" in args
assert "-valonly" in args
assert "/vsicurl/https://example.com/tile.tif" in args
# Args should also include lon then lat at the end
[lon_str, lat_str] = Enum.take(args, -2)
assert lon_str == "-97.7431"
assert lat_str == "30.2672"
{"149.32\n", 0}
end)
assert {:ok, 149.32} = Reader.elevation_at(tile(), 30.2672, -97.7431)
end
test "returns :nodata when stdout is empty or all whitespace" do
stub_runner(fn _cmd, _args, _opts -> {"\n", 0} end)
assert {:error, :nodata} = Reader.elevation_at(tile(), 30.0, -97.0)
end
test "returns :nodata when value matches the GDAL nodata sentinel" do
stub_runner(fn _cmd, _args, _opts -> {"-3.4028235e+38\n", 0} end)
assert {:error, :nodata} = Reader.elevation_at(tile(), 30.0, -97.0)
end
test "returns {:error, {:gdal, code, output}} on non-zero exit" do
stub_runner(fn _cmd, _args, _opts -> {"some error\n", 2} end)
assert {:error, {:gdal, 2, "some error\n"}} = Reader.elevation_at(tile(), 30.0, -97.0)
end
test "returns :nodata on non-numeric stdout" do
stub_runner(fn _cmd, _args, _opts -> {"junk output\n", 0} end)
assert {:error, :nodata} = Reader.elevation_at(tile(), 30.0, -97.0)
end
end
describe "healthcheck/0" do
test "returns :ok when the runner returns version output" do
stub_runner(fn "gdallocationinfo", ["--version"], _opts ->
{"GDAL 3.9.0, released 2024/05/10\n", 0}
end)
assert :ok = Reader.healthcheck()
end
test "returns {:error, :gdal_missing} when the runner errors" do
stub_runner(fn _cmd, _args, _opts -> {"command not found", 127} end)
assert {:error, :gdal_missing} = Reader.healthcheck()
end
end
describe "parse_aaigrid/1" do
test "parses a valid grid with NODATA_value" do
grid = """
ncols 2
nrows 2
xllcorner -97.8
yllcorner 30.2
cellsize 0.05
NODATA_value -9999
1.0 2.0
3.0 4.0
"""
assert {:ok,
%{
ncols: 2,
nrows: 2,
xllcorner: -97.8,
yllcorner: 30.2,
cellsize: 0.05,
nodata_value: -9999.0,
cells: [[1.0, 2.0], [3.0, 4.0]]
}} = Reader.parse_aaigrid(grid)
end
test "defaults nodata_value to -9999.0 when missing" do
grid = """
ncols 1
nrows 1
xllcorner 0.0
yllcorner 0.0
cellsize 1.0
42.0
"""
assert {:ok, %{nodata_value: -9999.0, cells: [[42.0]]}} = Reader.parse_aaigrid(grid)
end
test "returns error on missing required header" do
grid = """
ncols 2
xllcorner 0.0
yllcorner 0.0
cellsize 1.0
1.0 2.0
"""
assert {:error, {:missing_header, "nrows"}} = Reader.parse_aaigrid(grid)
end
test "returns error on missing required float header" do
grid = """
ncols 1
nrows 1
xllcorner 0.0
cellsize 1.0
1.0
"""
assert {:error, {:missing_header, "yllcorner"}} = Reader.parse_aaigrid(grid)
end
test "returns error on bad integer header" do
grid = """
ncols not_a_number
nrows 1
xllcorner 0.0
yllcorner 0.0
cellsize 1.0
1.0
"""
assert {:error, {:bad_header, "ncols"}} = Reader.parse_aaigrid(grid)
end
test "returns error on bad float header" do
grid = """
ncols 1
nrows 1
xllcorner not_a_float
yllcorner 0.0
cellsize 1.0
1.0
"""
assert {:error, {:bad_header, "xllcorner"}} = Reader.parse_aaigrid(grid)
end
test "returns error on bad nodata_value" do
grid = """
ncols 1
nrows 1
xllcorner 0.0
yllcorner 0.0
cellsize 1.0
NODATA_value abc
1.0
"""
assert {:error, {:bad_header, "nodata_value"}} = Reader.parse_aaigrid(grid)
end
test "treats unparseable cell tokens as nil" do
grid = """
ncols 2
nrows 1
xllcorner 0.0
yllcorner 0.0
cellsize 1.0
NODATA_value -9999
not_a_number 2.0
"""
assert {:ok, %{cells: [[nil, 2.0]]}} = Reader.parse_aaigrid(grid)
end
end
describe "elevation_grid/3" do
test "passes /vsicurl/ paths through without re-prefixing" do
Application.put_env(:towerops, :lidar_runner, fn _cmd, args, _opts ->
url_arg = Enum.find(args, &String.starts_with?(&1, "/vsicurl/"))
# If the input already has the prefix, vsicurl/2 should not double it
refute String.starts_with?(url_arg, "/vsicurl//vsicurl/")
{"""
ncols 1
nrows 1
xllcorner 0.0
yllcorner 0.0
cellsize 1.0
1.0
""", 0}
end)
tile_with_vsi = %Tile{id: Ecto.UUID.generate(), url: "/vsicurl/https://x/y.tif"}
assert {:ok, %{cells: [[1.0]]}} =
Reader.elevation_grid(tile_with_vsi, {-97.8, 30.2, -97.7, 30.3}, 0.05)
end
end
defp tile do
%Tile{
id: Ecto.UUID.generate(),
url: "https://example.com/tile.tif",
crs: "EPSG:4326"
}
end
defp stub_runner(fun) when is_function(fun, 3) do
Application.put_env(:towerops, :lidar_runner, fun)
end
end

235
test/towerops/lidar/sources/three_dep/gdal_parser_test.exs Normal file
View file

@ -0,0 +1,235 @@
defmodule Towerops.Lidar.Sources.ThreeDep.GdalParserTest do
use ExUnit.Case, async: true
alias Towerops.Lidar.Sources.ThreeDep.GdalParser
describe "decode_geojson/2" do
test "extracts tile maps from a feature collection" do
json =
Jason.encode!(%{
"features" => [
%{
"properties" => %{
"tile_id" => "tile_a",
"download_url" => "https://example.com/tile_a.tif",
"crs" => "EPSG:6343",
"file_size" => 9_876_543
},
"geometry" => %{
"type" => "Polygon",
"coordinates" => [
[[-97.8, 30.2], [-97.7, 30.2], [-97.7, 30.3], [-97.8, 30.3], [-97.8, 30.2]]
]
}
}
]
})
assert {:ok, [tile]} = GdalParser.decode_geojson(json, "TX_Foo_2024")
assert tile.tile_name == "tile_a"
assert tile.url == "https://example.com/tile_a.tif"
assert tile.year == 2024
assert tile.crs == "EPSG:6343"
assert tile.file_size_bytes == 9_876_543
assert tile.footprint_wkt =~ "POLYGON"
assert tile.resolution_m == Decimal.new("1.0")
end
test "falls back to derived tile URL when download_url is absent" do
json =
Jason.encode!(%{
"features" => [
%{
"properties" => %{"tile_id" => "tile_b"},
"geometry" => %{
"type" => "Polygon",
"coordinates" => [[[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 1.0], [0.0, 0.0]]]
}
}
]
})
assert {:ok, [tile]} = GdalParser.decode_geojson(json, "TX_Bar_2019")
assert tile.url =~ "TX_Bar_2019/TIFF/tile_b"
assert tile.year == 2019
end
test "tries TILE_ID and filename keys for tile name" do
json =
Jason.encode!(%{
"features" => [
%{
"properties" => %{"TILE_ID" => "upper_case_id"},
"geometry" => %{"type" => "Polygon", "coordinates" => [[[0.0, 0.0]]]}
},
%{
"properties" => %{"filename" => "via_filename"},
"geometry" => %{"type" => "Polygon", "coordinates" => [[[0.0, 0.0]]]}
}
]
})
assert {:ok, tiles} = GdalParser.decode_geojson(json, "TX_Test_2024")
names = Enum.map(tiles, & &1.tile_name)
assert "upper_case_id" in names
assert "via_filename" in names
end
test "skips features missing tile name entirely" do
json =
Jason.encode!(%{
"features" => [
%{
"properties" => %{"unknown_key" => "x"},
"geometry" => %{"type" => "Polygon", "coordinates" => [[[0.0, 0.0]]]}
}
]
})
assert {:ok, []} = GdalParser.decode_geojson(json, "TX_Test_2024")
end
test "skips features whose properties is empty (no name resolvable)" do
json =
Jason.encode!(%{
"features" => [
%{
"properties" => %{},
"geometry" => %{"type" => "Polygon", "coordinates" => [[[0.0, 0.0]]]}
}
]
})
assert {:ok, []} = GdalParser.decode_geojson(json, "TX_Test_2024")
end
test "skips features with empty-string tile names" do
json =
Jason.encode!(%{
"features" => [
%{
"properties" => %{"tile_id" => "", "TILE_ID" => "", "filename" => ""},
"geometry" => %{"type" => "Polygon", "coordinates" => [[[0.0, 0.0]]]}
}
]
})
assert {:ok, []} = GdalParser.decode_geojson(json, "TX_Test_2024")
end
test "skips features whose geometry is non-Polygon (no WKT)" do
json =
Jason.encode!(%{
"features" => [
%{
"properties" => %{"tile_id" => "tile_c"},
"geometry" => %{"type" => "Point", "coordinates" => [0.0, 0.0]}
}
]
})
assert {:ok, [tile]} = GdalParser.decode_geojson(json, "TX_Test_2024")
assert tile.footprint_wkt == nil
end
test "returns :no_features when payload lacks features key" do
assert {:error, :no_features} =
GdalParser.decode_geojson(Jason.encode!(%{"x" => 1}), "TX_X_2024")
end
test "returns json_decode error on bad JSON" do
assert {:error, {:json_decode, _}} = GdalParser.decode_geojson("not json", "TX_X_2024")
end
test "extracts no year when project name has none" do
json =
Jason.encode!(%{
"features" => [
%{
"properties" => %{"tile_id" => "x"},
"geometry" => %{"type" => "Polygon", "coordinates" => [[[0.0, 0.0]]]}
}
]
})
assert {:ok, [tile]} = GdalParser.decode_geojson(json, "TX_NoYear")
assert tile.year == nil
end
end
describe "enumerate/2" do
test "returns :ogr2ogr_not_found when binary is missing" do
assert {:error, :ogr2ogr_not_found} =
GdalParser.enumerate("TX_X_2024",
ogr_bin: "definitely-not-a-real-bin-#{System.unique_integer()}"
)
end
test "shells out via the injected runner and returns parsed tiles on success" do
Application.put_env(:towerops, :lidar_ogr_runner, fn _bin, args, _opts ->
# Sanity-check the args so any breakage in the constructor surfaces here
assert "-f" in args and "GeoJSON" in args
assert Enum.any?(args, &String.starts_with?(&1, "/vsicurl/"))
json =
Jason.encode!(%{
"features" => [
%{
"properties" => %{"tile_id" => "tile_x"},
"geometry" => %{
"type" => "Polygon",
"coordinates" => [[[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 0.0]]]
}
}
]
})
{json, 0}
end)
on_exit(fn -> Application.delete_env(:towerops, :lidar_ogr_runner) end)
assert {:ok, [tile]} = GdalParser.enumerate("TX_Foo_2020")
assert tile.tile_name == "tile_x"
end
test "returns {:error, {:ogr2ogr, code, output}} on non-zero exit" do
Application.put_env(:towerops, :lidar_ogr_runner, fn _bin, _args, _opts ->
{"failed to read manifest", 2}
end)
on_exit(fn -> Application.delete_env(:towerops, :lidar_ogr_runner) end)
assert {:error, {:ogr2ogr, 2, "failed to read manifest"}} =
GdalParser.enumerate("TX_Foo_2020")
end
end
describe "decode_geojson/2 — defensive paths" do
test "skips features whose properties is non-map (e.g. nil)" do
json =
Jason.encode!(%{
"features" => [%{"properties" => nil, "geometry" => nil}]
})
assert {:ok, []} = GdalParser.decode_geojson(json, "TX_X_2024")
end
test "skips features where the tile name resolves but URL cannot be derived" do
# tile_id is a number — first_present returns the number (truthy),
# but derive_tile_url returns nil since tile_name isn't a binary,
# exercising the build_tile/_, nil, ...) clause.
json =
Jason.encode!(%{
"features" => [
%{
"properties" => %{"tile_id" => 42},
"geometry" => %{"type" => "Polygon", "coordinates" => [[[0.0, 0.0]]]}
}
]
})
assert {:ok, []} = GdalParser.decode_geojson(json, "TX_X_2024")
end
end
end

121
test/towerops/lidar/sources/three_dep_test.exs Normal file
View file

@ -0,0 +1,121 @@
defmodule Towerops.Lidar.Sources.ThreeDepTest do
use ExUnit.Case, async: true
alias Towerops.Lidar.Sources.ThreeDep
describe "list_texas_projects/0" do
test "parses CommonPrefixes and returns project names without prefix path" do
Req.Test.stub(ThreeDep, fn conn ->
assert conn.method == "GET"
assert conn.host == "prd-tnm.s3.amazonaws.com"
assert conn.query_string =~ "list-type=2"
assert conn.query_string =~ "prefix=StagedProducts%2FElevation%2F1m%2FProjects%2FTX_"
assert conn.query_string =~ "delimiter=%2F"
Plug.Conn.send_resp(conn, 200, list_xml_response(["TX_Foo_2018", "TX_Bar_2020"], false))
end)
assert {:ok, projects} = ThreeDep.list_texas_projects()
assert projects == ["TX_Foo_2018", "TX_Bar_2020"]
end
test "follows NextContinuationToken across pages" do
1 |> :counters.new([:atomics]) |> tap(fn c -> Process.put(:counter, c) end)
Req.Test.stub(ThreeDep, fn conn ->
c = Process.get(:counter)
:counters.add(c, 1, 1)
count = :counters.get(c, 1)
if count == 1 do
refute conn.query_string =~ "continuation-token"
Plug.Conn.send_resp(conn, 200, list_xml_response(["TX_A_2018"], true))
else
assert conn.query_string =~ "continuation-token=NEXT-TOKEN"
Plug.Conn.send_resp(conn, 200, list_xml_response(["TX_B_2020"], false))
end
end)
assert {:ok, ["TX_A_2018", "TX_B_2020"]} = ThreeDep.list_texas_projects()
end
test "returns error tuple on HTTP failure" do
Req.Test.stub(ThreeDep, fn conn ->
Plug.Conn.send_resp(conn, 500, "boom")
end)
assert {:error, {:http, 500, _}} = ThreeDep.list_texas_projects()
end
test "returns error tuple when no test stub is configured (transport-failure path)" do
# No Req.Test.stub, so HTTP.request returns {:error, :no_test_stub} —
# exercises the {:error, reason} branch in list_texas_projects/0
assert {:error, :no_test_stub} = ThreeDep.list_texas_projects()
end
end
describe "enumerate_tiles/1" do
test "delegates to the configured parser module" do
tiles = [
%{
tile_name: "tile1",
url: "https://prd-tnm.s3.amazonaws.com/example/tile1.tif",
footprint_wkt: "POLYGON((-97.9 30.2, -97.7 30.2, -97.7 30.4, -97.9 30.4, -97.9 30.2))",
resolution_m: Decimal.new("1.0"),
year: 2024,
crs: "EPSG:6343",
file_size_bytes: 12_345
}
]
defmodule FakeParser do
@moduledoc false
def enumerate("TX_Test_2024", _opts),
do:
{:ok,
[
%{
tile_name: "tile1",
url: "https://prd-tnm.s3.amazonaws.com/example/tile1.tif",
footprint_wkt: "POLYGON((-97.9 30.2, -97.7 30.2, -97.7 30.4, -97.9 30.4, -97.9 30.2))",
resolution_m: Decimal.new("1.0"),
year: 2024,
crs: "EPSG:6343",
file_size_bytes: 12_345
}
]}
end
Application.put_env(:towerops, :lidar_3dep_parser, FakeParser)
on_exit(fn -> Application.delete_env(:towerops, :lidar_3dep_parser) end)
assert {:ok, ^tiles} = ThreeDep.enumerate_tiles("TX_Test_2024")
end
end
defp list_xml_response(projects, has_more?) do
prefixes =
Enum.map_join(projects, "", fn p ->
"<CommonPrefixes><Prefix>StagedProducts/Elevation/1m/Projects/#{p}/</Prefix></CommonPrefixes>"
end)
next_token =
if has_more?, do: "<NextContinuationToken>NEXT-TOKEN</NextContinuationToken>", else: ""
truncated = if has_more?, do: "true", else: "false"
"""
<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>prd-tnm</Name>
<Prefix>StagedProducts/Elevation/1m/Projects/TX_</Prefix>
<KeyCount>#{length(projects)}</KeyCount>
<MaxKeys>1000</MaxKeys>
<Delimiter>/</Delimiter>
<IsTruncated>#{truncated}</IsTruncated>
#{prefixes}
#{next_token}
</ListBucketResult>
"""
end
end

87
test/towerops/lidar/sources/tnris_test.exs Normal file
View file

@ -0,0 +1,87 @@
defmodule Towerops.Lidar.Sources.TnrisTest do
use ExUnit.Case, async: true
alias Towerops.Lidar.Sources.Tnris
describe "list_dem_resources/0" do
test "returns parsed DEM resources, skipping LAZ-only entries" do
Req.Test.stub(Tnris, fn conn ->
assert conn.host == "api.tnris.org"
assert conn.request_path == "/api/v1/resources"
assert conn.query_string =~ "collection=lidar"
conn
|> Plug.Conn.put_resp_content_type("application/json")
|> Plug.Conn.send_resp(200, Jason.encode!(api_response()))
end)
assert {:ok, [tile]} = Tnris.list_dem_resources()
assert tile.tile_name == "tile_1.tif"
assert tile.url =~ "tile_1.tif"
assert tile.year == 2019
assert tile.footprint_wkt =~ "POLYGON"
end
test "returns error tuple on HTTP failure" do
Req.Test.stub(Tnris, fn conn ->
Plug.Conn.send_resp(conn, 503, "down")
end)
assert {:error, {:http, 503, _}} = Tnris.list_dem_resources()
end
test "returns error tuple when no test stub is configured (transport-failure path)" do
assert {:error, :no_test_stub} = Tnris.list_dem_resources()
end
test "returns empty list when body has no results key" do
Req.Test.stub(Tnris, fn conn ->
conn
|> Plug.Conn.put_resp_content_type("application/json")
|> Plug.Conn.send_resp(200, Jason.encode!(%{"unrelated" => "data"}))
end)
assert {:ok, []} = Tnris.list_dem_resources()
end
test "skips resources missing resource_type or required fields" do
Req.Test.stub(Tnris, fn conn ->
conn
|> Plug.Conn.put_resp_content_type("application/json")
|> Plug.Conn.send_resp(
200,
Jason.encode!(%{
"results" => [
%{"missing" => "type"},
%{"resource_type" => 42},
%{"resource_type" => "DEM", "incomplete" => true}
]
})
)
end)
assert {:ok, []} = Tnris.list_dem_resources()
end
end
defp api_response do
%{
"results" => [
%{
"resource_type" => "DEM",
"name" => "tile_1.tif",
"year" => 2019,
"download_url" => "https://example.tnris.org/tile_1.tif",
"bbox" => [-97.8, 30.2, -97.7, 30.3]
},
%{
"resource_type" => "LAZ",
"name" => "points.laz",
"year" => 2019,
"download_url" => "https://example.tnris.org/points.laz",
"bbox" => [-97.8, 30.2, -97.7, 30.3]
}
]
}
end
end

223
test/towerops/lidar_test.exs Normal file
View file

@ -0,0 +1,223 @@
defmodule Towerops.LidarTest do
use Towerops.DataCase
alias Towerops.Lidar
alias Towerops.Lidar.Tile
alias Towerops.Repo
# Build a small square polygon footprint around (lat, lon) of size deg.
defp square_footprint(lat, lon, deg \\ 0.05) do
coords = [
{lon - deg, lat - deg},
{lon + deg, lat - deg},
{lon + deg, lat + deg},
{lon - deg, lat + deg},
{lon - deg, lat - deg}
]
%Geo.Polygon{coordinates: [coords], srid: 4326}
end
defp insert_tile!(attrs) do
defaults = %{
source: "3dep",
project_name: "TX_Test_2024",
tile_name: "tile_#{System.unique_integer([:positive])}",
year: 2024,
resolution_m: Decimal.new("1.0"),
url: "https://prd-tnm.s3.amazonaws.com/example.tif",
crs: "EPSG:6343",
footprint: square_footprint(30.27, -97.74),
availability: "available",
metadata: %{},
inserted_at: DateTime.truncate(DateTime.utc_now(), :second),
updated_at: DateTime.truncate(DateTime.utc_now(), :second)
}
%Tile{}
|> Tile.changeset(Map.merge(defaults, attrs))
|> Repo.insert!()
end
describe "list_tiles_for_point/2" do
test "returns tiles whose footprint contains the point" do
tile = insert_tile!(%{footprint: square_footprint(30.27, -97.74, 0.1)})
_other = insert_tile!(%{footprint: square_footprint(40.0, -100.0, 0.1)})
[result] = Lidar.list_tiles_for_point(30.27, -97.74)
assert result.id == tile.id
end
test "returns empty list when no tile covers the point" do
_tile = insert_tile!(%{footprint: square_footprint(40.0, -100.0, 0.05)})
assert Lidar.list_tiles_for_point(30.27, -97.74) == []
end
test "orders by resolution_m ascending then year descending (best tile first)" do
coarse_old =
insert_tile!(%{
tile_name: "coarse_old",
resolution_m: Decimal.new("3.0"),
year: 2018
})
coarse_new =
insert_tile!(%{
tile_name: "coarse_new",
resolution_m: Decimal.new("3.0"),
year: 2024
})
fine_old =
insert_tile!(%{
tile_name: "fine_old",
resolution_m: Decimal.new("1.0"),
year: 2018
})
fine_new =
insert_tile!(%{
tile_name: "fine_new",
resolution_m: Decimal.new("1.0"),
year: 2024
})
result = Lidar.list_tiles_for_point(30.27, -97.74)
ids = Enum.map(result, & &1.id)
assert ids == [fine_new.id, fine_old.id, coarse_new.id, coarse_old.id]
end
test "excludes tiles with availability != \"available\"" do
_missing =
insert_tile!(%{tile_name: "missing", availability: "missing"})
ok = insert_tile!(%{tile_name: "ok", availability: "available"})
[result] = Lidar.list_tiles_for_point(30.27, -97.74)
assert result.id == ok.id
end
end
describe "list_tiles_for_area/1" do
test "returns tiles intersecting the bbox" do
inside = insert_tile!(%{footprint: square_footprint(30.27, -97.74, 0.05)})
_outside = insert_tile!(%{footprint: square_footprint(40.0, -100.0, 0.05)})
bbox = {-97.8, 30.2, -97.7, 30.3}
[result] = Lidar.list_tiles_for_area(bbox)
assert result.id == inside.id
end
test "returns multiple tiles ordered best-first" do
_coarse =
insert_tile!(%{
tile_name: "coarse",
resolution_m: Decimal.new("3.0"),
footprint: square_footprint(30.27, -97.74, 0.1)
})
fine =
insert_tile!(%{
tile_name: "fine",
resolution_m: Decimal.new("1.0"),
footprint: square_footprint(30.27, -97.74, 0.1)
})
bbox = {-97.8, 30.2, -97.7, 30.3}
[first | _] = Lidar.list_tiles_for_area(bbox)
assert first.id == fine.id
end
end
describe "get_elevation/2" do
test "returns :no_tile when no catalog tile covers the point" do
assert {:error, :no_tile} = Lidar.get_elevation(30.27, -97.74)
end
test "returns the reader's value when a tile covers the point" do
_ = insert_tile!(%{footprint: square_footprint(30.27, -97.74, 0.1)})
Application.put_env(:towerops, :lidar_runner, fn _cmd, _args, _opts ->
{"152.5\n", 0}
end)
on_exit(fn -> Application.delete_env(:towerops, :lidar_runner) end)
assert {:ok, 152.5} = Lidar.get_elevation(30.27, -97.74)
end
test "falls through to the next-best tile on nodata" do
_ = insert_tile!(%{tile_name: "best", resolution_m: Decimal.new("1.0")})
_ = insert_tile!(%{tile_name: "fallback", resolution_m: Decimal.new("3.0")})
1 |> :counters.new([:atomics]) |> tap(&Process.put(:c, &1))
Application.put_env(:towerops, :lidar_runner, fn _cmd, _args, _opts ->
c = Process.get(:c)
:counters.add(c, 1, 1)
n = :counters.get(c, 1)
if n == 1, do: {"\n", 0}, else: {"42.0\n", 0}
end)
on_exit(fn -> Application.delete_env(:towerops, :lidar_runner) end)
assert {:ok, 42.0} = Lidar.get_elevation(30.27, -97.74)
end
test "falls through to next tile on non-nodata reader errors" do
_ = insert_tile!(%{tile_name: "first"})
_ = insert_tile!(%{tile_name: "second"})
1 |> :counters.new([:atomics]) |> tap(&Process.put(:c, &1))
Application.put_env(:towerops, :lidar_runner, fn _cmd, _args, _opts ->
c = Process.get(:c)
:counters.add(c, 1, 1)
n = :counters.get(c, 1)
if n == 1 do
{"network down", 1}
else
{"77.0\n", 0}
end
end)
on_exit(fn -> Application.delete_env(:towerops, :lidar_runner) end)
assert {:ok, 77.0} = Lidar.get_elevation(30.27, -97.74)
end
test "returns the last error when only one tile and it returns a non-nodata error" do
_ = insert_tile!(%{})
Application.put_env(:towerops, :lidar_runner, fn _cmd, _args, _opts ->
{"boom", 5}
end)
on_exit(fn -> Application.delete_env(:towerops, :lidar_runner) end)
assert {:error, {:gdal, 5, "boom"}} = Lidar.get_elevation(30.27, -97.74)
end
test "returns :nodata when every covering tile lacks data" do
_ = insert_tile!(%{tile_name: "a"})
_ = insert_tile!(%{tile_name: "b"})
Application.put_env(:towerops, :lidar_runner, fn _cmd, _args, _opts ->
{"\n", 0}
end)
on_exit(fn -> Application.delete_env(:towerops, :lidar_runner) end)
assert {:error, :nodata} = Lidar.get_elevation(30.27, -97.74)
end
end
describe "get_elevation_grid/3" do
test "returns :no_tile when no catalog tile intersects the bbox" do
bbox = {-97.8, 30.2, -97.7, 30.3}
assert {:error, :no_tile} = Lidar.get_elevation_grid(bbox, 0.05)
end
end
end

60
test/towerops/workers/lidar_catalog_sync_worker_test.exs Normal file
View file

@ -0,0 +1,60 @@
defmodule Towerops.Workers.LidarCatalogSyncWorkerTest do
use Towerops.DataCase
alias Towerops.Workers.LidarCatalogSyncWorker
require Logger
describe "perform/1" do
test "calls Catalog.sync_3dep and returns :ok on success" do
stub_sync({:ok, %{projects_synced: 0, tiles_upserted: 0, errors: %{}, duration_ms: 1}})
assert :ok = LidarCatalogSyncWorker.perform(%Oban.Job{})
end
test "returns the error on failure so Oban retries" do
stub_sync({:error, :boom})
assert {:error, :boom} = LidarCatalogSyncWorker.perform(%Oban.Job{})
end
test "exercises the full success log branch" do
stub_sync({:ok, %{projects_synced: 3, tiles_upserted: 250, errors: %{}, duration_ms: 1234}})
previous = Logger.level()
Logger.configure(level: :info)
on_exit(fn -> Logger.configure(level: previous) end)
log =
ExUnit.CaptureLog.capture_log(fn ->
assert :ok = LidarCatalogSyncWorker.perform(%Oban.Job{})
end)
assert log =~ "250 tiles"
assert log =~ "3 projects"
assert log =~ "1234ms"
end
test "calls real Catalog.sync_3dep when no runner override is configured" do
Application.delete_env(:towerops, :lidar_catalog_runner)
Req.Test.stub(Towerops.Lidar.Sources.ThreeDep, fn conn ->
Plug.Conn.send_resp(
conn,
200,
"""
<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult><IsTruncated>false</IsTruncated></ListBucketResult>
"""
)
end)
assert :ok = LidarCatalogSyncWorker.perform(%Oban.Job{})
end
end
defp stub_sync(result) do
Application.put_env(:towerops, :lidar_catalog_runner, fn -> result end)
on_exit(fn -> Application.delete_env(:towerops, :lidar_catalog_runner) end)
end
end