diff --git a/CLAUDE.md b/CLAUDE.md index f138137f..d78cf58b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -192,6 +192,26 @@ Pure C NIF calling libnetsnmp directly for fast in-process MIB resolution Note: SnmpKit still used for SNMP protocol operations (get, walk, etc.). +### LIDAR Elevation Catalog + +Catalog-only system for accessing LIDAR-derived DEMs covering Texas. We do +not mirror raster data — the DB stores tile metadata (URL + footprint +geometry) and elevation values are streamed on-demand from public USGS +3DEP / TNRIS COGs via GDAL `/vsicurl/` HTTP byte-range reads. + +- **Catalog**: `lib/towerops/lidar/catalog.ex`, schemas in `lib/towerops/lidar/` +- **Sources**: `lib/towerops/lidar/sources/three_dep.ex` (primary), + `lib/towerops/lidar/sources/tnris.ex` (fallback for areas not in 3DEP) +- **Reader**: `lib/towerops/lidar/reader.ex` shells to `gdallocationinfo` / + `gdal_translate` against `/vsicurl/` — only fetches the bytes needed. +- **Sync**: `Towerops.Workers.LidarCatalogSyncWorker` runs monthly via Oban cron +- **Dependencies**: `brew install gdal postgis` (macOS), `apt-get install + gdal-bin postgis` (Docker — `gdal-bin` is in `k8s/Dockerfile`) +- **PostGIS**: enabled by migration `20260504123340_enable_postgis`, used for + spatial indexes on tile footprints (GiST) and `ST_Contains` dedup checks +- **Geometry types**: registered via `lib/towerops/postgrex_types.ex`, wired + into the Repo via `config :towerops, Towerops.Repo, types: ...` + ## Admin Features ### User Impersonation diff --git a/config/config.exs b/config/config.exs index 28cd9a3b..9ddc7489 100644 --- a/config/config.exs +++ b/config/config.exs @@ -109,7 +109,8 @@ config :towerops, Towerops.Mailer, adapter: Swoosh.Adapters.Local # This allows CI to load the schema instantly instead of running 172+ migrations config :towerops, Towerops.Repo, dump_path: "priv/repo/structure.sql", - migration_timestamps: [type: :naive_datetime_usec] + migration_timestamps: [type: :naive_datetime_usec], + types: Towerops.PostgrexTypes # Configure the endpoint config :towerops, ToweropsWeb.Endpoint, diff --git a/config/runtime.exs b/config/runtime.exs index d970959c..c23978ce 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -187,7 +187,8 @@ if config_env() == :prod do # Alert notifications (PagerDuty, email, etc.) notifications: max(1, div(25, oban_scale)), maintenance: max(1, div(5, oban_scale)), - weather: max(1, div(2, oban_scale)) + weather: max(1, div(2, oban_scale)), + lidar: max(1, div(2, oban_scale)) ], plugins: [ # Cron jobs for periodic maintenance tasks @@ -254,7 +255,9 @@ if config_env() == :prod do # Sync device usage to Stripe daily at 3 AM UTC {"0 3 * * *", Towerops.Workers.BillingSyncWorker}, # Data retention cleanup nightly at 1 AM - {"0 1 * * *", Towerops.Workers.DataRetentionWorker} + {"0 1 * * *", Towerops.Workers.DataRetentionWorker}, + # Refresh LIDAR tile catalog from public 3DEP/TNRIS sources monthly + {"0 6 1 * *", Towerops.Workers.LidarCatalogSyncWorker} ]}, # Automatically delete completed jobs after 60 seconds. # Default limit of 10k/30s (333/s) can't keep up with job throughput (~600+/s), diff --git a/k8s/Dockerfile b/k8s/Dockerfile index 884e9228..337c10b0 100644 --- a/k8s/Dockerfile +++ b/k8s/Dockerfile @@ -82,7 +82,7 @@ FROM ${RUNNER_IMAGE} AS final RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends libstdc++6 openssl libncurses6 locales ca-certificates iputils-ping snmp libsnmp40 \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends libstdc++6 openssl libncurses6 locales ca-certificates iputils-ping snmp libsnmp40 gdal-bin \ && rm -rf /var/lib/apt/lists/* # Set the locale diff --git a/lib/towerops/lidar.ex b/lib/towerops/lidar.ex new file mode 100644 index 00000000..c5c613a8 --- /dev/null +++ b/lib/towerops/lidar.ex @@ -0,0 +1,162 @@ +defmodule Towerops.Lidar do + @moduledoc """ + Catalog and on-demand retrieval of LIDAR-derived DEMs covering Texas. + + We do not mirror raster data. The catalog stores tile metadata and the + public HTTPS URL of each Cloud-Optimized GeoTIFF (USGS 3DEP, with TNRIS + as fallback). Elevation reads stream byte ranges directly from those + public buckets via GDAL's `/vsicurl/` driver. + """ + + import Ecto.Query + + alias Towerops.Lidar.Reader + alias Towerops.Lidar.Tile + alias Towerops.Repo + + @typedoc "{west, south, east, north} in WGS84 degrees" + @type bbox :: {number(), number(), number(), number()} + + @doc """ + Lists catalog tiles whose footprint contains the given point, ordered + best-first (highest resolution, then newest year). + + Filters out tiles flagged as unavailable. + """ + @spec list_tiles_for_point(number(), number()) :: [Tile.t()] + def list_tiles_for_point(lat, lon) do + point = %Geo.Point{coordinates: {lon, lat}, srid: 4326} + + Tile + |> where([t], t.availability == "available") + |> where([t], fragment("ST_Intersects(?, ?)", t.footprint, ^point)) + |> order_by([t], asc: t.resolution_m, desc: t.year) + |> Repo.all() + end + + @doc """ + Lists catalog tiles whose footprint intersects the given bbox, ordered + best-first (highest resolution, then newest year). + """ + @spec list_tiles_for_area(bbox()) :: [Tile.t()] + def list_tiles_for_area({west, south, east, north}) do + envelope = + %Geo.Polygon{ + coordinates: [ + [ + {west, south}, + {east, south}, + {east, north}, + {west, north}, + {west, south} + ] + ], + srid: 4326 + } + + Tile + |> where([t], t.availability == "available") + |> where([t], fragment("ST_Intersects(?, ?)", t.footprint, ^envelope)) + |> order_by([t], asc: t.resolution_m, desc: t.year) + |> Repo.all() + end + + @doc """ + Returns the elevation in meters at the given (lat, lon). + + Picks the best (highest-resolution, newest) tile that covers the point. + Falls through to the next-best tile if the chosen one returns nodata. + Returns `{:error, :no_tile}` if no catalog tile covers the point, or + `{:error, :nodata}` if every covering tile lacks data at that pixel. + """ + @spec get_elevation(number(), number()) :: {:ok, float()} | {:error, term()} + def get_elevation(lat, lon) do + case list_tiles_for_point(lat, lon) do + [] -> {:error, :no_tile} + tiles -> read_first_value(tiles, lat, lon) + end + end + + defp read_first_value([], _lat, _lon), do: {:error, :nodata} + + defp read_first_value([tile | rest], lat, lon) do + case Reader.elevation_at(tile, lat, lon) do + {:ok, value} -> {:ok, value} + {:error, :nodata} -> read_first_value(rest, lat, lon) + {:error, _} = err when rest == [] -> err + {:error, _} -> read_first_value(rest, lat, lon) + end + end + + @max_grid_cells 4_000_000 + + @doc """ + Returns a 2D elevation grid covering the bbox at the given cell size + (in degrees, EPSG:4326). + + Mosaics across all intersecting catalog tiles, filling nodata cells from + next-best tiles. Caps at #{@max_grid_cells} cells to bound memory. + """ + @spec get_elevation_grid(bbox(), number()) :: + {:ok, map()} | {:error, term()} + def get_elevation_grid({west, south, east, north} = bbox, cell_size_deg) do + ncols = ceil((east - west) / cell_size_deg) + nrows = ceil((north - south) / cell_size_deg) + + if ncols * nrows > @max_grid_cells do + {:error, :grid_too_large} + else + case list_tiles_for_area(bbox) do + [] -> {:error, :no_tile} + tiles -> mosaic(tiles, bbox, cell_size_deg) + end + end + end + + defp mosaic(tiles, bbox, cell_size_deg) do + {grids, last_error} = + Enum.reduce(tiles, {[], nil}, fn tile, {acc, err} -> + case Reader.elevation_grid(tile, bbox, cell_size_deg) do + {:ok, grid} -> {[grid | acc], err} + {:error, reason} -> {acc, reason} + end + end) + + case Enum.reverse(grids) do + [] -> {:error, last_error || :no_data} + [first | rest] -> finalize_mosaic(first, rest) + end + end + + defp finalize_mosaic(base, additional) do + nodata = base.nodata_value + + merged_cells = + Enum.reduce(additional, base.cells, fn next, current -> + merge_grid_cells(current, next.cells, nodata, next.nodata_value) + end) + + if all_nodata?(merged_cells, nodata) do + {:error, :nodata} + else + {:ok, %{base | cells: merged_cells}} + end + end + + defp merge_grid_cells(current_rows, next_rows, base_nodata, next_nodata) do + current_rows + |> Enum.zip(next_rows) + |> Enum.map(fn {current_row, next_row} -> + current_row + |> Enum.zip(next_row) + |> Enum.map(fn + {cur, next} when cur == base_nodata and next != next_nodata -> next + {cur, _next} -> cur + end) + end) + end + + defp all_nodata?(rows, nodata) do + Enum.all?(rows, fn row -> Enum.all?(row, &(&1 == nodata)) end) + end +end diff --git a/lib/towerops/lidar/catalog.ex b/lib/towerops/lidar/catalog.ex new file mode 100644 index 00000000..a600c32b --- /dev/null +++ b/lib/towerops/lidar/catalog.ex @@ -0,0 +1,278 @@ +defmodule Towerops.Lidar.Catalog do + @moduledoc """ + Orchestrates LIDAR tile catalog discovery + verification. + + Sync runs do NOT download raster data — they only enumerate which tiles + exist publicly and store HTTPS URLs + footprint metadata in our DB. + """ + + import Ecto.Query + + alias Towerops.HTTP + alias Towerops.Lidar.Sources.ThreeDep + alias Towerops.Lidar.Sources.Tnris + alias Towerops.Lidar.SyncLog + alias Towerops.Lidar.Tile + alias Towerops.Repo + + require Logger + + @batch_size 5_000 + + @type sync_summary :: %{ + projects_synced: non_neg_integer(), + tiles_upserted: non_neg_integer(), + errors: map(), + duration_ms: integer() + } + + @doc """ + Syncs the 3DEP catalog: lists Texas-prefixed projects, enumerates tiles + per project, batches upserts into `lidar_tiles`, and records a `SyncLog`. + + Returns `{:ok, summary}` on success/partial, `{:error, reason}` on total + failure. + """ + @spec sync_3dep() :: {:ok, sync_summary()} | {:error, term()} + def sync_3dep do + started_at = System.monotonic_time(:millisecond) + + case list_projects() do + {:ok, projects} -> + run_sync(projects, started_at) + + {:error, reason} -> + record_failed_log(started_at) + {:error, reason} + end + end + + defp list_projects do + case Application.get_env(:towerops, :lidar_3dep_lister) do + nil -> ThreeDep.list_texas_projects() + fun when is_function(fun, 0) -> fun.() + end + end + + defp run_sync(projects, started_at) do + {tiles_upserted, errors} = + Enum.reduce(projects, {0, %{}}, fn project, {count, errs} -> + case ThreeDep.enumerate_tiles(project) do + {:ok, tiles} -> + inserted = upsert_tiles(project, tiles) + {count + inserted, errs} + + {:error, reason} -> + Logger.warning("Lidar sync: enumerate_tiles failed for #{project}: #{inspect(reason)}") + {count, Map.put(errs, project, inspect(reason))} + end + end) + + duration_ms = System.monotonic_time(:millisecond) - started_at + + summary = %{ + projects_synced: length(projects) - map_size(errors), + tiles_upserted: tiles_upserted, + errors: errors, + duration_ms: duration_ms + } + + status = if map_size(errors) == 0, do: "success", else: "partial" + + record_log(status, summary) + + {:ok, summary} + end + + defp upsert_tiles(project, tiles) when is_list(tiles) do + now = DateTime.truncate(DateTime.utc_now(), :second) + + rows = + Enum.map(tiles, fn t -> + %{ + id: Ecto.UUID.generate(), + source: "3dep", + project_name: project, + tile_name: t.tile_name, + year: t.year, + resolution_m: t.resolution_m, + url: t.url, + crs: t.crs, + file_size_bytes: t.file_size_bytes, + footprint: wkt_to_geometry(t.footprint_wkt), + metadata: %{}, + availability: "available", + inserted_at: now, + updated_at: now + } + end) + + rows + |> Enum.chunk_every(@batch_size) + |> Enum.reduce(0, fn batch, acc -> + {count, _} = + Repo.insert_all(Tile, batch, + on_conflict: {:replace_all_except, [:id, :inserted_at]}, + conflict_target: [:source, :project_name, :tile_name] + ) + + acc + count + end) + end + + defp wkt_to_geometry(wkt) when is_binary(wkt) do + geo = Geo.WKT.decode!(wkt) + %{geo | srid: 4326} + end + + @doc """ + Syncs the TNRIS catalog as a fallback. Skips any tile whose footprint + centroid is already covered by a 3DEP tile. + + Records a `SyncLog` with `source: "tnris"`. + """ + @spec sync_tnris() :: {:ok, sync_summary()} | {:error, term()} + def sync_tnris do + started_at = System.monotonic_time(:millisecond) + + case Tnris.list_dem_resources() do + {:ok, resources} -> + {kept, skipped} = + Enum.split_with(resources, fn r -> not covered_by_3dep?(r.footprint_wkt) end) + + upserted = + if kept == [] do + 0 + else + upsert_tnris_tiles(kept) + end + + duration_ms = System.monotonic_time(:millisecond) - started_at + + summary = %{ + projects_synced: 1, + tiles_upserted: upserted, + errors: %{"skipped_covered" => length(skipped)}, + duration_ms: duration_ms + } + + record_log("tnris", "success", summary) + {:ok, summary} + + {:error, reason} -> + record_log("tnris", "failed", %{ + projects_synced: 0, + tiles_upserted: 0, + errors: %{"list" => inspect(reason)}, + duration_ms: System.monotonic_time(:millisecond) - started_at + }) + + {:error, reason} + end + end + + defp covered_by_3dep?(wkt) when is_binary(wkt) do + geom = wkt_to_geometry(wkt) + + query = + from t in Tile, + where: t.source == "3dep", + where: + fragment( + "ST_Contains(?, ST_Centroid(?))", + t.footprint, + type(^geom, Geo.PostGIS.Geometry) + ), + select: count(t.id) + + Repo.one(query) > 0 + end + + defp upsert_tnris_tiles(resources) do + now = DateTime.truncate(DateTime.utc_now(), :second) + + rows = + Enum.map(resources, fn r -> + %{ + id: Ecto.UUID.generate(), + source: "tnris", + project_name: "tnris", + tile_name: r.tile_name, + year: r.year, + resolution_m: r.resolution_m, + url: r.url, + crs: r.crs, + file_size_bytes: r.file_size_bytes, + footprint: wkt_to_geometry(r.footprint_wkt), + metadata: %{}, + availability: "available", + inserted_at: now, + updated_at: now + } + end) + + rows + |> Enum.chunk_every(@batch_size) + |> Enum.reduce(0, fn batch, acc -> + {count, _} = + Repo.insert_all(Tile, batch, + on_conflict: {:replace_all_except, [:id, :inserted_at]}, + conflict_target: [:source, :project_name, :tile_name] + ) + + acc + count + end) + end + + @doc """ + HEAD-checks a tile's URL and updates `last_verified_at` + `availability`. + """ + @spec verify_tile(Tile.t()) :: {:ok, Tile.t()} | {:error, term()} + def verify_tile(%Tile{} = tile) do + availability = + case HTTP.request(__MODULE__, method: :head, url: tile.url) do + {:ok, %Req.Response{status: 200}} -> "available" + {:ok, %Req.Response{status: 404}} -> "missing" + {:ok, %Req.Response{}} -> "error" + {:error, _} -> "error" + end + + tile + |> Tile.changeset(%{ + availability: availability, + last_verified_at: DateTime.truncate(DateTime.utc_now(), :second) + }) + |> Repo.update() + end + + defp record_log(status, summary), do: record_log("3dep", status, summary) + + defp record_log(source, status, summary) do + %SyncLog{} + |> SyncLog.changeset(%{ + source: source, + status: status, + projects_synced: summary.projects_synced, + tiles_upserted: summary.tiles_upserted, + errors: summary.errors, + duration_ms: summary.duration_ms, + inserted_at: DateTime.truncate(DateTime.utc_now(), :second) + }) + |> Repo.insert() + end + + defp record_failed_log(started_at) do + duration_ms = System.monotonic_time(:millisecond) - started_at + + %SyncLog{} + |> SyncLog.changeset(%{ + source: "3dep", + status: "failed", + projects_synced: 0, + tiles_upserted: 0, + duration_ms: duration_ms, + inserted_at: DateTime.truncate(DateTime.utc_now(), :second) + }) + |> Repo.insert() + end +end diff --git a/lib/towerops/lidar/reader.ex b/lib/towerops/lidar/reader.ex new file mode 100644 index 00000000..93167bf3 --- /dev/null +++ b/lib/towerops/lidar/reader.ex @@ -0,0 +1,214 @@ +defmodule Towerops.Lidar.Reader do + @moduledoc """ + Reads elevation values from public Cloud-Optimized GeoTIFFs via GDAL. + + Uses `gdallocationinfo` with `/vsicurl/` to perform HTTP byte-range + reads — only the bytes needed for a single pixel are fetched. Tiles + are never downloaded in full and never persisted on our side. + """ + + alias Towerops.Lidar.Tile + + require Logger + + @gdal_bin "gdallocationinfo" + # GDAL emits this sentinel for nodata in -valonly mode + @nodata_sentinels ["-3.4028235e+38", "-3.402823466e+38", "-9999"] + + @doc """ + Returns `{:ok, elevation_m}` for the given (lat, lon) within the tile, + or `{:error, :nodata}` if no value at that point. + """ + @spec elevation_at(Tile.t(), float(), float()) :: + {:ok, float()} | {:error, :nodata | term()} + def elevation_at(%Tile{url: url}, lat, lon) when is_binary(url) and is_number(lat) and is_number(lon) do + args = [ + "-wgs84", + "-valonly", + vsicurl(url), + to_string(lon), + to_string(lat) + ] + + case run(@gdal_bin, args) do + {output, 0} -> parse_value(output) + {output, code} -> {:error, {:gdal, code, output}} + end + end + + @doc """ + Verifies GDAL is callable. Intended to be invoked at boot in non-test envs. + """ + @spec healthcheck() :: :ok | {:error, :gdal_missing} + def healthcheck do + case run(@gdal_bin, ["--version"]) do + {_output, 0} -> :ok + _ -> {:error, :gdal_missing} + end + end + + @doc """ + Reads an elevation grid for the given bbox at the requested cell size + (degrees), reprojecting to EPSG:4326 on the fly. Returns a parsed AAIGrid. + + Returns `{:ok, grid}` or `{:error, reason}`. If the underlying COG has no + data over the bbox, the cells field will be all-nodata; the caller is + responsible for treating that as `:nodata` if every tile produces nodata. + """ + @spec elevation_grid(Tile.t(), {number(), number(), number(), number()}, number()) :: + {:ok, map()} | {:error, term()} + def elevation_grid(%Tile{url: url}, {west, south, east, north}, cell_size_deg) do + args = [ + "-of", + "AAIGrid", + "-projwin_srs", + "EPSG:4326", + "-projwin", + to_string(west), + to_string(north), + to_string(east), + to_string(south), + "-tr", + to_string(cell_size_deg), + to_string(cell_size_deg), + "-t_srs", + "EPSG:4326", + vsicurl(url), + "/vsistdout/" + ] + + case run("gdal_translate", args) do + {output, 0} -> parse_aaigrid(output) + {output, code} -> {:error, {:gdal, code, output}} + end + end + + @doc """ + Parses an Esri ASCII Grid (AAIGrid) text into a structured map. + Public for testability. + """ + @spec parse_aaigrid(String.t()) :: {:ok, map()} | {:error, term()} + def parse_aaigrid(text) when is_binary(text) do + {header, body_lines} = split_header(text) + + with {:ok, ncols} <- header_int(header, "ncols"), + {:ok, nrows} <- header_int(header, "nrows"), + {:ok, xll} <- header_float(header, "xllcorner"), + {:ok, yll} <- header_float(header, "yllcorner"), + {:ok, cellsize} <- header_float(header, "cellsize"), + {:ok, nodata} <- header_nodata(header) do + cells = + body_lines + |> Enum.take(nrows) + |> Enum.map(&parse_row/1) + + {:ok, + %{ + ncols: ncols, + nrows: nrows, + xllcorner: xll, + yllcorner: yll, + cellsize: cellsize, + nodata_value: nodata, + cells: cells + }} + end + end + + defp split_header(text) do + lines = text |> String.split(~r/\r?\n/, trim: true) |> Enum.map(&String.trim/1) + {header, body} = Enum.split_while(lines, &header_line?/1) + {Map.new(header, &parse_header_line/1), body} + end + + defp header_line?(line) do + case Regex.run(~r/^([A-Za-z_]+)\s+/, line) do + [_, key] -> + String.downcase(key) in ~w(ncols nrows xllcorner yllcorner xllcenter yllcenter cellsize dx dy nodata_value) + + _ -> + false + end + end + + defp parse_header_line(line) do + [k, v] = String.split(line, ~r/\s+/, parts: 2) + {String.downcase(k), v} + end + + defp header_int(header, key) do + case Map.fetch(header, key) do + {:ok, val} -> + case Integer.parse(val) do + {n, _} -> {:ok, n} + _ -> {:error, {:bad_header, key}} + end + + :error -> + {:error, {:missing_header, key}} + end + end + + defp header_float(header, key) do + case Map.fetch(header, key) do + {:ok, val} -> + case Float.parse(val) do + {n, _} -> {:ok, n} + _ -> {:error, {:bad_header, key}} + end + + :error -> + {:error, {:missing_header, key}} + end + end + + defp header_nodata(header) do + case Map.get(header, "nodata_value") do + nil -> + {:ok, -9999.0} + + val -> + case Float.parse(val) do + {n, _} -> {:ok, n} + _ -> {:error, {:bad_header, "nodata_value"}} + end + end + end + + defp parse_row(line) do + line + |> String.split(~r/\s+/, trim: true) + |> Enum.map(fn token -> + case Float.parse(token) do + {n, _} -> n + :error -> nil + end + end) + end + + defp vsicurl("/vsicurl/" <> _rest = path), do: path + defp vsicurl(url), do: "/vsicurl/#{url}" + + defp parse_value(output) do + trimmed = String.trim(output) + + cond do + trimmed == "" -> + {:error, :nodata} + + Enum.any?(@nodata_sentinels, &String.starts_with?(trimmed, &1)) -> + {:error, :nodata} + + true -> + case Float.parse(trimmed) do + {value, _rest} -> {:ok, value} + :error -> {:error, :nodata} + end + end + end + + defp run(cmd, args, opts \\ [stderr_to_stdout: true]) do + runner = Application.get_env(:towerops, :lidar_runner, &System.cmd/3) + runner.(cmd, args, opts) + end +end diff --git a/lib/towerops/lidar/sources/three_dep.ex b/lib/towerops/lidar/sources/three_dep.ex new file mode 100644 index 00000000..bd46c9a1 --- /dev/null +++ b/lib/towerops/lidar/sources/three_dep.ex @@ -0,0 +1,104 @@ +defmodule Towerops.Lidar.Sources.ThreeDep do + @moduledoc """ + USGS 3DEP catalog discovery. + + Lists Texas-prefixed LIDAR projects on the public `prd-tnm` S3 bucket and + enumerates per-project tiles via an injectable manifest parser. + """ + + import SweetXml + + alias Towerops.HTTP + + @s3_host "https://prd-tnm.s3.amazonaws.com" + @projects_prefix "StagedProducts/Elevation/1m/Projects/" + @texas_prefix "TX_" + + @type project_name :: String.t() + + @type tile :: %{ + tile_name: String.t(), + url: String.t(), + footprint_wkt: String.t(), + resolution_m: Decimal.t(), + year: integer() | nil, + crs: String.t() | nil, + file_size_bytes: integer() | nil + } + + @doc """ + Lists Texas-prefixed project names by paginating through S3 ListObjectsV2. + """ + @spec list_texas_projects() :: {:ok, [project_name()]} | {:error, term()} + def list_texas_projects do + do_list_projects(nil, []) + end + + defp do_list_projects(continuation_token, acc) do + base_query = [ + {"list-type", "2"}, + {"prefix", @projects_prefix <> @texas_prefix}, + {"delimiter", "/"} + ] + + query = + if continuation_token do + base_query ++ [{"continuation-token", continuation_token}] + else + base_query + end + + case HTTP.get(__MODULE__, url: @s3_host <> "/", params: query) do + {:ok, %Req.Response{status: 200, body: body}} -> + {projects, next} = parse_list_response(body) + merged = acc ++ projects + + if next do + do_list_projects(next, merged) + else + {:ok, merged} + end + + {:ok, %Req.Response{status: status, body: body}} -> + {:error, {:http, status, body}} + + {:error, reason} -> + {:error, reason} + end + end + + @doc false + # Public for tests: parses the S3 ListObjectsV2 XML response. + def parse_list_response(xml) when is_binary(xml) do + prefixes = xpath(xml, ~x"//CommonPrefixes/Prefix/text()"sl) + truncated = xpath(xml, ~x"//IsTruncated/text()"s) + next_token = xpath(xml, ~x"//NextContinuationToken/text()"s) + + projects = + prefixes + |> Enum.map(&strip_project_path/1) + |> Enum.reject(&(&1 == "")) + + next = if truncated == "true" and next_token != "", do: next_token + + {projects, next} + end + + defp strip_project_path(prefix) do + prefix + |> String.replace_prefix(@projects_prefix, "") + |> String.trim_trailing("/") + end + + @doc """ + Enumerates tiles for a project using the configured parser module. + + Tests inject a fake parser via `Application.put_env(:towerops, :lidar_3dep_parser, ...)`. + Production uses `Towerops.Lidar.Sources.ThreeDep.GdalParser` (default). + """ + @spec enumerate_tiles(project_name(), keyword()) :: {:ok, [tile()]} | {:error, term()} + def enumerate_tiles(project_name, opts \\ []) do + parser = Application.get_env(:towerops, :lidar_3dep_parser, __MODULE__.GdalParser) + parser.enumerate(project_name, opts) + end +end diff --git a/lib/towerops/lidar/sources/three_dep/gdal_parser.ex b/lib/towerops/lidar/sources/three_dep/gdal_parser.ex new file mode 100644 index 00000000..340053b5 --- /dev/null +++ b/lib/towerops/lidar/sources/three_dep/gdal_parser.ex @@ -0,0 +1,150 @@ +defmodule Towerops.Lidar.Sources.ThreeDep.GdalParser do + @moduledoc """ + Default 3DEP manifest parser. + + Each Texas 3DEP project ships with a tile index — typically a GeoPackage + (`.gpkg`) under the `metadata/` prefix. This parser shells out to + `ogr2ogr` to convert the GeoPackage to GeoJSON in stdout, then maps each + feature to a tile row. + + The implementation depends on a runtime `ogr2ogr` binary. Tests inject a + fake parser via `Application.put_env(:towerops, :lidar_3dep_parser, ...)` + to avoid this dependency. + """ + + require Logger + + @s3_host "https://prd-tnm.s3.amazonaws.com" + + @doc """ + Returns `{:ok, [tile_map]}` for the project, or `{:error, reason}`. + + Tile map shape: + + %{ + tile_name: String.t(), + url: String.t(), + footprint_wkt: String.t(), + resolution_m: Decimal.t(), + year: integer() | nil, + crs: String.t() | nil, + file_size_bytes: integer() | nil + } + """ + def enumerate(project_name, opts \\ []) do + manifest_url = manifest_url(project_name) + ogr_bin = Keyword.get(opts, :ogr_bin, Application.get_env(:towerops, :ogr2ogr_bin, "ogr2ogr")) + + if executable_available?(ogr_bin) do + run_ogr2ogr(ogr_bin, manifest_url, project_name) + else + {:error, :ogr2ogr_not_found} + end + end + + defp executable_available?(bin) do + if Application.get_env(:towerops, :lidar_ogr_runner) do + true + else + System.find_executable(bin) != nil + end + end + + defp manifest_url(project_name) do + # Convention: metadata GeoPackage lives at metadata//.gpkg. + # Some older projects use spatial_metadata/ — the parser falls back if the + # primary URL 404s (handled at the catalog layer, not here). + "/vsicurl/#{@s3_host}/StagedProducts/Elevation/metadata/#{project_name}/#{project_name}.gpkg" + end + + defp run_ogr2ogr(ogr_bin, manifest_url, project_name) do + args = ["-f", "GeoJSON", "-t_srs", "EPSG:4326", "/vsistdout/", manifest_url] + runner = Application.get_env(:towerops, :lidar_ogr_runner, &System.cmd/3) + + case runner.(ogr_bin, args, stderr_to_stdout: true) do + {output, 0} -> + decode_geojson(output, project_name) + + {output, code} -> + Logger.warning("ogr2ogr failed (#{code}) for #{project_name}: #{output}") + {:error, {:ogr2ogr, code, output}} + end + end + + @doc """ + Decodes a GeoJSON FeatureCollection (as text) into tile maps. Public for + testability — exercised directly without invoking `ogr2ogr`. + """ + @spec decode_geojson(String.t(), String.t()) :: {:ok, list(map())} | {:error, term()} + def decode_geojson(json, project_name) do + case Jason.decode(json) do + {:ok, %{"features" => features}} -> + tiles = + features + |> Enum.map(&feature_to_tile(&1, project_name)) + |> Enum.reject(&is_nil/1) + + {:ok, tiles} + + {:ok, _other} -> + {:error, :no_features} + + {:error, reason} -> + {:error, {:json_decode, reason}} + end + end + + defp feature_to_tile(%{"properties" => props, "geometry" => geom}, project_name) when is_map(props) and is_map(geom) do + tile_name = first_present(props, ["tile_id", "TILE_ID", "filename"]) + url = first_present(props, ["download_url", "DOWNLOAD_URL"]) || derive_tile_url(project_name, tile_name) + + build_tile(tile_name, url, geom, props, project_name) + end + + defp feature_to_tile(_, _), do: nil + + defp first_present(_props, []), do: nil + + defp first_present(props, [key | rest]) do + case Map.get(props, key) do + nil -> first_present(props, rest) + "" -> first_present(props, rest) + value -> value + end + end + + defp build_tile(nil, _url, _geom, _props, _project_name), do: nil + defp build_tile(_tile_name, nil, _geom, _props, _project_name), do: nil + + defp build_tile(tile_name, url, geom, props, project_name) do + %{ + tile_name: to_string(tile_name), + url: to_string(url), + footprint_wkt: geometry_to_wkt(geom), + resolution_m: Decimal.new("1.0"), + year: extract_year(project_name), + crs: props["crs"], + file_size_bytes: props["file_size"] + } + end + + defp derive_tile_url(project_name, tile_name) when is_binary(tile_name) do + "#{@s3_host}/StagedProducts/Elevation/1m/Projects/#{project_name}/TIFF/#{tile_name}" + end + + defp derive_tile_url(_, _), do: nil + + defp geometry_to_wkt(%{"type" => "Polygon", "coordinates" => [ring | _]}) do + points = Enum.map_join(ring, ", ", fn [lon, lat | _] -> "#{lon} #{lat}" end) + "POLYGON((#{points}))" + end + + defp geometry_to_wkt(_), do: nil + + defp extract_year(project_name) do + case Regex.run(~r/(\d{4})/, project_name) do + [_, year] -> String.to_integer(year) + _ -> nil + end + end +end diff --git a/lib/towerops/lidar/sources/tnris.ex b/lib/towerops/lidar/sources/tnris.ex new file mode 100644 index 00000000..8ecfeb9b --- /dev/null +++ b/lib/towerops/lidar/sources/tnris.ex @@ -0,0 +1,74 @@ +defmodule Towerops.Lidar.Sources.Tnris do + @moduledoc """ + TNRIS (TxGIO) catalog discovery — used as a fallback for areas not yet + covered by USGS 3DEP. + + Only DEM/COG-like resources are surfaced; raw LAZ point clouds are + filtered out. + """ + + alias Towerops.HTTP + + @api_base "https://api.tnris.org/api/v1/resources" + + @type tile :: %{ + tile_name: String.t(), + url: String.t(), + footprint_wkt: String.t(), + resolution_m: Decimal.t(), + year: integer() | nil, + crs: String.t() | nil, + file_size_bytes: integer() | nil + } + + @doc """ + Fetches the LIDAR collection resources from TNRIS, filters to DEM-style + entries, and converts each to a tile map. + """ + @spec list_dem_resources() :: {:ok, [tile()]} | {:error, term()} + def list_dem_resources do + case HTTP.get(__MODULE__, url: @api_base, params: [{"collection", "lidar"}]) do + {:ok, %Req.Response{status: 200, body: body}} -> + {:ok, parse_resources(body)} + + {:ok, %Req.Response{status: status, body: body}} -> + {:error, {:http, status, body}} + + {:error, reason} -> + {:error, reason} + end + end + + defp parse_resources(%{"results" => results}) when is_list(results) do + results + |> Enum.filter(&dem_resource?/1) + |> Enum.map(&resource_to_tile/1) + |> Enum.reject(&is_nil/1) + end + + defp parse_resources(_), do: [] + + defp dem_resource?(%{"resource_type" => type}) when is_binary(type) do + String.upcase(type) =~ "DEM" + end + + defp dem_resource?(_), do: false + + defp resource_to_tile(%{"name" => name, "download_url" => url, "bbox" => [w, s, e, n]} = r) do + %{ + tile_name: to_string(name), + url: to_string(url), + footprint_wkt: bbox_to_wkt(w, s, e, n), + resolution_m: Decimal.new("1.0"), + year: r["year"], + crs: r["crs"], + file_size_bytes: r["file_size"] + } + end + + defp resource_to_tile(_), do: nil + + defp bbox_to_wkt(w, s, e, n) do + "POLYGON((#{w} #{s}, #{e} #{s}, #{e} #{n}, #{w} #{n}, #{w} #{s}))" + end +end diff --git a/lib/towerops/lidar/sync_log.ex b/lib/towerops/lidar/sync_log.ex new file mode 100644 index 00000000..ac3ac967 --- /dev/null +++ b/lib/towerops/lidar/sync_log.ex @@ -0,0 +1,39 @@ +defmodule Towerops.Lidar.SyncLog do + @moduledoc """ + Audit log for LIDAR catalog sync operations. + """ + use Ecto.Schema + + import Ecto.Changeset + + @primary_key {:id, :binary_id, autogenerate: true} + + @valid_statuses ~w(success partial failed) + @valid_sources ~w(3dep tnris) + + schema "lidar_sync_logs" do + field :source, :string + field :status, :string + field :projects_synced, :integer, default: 0 + field :tiles_upserted, :integer, default: 0 + field :errors, :map + field :duration_ms, :integer + field :inserted_at, :utc_datetime + end + + def changeset(sync_log, attrs) do + sync_log + |> cast(attrs, [ + :source, + :status, + :projects_synced, + :tiles_upserted, + :errors, + :duration_ms, + :inserted_at + ]) + |> validate_required([:source, :status, :inserted_at]) + |> validate_inclusion(:source, @valid_sources) + |> validate_inclusion(:status, @valid_statuses) + end +end diff --git a/lib/towerops/lidar/tile.ex b/lib/towerops/lidar/tile.ex new file mode 100644 index 00000000..4f7dde90 --- /dev/null +++ b/lib/towerops/lidar/tile.ex @@ -0,0 +1,60 @@ +defmodule Towerops.Lidar.Tile do + @moduledoc """ + A single LIDAR-derived DEM tile in the catalog. + + We do not store raster bytes — only the public HTTPS URL and footprint + metadata. Reads happen on-demand via GDAL's `/vsicurl/` driver. + """ + use Ecto.Schema + + import Ecto.Changeset + + @primary_key {:id, :binary_id, autogenerate: true} + @foreign_key_type :binary_id + + @valid_sources ~w(3dep tnris) + @valid_availability ~w(available missing error no_cog) + + schema "lidar_tiles" do + field :source, :string + field :project_name, :string + field :tile_name, :string + field :year, :integer + field :resolution_m, :decimal + field :url, :string + field :crs, :string + field :file_size_bytes, :integer + field :metadata, :map, default: %{} + field :last_verified_at, :utc_datetime + field :availability, :string, default: "available" + field :footprint, Geo.PostGIS.Geometry + + timestamps(type: :utc_datetime) + end + + @castable [ + :source, + :project_name, + :tile_name, + :year, + :resolution_m, + :url, + :crs, + :file_size_bytes, + :metadata, + :last_verified_at, + :availability, + :footprint, + :inserted_at, + :updated_at + ] + + def changeset(tile, attrs) do + tile + |> cast(attrs, @castable) + |> validate_required([:source, :project_name, :tile_name, :url, :footprint]) + |> validate_inclusion(:source, @valid_sources) + |> validate_inclusion(:availability, @valid_availability) + |> unique_constraint([:source, :project_name, :tile_name]) + end +end diff --git a/lib/towerops/postgrex_types.ex b/lib/towerops/postgrex_types.ex new file mode 100644 index 00000000..516917b6 --- /dev/null +++ b/lib/towerops/postgrex_types.ex @@ -0,0 +1,5 @@ +Postgrex.Types.define( + Towerops.PostgrexTypes, + [Geo.PostGIS.Extension] ++ Ecto.Adapters.Postgres.extensions(), + json: Jason +) diff --git a/lib/towerops/workers/lidar_catalog_sync_worker.ex b/lib/towerops/workers/lidar_catalog_sync_worker.ex new file mode 100644 index 00000000..bfa2302f --- /dev/null +++ b/lib/towerops/workers/lidar_catalog_sync_worker.ex @@ -0,0 +1,39 @@ +defmodule Towerops.Workers.LidarCatalogSyncWorker do + @moduledoc """ + Oban worker that syncs the LIDAR tile catalog from public sources. + + Runs monthly via cron. Discovers Texas LIDAR projects on USGS 3DEP and + upserts tile metadata into `lidar_tiles`. Does not download raster data. + """ + use Oban.Worker, + queue: :lidar, + unique: [period: 86_400, states: [:available, :scheduled, :executing]] + + alias Towerops.Lidar.Catalog + + require Logger + + @impl Oban.Worker + def perform(%Oban.Job{}) do + case run_sync() do + {:ok, summary} -> + Logger.info( + "Lidar catalog sync: #{summary.tiles_upserted} tiles across " <> + "#{summary.projects_synced} projects in #{summary.duration_ms}ms" + ) + + :ok + + {:error, reason} = error -> + Logger.error("Lidar catalog sync failed: #{inspect(reason)}") + error + end + end + + defp run_sync do + case Application.get_env(:towerops, :lidar_catalog_runner) do + nil -> Catalog.sync_3dep() + fun when is_function(fun, 0) -> fun.() + end + end +end diff --git a/mix.exs b/mix.exs index 29528d0b..42d2e20c 100644 --- a/mix.exs +++ b/mix.exs @@ -107,6 +107,7 @@ defmodule Towerops.MixProject do {:mix_audit, "~> 2.1", only: [:dev, :test], runtime: false}, {:inet_cidr, "~> 1.0"}, {:cloak_ecto, "~> 1.3"}, + {:geo_postgis, "~> 3.7"}, {:logger_file_backend, "~> 0.0.13", only: :dev}, {:logger_backends, "~> 1.0", only: :dev}, {:tidewave, "~> 0.5", only: :dev} diff --git a/mix.lock b/mix.lock index c39fffd1..7fa549be 100644 --- a/mix.lock +++ b/mix.lock @@ -32,6 +32,8 @@ "finch": {:hex, :finch, "0.21.0", "b1c3b2d48af02d0c66d2a9ebfb5622be5c5ecd62937cf79a88a7f98d48a8290c", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "87dc6e169794cb2570f75841a19da99cfde834249568f2a5b121b809588a4377"}, "fine": {:hex, :fine, "0.1.6", "4bf7151493443c454aac9f2fa2f34f5fefd0346a83fb5586a016c4a135c63247", [:mix], [], "hexpm", "5638eb4495488e885ebec167fa57973e5c35e1a50c344eb7666c90ec1c4e3b12"}, "gen_smtp": {:hex, :gen_smtp, "1.3.0", "62c3d91f0dcf6ce9db71bcb6881d7ad0d1d834c7f38c13fa8e952f4104a8442e", [:rebar3], [{:ranch, ">= 1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "0b73fbf069864ecbce02fe653b16d3f35fd889d0fdd4e14527675565c39d84e6"}, + "geo": {:hex, :geo, "4.1.0", "64ba89a64cc400b5b16dd2f5bd644cb141776eb8c2ac5a983332c8d944936c12", [:mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "19edb2b3398ca9f701b573b1fb11bc90951ebd64f18b06bd1bf35abe509a2934"}, + "geo_postgis": {:hex, :geo_postgis, "3.7.1", "614f25b42334a615bd54bb09c22030b1aac7bac8f829bd823ab1faccf093a324", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:geo, "~> 3.6 or ~> 4.0", [hex: :geo, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0 or ~> 4.0 or ~> 5.0 or ~> 6.0", [hex: :poison, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: false]}], "hexpm", "c20d823c600d35b7fe9ddd5be03052bb7136c57d6f1775dbd46871545e405280"}, "gettext": {:hex, :gettext, "1.0.2", "5457e1fd3f4abe47b0e13ff85086aabae760497a3497909b8473e0acee57673b", [:mix], [{:expo, "~> 0.5.1 or ~> 1.0", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "eab805501886802071ad290714515c8c4a17196ea76e5afc9d06ca85fb1bfeb3"}, "hackney": {:hex, :hackney, "1.25.0", "390e9b83f31e5b325b9f43b76e1a785cbdb69b5b6cd4e079aa67835ded046867", [:rebar3], [{:certifi, "~> 2.15.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.4", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "7209bfd75fd1f42467211ff8f59ea74d6f2a9e81cbcee95a56711ee79fd6b1d4"}, "heroicons": {:git, "https://github.com/tailwindlabs/heroicons.git", "0435d4ca364a608cc75e2f8683d374e55abbae26", [tag: "v2.2.0", sparse: "optimized", depth: 1]}, diff --git a/priv/repo/migrations/20260504123340_enable_postgis.exs b/priv/repo/migrations/20260504123340_enable_postgis.exs new file mode 100644 index 00000000..051da333 --- /dev/null +++ b/priv/repo/migrations/20260504123340_enable_postgis.exs @@ -0,0 +1,11 @@ +defmodule Towerops.Repo.Migrations.EnablePostgis do + use Ecto.Migration + + def up do + execute "CREATE EXTENSION IF NOT EXISTS postgis" + end + + def down do + execute "DROP EXTENSION IF EXISTS postgis" + end +end diff --git a/priv/repo/migrations/20260504123341_create_lidar_tiles.exs b/priv/repo/migrations/20260504123341_create_lidar_tiles.exs new file mode 100644 index 00000000..ef45b6e2 --- /dev/null +++ b/priv/repo/migrations/20260504123341_create_lidar_tiles.exs @@ -0,0 +1,46 @@ +defmodule Towerops.Repo.Migrations.CreateLidarTiles do + use Ecto.Migration + + def change do + create table(:lidar_tiles, primary_key: false) do + add :id, :binary_id, primary_key: true + add :source, :string, null: false + add :project_name, :string, null: false + add :tile_name, :string, null: false + add :year, :integer + add :resolution_m, :decimal, precision: 6, scale: 3 + add :url, :text, null: false + add :crs, :string + add :file_size_bytes, :bigint + add :metadata, :map, default: %{} + add :last_verified_at, :utc_datetime + add :availability, :string, null: false, default: "available" + + timestamps(type: :utc_datetime) + end + + # PostGIS geometry column. AddGeometryColumn registers the column in + # geometry_columns metadata and applies SRID/type constraints. + execute( + """ + SELECT AddGeometryColumn('lidar_tiles', 'footprint', 4326, 'POLYGON', 2) + """, + "SELECT DropGeometryColumn('lidar_tiles', 'footprint')" + ) + + execute( + "ALTER TABLE lidar_tiles ALTER COLUMN footprint SET NOT NULL", + "ALTER TABLE lidar_tiles ALTER COLUMN footprint DROP NOT NULL" + ) + + create unique_index(:lidar_tiles, [:source, :project_name, :tile_name]) + create index(:lidar_tiles, [:year]) + create index(:lidar_tiles, [:availability]) + + # GiST index on footprint for fast spatial intersection queries. + execute( + "CREATE INDEX lidar_tiles_footprint_gist ON lidar_tiles USING gist (footprint)", + "DROP INDEX lidar_tiles_footprint_gist" + ) + end +end diff --git a/priv/repo/migrations/20260504123342_create_lidar_sync_logs.exs b/priv/repo/migrations/20260504123342_create_lidar_sync_logs.exs new file mode 100644 index 00000000..88c8d521 --- /dev/null +++ b/priv/repo/migrations/20260504123342_create_lidar_sync_logs.exs @@ -0,0 +1,18 @@ +defmodule Towerops.Repo.Migrations.CreateLidarSyncLogs do + use Ecto.Migration + + def change do + create table(:lidar_sync_logs, primary_key: false) do + add :id, :binary_id, primary_key: true + add :source, :string, null: false + add :status, :string, null: false + add :projects_synced, :integer, default: 0 + add :tiles_upserted, :integer, default: 0 + add :errors, :map + add :duration_ms, :integer + add :inserted_at, :utc_datetime, null: false + end + + create index(:lidar_sync_logs, [:source, :inserted_at]) + end +end diff --git a/test/support/lidar_fake_parser.ex b/test/support/lidar_fake_parser.ex new file mode 100644 index 00000000..9acdb675 --- /dev/null +++ b/test/support/lidar_fake_parser.ex @@ -0,0 +1,39 @@ +defmodule Towerops.Lidar.Test.FakeParser do + @moduledoc """ + Test-only stand-in for `Towerops.Lidar.Sources.ThreeDep.GdalParser`. + + Tests configure canned tile responses per project name via `put_state/1`, + which are stored in an ETS table so the parser process can be the test + process or any Oban-spawned worker. + """ + + @table :lidar_catalog_fake_parser + + def put_state(map) do + ensure_table() + :ets.insert(@table, {:state, map}) + end + + def enumerate(project_name, _opts) do + ensure_table() + + case :ets.lookup(@table, :state) do + [{:state, map}] -> + case Map.fetch(map, project_name) do + {:ok, {:error, _} = err} -> err + {:ok, tiles} -> {:ok, tiles} + :error -> {:error, :unknown_project} + end + + _ -> + {:error, :no_state} + end + end + + defp ensure_table do + case :ets.info(@table) do + :undefined -> :ets.new(@table, [:named_table, :public, :set]) + _ -> @table + end + end +end diff --git a/test/towerops/lidar/catalog_test.exs b/test/towerops/lidar/catalog_test.exs new file mode 100644 index 00000000..ad20924d --- /dev/null +++ b/test/towerops/lidar/catalog_test.exs @@ -0,0 +1,293 @@ +defmodule Towerops.Lidar.CatalogTest do + use Towerops.DataCase + + alias Towerops.Lidar.Catalog + alias Towerops.Lidar.Sources.ThreeDep + alias Towerops.Lidar.Sources.Tnris + alias Towerops.Lidar.SyncLog + alias Towerops.Lidar.Test.FakeParser + alias Towerops.Lidar.Tile + alias Towerops.Repo + + setup do + on_exit(fn -> + Application.delete_env(:towerops, :lidar_3dep_parser) + end) + + :ok + end + + describe "sync_3dep/0" do + test "upserts tiles for each project and records a SyncLog with status :success" do + stub_projects(["TX_Foo_2024"]) + stub_parser(%{"TX_Foo_2024" => sample_tiles(2, "TX_Foo_2024")}) + + assert {:ok, summary} = Catalog.sync_3dep() + assert summary.projects_synced == 1 + assert summary.tiles_upserted == 2 + assert summary.errors == %{} + + tiles = Repo.all(Tile) + assert length(tiles) == 2 + assert Enum.all?(tiles, &(&1.source == "3dep")) + assert Enum.all?(tiles, &(&1.project_name == "TX_Foo_2024")) + + [log] = Repo.all(SyncLog) + assert log.source == "3dep" + assert log.status == "success" + assert log.projects_synced == 1 + assert log.tiles_upserted == 2 + assert is_integer(log.duration_ms) + end + + test "is idempotent: re-running with the same data does not duplicate rows" do + stub_projects(["TX_Foo_2024"]) + stub_parser(%{"TX_Foo_2024" => sample_tiles(3, "TX_Foo_2024")}) + + assert {:ok, _} = Catalog.sync_3dep() + assert Repo.aggregate(Tile, :count) == 3 + + assert {:ok, _} = Catalog.sync_3dep() + assert Repo.aggregate(Tile, :count) == 3 + end + + test "records SyncLog with status :partial when one project fails" do + stub_projects(["TX_Good_2024", "TX_Bad_2018"]) + + stub_parser(%{ + "TX_Good_2024" => sample_tiles(1, "TX_Good_2024"), + "TX_Bad_2018" => {:error, :ogr2ogr_not_found} + }) + + assert {:ok, summary} = Catalog.sync_3dep() + assert summary.projects_synced == 1 + assert summary.tiles_upserted == 1 + assert Map.has_key?(summary.errors, "TX_Bad_2018") + + [log] = Repo.all(SyncLog) + assert log.status == "partial" + end + + test "records SyncLog with status :failed when project listing fails" do + stub_projects({:error, {:http, 500, "boom"}}) + + assert {:error, _} = Catalog.sync_3dep() + + [log] = Repo.all(SyncLog) + assert log.status == "failed" + assert log.projects_synced == 0 + assert log.tiles_upserted == 0 + end + end + + describe "sync_tnris/0" do + test "upserts TNRIS tiles not already covered by 3DEP" do + Req.Test.stub(Tnris, fn conn -> + conn + |> Plug.Conn.put_resp_content_type("application/json") + |> Plug.Conn.send_resp( + 200, + Jason.encode!(%{ + "results" => [ + %{ + "resource_type" => "DEM", + "name" => "tnris_tile_1.tif", + "year" => 2018, + "download_url" => "https://tnris.example.com/t1.tif", + "bbox" => [-100.0, 32.0, -99.9, 32.1] + } + ] + }) + ) + end) + + assert {:ok, summary} = Catalog.sync_tnris() + assert summary.tiles_upserted == 1 + + [tile] = Repo.all(Tile) + assert tile.source == "tnris" + assert tile.tile_name == "tnris_tile_1.tif" + end + + test "skips TNRIS tiles whose centroid is contained by an existing 3DEP tile" do + _existing = insert_tile!(%{source: "3dep"}) + + Req.Test.stub(Tnris, fn conn -> + # bbox centroid is well inside the 3DEP tile fixture footprint + # (the helper insert_tile! places 3DEP at -97.7..-97.8, 30.2..30.3) + conn + |> Plug.Conn.put_resp_content_type("application/json") + |> Plug.Conn.send_resp( + 200, + Jason.encode!(%{ + "results" => [ + %{ + "resource_type" => "DEM", + "name" => "tnris_overlap.tif", + "year" => 2018, + "download_url" => "https://tnris.example.com/t1.tif", + "bbox" => [-97.78, 30.22, -97.72, 30.28] + } + ] + }) + ) + end) + + assert {:ok, summary} = Catalog.sync_tnris() + assert summary.tiles_upserted == 0 + assert summary.errors["skipped_covered"] == 1 + end + + test "records SyncLog with status :failed when listing fails" do + Req.Test.stub(Tnris, fn conn -> + Plug.Conn.send_resp(conn, 503, "down") + end) + + assert {:error, _} = Catalog.sync_tnris() + + [log] = Repo.all(SyncLog) + assert log.source == "tnris" + assert log.status == "failed" + end + end + + describe "verify_tile/1" do + test "sets last_verified_at on a 200 response" do + tile = insert_tile!() + + Req.Test.stub(Catalog, fn conn -> + assert conn.method == "HEAD" + Plug.Conn.send_resp(conn, 200, "") + end) + + assert {:ok, %Tile{} = updated} = Catalog.verify_tile(tile) + assert updated.availability == "available" + assert updated.last_verified_at + end + + test "flips availability to \"missing\" on 404" do + tile = insert_tile!() + + Req.Test.stub(Catalog, fn conn -> + Plug.Conn.send_resp(conn, 404, "") + end) + + assert {:ok, %Tile{} = updated} = Catalog.verify_tile(tile) + assert updated.availability == "missing" + assert updated.last_verified_at + end + + test "flips availability to \"error\" on transport failures" do + tile = insert_tile!() + + Req.Test.stub(Catalog, fn conn -> + Plug.Conn.send_resp(conn, 503, "") + end) + + assert {:ok, %Tile{} = updated} = Catalog.verify_tile(tile) + assert updated.availability == "error" + end + + test "flips availability to \"error\" when no stub is configured (transport-failure path)" do + tile = insert_tile!() + # No Req.Test.stub — HTTP returns {:error, :no_test_stub} + assert {:ok, %Tile{availability: "error"}} = Catalog.verify_tile(tile) + end + end + + describe "sync_3dep/0 with default lister (no app config override)" do + test "calls ThreeDep.list_texas_projects via Req.Test stub" do + Req.Test.stub(ThreeDep, fn conn -> + Plug.Conn.send_resp( + conn, + 200, + """ + + + false + + StagedProducts/Elevation/1m/Projects/TX_Default_2024/ + + + """ + ) + end) + + stub_parser(%{"TX_Default_2024" => sample_tiles(1, "TX_Default_2024")}) + + assert {:ok, summary} = Catalog.sync_3dep() + assert summary.tiles_upserted == 1 + end + end + + # ---------- helpers ---------- + + defp insert_tile!(attrs \\ %{}) do + poly = %Geo.Polygon{ + coordinates: [ + [ + {-97.8, 30.2}, + {-97.7, 30.2}, + {-97.7, 30.3}, + {-97.8, 30.3}, + {-97.8, 30.2} + ] + ], + srid: 4326 + } + + defaults = %{ + source: "3dep", + project_name: "TX_Test_2024", + tile_name: "tile_#{System.unique_integer([:positive])}", + year: 2024, + resolution_m: Decimal.new("1.0"), + url: "https://prd-tnm.s3.amazonaws.com/example.tif", + crs: "EPSG:6343", + footprint: poly, + availability: "available", + metadata: %{}, + inserted_at: DateTime.truncate(DateTime.utc_now(), :second), + updated_at: DateTime.truncate(DateTime.utc_now(), :second) + } + + %Tile{} + |> Tile.changeset(Map.merge(defaults, attrs)) + |> Repo.insert!() + end + + defp sample_tiles(count, project) do + Enum.map(1..count, fn idx -> + %{ + tile_name: "#{project}_tile_#{idx}", + url: "https://prd-tnm.s3.amazonaws.com/#{project}/tile_#{idx}.tif", + footprint_wkt: + "POLYGON((-97.#{idx} 30.2, -97.#{idx + 1} 30.2, -97.#{idx + 1} 30.3, -97.#{idx} 30.3, -97.#{idx} 30.2))", + resolution_m: Decimal.new("1.0"), + year: 2024, + crs: "EPSG:6343", + file_size_bytes: 1_000_000 + } + end) + end + + defp stub_projects({:error, _} = err) do + test_pid = self() + Application.put_env(:towerops, :lidar_3dep_lister, fn -> send(test_pid, :listed) && err end) + on_exit(fn -> Application.delete_env(:towerops, :lidar_3dep_lister) end) + end + + defp stub_projects(projects) when is_list(projects) do + Application.put_env(:towerops, :lidar_3dep_lister, fn -> {:ok, projects} end) + on_exit(fn -> Application.delete_env(:towerops, :lidar_3dep_lister) end) + + # Some tests may exercise the real ThreeDep.list_texas_projects path through + # Req.Test stubs; we don't need that here, but defensively no-op. + Req.Test.stub(ThreeDep, fn conn -> Plug.Conn.send_resp(conn, 500, "") end) + end + + defp stub_parser(map) do + FakeParser.put_state(map) + Application.put_env(:towerops, :lidar_3dep_parser, FakeParser) + end +end diff --git a/test/towerops/lidar/grid_test.exs b/test/towerops/lidar/grid_test.exs new file mode 100644 index 00000000..1a124f52 --- /dev/null +++ b/test/towerops/lidar/grid_test.exs @@ -0,0 +1,152 @@ +defmodule Towerops.Lidar.GridTest do + use Towerops.DataCase, async: false + + alias Towerops.Lidar + alias Towerops.Lidar.Tile + alias Towerops.Repo + + setup do + on_exit(fn -> Application.delete_env(:towerops, :lidar_runner) end) + :ok + end + + describe "get_elevation_grid/3" do + test "returns :no_tile when no catalog tile intersects the bbox" do + assert {:error, :no_tile} = + Lidar.get_elevation_grid({-97.8, 30.2, -97.7, 30.3}, 0.01) + end + + test "returns :grid_too_large when requested cells exceed the cap" do + _ = insert_tile!() + + # 1 degree x 1 degree at 0.0001 = ~10000x10000 = 100M cells + bbox = {-98.0, 30.0, -97.0, 31.0} + assert {:error, :grid_too_large} = Lidar.get_elevation_grid(bbox, 0.0001) + end + + test "parses an AAIGrid response from a single tile" do + _ = insert_tile!() + + stub_grid_runner(""" + ncols 3 + nrows 2 + xllcorner -97.8 + yllcorner 30.2 + cellsize 0.05 + NODATA_value -9999 + 100.0 101.0 102.0 + 103.0 104.0 105.0 + """) + + assert {:ok, grid} = + Lidar.get_elevation_grid({-97.8, 30.2, -97.65, 30.3}, 0.05) + + assert grid.nrows == 2 + assert grid.ncols == 3 + assert grid.cells == [[100.0, 101.0, 102.0], [103.0, 104.0, 105.0]] + end + + test "mosaics across two tiles by filling nodata cells from the next-best tile" do + _ = insert_tile!(%{tile_name: "best", resolution_m: Decimal.new("1.0")}) + _ = insert_tile!(%{tile_name: "fallback", resolution_m: Decimal.new("3.0")}) + + 1 |> :counters.new([:atomics]) |> tap(&Process.put(:c, &1)) + + Application.put_env(:towerops, :lidar_runner, fn _cmd, args, _opts -> + c = Process.get(:c) + :counters.add(c, 1, 1) + n = :counters.get(c, 1) + + # Detect which tile is being read by the URL position in args + if String.contains?(Enum.join(args, " "), "best") or n == 1 do + {grid_aai([[100.0, -9999], [-9999, 105.0]]), 0} + else + {grid_aai([[200.0, 201.0], [202.0, 203.0]]), 0} + end + end) + + assert {:ok, grid} = + Lidar.get_elevation_grid({-97.8, 30.2, -97.7, 30.3}, 0.05) + + # Best tile contributes (0,0) and (1,1); fallback fills (0,1) and (1,0) + assert grid.cells == [[100.0, 201.0], [202.0, 105.0]] + end + + test "returns :nodata when all tiles return only nodata" do + _ = insert_tile!() + stub_grid_runner(grid_aai([[-9999.0, -9999.0], [-9999.0, -9999.0]])) + + assert {:error, :nodata} = + Lidar.get_elevation_grid({-97.8, 30.2, -97.7, 30.3}, 0.05) + end + + test "propagates GDAL errors" do + _ = insert_tile!() + + Application.put_env(:towerops, :lidar_runner, fn _cmd, _args, _opts -> + {"network unreachable", 1} + end) + + assert {:error, {:gdal, 1, "network unreachable"}} = + Lidar.get_elevation_grid({-97.8, 30.2, -97.7, 30.3}, 0.05) + end + end + + defp insert_tile!(attrs \\ %{}) do + poly = %Geo.Polygon{ + coordinates: [ + [ + {-97.9, 30.1}, + {-97.6, 30.1}, + {-97.6, 30.4}, + {-97.9, 30.4}, + {-97.9, 30.1} + ] + ], + srid: 4326 + } + + defaults = %{ + source: "3dep", + project_name: "TX_Test_2024", + tile_name: "tile_#{System.unique_integer([:positive])}", + year: 2024, + resolution_m: Decimal.new("1.0"), + url: "https://prd-tnm.s3.amazonaws.com/example.tif", + crs: "EPSG:6343", + footprint: poly, + availability: "available", + metadata: %{}, + inserted_at: DateTime.truncate(DateTime.utc_now(), :second), + updated_at: DateTime.truncate(DateTime.utc_now(), :second) + } + + %Tile{} + |> Tile.changeset(Map.merge(defaults, attrs)) + |> Repo.insert!() + end + + defp stub_grid_runner(output) do + Application.put_env(:towerops, :lidar_runner, fn _cmd, _args, _opts -> + {output, 0} + end) + end + + defp grid_aai(rows) do + ncols = length(hd(rows)) + nrows = length(rows) + + body = + Enum.map_join(rows, "\n", fn row -> Enum.map_join(row, " ", &to_string/1) end) + + """ + ncols #{ncols} + nrows #{nrows} + xllcorner -97.8 + yllcorner 30.2 + cellsize 0.05 + NODATA_value -9999 + #{body} + """ + end +end diff --git a/test/towerops/lidar/reader_test.exs b/test/towerops/lidar/reader_test.exs new file mode 100644 index 00000000..1d1e5ee5 --- /dev/null +++ b/test/towerops/lidar/reader_test.exs @@ -0,0 +1,218 @@ +defmodule Towerops.Lidar.ReaderTest do + use ExUnit.Case, async: false + + alias Towerops.Lidar.Reader + alias Towerops.Lidar.Tile + + setup do + on_exit(fn -> Application.delete_env(:towerops, :lidar_runner) end) + :ok + end + + describe "elevation_at/3" do + test "returns the parsed float on success" do + stub_runner(fn cmd, args, _opts -> + assert cmd == "gdallocationinfo" + assert "-wgs84" in args + assert "-valonly" in args + assert "/vsicurl/https://example.com/tile.tif" in args + # Args should also include lon then lat at the end + [lon_str, lat_str] = Enum.take(args, -2) + assert lon_str == "-97.7431" + assert lat_str == "30.2672" + + {"149.32\n", 0} + end) + + assert {:ok, 149.32} = Reader.elevation_at(tile(), 30.2672, -97.7431) + end + + test "returns :nodata when stdout is empty or all whitespace" do + stub_runner(fn _cmd, _args, _opts -> {"\n", 0} end) + assert {:error, :nodata} = Reader.elevation_at(tile(), 30.0, -97.0) + end + + test "returns :nodata when value matches the GDAL nodata sentinel" do + stub_runner(fn _cmd, _args, _opts -> {"-3.4028235e+38\n", 0} end) + assert {:error, :nodata} = Reader.elevation_at(tile(), 30.0, -97.0) + end + + test "returns {:error, {:gdal, code, output}} on non-zero exit" do + stub_runner(fn _cmd, _args, _opts -> {"some error\n", 2} end) + assert {:error, {:gdal, 2, "some error\n"}} = Reader.elevation_at(tile(), 30.0, -97.0) + end + + test "returns :nodata on non-numeric stdout" do + stub_runner(fn _cmd, _args, _opts -> {"junk output\n", 0} end) + assert {:error, :nodata} = Reader.elevation_at(tile(), 30.0, -97.0) + end + end + + describe "healthcheck/0" do + test "returns :ok when the runner returns version output" do + stub_runner(fn "gdallocationinfo", ["--version"], _opts -> + {"GDAL 3.9.0, released 2024/05/10\n", 0} + end) + + assert :ok = Reader.healthcheck() + end + + test "returns {:error, :gdal_missing} when the runner errors" do + stub_runner(fn _cmd, _args, _opts -> {"command not found", 127} end) + assert {:error, :gdal_missing} = Reader.healthcheck() + end + end + + describe "parse_aaigrid/1" do + test "parses a valid grid with NODATA_value" do + grid = """ + ncols 2 + nrows 2 + xllcorner -97.8 + yllcorner 30.2 + cellsize 0.05 + NODATA_value -9999 + 1.0 2.0 + 3.0 4.0 + """ + + assert {:ok, + %{ + ncols: 2, + nrows: 2, + xllcorner: -97.8, + yllcorner: 30.2, + cellsize: 0.05, + nodata_value: -9999.0, + cells: [[1.0, 2.0], [3.0, 4.0]] + }} = Reader.parse_aaigrid(grid) + end + + test "defaults nodata_value to -9999.0 when missing" do + grid = """ + ncols 1 + nrows 1 + xllcorner 0.0 + yllcorner 0.0 + cellsize 1.0 + 42.0 + """ + + assert {:ok, %{nodata_value: -9999.0, cells: [[42.0]]}} = Reader.parse_aaigrid(grid) + end + + test "returns error on missing required header" do + grid = """ + ncols 2 + xllcorner 0.0 + yllcorner 0.0 + cellsize 1.0 + 1.0 2.0 + """ + + assert {:error, {:missing_header, "nrows"}} = Reader.parse_aaigrid(grid) + end + + test "returns error on missing required float header" do + grid = """ + ncols 1 + nrows 1 + xllcorner 0.0 + cellsize 1.0 + 1.0 + """ + + assert {:error, {:missing_header, "yllcorner"}} = Reader.parse_aaigrid(grid) + end + + test "returns error on bad integer header" do + grid = """ + ncols not_a_number + nrows 1 + xllcorner 0.0 + yllcorner 0.0 + cellsize 1.0 + 1.0 + """ + + assert {:error, {:bad_header, "ncols"}} = Reader.parse_aaigrid(grid) + end + + test "returns error on bad float header" do + grid = """ + ncols 1 + nrows 1 + xllcorner not_a_float + yllcorner 0.0 + cellsize 1.0 + 1.0 + """ + + assert {:error, {:bad_header, "xllcorner"}} = Reader.parse_aaigrid(grid) + end + + test "returns error on bad nodata_value" do + grid = """ + ncols 1 + nrows 1 + xllcorner 0.0 + yllcorner 0.0 + cellsize 1.0 + NODATA_value abc + 1.0 + """ + + assert {:error, {:bad_header, "nodata_value"}} = Reader.parse_aaigrid(grid) + end + + test "treats unparseable cell tokens as nil" do + grid = """ + ncols 2 + nrows 1 + xllcorner 0.0 + yllcorner 0.0 + cellsize 1.0 + NODATA_value -9999 + not_a_number 2.0 + """ + + assert {:ok, %{cells: [[nil, 2.0]]}} = Reader.parse_aaigrid(grid) + end + end + + describe "elevation_grid/3" do + test "passes /vsicurl/ paths through without re-prefixing" do + Application.put_env(:towerops, :lidar_runner, fn _cmd, args, _opts -> + url_arg = Enum.find(args, &String.starts_with?(&1, "/vsicurl/")) + # If the input already has the prefix, vsicurl/2 should not double it + refute String.starts_with?(url_arg, "/vsicurl//vsicurl/") + + {""" + ncols 1 + nrows 1 + xllcorner 0.0 + yllcorner 0.0 + cellsize 1.0 + 1.0 + """, 0} + end) + + tile_with_vsi = %Tile{id: Ecto.UUID.generate(), url: "/vsicurl/https://x/y.tif"} + + assert {:ok, %{cells: [[1.0]]}} = + Reader.elevation_grid(tile_with_vsi, {-97.8, 30.2, -97.7, 30.3}, 0.05) + end + end + + defp tile do + %Tile{ + id: Ecto.UUID.generate(), + url: "https://example.com/tile.tif", + crs: "EPSG:4326" + } + end + + defp stub_runner(fun) when is_function(fun, 3) do + Application.put_env(:towerops, :lidar_runner, fun) + end +end diff --git a/test/towerops/lidar/sources/three_dep/gdal_parser_test.exs b/test/towerops/lidar/sources/three_dep/gdal_parser_test.exs new file mode 100644 index 00000000..c27b1481 --- /dev/null +++ b/test/towerops/lidar/sources/three_dep/gdal_parser_test.exs @@ -0,0 +1,235 @@ +defmodule Towerops.Lidar.Sources.ThreeDep.GdalParserTest do + use ExUnit.Case, async: true + + alias Towerops.Lidar.Sources.ThreeDep.GdalParser + + describe "decode_geojson/2" do + test "extracts tile maps from a feature collection" do + json = + Jason.encode!(%{ + "features" => [ + %{ + "properties" => %{ + "tile_id" => "tile_a", + "download_url" => "https://example.com/tile_a.tif", + "crs" => "EPSG:6343", + "file_size" => 9_876_543 + }, + "geometry" => %{ + "type" => "Polygon", + "coordinates" => [ + [[-97.8, 30.2], [-97.7, 30.2], [-97.7, 30.3], [-97.8, 30.3], [-97.8, 30.2]] + ] + } + } + ] + }) + + assert {:ok, [tile]} = GdalParser.decode_geojson(json, "TX_Foo_2024") + assert tile.tile_name == "tile_a" + assert tile.url == "https://example.com/tile_a.tif" + assert tile.year == 2024 + assert tile.crs == "EPSG:6343" + assert tile.file_size_bytes == 9_876_543 + assert tile.footprint_wkt =~ "POLYGON" + assert tile.resolution_m == Decimal.new("1.0") + end + + test "falls back to derived tile URL when download_url is absent" do + json = + Jason.encode!(%{ + "features" => [ + %{ + "properties" => %{"tile_id" => "tile_b"}, + "geometry" => %{ + "type" => "Polygon", + "coordinates" => [[[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 1.0], [0.0, 0.0]]] + } + } + ] + }) + + assert {:ok, [tile]} = GdalParser.decode_geojson(json, "TX_Bar_2019") + assert tile.url =~ "TX_Bar_2019/TIFF/tile_b" + assert tile.year == 2019 + end + + test "tries TILE_ID and filename keys for tile name" do + json = + Jason.encode!(%{ + "features" => [ + %{ + "properties" => %{"TILE_ID" => "upper_case_id"}, + "geometry" => %{"type" => "Polygon", "coordinates" => [[[0.0, 0.0]]]} + }, + %{ + "properties" => %{"filename" => "via_filename"}, + "geometry" => %{"type" => "Polygon", "coordinates" => [[[0.0, 0.0]]]} + } + ] + }) + + assert {:ok, tiles} = GdalParser.decode_geojson(json, "TX_Test_2024") + names = Enum.map(tiles, & &1.tile_name) + assert "upper_case_id" in names + assert "via_filename" in names + end + + test "skips features missing tile name entirely" do + json = + Jason.encode!(%{ + "features" => [ + %{ + "properties" => %{"unknown_key" => "x"}, + "geometry" => %{"type" => "Polygon", "coordinates" => [[[0.0, 0.0]]]} + } + ] + }) + + assert {:ok, []} = GdalParser.decode_geojson(json, "TX_Test_2024") + end + + test "skips features whose properties is empty (no name resolvable)" do + json = + Jason.encode!(%{ + "features" => [ + %{ + "properties" => %{}, + "geometry" => %{"type" => "Polygon", "coordinates" => [[[0.0, 0.0]]]} + } + ] + }) + + assert {:ok, []} = GdalParser.decode_geojson(json, "TX_Test_2024") + end + + test "skips features with empty-string tile names" do + json = + Jason.encode!(%{ + "features" => [ + %{ + "properties" => %{"tile_id" => "", "TILE_ID" => "", "filename" => ""}, + "geometry" => %{"type" => "Polygon", "coordinates" => [[[0.0, 0.0]]]} + } + ] + }) + + assert {:ok, []} = GdalParser.decode_geojson(json, "TX_Test_2024") + end + + test "skips features whose geometry is non-Polygon (no WKT)" do + json = + Jason.encode!(%{ + "features" => [ + %{ + "properties" => %{"tile_id" => "tile_c"}, + "geometry" => %{"type" => "Point", "coordinates" => [0.0, 0.0]} + } + ] + }) + + assert {:ok, [tile]} = GdalParser.decode_geojson(json, "TX_Test_2024") + assert tile.footprint_wkt == nil + end + + test "returns :no_features when payload lacks features key" do + assert {:error, :no_features} = + GdalParser.decode_geojson(Jason.encode!(%{"x" => 1}), "TX_X_2024") + end + + test "returns json_decode error on bad JSON" do + assert {:error, {:json_decode, _}} = GdalParser.decode_geojson("not json", "TX_X_2024") + end + + test "extracts no year when project name has none" do + json = + Jason.encode!(%{ + "features" => [ + %{ + "properties" => %{"tile_id" => "x"}, + "geometry" => %{"type" => "Polygon", "coordinates" => [[[0.0, 0.0]]]} + } + ] + }) + + assert {:ok, [tile]} = GdalParser.decode_geojson(json, "TX_NoYear") + assert tile.year == nil + end + end + + describe "enumerate/2" do + test "returns :ogr2ogr_not_found when binary is missing" do + assert {:error, :ogr2ogr_not_found} = + GdalParser.enumerate("TX_X_2024", + ogr_bin: "definitely-not-a-real-bin-#{System.unique_integer()}" + ) + end + + test "shells out via the injected runner and returns parsed tiles on success" do + Application.put_env(:towerops, :lidar_ogr_runner, fn _bin, args, _opts -> + # Sanity-check the args so any breakage in the constructor surfaces here + assert "-f" in args and "GeoJSON" in args + assert Enum.any?(args, &String.starts_with?(&1, "/vsicurl/")) + + json = + Jason.encode!(%{ + "features" => [ + %{ + "properties" => %{"tile_id" => "tile_x"}, + "geometry" => %{ + "type" => "Polygon", + "coordinates" => [[[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 0.0]]] + } + } + ] + }) + + {json, 0} + end) + + on_exit(fn -> Application.delete_env(:towerops, :lidar_ogr_runner) end) + + assert {:ok, [tile]} = GdalParser.enumerate("TX_Foo_2020") + assert tile.tile_name == "tile_x" + end + + test "returns {:error, {:ogr2ogr, code, output}} on non-zero exit" do + Application.put_env(:towerops, :lidar_ogr_runner, fn _bin, _args, _opts -> + {"failed to read manifest", 2} + end) + + on_exit(fn -> Application.delete_env(:towerops, :lidar_ogr_runner) end) + + assert {:error, {:ogr2ogr, 2, "failed to read manifest"}} = + GdalParser.enumerate("TX_Foo_2020") + end + end + + describe "decode_geojson/2 — defensive paths" do + test "skips features whose properties is non-map (e.g. nil)" do + json = + Jason.encode!(%{ + "features" => [%{"properties" => nil, "geometry" => nil}] + }) + + assert {:ok, []} = GdalParser.decode_geojson(json, "TX_X_2024") + end + + test "skips features where the tile name resolves but URL cannot be derived" do + # tile_id is a number — first_present returns the number (truthy), + # but derive_tile_url returns nil since tile_name isn't a binary, + # exercising the build_tile/_, nil, ...) clause. + json = + Jason.encode!(%{ + "features" => [ + %{ + "properties" => %{"tile_id" => 42}, + "geometry" => %{"type" => "Polygon", "coordinates" => [[[0.0, 0.0]]]} + } + ] + }) + + assert {:ok, []} = GdalParser.decode_geojson(json, "TX_X_2024") + end + end +end diff --git a/test/towerops/lidar/sources/three_dep_test.exs b/test/towerops/lidar/sources/three_dep_test.exs new file mode 100644 index 00000000..55273511 --- /dev/null +++ b/test/towerops/lidar/sources/three_dep_test.exs @@ -0,0 +1,121 @@ +defmodule Towerops.Lidar.Sources.ThreeDepTest do + use ExUnit.Case, async: true + + alias Towerops.Lidar.Sources.ThreeDep + + describe "list_texas_projects/0" do + test "parses CommonPrefixes and returns project names without prefix path" do + Req.Test.stub(ThreeDep, fn conn -> + assert conn.method == "GET" + assert conn.host == "prd-tnm.s3.amazonaws.com" + assert conn.query_string =~ "list-type=2" + assert conn.query_string =~ "prefix=StagedProducts%2FElevation%2F1m%2FProjects%2FTX_" + assert conn.query_string =~ "delimiter=%2F" + + Plug.Conn.send_resp(conn, 200, list_xml_response(["TX_Foo_2018", "TX_Bar_2020"], false)) + end) + + assert {:ok, projects} = ThreeDep.list_texas_projects() + assert projects == ["TX_Foo_2018", "TX_Bar_2020"] + end + + test "follows NextContinuationToken across pages" do + 1 |> :counters.new([:atomics]) |> tap(fn c -> Process.put(:counter, c) end) + + Req.Test.stub(ThreeDep, fn conn -> + c = Process.get(:counter) + :counters.add(c, 1, 1) + count = :counters.get(c, 1) + + if count == 1 do + refute conn.query_string =~ "continuation-token" + Plug.Conn.send_resp(conn, 200, list_xml_response(["TX_A_2018"], true)) + else + assert conn.query_string =~ "continuation-token=NEXT-TOKEN" + Plug.Conn.send_resp(conn, 200, list_xml_response(["TX_B_2020"], false)) + end + end) + + assert {:ok, ["TX_A_2018", "TX_B_2020"]} = ThreeDep.list_texas_projects() + end + + test "returns error tuple on HTTP failure" do + Req.Test.stub(ThreeDep, fn conn -> + Plug.Conn.send_resp(conn, 500, "boom") + end) + + assert {:error, {:http, 500, _}} = ThreeDep.list_texas_projects() + end + + test "returns error tuple when no test stub is configured (transport-failure path)" do + # No Req.Test.stub, so HTTP.request returns {:error, :no_test_stub} — + # exercises the {:error, reason} branch in list_texas_projects/0 + assert {:error, :no_test_stub} = ThreeDep.list_texas_projects() + end + end + + describe "enumerate_tiles/1" do + test "delegates to the configured parser module" do + tiles = [ + %{ + tile_name: "tile1", + url: "https://prd-tnm.s3.amazonaws.com/example/tile1.tif", + footprint_wkt: "POLYGON((-97.9 30.2, -97.7 30.2, -97.7 30.4, -97.9 30.4, -97.9 30.2))", + resolution_m: Decimal.new("1.0"), + year: 2024, + crs: "EPSG:6343", + file_size_bytes: 12_345 + } + ] + + defmodule FakeParser do + @moduledoc false + def enumerate("TX_Test_2024", _opts), + do: + {:ok, + [ + %{ + tile_name: "tile1", + url: "https://prd-tnm.s3.amazonaws.com/example/tile1.tif", + footprint_wkt: "POLYGON((-97.9 30.2, -97.7 30.2, -97.7 30.4, -97.9 30.4, -97.9 30.2))", + resolution_m: Decimal.new("1.0"), + year: 2024, + crs: "EPSG:6343", + file_size_bytes: 12_345 + } + ]} + end + + Application.put_env(:towerops, :lidar_3dep_parser, FakeParser) + on_exit(fn -> Application.delete_env(:towerops, :lidar_3dep_parser) end) + + assert {:ok, ^tiles} = ThreeDep.enumerate_tiles("TX_Test_2024") + end + end + + defp list_xml_response(projects, has_more?) do + prefixes = + Enum.map_join(projects, "", fn p -> + "StagedProducts/Elevation/1m/Projects/#{p}/" + end) + + next_token = + if has_more?, do: "NEXT-TOKEN", else: "" + + truncated = if has_more?, do: "true", else: "false" + + """ + + + prd-tnm + StagedProducts/Elevation/1m/Projects/TX_ + #{length(projects)} + 1000 + / + #{truncated} + #{prefixes} + #{next_token} + + """ + end +end diff --git a/test/towerops/lidar/sources/tnris_test.exs b/test/towerops/lidar/sources/tnris_test.exs new file mode 100644 index 00000000..14899784 --- /dev/null +++ b/test/towerops/lidar/sources/tnris_test.exs @@ -0,0 +1,87 @@ +defmodule Towerops.Lidar.Sources.TnrisTest do + use ExUnit.Case, async: true + + alias Towerops.Lidar.Sources.Tnris + + describe "list_dem_resources/0" do + test "returns parsed DEM resources, skipping LAZ-only entries" do + Req.Test.stub(Tnris, fn conn -> + assert conn.host == "api.tnris.org" + assert conn.request_path == "/api/v1/resources" + assert conn.query_string =~ "collection=lidar" + + conn + |> Plug.Conn.put_resp_content_type("application/json") + |> Plug.Conn.send_resp(200, Jason.encode!(api_response())) + end) + + assert {:ok, [tile]} = Tnris.list_dem_resources() + assert tile.tile_name == "tile_1.tif" + assert tile.url =~ "tile_1.tif" + assert tile.year == 2019 + assert tile.footprint_wkt =~ "POLYGON" + end + + test "returns error tuple on HTTP failure" do + Req.Test.stub(Tnris, fn conn -> + Plug.Conn.send_resp(conn, 503, "down") + end) + + assert {:error, {:http, 503, _}} = Tnris.list_dem_resources() + end + + test "returns error tuple when no test stub is configured (transport-failure path)" do + assert {:error, :no_test_stub} = Tnris.list_dem_resources() + end + + test "returns empty list when body has no results key" do + Req.Test.stub(Tnris, fn conn -> + conn + |> Plug.Conn.put_resp_content_type("application/json") + |> Plug.Conn.send_resp(200, Jason.encode!(%{"unrelated" => "data"})) + end) + + assert {:ok, []} = Tnris.list_dem_resources() + end + + test "skips resources missing resource_type or required fields" do + Req.Test.stub(Tnris, fn conn -> + conn + |> Plug.Conn.put_resp_content_type("application/json") + |> Plug.Conn.send_resp( + 200, + Jason.encode!(%{ + "results" => [ + %{"missing" => "type"}, + %{"resource_type" => 42}, + %{"resource_type" => "DEM", "incomplete" => true} + ] + }) + ) + end) + + assert {:ok, []} = Tnris.list_dem_resources() + end + end + + defp api_response do + %{ + "results" => [ + %{ + "resource_type" => "DEM", + "name" => "tile_1.tif", + "year" => 2019, + "download_url" => "https://example.tnris.org/tile_1.tif", + "bbox" => [-97.8, 30.2, -97.7, 30.3] + }, + %{ + "resource_type" => "LAZ", + "name" => "points.laz", + "year" => 2019, + "download_url" => "https://example.tnris.org/points.laz", + "bbox" => [-97.8, 30.2, -97.7, 30.3] + } + ] + } + end +end diff --git a/test/towerops/lidar_test.exs b/test/towerops/lidar_test.exs new file mode 100644 index 00000000..a7087bdf --- /dev/null +++ b/test/towerops/lidar_test.exs @@ -0,0 +1,223 @@ +defmodule Towerops.LidarTest do + use Towerops.DataCase + + alias Towerops.Lidar + alias Towerops.Lidar.Tile + alias Towerops.Repo + + # Build a small square polygon footprint around (lat, lon) of size deg. + defp square_footprint(lat, lon, deg \\ 0.05) do + coords = [ + {lon - deg, lat - deg}, + {lon + deg, lat - deg}, + {lon + deg, lat + deg}, + {lon - deg, lat + deg}, + {lon - deg, lat - deg} + ] + + %Geo.Polygon{coordinates: [coords], srid: 4326} + end + + defp insert_tile!(attrs) do + defaults = %{ + source: "3dep", + project_name: "TX_Test_2024", + tile_name: "tile_#{System.unique_integer([:positive])}", + year: 2024, + resolution_m: Decimal.new("1.0"), + url: "https://prd-tnm.s3.amazonaws.com/example.tif", + crs: "EPSG:6343", + footprint: square_footprint(30.27, -97.74), + availability: "available", + metadata: %{}, + inserted_at: DateTime.truncate(DateTime.utc_now(), :second), + updated_at: DateTime.truncate(DateTime.utc_now(), :second) + } + + %Tile{} + |> Tile.changeset(Map.merge(defaults, attrs)) + |> Repo.insert!() + end + + describe "list_tiles_for_point/2" do + test "returns tiles whose footprint contains the point" do + tile = insert_tile!(%{footprint: square_footprint(30.27, -97.74, 0.1)}) + _other = insert_tile!(%{footprint: square_footprint(40.0, -100.0, 0.1)}) + + [result] = Lidar.list_tiles_for_point(30.27, -97.74) + assert result.id == tile.id + end + + test "returns empty list when no tile covers the point" do + _tile = insert_tile!(%{footprint: square_footprint(40.0, -100.0, 0.05)}) + assert Lidar.list_tiles_for_point(30.27, -97.74) == [] + end + + test "orders by resolution_m ascending then year descending (best tile first)" do + coarse_old = + insert_tile!(%{ + tile_name: "coarse_old", + resolution_m: Decimal.new("3.0"), + year: 2018 + }) + + coarse_new = + insert_tile!(%{ + tile_name: "coarse_new", + resolution_m: Decimal.new("3.0"), + year: 2024 + }) + + fine_old = + insert_tile!(%{ + tile_name: "fine_old", + resolution_m: Decimal.new("1.0"), + year: 2018 + }) + + fine_new = + insert_tile!(%{ + tile_name: "fine_new", + resolution_m: Decimal.new("1.0"), + year: 2024 + }) + + result = Lidar.list_tiles_for_point(30.27, -97.74) + ids = Enum.map(result, & &1.id) + + assert ids == [fine_new.id, fine_old.id, coarse_new.id, coarse_old.id] + end + + test "excludes tiles with availability != \"available\"" do + _missing = + insert_tile!(%{tile_name: "missing", availability: "missing"}) + + ok = insert_tile!(%{tile_name: "ok", availability: "available"}) + + [result] = Lidar.list_tiles_for_point(30.27, -97.74) + assert result.id == ok.id + end + end + + describe "list_tiles_for_area/1" do + test "returns tiles intersecting the bbox" do + inside = insert_tile!(%{footprint: square_footprint(30.27, -97.74, 0.05)}) + _outside = insert_tile!(%{footprint: square_footprint(40.0, -100.0, 0.05)}) + + bbox = {-97.8, 30.2, -97.7, 30.3} + [result] = Lidar.list_tiles_for_area(bbox) + assert result.id == inside.id + end + + test "returns multiple tiles ordered best-first" do + _coarse = + insert_tile!(%{ + tile_name: "coarse", + resolution_m: Decimal.new("3.0"), + footprint: square_footprint(30.27, -97.74, 0.1) + }) + + fine = + insert_tile!(%{ + tile_name: "fine", + resolution_m: Decimal.new("1.0"), + footprint: square_footprint(30.27, -97.74, 0.1) + }) + + bbox = {-97.8, 30.2, -97.7, 30.3} + [first | _] = Lidar.list_tiles_for_area(bbox) + assert first.id == fine.id + end + end + + describe "get_elevation/2" do + test "returns :no_tile when no catalog tile covers the point" do + assert {:error, :no_tile} = Lidar.get_elevation(30.27, -97.74) + end + + test "returns the reader's value when a tile covers the point" do + _ = insert_tile!(%{footprint: square_footprint(30.27, -97.74, 0.1)}) + + Application.put_env(:towerops, :lidar_runner, fn _cmd, _args, _opts -> + {"152.5\n", 0} + end) + + on_exit(fn -> Application.delete_env(:towerops, :lidar_runner) end) + + assert {:ok, 152.5} = Lidar.get_elevation(30.27, -97.74) + end + + test "falls through to the next-best tile on nodata" do + _ = insert_tile!(%{tile_name: "best", resolution_m: Decimal.new("1.0")}) + _ = insert_tile!(%{tile_name: "fallback", resolution_m: Decimal.new("3.0")}) + + 1 |> :counters.new([:atomics]) |> tap(&Process.put(:c, &1)) + + Application.put_env(:towerops, :lidar_runner, fn _cmd, _args, _opts -> + c = Process.get(:c) + :counters.add(c, 1, 1) + n = :counters.get(c, 1) + if n == 1, do: {"\n", 0}, else: {"42.0\n", 0} + end) + + on_exit(fn -> Application.delete_env(:towerops, :lidar_runner) end) + + assert {:ok, 42.0} = Lidar.get_elevation(30.27, -97.74) + end + + test "falls through to next tile on non-nodata reader errors" do + _ = insert_tile!(%{tile_name: "first"}) + _ = insert_tile!(%{tile_name: "second"}) + + 1 |> :counters.new([:atomics]) |> tap(&Process.put(:c, &1)) + + Application.put_env(:towerops, :lidar_runner, fn _cmd, _args, _opts -> + c = Process.get(:c) + :counters.add(c, 1, 1) + n = :counters.get(c, 1) + + if n == 1 do + {"network down", 1} + else + {"77.0\n", 0} + end + end) + + on_exit(fn -> Application.delete_env(:towerops, :lidar_runner) end) + + assert {:ok, 77.0} = Lidar.get_elevation(30.27, -97.74) + end + + test "returns the last error when only one tile and it returns a non-nodata error" do + _ = insert_tile!(%{}) + + Application.put_env(:towerops, :lidar_runner, fn _cmd, _args, _opts -> + {"boom", 5} + end) + + on_exit(fn -> Application.delete_env(:towerops, :lidar_runner) end) + + assert {:error, {:gdal, 5, "boom"}} = Lidar.get_elevation(30.27, -97.74) + end + + test "returns :nodata when every covering tile lacks data" do + _ = insert_tile!(%{tile_name: "a"}) + _ = insert_tile!(%{tile_name: "b"}) + + Application.put_env(:towerops, :lidar_runner, fn _cmd, _args, _opts -> + {"\n", 0} + end) + + on_exit(fn -> Application.delete_env(:towerops, :lidar_runner) end) + + assert {:error, :nodata} = Lidar.get_elevation(30.27, -97.74) + end + end + + describe "get_elevation_grid/3" do + test "returns :no_tile when no catalog tile intersects the bbox" do + bbox = {-97.8, 30.2, -97.7, 30.3} + assert {:error, :no_tile} = Lidar.get_elevation_grid(bbox, 0.05) + end + end +end diff --git a/test/towerops/workers/lidar_catalog_sync_worker_test.exs b/test/towerops/workers/lidar_catalog_sync_worker_test.exs new file mode 100644 index 00000000..2554f443 --- /dev/null +++ b/test/towerops/workers/lidar_catalog_sync_worker_test.exs @@ -0,0 +1,60 @@ +defmodule Towerops.Workers.LidarCatalogSyncWorkerTest do + use Towerops.DataCase + + alias Towerops.Workers.LidarCatalogSyncWorker + + require Logger + + describe "perform/1" do + test "calls Catalog.sync_3dep and returns :ok on success" do + stub_sync({:ok, %{projects_synced: 0, tiles_upserted: 0, errors: %{}, duration_ms: 1}}) + + assert :ok = LidarCatalogSyncWorker.perform(%Oban.Job{}) + end + + test "returns the error on failure so Oban retries" do + stub_sync({:error, :boom}) + + assert {:error, :boom} = LidarCatalogSyncWorker.perform(%Oban.Job{}) + end + + test "exercises the full success log branch" do + stub_sync({:ok, %{projects_synced: 3, tiles_upserted: 250, errors: %{}, duration_ms: 1234}}) + + previous = Logger.level() + Logger.configure(level: :info) + on_exit(fn -> Logger.configure(level: previous) end) + + log = + ExUnit.CaptureLog.capture_log(fn -> + assert :ok = LidarCatalogSyncWorker.perform(%Oban.Job{}) + end) + + assert log =~ "250 tiles" + assert log =~ "3 projects" + assert log =~ "1234ms" + end + + test "calls real Catalog.sync_3dep when no runner override is configured" do + Application.delete_env(:towerops, :lidar_catalog_runner) + + Req.Test.stub(Towerops.Lidar.Sources.ThreeDep, fn conn -> + Plug.Conn.send_resp( + conn, + 200, + """ + + false + """ + ) + end) + + assert :ok = LidarCatalogSyncWorker.perform(%Oban.Job{}) + end + end + + defp stub_sync(result) do + Application.put_env(:towerops, :lidar_catalog_runner, fn -> result end) + on_exit(fn -> Application.delete_env(:towerops, :lidar_catalog_runner) end) + end +end