defmodule Microwaveprop.Buildings.MsFootprints do @moduledoc """ Bing-style quadkey math, dataset-index lookup, and tile downloader for Microsoft's Global ML Building Footprints (https://github.com/microsoft/GlobalMLBuildingFootprints). We use quadkey level 9 — each tile covers ~80 km × 65 km at DFW latitude, which matches the rover Calculate radius. Files are line-delimited GeoJSON inside csv.gz, hosted on Azure Blob Storage. """ require Logger @dataset_links_url "https://minedbuildings.z5.web.core.windows.net/global-buildings/dataset-links.csv" @region "UnitedStates" @doc """ Returns the Bing/MS quadkey string for `lat, lon` at the given `zoom`. Encodes via Web Mercator pixel coordinates → tile XY → quadkey, per https://learn.microsoft.com/en-us/bingmaps/articles/bing-maps-tile-system. """ @spec quadkey(float(), float(), pos_integer()) :: String.t() def quadkey(lat, lon, zoom) when zoom in 1..23 do {tx, ty} = tile_xy(lat, lon, zoom) encode_quadkey(tx, ty, zoom) end @doc """ Returns the unique set of quadkeys at `zoom` whose tile boundary intersects the bbox. Used to discover which MS tiles need to be downloaded for a Calculate. """ @spec quadkeys_for_bbox(map(), pos_integer()) :: [String.t()] def quadkeys_for_bbox(%{"south" => s, "north" => n, "west" => w, "east" => e}, zoom) do {tx_min, ty_max} = tile_xy(s, w, zoom) {tx_max, ty_min} = tile_xy(n, e, zoom) for tx <- tx_min..tx_max, ty <- ty_min..ty_max, uniq: true do encode_quadkey(tx, ty, zoom) end end defp tile_xy(lat, lon, zoom) do lat_clamped = lat |> max(-85.05112878) |> min(85.05112878) n = :math.pow(2, zoom) x = (lon + 180.0) / 360.0 * n sin_lat = :math.sin(lat_clamped * :math.pi() / 180.0) y = (0.5 - :math.log((1 + sin_lat) / (1 - sin_lat)) / (4 * :math.pi())) * n {clamp_tile(trunc(x), zoom), clamp_tile(trunc(y), zoom)} end defp clamp_tile(t, zoom) do max_tile = trunc(:math.pow(2, zoom)) - 1 t |> max(0) |> min(max_tile) end defp encode_quadkey(tx, ty, zoom) do Enum.reduce(zoom..1//-1, "", fn i, acc -> mask = Bitwise.bsl(1, i - 1) digit = digit_for(tx, ty, mask) acc <> Integer.to_string(digit) end) end defp digit_for(tx, ty, mask) do x_bit = if Bitwise.band(tx, mask) == 0, do: 0, else: 1 y_bit = if Bitwise.band(ty, mask) == 0, do: 0, else: 1 Bitwise.bor(Bitwise.bsl(y_bit, 1), x_bit) end @doc """ Returns `%{quadkey => url}` for every UnitedStates row in the MS dataset-links CSV. Result is memoized in `Microwaveprop.Cache` for 24 hours; the index file is ~7 MB so we don't want to refetch it on every Calculate. """ @spec dataset_index() :: %{String.t() => String.t()} def dataset_index do Microwaveprop.Cache.fetch_or_store(:ms_footprints_index, 24 * 60 * 60 * 1_000, fn -> case http_get(@dataset_links_url, retry: false, receive_timeout: 30_000) do {:ok, %{status: 200, body: body}} -> parse_dataset_index(body) other -> raise "ms footprints dataset index fetch failed: #{inspect(other)}" end end) end defp http_get(url, opts) do runner = Application.get_env(:microwaveprop, :ms_footprints_http_get, &Req.get/2) runner.(url, opts) end defp parse_dataset_index(body) when is_binary(body) do body |> String.split("\n", trim: true) |> Enum.drop(1) |> Enum.flat_map(fn line -> case String.split(line, ",", parts: 5) do [@region, qk, url, _size, _date] -> [{qk, url}] _ -> [] end end) |> Map.new() end @doc """ Local cache directory for downloaded csv.gz files. Override via `:microwaveprop, :buildings_cache_dir` to redirect into NFS in prod. """ @spec cache_dir() :: String.t() def cache_dir do Application.get_env(:microwaveprop, :buildings_cache_dir, "/data/buildings") end @doc """ Ensure `quadkey` is downloaded to disk; returns `{:ok, path}` or `{:error, reason}`. Re-downloads are skipped if the cached file is already present. """ @spec download_tile(String.t()) :: {:ok, String.t()} | {:error, term()} def download_tile(quadkey) when is_binary(quadkey) do path = Path.join(cache_dir(), "#{quadkey}.csv.gz") if File.exists?(path) do {:ok, path} else do_download_tile(quadkey, path) end end defp do_download_tile(quadkey, path) do case Map.fetch(dataset_index(), quadkey) do :error -> {:error, :unknown_quadkey} {:ok, url} -> File.mkdir_p!(cache_dir()) Logger.info("downloading MS footprint tile #{quadkey} from #{url}") case http_get(url, retry: false, receive_timeout: 120_000) do {:ok, %{status: 200, body: body}} -> File.write!(path, body) {:ok, path} {:ok, %{status: status}} -> {:error, {:http, status}} {:error, reason} -> {:error, reason} end end end @doc """ Delete building cache files whose mtime is older than `cutoff`. Returns the number of files deleted. """ @spec prune_older_than(DateTime.t()) :: integer() def prune_older_than(%DateTime{} = cutoff) do cutoff_unix = DateTime.to_unix(cutoff) case File.ls(cache_dir()) do {:ok, files} -> Enum.reduce(files, 0, &prune_file(&1, &2, cutoff_unix)) _ -> 0 end end defp prune_file(file, acc, cutoff_unix) do path = Path.join(cache_dir(), file) case File.stat(path) do {:ok, stat} -> mtime_unix = stat.mtime |> NaiveDateTime.from_erl!() |> DateTime.from_naive!("Etc/UTC") |> DateTime.to_unix() if mtime_unix < cutoff_unix do _ = File.rm(path) acc + 1 else acc end {:error, :enoent} -> # Race: file deleted between File.ls and File.stat. Skip it. acc {:error, reason} -> Logger.warning("MsFootprints.prune_older_than: stat failed for #{file}: #{inspect(reason)}") acc end end end