defmodule Microwaveprop.Buildings.BulkFetch do @moduledoc """ One-shot bulk download of Microsoft Building Footprint quadkey tiles for a `radius_mi`-mile bbox around a `(lat, lon)` center. Used to pre-warm the cache so per-Calculate runs never have to hit the Microsoft CDN. Runs in any environment, including prod. From an attached `iex` on the pod: iex> Microwaveprop.Buildings.BulkFetch.run(32.8, -97.0, 500) Tiles already on disk are skipped. Quadkeys not present in the MS dataset index (water, outside the US) are silently dropped. """ alias Microwaveprop.Buildings.MsFootprints require Logger @zoom 9 @default_concurrency 4 @task_timeout_ms 180_000 @type result :: %{ downloaded: non_neg_integer(), cached: non_neg_integer(), failed: non_neg_integer(), missing: non_neg_integer(), cache_dir: String.t() } @spec run(float(), float(), pos_integer(), keyword()) :: result() def run(lat, lon, radius_mi, opts \\ []) do concurrency = Keyword.get(opts, :concurrency, @default_concurrency) radius_km = radius_mi * 1.609 bbox = bbox_around(lat, lon, radius_km) Logger.info("buildings bulk fetch: ~#{radius_mi} mi (#{Float.round(radius_km)} km) around (#{lat}, #{lon})") candidate_quadkeys = MsFootprints.quadkeys_for_bbox(bbox, @zoom) index = MsFootprints.dataset_index() quadkeys = Enum.filter(candidate_quadkeys, &Map.has_key?(index, &1)) missing = length(candidate_quadkeys) - length(quadkeys) Logger.info( "buildings bulk fetch: #{length(candidate_quadkeys)} candidate, #{length(quadkeys)} present in index, #{missing} not in index" ) {downloaded, cached, failed} = run_downloads(quadkeys, concurrency) result = %{ downloaded: downloaded, cached: cached, failed: failed, missing: missing, cache_dir: MsFootprints.cache_dir() } Logger.info("buildings bulk fetch: #{inspect(result)}") result end defp run_downloads(quadkeys, concurrency) do quadkeys |> Task.async_stream(&fetch_one/1, max_concurrency: concurrency, ordered: false, timeout: @task_timeout_ms, on_timeout: :kill_task ) |> Enum.reduce({0, 0, 0}, &accumulate/2) end defp fetch_one(qk) do path = Path.join(MsFootprints.cache_dir(), "#{qk}.csv.gz") if File.exists?(path) do :cached else case MsFootprints.download_tile(qk) do {:ok, _} -> :downloaded {:error, reason} -> {:failed, qk, reason} end end end defp accumulate({:ok, :downloaded}, {d, s, f}), do: {d + 1, s, f} defp accumulate({:ok, :cached}, {d, s, f}), do: {d, s + 1, f} defp accumulate({:ok, {:failed, qk, reason}}, {d, s, f}) do Logger.warning("buildings bulk fetch failed #{qk}: #{inspect(reason)}") {d, s, f + 1} end defp accumulate({:exit, reason}, {d, s, f}) do Logger.warning("buildings bulk fetch task crashed: #{inspect(reason)}") {d, s, f + 1} end defp bbox_around(lat, lon, radius_km) do dlat = radius_km / 111.0 dlon = radius_km / (111.0 * max(:math.cos(lat * :math.pi() / 180.0), 0.1)) %{ "south" => lat - dlat, "north" => lat + dlat, "west" => lon - dlon, "east" => lon + dlon } end end