diff --git a/lib/microwaveprop/buildings/bulk_fetch.ex b/lib/microwaveprop/buildings/bulk_fetch.ex new file mode 100644 index 00000000..f8c374b1 --- /dev/null +++ b/lib/microwaveprop/buildings/bulk_fetch.ex @@ -0,0 +1,112 @@ +defmodule Microwaveprop.Buildings.BulkFetch do + @moduledoc """ + One-shot bulk download of Microsoft Building Footprint quadkey tiles + for a `radius_mi`-mile bbox around a `(lat, lon)` center. Used to + pre-warm the cache so per-Calculate runs never have to hit the + Microsoft CDN. + + Runs in any environment, including prod. From an attached `iex` on the + pod: + + iex> Microwaveprop.Buildings.BulkFetch.run(32.8, -97.0, 500) + + Tiles already on disk are skipped. Quadkeys not present in the MS + dataset index (water, outside the US) are silently dropped. + """ + + alias Microwaveprop.Buildings.MsFootprints + + require Logger + + @zoom 9 + @default_concurrency 4 + @task_timeout_ms 180_000 + + @type result :: %{ + downloaded: non_neg_integer(), + cached: non_neg_integer(), + failed: non_neg_integer(), + missing: non_neg_integer(), + cache_dir: String.t() + } + + @spec run(float(), float(), pos_integer(), keyword()) :: result() + def run(lat, lon, radius_mi, opts \\ []) do + concurrency = Keyword.get(opts, :concurrency, @default_concurrency) + radius_km = radius_mi * 1.609 + bbox = bbox_around(lat, lon, radius_km) + + Logger.info("buildings bulk fetch: ~#{radius_mi} mi (#{Float.round(radius_km)} km) around (#{lat}, #{lon})") + + candidate_quadkeys = MsFootprints.quadkeys_for_bbox(bbox, @zoom) + index = MsFootprints.dataset_index() + quadkeys = Enum.filter(candidate_quadkeys, &Map.has_key?(index, &1)) + missing = length(candidate_quadkeys) - length(quadkeys) + + Logger.info( + "buildings bulk fetch: #{length(candidate_quadkeys)} candidate, #{length(quadkeys)} present in index, #{missing} not in index" + ) + + {downloaded, cached, failed} = run_downloads(quadkeys, concurrency) + + result = %{ + downloaded: downloaded, + cached: cached, + failed: failed, + missing: missing, + cache_dir: MsFootprints.cache_dir() + } + + Logger.info("buildings bulk fetch: #{inspect(result)}") + result + end + + defp run_downloads(quadkeys, concurrency) do + quadkeys + |> Task.async_stream(&fetch_one/1, + max_concurrency: concurrency, + ordered: false, + timeout: @task_timeout_ms, + on_timeout: :kill_task + ) + |> Enum.reduce({0, 0, 0}, &accumulate/2) + end + + defp fetch_one(qk) do + path = Path.join(MsFootprints.cache_dir(), "#{qk}.csv.gz") + + if File.exists?(path) do + :cached + else + case MsFootprints.download_tile(qk) do + {:ok, _} -> :downloaded + {:error, reason} -> {:failed, qk, reason} + end + end + end + + defp accumulate({:ok, :downloaded}, {d, s, f}), do: {d + 1, s, f} + defp accumulate({:ok, :cached}, {d, s, f}), do: {d, s + 1, f} + + defp accumulate({:ok, {:failed, qk, reason}}, {d, s, f}) do + Logger.warning("buildings bulk fetch failed #{qk}: #{inspect(reason)}") + {d, s, f + 1} + end + + defp accumulate({:exit, reason}, {d, s, f}) do + Logger.warning("buildings bulk fetch task crashed: #{inspect(reason)}") + {d, s, f + 1} + end + + defp bbox_around(lat, lon, radius_km) do + dlat = radius_km / 111.0 + dlon = radius_km / (111.0 * max(:math.cos(lat * :math.pi() / 180.0), 0.1)) + + %{ + "south" => lat - dlat, + "north" => lat + dlat, + "west" => lon - dlon, + "east" => lon + dlon + } + end +end diff --git a/lib/mix/tasks/buildings_fetch.ex b/lib/mix/tasks/buildings_fetch.ex new file mode 100644 index 00000000..87eddd16 --- /dev/null +++ b/lib/mix/tasks/buildings_fetch.ex @@ -0,0 +1,49 @@ +defmodule Mix.Tasks.BuildingsFetch do + @shortdoc "Bulk-download Microsoft Building Footprint tiles around DFW (DEV ONLY)" + + @moduledoc """ + Pre-warms the local Microsoft Building Footprints cache so per-Calculate + requests never have to wait on a tile download. + + mix buildings.fetch # default: 500 mi radius around EM13 + mix buildings.fetch --lat 32.8 --lon -97 --radius-mi 500 + mix buildings.fetch --concurrency 8 + + **Mix tasks do not run in production.** From an attached `iex` on the + prod pod, call the underlying module directly: + + iex> Microwaveprop.Buildings.BulkFetch.run(32.8, -97.0, 500) + """ + + use Mix.Task + + alias Microwaveprop.Buildings.BulkFetch + + @default_lat 32.8 + @default_lon -97.0 + @default_radius_mi 500 + + @switches [ + lat: :float, + lon: :float, + radius_mi: :integer, + concurrency: :integer + ] + + @impl Mix.Task + def run(argv) do + {opts, _, _} = OptionParser.parse(argv, switches: @switches) + + lat = Keyword.get(opts, :lat, @default_lat) + lon = Keyword.get(opts, :lon, @default_lon) + radius_mi = Keyword.get(opts, :radius_mi, @default_radius_mi) + concurrency = Keyword.get(opts, :concurrency) + + Mix.Task.run("app.start") + + bulk_opts = if concurrency, do: [concurrency: concurrency], else: [] + result = BulkFetch.run(lat, lon, radius_mi, bulk_opts) + + Mix.shell().info("Done. #{inspect(result)}") + end +end