feat(buildings): bulk-prefetch tiles via BulkFetch.run/4 (works in prod)

Mix tasks don't run in production, so the orchestration lives in
Microwaveprop.Buildings.BulkFetch — callable from an attached iex on
the prod pod via:

    iex> Microwaveprop.Buildings.BulkFetch.run(32.8, -97.0, 500)

Default 500 mi radius around DFW yields ~440 candidate quadkeys at
zoom 9; only those present in the MS dataset index (typically a few
hundred over CONUS) get downloaded. Concurrency 4, per-task timeout
3 min, results logged with downloaded/cached/failed/missing counts.
The mix.task is kept as a dev-only thin wrapper.
This commit is contained in:
Graham McIntire 2026-04-26 10:37:13 -05:00
parent 5a97776fd5
commit 021608dc90
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
2 changed files with 161 additions and 0 deletions

View file

@ -0,0 +1,112 @@
defmodule Microwaveprop.Buildings.BulkFetch do
@moduledoc """
One-shot bulk download of Microsoft Building Footprint quadkey tiles
for a `radius_mi`-mile bbox around a `(lat, lon)` center. Used to
pre-warm the cache so per-Calculate runs never have to hit the
Microsoft CDN.
Runs in any environment, including prod. From an attached `iex` on the
pod:
iex> Microwaveprop.Buildings.BulkFetch.run(32.8, -97.0, 500)
Tiles already on disk are skipped. Quadkeys not present in the MS
dataset index (water, outside the US) are silently dropped.
"""
alias Microwaveprop.Buildings.MsFootprints
require Logger
@zoom 9
@default_concurrency 4
@task_timeout_ms 180_000
@type result :: %{
downloaded: non_neg_integer(),
cached: non_neg_integer(),
failed: non_neg_integer(),
missing: non_neg_integer(),
cache_dir: String.t()
}
@spec run(float(), float(), pos_integer(), keyword()) :: result()
def run(lat, lon, radius_mi, opts \\ []) do
concurrency = Keyword.get(opts, :concurrency, @default_concurrency)
radius_km = radius_mi * 1.609
bbox = bbox_around(lat, lon, radius_km)
Logger.info("buildings bulk fetch: ~#{radius_mi} mi (#{Float.round(radius_km)} km) around (#{lat}, #{lon})")
candidate_quadkeys = MsFootprints.quadkeys_for_bbox(bbox, @zoom)
index = MsFootprints.dataset_index()
quadkeys = Enum.filter(candidate_quadkeys, &Map.has_key?(index, &1))
missing = length(candidate_quadkeys) - length(quadkeys)
Logger.info(
"buildings bulk fetch: #{length(candidate_quadkeys)} candidate, #{length(quadkeys)} present in index, #{missing} not in index"
)
{downloaded, cached, failed} = run_downloads(quadkeys, concurrency)
result = %{
downloaded: downloaded,
cached: cached,
failed: failed,
missing: missing,
cache_dir: MsFootprints.cache_dir()
}
Logger.info("buildings bulk fetch: #{inspect(result)}")
result
end
defp run_downloads(quadkeys, concurrency) do
quadkeys
|> Task.async_stream(&fetch_one/1,
max_concurrency: concurrency,
ordered: false,
timeout: @task_timeout_ms,
on_timeout: :kill_task
)
|> Enum.reduce({0, 0, 0}, &accumulate/2)
end
defp fetch_one(qk) do
path = Path.join(MsFootprints.cache_dir(), "#{qk}.csv.gz")
if File.exists?(path) do
:cached
else
case MsFootprints.download_tile(qk) do
{:ok, _} -> :downloaded
{:error, reason} -> {:failed, qk, reason}
end
end
end
defp accumulate({:ok, :downloaded}, {d, s, f}), do: {d + 1, s, f}
defp accumulate({:ok, :cached}, {d, s, f}), do: {d, s + 1, f}
defp accumulate({:ok, {:failed, qk, reason}}, {d, s, f}) do
Logger.warning("buildings bulk fetch failed #{qk}: #{inspect(reason)}")
{d, s, f + 1}
end
defp accumulate({:exit, reason}, {d, s, f}) do
Logger.warning("buildings bulk fetch task crashed: #{inspect(reason)}")
{d, s, f + 1}
end
defp bbox_around(lat, lon, radius_km) do
dlat = radius_km / 111.0
dlon = radius_km / (111.0 * max(:math.cos(lat * :math.pi() / 180.0), 0.1))
%{
"south" => lat - dlat,
"north" => lat + dlat,
"west" => lon - dlon,
"east" => lon + dlon
}
end
end

View file

@ -0,0 +1,49 @@
defmodule Mix.Tasks.BuildingsFetch do
@shortdoc "Bulk-download Microsoft Building Footprint tiles around DFW (DEV ONLY)"
@moduledoc """
Pre-warms the local Microsoft Building Footprints cache so per-Calculate
requests never have to wait on a tile download.
mix buildings.fetch # default: 500 mi radius around EM13
mix buildings.fetch --lat 32.8 --lon -97 --radius-mi 500
mix buildings.fetch --concurrency 8
**Mix tasks do not run in production.** From an attached `iex` on the
prod pod, call the underlying module directly:
iex> Microwaveprop.Buildings.BulkFetch.run(32.8, -97.0, 500)
"""
use Mix.Task
alias Microwaveprop.Buildings.BulkFetch
@default_lat 32.8
@default_lon -97.0
@default_radius_mi 500
@switches [
lat: :float,
lon: :float,
radius_mi: :integer,
concurrency: :integer
]
@impl Mix.Task
def run(argv) do
{opts, _, _} = OptionParser.parse(argv, switches: @switches)
lat = Keyword.get(opts, :lat, @default_lat)
lon = Keyword.get(opts, :lon, @default_lon)
radius_mi = Keyword.get(opts, :radius_mi, @default_radius_mi)
concurrency = Keyword.get(opts, :concurrency)
Mix.Task.run("app.start")
bulk_opts = if concurrency, do: [concurrency: concurrency], else: []
result = BulkFetch.run(lat, lon, radius_mi, bulk_opts)
Mix.shell().info("Done. #{inspect(result)}")
end
end