Graham McIntire
021608dc90
Mix tasks don't run in production, so the orchestration lives in
Microwaveprop.Buildings.BulkFetch — callable from an attached iex on
the prod pod via:
iex> Microwaveprop.Buildings.BulkFetch.run(32.8, -97.0, 500)
Default 500 mi radius around DFW yields ~440 candidate quadkeys at
zoom 9; only those present in the MS dataset index (typically a few
hundred over CONUS) get downloaded. Concurrency 4, per-task timeout
3 min, results logged with downloaded/cached/failed/missing counts.
The mix.task is kept as a dev-only thin wrapper.
112 lines
3.2 KiB
Elixir
112 lines
3.2 KiB
Elixir
defmodule Microwaveprop.Buildings.BulkFetch do
|
|
@moduledoc """
|
|
One-shot bulk download of Microsoft Building Footprint quadkey tiles
|
|
for a `radius_mi`-mile bbox around a `(lat, lon)` center. Used to
|
|
pre-warm the cache so per-Calculate runs never have to hit the
|
|
Microsoft CDN.
|
|
|
|
Runs in any environment, including prod. From an attached `iex` on the
|
|
pod:
|
|
|
|
iex> Microwaveprop.Buildings.BulkFetch.run(32.8, -97.0, 500)
|
|
|
|
Tiles already on disk are skipped. Quadkeys not present in the MS
|
|
dataset index (water, outside the US) are silently dropped.
|
|
"""
|
|
|
|
alias Microwaveprop.Buildings.MsFootprints
|
|
|
|
require Logger
|
|
|
|
@zoom 9
|
|
@default_concurrency 4
|
|
@task_timeout_ms 180_000
|
|
|
|
@type result :: %{
|
|
downloaded: non_neg_integer(),
|
|
cached: non_neg_integer(),
|
|
failed: non_neg_integer(),
|
|
missing: non_neg_integer(),
|
|
cache_dir: String.t()
|
|
}
|
|
|
|
@spec run(float(), float(), pos_integer(), keyword()) :: result()
|
|
def run(lat, lon, radius_mi, opts \\ []) do
|
|
concurrency = Keyword.get(opts, :concurrency, @default_concurrency)
|
|
radius_km = radius_mi * 1.609
|
|
bbox = bbox_around(lat, lon, radius_km)
|
|
|
|
Logger.info("buildings bulk fetch: ~#{radius_mi} mi (#{Float.round(radius_km)} km) around (#{lat}, #{lon})")
|
|
|
|
candidate_quadkeys = MsFootprints.quadkeys_for_bbox(bbox, @zoom)
|
|
index = MsFootprints.dataset_index()
|
|
quadkeys = Enum.filter(candidate_quadkeys, &Map.has_key?(index, &1))
|
|
missing = length(candidate_quadkeys) - length(quadkeys)
|
|
|
|
Logger.info(
|
|
"buildings bulk fetch: #{length(candidate_quadkeys)} candidate, #{length(quadkeys)} present in index, #{missing} not in index"
|
|
)
|
|
|
|
{downloaded, cached, failed} = run_downloads(quadkeys, concurrency)
|
|
|
|
result = %{
|
|
downloaded: downloaded,
|
|
cached: cached,
|
|
failed: failed,
|
|
missing: missing,
|
|
cache_dir: MsFootprints.cache_dir()
|
|
}
|
|
|
|
Logger.info("buildings bulk fetch: #{inspect(result)}")
|
|
result
|
|
end
|
|
|
|
defp run_downloads(quadkeys, concurrency) do
|
|
quadkeys
|
|
|> Task.async_stream(&fetch_one/1,
|
|
max_concurrency: concurrency,
|
|
ordered: false,
|
|
timeout: @task_timeout_ms,
|
|
on_timeout: :kill_task
|
|
)
|
|
|> Enum.reduce({0, 0, 0}, &accumulate/2)
|
|
end
|
|
|
|
defp fetch_one(qk) do
|
|
path = Path.join(MsFootprints.cache_dir(), "#{qk}.csv.gz")
|
|
|
|
if File.exists?(path) do
|
|
:cached
|
|
else
|
|
case MsFootprints.download_tile(qk) do
|
|
{:ok, _} -> :downloaded
|
|
{:error, reason} -> {:failed, qk, reason}
|
|
end
|
|
end
|
|
end
|
|
|
|
defp accumulate({:ok, :downloaded}, {d, s, f}), do: {d + 1, s, f}
|
|
defp accumulate({:ok, :cached}, {d, s, f}), do: {d, s + 1, f}
|
|
|
|
defp accumulate({:ok, {:failed, qk, reason}}, {d, s, f}) do
|
|
Logger.warning("buildings bulk fetch failed #{qk}: #{inspect(reason)}")
|
|
{d, s, f + 1}
|
|
end
|
|
|
|
defp accumulate({:exit, reason}, {d, s, f}) do
|
|
Logger.warning("buildings bulk fetch task crashed: #{inspect(reason)}")
|
|
{d, s, f + 1}
|
|
end
|
|
|
|
defp bbox_around(lat, lon, radius_km) do
|
|
dlat = radius_km / 111.0
|
|
dlon = radius_km / (111.0 * max(:math.cos(lat * :math.pi() / 180.0), 0.1))
|
|
|
|
%{
|
|
"south" => lat - dlat,
|
|
"north" => lat + dlat,
|
|
"west" => lon - dlon,
|
|
"east" => lon + dlon
|
|
}
|
|
end
|
|
end
|