feat(buildings): bulk-prefetch tiles via BulkFetch.run/4 (works in prod)
Mix tasks don't run in production, so the orchestration lives in
Microwaveprop.Buildings.BulkFetch — callable from an attached iex on
the prod pod via:
iex> Microwaveprop.Buildings.BulkFetch.run(32.8, -97.0, 500)
Default 500 mi radius around DFW yields ~440 candidate quadkeys at
zoom 9; only those present in the MS dataset index (typically a few
hundred over CONUS) get downloaded. Concurrency 4, per-task timeout
3 min, results logged with downloaded/cached/failed/missing counts.
The mix.task is kept as a dev-only thin wrapper.
This commit is contained in:
parent
5a97776fd5
commit
021608dc90
2 changed files with 161 additions and 0 deletions
112
lib/microwaveprop/buildings/bulk_fetch.ex
Normal file
112
lib/microwaveprop/buildings/bulk_fetch.ex
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
defmodule Microwaveprop.Buildings.BulkFetch do
|
||||
@moduledoc """
|
||||
One-shot bulk download of Microsoft Building Footprint quadkey tiles
|
||||
for a `radius_mi`-mile bbox around a `(lat, lon)` center. Used to
|
||||
pre-warm the cache so per-Calculate runs never have to hit the
|
||||
Microsoft CDN.
|
||||
|
||||
Runs in any environment, including prod. From an attached `iex` on the
|
||||
pod:
|
||||
|
||||
iex> Microwaveprop.Buildings.BulkFetch.run(32.8, -97.0, 500)
|
||||
|
||||
Tiles already on disk are skipped. Quadkeys not present in the MS
|
||||
dataset index (water, outside the US) are silently dropped.
|
||||
"""
|
||||
|
||||
alias Microwaveprop.Buildings.MsFootprints
|
||||
|
||||
require Logger
|
||||
|
||||
@zoom 9
|
||||
@default_concurrency 4
|
||||
@task_timeout_ms 180_000
|
||||
|
||||
@type result :: %{
|
||||
downloaded: non_neg_integer(),
|
||||
cached: non_neg_integer(),
|
||||
failed: non_neg_integer(),
|
||||
missing: non_neg_integer(),
|
||||
cache_dir: String.t()
|
||||
}
|
||||
|
||||
@spec run(float(), float(), pos_integer(), keyword()) :: result()
|
||||
def run(lat, lon, radius_mi, opts \\ []) do
|
||||
concurrency = Keyword.get(opts, :concurrency, @default_concurrency)
|
||||
radius_km = radius_mi * 1.609
|
||||
bbox = bbox_around(lat, lon, radius_km)
|
||||
|
||||
Logger.info("buildings bulk fetch: ~#{radius_mi} mi (#{Float.round(radius_km)} km) around (#{lat}, #{lon})")
|
||||
|
||||
candidate_quadkeys = MsFootprints.quadkeys_for_bbox(bbox, @zoom)
|
||||
index = MsFootprints.dataset_index()
|
||||
quadkeys = Enum.filter(candidate_quadkeys, &Map.has_key?(index, &1))
|
||||
missing = length(candidate_quadkeys) - length(quadkeys)
|
||||
|
||||
Logger.info(
|
||||
"buildings bulk fetch: #{length(candidate_quadkeys)} candidate, #{length(quadkeys)} present in index, #{missing} not in index"
|
||||
)
|
||||
|
||||
{downloaded, cached, failed} = run_downloads(quadkeys, concurrency)
|
||||
|
||||
result = %{
|
||||
downloaded: downloaded,
|
||||
cached: cached,
|
||||
failed: failed,
|
||||
missing: missing,
|
||||
cache_dir: MsFootprints.cache_dir()
|
||||
}
|
||||
|
||||
Logger.info("buildings bulk fetch: #{inspect(result)}")
|
||||
result
|
||||
end
|
||||
|
||||
defp run_downloads(quadkeys, concurrency) do
|
||||
quadkeys
|
||||
|> Task.async_stream(&fetch_one/1,
|
||||
max_concurrency: concurrency,
|
||||
ordered: false,
|
||||
timeout: @task_timeout_ms,
|
||||
on_timeout: :kill_task
|
||||
)
|
||||
|> Enum.reduce({0, 0, 0}, &accumulate/2)
|
||||
end
|
||||
|
||||
defp fetch_one(qk) do
|
||||
path = Path.join(MsFootprints.cache_dir(), "#{qk}.csv.gz")
|
||||
|
||||
if File.exists?(path) do
|
||||
:cached
|
||||
else
|
||||
case MsFootprints.download_tile(qk) do
|
||||
{:ok, _} -> :downloaded
|
||||
{:error, reason} -> {:failed, qk, reason}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
defp accumulate({:ok, :downloaded}, {d, s, f}), do: {d + 1, s, f}
|
||||
defp accumulate({:ok, :cached}, {d, s, f}), do: {d, s + 1, f}
|
||||
|
||||
defp accumulate({:ok, {:failed, qk, reason}}, {d, s, f}) do
|
||||
Logger.warning("buildings bulk fetch failed #{qk}: #{inspect(reason)}")
|
||||
{d, s, f + 1}
|
||||
end
|
||||
|
||||
defp accumulate({:exit, reason}, {d, s, f}) do
|
||||
Logger.warning("buildings bulk fetch task crashed: #{inspect(reason)}")
|
||||
{d, s, f + 1}
|
||||
end
|
||||
|
||||
defp bbox_around(lat, lon, radius_km) do
|
||||
dlat = radius_km / 111.0
|
||||
dlon = radius_km / (111.0 * max(:math.cos(lat * :math.pi() / 180.0), 0.1))
|
||||
|
||||
%{
|
||||
"south" => lat - dlat,
|
||||
"north" => lat + dlat,
|
||||
"west" => lon - dlon,
|
||||
"east" => lon + dlon
|
||||
}
|
||||
end
|
||||
end
|
||||
49
lib/mix/tasks/buildings_fetch.ex
Normal file
49
lib/mix/tasks/buildings_fetch.ex
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
defmodule Mix.Tasks.BuildingsFetch do
|
||||
@shortdoc "Bulk-download Microsoft Building Footprint tiles around DFW (DEV ONLY)"
|
||||
|
||||
@moduledoc """
|
||||
Pre-warms the local Microsoft Building Footprints cache so per-Calculate
|
||||
requests never have to wait on a tile download.
|
||||
|
||||
mix buildings.fetch # default: 500 mi radius around EM13
|
||||
mix buildings.fetch --lat 32.8 --lon -97 --radius-mi 500
|
||||
mix buildings.fetch --concurrency 8
|
||||
|
||||
**Mix tasks do not run in production.** From an attached `iex` on the
|
||||
prod pod, call the underlying module directly:
|
||||
|
||||
iex> Microwaveprop.Buildings.BulkFetch.run(32.8, -97.0, 500)
|
||||
"""
|
||||
|
||||
use Mix.Task
|
||||
|
||||
alias Microwaveprop.Buildings.BulkFetch
|
||||
|
||||
@default_lat 32.8
|
||||
@default_lon -97.0
|
||||
@default_radius_mi 500
|
||||
|
||||
@switches [
|
||||
lat: :float,
|
||||
lon: :float,
|
||||
radius_mi: :integer,
|
||||
concurrency: :integer
|
||||
]
|
||||
|
||||
@impl Mix.Task
|
||||
def run(argv) do
|
||||
{opts, _, _} = OptionParser.parse(argv, switches: @switches)
|
||||
|
||||
lat = Keyword.get(opts, :lat, @default_lat)
|
||||
lon = Keyword.get(opts, :lon, @default_lon)
|
||||
radius_mi = Keyword.get(opts, :radius_mi, @default_radius_mi)
|
||||
concurrency = Keyword.get(opts, :concurrency)
|
||||
|
||||
Mix.Task.run("app.start")
|
||||
|
||||
bulk_opts = if concurrency, do: [concurrency: concurrency], else: []
|
||||
result = BulkFetch.run(lat, lon, radius_mi, bulk_opts)
|
||||
|
||||
Mix.shell().info("Done. #{inspect(result)}")
|
||||
end
|
||||
end
|
||||
Loading…
Add table
Reference in a new issue