Filter weather grid by bounds before deriving sounding params

Timeline scrubs on /weather were reading a 21 MB NFS ProfilesFile and
then calling SoundingParams.derive on all 92k CONUS points before
filtering to the viewport. Flip the order: filter the raw grid_data map
by bounds first, so derivation work is bounded by the viewport (~4k
points on a DFW box) instead of the full grid.

Add build_grid_cache_rows/3 with an optional bounds argument; the /2
arity is unchanged so PropagationGridWorker still derives the full grid
on f00.
This commit is contained in:
Graham McIntire 2026-04-15 15:13:55 -05:00
parent ba23ca2b3e
commit 529102d551
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
2 changed files with 84 additions and 6 deletions

View file

@ -466,9 +466,11 @@ defmodule Microwaveprop.Weather do
:miss ->
case ProfilesFile.read(valid_time) do
{:ok, grid_data} ->
grid_data
|> build_grid_cache_rows(valid_time)
|> filter_weather_bounds(bounds)
# Filter-before-derive: only derive the points inside the
# viewport instead of all 92k CONUS grid points. On a DFW
# viewport that's ~20× less `SoundingParams.derive` work
# per timeline scrub.
build_grid_cache_rows(grid_data, valid_time, bounds)
{:error, _} ->
[]
@ -599,13 +601,22 @@ defmodule Microwaveprop.Weather do
output matches the shape of `load_weather_grid_from_db/1` after
`derive_and_clean/1`.
"""
@spec build_grid_cache_rows(%{{float(), float()} => map()}, DateTime.t()) :: [map()]
def build_grid_cache_rows(grid_data, valid_time) do
Enum.flat_map(grid_data, fn {{lat, lon}, profile} ->
@spec build_grid_cache_rows(%{{float(), float()} => map()}, DateTime.t(), map() | nil) ::
[map()]
def build_grid_cache_rows(grid_data, valid_time, bounds \\ nil) do
grid_data
|> filter_grid_data_bounds(bounds)
|> Enum.flat_map(fn {{lat, lon}, profile} ->
build_grid_cache_row(lat, lon, profile, valid_time)
end)
end
defp filter_grid_data_bounds(grid_data, nil), do: grid_data
defp filter_grid_data_bounds(grid_data, %{"south" => s, "north" => n, "west" => w, "east" => e}) do
:maps.filter(fn {lat, lon}, _ -> lat >= s and lat <= n and lon >= w and lon <= e end, grid_data)
end
defp build_grid_cache_row(lat, lon, profile, valid_time) do
temp_c = profile[:surface_temp_c]

View file

@ -302,6 +302,73 @@ defmodule Microwaveprop.WeatherGridTest do
assert hd(result).lat == 32.375
end
test "with a bounds filter only derives in-bounds points" do
# Filter-before-derive: when called with bounds, the builder must
# skip the `SoundingParams.derive` work for every point outside
# the viewport. Out-of-bounds points are left as raw profile maps
# so the heavy per-point derivation cost is bounded by the
# viewport size, not the full CONUS grid.
now = DateTime.truncate(DateTime.utc_now(), :second)
profile = [
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
%{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 14.0, "hght" => 800.0},
%{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0}
]
base_profile = %{
profile: profile,
surface_temp_c: 25.0,
surface_dewpoint_c: 18.0,
surface_pressure_mb: 1013.0,
hpbl_m: 500.0,
pwat_mm: 30.0
}
grid_data = %{
{32.125, -97.375} => base_profile,
{40.000, -75.000} => base_profile,
{45.000, -120.000} => base_profile
}
bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0}
rows = Weather.build_grid_cache_rows(grid_data, now, bounds)
assert length(rows) == 1
assert hd(rows).lat == 32.125
assert hd(rows).lon == -97.375
end
test "with nil bounds derives every point (backwards compatible)" do
now = DateTime.truncate(DateTime.utc_now(), :second)
profile = [
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
%{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 14.0, "hght" => 800.0}
]
grid_data = %{
{32.125, -97.375} => %{
profile: profile,
surface_temp_c: 25.0,
surface_dewpoint_c: 18.0,
surface_pressure_mb: 1013.0,
hpbl_m: 500.0,
pwat_mm: 30.0
},
{40.000, -75.000} => %{
profile: profile,
surface_temp_c: 22.0,
surface_dewpoint_c: 15.0,
surface_pressure_mb: 1015.0,
hpbl_m: 400.0,
pwat_mm: 25.0
}
}
assert length(Weather.build_grid_cache_rows(grid_data, now, nil)) == 2
end
test "strips raw profile/duct_characteristics/surface_temp_c/surface_dewpoint_c fields" do
now = DateTime.truncate(DateTime.utc_now(), :second)