Filter weather grid by bounds before deriving sounding params
Timeline scrubs on /weather were reading a 21 MB NFS ProfilesFile and then calling SoundingParams.derive on all 92k CONUS points before filtering to the viewport. Flip the order: filter the raw grid_data map by bounds first, so derivation work is bounded by the viewport (~4k points on a DFW box) instead of the full grid. Add build_grid_cache_rows/3 with an optional bounds argument; the /2 arity is unchanged so PropagationGridWorker still derives the full grid on f00.
This commit is contained in:
parent
ba23ca2b3e
commit
529102d551
2 changed files with 84 additions and 6 deletions
|
|
@ -466,9 +466,11 @@ defmodule Microwaveprop.Weather do
|
||||||
:miss ->
|
:miss ->
|
||||||
case ProfilesFile.read(valid_time) do
|
case ProfilesFile.read(valid_time) do
|
||||||
{:ok, grid_data} ->
|
{:ok, grid_data} ->
|
||||||
grid_data
|
# Filter-before-derive: only derive the points inside the
|
||||||
|> build_grid_cache_rows(valid_time)
|
# viewport instead of all 92k CONUS grid points. On a DFW
|
||||||
|> filter_weather_bounds(bounds)
|
# viewport that's ~20× less `SoundingParams.derive` work
|
||||||
|
# per timeline scrub.
|
||||||
|
build_grid_cache_rows(grid_data, valid_time, bounds)
|
||||||
|
|
||||||
{:error, _} ->
|
{:error, _} ->
|
||||||
[]
|
[]
|
||||||
|
|
@ -599,13 +601,22 @@ defmodule Microwaveprop.Weather do
|
||||||
output matches the shape of `load_weather_grid_from_db/1` after
|
output matches the shape of `load_weather_grid_from_db/1` after
|
||||||
`derive_and_clean/1`.
|
`derive_and_clean/1`.
|
||||||
"""
|
"""
|
||||||
@spec build_grid_cache_rows(%{{float(), float()} => map()}, DateTime.t()) :: [map()]
|
@spec build_grid_cache_rows(%{{float(), float()} => map()}, DateTime.t(), map() | nil) ::
|
||||||
def build_grid_cache_rows(grid_data, valid_time) do
|
[map()]
|
||||||
Enum.flat_map(grid_data, fn {{lat, lon}, profile} ->
|
def build_grid_cache_rows(grid_data, valid_time, bounds \\ nil) do
|
||||||
|
grid_data
|
||||||
|
|> filter_grid_data_bounds(bounds)
|
||||||
|
|> Enum.flat_map(fn {{lat, lon}, profile} ->
|
||||||
build_grid_cache_row(lat, lon, profile, valid_time)
|
build_grid_cache_row(lat, lon, profile, valid_time)
|
||||||
end)
|
end)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
defp filter_grid_data_bounds(grid_data, nil), do: grid_data
|
||||||
|
|
||||||
|
defp filter_grid_data_bounds(grid_data, %{"south" => s, "north" => n, "west" => w, "east" => e}) do
|
||||||
|
:maps.filter(fn {lat, lon}, _ -> lat >= s and lat <= n and lon >= w and lon <= e end, grid_data)
|
||||||
|
end
|
||||||
|
|
||||||
defp build_grid_cache_row(lat, lon, profile, valid_time) do
|
defp build_grid_cache_row(lat, lon, profile, valid_time) do
|
||||||
temp_c = profile[:surface_temp_c]
|
temp_c = profile[:surface_temp_c]
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -302,6 +302,73 @@ defmodule Microwaveprop.WeatherGridTest do
|
||||||
assert hd(result).lat == 32.375
|
assert hd(result).lat == 32.375
|
||||||
end
|
end
|
||||||
|
|
||||||
|
test "with a bounds filter only derives in-bounds points" do
|
||||||
|
# Filter-before-derive: when called with bounds, the builder must
|
||||||
|
# skip the `SoundingParams.derive` work for every point outside
|
||||||
|
# the viewport. Out-of-bounds points are left as raw profile maps
|
||||||
|
# so the heavy per-point derivation cost is bounded by the
|
||||||
|
# viewport size, not the full CONUS grid.
|
||||||
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
||||||
|
|
||||||
|
profile = [
|
||||||
|
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
|
||||||
|
%{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 14.0, "hght" => 800.0},
|
||||||
|
%{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0}
|
||||||
|
]
|
||||||
|
|
||||||
|
base_profile = %{
|
||||||
|
profile: profile,
|
||||||
|
surface_temp_c: 25.0,
|
||||||
|
surface_dewpoint_c: 18.0,
|
||||||
|
surface_pressure_mb: 1013.0,
|
||||||
|
hpbl_m: 500.0,
|
||||||
|
pwat_mm: 30.0
|
||||||
|
}
|
||||||
|
|
||||||
|
grid_data = %{
|
||||||
|
{32.125, -97.375} => base_profile,
|
||||||
|
{40.000, -75.000} => base_profile,
|
||||||
|
{45.000, -120.000} => base_profile
|
||||||
|
}
|
||||||
|
|
||||||
|
bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0}
|
||||||
|
rows = Weather.build_grid_cache_rows(grid_data, now, bounds)
|
||||||
|
|
||||||
|
assert length(rows) == 1
|
||||||
|
assert hd(rows).lat == 32.125
|
||||||
|
assert hd(rows).lon == -97.375
|
||||||
|
end
|
||||||
|
|
||||||
|
test "with nil bounds derives every point (backwards compatible)" do
|
||||||
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
||||||
|
|
||||||
|
profile = [
|
||||||
|
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
|
||||||
|
%{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 14.0, "hght" => 800.0}
|
||||||
|
]
|
||||||
|
|
||||||
|
grid_data = %{
|
||||||
|
{32.125, -97.375} => %{
|
||||||
|
profile: profile,
|
||||||
|
surface_temp_c: 25.0,
|
||||||
|
surface_dewpoint_c: 18.0,
|
||||||
|
surface_pressure_mb: 1013.0,
|
||||||
|
hpbl_m: 500.0,
|
||||||
|
pwat_mm: 30.0
|
||||||
|
},
|
||||||
|
{40.000, -75.000} => %{
|
||||||
|
profile: profile,
|
||||||
|
surface_temp_c: 22.0,
|
||||||
|
surface_dewpoint_c: 15.0,
|
||||||
|
surface_pressure_mb: 1015.0,
|
||||||
|
hpbl_m: 400.0,
|
||||||
|
pwat_mm: 25.0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
assert length(Weather.build_grid_cache_rows(grid_data, now, nil)) == 2
|
||||||
|
end
|
||||||
|
|
||||||
test "strips raw profile/duct_characteristics/surface_temp_c/surface_dewpoint_c fields" do
|
test "strips raw profile/duct_characteristics/surface_temp_c/surface_dewpoint_c fields" do
|
||||||
now = DateTime.truncate(DateTime.utc_now(), :second)
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue