Filter weather grid by bounds before deriving sounding params
Timeline scrubs on /weather were reading a 21 MB NFS ProfilesFile and then calling SoundingParams.derive on all 92k CONUS points before filtering to the viewport. Flip the order: filter the raw grid_data map by bounds first, so derivation work is bounded by the viewport (~4k points on a DFW box) instead of the full grid. Add build_grid_cache_rows/3 with an optional bounds argument; the /2 arity is unchanged so PropagationGridWorker still derives the full grid on f00.
This commit is contained in:
parent
ba23ca2b3e
commit
529102d551
2 changed files with 84 additions and 6 deletions
|
|
@ -466,9 +466,11 @@ defmodule Microwaveprop.Weather do
|
|||
:miss ->
|
||||
case ProfilesFile.read(valid_time) do
|
||||
{:ok, grid_data} ->
|
||||
grid_data
|
||||
|> build_grid_cache_rows(valid_time)
|
||||
|> filter_weather_bounds(bounds)
|
||||
# Filter-before-derive: only derive the points inside the
|
||||
# viewport instead of all 92k CONUS grid points. On a DFW
|
||||
# viewport that's ~20× less `SoundingParams.derive` work
|
||||
# per timeline scrub.
|
||||
build_grid_cache_rows(grid_data, valid_time, bounds)
|
||||
|
||||
{:error, _} ->
|
||||
[]
|
||||
|
|
@ -599,13 +601,22 @@ defmodule Microwaveprop.Weather do
|
|||
output matches the shape of `load_weather_grid_from_db/1` after
|
||||
`derive_and_clean/1`.
|
||||
"""
|
||||
@spec build_grid_cache_rows(%{{float(), float()} => map()}, DateTime.t()) :: [map()]
|
||||
def build_grid_cache_rows(grid_data, valid_time) do
|
||||
Enum.flat_map(grid_data, fn {{lat, lon}, profile} ->
|
||||
@spec build_grid_cache_rows(%{{float(), float()} => map()}, DateTime.t(), map() | nil) ::
|
||||
[map()]
|
||||
def build_grid_cache_rows(grid_data, valid_time, bounds \\ nil) do
|
||||
grid_data
|
||||
|> filter_grid_data_bounds(bounds)
|
||||
|> Enum.flat_map(fn {{lat, lon}, profile} ->
|
||||
build_grid_cache_row(lat, lon, profile, valid_time)
|
||||
end)
|
||||
end
|
||||
|
||||
defp filter_grid_data_bounds(grid_data, nil), do: grid_data
|
||||
|
||||
defp filter_grid_data_bounds(grid_data, %{"south" => s, "north" => n, "west" => w, "east" => e}) do
|
||||
:maps.filter(fn {lat, lon}, _ -> lat >= s and lat <= n and lon >= w and lon <= e end, grid_data)
|
||||
end
|
||||
|
||||
defp build_grid_cache_row(lat, lon, profile, valid_time) do
|
||||
temp_c = profile[:surface_temp_c]
|
||||
|
||||
|
|
|
|||
|
|
@ -302,6 +302,73 @@ defmodule Microwaveprop.WeatherGridTest do
|
|||
assert hd(result).lat == 32.375
|
||||
end
|
||||
|
||||
test "with a bounds filter only derives in-bounds points" do
|
||||
# Filter-before-derive: when called with bounds, the builder must
|
||||
# skip the `SoundingParams.derive` work for every point outside
|
||||
# the viewport. Out-of-bounds points are left as raw profile maps
|
||||
# so the heavy per-point derivation cost is bounded by the
|
||||
# viewport size, not the full CONUS grid.
|
||||
now = DateTime.truncate(DateTime.utc_now(), :second)
|
||||
|
||||
profile = [
|
||||
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
|
||||
%{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 14.0, "hght" => 800.0},
|
||||
%{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0}
|
||||
]
|
||||
|
||||
base_profile = %{
|
||||
profile: profile,
|
||||
surface_temp_c: 25.0,
|
||||
surface_dewpoint_c: 18.0,
|
||||
surface_pressure_mb: 1013.0,
|
||||
hpbl_m: 500.0,
|
||||
pwat_mm: 30.0
|
||||
}
|
||||
|
||||
grid_data = %{
|
||||
{32.125, -97.375} => base_profile,
|
||||
{40.000, -75.000} => base_profile,
|
||||
{45.000, -120.000} => base_profile
|
||||
}
|
||||
|
||||
bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0}
|
||||
rows = Weather.build_grid_cache_rows(grid_data, now, bounds)
|
||||
|
||||
assert length(rows) == 1
|
||||
assert hd(rows).lat == 32.125
|
||||
assert hd(rows).lon == -97.375
|
||||
end
|
||||
|
||||
test "with nil bounds derives every point (backwards compatible)" do
|
||||
now = DateTime.truncate(DateTime.utc_now(), :second)
|
||||
|
||||
profile = [
|
||||
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
|
||||
%{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 14.0, "hght" => 800.0}
|
||||
]
|
||||
|
||||
grid_data = %{
|
||||
{32.125, -97.375} => %{
|
||||
profile: profile,
|
||||
surface_temp_c: 25.0,
|
||||
surface_dewpoint_c: 18.0,
|
||||
surface_pressure_mb: 1013.0,
|
||||
hpbl_m: 500.0,
|
||||
pwat_mm: 30.0
|
||||
},
|
||||
{40.000, -75.000} => %{
|
||||
profile: profile,
|
||||
surface_temp_c: 22.0,
|
||||
surface_dewpoint_c: 15.0,
|
||||
surface_pressure_mb: 1015.0,
|
||||
hpbl_m: 400.0,
|
||||
pwat_mm: 25.0
|
||||
}
|
||||
}
|
||||
|
||||
assert length(Weather.build_grid_cache_rows(grid_data, now, nil)) == 2
|
||||
end
|
||||
|
||||
test "strips raw profile/duct_characteristics/surface_temp_c/surface_dewpoint_c fields" do
|
||||
now = DateTime.truncate(DateTime.utc_now(), :second)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue