graham/prop
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use batch insert for HRRR profiles, tune Postgres for 16GB host

Browse source
This commit is contained in:
Graham McIntire 2026-04-03 09:12:24 -05:00
parent 83ab40fdad
commit ebeaaa06e4
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
1 changed files with 28 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
lib/microwaveprop/workers/hrrr_fetch_worker.ex
View file

@ -32,8 +32,14 @@ defmodule Microwaveprop.Workers.HrrrFetchWorker do
case HrrrClient.fetch_grid(point_tuples, valid_time) do
{:ok, grid_data} ->
Enum.each(grid_data, fn {{lat, lon}, data} ->
store_profile(lat, lon, valid_time, data)
profiles =
Enum.map(grid_data, fn {{lat, lon}, data} ->
build_profile_attrs(lat, lon, valid_time, data)
end)
Weather.upsert_hrrr_profiles_batch(profiles)
Enum.each(grid_data, fn {{lat, lon}, _data} ->
broadcast_enrichment(lat, lon, valid_time)
end)
@ -70,26 +76,28 @@ defmodule Microwaveprop.Workers.HrrrFetchWorker do
end
defp store_profile(lat, lon, valid_time, data) do
attrs = build_profile_attrs(lat, lon, valid_time, data)
Weather.upsert_hrrr_profile(attrs)
end
defp build_profile_attrs(lat, lon, valid_time, data) do
params = SoundingParams.derive(data.profile)
attrs =
maybe_add_derived_params(
%{
valid_time: valid_time,
lat: lat,
lon: lon,
run_time: data.run_time,
profile: data.profile,
hpbl_m: data.hpbl_m,
pwat_mm: data.pwat_mm,
surface_temp_c: data.surface_temp_c,
surface_dewpoint_c: data.surface_dewpoint_c,
surface_pressure_mb: data.surface_pressure_mb
},
params
)
Weather.upsert_hrrr_profile(attrs)
maybe_add_derived_params(
%{
valid_time: valid_time,
lat: lat,
lon: lon,
run_time: data.run_time,
profile: data.profile,
hpbl_m: data.hpbl_m,
pwat_mm: data.pwat_mm,
surface_temp_c: data.surface_temp_c,
surface_dewpoint_c: data.surface_dewpoint_c,
surface_pressure_mb: data.surface_pressure_mb
},
params
)
end
defp handle_error(reason, label) do