graham/prop
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use batch insert for HRRR profiles, tune Postgres for 16GB host

Browse source
This commit is contained in:
Graham McIntire 2026-04-03 09:12:24 -05:00
parent 83ab40fdad
commit ebeaaa06e4
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
1 changed files with 28 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
lib/microwaveprop/workers/hrrr_fetch_worker.ex
View file

@ -32,8 +32,14 @@ defmodule Microwaveprop.Workers.HrrrFetchWorker do
case HrrrClient.fetch_grid(point_tuples, valid_time) do case HrrrClient.fetch_grid(point_tuples, valid_time) do
{:ok, grid_data} -> {:ok, grid_data} ->
Enum.each(grid_data, fn {{lat, lon}, data} -> profiles =
store_profile(lat, lon, valid_time, data) Enum.map(grid_data, fn {{lat, lon}, data} ->
build_profile_attrs(lat, lon, valid_time, data)
end)
Weather.upsert_hrrr_profiles_batch(profiles)
Enum.each(grid_data, fn {{lat, lon}, _data} ->
broadcast_enrichment(lat, lon, valid_time) broadcast_enrichment(lat, lon, valid_time)
end) end)
@ -70,26 +76,28 @@ defmodule Microwaveprop.Workers.HrrrFetchWorker do
end end
defp store_profile(lat, lon, valid_time, data) do defp store_profile(lat, lon, valid_time, data) do
attrs = build_profile_attrs(lat, lon, valid_time, data)
Weather.upsert_hrrr_profile(attrs)
end
defp build_profile_attrs(lat, lon, valid_time, data) do
params = SoundingParams.derive(data.profile) params = SoundingParams.derive(data.profile)
attrs = maybe_add_derived_params(
maybe_add_derived_params( %{
%{ valid_time: valid_time,
valid_time: valid_time, lat: lat,
lat: lat, lon: lon,
lon: lon, run_time: data.run_time,
run_time: data.run_time, profile: data.profile,
profile: data.profile, hpbl_m: data.hpbl_m,
hpbl_m: data.hpbl_m, pwat_mm: data.pwat_mm,
pwat_mm: data.pwat_mm, surface_temp_c: data.surface_temp_c,
surface_temp_c: data.surface_temp_c, surface_dewpoint_c: data.surface_dewpoint_c,
surface_dewpoint_c: data.surface_dewpoint_c, surface_pressure_mb: data.surface_pressure_mb
surface_pressure_mb: data.surface_pressure_mb },
}, params
params )
)
Weather.upsert_hrrr_profile(attrs)
end end
defp handle_error(reason, label) do defp handle_error(reason, label) do