Graham McIntire
900685aa06
- Spike docs at docs/research/hrrr_native_levels.md confirming files
are on AWS S3 for 5+ years, 50 hybrid levels, and include TKE and
SPFH needed for Phase 2 turbulence features. Architectural finding:
per-point on-demand fetching is impractical (~530 MB/file), so
the ingestion worker batches per (date, hour) instead.
- hrrr_native_profiles schema: arrays per level plus cached surface
scalars and placeholder columns for Phase 2/4 derived fields.
Strictly additive — the existing hrrr_profiles table is untouched.
- HrrrNativeClient: pure URL/message-list helpers, build_native_profile/1
that turns a parsed wgrib2 map into the schema shape (TDD'd).
- Exposed HrrrClient.download_grib_ranges/2 so the native client
reuses the existing parallel byte-range download + disk cache.
- HrrrNativeGridWorker: Oban worker keyed on {year, month, day, hour},
unique at :infinity, pulls distinct (lat, lon) points from contacts
in the ±30 min window, downloads the native grib2, extracts per
point, bulk-upserts.
- mix hrrr_native_backfill --limit N enqueues the top-N hours by
contact count.
Phase 1 gate still pending Task 1.6 (sanity-check backtest after
live data lands).
93 lines
3 KiB
Elixir
93 lines
3 KiB
Elixir
defmodule Microwaveprop.Weather.HrrrNativeProfile do
|
|
@moduledoc """
|
|
A single HRRR profile extracted on the native hybrid-sigma vertical
|
|
coordinate (50 levels), as opposed to `HrrrProfile` which is
|
|
interpolated to a coarse set of pressure levels.
|
|
|
|
This schema is intentionally additive: it does not replace
|
|
`HrrrProfile`. The legacy scorer keeps running against
|
|
`hrrr_profiles` until Phase 9 swaps it out.
|
|
|
|
Columns marked "derived (Phase 2/4)" are written by the
|
|
boundary-layer turbulence and duct-geometry phases — they stay
|
|
nullable so the ingestion worker can populate just the raw profile
|
|
columns and a later `mix hrrr_native_derive` task fills the rest in.
|
|
"""
|
|
use Ecto.Schema
|
|
|
|
import Ecto.Changeset
|
|
|
|
@primary_key {:id, :binary_id, autogenerate: true}
|
|
@foreign_key_type :binary_id
|
|
|
|
schema "hrrr_native_profiles" do
|
|
field :valid_time, :utc_datetime
|
|
field :run_time, :utc_datetime
|
|
field :lat, :float
|
|
field :lon, :float
|
|
|
|
# Raw native-level columns, level 1 (surface) first, strictly
|
|
# monotone in height.
|
|
field :level_count, :integer
|
|
field :heights_m, {:array, :float}
|
|
field :temp_k, {:array, :float}
|
|
field :spfh, {:array, :float}
|
|
field :pressure_pa, {:array, :float}
|
|
field :u_wind_ms, {:array, :float}
|
|
field :v_wind_ms, {:array, :float}
|
|
field :tke_m2s2, {:array, :float}
|
|
|
|
# Cached surface scalars — convenient for backtests that only need
|
|
# the lowest level without unpacking the whole profile.
|
|
field :surface_temp_k, :float
|
|
field :surface_spfh, :float
|
|
field :surface_pressure_pa, :float
|
|
|
|
# Derived (Phase 2/4).
|
|
field :inversion_top_m, :float
|
|
field :bulk_richardson, :float
|
|
field :theta_e_jump_k, :float
|
|
field :shear_at_top_ms, :float
|
|
field :ducts, {:array, :map}
|
|
field :best_duct_band_ghz, :float
|
|
|
|
timestamps(type: :utc_datetime)
|
|
end
|
|
|
|
@required ~w(valid_time lat lon)a
|
|
@optional ~w(
|
|
run_time level_count heights_m temp_k spfh pressure_pa
|
|
u_wind_ms v_wind_ms tke_m2s2
|
|
surface_temp_k surface_spfh surface_pressure_pa
|
|
inversion_top_m bulk_richardson theta_e_jump_k shear_at_top_ms
|
|
ducts best_duct_band_ghz
|
|
)a
|
|
|
|
def changeset(profile, attrs) do
|
|
profile
|
|
|> cast(attrs, @required ++ @optional)
|
|
|> validate_required(@required)
|
|
|> validate_array_lengths()
|
|
|> unique_constraint([:lat, :lon, :valid_time])
|
|
end
|
|
|
|
# All level arrays must have the same length, matching level_count.
|
|
defp validate_array_lengths(changeset) do
|
|
level_count = get_field(changeset, :level_count)
|
|
|
|
arrays =
|
|
[:heights_m, :temp_k, :spfh, :pressure_pa, :u_wind_ms, :v_wind_ms, :tke_m2s2]
|
|
|> Enum.map(&{&1, get_field(changeset, &1)})
|
|
|> Enum.reject(fn {_k, v} -> is_nil(v) end)
|
|
|
|
expected = level_count || (arrays |> Enum.map(fn {_, v} -> length(v) end) |> Enum.max(fn -> 0 end))
|
|
|
|
Enum.reduce(arrays, changeset, fn {key, values}, acc ->
|
|
if length(values) == expected do
|
|
acc
|
|
else
|
|
add_error(acc, key, "length #{length(values)} does not match expected #{expected}")
|
|
end
|
|
end)
|
|
end
|
|
end
|