Graham McIntire
e7a7ae073d
Task 9.3 - Weight recalibration via gradient descent:
- Recalibrator module fits logistic regression weights using Nx
- Trains on QSO positives vs random baseline negatives
- Cross-validates by month, normalizes weights to sum to 1.0
- Mix task: mix recalibrate_scorer --sample 5000 --epochs 2000
Task 9.4 - Side-by-side scorer comparison:
- ScorerDiff.compare/3 re-scores grid with old vs new weights
- Reports mean diff, regressions, improvements, per-band breakdown
- Mix task: mix scorer_diff --new-weights '{...}'
Phase 3 - NEXRAD ingestion pipeline:
- NexradClient fetches IEM n0q composite PNGs, extracts per-point
box statistics (mean/max dBZ, texture variance)
- NexradObservation schema with unique (lat, lon, observed_at)
- NexradWorker on :nexrad queue for background processing
- nexrad_texture backtest feature in Features module
- mix nexrad_backfill --limit 200
All tasks added to AdminTaskWorker and Release for production use.
1116 tests, 0 failures.
93 lines
3 KiB
Elixir
93 lines
3 KiB
Elixir
defmodule Microwaveprop.Weather.HrrrNativeProfile do
|
|
@moduledoc """
|
|
A single HRRR profile extracted on the native hybrid-sigma vertical
|
|
coordinate (50 levels), as opposed to `HrrrProfile` which is
|
|
interpolated to a coarse set of pressure levels.
|
|
|
|
This schema is intentionally additive: it does not replace
|
|
`HrrrProfile`. The legacy scorer keeps running against
|
|
`hrrr_profiles` until Phase 9 swaps it out.
|
|
|
|
Columns marked "derived (Phase 2/4)" are written by the
|
|
boundary-layer turbulence and duct-geometry phases — they stay
|
|
nullable so the ingestion worker can populate just the raw profile
|
|
columns and a later `mix hrrr_native_derive` task fills the rest in.
|
|
"""
|
|
use Ecto.Schema
|
|
|
|
import Ecto.Changeset
|
|
|
|
@primary_key {:id, :binary_id, autogenerate: true}
|
|
@foreign_key_type :binary_id
|
|
|
|
schema "hrrr_native_profiles" do
|
|
field :valid_time, :utc_datetime
|
|
field :run_time, :utc_datetime
|
|
field :lat, :float
|
|
field :lon, :float
|
|
|
|
# Raw native-level columns, level 1 (surface) first, strictly
|
|
# monotone in height.
|
|
field :level_count, :integer
|
|
field :heights_m, {:array, :float}
|
|
field :temp_k, {:array, :float}
|
|
field :spfh, {:array, :float}
|
|
field :pressure_pa, {:array, :float}
|
|
field :u_wind_ms, {:array, :float}
|
|
field :v_wind_ms, {:array, :float}
|
|
field :tke_m2s2, {:array, :float}
|
|
|
|
# Cached surface scalars — convenient for backtests that only need
|
|
# the lowest level without unpacking the whole profile.
|
|
field :surface_temp_k, :float
|
|
field :surface_spfh, :float
|
|
field :surface_pressure_pa, :float
|
|
|
|
# Derived (Phase 2/4).
|
|
field :inversion_top_m, :float
|
|
field :bulk_richardson, :float
|
|
field :theta_e_jump_k, :float
|
|
field :shear_at_top_ms, :float
|
|
field :ducts, {:array, :map}
|
|
field :best_duct_band_ghz, :float
|
|
|
|
timestamps(type: :utc_datetime)
|
|
end
|
|
|
|
@required ~w(valid_time lat lon)a
|
|
@optional ~w(
|
|
run_time level_count heights_m temp_k spfh pressure_pa
|
|
u_wind_ms v_wind_ms tke_m2s2
|
|
surface_temp_k surface_spfh surface_pressure_pa
|
|
inversion_top_m bulk_richardson theta_e_jump_k shear_at_top_ms
|
|
ducts best_duct_band_ghz
|
|
)a
|
|
|
|
def changeset(profile, attrs) do
|
|
profile
|
|
|> cast(attrs, @required ++ @optional)
|
|
|> validate_required(@required)
|
|
|> validate_array_lengths()
|
|
|> unique_constraint([:lat, :lon, :valid_time])
|
|
end
|
|
|
|
# All level arrays must have the same length, matching level_count.
|
|
defp validate_array_lengths(changeset) do
|
|
level_count = get_field(changeset, :level_count)
|
|
|
|
arrays =
|
|
[:heights_m, :temp_k, :spfh, :pressure_pa, :u_wind_ms, :v_wind_ms, :tke_m2s2]
|
|
|> Enum.map(&{&1, get_field(changeset, &1)})
|
|
|> Enum.reject(fn {_k, v} -> is_nil(v) end)
|
|
|
|
expected = level_count || arrays |> Enum.map(fn {_, v} -> length(v) end) |> Enum.max(fn -> 0 end)
|
|
|
|
Enum.reduce(arrays, changeset, fn {key, values}, acc ->
|
|
if length(values) == expected do
|
|
acc
|
|
else
|
|
add_error(acc, key, "length #{length(values)} does not match expected #{expected}")
|
|
end
|
|
end)
|
|
end
|
|
end
|