Graham McIntire
900685aa06
- Spike docs at docs/research/hrrr_native_levels.md confirming files
are on AWS S3 for 5+ years, 50 hybrid levels, and include TKE and
SPFH needed for Phase 2 turbulence features. Architectural finding:
per-point on-demand fetching is impractical (~530 MB/file), so
the ingestion worker batches per (date, hour) instead.
- hrrr_native_profiles schema: arrays per level plus cached surface
scalars and placeholder columns for Phase 2/4 derived fields.
Strictly additive — the existing hrrr_profiles table is untouched.
- HrrrNativeClient: pure URL/message-list helpers, build_native_profile/1
that turns a parsed wgrib2 map into the schema shape (TDD'd).
- Exposed HrrrClient.download_grib_ranges/2 so the native client
reuses the existing parallel byte-range download + disk cache.
- HrrrNativeGridWorker: Oban worker keyed on {year, month, day, hour},
unique at :infinity, pulls distinct (lat, lon) points from contacts
in the ±30 min window, downloads the native grib2, extracts per
point, bulk-upserts.
- mix hrrr_native_backfill --limit N enqueues the top-N hours by
contact count.
Phase 1 gate still pending Task 1.6 (sanity-check backtest after
live data lands).
45 lines
1.6 KiB
Elixir
45 lines
1.6 KiB
Elixir
defmodule Microwaveprop.Repo.Migrations.CreateHrrrNativeProfiles do
|
|
use Ecto.Migration
|
|
|
|
def change do
|
|
create table(:hrrr_native_profiles, primary_key: false) do
|
|
add :id, :binary_id, primary_key: true, null: false
|
|
add :valid_time, :utc_datetime, null: false
|
|
add :run_time, :utc_datetime
|
|
add :lat, :float, null: false
|
|
add :lon, :float, null: false
|
|
|
|
# Hybrid-sigma level data, sorted ascending by height (level 1 first).
|
|
# Each array has the same length (HRRR native has 50 hybrid levels);
|
|
# leaving them as arrays avoids a sidecar table and keeps per-profile
|
|
# queries cheap.
|
|
add :level_count, :integer
|
|
add :heights_m, {:array, :float}
|
|
add :temp_k, {:array, :float}
|
|
add :spfh, {:array, :float}
|
|
add :pressure_pa, {:array, :float}
|
|
add :u_wind_ms, {:array, :float}
|
|
add :v_wind_ms, {:array, :float}
|
|
add :tke_m2s2, {:array, :float}
|
|
|
|
# Cached surface-level scalars for quick lookup without unpacking arrays.
|
|
add :surface_temp_k, :float
|
|
add :surface_spfh, :float
|
|
add :surface_pressure_pa, :float
|
|
|
|
# Derived fields populated by Phase 2/4. Nullable so the schema
|
|
# migration stays additive; the derive task fills them in later.
|
|
add :inversion_top_m, :float
|
|
add :bulk_richardson, :float
|
|
add :theta_e_jump_k, :float
|
|
add :shear_at_top_ms, :float
|
|
add :ducts, :map
|
|
add :best_duct_band_ghz, :float
|
|
|
|
timestamps(type: :utc_datetime)
|
|
end
|
|
|
|
create unique_index(:hrrr_native_profiles, [:lat, :lon, :valid_time])
|
|
create index(:hrrr_native_profiles, [:valid_time])
|
|
end
|
|
end
|