feat(recalibrate): classify data gaps + add NARR/HRRR coverage breakdowns
The "Data gaps" section was a flat key/value dump that buried what was
actually broken (hrrr_climatology=0, rtma=0, metar5=0) under metrics
that are merely informational (pre2014_contacts=207). Operators reading
the report had to know each metric's healthy range by heart.
This commit:
* Classifies every gap metric BROKEN / WARN / OK via DATA_GAP_RULES,
with a one-line remediation hint per non-OK row (e.g. "run
`mix hrrr_climatology`").
* Prints BROKEN items to stderr at runtime so a CI/cron run surfaces
them without anyone opening the markdown file.
* Adds NARR-coverage-by-year and HRRR-status-by-year tables. With
only 358 NARR rows, the year breakdown shows whether the backfill
ever started; with 36k pending HRRR contacts, the year breakdown
shows whether the backlog is a stalled historical backfill or a
stuck live queue — different remediations.
* Reports HRRR enrichment completeness as a percentage instead of
forcing the reader to do the division.
This commit is contained in:
parent
0f21821ca5
commit
2b071c3b56
1 changed files with 217 additions and 3 deletions
|
|
@ -270,7 +270,9 @@ FROM baseline
|
|||
WHERE baseline_rx IS NOT NULL;
|
||||
"""
|
||||
|
||||
# Empty-table / data-quality smoke checks.
|
||||
# Empty-table / data-quality smoke checks. Each metric is classified in
|
||||
# DATA_GAP_RULES below so the report can flag BROKEN vs WARN vs OK with
|
||||
# concrete remediation hints instead of dumping a flat key/value list.
|
||||
DATA_GAP_SQL = """
|
||||
SELECT
|
||||
(SELECT count(*) FROM narr_profiles) AS narr_rows,
|
||||
|
|
@ -286,6 +288,110 @@ SELECT
|
|||
AND flagged_invalid = false) AS pre2014_contacts;
|
||||
"""
|
||||
|
||||
# Coverage breakdown for tables whose total row count alone hides the
|
||||
# real gap — e.g. narr_profiles with 358 rows could be one good year or
|
||||
# 358 stragglers spread across 30. Year-bucketed counts let the report
|
||||
# flag whether the backfill ever made meaningful progress.
|
||||
NARR_COVERAGE_BY_YEAR_SQL = """
|
||||
SELECT EXTRACT(YEAR FROM valid_time)::int AS year, count(*)::bigint AS profiles
|
||||
FROM narr_profiles
|
||||
GROUP BY 1 ORDER BY 1;
|
||||
"""
|
||||
|
||||
# Where is the HRRR enrichment backlog? A 36k pending count could mean
|
||||
# "the backfill stalled in 2018" or "every contact submitted in the last
|
||||
# week is queued". The breakdown drives a different remediation: rerun
|
||||
# HrrrFetchWorker.enqueue_for_qso/1 over a year range vs. unstick the
|
||||
# Oban queue.
|
||||
HRRR_STATUS_BY_YEAR_SQL = """
|
||||
SELECT EXTRACT(YEAR FROM qso_timestamp)::int AS year,
|
||||
count(*) FILTER (WHERE hrrr_status = 'complete')::bigint AS complete,
|
||||
count(*) FILTER (WHERE hrrr_status = 'queued')::bigint AS queued,
|
||||
count(*) FILTER (WHERE hrrr_status = 'failed')::bigint AS failed,
|
||||
count(*) FILTER (WHERE hrrr_status IS NULL)::bigint AS unset
|
||||
FROM contacts
|
||||
WHERE pos1 IS NOT NULL AND distance_km < 3000 AND flagged_invalid = false
|
||||
AND qso_timestamp >= '2014-10-02'
|
||||
GROUP BY 1 ORDER BY 1;
|
||||
"""
|
||||
|
||||
# Severity rules for each metric in DATA_GAP_SQL.
|
||||
# key metric name returned by DATA_GAP_SQL
|
||||
# broken_when lambda(value) → True means BROKEN (red)
|
||||
# warn_when lambda(value) → True means WARN (yellow). Skipped if BROKEN.
|
||||
# remediation one-line action the operator should take
|
||||
# A metric not flagged by either lambda is OK (green).
|
||||
DATA_GAP_RULES: list[dict] = [
|
||||
{
|
||||
"key": "narr_rows",
|
||||
"broken_when": lambda v: v == 0,
|
||||
# 358 rows is the symptom that triggered this rewrite — anything
|
||||
# under ~5k means the NARR backfill hasn't meaningfully started.
|
||||
"warn_when": lambda v: v < 5_000,
|
||||
"remediation": "Run `mix narr.backfill` (see lib/microwaveprop/workers/narr_fetch_worker.ex).",
|
||||
},
|
||||
{
|
||||
"key": "narr_pre2014_rows",
|
||||
"broken_when": lambda v: v == 0,
|
||||
"warn_when": lambda v: v < 5_000,
|
||||
"remediation": (
|
||||
"NARR's purpose is filling the pre-2014-10 gap; without rows here "
|
||||
"the historical-sanity correlation table is empty."
|
||||
),
|
||||
},
|
||||
{
|
||||
"key": "hrrr_climatology_rows",
|
||||
"broken_when": lambda v: v == 0,
|
||||
"warn_when": lambda _v: False,
|
||||
"remediation": "Run `mix hrrr_climatology` (see lib/mix/tasks/hrrr_climatology.ex).",
|
||||
},
|
||||
{
|
||||
"key": "rtma_rows",
|
||||
"broken_when": lambda v: v == 0,
|
||||
"warn_when": lambda _v: False,
|
||||
"remediation": (
|
||||
"Enable the rtma queue and let RtmaFetchWorker run — needed for "
|
||||
"contacts whose HRRR enrichment misses the ±1 h window."
|
||||
),
|
||||
},
|
||||
{
|
||||
"key": "metar5_rows",
|
||||
"broken_when": lambda v: v == 0,
|
||||
"warn_when": lambda _v: False,
|
||||
"remediation": (
|
||||
"No 5-minute METAR ingestor exists yet; if this stays at 0 the "
|
||||
"schema metar_5min_observations is unused — consider dropping it "
|
||||
"or wiring an IEM 5-min ASOS fetcher."
|
||||
),
|
||||
},
|
||||
{
|
||||
"key": "hrrr_complete_contacts",
|
||||
# Healthy is the fraction complete, not the absolute number — handled
|
||||
# separately via _check_hrrr_completeness below.
|
||||
"broken_when": lambda _v: False,
|
||||
"warn_when": lambda _v: False,
|
||||
"remediation": "",
|
||||
},
|
||||
{
|
||||
"key": "hrrr_pending_contacts",
|
||||
"broken_when": lambda _v: False,
|
||||
# Anything over 5k pending is a real backlog worth chasing.
|
||||
"warn_when": lambda v: v > 5_000,
|
||||
"remediation": (
|
||||
"See `HRRR enrichment status by year` table below to locate the "
|
||||
"backlog, then `mix hrrr.backfill --year YYYY` (or rerun "
|
||||
"HrrrFetchWorker.enqueue_for_qso/1 over the affected ids)."
|
||||
),
|
||||
},
|
||||
{
|
||||
"key": "pre2014_contacts",
|
||||
# pre-2014 contacts existing is informational, not a gap on its own.
|
||||
"broken_when": lambda _v: False,
|
||||
"warn_when": lambda _v: False,
|
||||
"remediation": "",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
# ─── Reporting helpers ──────────────────────────────────────────────────────
|
||||
|
||||
|
|
@ -482,6 +588,61 @@ def commercial_degradation_summary(df: pd.DataFrame) -> pd.DataFrame:
|
|||
return out
|
||||
|
||||
|
||||
def classify_data_gaps(gaps: dict) -> pd.DataFrame:
|
||||
"""Apply DATA_GAP_RULES to the raw gap counts.
|
||||
|
||||
Returns a dataframe with columns: metric, value, severity, remediation.
|
||||
`severity` is one of "BROKEN", "WARN", "OK" (informational metrics with
|
||||
no rule are also "OK"). The dataframe order matches DATA_GAP_RULES so
|
||||
related metrics group naturally in the rendered table.
|
||||
"""
|
||||
rows = []
|
||||
for rule in DATA_GAP_RULES:
|
||||
key = rule["key"]
|
||||
if key not in gaps:
|
||||
continue
|
||||
value = gaps[key]
|
||||
if rule["broken_when"](value):
|
||||
severity = "BROKEN"
|
||||
elif rule["warn_when"](value):
|
||||
severity = "WARN"
|
||||
else:
|
||||
severity = "OK"
|
||||
rows.append(
|
||||
{
|
||||
"metric": key,
|
||||
"value": int(value) if value is not None else None,
|
||||
"severity": severity,
|
||||
"remediation": rule["remediation"] if severity != "OK" else "",
|
||||
}
|
||||
)
|
||||
# Append any metric DATA_GAP_SQL returned that we don't have a rule for,
|
||||
# so the report stays exhaustive even if SQL changes ahead of the rules.
|
||||
handled = {r["key"] for r in DATA_GAP_RULES}
|
||||
for key, value in gaps.items():
|
||||
if key in handled:
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"metric": key,
|
||||
"value": int(value) if value is not None else None,
|
||||
"severity": "OK",
|
||||
"remediation": "",
|
||||
}
|
||||
)
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
def hrrr_completeness_pct(gaps: dict) -> float | None:
|
||||
"""Fraction of eligible contacts with hrrr_status = 'complete'."""
|
||||
complete = gaps.get("hrrr_complete_contacts") or 0
|
||||
pending = gaps.get("hrrr_pending_contacts") or 0
|
||||
total = complete + pending
|
||||
if total == 0:
|
||||
return None
|
||||
return round(100.0 * complete / total, 1)
|
||||
|
||||
|
||||
def commercial_correlation(df: pd.DataFrame) -> float | None:
|
||||
"""Pearson correlation of degradation_db with contact distance. None if n<5."""
|
||||
if df.empty or len(df) < 5:
|
||||
|
|
@ -561,10 +722,63 @@ def main() -> int:
|
|||
|
||||
print("• data gaps", file=sys.stderr)
|
||||
gaps = fetch_df(conn, DATA_GAP_SQL).iloc[0].to_dict()
|
||||
sections.append("\n## Data gaps (these should NOT be zero in a healthy prod)\n")
|
||||
gap_table = classify_data_gaps(gaps)
|
||||
completeness = hrrr_completeness_pct(gaps)
|
||||
|
||||
broken = gap_table[gap_table["severity"] == "BROKEN"]
|
||||
for _, row in broken.iterrows():
|
||||
print(
|
||||
f" ⚠ BROKEN: {row['metric']}={row['value']} — {row['remediation']}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
sections.append("\n## Data gaps and ingestion health\n")
|
||||
sections.append(
|
||||
"\n".join(f"- **{k}**: {v}" for k, v in gaps.items()) + "\n"
|
||||
"Severity legend: **BROKEN** = table is empty and feeds an "
|
||||
"active scoring path; **WARN** = populated but coverage is "
|
||||
"low enough to skew per-band statistics; **OK** = within "
|
||||
"spec or informational. Each non-OK row carries a one-line "
|
||||
"remediation hint.\n"
|
||||
)
|
||||
sections.append(md_table(gap_table))
|
||||
if completeness is not None:
|
||||
sections.append(
|
||||
f"\nHRRR enrichment completeness: **{completeness}%** "
|
||||
f"({gaps.get('hrrr_complete_contacts', 0):,} complete / "
|
||||
f"{gaps.get('hrrr_pending_contacts', 0):,} pending).\n"
|
||||
)
|
||||
|
||||
print("• narr coverage by year", file=sys.stderr)
|
||||
narr_year_df = fetch_df(conn, NARR_COVERAGE_BY_YEAR_SQL)
|
||||
sections.append("\n### NARR coverage by year\n")
|
||||
if narr_year_df.empty:
|
||||
sections.append(
|
||||
"_narr_profiles is empty. Run `mix narr.backfill` to "
|
||||
"populate the pre-2014 corpus._\n"
|
||||
)
|
||||
else:
|
||||
sections.append(
|
||||
"Year-bucketed counts. NARR is supposed to span 1979 → "
|
||||
"2014-10 at 3-hourly cadence (~2,920 timestamps per year "
|
||||
"per grid cell). Years far below the others are partial "
|
||||
"backfills the worker hasn't finished.\n"
|
||||
)
|
||||
sections.append(md_table(narr_year_df))
|
||||
|
||||
print("• hrrr status by year", file=sys.stderr)
|
||||
hrrr_year_df = fetch_df(conn, HRRR_STATUS_BY_YEAR_SQL)
|
||||
sections.append("\n### HRRR enrichment status by year\n")
|
||||
if hrrr_year_df.empty:
|
||||
sections.append("_(no contacts with `pos1` set after 2014-10-02)_\n")
|
||||
else:
|
||||
sections.append(
|
||||
"Locate the pending backlog: a year with high `queued`/"
|
||||
"`failed`/`unset` counts is where HrrrFetchWorker needs "
|
||||
"to be rerun. Recent years dominating `queued` usually "
|
||||
"means the live Oban queue is stuck rather than a "
|
||||
"historical backfill gap.\n"
|
||||
)
|
||||
sections.append(md_table(hrrr_year_df))
|
||||
|
||||
print("• contacts by band", file=sys.stderr)
|
||||
sections.append("\n## Contacts by band (≥50 MHz)\n")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue