From 2b071c3b56bbb5cdffd4ce1188192f28f3459d79 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Sat, 25 Apr 2026 12:17:39 -0500 Subject: [PATCH] feat(recalibrate): classify data gaps + add NARR/HRRR coverage breakdowns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "Data gaps" section was a flat key/value dump that buried what was actually broken (hrrr_climatology=0, rtma=0, metar5=0) under metrics that are merely informational (pre2014_contacts=207). Operators reading the report had to know each metric's healthy range by heart. This commit: * Classifies every gap metric BROKEN / WARN / OK via DATA_GAP_RULES, with a one-line remediation hint per non-OK row (e.g. "run `mix hrrr_climatology`"). * Prints BROKEN items to stderr at runtime so a CI/cron run surfaces them without anyone opening the markdown file. * Adds NARR-coverage-by-year and HRRR-status-by-year tables. With only 358 NARR rows, the year breakdown shows whether the backfill ever started; with 36k pending HRRR contacts, the year breakdown shows whether the backlog is a stalled historical backfill or a stuck live queue — different remediations. * Reports HRRR enrichment completeness as a percentage instead of forcing the reader to do the division. --- scripts/recalibrate_algo.py | 220 +++++++++++++++++++++++++++++++++++- 1 file changed, 217 insertions(+), 3 deletions(-) diff --git a/scripts/recalibrate_algo.py b/scripts/recalibrate_algo.py index 80a6ffe5..7b86a4d9 100755 --- a/scripts/recalibrate_algo.py +++ b/scripts/recalibrate_algo.py @@ -270,7 +270,9 @@ FROM baseline WHERE baseline_rx IS NOT NULL; """ -# Empty-table / data-quality smoke checks. +# Empty-table / data-quality smoke checks. Each metric is classified in +# DATA_GAP_RULES below so the report can flag BROKEN vs WARN vs OK with +# concrete remediation hints instead of dumping a flat key/value list. DATA_GAP_SQL = """ SELECT (SELECT count(*) FROM narr_profiles) AS narr_rows, @@ -286,6 +288,110 @@ SELECT AND flagged_invalid = false) AS pre2014_contacts; """ +# Coverage breakdown for tables whose total row count alone hides the +# real gap — e.g. narr_profiles with 358 rows could be one good year or +# 358 stragglers spread across 30. Year-bucketed counts let the report +# flag whether the backfill ever made meaningful progress. +NARR_COVERAGE_BY_YEAR_SQL = """ +SELECT EXTRACT(YEAR FROM valid_time)::int AS year, count(*)::bigint AS profiles +FROM narr_profiles +GROUP BY 1 ORDER BY 1; +""" + +# Where is the HRRR enrichment backlog? A 36k pending count could mean +# "the backfill stalled in 2018" or "every contact submitted in the last +# week is queued". The breakdown drives a different remediation: rerun +# HrrrFetchWorker.enqueue_for_qso/1 over a year range vs. unstick the +# Oban queue. +HRRR_STATUS_BY_YEAR_SQL = """ +SELECT EXTRACT(YEAR FROM qso_timestamp)::int AS year, + count(*) FILTER (WHERE hrrr_status = 'complete')::bigint AS complete, + count(*) FILTER (WHERE hrrr_status = 'queued')::bigint AS queued, + count(*) FILTER (WHERE hrrr_status = 'failed')::bigint AS failed, + count(*) FILTER (WHERE hrrr_status IS NULL)::bigint AS unset +FROM contacts +WHERE pos1 IS NOT NULL AND distance_km < 3000 AND flagged_invalid = false + AND qso_timestamp >= '2014-10-02' +GROUP BY 1 ORDER BY 1; +""" + +# Severity rules for each metric in DATA_GAP_SQL. +# key metric name returned by DATA_GAP_SQL +# broken_when lambda(value) → True means BROKEN (red) +# warn_when lambda(value) → True means WARN (yellow). Skipped if BROKEN. +# remediation one-line action the operator should take +# A metric not flagged by either lambda is OK (green). +DATA_GAP_RULES: list[dict] = [ + { + "key": "narr_rows", + "broken_when": lambda v: v == 0, + # 358 rows is the symptom that triggered this rewrite — anything + # under ~5k means the NARR backfill hasn't meaningfully started. + "warn_when": lambda v: v < 5_000, + "remediation": "Run `mix narr.backfill` (see lib/microwaveprop/workers/narr_fetch_worker.ex).", + }, + { + "key": "narr_pre2014_rows", + "broken_when": lambda v: v == 0, + "warn_when": lambda v: v < 5_000, + "remediation": ( + "NARR's purpose is filling the pre-2014-10 gap; without rows here " + "the historical-sanity correlation table is empty." + ), + }, + { + "key": "hrrr_climatology_rows", + "broken_when": lambda v: v == 0, + "warn_when": lambda _v: False, + "remediation": "Run `mix hrrr_climatology` (see lib/mix/tasks/hrrr_climatology.ex).", + }, + { + "key": "rtma_rows", + "broken_when": lambda v: v == 0, + "warn_when": lambda _v: False, + "remediation": ( + "Enable the rtma queue and let RtmaFetchWorker run — needed for " + "contacts whose HRRR enrichment misses the ±1 h window." + ), + }, + { + "key": "metar5_rows", + "broken_when": lambda v: v == 0, + "warn_when": lambda _v: False, + "remediation": ( + "No 5-minute METAR ingestor exists yet; if this stays at 0 the " + "schema metar_5min_observations is unused — consider dropping it " + "or wiring an IEM 5-min ASOS fetcher." + ), + }, + { + "key": "hrrr_complete_contacts", + # Healthy is the fraction complete, not the absolute number — handled + # separately via _check_hrrr_completeness below. + "broken_when": lambda _v: False, + "warn_when": lambda _v: False, + "remediation": "", + }, + { + "key": "hrrr_pending_contacts", + "broken_when": lambda _v: False, + # Anything over 5k pending is a real backlog worth chasing. + "warn_when": lambda v: v > 5_000, + "remediation": ( + "See `HRRR enrichment status by year` table below to locate the " + "backlog, then `mix hrrr.backfill --year YYYY` (or rerun " + "HrrrFetchWorker.enqueue_for_qso/1 over the affected ids)." + ), + }, + { + "key": "pre2014_contacts", + # pre-2014 contacts existing is informational, not a gap on its own. + "broken_when": lambda _v: False, + "warn_when": lambda _v: False, + "remediation": "", + }, +] + # ─── Reporting helpers ────────────────────────────────────────────────────── @@ -482,6 +588,61 @@ def commercial_degradation_summary(df: pd.DataFrame) -> pd.DataFrame: return out +def classify_data_gaps(gaps: dict) -> pd.DataFrame: + """Apply DATA_GAP_RULES to the raw gap counts. + + Returns a dataframe with columns: metric, value, severity, remediation. + `severity` is one of "BROKEN", "WARN", "OK" (informational metrics with + no rule are also "OK"). The dataframe order matches DATA_GAP_RULES so + related metrics group naturally in the rendered table. + """ + rows = [] + for rule in DATA_GAP_RULES: + key = rule["key"] + if key not in gaps: + continue + value = gaps[key] + if rule["broken_when"](value): + severity = "BROKEN" + elif rule["warn_when"](value): + severity = "WARN" + else: + severity = "OK" + rows.append( + { + "metric": key, + "value": int(value) if value is not None else None, + "severity": severity, + "remediation": rule["remediation"] if severity != "OK" else "", + } + ) + # Append any metric DATA_GAP_SQL returned that we don't have a rule for, + # so the report stays exhaustive even if SQL changes ahead of the rules. + handled = {r["key"] for r in DATA_GAP_RULES} + for key, value in gaps.items(): + if key in handled: + continue + rows.append( + { + "metric": key, + "value": int(value) if value is not None else None, + "severity": "OK", + "remediation": "", + } + ) + return pd.DataFrame(rows) + + +def hrrr_completeness_pct(gaps: dict) -> float | None: + """Fraction of eligible contacts with hrrr_status = 'complete'.""" + complete = gaps.get("hrrr_complete_contacts") or 0 + pending = gaps.get("hrrr_pending_contacts") or 0 + total = complete + pending + if total == 0: + return None + return round(100.0 * complete / total, 1) + + def commercial_correlation(df: pd.DataFrame) -> float | None: """Pearson correlation of degradation_db with contact distance. None if n<5.""" if df.empty or len(df) < 5: @@ -561,10 +722,63 @@ def main() -> int: print("• data gaps", file=sys.stderr) gaps = fetch_df(conn, DATA_GAP_SQL).iloc[0].to_dict() - sections.append("\n## Data gaps (these should NOT be zero in a healthy prod)\n") + gap_table = classify_data_gaps(gaps) + completeness = hrrr_completeness_pct(gaps) + + broken = gap_table[gap_table["severity"] == "BROKEN"] + for _, row in broken.iterrows(): + print( + f" ⚠ BROKEN: {row['metric']}={row['value']} — {row['remediation']}", + file=sys.stderr, + ) + + sections.append("\n## Data gaps and ingestion health\n") sections.append( - "\n".join(f"- **{k}**: {v}" for k, v in gaps.items()) + "\n" + "Severity legend: **BROKEN** = table is empty and feeds an " + "active scoring path; **WARN** = populated but coverage is " + "low enough to skew per-band statistics; **OK** = within " + "spec or informational. Each non-OK row carries a one-line " + "remediation hint.\n" ) + sections.append(md_table(gap_table)) + if completeness is not None: + sections.append( + f"\nHRRR enrichment completeness: **{completeness}%** " + f"({gaps.get('hrrr_complete_contacts', 0):,} complete / " + f"{gaps.get('hrrr_pending_contacts', 0):,} pending).\n" + ) + + print("• narr coverage by year", file=sys.stderr) + narr_year_df = fetch_df(conn, NARR_COVERAGE_BY_YEAR_SQL) + sections.append("\n### NARR coverage by year\n") + if narr_year_df.empty: + sections.append( + "_narr_profiles is empty. Run `mix narr.backfill` to " + "populate the pre-2014 corpus._\n" + ) + else: + sections.append( + "Year-bucketed counts. NARR is supposed to span 1979 → " + "2014-10 at 3-hourly cadence (~2,920 timestamps per year " + "per grid cell). Years far below the others are partial " + "backfills the worker hasn't finished.\n" + ) + sections.append(md_table(narr_year_df)) + + print("• hrrr status by year", file=sys.stderr) + hrrr_year_df = fetch_df(conn, HRRR_STATUS_BY_YEAR_SQL) + sections.append("\n### HRRR enrichment status by year\n") + if hrrr_year_df.empty: + sections.append("_(no contacts with `pos1` set after 2014-10-02)_\n") + else: + sections.append( + "Locate the pending backlog: a year with high `queued`/" + "`failed`/`unset` counts is where HrrrFetchWorker needs " + "to be rerun. Recent years dominating `queued` usually " + "means the live Oban queue is stuck rather than a " + "historical backfill gap.\n" + ) + sections.append(md_table(hrrr_year_df)) print("• contacts by band", file=sys.stderr) sections.append("\n## Contacts by band (≥50 MHz)\n")