prop/lib/microwaveprop/partition_manager.ex
Graham McIntire d67186176f
fix: resolve all dialyzer type errors across project (46→0 project errors)
Type spec fixes:
- duct_usable_* return boolean→float (delegate to duct_usable_for_band)
- sanitize/1 spec includes :unicode error tuples
- match_delete/1 broadened from :ets.match_spec()
- telemetry_event local type replaces :telemetry.event/0
- wgrib2 parse_lon_val_segment corrected to tuple spec
- preloaded Ecto assoc types in get_mission/get_contact! specs

Unmatched returns:
- _ = prefix on Task.start, Oban.insert, Repo.query!, PubSub.subscribe,
  :ets.new, and if-expression returns across 18 files

Pattern match fixes:
- markdown: restructure acc!=[] guard as direct pattern match
- path_compute/pskr/skewt_location_resolver: remove dead clauses
- calibrate.aprs_144: remove unreachable format_float catch-all
- unused.ex: suppress MapSet.union no_opaque

Also: remove unused unicode_util_compat from mix.lock
2026-06-08 17:51:13 -05:00

154 lines
6 KiB
Elixir
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

defmodule Microwaveprop.PartitionManager do
@moduledoc """
Ensures rolling quarterly partitions exist on the time-partitioned
tables (`hrrr_profiles`, `hrdps_profiles`).
Historically the partition list was hand-edited in `priv/repo/structure.sql`
whenever someone remembered. When that fell behind, writes for the
next quarter would start failing with `no partition of relation "..."
found for row` — silent at the worker level (the Rust hrrr-point-worker
marked tasks as failed) but blocking the calibration pipeline.
This module is the runtime safety net. The companion
`Microwaveprop.Workers.PartitionMaintenanceWorker` calls it daily so
the parent table always has at least `lookahead_quarters` quarters of
runway.
Design notes:
* Partitions are quarterly. `MM` in the partition name is the
starting month of the quarter (`01`, `04`, `07`, `10`) — matches
what `priv/repo/structure.sql` already uses for recent quarters.
* Some older partitions in the parent table cover wider ranges
(e.g. `hrrr_profiles_2027_01` covers 2027-01 → 2027-07). To
avoid colliding with those, every CREATE is gated by a
pg_inherits coverage check: if any existing child already
contains the target range, the call is a no-op. Postgres still
raises on partial overlap — that's a structural mismatch the
operator should resolve manually rather than have a worker
paper over.
* Postgres auto-attaches the parent's indexes on the new child,
so we only need to issue the `CREATE TABLE ... PARTITION OF`
DDL — no per-index follow-up.
* Quarter math is done on a single integer index
(`year * 4 + (month-1) ÷ 3`) to avoid month-rollover bugs.
"""
alias Microwaveprop.Repo
@default_parents ~w(hrrr_profiles hrdps_profiles)
@typedoc """
Per-partition result. `parent` is the parent table name, `name` is
the resolved child partition name, and the status flags whether
this call performed the CREATE or found an existing partition that
already covered the range.
"""
@type result :: {parent :: String.t(), name :: String.t(), :created | :exists}
@doc """
For each configured parent table, ensure partitions covering the
current quarter and the next `lookahead_quarters` quarters exist.
Returns a list of `{parent, partition_name, :created | :exists}`
entries so the worker can log a single summary line.
"""
@spec ensure_quarterly_partitions(non_neg_integer(), [String.t()]) :: [result()]
def ensure_quarterly_partitions(lookahead_quarters \\ 4, parents \\ @default_parents)
when is_integer(lookahead_quarters) and lookahead_quarters >= 0 and is_list(parents) do
today = Date.utc_today()
quarters = quarter_starts(today, lookahead_quarters)
# One pg_inherits scan per parent (not per parent × lookahead).
# The result is a list of {lo, hi} NaiveDateTime tuples; coverage
# checks then run in-memory.
for parent <- parents, existing = existing_bounds(parent), {q_start, q_end} <- quarters do
ensure_partition_with_bounds(parent, q_start, q_end, existing)
end
end
# Public for direct access from tests / mix tasks. Issues its own
# pg_inherits scan; prefer the bulk path through
# `ensure_quarterly_partitions/2` to amortize it.
@spec ensure_partition(String.t(), Date.t(), Date.t()) :: result()
def ensure_partition(parent, %Date{} = q_start, %Date{} = q_end) do
ensure_partition_with_bounds(parent, q_start, q_end, existing_bounds(parent))
end
defp ensure_partition_with_bounds(parent, q_start, q_end, existing) do
name = partition_name(parent, q_start)
q_start_dt = NaiveDateTime.new!(q_start, ~T[00:00:00])
q_end_dt = NaiveDateTime.new!(q_end, ~T[00:00:00])
if covered_in_memory?(existing, q_start_dt, q_end_dt) do
{parent, name, :exists}
else
_ =
Repo.query!("""
CREATE TABLE public."#{name}"
PARTITION OF public."#{parent}"
FOR VALUES FROM ('#{Date.to_iso8601(q_start)} 00:00:00')
TO ('#{Date.to_iso8601(q_end)} 00:00:00')
""")
{parent, name, :created}
end
end
# Pull every child partition's bound expression for `parent`. Each
# row that matches the standard `FROM ('iso') TO ('iso')` shape is
# parsed into a `{lo_naive, hi_naive}` tuple; rows that don't match
# (DEFAULT partitions, MINVALUE/MAXVALUE, multi-column boundaries,
# etc.) are dropped — they can't usefully participate in a
# range-contains check anyway.
@spec existing_bounds(String.t()) :: [{NaiveDateTime.t(), NaiveDateTime.t()}]
defp existing_bounds(parent) do
%Postgrex.Result{rows: rows} =
Repo.query!(
"""
SELECT pg_get_expr(child.relpartbound, child.oid)
FROM pg_inherits
JOIN pg_class parent ON pg_inherits.inhparent = parent.oid
JOIN pg_class child ON pg_inherits.inhrelid = child.oid
WHERE parent.relname = $1
""",
[parent]
)
Enum.flat_map(rows, fn [bound] ->
case Regex.run(~r/FROM \('([^']+)'\) TO \('([^']+)'\)/, bound) do
[_, lo, hi] -> [{NaiveDateTime.from_iso8601!(lo), NaiveDateTime.from_iso8601!(hi)}]
_ -> []
end
end)
end
defp covered_in_memory?(existing, q_start_dt, q_end_dt) do
Enum.any?(existing, fn {lo, hi} ->
NaiveDateTime.compare(lo, q_start_dt) != :gt and
NaiveDateTime.compare(hi, q_end_dt) != :lt
end)
end
defp partition_name(parent, %Date{year: y, month: m}) do
"#{parent}_#{y}_#{String.pad_leading(Integer.to_string(m), 2, "0")}"
end
# Quarter index = year * 4 + zero-based-quarter-of-year. Going
# through this single integer makes "next quarter" trivial and
# year-rollover free.
defp quarter_starts(%Date{} = today, lookahead) do
current = quarter_index(today)
Enum.map(0..lookahead, fn n ->
q = current + n
{quarter_to_date(q), quarter_to_date(q + 1)}
end)
end
defp quarter_index(%Date{year: y, month: m}), do: y * 4 + div(m - 1, 3)
defp quarter_to_date(qi) do
%Date{year: div(qi, 4), month: rem(qi, 4) * 3 + 1, day: 1}
end
end