Adds the test that verifies the full producer→drain→re-pass loop: sample written with NULL HRRR fields + fetch task enqueued, then once an HRRR profile lands, the next sampler pass UPSERTs the same calibration_samples row (id preserved) with non-NULL weather data. Also tightens dialyzer surface: * PartitionManager defines a `result()` typedoc and uses it on both public functions instead of inlined tuple types. * PartitionMaintenanceWorker.perform/1 + the lookahead/1 helper get explicit specs. * CalibrationSampler.enqueue_missing_hrrr/3 gets a spec covering its group_by-shape input map and HRRR profile list. `mix precommit` clean (3310 tests, 0 failures). Local `mix dialyzer` hits an OTP 28.4 vs 28.5 toolchain mismatch unrelated to this change.
143 lines
5.3 KiB
Elixir
143 lines
5.3 KiB
Elixir
defmodule Microwaveprop.PartitionManager do
|
|
@moduledoc """
|
|
Ensures rolling quarterly partitions exist on the time-partitioned
|
|
tables (`hrrr_profiles`, `hrdps_profiles`).
|
|
|
|
Historically the partition list was hand-edited in `priv/repo/structure.sql`
|
|
whenever someone remembered. When that fell behind, writes for the
|
|
next quarter would start failing with `no partition of relation "..."
|
|
found for row` — silent at the worker level (the Rust hrrr-point-worker
|
|
marked tasks as failed) but blocking the calibration pipeline.
|
|
|
|
This module is the runtime safety net. The companion
|
|
`Microwaveprop.Workers.PartitionMaintenanceWorker` calls it daily so
|
|
the parent table always has at least `lookahead_quarters` quarters of
|
|
runway.
|
|
|
|
Design notes:
|
|
|
|
* Partitions are quarterly. `MM` in the partition name is the
|
|
starting month of the quarter (`01`, `04`, `07`, `10`) — matches
|
|
what `priv/repo/structure.sql` already uses for recent quarters.
|
|
* Some older partitions in the parent table cover wider ranges
|
|
(e.g. `hrrr_profiles_2027_01` covers 2027-01 → 2027-07). To
|
|
avoid colliding with those, every CREATE is gated by a
|
|
pg_inherits coverage check: if any existing child already
|
|
contains the target range, the call is a no-op. Postgres still
|
|
raises on partial overlap — that's a structural mismatch the
|
|
operator should resolve manually rather than have a worker
|
|
paper over.
|
|
* Postgres auto-attaches the parent's indexes on the new child,
|
|
so we only need to issue the `CREATE TABLE ... PARTITION OF`
|
|
DDL — no per-index follow-up.
|
|
* Quarter math is done on a single integer index
|
|
(`year * 4 + (month-1) ÷ 3`) to avoid month-rollover bugs.
|
|
"""
|
|
|
|
alias Microwaveprop.Repo
|
|
|
|
require Logger
|
|
|
|
@default_parents ~w(hrrr_profiles hrdps_profiles)
|
|
|
|
@typedoc """
|
|
Per-partition result. `parent` is the parent table name, `name` is
|
|
the resolved child partition name, and the status flags whether
|
|
this call performed the CREATE or found an existing partition that
|
|
already covered the range.
|
|
"""
|
|
@type result :: {parent :: String.t(), name :: String.t(), :created | :exists}
|
|
|
|
@doc """
|
|
For each configured parent table, ensure partitions covering the
|
|
current quarter and the next `lookahead_quarters` quarters exist.
|
|
|
|
Returns a list of `{parent, partition_name, :created | :exists}`
|
|
entries so the worker can log a single summary line.
|
|
"""
|
|
@spec ensure_quarterly_partitions(non_neg_integer(), [String.t()]) :: [result()]
|
|
def ensure_quarterly_partitions(lookahead_quarters \\ 4, parents \\ @default_parents)
|
|
when is_integer(lookahead_quarters) and lookahead_quarters >= 0 and is_list(parents) do
|
|
today = Date.utc_today()
|
|
quarters = quarter_starts(today, lookahead_quarters)
|
|
|
|
for parent <- parents, {q_start, q_end} <- quarters do
|
|
ensure_partition(parent, q_start, q_end)
|
|
end
|
|
end
|
|
|
|
# Public for direct access from tests / mix tasks.
|
|
@spec ensure_partition(String.t(), Date.t(), Date.t()) :: result()
|
|
def ensure_partition(parent, %Date{} = q_start, %Date{} = q_end) do
|
|
name = partition_name(parent, q_start)
|
|
|
|
if covered?(parent, q_start, q_end) do
|
|
{parent, name, :exists}
|
|
else
|
|
Repo.query!("""
|
|
CREATE TABLE public."#{name}"
|
|
PARTITION OF public."#{parent}"
|
|
FOR VALUES FROM ('#{Date.to_iso8601(q_start)} 00:00:00')
|
|
TO ('#{Date.to_iso8601(q_end)} 00:00:00')
|
|
""")
|
|
|
|
{parent, name, :created}
|
|
end
|
|
end
|
|
|
|
# Returns true when at least one existing partition of `parent`
|
|
# fully contains the [q_start, q_end) range. Partial overlaps fall
|
|
# through to CREATE, where Postgres raises a clear error rather
|
|
# than letting the worker pretend success.
|
|
defp covered?(parent, q_start, q_end) do
|
|
%Postgrex.Result{rows: rows} =
|
|
Repo.query!(
|
|
"""
|
|
SELECT pg_get_expr(child.relpartbound, child.oid)
|
|
FROM pg_inherits
|
|
JOIN pg_class parent ON pg_inherits.inhparent = parent.oid
|
|
JOIN pg_class child ON pg_inherits.inhrelid = child.oid
|
|
WHERE parent.relname = $1
|
|
""",
|
|
[parent]
|
|
)
|
|
|
|
Enum.any?(rows, fn [bound] ->
|
|
case Regex.run(~r/FROM \('([^']+)'\) TO \('([^']+)'\)/, bound) do
|
|
[_, lo, hi] ->
|
|
lo_dt = NaiveDateTime.from_iso8601!(lo)
|
|
hi_dt = NaiveDateTime.from_iso8601!(hi)
|
|
q_start_dt = NaiveDateTime.new!(q_start, ~T[00:00:00])
|
|
q_end_dt = NaiveDateTime.new!(q_end, ~T[00:00:00])
|
|
|
|
NaiveDateTime.compare(lo_dt, q_start_dt) != :gt and
|
|
NaiveDateTime.compare(hi_dt, q_end_dt) != :lt
|
|
|
|
_ ->
|
|
false
|
|
end
|
|
end)
|
|
end
|
|
|
|
defp partition_name(parent, %Date{year: y, month: m}) do
|
|
"#{parent}_#{y}_#{String.pad_leading(Integer.to_string(m), 2, "0")}"
|
|
end
|
|
|
|
# Quarter index = year * 4 + zero-based-quarter-of-year. Going
|
|
# through this single integer makes "next quarter" trivial and
|
|
# year-rollover free.
|
|
defp quarter_starts(%Date{} = today, lookahead) do
|
|
current = quarter_index(today)
|
|
|
|
Enum.map(0..lookahead, fn n ->
|
|
q = current + n
|
|
{quarter_to_date(q), quarter_to_date(q + 1)}
|
|
end)
|
|
end
|
|
|
|
defp quarter_index(%Date{year: y, month: m}), do: y * 4 + div(m - 1, 3)
|
|
|
|
defp quarter_to_date(qi) do
|
|
%Date{year: div(qi, 4), month: rem(qi, 4) * 3 + 1, day: 1}
|
|
end
|
|
end
|