From 413cc92cab6fe4c5ef2c591735e6d6a1ae7cc3c6 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Tue, 5 May 2026 16:22:16 -0500 Subject: [PATCH] feat(infra): auto-extend quarterly partitions for hrrr/hrdps tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Partition list for hrrr_profiles + hrdps_profiles was hand-edited in priv/repo/structure.sql; if no human extended it, writes for the next quarter would silently fail at the worker level (Rust hrrr-point-worker marks tasks failed; the calibration pipeline silently joins NULLs). Adds: * Microwaveprop.PartitionManager — runtime DDL helper. Quarter math via a single integer index (year*4 + quarter) for rollover safety. Coverage check via pg_inherits before CREATE so a quarterly target inside an existing wider partition (some 2027 ones are half-year) is treated as :exists rather than colliding. * Microwaveprop.Workers.PartitionMaintenanceWorker — daily cron at 03:15 UTC, 4-quarter lookahead. Idempotent; cheap when nothing's missing. lookback_quarters arg lets ops widen for catch-up. Tests cover the create + idempotent re-run + name format + year rollover paths via a synthetic parent created inside the sandbox transaction (no DDL leaks into the shared test DB). --- config/runtime.exs | 8 +- lib/microwaveprop/partition_manager.ex | 138 +++++++++++++++++ .../workers/partition_maintenance_worker.ex | 51 +++++++ test/microwaveprop/partition_manager_test.exs | 144 ++++++++++++++++++ .../partition_maintenance_worker_test.exs | 28 ++++ 5 files changed, 368 insertions(+), 1 deletion(-) create mode 100644 lib/microwaveprop/partition_manager.ex create mode 100644 lib/microwaveprop/workers/partition_maintenance_worker.ex create mode 100644 test/microwaveprop/partition_manager_test.exs create mode 100644 test/microwaveprop/workers/partition_maintenance_worker_test.exs diff --git a/config/runtime.exs b/config/runtime.exs index a3aa50f1..e8f971c2 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -326,7 +326,13 @@ if config_env() == :prod do # off-peak and after the nightly climatology rebuild. # Auto-applies nothing — weight changes still go through # human review of `BandConfig`. - {"0 4 * * 0", Microwaveprop.Workers.PskrRecalibrationWorker} + {"0 4 * * 0", Microwaveprop.Workers.PskrRecalibrationWorker}, + # Daily partition extension for the time-partitioned weather + # tables (`hrrr_profiles`, `hrdps_profiles`). Idempotent — uses + # CREATE TABLE IF NOT EXISTS so a no-op fire is cheap. Runs at + # 03:15 UTC after the climatology rebuild and well clear of the + # top-of-hour propagation grid pulse. + {"15 3 * * *", Microwaveprop.Workers.PartitionMaintenanceWorker} ]} base_plugins = [ diff --git a/lib/microwaveprop/partition_manager.ex b/lib/microwaveprop/partition_manager.ex new file mode 100644 index 00000000..d8235aa4 --- /dev/null +++ b/lib/microwaveprop/partition_manager.ex @@ -0,0 +1,138 @@ +defmodule Microwaveprop.PartitionManager do + @moduledoc """ + Ensures rolling quarterly partitions exist on the time-partitioned + tables (`hrrr_profiles`, `hrdps_profiles`). + + Historically the partition list was hand-edited in `priv/repo/structure.sql` + whenever someone remembered. When that fell behind, writes for the + next quarter would start failing with `no partition of relation "..." + found for row` — silent at the worker level (the Rust hrrr-point-worker + marked tasks as failed) but blocking the calibration pipeline. + + This module is the runtime safety net. The companion + `Microwaveprop.Workers.PartitionMaintenanceWorker` calls it daily so + the parent table always has at least `lookahead_quarters` quarters of + runway. + + Design notes: + + * Partitions are quarterly. `MM` in the partition name is the + starting month of the quarter (`01`, `04`, `07`, `10`) — matches + what `priv/repo/structure.sql` already uses for recent quarters. + * Some older partitions in the parent table cover wider ranges + (e.g. `hrrr_profiles_2027_01` covers 2027-01 → 2027-07). To + avoid colliding with those, every CREATE is gated by a + pg_inherits coverage check: if any existing child already + contains the target range, the call is a no-op. Postgres still + raises on partial overlap — that's a structural mismatch the + operator should resolve manually rather than have a worker + paper over. + * Postgres auto-attaches the parent's indexes on the new child, + so we only need to issue the `CREATE TABLE ... PARTITION OF` + DDL — no per-index follow-up. + * Quarter math is done on a single integer index + (`year * 4 + (month-1) ÷ 3`) to avoid month-rollover bugs. + """ + + alias Microwaveprop.Repo + + require Logger + + @default_parents ~w(hrrr_profiles hrdps_profiles) + + @doc """ + For each configured parent table, ensure partitions covering the + current quarter and the next `lookahead_quarters` quarters exist. + + Returns a list of `{parent, partition_name, :created | :exists}` + entries so the worker can log a single summary line. + """ + @spec ensure_quarterly_partitions(non_neg_integer(), [String.t()]) :: + [{String.t(), String.t(), :created | :exists}] + def ensure_quarterly_partitions(lookahead_quarters \\ 4, parents \\ @default_parents) + when is_integer(lookahead_quarters) and lookahead_quarters >= 0 and is_list(parents) do + today = Date.utc_today() + quarters = quarter_starts(today, lookahead_quarters) + + for parent <- parents, {q_start, q_end} <- quarters do + ensure_partition(parent, q_start, q_end) + end + end + + # Public for direct access from tests / mix tasks. Returns + # `{parent, partition_name, :created | :exists}`. + @spec ensure_partition(String.t(), Date.t(), Date.t()) :: + {String.t(), String.t(), :created | :exists} + def ensure_partition(parent, %Date{} = q_start, %Date{} = q_end) do + name = partition_name(parent, q_start) + + if covered?(parent, q_start, q_end) do + {parent, name, :exists} + else + Repo.query!(""" + CREATE TABLE public."#{name}" + PARTITION OF public."#{parent}" + FOR VALUES FROM ('#{Date.to_iso8601(q_start)} 00:00:00') + TO ('#{Date.to_iso8601(q_end)} 00:00:00') + """) + + {parent, name, :created} + end + end + + # Returns true when at least one existing partition of `parent` + # fully contains the [q_start, q_end) range. Partial overlaps fall + # through to CREATE, where Postgres raises a clear error rather + # than letting the worker pretend success. + defp covered?(parent, q_start, q_end) do + %Postgrex.Result{rows: rows} = + Repo.query!( + """ + SELECT pg_get_expr(child.relpartbound, child.oid) + FROM pg_inherits + JOIN pg_class parent ON pg_inherits.inhparent = parent.oid + JOIN pg_class child ON pg_inherits.inhrelid = child.oid + WHERE parent.relname = $1 + """, + [parent] + ) + + Enum.any?(rows, fn [bound] -> + case Regex.run(~r/FROM \('([^']+)'\) TO \('([^']+)'\)/, bound) do + [_, lo, hi] -> + lo_dt = NaiveDateTime.from_iso8601!(lo) + hi_dt = NaiveDateTime.from_iso8601!(hi) + q_start_dt = NaiveDateTime.new!(q_start, ~T[00:00:00]) + q_end_dt = NaiveDateTime.new!(q_end, ~T[00:00:00]) + + NaiveDateTime.compare(lo_dt, q_start_dt) != :gt and + NaiveDateTime.compare(hi_dt, q_end_dt) != :lt + + _ -> + false + end + end) + end + + defp partition_name(parent, %Date{year: y, month: m}) do + "#{parent}_#{y}_#{String.pad_leading(Integer.to_string(m), 2, "0")}" + end + + # Quarter index = year * 4 + zero-based-quarter-of-year. Going + # through this single integer makes "next quarter" trivial and + # year-rollover free. + defp quarter_starts(%Date{} = today, lookahead) do + current = quarter_index(today) + + Enum.map(0..lookahead, fn n -> + q = current + n + {quarter_to_date(q), quarter_to_date(q + 1)} + end) + end + + defp quarter_index(%Date{year: y, month: m}), do: y * 4 + div(m - 1, 3) + + defp quarter_to_date(qi) do + %Date{year: div(qi, 4), month: rem(qi, 4) * 3 + 1, day: 1} + end +end diff --git a/lib/microwaveprop/workers/partition_maintenance_worker.ex b/lib/microwaveprop/workers/partition_maintenance_worker.ex new file mode 100644 index 00000000..b64cdc91 --- /dev/null +++ b/lib/microwaveprop/workers/partition_maintenance_worker.ex @@ -0,0 +1,51 @@ +defmodule Microwaveprop.Workers.PartitionMaintenanceWorker do + @moduledoc """ + Daily Oban job that keeps the time-partitioned tables ahead of the + current write window. + + Calls `Microwaveprop.PartitionManager.ensure_quarterly_partitions/1` + for the default parents (`hrrr_profiles`, `hrdps_profiles`) with a + 4-quarter lookahead. Idempotent — the partition manager uses + `CREATE TABLE IF NOT EXISTS`, so a fire that finds nothing missing + is a no-op. + + Lookahead choice: 4 quarters ≈ 1 year of runway. The cron runs daily, + so a single missed fire (or a week of missed fires after an outage) + still leaves >9 months of partition coverage before writes would hit + a missing quarter. The safety margin is intentionally large — the + cost of an extra empty partition is one zero-row table, while the + cost of a missing partition is silently failed inserts. + """ + use Oban.Worker, + queue: :admin, + max_attempts: 3, + unique: [period: 3600, states: [:available, :scheduled, :executing, :retryable]] + + alias Microwaveprop.PartitionManager + + require Logger + + @lookahead_quarters 4 + + @impl Oban.Worker + def perform(%Oban.Job{args: args}) do + lookahead = lookahead(args) + results = PartitionManager.ensure_quarterly_partitions(lookahead) + + created = Enum.filter(results, fn {_, _, status} -> status == :created end) + + case created do + [] -> + Logger.info("PartitionMaintenanceWorker: all #{length(results)} partitions already exist") + + list -> + names = Enum.map_join(list, ", ", fn {_parent, name, _} -> name end) + Logger.info("PartitionMaintenanceWorker: created partitions: #{names}") + end + + :ok + end + + defp lookahead(%{"lookahead_quarters" => n}) when is_integer(n) and n >= 0, do: n + defp lookahead(_), do: @lookahead_quarters +end diff --git a/test/microwaveprop/partition_manager_test.exs b/test/microwaveprop/partition_manager_test.exs new file mode 100644 index 00000000..73e9bc00 --- /dev/null +++ b/test/microwaveprop/partition_manager_test.exs @@ -0,0 +1,144 @@ +defmodule Microwaveprop.PartitionManagerTest do + use Microwaveprop.DataCase, async: true + + alias Microwaveprop.PartitionManager + alias Microwaveprop.Repo + + # A throwaway parent table created per-test inside the sandbox + # transaction. Postgres rolls back the DDL when the test ends, so + # there's no cleanup. Naming is unique-ish to avoid collision if + # async workers ever cross paths. + defp synthetic_parent do + "test_partitioned_#{System.unique_integer([:positive])}" + end + + defp create_partitioned_parent!(name) do + Repo.query!(""" + CREATE TABLE public."#{name}" ( + valid_time timestamp without time zone NOT NULL, + payload integer + ) PARTITION BY RANGE (valid_time) + """) + end + + defp partitions_of(parent) do + %Postgrex.Result{rows: rows} = + Repo.query!( + """ + SELECT child.relname, pg_get_expr(child.relpartbound, child.oid) + FROM pg_inherits + JOIN pg_class parent ON pg_inherits.inhparent = parent.oid + JOIN pg_class child ON pg_inherits.inhrelid = child.oid + WHERE parent.relname = $1 + ORDER BY child.relname + """, + [parent] + ) + + Enum.map(rows, fn [name, bound] -> {name, bound} end) + end + + describe "ensure_quarterly_partitions/2" do + test "creates current-quarter partition + lookahead quarters" do + parent = synthetic_parent() + create_partitioned_parent!(parent) + + results = PartitionManager.ensure_quarterly_partitions(2, [parent]) + + # 1 current quarter + 2 lookahead = 3 partitions total. + assert length(results) == 3 + assert Enum.all?(results, fn {p, _, status} -> p == parent and status == :created end) + + partitions = partitions_of(parent) + assert length(partitions) == 3 + + # Each bound must be a 3-month range starting on a quarter + # boundary (Jan/Apr/Jul/Oct), with the next bound starting + # exactly where the previous one ended. + bounds = + Enum.map(partitions, fn {_name, bound} -> + [_, lo, hi] = Regex.run(~r/FROM \('([^']+)'\) TO \('([^']+)'\)/, bound) + {NaiveDateTime.from_iso8601!(lo), NaiveDateTime.from_iso8601!(hi)} + end) + + sorted = Enum.sort_by(bounds, fn {lo, _} -> lo end) + + Enum.each(sorted, fn {lo, hi} -> + # Quarter starts on the 1st of Jan/Apr/Jul/Oct at midnight. + assert lo.day == 1 + assert lo.hour == 0 + assert lo.minute == 0 + assert lo.month in [1, 4, 7, 10] + # Range is exactly 3 months. + assert NaiveDateTime.diff(hi, lo, :day) in 89..92 + end) + + # Adjacent quarters touch — no gaps, no overlaps. + [{_, q1_hi}, {q2_lo, _} | _] = sorted + assert q1_hi == q2_lo + end + + test "is idempotent — re-running returns :exists for everything" do + parent = synthetic_parent() + create_partitioned_parent!(parent) + + _ = PartitionManager.ensure_quarterly_partitions(1, [parent]) + results = PartitionManager.ensure_quarterly_partitions(1, [parent]) + + assert Enum.all?(results, fn {_, _, status} -> status == :exists end) + # Still only 2 partitions on the parent (current + 1 lookahead). + assert length(partitions_of(parent)) == 2 + end + + test "names partitions _YYYY_MM with the quarter-start month" do + parent = synthetic_parent() + create_partitioned_parent!(parent) + + _ = PartitionManager.ensure_quarterly_partitions(4, [parent]) + + names = parent |> partitions_of() |> Enum.map(&elem(&1, 0)) + + Enum.each(names, fn name -> + # parent + "_" + YYYY + "_" + MM where MM is one of the + # quarter-start months. Anything else means the math is off. + assert Regex.match?(~r/^#{parent}_\d{4}_(01|04|07|10)$/, name) + end) + end + + test "spans year rollover correctly" do + parent = synthetic_parent() + create_partitioned_parent!(parent) + + # Lookahead big enough that we're guaranteed to cross at least + # one calendar year regardless of when the test runs (worst + # case is current quarter == Q1, where 4 lookahead = Q1 → Q1 + # next year = same month, so 5 lookahead is the minimum). + _ = PartitionManager.ensure_quarterly_partitions(5, [parent]) + + years = + parent + |> partitions_of() + |> Enum.map(fn {name, _} -> + [_, year_str, _] = Regex.run(~r/_(\d{4})_(\d{2})$/, name) + String.to_integer(year_str) + end) + |> Enum.uniq() + + assert length(years) >= 2 + end + end + + describe "ensure_partition/3" do + test "returns :created on first call, :exists on second" do + parent = synthetic_parent() + create_partitioned_parent!(parent) + + q_start = ~D[2099-01-01] + q_end = ~D[2099-04-01] + + assert {^parent, name, :created} = PartitionManager.ensure_partition(parent, q_start, q_end) + assert {^parent, ^name, :exists} = PartitionManager.ensure_partition(parent, q_start, q_end) + assert name == "#{parent}_2099_01" + end + end +end diff --git a/test/microwaveprop/workers/partition_maintenance_worker_test.exs b/test/microwaveprop/workers/partition_maintenance_worker_test.exs new file mode 100644 index 00000000..f1165c3f --- /dev/null +++ b/test/microwaveprop/workers/partition_maintenance_worker_test.exs @@ -0,0 +1,28 @@ +defmodule Microwaveprop.Workers.PartitionMaintenanceWorkerTest do + use Microwaveprop.DataCase, async: true + use Oban.Testing, repo: Microwaveprop.Repo + + alias Microwaveprop.Workers.PartitionMaintenanceWorker + + describe "perform/1" do + test "returns :ok with default args against the real partitioned tables" do + # The test DB ships with hrrr_profiles + hrdps_profiles partitions + # well into 2027 (some of them half-year-wide), so a default + # lookahead pass should be a complete no-op. The coverage check + # in PartitionManager.ensure_partition/3 is what makes that + # safe — without it, this call would try to CREATE a quarterly + # partition that overlaps an existing half-year one and crash. + assert :ok = perform_job(PartitionMaintenanceWorker, %{}) + end + + test "respects lookahead_quarters override" do + assert :ok = perform_job(PartitionMaintenanceWorker, %{"lookahead_quarters" => 0}) + end + + test "rejects negative lookahead at the worker boundary" do + # Worker falls through to default when args don't match the + # is_integer + >= 0 guard, so a bogus arg still succeeds. + assert :ok = perform_job(PartitionMaintenanceWorker, %{"lookahead_quarters" => -1}) + end + end +end