feat(infra): auto-extend quarterly partitions for hrrr/hrdps tables

Partition list for hrrr_profiles + hrdps_profiles was hand-edited in
priv/repo/structure.sql; if no human extended it, writes for the next
quarter would silently fail at the worker level (Rust hrrr-point-worker
marks tasks failed; the calibration pipeline silently joins NULLs).

Adds:

* Microwaveprop.PartitionManager — runtime DDL helper. Quarter math via
  a single integer index (year*4 + quarter) for rollover safety.
  Coverage check via pg_inherits before CREATE so a quarterly target
  inside an existing wider partition (some 2027 ones are half-year)
  is treated as :exists rather than colliding.
* Microwaveprop.Workers.PartitionMaintenanceWorker — daily cron at
  03:15 UTC, 4-quarter lookahead. Idempotent; cheap when nothing's
  missing. lookback_quarters arg lets ops widen for catch-up.

Tests cover the create + idempotent re-run + name format + year
rollover paths via a synthetic parent created inside the sandbox
transaction (no DDL leaks into the shared test DB).

config/runtime.exs

@@ -326,7 +326,13 @@ if config_env() == :prod do
        # off-peak and after the nightly climatology rebuild.
        # Auto-applies nothing — weight changes still go through
        # human review of `BandConfig`.
-       {"0 4 * * 0", Microwaveprop.Workers.PskrRecalibrationWorker}
+       {"0 4 * * 0", Microwaveprop.Workers.PskrRecalibrationWorker},
+       # Daily partition extension for the time-partitioned weather
+       # tables (`hrrr_profiles`, `hrdps_profiles`). Idempotent — uses
+       # CREATE TABLE IF NOT EXISTS so a no-op fire is cheap. Runs at
+       # 03:15 UTC after the climatology rebuild and well clear of the
+       # top-of-hour propagation grid pulse.
+       {"15 3 * * *", Microwaveprop.Workers.PartitionMaintenanceWorker}
      ]}
 
   base_plugins = [

lib/microwaveprop/partition_manager.ex

@@ -0,0 +1,138 @@
+defmodule Microwaveprop.PartitionManager do
+  @moduledoc """
+  Ensures rolling quarterly partitions exist on the time-partitioned
+  tables (`hrrr_profiles`, `hrdps_profiles`).
+
+  Historically the partition list was hand-edited in `priv/repo/structure.sql`
+  whenever someone remembered. When that fell behind, writes for the
+  next quarter would start failing with `no partition of relation "..."
+  found for row` — silent at the worker level (the Rust hrrr-point-worker
+  marked tasks as failed) but blocking the calibration pipeline.
+
+  This module is the runtime safety net. The companion
+  `Microwaveprop.Workers.PartitionMaintenanceWorker` calls it daily so
+  the parent table always has at least `lookahead_quarters` quarters of
+  runway.
+
+  Design notes:
+
+    * Partitions are quarterly. `MM` in the partition name is the
+      starting month of the quarter (`01`, `04`, `07`, `10`) — matches
+      what `priv/repo/structure.sql` already uses for recent quarters.
+    * Some older partitions in the parent table cover wider ranges
+      (e.g. `hrrr_profiles_2027_01` covers 2027-01 → 2027-07). To
+      avoid colliding with those, every CREATE is gated by a
+      pg_inherits coverage check: if any existing child already
+      contains the target range, the call is a no-op. Postgres still
+      raises on partial overlap — that's a structural mismatch the
+      operator should resolve manually rather than have a worker
+      paper over.
+    * Postgres auto-attaches the parent's indexes on the new child,
+      so we only need to issue the `CREATE TABLE ... PARTITION OF`
+      DDL — no per-index follow-up.
+    * Quarter math is done on a single integer index
+      (`year * 4 + (month-1) ÷ 3`) to avoid month-rollover bugs.
+  """
+
+  alias Microwaveprop.Repo
+
+  require Logger
+
+  @default_parents ~w(hrrr_profiles hrdps_profiles)
+
+  @doc """
+  For each configured parent table, ensure partitions covering the
+  current quarter and the next `lookahead_quarters` quarters exist.
+
+  Returns a list of `{parent, partition_name, :created | :exists}`
+  entries so the worker can log a single summary line.
+  """
+  @spec ensure_quarterly_partitions(non_neg_integer(), [String.t()]) ::
+          [{String.t(), String.t(), :created | :exists}]
+  def ensure_quarterly_partitions(lookahead_quarters \\ 4, parents \\ @default_parents)
+      when is_integer(lookahead_quarters) and lookahead_quarters >= 0 and is_list(parents) do
+    today = Date.utc_today()
+    quarters = quarter_starts(today, lookahead_quarters)
+
+    for parent <- parents, {q_start, q_end} <- quarters do
+      ensure_partition(parent, q_start, q_end)
+    end
+  end
+
+  # Public for direct access from tests / mix tasks. Returns
+  # `{parent, partition_name, :created | :exists}`.
+  @spec ensure_partition(String.t(), Date.t(), Date.t()) ::
+          {String.t(), String.t(), :created | :exists}
+  def ensure_partition(parent, %Date{} = q_start, %Date{} = q_end) do
+    name = partition_name(parent, q_start)
+
+    if covered?(parent, q_start, q_end) do
+      {parent, name, :exists}
+    else
+      Repo.query!("""
+      CREATE TABLE public."#{name}"
+      PARTITION OF public."#{parent}"
+      FOR VALUES FROM ('#{Date.to_iso8601(q_start)} 00:00:00')
+                   TO ('#{Date.to_iso8601(q_end)} 00:00:00')
+      """)
+
+      {parent, name, :created}
+    end
+  end
+
+  # Returns true when at least one existing partition of `parent`
+  # fully contains the [q_start, q_end) range. Partial overlaps fall
+  # through to CREATE, where Postgres raises a clear error rather
+  # than letting the worker pretend success.
+  defp covered?(parent, q_start, q_end) do
+    %Postgrex.Result{rows: rows} =
+      Repo.query!(
+        """
+        SELECT pg_get_expr(child.relpartbound, child.oid)
+        FROM pg_inherits
+        JOIN pg_class parent ON pg_inherits.inhparent = parent.oid
+        JOIN pg_class child  ON pg_inherits.inhrelid  = child.oid
+        WHERE parent.relname = $1
+        """,
+        [parent]
+      )
+
+    Enum.any?(rows, fn [bound] ->
+      case Regex.run(~r/FROM \('([^']+)'\) TO \('([^']+)'\)/, bound) do
+        [_, lo, hi] ->
+          lo_dt = NaiveDateTime.from_iso8601!(lo)
+          hi_dt = NaiveDateTime.from_iso8601!(hi)
+          q_start_dt = NaiveDateTime.new!(q_start, ~T[00:00:00])
+          q_end_dt = NaiveDateTime.new!(q_end, ~T[00:00:00])
+
+          NaiveDateTime.compare(lo_dt, q_start_dt) != :gt and
+            NaiveDateTime.compare(hi_dt, q_end_dt) != :lt
+
+        _ ->
+          false
+      end
+    end)
+  end
+
+  defp partition_name(parent, %Date{year: y, month: m}) do
+    "#{parent}_#{y}_#{String.pad_leading(Integer.to_string(m), 2, "0")}"
+  end
+
+  # Quarter index = year * 4 + zero-based-quarter-of-year. Going
+  # through this single integer makes "next quarter" trivial and
+  # year-rollover free.
+  defp quarter_starts(%Date{} = today, lookahead) do
+    current = quarter_index(today)
+
+    Enum.map(0..lookahead, fn n ->
+      q = current + n
+      {quarter_to_date(q), quarter_to_date(q + 1)}
+    end)
+  end
+
+  defp quarter_index(%Date{year: y, month: m}), do: y * 4 + div(m - 1, 3)
+
+  defp quarter_to_date(qi) do
+    %Date{year: div(qi, 4), month: rem(qi, 4) * 3 + 1, day: 1}
+  end
+end

lib/microwaveprop/workers/partition_maintenance_worker.ex

@@ -0,0 +1,51 @@
+defmodule Microwaveprop.Workers.PartitionMaintenanceWorker do
+  @moduledoc """
+  Daily Oban job that keeps the time-partitioned tables ahead of the
+  current write window.
+
+  Calls `Microwaveprop.PartitionManager.ensure_quarterly_partitions/1`
+  for the default parents (`hrrr_profiles`, `hrdps_profiles`) with a
+  4-quarter lookahead. Idempotent — the partition manager uses
+  `CREATE TABLE IF NOT EXISTS`, so a fire that finds nothing missing
+  is a no-op.
+
+  Lookahead choice: 4 quarters ≈ 1 year of runway. The cron runs daily,
+  so a single missed fire (or a week of missed fires after an outage)
+  still leaves >9 months of partition coverage before writes would hit
+  a missing quarter. The safety margin is intentionally large — the
+  cost of an extra empty partition is one zero-row table, while the
+  cost of a missing partition is silently failed inserts.
+  """
+  use Oban.Worker,
+    queue: :admin,
+    max_attempts: 3,
+    unique: [period: 3600, states: [:available, :scheduled, :executing, :retryable]]
+
+  alias Microwaveprop.PartitionManager
+
+  require Logger
+
+  @lookahead_quarters 4
+
+  @impl Oban.Worker
+  def perform(%Oban.Job{args: args}) do
+    lookahead = lookahead(args)
+    results = PartitionManager.ensure_quarterly_partitions(lookahead)
+
+    created = Enum.filter(results, fn {_, _, status} -> status == :created end)
+
+    case created do
+      [] ->
+        Logger.info("PartitionMaintenanceWorker: all #{length(results)} partitions already exist")
+
+      list ->
+        names = Enum.map_join(list, ", ", fn {_parent, name, _} -> name end)
+        Logger.info("PartitionMaintenanceWorker: created partitions: #{names}")
+    end
+
+    :ok
+  end
+
+  defp lookahead(%{"lookahead_quarters" => n}) when is_integer(n) and n >= 0, do: n
+  defp lookahead(_), do: @lookahead_quarters
+end

test/microwaveprop/partition_manager_test.exs

@@ -0,0 +1,144 @@
+defmodule Microwaveprop.PartitionManagerTest do
+  use Microwaveprop.DataCase, async: true
+
+  alias Microwaveprop.PartitionManager
+  alias Microwaveprop.Repo
+
+  # A throwaway parent table created per-test inside the sandbox
+  # transaction. Postgres rolls back the DDL when the test ends, so
+  # there's no cleanup. Naming is unique-ish to avoid collision if
+  # async workers ever cross paths.
+  defp synthetic_parent do
+    "test_partitioned_#{System.unique_integer([:positive])}"
+  end
+
+  defp create_partitioned_parent!(name) do
+    Repo.query!("""
+    CREATE TABLE public."#{name}" (
+      valid_time timestamp without time zone NOT NULL,
+      payload integer
+    ) PARTITION BY RANGE (valid_time)
+    """)
+  end
+
+  defp partitions_of(parent) do
+    %Postgrex.Result{rows: rows} =
+      Repo.query!(
+        """
+        SELECT child.relname, pg_get_expr(child.relpartbound, child.oid)
+        FROM pg_inherits
+        JOIN pg_class parent ON pg_inherits.inhparent = parent.oid
+        JOIN pg_class child  ON pg_inherits.inhrelid  = child.oid
+        WHERE parent.relname = $1
+        ORDER BY child.relname
+        """,
+        [parent]
+      )
+
+    Enum.map(rows, fn [name, bound] -> {name, bound} end)
+  end
+
+  describe "ensure_quarterly_partitions/2" do
+    test "creates current-quarter partition + lookahead quarters" do
+      parent = synthetic_parent()
+      create_partitioned_parent!(parent)
+
+      results = PartitionManager.ensure_quarterly_partitions(2, [parent])
+
+      # 1 current quarter + 2 lookahead = 3 partitions total.
+      assert length(results) == 3
+      assert Enum.all?(results, fn {p, _, status} -> p == parent and status == :created end)
+
+      partitions = partitions_of(parent)
+      assert length(partitions) == 3
+
+      # Each bound must be a 3-month range starting on a quarter
+      # boundary (Jan/Apr/Jul/Oct), with the next bound starting
+      # exactly where the previous one ended.
+      bounds =
+        Enum.map(partitions, fn {_name, bound} ->
+          [_, lo, hi] = Regex.run(~r/FROM \('([^']+)'\) TO \('([^']+)'\)/, bound)
+          {NaiveDateTime.from_iso8601!(lo), NaiveDateTime.from_iso8601!(hi)}
+        end)
+
+      sorted = Enum.sort_by(bounds, fn {lo, _} -> lo end)
+
+      Enum.each(sorted, fn {lo, hi} ->
+        # Quarter starts on the 1st of Jan/Apr/Jul/Oct at midnight.
+        assert lo.day == 1
+        assert lo.hour == 0
+        assert lo.minute == 0
+        assert lo.month in [1, 4, 7, 10]
+        # Range is exactly 3 months.
+        assert NaiveDateTime.diff(hi, lo, :day) in 89..92
+      end)
+
+      # Adjacent quarters touch — no gaps, no overlaps.
+      [{_, q1_hi}, {q2_lo, _} | _] = sorted
+      assert q1_hi == q2_lo
+    end
+
+    test "is idempotent — re-running returns :exists for everything" do
+      parent = synthetic_parent()
+      create_partitioned_parent!(parent)
+
+      _ = PartitionManager.ensure_quarterly_partitions(1, [parent])
+      results = PartitionManager.ensure_quarterly_partitions(1, [parent])
+
+      assert Enum.all?(results, fn {_, _, status} -> status == :exists end)
+      # Still only 2 partitions on the parent (current + 1 lookahead).
+      assert length(partitions_of(parent)) == 2
+    end
+
+    test "names partitions <parent>_YYYY_MM with the quarter-start month" do
+      parent = synthetic_parent()
+      create_partitioned_parent!(parent)
+
+      _ = PartitionManager.ensure_quarterly_partitions(4, [parent])
+
+      names = parent |> partitions_of() |> Enum.map(&elem(&1, 0))
+
+      Enum.each(names, fn name ->
+        # parent + "_" + YYYY + "_" + MM where MM is one of the
+        # quarter-start months. Anything else means the math is off.
+        assert Regex.match?(~r/^#{parent}_\d{4}_(01|04|07|10)$/, name)
+      end)
+    end
+
+    test "spans year rollover correctly" do
+      parent = synthetic_parent()
+      create_partitioned_parent!(parent)
+
+      # Lookahead big enough that we're guaranteed to cross at least
+      # one calendar year regardless of when the test runs (worst
+      # case is current quarter == Q1, where 4 lookahead = Q1 → Q1
+      # next year = same month, so 5 lookahead is the minimum).
+      _ = PartitionManager.ensure_quarterly_partitions(5, [parent])
+
+      years =
+        parent
+        |> partitions_of()
+        |> Enum.map(fn {name, _} ->
+          [_, year_str, _] = Regex.run(~r/_(\d{4})_(\d{2})$/, name)
+          String.to_integer(year_str)
+        end)
+        |> Enum.uniq()
+
+      assert length(years) >= 2
+    end
+  end
+
+  describe "ensure_partition/3" do
+    test "returns :created on first call, :exists on second" do
+      parent = synthetic_parent()
+      create_partitioned_parent!(parent)
+
+      q_start = ~D[2099-01-01]
+      q_end = ~D[2099-04-01]
+
+      assert {^parent, name, :created} = PartitionManager.ensure_partition(parent, q_start, q_end)
+      assert {^parent, ^name, :exists} = PartitionManager.ensure_partition(parent, q_start, q_end)
+      assert name == "#{parent}_2099_01"
+    end
+  end
+end

test/microwaveprop/workers/partition_maintenance_worker_test.exs

@@ -0,0 +1,28 @@
+defmodule Microwaveprop.Workers.PartitionMaintenanceWorkerTest do
+  use Microwaveprop.DataCase, async: true
+  use Oban.Testing, repo: Microwaveprop.Repo
+
+  alias Microwaveprop.Workers.PartitionMaintenanceWorker
+
+  describe "perform/1" do
+    test "returns :ok with default args against the real partitioned tables" do
+      # The test DB ships with hrrr_profiles + hrdps_profiles partitions
+      # well into 2027 (some of them half-year-wide), so a default
+      # lookahead pass should be a complete no-op. The coverage check
+      # in PartitionManager.ensure_partition/3 is what makes that
+      # safe — without it, this call would try to CREATE a quarterly
+      # partition that overlaps an existing half-year one and crash.
+      assert :ok = perform_job(PartitionMaintenanceWorker, %{})
+    end
+
+    test "respects lookahead_quarters override" do
+      assert :ok = perform_job(PartitionMaintenanceWorker, %{"lookahead_quarters" => 0})
+    end
+
+    test "rejects negative lookahead at the worker boundary" do
+      # Worker falls through to default when args don't match the
+      # is_integer + >= 0 guard, so a bogus arg still succeeds.
+      assert :ok = perform_job(PartitionMaintenanceWorker, %{"lookahead_quarters" => -1})
+    end
+  end
+end