prop/test/microwaveprop/partition_manager_test.exs
Graham McIntire 7ee19c5b6a
perf+test: cache nearest_hrrr per cell, batch pg_inherits, +property tests
Two performance fixes the property tests pinned down:

* CalibrationSampler.build_for_hour/1 walked every cell through
  nearest_hrrr/3 twice — once in the producer and again in
  build_sample. Now computed once into a (band, lat, lon) → hrrr_or_nil
  map, halving the O(cells × profiles) scan (~10M comparisons saved
  per fire at typical sizes).
* PartitionManager.ensure_quarterly_partitions/2 issued one
  pg_inherits scan per (parent × lookahead). Refactored to one scan
  per parent with in-memory coverage check across all quarters.

Property tests for both modules:

* PartitionManager: tile-coverage invariant (no gaps, no overlaps),
  exact 3-month bounds, name format, multi-call idempotence,
  multi-parent independence — caught a real NaiveDateTime sort bug
  in the original test helpers. Default Enum.sort_by/2 falls back to
  term comparison on NaiveDateTime; now uses NaiveDateTime as the
  comparator module.
* CalibrationSampler: enqueued points = unique missing midpoints,
  empty enqueue when fully covered, sample row count invariant under
  HRRR availability.

Saved operational gotchas to CLAUDE.md (Oban leader election spans
all app=prop pods, kubectl exec deploy/prop ambiguity, half-year
partition coexistence, HRRR f000 publish lag).

3310 tests + 228 properties, 0 failures.
2026-05-05 17:40:13 -05:00

146 lines
5 KiB
Elixir

defmodule Microwaveprop.PartitionManagerTest do
use Microwaveprop.DataCase, async: true
alias Microwaveprop.PartitionManager
alias Microwaveprop.Repo
# A throwaway parent table created per-test inside the sandbox
# transaction. Postgres rolls back the DDL when the test ends, so
# there's no cleanup. Naming is unique-ish to avoid collision if
# async workers ever cross paths.
defp synthetic_parent do
"test_partitioned_#{System.unique_integer([:positive])}"
end
defp create_partitioned_parent!(name) do
Repo.query!("""
CREATE TABLE public."#{name}" (
valid_time timestamp without time zone NOT NULL,
payload integer
) PARTITION BY RANGE (valid_time)
""")
end
defp partitions_of(parent) do
%Postgrex.Result{rows: rows} =
Repo.query!(
"""
SELECT child.relname, pg_get_expr(child.relpartbound, child.oid)
FROM pg_inherits
JOIN pg_class parent ON pg_inherits.inhparent = parent.oid
JOIN pg_class child ON pg_inherits.inhrelid = child.oid
WHERE parent.relname = $1
ORDER BY child.relname
""",
[parent]
)
Enum.map(rows, fn [name, bound] -> {name, bound} end)
end
describe "ensure_quarterly_partitions/2" do
test "creates current-quarter partition + lookahead quarters" do
parent = synthetic_parent()
create_partitioned_parent!(parent)
results = PartitionManager.ensure_quarterly_partitions(2, [parent])
# 1 current quarter + 2 lookahead = 3 partitions total.
assert length(results) == 3
assert Enum.all?(results, fn {p, _, status} -> p == parent and status == :created end)
partitions = partitions_of(parent)
assert length(partitions) == 3
# Each bound must be a 3-month range starting on a quarter
# boundary (Jan/Apr/Jul/Oct), with the next bound starting
# exactly where the previous one ended.
bounds =
Enum.map(partitions, fn {_name, bound} ->
[_, lo, hi] = Regex.run(~r/FROM \('([^']+)'\) TO \('([^']+)'\)/, bound)
{NaiveDateTime.from_iso8601!(lo), NaiveDateTime.from_iso8601!(hi)}
end)
# Pass `NaiveDateTime` as the comparator module — default
# `<=` falls back to term comparison, which is not chronological.
sorted = Enum.sort_by(bounds, fn {lo, _} -> lo end, NaiveDateTime)
Enum.each(sorted, fn {lo, hi} ->
# Quarter starts on the 1st of Jan/Apr/Jul/Oct at midnight.
assert lo.day == 1
assert lo.hour == 0
assert lo.minute == 0
assert lo.month in [1, 4, 7, 10]
# Range is exactly 3 months.
assert NaiveDateTime.diff(hi, lo, :day) in 89..92
end)
# Adjacent quarters touch — no gaps, no overlaps.
[{_, q1_hi}, {q2_lo, _} | _] = sorted
assert q1_hi == q2_lo
end
test "is idempotent — re-running returns :exists for everything" do
parent = synthetic_parent()
create_partitioned_parent!(parent)
_ = PartitionManager.ensure_quarterly_partitions(1, [parent])
results = PartitionManager.ensure_quarterly_partitions(1, [parent])
assert Enum.all?(results, fn {_, _, status} -> status == :exists end)
# Still only 2 partitions on the parent (current + 1 lookahead).
assert length(partitions_of(parent)) == 2
end
test "names partitions <parent>_YYYY_MM with the quarter-start month" do
parent = synthetic_parent()
create_partitioned_parent!(parent)
_ = PartitionManager.ensure_quarterly_partitions(4, [parent])
names = parent |> partitions_of() |> Enum.map(&elem(&1, 0))
Enum.each(names, fn name ->
# parent + "_" + YYYY + "_" + MM where MM is one of the
# quarter-start months. Anything else means the math is off.
assert Regex.match?(~r/^#{parent}_\d{4}_(01|04|07|10)$/, name)
end)
end
test "spans year rollover correctly" do
parent = synthetic_parent()
create_partitioned_parent!(parent)
# Lookahead big enough that we're guaranteed to cross at least
# one calendar year regardless of when the test runs (worst
# case is current quarter == Q1, where 4 lookahead = Q1 → Q1
# next year = same month, so 5 lookahead is the minimum).
_ = PartitionManager.ensure_quarterly_partitions(5, [parent])
years =
parent
|> partitions_of()
|> Enum.map(fn {name, _} ->
[_, year_str, _] = Regex.run(~r/_(\d{4})_(\d{2})$/, name)
String.to_integer(year_str)
end)
|> Enum.uniq()
assert length(years) >= 2
end
end
describe "ensure_partition/3" do
test "returns :created on first call, :exists on second" do
parent = synthetic_parent()
create_partitioned_parent!(parent)
q_start = ~D[2099-01-01]
q_end = ~D[2099-04-01]
assert {^parent, name, :created} = PartitionManager.ensure_partition(parent, q_start, q_end)
assert {^parent, ^name, :exists} = PartitionManager.ensure_partition(parent, q_start, q_end)
assert name == "#{parent}_2099_01"
end
end
end