Two performance fixes the property tests pinned down: * CalibrationSampler.build_for_hour/1 walked every cell through nearest_hrrr/3 twice — once in the producer and again in build_sample. Now computed once into a (band, lat, lon) → hrrr_or_nil map, halving the O(cells × profiles) scan (~10M comparisons saved per fire at typical sizes). * PartitionManager.ensure_quarterly_partitions/2 issued one pg_inherits scan per (parent × lookahead). Refactored to one scan per parent with in-memory coverage check across all quarters. Property tests for both modules: * PartitionManager: tile-coverage invariant (no gaps, no overlaps), exact 3-month bounds, name format, multi-call idempotence, multi-parent independence — caught a real NaiveDateTime sort bug in the original test helpers. Default Enum.sort_by/2 falls back to term comparison on NaiveDateTime; now uses NaiveDateTime as the comparator module. * CalibrationSampler: enqueued points = unique missing midpoints, empty enqueue when fully covered, sample row count invariant under HRRR availability. Saved operational gotchas to CLAUDE.md (Oban leader election spans all app=prop pods, kubectl exec deploy/prop ambiguity, half-year partition coexistence, HRRR f000 publish lag). 3310 tests + 228 properties, 0 failures.
146 lines
5 KiB
Elixir
146 lines
5 KiB
Elixir
defmodule Microwaveprop.PartitionManagerTest do
|
|
use Microwaveprop.DataCase, async: true
|
|
|
|
alias Microwaveprop.PartitionManager
|
|
alias Microwaveprop.Repo
|
|
|
|
# A throwaway parent table created per-test inside the sandbox
|
|
# transaction. Postgres rolls back the DDL when the test ends, so
|
|
# there's no cleanup. Naming is unique-ish to avoid collision if
|
|
# async workers ever cross paths.
|
|
defp synthetic_parent do
|
|
"test_partitioned_#{System.unique_integer([:positive])}"
|
|
end
|
|
|
|
defp create_partitioned_parent!(name) do
|
|
Repo.query!("""
|
|
CREATE TABLE public."#{name}" (
|
|
valid_time timestamp without time zone NOT NULL,
|
|
payload integer
|
|
) PARTITION BY RANGE (valid_time)
|
|
""")
|
|
end
|
|
|
|
defp partitions_of(parent) do
|
|
%Postgrex.Result{rows: rows} =
|
|
Repo.query!(
|
|
"""
|
|
SELECT child.relname, pg_get_expr(child.relpartbound, child.oid)
|
|
FROM pg_inherits
|
|
JOIN pg_class parent ON pg_inherits.inhparent = parent.oid
|
|
JOIN pg_class child ON pg_inherits.inhrelid = child.oid
|
|
WHERE parent.relname = $1
|
|
ORDER BY child.relname
|
|
""",
|
|
[parent]
|
|
)
|
|
|
|
Enum.map(rows, fn [name, bound] -> {name, bound} end)
|
|
end
|
|
|
|
describe "ensure_quarterly_partitions/2" do
|
|
test "creates current-quarter partition + lookahead quarters" do
|
|
parent = synthetic_parent()
|
|
create_partitioned_parent!(parent)
|
|
|
|
results = PartitionManager.ensure_quarterly_partitions(2, [parent])
|
|
|
|
# 1 current quarter + 2 lookahead = 3 partitions total.
|
|
assert length(results) == 3
|
|
assert Enum.all?(results, fn {p, _, status} -> p == parent and status == :created end)
|
|
|
|
partitions = partitions_of(parent)
|
|
assert length(partitions) == 3
|
|
|
|
# Each bound must be a 3-month range starting on a quarter
|
|
# boundary (Jan/Apr/Jul/Oct), with the next bound starting
|
|
# exactly where the previous one ended.
|
|
bounds =
|
|
Enum.map(partitions, fn {_name, bound} ->
|
|
[_, lo, hi] = Regex.run(~r/FROM \('([^']+)'\) TO \('([^']+)'\)/, bound)
|
|
{NaiveDateTime.from_iso8601!(lo), NaiveDateTime.from_iso8601!(hi)}
|
|
end)
|
|
|
|
# Pass `NaiveDateTime` as the comparator module — default
|
|
# `<=` falls back to term comparison, which is not chronological.
|
|
sorted = Enum.sort_by(bounds, fn {lo, _} -> lo end, NaiveDateTime)
|
|
|
|
Enum.each(sorted, fn {lo, hi} ->
|
|
# Quarter starts on the 1st of Jan/Apr/Jul/Oct at midnight.
|
|
assert lo.day == 1
|
|
assert lo.hour == 0
|
|
assert lo.minute == 0
|
|
assert lo.month in [1, 4, 7, 10]
|
|
# Range is exactly 3 months.
|
|
assert NaiveDateTime.diff(hi, lo, :day) in 89..92
|
|
end)
|
|
|
|
# Adjacent quarters touch — no gaps, no overlaps.
|
|
[{_, q1_hi}, {q2_lo, _} | _] = sorted
|
|
assert q1_hi == q2_lo
|
|
end
|
|
|
|
test "is idempotent — re-running returns :exists for everything" do
|
|
parent = synthetic_parent()
|
|
create_partitioned_parent!(parent)
|
|
|
|
_ = PartitionManager.ensure_quarterly_partitions(1, [parent])
|
|
results = PartitionManager.ensure_quarterly_partitions(1, [parent])
|
|
|
|
assert Enum.all?(results, fn {_, _, status} -> status == :exists end)
|
|
# Still only 2 partitions on the parent (current + 1 lookahead).
|
|
assert length(partitions_of(parent)) == 2
|
|
end
|
|
|
|
test "names partitions <parent>_YYYY_MM with the quarter-start month" do
|
|
parent = synthetic_parent()
|
|
create_partitioned_parent!(parent)
|
|
|
|
_ = PartitionManager.ensure_quarterly_partitions(4, [parent])
|
|
|
|
names = parent |> partitions_of() |> Enum.map(&elem(&1, 0))
|
|
|
|
Enum.each(names, fn name ->
|
|
# parent + "_" + YYYY + "_" + MM where MM is one of the
|
|
# quarter-start months. Anything else means the math is off.
|
|
assert Regex.match?(~r/^#{parent}_\d{4}_(01|04|07|10)$/, name)
|
|
end)
|
|
end
|
|
|
|
test "spans year rollover correctly" do
|
|
parent = synthetic_parent()
|
|
create_partitioned_parent!(parent)
|
|
|
|
# Lookahead big enough that we're guaranteed to cross at least
|
|
# one calendar year regardless of when the test runs (worst
|
|
# case is current quarter == Q1, where 4 lookahead = Q1 → Q1
|
|
# next year = same month, so 5 lookahead is the minimum).
|
|
_ = PartitionManager.ensure_quarterly_partitions(5, [parent])
|
|
|
|
years =
|
|
parent
|
|
|> partitions_of()
|
|
|> Enum.map(fn {name, _} ->
|
|
[_, year_str, _] = Regex.run(~r/_(\d{4})_(\d{2})$/, name)
|
|
String.to_integer(year_str)
|
|
end)
|
|
|> Enum.uniq()
|
|
|
|
assert length(years) >= 2
|
|
end
|
|
end
|
|
|
|
describe "ensure_partition/3" do
|
|
test "returns :created on first call, :exists on second" do
|
|
parent = synthetic_parent()
|
|
create_partitioned_parent!(parent)
|
|
|
|
q_start = ~D[2099-01-01]
|
|
q_end = ~D[2099-04-01]
|
|
|
|
assert {^parent, name, :created} = PartitionManager.ensure_partition(parent, q_start, q_end)
|
|
assert {^parent, ^name, :exists} = PartitionManager.ensure_partition(parent, q_start, q_end)
|
|
assert name == "#{parent}_2099_01"
|
|
end
|
|
end
|
|
end
|