Graham McIntire
5ec66df135
The scoring+upsert phase was ~4m40s per forecast hour and dominated wall time. Three stacked optimizations attack it from different angles. replace_scores/2 is a new hot-path writer that does DELETE WHERE valid_time = $1 followed by a plain insert_all (no ON CONFLICT resolution). The chain worker rewrites the full (valid_time, all bands) slice every forecast hour, so conflict detection was pure waste. AsosAdjustmentWorker still uses upsert_scores because it only rewrites the subset of cells near a station. factors is now nullable. Forecast hours f01-f18 pass factors: nil so the JSONB encode + toast write is skipped entirely — roughly halves the data volume per run. point_detail/4 coalesces nil to an empty map so the JS popup renders without a TypeError, and scorer_diff only pulls the most recent valid_time that still has factors (the f00 row). propagation_scores is now UNLOGGED, so inserts bypass WAL entirely. Durability tradeoff: an unclean shutdown truncates the table, but PropagationGridWorker rebuilds it from HRRR every 3h so a lost table is re-populated within one cron cycle. Also adds docs/plans/2026-04-14-duckdb-scores-storage.md — a speculative plan for a flat-file / DuckDB rewrite with explicit trigger conditions for when to pick it up (partitioning deferred too; revisit only if these three don't solve it).
95 lines
2.8 KiB
Elixir
95 lines
2.8 KiB
Elixir
defmodule Microwaveprop.Propagation.GridScoreTest do
|
|
use Microwaveprop.DataCase, async: true
|
|
|
|
alias Microwaveprop.Propagation.GridScore
|
|
|
|
@valid_attrs %{
|
|
lat: 32.875,
|
|
lon: -97.0,
|
|
valid_time: ~U[2026-03-28 18:00:00Z],
|
|
band_mhz: 10_000,
|
|
score: 72,
|
|
factors: %{
|
|
"humidity" => 85,
|
|
"time_of_day" => 60,
|
|
"td_depression" => 70,
|
|
"refractivity" => 55,
|
|
"sky" => 80,
|
|
"season" => 65,
|
|
"wind" => 50,
|
|
"rain" => 90,
|
|
"pressure" => 45
|
|
}
|
|
}
|
|
|
|
describe "changeset/2" do
|
|
test "valid attributes" do
|
|
changeset = GridScore.changeset(%GridScore{}, @valid_attrs)
|
|
assert changeset.valid?
|
|
end
|
|
|
|
test "requires the core fields except factors" do
|
|
changeset = GridScore.changeset(%GridScore{}, %{})
|
|
|
|
errors = errors_on(changeset)
|
|
|
|
assert %{
|
|
lat: ["can't be blank"],
|
|
lon: ["can't be blank"],
|
|
valid_time: ["can't be blank"],
|
|
band_mhz: ["can't be blank"],
|
|
score: ["can't be blank"]
|
|
} = errors
|
|
|
|
refute Map.has_key?(errors, :factors)
|
|
end
|
|
|
|
test "persists with nil factors (forecast-hour rows skip the factor breakdown)" do
|
|
attrs = Map.delete(@valid_attrs, :factors)
|
|
changeset = GridScore.changeset(%GridScore{}, attrs)
|
|
assert changeset.valid?
|
|
assert {:ok, score} = Repo.insert(changeset)
|
|
assert score.factors == nil
|
|
end
|
|
|
|
test "persists to database" do
|
|
changeset = GridScore.changeset(%GridScore{}, @valid_attrs)
|
|
assert {:ok, score} = Repo.insert(changeset)
|
|
assert score.lat == 32.875
|
|
assert score.lon == -97.0
|
|
assert score.band_mhz == 10_000
|
|
assert score.score == 72
|
|
assert score.factors["humidity"] == 85
|
|
end
|
|
|
|
test "enforces unique lat + lon + valid_time + band_mhz" do
|
|
assert {:ok, _} =
|
|
%GridScore{} |> GridScore.changeset(@valid_attrs) |> Repo.insert()
|
|
|
|
assert {:error, changeset} =
|
|
%GridScore{} |> GridScore.changeset(@valid_attrs) |> Repo.insert()
|
|
|
|
assert %{lat: ["has already been taken"]} = errors_on(changeset)
|
|
end
|
|
|
|
test "allows same location at different times" do
|
|
assert {:ok, _} =
|
|
%GridScore{} |> GridScore.changeset(@valid_attrs) |> Repo.insert()
|
|
|
|
later_attrs = Map.put(@valid_attrs, :valid_time, ~U[2026-03-28 19:00:00Z])
|
|
|
|
assert {:ok, _} =
|
|
%GridScore{} |> GridScore.changeset(later_attrs) |> Repo.insert()
|
|
end
|
|
|
|
test "allows same location and time for different bands" do
|
|
assert {:ok, _} =
|
|
%GridScore{} |> GridScore.changeset(@valid_attrs) |> Repo.insert()
|
|
|
|
other_band_attrs = Map.put(@valid_attrs, :band_mhz, 24_000)
|
|
|
|
assert {:ok, _} =
|
|
%GridScore{} |> GridScore.changeset(other_band_attrs) |> Repo.insert()
|
|
end
|
|
end
|
|
end
|