Graham McIntire
b6b5945002
CommonVolumeRadarWorker was enqueuing one job per contact — every job fetched + decoded the ~5 MB n0q PNG for its own 5-min frame. At current backlog depth that's 17,588 contacts across just 1,747 distinct 5-min frames, i.e. ~10 contacts per frame paying for the same PNG decode over and over. New RadarFrameWorker takes a batch of contact_ids sharing one frame, fetches + decodes the frame ONCE, then walks the in-memory pixel buffer per contact. build_radar_jobs/1 in ContactWeatherEnqueueWorker now groups the input contacts by their rounded 5-min timestamp and emits one RadarFrameWorker job per frame instead of one CommonVolumeRadarWorker per contact. process_frame/4 is public so the same code path is used both by perform/1 (production fetch) and tests (pre-decoded pixel buffer); keeps the happy-path test hermetic without hand-crafting a PNG. CommonVolumeRadarWorker stays in the tree — still used from the single-contact submit path where batching is pointless and the aggregate_stats/5 pure helper is a dependency of the new batched worker. Expected impact on the backfill: ~10x fewer fetch + decode cycles, so the 17k queued-contact backlog drains in minutes instead of ~50.
107 lines
3.5 KiB
Elixir
107 lines
3.5 KiB
Elixir
defmodule Microwaveprop.Workers.RadarFrameWorkerTest do
|
|
use Microwaveprop.DataCase, async: false
|
|
use Oban.Testing, repo: Microwaveprop.Repo
|
|
|
|
alias Microwaveprop.Radio.Contact
|
|
alias Microwaveprop.Radio.ContactCommonVolumeRadar
|
|
alias Microwaveprop.Repo
|
|
alias Microwaveprop.Weather.NexradClient
|
|
alias Microwaveprop.Workers.RadarFrameWorker
|
|
|
|
setup do
|
|
Req.Test.set_req_test_from_context(%{async: false})
|
|
:ok
|
|
end
|
|
|
|
defp insert_contact(attrs \\ %{}) do
|
|
default = %{
|
|
station1: "W5XD",
|
|
station2: "K5TR",
|
|
qso_timestamp: ~U[2024-09-15 18:30:00Z],
|
|
mode: "CW",
|
|
band: Decimal.new("10000"),
|
|
grid1: "EM12",
|
|
grid2: "EM00",
|
|
pos1: %{"lat" => 32.9, "lon" => -97.0},
|
|
pos2: %{"lat" => 33.1, "lon" => -96.8},
|
|
distance_km: Decimal.new("30")
|
|
}
|
|
|
|
{:ok, contact} =
|
|
%Contact{}
|
|
|> Contact.changeset(Map.merge(default, attrs))
|
|
|> Repo.insert()
|
|
|
|
contact
|
|
end
|
|
|
|
describe "process_frame/4" do
|
|
test "writes a ContactCommonVolumeRadar row and marks :complete for every contact in the batch" do
|
|
c1 = insert_contact(%{station1: "A1"})
|
|
c2 = insert_contact(%{station1: "A2"})
|
|
|
|
# In-memory pixel buffer: 10x10 empty frame. The aggregator will
|
|
# observe zero echoes but still emit a stats row per contact.
|
|
pixels = :binary.copy(<<0>>, 100)
|
|
width = 10
|
|
frame_ts = ~U[2024-09-15 18:30:00Z]
|
|
|
|
assert :ok =
|
|
RadarFrameWorker.process_frame(
|
|
[Repo.reload!(c1), Repo.reload!(c2)],
|
|
frame_ts,
|
|
pixels,
|
|
width
|
|
)
|
|
|
|
assert Repo.get_by(ContactCommonVolumeRadar, contact_id: c1.id)
|
|
assert Repo.get_by(ContactCommonVolumeRadar, contact_id: c2.id)
|
|
|
|
assert %Contact{radar_status: :complete} = Repo.get!(Contact, c1.id)
|
|
assert %Contact{radar_status: :complete} = Repo.get!(Contact, c2.id)
|
|
end
|
|
end
|
|
|
|
describe "perform/1" do
|
|
test "marks every contact in the batch :unavailable when the frame 404s" do
|
|
c1 = insert_contact(%{station1: "B1"})
|
|
c2 = insert_contact(%{station1: "B2"})
|
|
|
|
Req.Test.stub(NexradClient, fn conn ->
|
|
Plug.Conn.send_resp(conn, 404, "not found")
|
|
end)
|
|
|
|
assert :ok =
|
|
perform_job(RadarFrameWorker, %{
|
|
"frame_ts" => "2024-09-15T18:30:00Z",
|
|
"contact_ids" => [c1.id, c2.id]
|
|
})
|
|
|
|
refute Repo.get_by(ContactCommonVolumeRadar, contact_id: c1.id)
|
|
refute Repo.get_by(ContactCommonVolumeRadar, contact_id: c2.id)
|
|
assert %Contact{radar_status: :unavailable} = Repo.get!(Contact, c1.id)
|
|
assert %Contact{radar_status: :unavailable} = Repo.get!(Contact, c2.id)
|
|
end
|
|
|
|
test "skips contacts already at a terminal status (idempotent replay)" do
|
|
c_done = insert_contact(%{station1: "DONE"})
|
|
{:ok, _} = c_done |> Ecto.Changeset.change(%{radar_status: :complete}) |> Repo.update()
|
|
|
|
c_new = insert_contact(%{station1: "NEW"})
|
|
|
|
Req.Test.stub(NexradClient, fn conn ->
|
|
Plug.Conn.send_resp(conn, 404, "not found")
|
|
end)
|
|
|
|
assert :ok =
|
|
perform_job(RadarFrameWorker, %{
|
|
"frame_ts" => "2024-09-15T18:30:00Z",
|
|
"contact_ids" => [c_done.id, c_new.id]
|
|
})
|
|
|
|
# Already-complete contact stays complete; new one is marked unavailable.
|
|
assert %Contact{radar_status: :complete} = Repo.get!(Contact, c_done.id)
|
|
assert %Contact{radar_status: :unavailable} = Repo.get!(Contact, c_new.id)
|
|
end
|
|
end
|
|
end
|