Graham McIntire
33f5d4edbe
Adds a per-contact enrichment pipeline that determines whether a QSO was
most likely carried by rain scatter, tropospheric ducting, or ordinary
troposcatter — using IEM n0q composite reflectivity sampled inside the
lens-shaped intersection of 400 km-radius disks around each endpoint.
Pieces:
* Microwaveprop.Propagation.CommonVolume — lens geometry (haversine,
in-CV test, bbox, area).
* contact_common_volume_radar table (1:1 per contact) storing
aggregate dBZ stats inside the CV + radar_status column on contacts.
* Microwaveprop.Workers.CommonVolumeRadarWorker — Oban :radar queue,
fetches the n0q frame at QSO time, iterates pixels inside the CV
bbox, aggregates rain/heavy/core-pixel counts, max/mean dBZ, and
coverage percentage.
* Microwaveprop.Propagation.RainScatterClassifier — rule-based mapper
from (band, distance, radar stats, duct flags) to one of
:likely_rainscatter | :rainscatter_possible | :tropo_duct |
:troposcatter | :unknown.
* ContactWeatherEnqueueWorker learns a :radar enrichment type and
enqueues the CV worker on contact submission; pre-2014 contacts
(outside IEM n0q coverage) are pinned to :unavailable.
* `mix radar_backfill` bulk-enqueues historical contacts with
--year / --limit / --dry-run.
* Contact detail page renders a mechanism badge with supporting
stats (common-volume area, max dBZ, heavy-rain pixel count,
coverage %).
101 lines
2.8 KiB
Elixir
101 lines
2.8 KiB
Elixir
defmodule Mix.Tasks.RadarBackfill do
|
|
@shortdoc "Enqueue CommonVolumeRadarWorker for contacts with radar_status :pending"
|
|
@moduledoc """
|
|
Bulk-enqueue per-contact common-volume radar enrichment for historical
|
|
contacts.
|
|
|
|
Eligible contacts: post-2014-10-02 (outside NARR coverage), both pos1
|
|
and pos2 present, `radar_status = :pending`. The worker's own unique
|
|
constraint on `contact_id` makes re-runs idempotent.
|
|
|
|
Options:
|
|
--limit N Maximum contacts to enqueue per run (default: all)
|
|
--year N Only enqueue contacts from this year (optional filter)
|
|
--dry-run Print counts; don't enqueue
|
|
|
|
Example:
|
|
|
|
mix radar_backfill --year 2024 --limit 5000
|
|
"""
|
|
use Mix.Task
|
|
|
|
import Ecto.Query
|
|
|
|
alias Microwaveprop.Radio
|
|
alias Microwaveprop.Radio.Contact
|
|
alias Microwaveprop.Repo
|
|
alias Microwaveprop.Workers.CommonVolumeRadarWorker
|
|
|
|
@narr_cutoff ~U[2014-10-02 00:00:00Z]
|
|
@chunk 400
|
|
|
|
@impl Mix.Task
|
|
def run(argv) do
|
|
Mix.Task.run("app.start")
|
|
|
|
{opts, _, _} =
|
|
OptionParser.parse(argv,
|
|
switches: [limit: :integer, year: :integer, dry_run: :boolean]
|
|
)
|
|
|
|
limit = Keyword.get(opts, :limit)
|
|
year = Keyword.get(opts, :year)
|
|
dry_run? = Keyword.get(opts, :dry_run, false)
|
|
|
|
total = count_eligible(year)
|
|
Mix.shell().info("Eligible contacts: #{total}#{if year, do: " (year #{year})", else: ""}")
|
|
|
|
if dry_run? or total == 0 do
|
|
:ok
|
|
else
|
|
ids = fetch_ids(year, limit)
|
|
|
|
Mix.shell().info("Enqueueing #{length(ids)} CommonVolumeRadarWorker jobs...")
|
|
|
|
ids
|
|
|> Enum.chunk_every(@chunk)
|
|
|> Enum.with_index(1)
|
|
|> Enum.each(fn {batch, i} ->
|
|
jobs = Enum.map(batch, &CommonVolumeRadarWorker.new(%{"contact_id" => &1}))
|
|
Oban.insert_all(jobs)
|
|
Radio.set_enrichment_status!(batch, :radar_status, :queued)
|
|
Mix.shell().info(" batch #{i}: #{length(batch)} enqueued")
|
|
end)
|
|
|
|
Mix.shell().info("Done.")
|
|
end
|
|
end
|
|
|
|
defp count_eligible(year) do
|
|
Contact
|
|
|> eligible_query(year)
|
|
|> Repo.aggregate(:count, :id)
|
|
end
|
|
|
|
defp fetch_ids(year, limit) do
|
|
query =
|
|
Contact
|
|
|> eligible_query(year)
|
|
|> order_by(asc: :qso_timestamp)
|
|
|> select([c], c.id)
|
|
|
|
query = if limit, do: limit(query, ^limit), else: query
|
|
Repo.all(query)
|
|
end
|
|
|
|
defp eligible_query(query, year) do
|
|
query
|
|
|> where([c], not is_nil(c.pos1) and not is_nil(c.pos2))
|
|
|> where([c], c.qso_timestamp >= ^@narr_cutoff)
|
|
|> where([c], c.radar_status == :pending)
|
|
|> maybe_filter_year(year)
|
|
end
|
|
|
|
defp maybe_filter_year(query, nil), do: query
|
|
|
|
defp maybe_filter_year(query, year) do
|
|
start_dt = DateTime.new!(Date.new!(year, 1, 1), ~T[00:00:00], "Etc/UTC")
|
|
end_dt = DateTime.new!(Date.new!(year, 12, 31), ~T[23:59:59], "Etc/UTC")
|
|
where(query, [c], c.qso_timestamp >= ^start_dt and c.qso_timestamp <= ^end_dt)
|
|
end
|
|
end
|