prop/lib/microwaveprop/radio/import.ex
Graham McIntire 0c3be97abb
Some checks failed
Build base image / Build and push base image (push) Successful in 3m10s
Build and Push / Build and Push Docker Image (push) Failing after 14s
Build prop-grid-rs / Test, build, push (push) Successful in 12m52s
fix: resolve 27 security, architecture, test, and performance audit findings
P0 (security-critical):
- Gate CSV/ADIF upload tabs behind authentication, add 30s cooldown to all upload handlers
- Cap CSV/ADIF imports at 2,000 rows server-side in both parsers
- Add submitter_verified boolean to contacts (client-cannot-set, anonymous=false)
- Create k8s/secret.example.yaml with placeholders, add LIVE_VIEW_SIGNING_SALT

P1 (high-priority):
- Add Mox.verify_on_exit!() to valkey_test.exs
- Replace DateTime.utc_now() truncation with static ~U literals in map_live_test.exs
- Replace Process.sleep with render_async in pskr_spots_live_test.exs (6 occurrences)
- Add MonitorLive.Show test coverage (4 tests: owner view, non-owner redirect, config success/error)
- Extract duct-detection and mechanism-classification logic from ContactLive.Show into Propagation.PathAnalysis
- Split ContactLive.Show render into 12 function components
- Update CLAUDE.md: remove stale ML model, mark HRDPS active, add backtest/pskr dirs
- Batch CSV import enrichment jobs via new enqueue_for_contacts/1

P2 (medium-priority):
- Set secure:true on session and remember-me cookies in production
- Change SMTP TLS from verify_none to verify_peer with public_key cacerts
- Make /metrics fail-closed in production when PROMETHEUS_AUTH_TOKEN unset
- Add RateLimiter (anon_limit:10, auth_limit:60) to /api/contacts/map
- Add content-security-policy-report-only header
- Add comment noting String.to_atom is compile-time safe in hrdps_client.ex
- Delegate duplicated haversine_km to canonical Microwaveprop.Geo.haversine_km/4
- Consolidate score-tier/color/verdict formatting into Microwaveprop.Format
- Update CLAUDE.md testing section to match actual raw-string-matching practice
- Batch HrrrPointEnqueuer Repo.insert_all calls to single round-trip
- Split weather.ex (1696→216 lines) and radio.ex (1285→54 lines) into purpose-based sub-facades

P3 (low-priority):
- Add LIVE_VIEW_SIGNING_SALT warning comment, extend filter_parameters
- Add host/community validation to snmp_client.ex
- Add raw/1 safety comment in algo_live.ex
- Add hex-audit and cargo-audit Makefile targets
- Add privacy_live smoke test
- Replace notify_listener busy-poll loop with Process.monitor/1 + assert_receive
- Add ContactCommonVolumeRadar changeset validation tests (5 tests)
2026-07-27 18:19:37 -05:00

385 lines
14 KiB
Elixir

defmodule Microwaveprop.Radio.Import do
@moduledoc false
import Ecto.Query
alias Microwaveprop.Accounts.User
alias Microwaveprop.Radio.Contact
alias Microwaveprop.Radio.ContactEdit
alias Microwaveprop.Radio.Contacts
alias Microwaveprop.Radio.Maidenhead
alias Microwaveprop.Repo
alias Microwaveprop.Workers.ContactWeatherEnqueueWorker
@refinement_fields ~w(grid1 grid2 mode)a
@refinement_allowed_modes ~w(CW SSB FM FT8 FT4 Q65)
@editable_proposed_keys ~w(station1 station2 grid1 grid2 band mode qso_timestamp height1_ft height2_ft private)
# ── Import refinement ──
@spec apply_contact_refinement(Contact.t(), map()) :: {:ok, Contact.t()} | {:error, Ecto.Changeset.t()}
def apply_contact_refinement(%Contact{} = contact, changes) when is_map(changes) do
cleaned = Map.take(changes, @refinement_fields)
if cleaned == %{}, do: {:ok, contact}, else: do_apply_contact_refinement(contact, cleaned)
end
defp do_apply_contact_refinement(contact, changes) do
changeset =
contact
|> Ecto.Changeset.cast(changes, @refinement_fields)
|> validate_refinement_mode()
|> maybe_recompute_refinement_positions(contact, changes)
if changeset.valid?, do: Repo.update(changeset), else: {:error, %{changeset | action: :update}}
end
defp validate_refinement_mode(changeset),
do: Ecto.Changeset.validate_inclusion(changeset, :mode, @refinement_allowed_modes)
defp maybe_recompute_refinement_positions(changeset, contact, changes) do
new_grid1 = Map.get(changes, :grid1)
new_grid2 = Map.get(changes, :grid2)
cond do
new_grid1 && new_grid2 -> put_pos_changes(changeset, new_grid1, new_grid2, :both)
new_grid1 -> put_pos_changes(changeset, new_grid1, contact.grid2, :side1)
new_grid2 -> put_pos_changes(changeset, contact.grid1, new_grid2, :side2)
true -> changeset
end
end
defp put_pos_changes(changeset, grid1, grid2, sides) do
case {Maidenhead.to_latlon(grid1), Maidenhead.to_latlon(grid2)} do
{{:ok, {lat1, lon1}}, {:ok, {lat2, lon2}}} ->
distance = lat1 |> Contacts.haversine_km(lon1, lat2, lon2) |> round() |> Decimal.new()
changeset
|> maybe_put_pos_for_side(:pos1, %{"lat" => lat1, "lon" => lon1}, sides)
|> maybe_put_pos_for_side(:pos2, %{"lat" => lat2, "lon" => lon2}, sides)
|> Ecto.Changeset.put_change(:distance_km, distance)
|> reset_all_enrichment_statuses()
_ ->
Ecto.Changeset.add_error(changeset, :grid1, "could not resolve grid to coordinates")
end
end
defp maybe_put_pos_for_side(changeset, :pos1, pos, sides) when sides in [:side1, :both],
do: Ecto.Changeset.put_change(changeset, :pos1, pos)
defp maybe_put_pos_for_side(changeset, :pos2, pos, sides) when sides in [:side2, :both],
do: Ecto.Changeset.put_change(changeset, :pos2, pos)
defp maybe_put_pos_for_side(changeset, _field, _pos, _sides), do: changeset
defp reset_all_enrichment_statuses(changeset) do
changeset
|> Ecto.Changeset.put_change(:hrrr_status, :pending)
|> Ecto.Changeset.put_change(:weather_status, :pending)
|> Ecto.Changeset.put_change(:terrain_status, :pending)
|> Ecto.Changeset.put_change(:iemre_status, :pending)
end
# ── Contact edits ──
@spec create_contact_edit(Contact.t(), User.t(), map()) :: {:ok, ContactEdit.t()} | {:error, Ecto.Changeset.t()}
def create_contact_edit(%Contact{} = contact, user, proposed_changes) when is_map(proposed_changes) do
normalized = normalize_proposed(proposed_changes)
diffed = diff_against_contact(contact, normalized)
attrs = %{contact_id: contact.id, user_id: user.id, proposed_changes: diffed}
%ContactEdit{} |> ContactEdit.changeset(attrs) |> Repo.insert()
end
@doc false
@spec normalize_proposed(map()) :: map()
def normalize_proposed(changes) do
changes
|> Map.take(@editable_proposed_keys)
|> normalize_string_field("station1")
|> normalize_string_field("station2")
|> normalize_string_field("grid1")
|> normalize_string_field("grid2")
|> normalize_string_field("mode")
|> normalize_integer_field("height1_ft")
|> normalize_integer_field("height2_ft")
|> normalize_boolean_field("private")
|> normalize_timestamp_field("qso_timestamp")
end
defp normalize_string_field(map, key) do
case Map.get(map, key) do
nil -> map
val when is_binary(val) -> Map.put(map, key, val |> String.trim() |> String.upcase())
val -> Map.put(map, key, val)
end
end
defp normalize_integer_field(map, key) do
case Map.get(map, key, :not_provided) do
:not_provided ->
map
nil ->
map
val when is_integer(val) ->
map
"" ->
Map.delete(map, key)
val when is_binary(val) ->
case Integer.parse(String.trim(val)) do
{int, ""} -> Map.put(map, key, int)
_ -> map
end
_ ->
map
end
end
defp normalize_boolean_field(map, key) do
case Map.get(map, key, :not_provided) do
:not_provided -> map
nil -> map
val when is_boolean(val) -> Map.put(map, key, val)
"true" -> Map.put(map, key, true)
"false" -> Map.put(map, key, false)
_ -> map
end
end
defp normalize_timestamp_field(map, key) do
case Map.get(map, key, :not_provided) do
:not_provided -> map
nil -> Map.delete(map, key)
"" -> Map.delete(map, key)
%DateTime{} = dt -> Map.put(map, key, dt)
val when is_binary(val) -> put_or_drop_timestamp(map, key, parse_timestamp(val))
_ -> Map.delete(map, key)
end
end
defp put_or_drop_timestamp(map, key, nil), do: Map.delete(map, key)
defp put_or_drop_timestamp(map, key, %DateTime{} = dt), do: Map.put(map, key, dt)
defp parse_timestamp(val) do
trimmed = String.trim(val)
iso = trimmed |> String.replace(" ", "T") |> ensure_seconds() |> String.trim_trailing("Z")
case NaiveDateTime.from_iso8601(iso) do
{:ok, ndt} -> DateTime.from_naive!(ndt, "Etc/UTC")
_ -> nil
end
end
defp ensure_seconds(s) do
case Regex.run(~r/^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2})(Z?)$/, s) do
[_, prefix, tail] -> prefix <> ":00" <> tail
_ -> s
end
end
@doc false
@spec diff_against_contact(Contact.t(), map()) :: map()
def diff_against_contact(contact, proposed) do
Enum.reduce(proposed, %{}, fn {key, new_val}, acc ->
current = current_value(contact, key)
if values_equal?(current, new_val), do: acc, else: Map.put(acc, key, new_val)
end)
end
defp current_value(contact, "station1"), do: contact.station1
defp current_value(contact, "station2"), do: contact.station2
defp current_value(contact, "grid1"), do: contact.grid1
defp current_value(contact, "grid2"), do: contact.grid2
defp current_value(contact, "mode"), do: contact.mode
defp current_value(contact, "band"), do: if(contact.band, do: Decimal.to_integer(contact.band))
defp current_value(contact, "qso_timestamp"), do: contact.qso_timestamp
defp current_value(contact, "height1_ft"), do: contact.height1_ft
defp current_value(contact, "height2_ft"), do: contact.height2_ft
defp current_value(contact, "private"), do: contact.private
defp current_value(_contact, _key), do: nil
defp values_equal?(a, b) when is_binary(a) and is_binary(b), do: String.upcase(a) == String.upcase(b)
defp values_equal?(a, b) when is_integer(a), do: a == to_integer(b)
defp values_equal?(a, b) when is_integer(b), do: to_integer(a) == b
defp values_equal?(a, b), do: a == b
defp to_integer(v) when is_integer(v), do: v
defp to_integer(v) when is_binary(v), do: String.to_integer(v)
defp to_integer(%Decimal{} = v), do: Decimal.to_integer(v)
defp to_integer(v), do: v
# ── Pending edits ──
@spec list_pending_edits() :: [ContactEdit.t()]
def list_pending_edits do
ContactEdit
|> where([e], e.status == :pending)
|> order_by([e], desc: e.inserted_at)
|> preload([:user, :contact])
|> Repo.all()
end
@spec pending_edits_query() :: Ecto.Query.t()
def pending_edits_query do
from e in ContactEdit, as: :resource, where: e.status == :pending, preload: [:user, :contact]
end
@spec list_contact_edits(Ecto.UUID.t()) :: [ContactEdit.t()]
def list_contact_edits(contact_id) do
ContactEdit
|> where([e], e.contact_id == ^contact_id)
|> order_by([e], desc: e.inserted_at)
|> preload([:user, :reviewed_by])
|> Repo.all()
end
@spec pending_edit_count() :: non_neg_integer()
def pending_edit_count, do: ContactEdit |> where([e], e.status == :pending) |> Repo.aggregate(:count)
@spec pending_edit_for_user(Ecto.UUID.t(), Ecto.UUID.t()) :: ContactEdit.t() | nil
def pending_edit_for_user(contact_id, user_id) do
ContactEdit
|> where([e], e.contact_id == ^contact_id and e.user_id == ^user_id and e.status == :pending)
|> preload([:user, :contact])
|> Repo.one()
end
@spec get_contact_edit(Ecto.UUID.t()) :: ContactEdit.t() | nil
def get_contact_edit(id), do: ContactEdit |> preload([:user, :contact, :reviewed_by]) |> Repo.get(id)
# ── Edit approval ──
@spec approve_edit(ContactEdit.t(), User.t(), String.t() | nil) :: {:ok, ContactEdit.t()} | {:error, any()}
def approve_edit(%ContactEdit{status: :pending} = edit, admin, note) do
Repo.transaction(fn ->
case Repo.get(Contact, edit.contact_id) do
nil ->
Repo.rollback(:contact_deleted)
contact ->
{:ok, approved} =
edit
|> ContactEdit.review_changeset(%{
status: :approved,
admin_note: note,
reviewed_by_id: admin.id,
reviewed_at: DateTime.truncate(DateTime.utc_now(), :second)
})
|> Repo.update()
_ = apply_edit_to_contact(contact, edit.proposed_changes)
Repo.preload(approved, [:user, :contact, :reviewed_by])
end
end)
end
@spec reject_edit(ContactEdit.t(), User.t(), String.t() | nil) ::
{:ok, ContactEdit.t()} | {:error, Ecto.Changeset.t()}
def reject_edit(%ContactEdit{status: :pending} = edit, admin, note) do
edit
|> ContactEdit.review_changeset(%{
status: :rejected,
admin_note: note,
reviewed_by_id: admin.id,
reviewed_at: DateTime.truncate(DateTime.utc_now(), :second)
})
|> Repo.update()
end
# ── Owner edit ──
@spec apply_owner_edit(Contact.t(), User.t(), map()) :: {:ok, Contact.t()} | {:error, :not_owner | :no_changes}
def apply_owner_edit(%Contact{} = contact, %User{} = user, proposed_changes) when is_map(proposed_changes) do
if Contacts.owner?(contact, user) do
diffed = diff_against_contact(contact, normalize_proposed(proposed_changes))
if diffed == %{}, do: {:error, :no_changes}, else: {:ok, apply_edit_to_contact(contact, diffed)}
else
{:error, :not_owner}
end
end
@doc "Apply proposed changes directly to a contact (admin use)."
@spec apply_edit_to_contact(Contact.t(), map()) :: Contact.t()
def apply_edit_to_contact(contact, proposed_changes) do
changes = build_contact_changes(contact, proposed_changes)
changes = maybe_recompute_positions(changes, contact, proposed_changes)
changes = maybe_reset_terrain_for_heights(changes, proposed_changes)
updated = contact |> Ecto.Changeset.change(changes) |> Repo.update!()
maybe_enqueue_terrain_for_heights(updated, proposed_changes)
Contacts.invalidate_contact_map_caches()
updated
end
defp maybe_enqueue_terrain_for_heights(contact, proposed) do
if Map.has_key?(proposed, "height1_ft") or Map.has_key?(proposed, "height2_ft") do
ContactWeatherEnqueueWorker.enqueue_for_contact(contact, [:terrain])
end
:ok
end
defp maybe_reset_terrain_for_heights(changes, proposed) do
if Map.has_key?(proposed, "height1_ft") or Map.has_key?(proposed, "height2_ft"),
do: Map.put(changes, :terrain_status, :pending),
else: changes
end
defp maybe_recompute_positions(changes, contact, proposed) do
needs_recompute =
Map.has_key?(proposed, "grid1") or Map.has_key?(proposed, "grid2") or Map.has_key?(proposed, "band") or
Map.has_key?(proposed, "qso_timestamp")
if needs_recompute, do: recompute_positions(changes, contact, proposed), else: changes
end
defp recompute_positions(changes, contact, proposed) do
pos1 = resolve_grid_position(Map.get(proposed, "grid1", contact.grid1)) || contact.pos1
pos2 = resolve_grid_position(Map.get(proposed, "grid2", contact.grid2)) || contact.pos2
distance = compute_distance(pos1, pos2)
changes
|> Map.put(:pos1, pos1)
|> Map.put(:pos2, pos2)
|> Map.put(:distance_km, distance)
|> Map.put(:hrrr_status, :pending)
|> Map.put(:weather_status, :pending)
|> Map.put(:terrain_status, :pending)
|> Map.put(:iemre_status, :pending)
end
defp compute_distance(pos1, pos2) when not is_nil(pos1) and not is_nil(pos2),
do: pos1["lat"] |> Contacts.haversine_km(pos1["lon"], pos2["lat"], pos2["lon"]) |> round() |> Decimal.new()
defp compute_distance(_, _), do: nil
defp build_contact_changes(_contact, proposed) do
Enum.reduce(proposed, %{}, fn
{"band", val}, acc ->
Map.put(acc, :band, Decimal.new(to_string(val)))
{"qso_timestamp", val}, acc when is_binary(val) ->
{:ok, dt, _} = DateTime.from_iso8601(val)
Map.put(acc, :qso_timestamp, dt)
{"qso_timestamp", %DateTime{} = dt}, acc ->
Map.put(acc, :qso_timestamp, dt)
{key, val}, acc ->
Map.put(acc, String.to_existing_atom(key), val)
end)
end
defp resolve_grid_position(nil), do: nil
defp resolve_grid_position(grid) do
case Maidenhead.to_latlon(grid) do
{:ok, {lat, lon}} -> %{"lat" => lat, "lon" => lon}
_ -> nil
end
end
end