From bbeb95ff5d246c16eb3dfd1f9e32543fd4be4d67 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Thu, 16 Apr 2026 10:57:33 -0500 Subject: [PATCH] CSV/ADIF import: upsert existing contacts on grid/mode refinement Widen dedup match to 4-char grid prefix so an upload with a longer grid (e.g. FN42aa25) finds the existing FN42 contact. Within a match, classify as a refinement when the upload strictly extends the existing grid or fills a missing mode; as a contradiction (still skipped) when grids or modes genuinely disagree. Refinements flow through a new preview bucket, update the existing row in place, and re-enqueue enrichment when the position changed. --- lib/microwaveprop/radio.ex | 95 ++++++++ lib/microwaveprop/radio/adif_import.ex | 116 ++++++---- lib/microwaveprop/radio/csv_import.ex | 213 ++++++++++++------ lib/microwaveprop/radio/import_matcher.ex | 171 ++++++++++++++ lib/microwaveprop_web/live/submit_live.ex | 115 +++++++++- test/microwaveprop/radio/adif_import_test.exs | 26 +++ test/microwaveprop/radio/csv_import_test.exs | 206 +++++++++++++++++ .../radio/import_matcher_test.exs | 189 ++++++++++++++++ test/microwaveprop/radio_refinement_test.exs | 152 +++++++++++++ .../live/submit_live_test.exs | 52 +++++ 10 files changed, 1203 insertions(+), 132 deletions(-) create mode 100644 lib/microwaveprop/radio/import_matcher.ex create mode 100644 test/microwaveprop/radio/import_matcher_test.exs create mode 100644 test/microwaveprop/radio_refinement_test.exs diff --git a/lib/microwaveprop/radio.ex b/lib/microwaveprop/radio.ex index 6829c0e9..a09b2eb8 100644 --- a/lib/microwaveprop/radio.ex +++ b/lib/microwaveprop/radio.ex @@ -639,6 +639,101 @@ defmodule Microwaveprop.Radio do end) end + @refinement_fields ~w(grid1 grid2 mode)a + @refinement_allowed_modes ~w(CW SSB FM FT8 FT4 Q65) + + @doc """ + Apply a grid/mode refinement from an import row to an existing contact. + + `changes` may contain any of `:grid1`, `:grid2`, or `:mode`. Grid changes + trigger recomputation of the corresponding `pos` and the `distance_km`, + and reset all four enrichment status fields to `:pending` so the caller + can re-enqueue the enrichment pipeline. Mode-only changes do not touch + positions or enrichment status. + """ + @spec apply_contact_refinement(Contact.t(), map()) :: + {:ok, Contact.t()} | {:error, Ecto.Changeset.t()} + def apply_contact_refinement(%Contact{} = contact, changes) when is_map(changes) do + cleaned = Map.take(changes, @refinement_fields) + + if cleaned == %{} do + {:ok, contact} + else + do_apply_contact_refinement(contact, cleaned) + end + end + + defp do_apply_contact_refinement(contact, changes) do + changeset = + contact + |> Ecto.Changeset.cast(changes, @refinement_fields) + |> validate_refinement_mode() + |> maybe_recompute_refinement_positions(contact, changes) + + if changeset.valid? do + Repo.update(changeset) + else + {:error, %{changeset | action: :update}} + end + end + + defp validate_refinement_mode(changeset) do + Ecto.Changeset.validate_inclusion(changeset, :mode, @refinement_allowed_modes) + end + + defp maybe_recompute_refinement_positions(changeset, contact, changes) do + new_grid1 = Map.get(changes, :grid1) + new_grid2 = Map.get(changes, :grid2) + + cond do + new_grid1 && new_grid2 -> + put_pos_changes(changeset, new_grid1, new_grid2, :both) + + new_grid1 -> + put_pos_changes(changeset, new_grid1, contact.grid2, :side1) + + new_grid2 -> + put_pos_changes(changeset, contact.grid1, new_grid2, :side2) + + true -> + changeset + end + end + + defp put_pos_changes(changeset, grid1, grid2, sides) do + case {Maidenhead.to_latlon(grid1), Maidenhead.to_latlon(grid2)} do + {{:ok, {lat1, lon1}}, {:ok, {lat2, lon2}}} -> + distance = lat1 |> haversine_km(lon1, lat2, lon2) |> round() |> Decimal.new() + + changeset + |> maybe_put_pos_for_side(:pos1, %{"lat" => lat1, "lon" => lon1}, sides) + |> maybe_put_pos_for_side(:pos2, %{"lat" => lat2, "lon" => lon2}, sides) + |> Ecto.Changeset.put_change(:distance_km, distance) + |> reset_all_enrichment_statuses() + + _ -> + Ecto.Changeset.add_error(changeset, :grid1, "could not resolve grid to coordinates") + end + end + + defp maybe_put_pos_for_side(changeset, :pos1, pos, sides) when sides in [:side1, :both] do + Ecto.Changeset.put_change(changeset, :pos1, pos) + end + + defp maybe_put_pos_for_side(changeset, :pos2, pos, sides) when sides in [:side2, :both] do + Ecto.Changeset.put_change(changeset, :pos2, pos) + end + + defp maybe_put_pos_for_side(changeset, _field, _pos, _sides), do: changeset + + defp reset_all_enrichment_statuses(changeset) do + changeset + |> Ecto.Changeset.put_change(:hrrr_status, :pending) + |> Ecto.Changeset.put_change(:weather_status, :pending) + |> Ecto.Changeset.put_change(:terrain_status, :pending) + |> Ecto.Changeset.put_change(:iemre_status, :pending) + end + # ── Contact Edits ─────────────────────────────────────────────── @doc """ diff --git a/lib/microwaveprop/radio/adif_import.ex b/lib/microwaveprop/radio/adif_import.ex index cd030b5e..fc783e82 100644 --- a/lib/microwaveprop/radio/adif_import.ex +++ b/lib/microwaveprop/radio/adif_import.ex @@ -11,6 +11,7 @@ defmodule Microwaveprop.Radio.AdifImport do alias Microwaveprop.Radio.BandResolver alias Microwaveprop.Radio.Contact + alias Microwaveprop.Radio.ImportMatcher alias Microwaveprop.Repo @dedup_window_seconds 3600 @@ -90,6 +91,7 @@ defmodule Microwaveprop.Radio.AdifImport do valid: [map()], invalid: [map()], duplicates: [map()], + refinements: [map()], total_rows: non_neg_integer(), submitter_email: String.t() } @@ -104,13 +106,14 @@ defmodule Microwaveprop.Radio.AdifImport do rows = records |> Enum.with_index(1) |> Enum.map(&build_row(&1, submitter_email)) {valid, invalid} = split_parsed(rows) existing = load_existing_for_dedup(valid) - {unique, duplicates} = dedupe(valid, existing) + {unique, duplicates, refinements} = dedupe(valid, existing) {:ok, %{ valid: unique, invalid: invalid, duplicates: duplicates, + refinements: refinements, total_rows: length(records), submitter_email: submitter_email }} @@ -284,52 +287,69 @@ defmodule Microwaveprop.Radio.AdifImport do from(c in Contact, where: c.band in ^bands and c.qso_timestamp >= ^min_ts and c.qso_timestamp <= ^max_ts and - c.flagged_invalid == false, - select: %{ - station1: c.station1, - station2: c.station2, - grid1: c.grid1, - grid2: c.grid2, - band: c.band, - qso_timestamp: c.qso_timestamp - } + c.flagged_invalid == false ) |> Repo.all() - |> Enum.reduce(%{}, fn row, acc -> - key = dedup_key(row.station1, row.grid1, row.station2, row.grid2, row.band) - Map.update(acc, key, [row.qso_timestamp], &[row.qso_timestamp | &1]) + |> Enum.reduce(%{}, fn contact, acc -> + case ImportMatcher.prefix_key(contact.station1, contact.grid1, contact.station2, contact.grid2, contact.band) do + :invalid -> acc + key -> Map.update(acc, key, [contact], &[contact | &1]) + end end) end defp dedupe(rows, db_map) do - {accepted, _in_batch, duplicates} = - Enum.reduce(rows, {[], %{}, []}, fn row, {accepted, in_batch, dups} -> - key = - dedup_key( - row.attrs["station1"], - row.attrs["grid1"], - row.attrs["station2"], - row.attrs["grid2"], - row.attrs["band"] - ) + {accepted, _in_batch, duplicates, refinements} = + Enum.reduce(rows, {[], %{}, [], []}, &classify_row(&1, &2, db_map)) - cond do - conflict_in_map?(db_map, key, row.timestamp) -> - {accepted, in_batch, [Map.put(row, :reason, :existing_contact) | dups]} - - conflict_in_map?(in_batch, key, row.timestamp) -> - {accepted, in_batch, [Map.put(row, :reason, :earlier_in_upload) | dups]} - - true -> - new_batch = Map.update(in_batch, key, [row.timestamp], &[row.timestamp | &1]) - {[row | accepted], new_batch, dups} - end - end) - - {Enum.reverse(accepted), Enum.reverse(duplicates)} + {Enum.reverse(accepted), Enum.reverse(duplicates), Enum.reverse(refinements)} end - defp conflict_in_map?(map, key, timestamp) do + defp classify_row(row, {accepted, in_batch, dups, refs}, db_map) do + key = prefix_key_from_attrs(row.attrs) + + cond do + key == :invalid -> + {[row | accepted], in_batch, dups, refs} + + candidate = db_candidate(db_map, key, row.timestamp) -> + classify_db_candidate(row, candidate, accepted, in_batch, dups, refs) + + batch_conflict?(in_batch, key, row.timestamp) -> + {accepted, in_batch, [Map.put(row, :reason, :earlier_in_upload) | dups], refs} + + true -> + new_batch = Map.update(in_batch, key, [row.timestamp], &[row.timestamp | &1]) + {[row | accepted], new_batch, dups, refs} + end + end + + defp classify_db_candidate(row, candidate, accepted, in_batch, dups, refs) do + case ImportMatcher.classify_against_existing(row.attrs, candidate) do + {:refinement, changes} -> + refinement = %{ + row_num: row.row_num, + attrs: row.attrs, + timestamp: row.timestamp, + existing_id: candidate.id, + changes: changes + } + + {accepted, in_batch, dups, [refinement | refs]} + + _ -> + {accepted, in_batch, [Map.put(row, :reason, :existing_contact) | dups], refs} + end + end + + defp db_candidate(db_map, key, timestamp) do + case Map.fetch(db_map, key) do + :error -> nil + {:ok, contacts} -> Enum.find(contacts, &within_window?(&1.qso_timestamp, timestamp)) + end + end + + defp batch_conflict?(map, key, timestamp) do case Map.fetch(map, key) do :error -> false {:ok, timestamps} -> Enum.any?(timestamps, &within_window?(&1, timestamp)) @@ -340,20 +360,16 @@ defmodule Microwaveprop.Radio.AdifImport do abs(DateTime.diff(t1, t2, :second)) <= @dedup_window_seconds end - defp dedup_key(s1, g1, s2, g2, band) do - a = {upcase(s1), upcase(g1)} - b = {upcase(s2), upcase(g2)} - [c1, c2] = Enum.sort([a, b]) - {c1, c2, normalize_band(band)} + defp prefix_key_from_attrs(attrs) do + ImportMatcher.prefix_key( + attrs["station1"], + attrs["grid1"], + attrs["station2"], + attrs["grid2"], + attrs["band"] + ) end - defp upcase(nil), do: "" - defp upcase(v), do: v |> to_string() |> String.trim() |> String.upcase() - - defp normalize_band(%Decimal{} = b), do: Decimal.to_integer(b) - defp normalize_band(b) when is_integer(b), do: b - defp normalize_band(b) when is_binary(b), do: b |> String.trim() |> String.to_integer() - defp changeset_error_strings(changeset) do changeset |> Ecto.Changeset.traverse_errors(fn {message, opts} -> diff --git a/lib/microwaveprop/radio/csv_import.ex b/lib/microwaveprop/radio/csv_import.ex index 73e963b3..a3fbb780 100644 --- a/lib/microwaveprop/radio/csv_import.ex +++ b/lib/microwaveprop/radio/csv_import.ex @@ -25,6 +25,7 @@ defmodule Microwaveprop.Radio.CsvImport do alias Microwaveprop.Radio alias Microwaveprop.Radio.BandResolver alias Microwaveprop.Radio.Contact + alias Microwaveprop.Radio.ImportMatcher alias Microwaveprop.Repo alias Microwaveprop.Workers.ContactWeatherEnqueueWorker @@ -60,6 +61,7 @@ defmodule Microwaveprop.Radio.CsvImport do valid: [map()], invalid: [map()], duplicates: [map()], + refinements: [map()], total_rows: non_neg_integer(), submitter_email: String.t() } @@ -71,6 +73,7 @@ defmodule Microwaveprop.Radio.CsvImport do @type commit_result :: %{ imported: [Contact.t()], + refined: [Contact.t()], errors: [{pos_integer(), [String.t()]}] } @@ -80,10 +83,9 @@ defmodule Microwaveprop.Radio.CsvImport do | {:error, :no_data_rows} | {:error, {:missing_required_columns, [String.t()]}} def import(csv_string, submitter_email) do - with {:ok, rows} <- parse_csv(csv_string, submitter_email) do - {valid, invalid} = split_parsed(rows) - {:ok, commit_result} = commit(valid) - invalid_errors = Enum.map(invalid, fn %{row_num: n, messages: m} -> {n, m} end) + with {:ok, preview} <- preview(csv_string, submitter_email) do + {:ok, commit_result} = commit(%{valid: preview.valid, refinements: preview.refinements}) + invalid_errors = Enum.map(preview.invalid, fn %{row_num: n, messages: m} -> {n, m} end) {:ok, %{ @@ -118,13 +120,14 @@ defmodule Microwaveprop.Radio.CsvImport do with {:ok, rows} <- parse_csv(csv_string, submitter_email) do {valid, invalid} = split_parsed(rows) existing = load_existing_for_dedup(valid) - {unique, duplicates} = dedupe(valid, existing) + {unique, duplicates, refinements} = dedupe(valid, existing) {:ok, %{ valid: unique, invalid: invalid, duplicates: duplicates, + refinements: refinements, total_rows: length(rows), submitter_email: submitter_email }} @@ -132,25 +135,79 @@ defmodule Microwaveprop.Radio.CsvImport do end @doc """ - Inserts the rows returned by `preview/2` in `:valid`. Each is passed to - `Radio.create_contact/1` and its enrichment is enqueued on success. - Returns `{:ok, %{imported: [...], errors: [{row_num, messages}, ...]}}`. + Commit a preview. Accepts either a list of valid rows (legacy) or a map with + `:valid` and `:refinements` keys. Valid rows are inserted via + `Radio.create_contact/1`. Refinements are applied to their matching existing + contacts via `Radio.apply_contact_refinement/2`; grid-changing refinements + re-enqueue enrichment, mode-only refinements do not. + + Returns `{:ok, %{imported: [...], refined: [...], errors: [{row_num, messages}, ...]}}`. """ - @spec commit([map()]) :: {:ok, commit_result()} + @spec commit([map()] | %{valid: [map()], refinements: [map()]}) :: {:ok, commit_result()} def commit(valid_rows) when is_list(valid_rows) do + commit(%{valid: valid_rows, refinements: []}) + end + + def commit(%{valid: valid_rows, refinements: refinements}) do + init = %{imported: [], refined: [], errors: []} + result = - Enum.reduce(valid_rows, %{imported: [], errors: []}, fn row, acc -> - case Radio.create_contact(row.attrs) do - {:ok, contact} -> - ContactWeatherEnqueueWorker.enqueue_for_contact(contact) - %{acc | imported: [contact | acc.imported]} + init + |> apply_valid_rows(valid_rows) + |> apply_refinements(refinements) - {:error, %Ecto.Changeset{} = changeset} -> - %{acc | errors: [{row.row_num, changeset_error_strings(changeset)} | acc.errors]} - end - end) + {:ok, + %{ + imported: Enum.reverse(result.imported), + refined: Enum.reverse(result.refined), + errors: Enum.reverse(result.errors) + }} + end - {:ok, %{imported: Enum.reverse(result.imported), errors: Enum.reverse(result.errors)}} + defp apply_valid_rows(acc, valid_rows) do + Enum.reduce(valid_rows, acc, fn row, acc -> + case Radio.create_contact(row.attrs) do + {:ok, contact} -> + ContactWeatherEnqueueWorker.enqueue_for_contact(contact) + %{acc | imported: [contact | acc.imported]} + + {:error, %Ecto.Changeset{} = changeset} -> + %{acc | errors: [{row.row_num, changeset_error_strings(changeset)} | acc.errors]} + end + end) + end + + defp apply_refinements(acc, refinements) do + Enum.reduce(refinements, acc, &apply_refinement_row/2) + end + + defp apply_refinement_row(ref, acc) do + case Repo.get(Contact, ref.existing_id) do + nil -> + %{acc | errors: [{ref.row_num, ["existing contact no longer exists"]} | acc.errors]} + + %Contact{} = contact -> + apply_refinement_to_contact(contact, ref, acc) + end + end + + defp apply_refinement_to_contact(contact, ref, acc) do + case Radio.apply_contact_refinement(contact, ref.changes) do + {:ok, refined} -> + maybe_reenqueue_after_refinement(refined, ref.changes) + %{acc | refined: [refined | acc.refined]} + + {:error, %Ecto.Changeset{} = changeset} -> + %{acc | errors: [{ref.row_num, changeset_error_strings(changeset)} | acc.errors]} + end + end + + defp maybe_reenqueue_after_refinement(contact, changes) do + if Map.has_key?(changes, :grid1) or Map.has_key?(changes, :grid2) do + ContactWeatherEnqueueWorker.enqueue_for_contact(contact) + end + + :ok end # -- parsing --------------------------------------------------------------- @@ -305,48 +362,72 @@ defmodule Microwaveprop.Radio.CsvImport do from(c in Contact, where: c.band in ^bands and c.qso_timestamp >= ^min_ts and c.qso_timestamp <= ^max_ts and - c.flagged_invalid == false, - select: %{ - station1: c.station1, - station2: c.station2, - grid1: c.grid1, - grid2: c.grid2, - band: c.band, - qso_timestamp: c.qso_timestamp - } + c.flagged_invalid == false ) |> Repo.all() - |> Enum.reduce(%{}, fn row, acc -> - key = dedup_key(row.station1, row.grid1, row.station2, row.grid2, row.band) - Map.update(acc, key, [row.qso_timestamp], &[row.qso_timestamp | &1]) + |> Enum.reduce(%{}, fn contact, acc -> + case ImportMatcher.prefix_key(contact.station1, contact.grid1, contact.station2, contact.grid2, contact.band) do + :invalid -> + acc + + key -> + Map.update(acc, key, [contact], &[contact | &1]) + end end) end defp dedupe(rows, db_map) do - # in_batch_map accumulates keys from rows already accepted in this CSV so - # we can distinguish DB duplicates from earlier-row duplicates when we - # report them. - {accepted, _in_batch, duplicates} = - Enum.reduce(rows, {[], %{}, []}, fn row, {accepted, in_batch, dups} -> - key = dedup_key_from_attrs(row.attrs) + {accepted, _in_batch, duplicates, refinements} = + Enum.reduce(rows, {[], %{}, [], []}, &classify_row(&1, &2, db_map)) - cond do - conflict_in_map?(db_map, key, row.timestamp) -> - {accepted, in_batch, [Map.put(row, :reason, :existing_contact) | dups]} - - conflict_in_map?(in_batch, key, row.timestamp) -> - {accepted, in_batch, [Map.put(row, :reason, :earlier_in_upload) | dups]} - - true -> - new_batch = Map.update(in_batch, key, [row.timestamp], &[row.timestamp | &1]) - {[row | accepted], new_batch, dups} - end - end) - - {Enum.reverse(accepted), Enum.reverse(duplicates)} + {Enum.reverse(accepted), Enum.reverse(duplicates), Enum.reverse(refinements)} end - defp conflict_in_map?(map, key, timestamp) do + defp classify_row(row, {accepted, in_batch, dups, refs}, db_map) do + key = prefix_key_from_attrs(row.attrs) + + cond do + key == :invalid -> + {[row | accepted], in_batch, dups, refs} + + candidate = db_candidate(db_map, key, row.timestamp) -> + classify_db_candidate(row, candidate, accepted, in_batch, dups, refs) + + batch_conflict?(in_batch, key, row.timestamp) -> + {accepted, in_batch, [Map.put(row, :reason, :earlier_in_upload) | dups], refs} + + true -> + new_batch = Map.update(in_batch, key, [row.timestamp], &[row.timestamp | &1]) + {[row | accepted], new_batch, dups, refs} + end + end + + defp classify_db_candidate(row, candidate, accepted, in_batch, dups, refs) do + case ImportMatcher.classify_against_existing(row.attrs, candidate) do + {:refinement, changes} -> + refinement = %{ + row_num: row.row_num, + attrs: row.attrs, + timestamp: row.timestamp, + existing_id: candidate.id, + changes: changes + } + + {accepted, in_batch, dups, [refinement | refs]} + + _ -> + {accepted, in_batch, [Map.put(row, :reason, :existing_contact) | dups], refs} + end + end + + defp db_candidate(db_map, key, timestamp) do + case Map.fetch(db_map, key) do + :error -> nil + {:ok, contacts} -> Enum.find(contacts, &within_window?(&1.qso_timestamp, timestamp)) + end + end + + defp batch_conflict?(map, key, timestamp) do case Map.fetch(map, key) do :error -> false {:ok, timestamps} -> Enum.any?(timestamps, &within_window?(&1, timestamp)) @@ -357,28 +438,16 @@ defmodule Microwaveprop.Radio.CsvImport do abs(DateTime.diff(t1, t2, :second)) <= @dedup_window_seconds end - defp dedup_key_from_attrs(attrs) do - dedup_key(attrs["station1"], attrs["grid1"], attrs["station2"], attrs["grid2"], attrs["band"]) + defp prefix_key_from_attrs(attrs) do + ImportMatcher.prefix_key( + attrs["station1"], + attrs["grid1"], + attrs["station2"], + attrs["grid2"], + attrs["band"] + ) end - defp dedup_key(s1, g1, s2, g2, band) do - a = normalize_callsite(s1, g1) - b = normalize_callsite(s2, g2) - [c1, c2] = Enum.sort([a, b]) - {c1, c2, normalize_band(band)} - end - - defp normalize_callsite(call, grid) do - {upcase_trim(call), upcase_trim(grid)} - end - - defp upcase_trim(nil), do: "" - defp upcase_trim(value), do: value |> to_string() |> String.trim() |> String.upcase() - - defp normalize_band(%Decimal{} = b), do: Decimal.to_integer(b) - defp normalize_band(b) when is_integer(b), do: b - defp normalize_band(b) when is_binary(b), do: b |> String.trim() |> String.to_integer() - defp to_decimal(%Decimal{} = d), do: d defp to_decimal(v) when is_integer(v), do: Decimal.new(v) defp to_decimal(v) when is_binary(v), do: v |> String.trim() |> Decimal.new() diff --git a/lib/microwaveprop/radio/import_matcher.ex b/lib/microwaveprop/radio/import_matcher.ex new file mode 100644 index 00000000..6c5ae860 --- /dev/null +++ b/lib/microwaveprop/radio/import_matcher.ex @@ -0,0 +1,171 @@ +defmodule Microwaveprop.Radio.ImportMatcher do + @moduledoc """ + Pure classification for CSV/ADIF upload rows against existing Contact records. + + `prefix_key/5` builds a sortable key from `{station, 4-char grid prefix, band}` + so callers can bulk-lookup candidates from the database. `classify_against_existing/2` + compares a single upload row against a single candidate contact and decides + whether the upload is a plain duplicate, a grid/mode refinement of the existing + record, or an irreconcilable contradiction that must be skipped. + """ + + alias Microwaveprop.Radio.Contact + + @allowed_modes ~w(CW SSB FM FT8 FT4 Q65) + + @type prefix_key :: + {{String.t(), String.t()}, {String.t(), String.t()}, integer()} | :invalid + + @type change_map :: %{optional(:grid1) => String.t(), optional(:grid2) => String.t(), optional(:mode) => String.t()} + @type classification :: :duplicate | {:refinement, change_map()} | :contradiction + + @spec prefix_key(String.t() | nil, String.t() | nil, String.t() | nil, String.t() | nil, term()) :: + prefix_key() + def prefix_key(station1, grid1, station2, grid2, band) do + with {:ok, s1, g1_prefix} <- build_side(station1, grid1), + {:ok, s2, g2_prefix} <- build_side(station2, grid2), + {:ok, band_int} <- normalize_band(band) do + [a, b] = Enum.sort([{s1, g1_prefix}, {s2, g2_prefix}]) + {a, b, band_int} + else + _ -> :invalid + end + end + + defp build_side(station, grid) do + with station_up when is_binary(station_up) <- upcase_trim(station), + grid_up when is_binary(grid_up) <- upcase_trim(grid), + true <- byte_size(grid_up) >= 4 do + {:ok, station_up, binary_part(grid_up, 0, 4)} + else + _ -> :error + end + end + + defp upcase_trim(nil), do: nil + + defp upcase_trim(value) when is_binary(value) do + trimmed = String.trim(value) + if trimmed == "", do: nil, else: String.upcase(trimmed) + end + + defp upcase_trim(_), do: nil + + defp normalize_band(%Decimal{} = d) do + {:ok, Decimal.to_integer(d)} + rescue + _ -> :error + end + + defp normalize_band(value) when is_integer(value), do: {:ok, value} + + defp normalize_band(value) when is_binary(value) do + case value |> String.trim() |> Integer.parse() do + {int, ""} -> {:ok, int} + _ -> :error + end + end + + defp normalize_band(_), do: :error + + @spec classify_against_existing(map(), Contact.t()) :: classification() + def classify_against_existing(upload_attrs, %Contact{} = existing) do + up_s1 = upcase_trim(upload_attrs["station1"]) + up_s2 = upcase_trim(upload_attrs["station2"]) + ex_s1 = upcase_trim(existing.station1) + ex_s2 = upcase_trim(existing.station2) + + case resolve_direction(ex_s1, ex_s2, up_s1, up_s2) do + :ambiguous -> :duplicate + {up_grid1_key, up_grid2_key} -> compare_sides(existing, upload_attrs, up_grid1_key, up_grid2_key) + end + end + + # Returns which upload keys ("grid1"/"grid2") correspond to existing's side1/side2. + # If direction matches as-is: {"grid1", "grid2"}. If swapped: {"grid2", "grid1"}. + # `:ambiguous` when callsigns don't pair cleanly (e.g. same callsign both sides). + defp resolve_direction(ex_s1, ex_s2, up_s1, up_s2) do + if pairable_callsigns?(ex_s1, ex_s2, up_s1, up_s2) do + match_direction(ex_s1, ex_s2, up_s1, up_s2) + else + :ambiguous + end + end + + defp pairable_callsigns?(ex_s1, ex_s2, up_s1, up_s2) do + not is_nil(ex_s1) and not is_nil(ex_s2) and not is_nil(up_s1) and not is_nil(up_s2) and + ex_s1 != ex_s2 and up_s1 != up_s2 + end + + defp match_direction(ex_s1, ex_s2, up_s1, up_s2) do + cond do + ex_s1 == up_s1 and ex_s2 == up_s2 -> {"grid1", "grid2"} + ex_s1 == up_s2 and ex_s2 == up_s1 -> {"grid2", "grid1"} + true -> :ambiguous + end + end + + defp compare_sides(existing, upload_attrs, up_grid1_key, up_grid2_key) do + up_g1 = upcase_trim(upload_attrs[up_grid1_key]) + up_g2 = upcase_trim(upload_attrs[up_grid2_key]) + ex_g1 = upcase_trim(existing.grid1) + ex_g2 = upcase_trim(existing.grid2) + + grid1_result = compare_grid(ex_g1, up_g1) + grid2_result = compare_grid(ex_g2, up_g2) + mode_result = compare_mode(existing.mode, upload_attrs["mode"]) + + combine_results(grid1_result, grid2_result, mode_result) + end + + # Each compare_* returns :same | :contradiction | {:refine, new_value} + defp compare_grid(nil, _up), do: :same + defp compare_grid(_ex, nil), do: :same + defp compare_grid(ex, up) when ex == up, do: :same + + defp compare_grid(ex, up) do + cond do + byte_size(up) > byte_size(ex) and String.starts_with?(up, ex) -> {:refine, up} + byte_size(up) < byte_size(ex) and String.starts_with?(ex, up) -> :same + true -> :contradiction + end + end + + defp compare_mode(_ex, nil), do: :same + + defp compare_mode(nil, up) do + normalized = upcase_trim(up) + + cond do + is_nil(normalized) -> :same + normalized in @allowed_modes -> {:refine, normalized} + true -> :same + end + end + + defp compare_mode(ex, up) do + up_norm = upcase_trim(up) + + cond do + is_nil(up_norm) -> :same + String.upcase(ex) == up_norm -> :same + true -> :contradiction + end + end + + defp combine_results(grid1_result, grid2_result, mode_result) do + results = [{:grid1, grid1_result}, {:grid2, grid2_result}, {:mode, mode_result}] + + if Enum.any?(results, fn {_k, r} -> r == :contradiction end) do + :contradiction + else + changes = + Enum.reduce(results, %{}, fn + {key, {:refine, value}}, acc -> Map.put(acc, key, value) + _, acc -> acc + end) + + if changes == %{}, do: :duplicate, else: {:refinement, changes} + end + end +end diff --git a/lib/microwaveprop_web/live/submit_live.ex b/lib/microwaveprop_web/live/submit_live.ex index aad099c0..40e3f893 100644 --- a/lib/microwaveprop_web/live/submit_live.ex +++ b/lib/microwaveprop_web/live/submit_live.ex @@ -163,13 +163,15 @@ defmodule MicrowavepropWeb.SubmitLive do def handle_event("confirm_csv", _params, socket) do case socket.assigns.csv_preview do - %{valid: valid_rows} when valid_rows != [] -> - {:ok, commit_result} = CsvImport.commit(valid_rows) + %{valid: valid_rows, refinements: refinements} + when valid_rows != [] or refinements != [] -> + {:ok, commit_result} = + CsvImport.commit(%{valid: valid_rows, refinements: refinements}) {:noreply, socket |> assign(csv_preview: nil, csv_result: commit_result) - |> put_flash(:info, "#{length(commit_result.imported)} contacts submitted.")} + |> put_flash(:info, commit_flash(commit_result))} _ -> {:noreply, put_flash(socket, :error, "Nothing to import.")} @@ -619,7 +621,9 @@ defmodule MicrowavepropWeb.SubmitLive do valid_count: length(assigns.preview.valid), invalid_count: length(assigns.preview.invalid), duplicate_count: length(assigns.preview.duplicates), - valid_sample: Enum.take(assigns.preview.valid, 20) + refinement_count: length(assigns.preview.refinements || []), + valid_sample: Enum.take(assigns.preview.valid, 20), + refinement_sample: Enum.take(assigns.preview.refinements || [], 50) ) ~H""" @@ -632,13 +636,19 @@ defmodule MicrowavepropWeb.SubmitLive do

-
+
<.summary_card tone="success" label="Valid" value={@valid_count} hint="Will be inserted" /> + <.summary_card + tone="info" + label="Refinements" + value={@refinement_count} + hint="Will update existing contacts" + /> <.summary_card tone="warning" label="Duplicates" @@ -678,6 +688,44 @@ defmodule MicrowavepropWeb.SubmitLive do

+
0}> +

Refinements ({@refinement_count})

+

+ These rows match an existing contact but add more precise data. + Confirming will update the existing contact in place. +

+
+ + + + + + + + + + + + + + + + + + + +
RowStation 1Station 2BandChanges
Row {row.row_num}{row.attrs["station1"]} {row.attrs["grid1"]}{row.attrs["station2"]} {row.attrs["grid2"]}{row.attrs["band"]} +
+ {field}: + {value} +
+
+
+

length(@refinement_sample)} class="text-xs opacity-60 mt-1"> + Showing first {length(@refinement_sample)} of {@refinement_count} refinements. +

+
+
0}>

Duplicates ({@duplicate_count})

@@ -749,17 +797,15 @@ defmodule MicrowavepropWeb.SubmitLive do

+
0} + class="alert alert-info p-8 text-center" + > + <.icon name="hero-sparkles" class="w-16 h-16 text-info mx-auto mb-3" /> +
+ {@refined_count} existing {if @refined_count == 1, do: "contact", else: "contacts"} refined +
+

+ Grid or mode updates have been applied. + Enrichment will re-run for any grid changes. +

+
+
0} class="space-y-2">
<.icon name="hero-exclamation-triangle" class="w-5 h-5" /> diff --git a/test/microwaveprop/radio/adif_import_test.exs b/test/microwaveprop/radio/adif_import_test.exs index 2fdc06b3..f9092d87 100644 --- a/test/microwaveprop/radio/adif_import_test.exs +++ b/test/microwaveprop/radio/adif_import_test.exs @@ -39,6 +39,7 @@ defmodule Microwaveprop.Radio.AdifImportTest do assert length(preview.valid) == 1 assert preview.invalid == [] assert preview.duplicates == [] + assert preview.refinements == [] [row] = preview.valid assert row.attrs["station1"] == "W5XD" @@ -174,9 +175,34 @@ defmodule Microwaveprop.Radio.AdifImportTest do assert {:ok, preview} = AdifImport.preview(@valid_adif, @submitter) assert preview.valid == [] + assert preview.refinements == [] assert length(preview.duplicates) == 1 end + test "upload with longer grid than existing routes to refinements" do + {:ok, existing} = + Microwaveprop.Radio.create_contact(%{ + station1: "W5XD", + station2: "K5TR", + grid1: "EM12", + grid2: "EM00", + band: "10000", + mode: "CW", + qso_timestamp: ~U[2026-03-28 18:00:00Z], + submitter_email: @submitter + }) + + adif = + "W5XDK5TREM00EM12kp3710368.000CW20260328180500" + + assert {:ok, preview} = AdifImport.preview(adif, @submitter) + assert preview.valid == [] + assert preview.duplicates == [] + assert [ref] = preview.refinements + assert ref.existing_id == existing.id + assert ref.changes == %{grid1: "EM12KP37"} + end + test "is case-insensitive for field names" do adif = """ W5XDK5TREM00EM1210368.000CW20260328180000 diff --git a/test/microwaveprop/radio/csv_import_test.exs b/test/microwaveprop/radio/csv_import_test.exs index 9634ba48..106ce5bd 100644 --- a/test/microwaveprop/radio/csv_import_test.exs +++ b/test/microwaveprop/radio/csv_import_test.exs @@ -527,6 +527,7 @@ defmodule Microwaveprop.Radio.CsvImportTest do assert attrs["station1"] == "W5XD" assert preview.invalid == [] assert preview.duplicates == [] + assert preview.refinements == [] assert preview.total_rows == 1 assert preview.submitter_email == @submitter assert Repo.aggregate(Contact, :count) == 0 @@ -556,6 +557,7 @@ defmodule Microwaveprop.Radio.CsvImportTest do assert {:ok, second} = CsvImport.preview(csv, @submitter) assert second.valid == [] + assert second.refinements == [] assert [%{row_num: 2, reason: :existing_contact}] = second.duplicates end @@ -631,6 +633,107 @@ defmodule Microwaveprop.Radio.CsvImportTest do assert {:ok, preview} = CsvImport.preview(csv, @submitter) assert length(preview.valid) == 2 + assert preview.refinements == [] + end + + test "upload with longer grid than existing routes to refinements" do + csv = @valid_header <> "\n" <> @valid_row + {:ok, preview} = CsvImport.preview(csv, @submitter) + {:ok, %{imported: [existing]}} = CsvImport.commit(preview.valid) + + refined_csv = + @valid_header <> "\n" <> "W5XD,K5TR,EM12kp37,EM00,10000,CW,2026-03-28T18:05:00Z" + + assert {:ok, p2} = CsvImport.preview(refined_csv, @submitter) + assert p2.valid == [] + assert p2.duplicates == [] + assert [ref] = p2.refinements + assert ref.row_num == 2 + assert ref.existing_id == existing.id + assert ref.changes == %{grid1: "EM12KP37"} + end + + test "direction-swapped refinement maps changes to existing orientation" do + csv = @valid_header <> "\n" <> @valid_row + {:ok, preview} = CsvImport.preview(csv, @submitter) + {:ok, %{imported: [existing]}} = CsvImport.commit(preview.valid) + + # Upload as K5TR's side with more-precise grids + swapped_csv = + @valid_header <> "\n" <> "K5TR,W5XD,EM00bb,EM12aa,10000,CW,2026-03-28T18:05:00Z" + + assert {:ok, p2} = CsvImport.preview(swapped_csv, @submitter) + assert p2.valid == [] + assert [ref] = p2.refinements + assert ref.existing_id == existing.id + assert ref.changes == %{grid1: "EM12AA", grid2: "EM00BB"} + end + + test "upload with mode filling in missing mode routes to refinements" do + {:ok, existing} = + Microwaveprop.Radio.create_contact(%{ + station1: "W5XD", + station2: "K5TR", + grid1: "EM12", + grid2: "EM00", + band: "10000", + qso_timestamp: ~U[2026-03-28 18:00:00Z], + submitter_email: @submitter + }) + + refined_csv = @valid_header <> "\n" <> @valid_row + + assert {:ok, p2} = CsvImport.preview(refined_csv, @submitter) + assert p2.valid == [] + assert [ref] = p2.refinements + assert ref.existing_id == existing.id + assert ref.changes == %{mode: "CW"} + end + + test "contradictory grid (same 4-char prefix, different 6th char) is a duplicate not refinement" do + csv = + @valid_header <> "\n" <> "W5XD,K5TR,EM12kp,EM00,10000,CW,2026-03-28T18:00:00Z" + + {:ok, preview} = CsvImport.preview(csv, @submitter) + {:ok, %{imported: [_existing]}} = CsvImport.commit(preview.valid) + + contradictory_csv = + @valid_header <> "\n" <> "W5XD,K5TR,EM12qq,EM00,10000,CW,2026-03-28T18:05:00Z" + + assert {:ok, p2} = CsvImport.preview(contradictory_csv, @submitter) + assert p2.valid == [] + assert p2.refinements == [] + assert [%{reason: :existing_contact}] = p2.duplicates + end + + test "contradictory mode is a duplicate not refinement" do + csv = @valid_header <> "\n" <> @valid_row + {:ok, preview} = CsvImport.preview(csv, @submitter) + {:ok, %{imported: [_]}} = CsvImport.commit(preview.valid) + + mismatched_csv = + @valid_header <> "\n" <> "W5XD,K5TR,EM12,EM00,10000,SSB,2026-03-28T18:05:00Z" + + assert {:ok, p2} = CsvImport.preview(mismatched_csv, @submitter) + assert p2.valid == [] + assert p2.refinements == [] + assert [%{reason: :existing_contact}] = p2.duplicates + end + + test "rover with different 4-char prefix still classifies as valid" do + csv = + @valid_header <> "\n" <> "W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z" + + {:ok, preview} = CsvImport.preview(csv, @submitter) + {:ok, %{imported: [_]}} = CsvImport.commit(preview.valid) + + rover_csv = + @valid_header <> "\n" <> "W5XD,K5TR,EM13,EM00,10000,CW,2026-03-28T18:15:00Z" + + assert {:ok, p2} = CsvImport.preview(rover_csv, @submitter) + assert length(p2.valid) == 1 + assert p2.refinements == [] + assert p2.duplicates == [] end end @@ -656,5 +759,108 @@ defmodule Microwaveprop.Radio.CsvImportTest do test "commit of [] is a no-op" do assert {:ok, %{imported: [], errors: []}} = CsvImport.commit([]) end + + test "commit accepts map with :valid and :refinements and applies both" do + csv = @valid_header <> "\n" <> @valid_row + {:ok, p1} = CsvImport.preview(csv, @submitter) + {:ok, %{imported: [existing]}} = CsvImport.commit(p1.valid) + + refined_csv = + @valid_header <> "\n" <> "W5XD,K5TR,EM12kp37,EM00,10000,CW,2026-03-28T18:05:00Z" + + {:ok, p2} = CsvImport.preview(refined_csv, @submitter) + assert [_ref] = p2.refinements + + assert {:ok, %{imported: [], refined: [refined], errors: []}} = + CsvImport.commit(%{valid: p2.valid, refinements: p2.refinements}) + + assert refined.id == existing.id + assert refined.grid1 == "EM12KP37" + end + + test "commit with grid refinement re-enqueues enrichment" do + csv = @valid_header <> "\n" <> @valid_row + {:ok, p1} = CsvImport.preview(csv, @submitter) + {:ok, %{imported: [existing]}} = CsvImport.commit(p1.valid) + + # Drive enrichment statuses to :complete so a reset is observable. + Repo.update!( + Ecto.Changeset.change(existing, %{ + hrrr_status: :complete, + weather_status: :complete, + terrain_status: :complete, + iemre_status: :complete + }) + ) + + refined_csv = + @valid_header <> "\n" <> "W5XD,K5TR,EM12kp37,EM00,10000,CW,2026-03-28T18:05:00Z" + + {:ok, p2} = CsvImport.preview(refined_csv, @submitter) + + assert {:ok, %{refined: [refined]}} = + CsvImport.commit(%{valid: p2.valid, refinements: p2.refinements}) + + # After apply_contact_refinement + re-enqueue, HRRR enrichment ran + # again and (because the HRRR stub returns 404) lands at :unavailable. + # Weather/iemre/terrain may resolve back to :complete if no jobs could + # be built, so HRRR is the definitive "was it re-enqueued" signal. + reloaded = Repo.reload!(refined) + refute reloaded.hrrr_status == :complete + end + + test "commit with mode-only refinement does not re-enqueue enrichment" do + {:ok, existing} = + Microwaveprop.Radio.create_contact(%{ + station1: "W5XD", + station2: "K5TR", + grid1: "EM12", + grid2: "EM00", + band: "10000", + qso_timestamp: ~U[2026-03-28 18:00:00Z], + submitter_email: @submitter + }) + + Repo.update!( + Ecto.Changeset.change(existing, %{ + hrrr_status: :complete, + weather_status: :complete, + terrain_status: :complete, + iemre_status: :complete + }) + ) + + refined_csv = @valid_header <> "\n" <> @valid_row + {:ok, p} = CsvImport.preview(refined_csv, @submitter) + assert [%{changes: %{mode: "CW"}}] = p.refinements + + assert {:ok, %{refined: [refined]}} = + CsvImport.commit(%{valid: p.valid, refinements: p.refinements}) + + reloaded = Repo.reload!(refined) + assert reloaded.mode == "CW" + assert reloaded.hrrr_status == :complete + assert reloaded.weather_status == :complete + assert reloaded.terrain_status == :complete + assert reloaded.iemre_status == :complete + end + + test "commit(list) backward compat still works" do + csv = + Enum.join( + [ + @valid_header, + "W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z" + ], + "\n" + ) + + {:ok, preview} = CsvImport.preview(csv, @submitter) + assert {:ok, result} = CsvImport.commit(preview.valid) + assert [_contact] = result.imported + assert result.errors == [] + # The legacy list form still ships a :refined key with an empty list. + assert result.refined == [] + end end end diff --git a/test/microwaveprop/radio/import_matcher_test.exs b/test/microwaveprop/radio/import_matcher_test.exs new file mode 100644 index 00000000..70b6e748 --- /dev/null +++ b/test/microwaveprop/radio/import_matcher_test.exs @@ -0,0 +1,189 @@ +defmodule Microwaveprop.Radio.ImportMatcherTest do + use ExUnit.Case, async: true + + alias Microwaveprop.Radio.Contact + alias Microwaveprop.Radio.ImportMatcher + + defp build_contact(attrs \\ %{}) do + defaults = %{ + id: Ecto.UUID.generate(), + station1: "W5XD", + station2: "K5TR", + grid1: "EM12", + grid2: "EM00", + band: Decimal.new("10000"), + mode: "CW", + qso_timestamp: ~U[2026-03-28 18:00:00Z] + } + + struct(Contact, Map.merge(defaults, attrs)) + end + + defp upload(attrs) do + defaults = %{ + "station1" => "W5XD", + "station2" => "K5TR", + "grid1" => "EM12", + "grid2" => "EM00", + "band" => "10000", + "mode" => "CW", + "qso_timestamp" => "2026-03-28T18:00:00Z" + } + + Map.merge(defaults, attrs) + end + + describe "prefix_key/5" do + test "is direction-agnostic" do + a = ImportMatcher.prefix_key("W5XD", "EM12kp", "K5TR", "EM00cd", 10_000) + b = ImportMatcher.prefix_key("K5TR", "EM00cd", "W5XD", "EM12kp", 10_000) + assert a == b + end + + test "uppercases and trims callsigns and grids" do + a = ImportMatcher.prefix_key(" w5xd ", " em12kp ", " k5tr ", " em00 ", 10_000) + b = ImportMatcher.prefix_key("W5XD", "EM12KP", "K5TR", "EM00", 10_000) + assert a == b + end + + test "uses only first 4 characters of the grid" do + with_precision = ImportMatcher.prefix_key("W5XD", "EM12kp", "K5TR", "EM00cd", 10_000) + just_four = ImportMatcher.prefix_key("W5XD", "EM12", "K5TR", "EM00", 10_000) + assert with_precision == just_four + end + + test "different band produces different key" do + a = ImportMatcher.prefix_key("W5XD", "EM12", "K5TR", "EM00", 10_000) + b = ImportMatcher.prefix_key("W5XD", "EM12", "K5TR", "EM00", 24_000) + refute a == b + end + + test "invalid when any grid is under 4 chars" do + assert ImportMatcher.prefix_key("W5XD", "EM1", "K5TR", "EM00", 10_000) == :invalid + assert ImportMatcher.prefix_key("W5XD", "EM12", "K5TR", "EM", 10_000) == :invalid + assert ImportMatcher.prefix_key("W5XD", nil, "K5TR", "EM00", 10_000) == :invalid + assert ImportMatcher.prefix_key("W5XD", "EM12", "K5TR", nil, 10_000) == :invalid + end + + test "band accepted as integer, string, or Decimal" do + int_key = ImportMatcher.prefix_key("W5XD", "EM12", "K5TR", "EM00", 10_000) + str_key = ImportMatcher.prefix_key("W5XD", "EM12", "K5TR", "EM00", "10000") + dec_key = ImportMatcher.prefix_key("W5XD", "EM12", "K5TR", "EM00", Decimal.new("10000")) + assert int_key == str_key + assert int_key == dec_key + end + end + + describe "classify_against_existing/2" do + test "identical grids and mode classify as duplicate" do + existing = build_contact() + up = upload(%{}) + assert ImportMatcher.classify_against_existing(up, existing) == :duplicate + end + + test "upload extends grid1 -> refinement" do + existing = build_contact() + up = upload(%{"grid1" => "EM12kp37"}) + + assert ImportMatcher.classify_against_existing(up, existing) == + {:refinement, %{grid1: "EM12KP37"}} + end + + test "upload extends grid2 -> refinement" do + existing = build_contact() + up = upload(%{"grid2" => "EM00cd"}) + + assert ImportMatcher.classify_against_existing(up, existing) == + {:refinement, %{grid2: "EM00CD"}} + end + + test "upload extends both grids -> refinement with both changes" do + existing = build_contact() + up = upload(%{"grid1" => "EM12kp", "grid2" => "EM00cd"}) + + assert ImportMatcher.classify_against_existing(up, existing) == + {:refinement, %{grid1: "EM12KP", grid2: "EM00CD"}} + end + + test "direction-swapped upload maps changes into existing's orientation" do + existing = build_contact(%{station1: "W5XD", grid1: "EM12", station2: "K5TR", grid2: "EM00"}) + + up = + upload(%{ + "station1" => "K5TR", + "grid1" => "EM00bb", + "station2" => "W5XD", + "grid2" => "EM12aa" + }) + + assert ImportMatcher.classify_against_existing(up, existing) == + {:refinement, %{grid1: "EM12AA", grid2: "EM00BB"}} + end + + test "shorter-than-existing upload grid -> duplicate (not refinement)" do + existing = build_contact(%{grid1: "EM12kp"}) + up = upload(%{"grid1" => "EM12"}) + assert ImportMatcher.classify_against_existing(up, existing) == :duplicate + end + + test "same-length but different grid -> contradiction" do + existing = build_contact(%{grid1: "EM12"}) + up = upload(%{"grid1" => "EM13"}) + assert ImportMatcher.classify_against_existing(up, existing) == :contradiction + end + + test "existing mode nil + upload mode CW -> mode refinement" do + existing = build_contact(%{mode: nil}) + up = upload(%{"mode" => "CW"}) + + assert ImportMatcher.classify_against_existing(up, existing) == + {:refinement, %{mode: "CW"}} + end + + test "existing mode CW + upload mode SSB -> contradiction" do + existing = build_contact(%{mode: "CW"}) + up = upload(%{"mode" => "SSB"}) + assert ImportMatcher.classify_against_existing(up, existing) == :contradiction + end + + test "existing mode CW + upload mode nil -> duplicate (no change)" do + existing = build_contact(%{mode: "CW"}) + up = upload(%{"mode" => nil}) + assert ImportMatcher.classify_against_existing(up, existing) == :duplicate + end + + test "both modes nil -> duplicate" do + existing = build_contact(%{mode: nil}) + up = upload(%{"mode" => nil}) + assert ImportMatcher.classify_against_existing(up, existing) == :duplicate + end + + test "same callsign on both sides of upload -> duplicate (ambiguous)" do + existing = build_contact(%{station1: "W5XD", station2: "K5TR"}) + up = upload(%{"station1" => "W5XD", "station2" => "W5XD", "grid1" => "EM12kp"}) + assert ImportMatcher.classify_against_existing(up, existing) == :duplicate + end + + test "grid refinement + mode refinement combined returns both keys" do + existing = build_contact(%{grid1: "EM12", mode: nil}) + up = upload(%{"grid1" => "EM12kp", "mode" => "CW"}) + + assert ImportMatcher.classify_against_existing(up, existing) == + {:refinement, %{grid1: "EM12KP", mode: "CW"}} + end + + test "grid contradiction on one side trumps refinement on other" do + existing = build_contact(%{grid1: "EM12", grid2: "EM00"}) + up = upload(%{"grid1" => "EM13", "grid2" => "EM00cd"}) + assert ImportMatcher.classify_against_existing(up, existing) == :contradiction + end + + test "refinement values are uppercased and trimmed" do + existing = build_contact() + up = upload(%{"grid1" => " em12kp "}) + + assert ImportMatcher.classify_against_existing(up, existing) == + {:refinement, %{grid1: "EM12KP"}} + end + end +end diff --git a/test/microwaveprop/radio_refinement_test.exs b/test/microwaveprop/radio_refinement_test.exs new file mode 100644 index 00000000..91ac75f5 --- /dev/null +++ b/test/microwaveprop/radio_refinement_test.exs @@ -0,0 +1,152 @@ +defmodule Microwaveprop.RadioRefinementTest do + use Microwaveprop.DataCase, async: true + + alias Microwaveprop.Radio + alias Microwaveprop.Radio.Contact + alias Microwaveprop.Terrain.ElevationClient + alias Microwaveprop.Weather.HrrrClient + alias Microwaveprop.Weather.IemClient + + setup do + Req.Test.stub(IemClient, fn conn -> + case conn.request_path do + "/cgi-bin/request/asos.py" -> Req.Test.text(conn, "#DEBUG,\nstation,valid,tmpf\n") + _ -> Req.Test.json(conn, %{"profiles" => []}) + end + end) + + Req.Test.stub(HrrrClient, fn conn -> + Plug.Conn.send_resp(conn, 404, "not found") + end) + + Req.Test.stub(ElevationClient, fn conn -> + params = Plug.Conn.fetch_query_params(conn).query_params + lat_count = params["latitude"] |> String.split(",") |> length() + Req.Test.json(conn, %{"elevation" => List.duplicate(200.0, lat_count)}) + end) + + :ok + end + + defp insert_contact(attrs \\ %{}) do + base = %{ + station1: "W5XD", + station2: "K5TR", + grid1: "EM12", + grid2: "EM00", + band: "10000", + mode: "CW", + qso_timestamp: ~U[2026-03-28 18:00:00Z], + submitter_email: "test@example.com" + } + + {:ok, contact} = Radio.create_contact(Map.merge(base, attrs)) + contact + end + + defp mark_all_enrichment_complete(%Contact{} = contact) do + contact + |> Ecto.Changeset.change(%{ + hrrr_status: :complete, + weather_status: :complete, + terrain_status: :complete, + iemre_status: :complete + }) + |> Repo.update!() + end + + describe "apply_contact_refinement/2" do + test "refining grid1 updates grid1, pos1, distance, and resets all enrichment statuses" do + contact = mark_all_enrichment_complete(insert_contact()) + original_grid2 = contact.grid2 + original_pos2 = contact.pos2 + + assert {:ok, refined} = Radio.apply_contact_refinement(contact, %{grid1: "EM12KP"}) + + assert refined.grid1 == "EM12KP" + assert refined.grid2 == original_grid2 + assert refined.pos2 == original_pos2 + refute refined.pos1 == contact.pos1 + assert refined.pos1["lat"] != contact.pos1["lat"] or refined.pos1["lon"] != contact.pos1["lon"] + assert refined.distance_km + assert refined.hrrr_status == :pending + assert refined.weather_status == :pending + assert refined.terrain_status == :pending + assert refined.iemre_status == :pending + end + + test "refining grid2 updates grid2, pos2, distance, and resets all enrichment statuses" do + contact = mark_all_enrichment_complete(insert_contact()) + original_grid1 = contact.grid1 + original_pos1 = contact.pos1 + + assert {:ok, refined} = Radio.apply_contact_refinement(contact, %{grid2: "EM00CD"}) + + assert refined.grid1 == original_grid1 + assert refined.pos1 == original_pos1 + assert refined.grid2 == "EM00CD" + refute refined.pos2 == contact.pos2 + assert refined.hrrr_status == :pending + assert refined.weather_status == :pending + assert refined.terrain_status == :pending + assert refined.iemre_status == :pending + end + + test "refining both grids updates both + recomputes distance" do + contact = mark_all_enrichment_complete(insert_contact()) + original_distance = contact.distance_km + + assert {:ok, refined} = + Radio.apply_contact_refinement(contact, %{grid1: "EM12KP", grid2: "EM00CD"}) + + assert refined.grid1 == "EM12KP" + assert refined.grid2 == "EM00CD" + assert refined.distance_km + # Distances remain similar (grids refined within same squares) but pos values change. + assert refined.pos1["lat"] != contact.pos1["lat"] or refined.pos1["lon"] != contact.pos1["lon"] + assert refined.pos2["lat"] != contact.pos2["lat"] or refined.pos2["lon"] != contact.pos2["lon"] + assert refined.distance_km != original_distance or refined.distance_km == original_distance + end + + test "refining only :mode updates mode without touching pos/distance or statuses" do + contact = %{mode: nil} |> insert_contact() |> mark_all_enrichment_complete() + original_pos1 = contact.pos1 + original_pos2 = contact.pos2 + original_distance = contact.distance_km + + assert {:ok, refined} = Radio.apply_contact_refinement(contact, %{mode: "CW"}) + + assert refined.mode == "CW" + assert refined.pos1 == original_pos1 + assert refined.pos2 == original_pos2 + assert refined.distance_km == original_distance + assert refined.hrrr_status == :complete + assert refined.weather_status == :complete + assert refined.terrain_status == :complete + assert refined.iemre_status == :complete + end + + test "empty changes map returns the contact unchanged" do + contact = mark_all_enrichment_complete(insert_contact()) + assert {:ok, unchanged} = Radio.apply_contact_refinement(contact, %{}) + assert unchanged.id == contact.id + assert unchanged.grid1 == contact.grid1 + assert unchanged.grid2 == contact.grid2 + assert unchanged.mode == contact.mode + assert unchanged.hrrr_status == :complete + end + + test "invalid Maidenhead grid returns error and does not mutate the DB row" do + contact = mark_all_enrichment_complete(insert_contact()) + + assert {:error, %Ecto.Changeset{} = changeset} = + Radio.apply_contact_refinement(contact, %{grid1: "ZZZZ"}) + + refute changeset.valid? + + reloaded = Repo.reload!(contact) + assert reloaded.grid1 == contact.grid1 + assert reloaded.hrrr_status == :complete + end + end +end diff --git a/test/microwaveprop_web/live/submit_live_test.exs b/test/microwaveprop_web/live/submit_live_test.exs index 87523ddd..c648c269 100644 --- a/test/microwaveprop_web/live/submit_live_test.exs +++ b/test/microwaveprop_web/live/submit_live_test.exs @@ -147,6 +147,58 @@ defmodule MicrowavepropWeb.SubmitLiveTest do refute html =~ "Looks good" end + test "preview shows refinements when upload extends an existing grid", %{conn: conn} do + {:ok, existing} = + Microwaveprop.Radio.create_contact(%{ + "station1" => "W5XD", + "station2" => "K5TR", + "grid1" => "EM12", + "grid2" => "EM00", + "band" => "10000", + "mode" => "CW", + "qso_timestamp" => "2026-03-28T18:00:00Z", + "submitter_email" => "seed@example.com" + }) + + {:ok, lv, _html} = live(conn, ~p"/submit") + + lv + |> element("[phx-value-tab=csv]") + |> render_click() + + csv_content = + "station1,station2,grid1,grid2,band,mode,qso_timestamp\n" <> + "W5XD,K5TR,EM12KP37,EM00CD22,10000,CW,2026-03-28T18:10:00Z\n" + + csv_input = + file_input(lv, "#csv-upload-form", :csv, [ + %{name: "contacts.csv", content: csv_content, type: "text/csv"} + ]) + + render_upload(csv_input, "contacts.csv") + + html = + lv + |> form("#csv-upload-form", %{submitter_email: "test@example.com"}) + |> render_submit() + + assert html =~ "Refinements" + assert html =~ "EM12KP37" + assert html =~ "refine 1 existing" + + confirm_html = + lv + |> element("button[phx-click=confirm_csv]") + |> render_click() + + assert confirm_html =~ "1 existing contact refined" + assert Repo.aggregate(Contact, :count) == 1 + + refreshed = Repo.reload(existing) + assert refreshed.grid1 == "EM12KP37" + assert refreshed.grid2 == "EM00CD22" + end + test "shows errors for invalid rows and still offers the confirm button", %{conn: conn} do {:ok, lv, _html} = live(conn, ~p"/submit")