defmodule Mix.Tasks.ImportContestLogs do @shortdoc "Import ARRL contest CSV files with deduplication" @moduledoc """ Imports one or more contest CSV files, deduplicating both across files and against existing contacts in the database. Uses the same direction-agnostic, 60-minute-window dedup logic as the web CSV uploader. mix import_contest_logs ~/Downloads/arrl_10ghz_2024.csv ~/Downloads/arrl_10ghz_2023.csv ... mix import_contest_logs ~/Downloads/arrl_10ghz_*.csv """ use Mix.Task alias Microwaveprop.Radio.CsvImport @impl Mix.Task def run(argv) do if argv == [] do Mix.raise("Usage: mix import_contest_logs FILE [FILE ...]") end Mix.Task.run("app.start") _ = Oban.pause_all_queues(Oban) # Concatenate all CSVs into one string (skip headers after the first file) combined = argv |> Enum.with_index() |> Enum.map(fn {path, idx} -> expanded = Path.expand(path) content = File.read!(expanded) lines = String.split(content, ~r/\r?\n/) # Skip header for all files after the first data_lines = if idx == 0, do: lines, else: tl(lines) # Drop blank trailing lines Enum.reject(data_lines, &(String.trim(&1) == "")) end) |> List.flatten() |> Enum.join("\n") total_lines = combined |> String.split("\n") |> length() |> Kernel.-(1) Mix.shell().info("Combined #{length(argv)} files: #{total_lines} data rows") Mix.shell().info("Running preview (validate + deduplicate)...") case CsvImport.preview(combined, "contest-import@ntms.org") do {:ok, preview} -> print_preview_summary(preview) print_invalid_rows(preview.invalid) if preview.valid == [] do Mix.shell().info("Nothing to import.") else import_batches(preview.valid) end {:error, reason} -> Mix.raise("Failed: #{inspect(reason)}") end end defp print_preview_summary(preview) do Mix.shell().info(""" Preview: Valid: #{length(preview.valid)} Duplicates: #{length(preview.duplicates)} Invalid: #{length(preview.invalid)} """) end defp print_invalid_rows([]), do: :ok defp print_invalid_rows(invalid) do invalid |> Enum.take(10) |> Enum.each(fn row -> Mix.shell().info(" Row #{row.row_num}: #{Enum.join(row.messages, ", ")}") end) if length(invalid) > 10 do Mix.shell().info(" ... and #{length(invalid) - 10} more") end end defp import_batches(valid) do Mix.shell().info("Importing #{length(valid)} contacts...") {imported, errors} = valid |> Enum.chunk_every(500) |> Enum.with_index(1) |> Enum.reduce({0, 0}, fn {batch, batch_num}, {imported, errors} -> {:ok, result} = CsvImport.commit(batch) new_imported = imported + length(result.imported) new_errors = errors + length(result.errors) Mix.shell().info( " Batch #{batch_num}: +#{length(result.imported)} imported, #{length(result.errors)} errors (total: #{new_imported})" ) {new_imported, new_errors} end) Mix.shell().info("Done! Imported #{imported} contacts, #{errors} errors.") end end