Graham McIntire
7fb340bc35
Aliases: add module aliases for 9 nested module references Apply: replace apply/3 with direct module attribute calls Line length: break 1 long spec line Refactoring: extract helpers to reduce complexity and nesting in show.ex, radio.ex, weather workers, terrain, duct detection, backfill dashboard, contact map, and mix tasks
109 lines
3.2 KiB
Elixir
109 lines
3.2 KiB
Elixir
defmodule Mix.Tasks.ImportContestLogs do
|
|
@shortdoc "Import ARRL contest CSV files with deduplication"
|
|
@moduledoc """
|
|
Imports one or more contest CSV files, deduplicating both across files
|
|
and against existing contacts in the database.
|
|
|
|
Uses the same direction-agnostic, 60-minute-window dedup logic as the
|
|
web CSV uploader.
|
|
|
|
mix import_contest_logs ~/Downloads/arrl_10ghz_2024.csv ~/Downloads/arrl_10ghz_2023.csv ...
|
|
mix import_contest_logs ~/Downloads/arrl_10ghz_*.csv
|
|
"""
|
|
use Mix.Task
|
|
|
|
alias Microwaveprop.Radio.CsvImport
|
|
|
|
@impl Mix.Task
|
|
def run(argv) do
|
|
if argv == [] do
|
|
Mix.raise("Usage: mix import_contest_logs FILE [FILE ...]")
|
|
end
|
|
|
|
Mix.Task.run("app.start")
|
|
Oban.pause_all_queues(Oban)
|
|
|
|
# Concatenate all CSVs into one string (skip headers after the first file)
|
|
combined =
|
|
argv
|
|
|> Enum.with_index()
|
|
|> Enum.map(fn {path, idx} ->
|
|
expanded = Path.expand(path)
|
|
content = File.read!(expanded)
|
|
lines = String.split(content, ~r/\r?\n/)
|
|
|
|
# Skip header for all files after the first
|
|
data_lines = if idx == 0, do: lines, else: tl(lines)
|
|
|
|
# Drop blank trailing lines
|
|
Enum.reject(data_lines, &(String.trim(&1) == ""))
|
|
end)
|
|
|> List.flatten()
|
|
|> Enum.join("\n")
|
|
|
|
total_lines = combined |> String.split("\n") |> length() |> Kernel.-(1)
|
|
Mix.shell().info("Combined #{length(argv)} files: #{total_lines} data rows")
|
|
|
|
Mix.shell().info("Running preview (validate + deduplicate)...")
|
|
|
|
case CsvImport.preview(combined, "contest-import@ntms.org") do
|
|
{:ok, preview} ->
|
|
print_preview_summary(preview)
|
|
print_invalid_rows(preview.invalid)
|
|
|
|
if preview.valid == [] do
|
|
Mix.shell().info("Nothing to import.")
|
|
else
|
|
import_batches(preview.valid)
|
|
end
|
|
|
|
{:error, reason} ->
|
|
Mix.raise("Failed: #{inspect(reason)}")
|
|
end
|
|
end
|
|
|
|
defp print_preview_summary(preview) do
|
|
Mix.shell().info("""
|
|
Preview:
|
|
Valid: #{length(preview.valid)}
|
|
Duplicates: #{length(preview.duplicates)}
|
|
Invalid: #{length(preview.invalid)}
|
|
""")
|
|
end
|
|
|
|
defp print_invalid_rows([]), do: :ok
|
|
|
|
defp print_invalid_rows(invalid) do
|
|
invalid
|
|
|> Enum.take(10)
|
|
|> Enum.each(fn row ->
|
|
Mix.shell().info(" Row #{row.row_num}: #{Enum.join(row.messages, ", ")}")
|
|
end)
|
|
|
|
if length(invalid) > 10 do
|
|
Mix.shell().info(" ... and #{length(invalid) - 10} more")
|
|
end
|
|
end
|
|
|
|
defp import_batches(valid) do
|
|
Mix.shell().info("Importing #{length(valid)} contacts...")
|
|
|
|
{imported, errors} =
|
|
valid
|
|
|> Enum.chunk_every(500)
|
|
|> Enum.with_index(1)
|
|
|> Enum.reduce({0, 0}, fn {batch, batch_num}, {imported, errors} ->
|
|
{:ok, result} = CsvImport.commit(batch)
|
|
new_imported = imported + length(result.imported)
|
|
new_errors = errors + length(result.errors)
|
|
|
|
Mix.shell().info(
|
|
" Batch #{batch_num}: +#{length(result.imported)} imported, #{length(result.errors)} errors (total: #{new_imported})"
|
|
)
|
|
|
|
{new_imported, new_errors}
|
|
end)
|
|
|
|
Mix.shell().info("Done! Imported #{imported} contacts, #{errors} errors.")
|
|
end
|
|
end
|