Graham McIntire
6d92973853
- Add `mix backtest --all` for consolidated pass/fail table across all features - Add Backtest.consolidated_report/2 and to_consolidated_markdown/1 - Add Features.all_features/0 to auto-discover backtestable features - Add `mix import_contest_logs` for bulk ARRL contest CSV import with dedup - Fix hrrr_climatology to batch by (month, hour) to avoid query timeout - Fix Repo.query! result pattern (Postgrex.Result, not tuple) - Backtest reports for all Phase 1-6 features
99 lines
3.2 KiB
Elixir
99 lines
3.2 KiB
Elixir
defmodule Mix.Tasks.ImportContestLogs do
|
|
@shortdoc "Import ARRL contest CSV files with deduplication"
|
|
@moduledoc """
|
|
Imports one or more contest CSV files, deduplicating both across files
|
|
and against existing contacts in the database.
|
|
|
|
Uses the same direction-agnostic, 60-minute-window dedup logic as the
|
|
web CSV uploader.
|
|
|
|
mix import_contest_logs ~/Downloads/arrl_10ghz_2024.csv ~/Downloads/arrl_10ghz_2023.csv ...
|
|
mix import_contest_logs ~/Downloads/arrl_10ghz_*.csv
|
|
"""
|
|
use Mix.Task
|
|
|
|
alias Microwaveprop.Radio.CsvImport
|
|
|
|
@impl Mix.Task
|
|
def run(argv) do
|
|
if argv == [] do
|
|
Mix.raise("Usage: mix import_contest_logs FILE [FILE ...]")
|
|
end
|
|
|
|
Mix.Task.run("app.start")
|
|
Oban.pause_all_queues(Oban)
|
|
|
|
# Concatenate all CSVs into one string (skip headers after the first file)
|
|
combined =
|
|
argv
|
|
|> Enum.with_index()
|
|
|> Enum.map(fn {path, idx} ->
|
|
expanded = Path.expand(path)
|
|
content = File.read!(expanded)
|
|
lines = String.split(content, ~r/\r?\n/)
|
|
|
|
# Skip header for all files after the first
|
|
data_lines = if idx == 0, do: lines, else: tl(lines)
|
|
|
|
# Drop blank trailing lines
|
|
Enum.reject(data_lines, &(String.trim(&1) == ""))
|
|
end)
|
|
|> List.flatten()
|
|
|> Enum.join("\n")
|
|
|
|
total_lines = combined |> String.split("\n") |> length() |> then(&(&1 - 1))
|
|
Mix.shell().info("Combined #{length(argv)} files: #{total_lines} data rows")
|
|
|
|
Mix.shell().info("Running preview (validate + deduplicate)...")
|
|
|
|
case CsvImport.preview(combined, "contest-import@ntms.org") do
|
|
{:ok, preview} ->
|
|
Mix.shell().info("""
|
|
Preview:
|
|
Valid: #{length(preview.valid)}
|
|
Duplicates: #{length(preview.duplicates)}
|
|
Invalid: #{length(preview.invalid)}
|
|
""")
|
|
|
|
if preview.invalid != [] do
|
|
preview.invalid
|
|
|> Enum.take(10)
|
|
|> Enum.each(fn row ->
|
|
Mix.shell().info(" Row #{row.row_num}: #{Enum.join(row.messages, ", ")}")
|
|
end)
|
|
|
|
if length(preview.invalid) > 10 do
|
|
Mix.shell().info(" ... and #{length(preview.invalid) - 10} more")
|
|
end
|
|
end
|
|
|
|
if preview.valid == [] do
|
|
Mix.shell().info("Nothing to import.")
|
|
else
|
|
Mix.shell().info("Importing #{length(preview.valid)} contacts...")
|
|
|
|
# Import in batches to show progress
|
|
preview.valid
|
|
|> Enum.chunk_every(500)
|
|
|> Enum.with_index(1)
|
|
|> Enum.reduce({0, 0}, fn {batch, batch_num}, {imported, errors} ->
|
|
{:ok, result} = CsvImport.commit(batch)
|
|
new_imported = imported + length(result.imported)
|
|
new_errors = errors + length(result.errors)
|
|
|
|
Mix.shell().info(
|
|
" Batch #{batch_num}: +#{length(result.imported)} imported, #{length(result.errors)} errors (total: #{new_imported})"
|
|
)
|
|
|
|
{new_imported, new_errors}
|
|
end)
|
|
|> then(fn {imported, errors} ->
|
|
Mix.shell().info("Done! Imported #{imported} contacts, #{errors} errors.")
|
|
end)
|
|
end
|
|
|
|
{:error, reason} ->
|
|
Mix.raise("Failed: #{inspect(reason)}")
|
|
end
|
|
end
|
|
end
|