Graham McIntire
079346a1b9
Some checks failed
Build and Push / Build and Push Docker Image (push) Failing after 4m38s
- F-level (228): replace length/1 with Enum.count_until/2 or pattern matching; convert Enum.flat_map+if to Enum.filter+Enum.map; fix identity case in narr_client - W-level (46): normalize dual atom/string key access in weather_layers, beacon_measurements, surface, skewt_live, contact_live/show; add :data_provider and weather-map assigns to ignored_assigns in credo config (consumed by child components credo can't trace); remove weak is_list assertion; remove explicit assert_receive timeout - R-level (6): replace 'This module provides...' moduledocs with meaningful descriptions - Also fix 4 compile-connected xref issues by deferring BandConfig.band_options() from module attribute to runtime Co-Authored-By: Claude <noreply@anthropic.com>
109 lines
3.2 KiB
Elixir
109 lines
3.2 KiB
Elixir
defmodule Mix.Tasks.ImportContestLogs do
|
|
@shortdoc "Import ARRL contest CSV files with deduplication"
|
|
@moduledoc """
|
|
Imports one or more contest CSV files, deduplicating both across files
|
|
and against existing contacts in the database.
|
|
|
|
Uses the same direction-agnostic, 60-minute-window dedup logic as the
|
|
web CSV uploader.
|
|
|
|
mix import_contest_logs ~/Downloads/arrl_10ghz_2024.csv ~/Downloads/arrl_10ghz_2023.csv ...
|
|
mix import_contest_logs ~/Downloads/arrl_10ghz_*.csv
|
|
"""
|
|
use Mix.Task
|
|
|
|
alias Microwaveprop.Radio.CsvImport
|
|
|
|
@impl Mix.Task
|
|
def run(argv) do
|
|
if argv == [] do
|
|
Mix.raise("Usage: mix import_contest_logs FILE [FILE ...]")
|
|
end
|
|
|
|
Mix.Task.run("app.start")
|
|
_ = Oban.pause_all_queues(Oban)
|
|
|
|
# Concatenate all CSVs into one string (skip headers after the first file)
|
|
combined =
|
|
argv
|
|
|> Enum.with_index()
|
|
|> Enum.map(fn {path, idx} ->
|
|
expanded = Path.expand(path)
|
|
content = File.read!(expanded)
|
|
lines = String.split(content, ~r/\r?\n/)
|
|
|
|
# Skip header for all files after the first
|
|
data_lines = if idx == 0, do: lines, else: tl(lines)
|
|
|
|
# Drop blank trailing lines
|
|
Enum.reject(data_lines, &(String.trim(&1) == ""))
|
|
end)
|
|
|> List.flatten()
|
|
|> Enum.join("\n")
|
|
|
|
total_lines = combined |> String.split("\n") |> length() |> Kernel.-(1)
|
|
Mix.shell().info("Combined #{length(argv)} files: #{total_lines} data rows")
|
|
|
|
Mix.shell().info("Running preview (validate + deduplicate)...")
|
|
|
|
case CsvImport.preview(combined, "contest-import@ntms.org") do
|
|
{:ok, preview} ->
|
|
print_preview_summary(preview)
|
|
print_invalid_rows(preview.invalid)
|
|
|
|
if preview.valid == [] do
|
|
Mix.shell().info("Nothing to import.")
|
|
else
|
|
import_batches(preview.valid)
|
|
end
|
|
|
|
{:error, reason} ->
|
|
Mix.raise("Failed: #{inspect(reason)}")
|
|
end
|
|
end
|
|
|
|
defp print_preview_summary(preview) do
|
|
Mix.shell().info("""
|
|
Preview:
|
|
Valid: #{length(preview.valid)}
|
|
Duplicates: #{length(preview.duplicates)}
|
|
Invalid: #{length(preview.invalid)}
|
|
""")
|
|
end
|
|
|
|
defp print_invalid_rows([]), do: :ok
|
|
|
|
defp print_invalid_rows(invalid) do
|
|
invalid
|
|
|> Enum.take(10)
|
|
|> Enum.each(fn row ->
|
|
Mix.shell().info(" Row #{row.row_num}: #{Enum.join(row.messages, ", ")}")
|
|
end)
|
|
|
|
if Enum.count_until(invalid, 11) == 11 do
|
|
Mix.shell().info(" ... and #{length(invalid) - 10} more")
|
|
end
|
|
end
|
|
|
|
defp import_batches(valid) do
|
|
Mix.shell().info("Importing #{length(valid)} contacts...")
|
|
|
|
{imported, errors} =
|
|
valid
|
|
|> Enum.chunk_every(500)
|
|
|> Enum.with_index(1)
|
|
|> Enum.reduce({0, 0}, fn {batch, batch_num}, {imported, errors} ->
|
|
{:ok, result} = CsvImport.commit(batch)
|
|
new_imported = imported + length(result.imported)
|
|
new_errors = errors + length(result.errors)
|
|
|
|
Mix.shell().info(
|
|
" Batch #{batch_num}: +#{length(result.imported)} imported, #{length(result.errors)} errors (total: #{new_imported})"
|
|
)
|
|
|
|
{new_imported, new_errors}
|
|
end)
|
|
|
|
Mix.shell().info("Done! Imported #{imported} contacts, #{errors} errors.")
|
|
end
|
|
end
|