Graham McIntire
e7a7ae073d
Task 9.3 - Weight recalibration via gradient descent:
- Recalibrator module fits logistic regression weights using Nx
- Trains on QSO positives vs random baseline negatives
- Cross-validates by month, normalizes weights to sum to 1.0
- Mix task: mix recalibrate_scorer --sample 5000 --epochs 2000
Task 9.4 - Side-by-side scorer comparison:
- ScorerDiff.compare/3 re-scores grid with old vs new weights
- Reports mean diff, regressions, improvements, per-band breakdown
- Mix task: mix scorer_diff --new-weights '{...}'
Phase 3 - NEXRAD ingestion pipeline:
- NexradClient fetches IEM n0q composite PNGs, extracts per-point
box statistics (mean/max dBZ, texture variance)
- NexradObservation schema with unique (lat, lon, observed_at)
- NexradWorker on :nexrad queue for background processing
- nexrad_texture backtest feature in Features module
- mix nexrad_backfill --limit 200
All tasks added to AdminTaskWorker and Release for production use.
1116 tests, 0 failures.
99 lines
3.2 KiB
Elixir
99 lines
3.2 KiB
Elixir
defmodule Mix.Tasks.ImportContestLogs do
|
|
@shortdoc "Import ARRL contest CSV files with deduplication"
|
|
@moduledoc """
|
|
Imports one or more contest CSV files, deduplicating both across files
|
|
and against existing contacts in the database.
|
|
|
|
Uses the same direction-agnostic, 60-minute-window dedup logic as the
|
|
web CSV uploader.
|
|
|
|
mix import_contest_logs ~/Downloads/arrl_10ghz_2024.csv ~/Downloads/arrl_10ghz_2023.csv ...
|
|
mix import_contest_logs ~/Downloads/arrl_10ghz_*.csv
|
|
"""
|
|
use Mix.Task
|
|
|
|
alias Microwaveprop.Radio.CsvImport
|
|
|
|
@impl Mix.Task
|
|
def run(argv) do
|
|
if argv == [] do
|
|
Mix.raise("Usage: mix import_contest_logs FILE [FILE ...]")
|
|
end
|
|
|
|
Mix.Task.run("app.start")
|
|
Oban.pause_all_queues(Oban)
|
|
|
|
# Concatenate all CSVs into one string (skip headers after the first file)
|
|
combined =
|
|
argv
|
|
|> Enum.with_index()
|
|
|> Enum.map(fn {path, idx} ->
|
|
expanded = Path.expand(path)
|
|
content = File.read!(expanded)
|
|
lines = String.split(content, ~r/\r?\n/)
|
|
|
|
# Skip header for all files after the first
|
|
data_lines = if idx == 0, do: lines, else: tl(lines)
|
|
|
|
# Drop blank trailing lines
|
|
Enum.reject(data_lines, &(String.trim(&1) == ""))
|
|
end)
|
|
|> List.flatten()
|
|
|> Enum.join("\n")
|
|
|
|
total_lines = combined |> String.split("\n") |> length() |> Kernel.-(1)
|
|
Mix.shell().info("Combined #{length(argv)} files: #{total_lines} data rows")
|
|
|
|
Mix.shell().info("Running preview (validate + deduplicate)...")
|
|
|
|
case CsvImport.preview(combined, "contest-import@ntms.org") do
|
|
{:ok, preview} ->
|
|
Mix.shell().info("""
|
|
Preview:
|
|
Valid: #{length(preview.valid)}
|
|
Duplicates: #{length(preview.duplicates)}
|
|
Invalid: #{length(preview.invalid)}
|
|
""")
|
|
|
|
if preview.invalid != [] do
|
|
preview.invalid
|
|
|> Enum.take(10)
|
|
|> Enum.each(fn row ->
|
|
Mix.shell().info(" Row #{row.row_num}: #{Enum.join(row.messages, ", ")}")
|
|
end)
|
|
|
|
if length(preview.invalid) > 10 do
|
|
Mix.shell().info(" ... and #{length(preview.invalid) - 10} more")
|
|
end
|
|
end
|
|
|
|
if preview.valid == [] do
|
|
Mix.shell().info("Nothing to import.")
|
|
else
|
|
Mix.shell().info("Importing #{length(preview.valid)} contacts...")
|
|
|
|
# Import in batches to show progress
|
|
preview.valid
|
|
|> Enum.chunk_every(500)
|
|
|> Enum.with_index(1)
|
|
|> Enum.reduce({0, 0}, fn {batch, batch_num}, {imported, errors} ->
|
|
{:ok, result} = CsvImport.commit(batch)
|
|
new_imported = imported + length(result.imported)
|
|
new_errors = errors + length(result.errors)
|
|
|
|
Mix.shell().info(
|
|
" Batch #{batch_num}: +#{length(result.imported)} imported, #{length(result.errors)} errors (total: #{new_imported})"
|
|
)
|
|
|
|
{new_imported, new_errors}
|
|
end)
|
|
|> then(fn {imported, errors} ->
|
|
Mix.shell().info("Done! Imported #{imported} contacts, #{errors} errors.")
|
|
end)
|
|
end
|
|
|
|
{:error, reason} ->
|
|
Mix.raise("Failed: #{inspect(reason)}")
|
|
end
|
|
end
|
|
end
|