Graham McIntire
d61fbd346e
Enabled :error_handling, :unknown, :unmatched_returns, :extra_return, :missing_return in an earlier commit and landed a 129-warning baseline. Four parallel agents each fixed a directory slice: - Core contexts (29): Radio, Release, Weather, Beacons, Cache, Backtest.Features, Terrain.Srtm, Ionosphere.GiroClient, Propagation.RunTiming, Accounts.Scope, RepoListener. Fixes were (a) prefix side-effect calls (Task.start, Phoenix.PubSub, Logger, :ets.new) with _ = ; (b) tighten/widen specs that didn't match actual returns; (c) add missing @type t declarations; (d) drop dead parse_int(nil) clause. - Propagation + weather subdirs (15): FreshnessMonitor, NotifyListener, ScoreCache, ScoreCacheReconciler, Weather.FrontalAnalysis, Weather.Grib2.Extractor, Weather.Grib2.Wgrib2, GridCache, HrrrPointEnqueuer, NexradCache. Same patterns — mostly _ = on PubSub / :ets / Repo.insert_all; widened two specs (float -> number) where integer returns were reachable. - Workers (35): BackfillEnqueue, CanadianSoundingFetch, ContactImport, ContactWeatherEnqueue, GefsFetch, IemreFetch, NarrFetch, SolarIndex, TerrainProfile, WeatherFetch. Prefixed Repo.update_all / Radio.set_enrichment_status! / Weather.upsert_* side-effect calls. Fixed one :pattern_match in CanadianSoundingFetch.most_recent_sounding_time/1 where a tautological cond guard generated unreachable code. - Web + Mix tasks + lib_ml (46 of 50): controllers, LiveViews, UserAuth, and 11 mix tasks. Same prefix strategy. 4 remaining warnings originate in LiveTable.LiveResource dep macro expansion and can't be fixed without forking the dep — added .dialyzer_ignore.exs to suppress just those specific file:line pairs. Also wired ignore_warnings in mix.exs dialyzer config. mix dialyzer --format short | grep ^lib/ | wc -l -> 0 mix test: 2163 tests, 3 pre-existing flakes, 0 regressions.
109 lines
3.2 KiB
Elixir
109 lines
3.2 KiB
Elixir
defmodule Mix.Tasks.ImportContestLogs do
|
|
@shortdoc "Import ARRL contest CSV files with deduplication"
|
|
@moduledoc """
|
|
Imports one or more contest CSV files, deduplicating both across files
|
|
and against existing contacts in the database.
|
|
|
|
Uses the same direction-agnostic, 60-minute-window dedup logic as the
|
|
web CSV uploader.
|
|
|
|
mix import_contest_logs ~/Downloads/arrl_10ghz_2024.csv ~/Downloads/arrl_10ghz_2023.csv ...
|
|
mix import_contest_logs ~/Downloads/arrl_10ghz_*.csv
|
|
"""
|
|
use Mix.Task
|
|
|
|
alias Microwaveprop.Radio.CsvImport
|
|
|
|
@impl Mix.Task
|
|
def run(argv) do
|
|
if argv == [] do
|
|
Mix.raise("Usage: mix import_contest_logs FILE [FILE ...]")
|
|
end
|
|
|
|
Mix.Task.run("app.start")
|
|
_ = Oban.pause_all_queues(Oban)
|
|
|
|
# Concatenate all CSVs into one string (skip headers after the first file)
|
|
combined =
|
|
argv
|
|
|> Enum.with_index()
|
|
|> Enum.map(fn {path, idx} ->
|
|
expanded = Path.expand(path)
|
|
content = File.read!(expanded)
|
|
lines = String.split(content, ~r/\r?\n/)
|
|
|
|
# Skip header for all files after the first
|
|
data_lines = if idx == 0, do: lines, else: tl(lines)
|
|
|
|
# Drop blank trailing lines
|
|
Enum.reject(data_lines, &(String.trim(&1) == ""))
|
|
end)
|
|
|> List.flatten()
|
|
|> Enum.join("\n")
|
|
|
|
total_lines = combined |> String.split("\n") |> length() |> Kernel.-(1)
|
|
Mix.shell().info("Combined #{length(argv)} files: #{total_lines} data rows")
|
|
|
|
Mix.shell().info("Running preview (validate + deduplicate)...")
|
|
|
|
case CsvImport.preview(combined, "contest-import@ntms.org") do
|
|
{:ok, preview} ->
|
|
print_preview_summary(preview)
|
|
print_invalid_rows(preview.invalid)
|
|
|
|
if preview.valid == [] do
|
|
Mix.shell().info("Nothing to import.")
|
|
else
|
|
import_batches(preview.valid)
|
|
end
|
|
|
|
{:error, reason} ->
|
|
Mix.raise("Failed: #{inspect(reason)}")
|
|
end
|
|
end
|
|
|
|
defp print_preview_summary(preview) do
|
|
Mix.shell().info("""
|
|
Preview:
|
|
Valid: #{length(preview.valid)}
|
|
Duplicates: #{length(preview.duplicates)}
|
|
Invalid: #{length(preview.invalid)}
|
|
""")
|
|
end
|
|
|
|
defp print_invalid_rows([]), do: :ok
|
|
|
|
defp print_invalid_rows(invalid) do
|
|
invalid
|
|
|> Enum.take(10)
|
|
|> Enum.each(fn row ->
|
|
Mix.shell().info(" Row #{row.row_num}: #{Enum.join(row.messages, ", ")}")
|
|
end)
|
|
|
|
if length(invalid) > 10 do
|
|
Mix.shell().info(" ... and #{length(invalid) - 10} more")
|
|
end
|
|
end
|
|
|
|
defp import_batches(valid) do
|
|
Mix.shell().info("Importing #{length(valid)} contacts...")
|
|
|
|
{imported, errors} =
|
|
valid
|
|
|> Enum.chunk_every(500)
|
|
|> Enum.with_index(1)
|
|
|> Enum.reduce({0, 0}, fn {batch, batch_num}, {imported, errors} ->
|
|
{:ok, result} = CsvImport.commit(batch)
|
|
new_imported = imported + length(result.imported)
|
|
new_errors = errors + length(result.errors)
|
|
|
|
Mix.shell().info(
|
|
" Batch #{batch_num}: +#{length(result.imported)} imported, #{length(result.errors)} errors (total: #{new_imported})"
|
|
)
|
|
|
|
{new_imported, new_errors}
|
|
end)
|
|
|
|
Mix.shell().info("Done! Imported #{imported} contacts, #{errors} errors.")
|
|
end
|
|
end
|