prop/priv/repo/import_weather.exs

import Ecto.Query

alias Microwaveprop.Repo
alias Microwaveprop.Weather
alias Microwaveprop.Weather.IemClient
alias Microwaveprop.Weather.SolarClient
alias Microwaveprop.Workers.WeatherFetchWorker

# --- Helpers ---

defmodule ImportHelpers do
  @moduledoc false

  @km_per_deg 111.0

  @us_networks ~w(
    AL_ASOS AK_ASOS AZ_ASOS AR_ASOS CA_ASOS CO_ASOS CT_ASOS DE_ASOS FL_ASOS
    GA_ASOS HI_ASOS ID_ASOS IL_ASOS IN_ASOS IA_ASOS KS_ASOS KY_ASOS LA_ASOS
    ME_ASOS MD_ASOS MA_ASOS MI_ASOS MN_ASOS MS_ASOS MO_ASOS MT_ASOS NE_ASOS
    NV_ASOS NH_ASOS NJ_ASOS NM_ASOS NY_ASOS NC_ASOS ND_ASOS OH_ASOS OK_ASOS
    OR_ASOS PA_ASOS RI_ASOS SC_ASOS SD_ASOS TN_ASOS TX_ASOS UT_ASOS VT_ASOS
    VA_ASOS WA_ASOS WV_ASOS WI_ASOS WY_ASOS DC_ASOS PR_ASOS
  )

  def us_networks, do: @us_networks

  def nearest_stations(lat, lon, stations, radius_km) do
    Enum.filter(stations, fn s ->
      dlat = (s.lat - lat) * @km_per_deg
      dlon = (s.lon - lon) * @km_per_deg * :math.cos(lat * :math.pi() / 180)
      :math.sqrt(dlat * dlat + dlon * dlon) <= radius_km
    end)
  end

  def sounding_times_around(dt) do
    # Return the 00Z and 12Z sounding times that bracket the QSO time
    date = DateTime.to_date(dt)
    hour = dt.hour

    times =
      if hour < 12 do
        [
          DateTime.new!(Date.add(date, -1), ~T[12:00:00], "Etc/UTC"),
          DateTime.new!(date, ~T[00:00:00], "Etc/UTC")
        ]
      else
        [
          DateTime.new!(date, ~T[00:00:00], "Etc/UTC"),
          DateTime.new!(date, ~T[12:00:00], "Etc/UTC")
        ]
      end

    Enum.uniq(times)
  end
end

# --- Phase 0: Import solar indices ---

IO.puts("=== Phase 0: Importing solar indices (since 2000-01-01) ===")

existing_solar = Weather.existing_solar_dates()
IO.puts("Found #{MapSet.size(existing_solar)} solar index dates already in DB")

IO.puts("Downloading GFZ solar index file...")

case SolarClient.fetch_solar_indices() do
  {:ok, all_records} ->
    IO.puts("Parsed #{length(all_records)} total daily records from GFZ file")

    records =
      all_records
      |> SolarClient.filter_since(~D[2000-01-01])
      |> Enum.reject(fn r -> MapSet.member?(existing_solar, r.date) end)

    IO.puts("#{length(records)} new records to import (skipping dates already in DB)")

    records
    |> Enum.with_index(1)
    |> Enum.each(fn {record, idx} ->
      if rem(idx, 500) == 0, do: IO.puts("  Upserting solar index #{idx}/#{length(records)}...")
      Weather.upsert_solar_index(record)
    end)

    IO.puts("Solar index import complete")

  {:error, reason} ->
    IO.puts("Warning: failed to fetch solar indices: #{inspect(reason)}")
end

# --- Phase 1: Discover and seed stations from IEM ---

IO.puts("=== Phase 1: Discovering stations from IEM ===")

# Fetch ASOS stations from all US state networks
IO.puts("Fetching ASOS stations from #{length(ImportHelpers.us_networks())} state networks...")

asos_stations =
  ImportHelpers.us_networks()
  |> Task.async_stream(
    fn network ->
      state = network |> String.split("_") |> hd()

      case IemClient.fetch_network(network) do
        {:ok, stations} ->
          Enum.map(stations, &Map.put(&1, :state, state))

        {:error, reason} ->
          IO.puts("  Warning: failed to fetch #{network}: #{inspect(reason)}")
          []
      end
    end,
    max_concurrency: 10,
    timeout: 30_000,
    ordered: false
  )
  |> Enum.flat_map(fn
    {:ok, stations} -> stations
    {:exit, _reason} -> []
  end)

IO.puts("Discovered #{length(asos_stations)} ASOS stations across all US networks")

# Fetch RAOB (sounding) stations
IO.puts("Fetching RAOB stations...")

raob_stations =
  case IemClient.fetch_network("RAOB") do
    {:ok, stations} ->
      stations

    {:error, reason} ->
      IO.puts("  Warning: failed to fetch RAOB network: #{inspect(reason)}")
      []
  end

IO.puts("Discovered #{length(raob_stations)} RAOB stations")

# Seed ASOS stations
IO.puts("Seeding ASOS stations...")

asos_station_map =
  Enum.reduce(asos_stations, %{}, fn attrs, acc ->
    {:ok, station} =
      Weather.find_or_create_station(Map.put(attrs, :station_type, "asos"))

    Map.put(acc, station.station_code, station)
  end)

IO.puts("Seeded #{map_size(asos_station_map)} unique ASOS stations")

# Seed sounding stations
IO.puts("Seeding sounding stations...")

sounding_station_map =
  Enum.reduce(raob_stations, %{}, fn attrs, acc ->
    {:ok, station} =
      Weather.find_or_create_station(Map.put(attrs, :station_type, "sounding"))

    Map.put(acc, station.station_code, station)
  end)

IO.puts("Seeded #{map_size(sounding_station_map)} unique sounding stations")

# Build lookup lists with lat/lon for nearest-station searches
asos_station_list =
  asos_stations
  |> Enum.map(fn s -> Map.put(s, :station_type, "asos") end)
  |> Enum.uniq_by(& &1.station_code)

sounding_station_list =
  raob_stations
  |> Enum.map(fn s -> Map.put(s, :station_type, "sounding") end)
  |> Enum.uniq_by(& &1.station_code)

# --- Phase 2: Enqueue weather fetches for each QSO ---

IO.puts("\n=== Phase 2: Enqueueing weather fetch jobs ===")

qsos =
  Microwaveprop.Radio.Qso
  |> where([q], not is_nil(q.pos1))
  |> order_by([q], q.qso_timestamp)
  |> Repo.all()

IO.puts("Processing #{length(qsos)} QSOs with positions...")

# Track which station+time combos we've already enqueued to avoid duplicates
asos_seen = :ets.new(:asos_seen, [:set, :public])
raob_seen = :ets.new(:raob_seen, [:set, :public])

total = length(qsos)
asos_enqueued = :counters.new(1, [:atomics])
raob_enqueued = :counters.new(1, [:atomics])
skipped = :counters.new(1, [:atomics])

Enum.with_index(qsos, 1)
|> Enum.each(fn {qso, idx} ->
  if rem(idx, 500) == 0, do: IO.puts("  Scanned #{idx}/#{total} QSOs")

  lat = qso.pos1["lat"] || qso.pos1["latitude"]
  lon = qso.pos1["lon"] || qso.pos1["longitude"]

  if lat && lon do
    # Find nearby ASOS stations (within 150km)
    nearby_asos = ImportHelpers.nearest_stations(lat, lon, asos_station_list, 150)

    Enum.each(nearby_asos, fn s ->
      start_dt = DateTime.add(qso.qso_timestamp, -2 * 3600, :second)
      end_dt = DateTime.add(qso.qso_timestamp, 2 * 3600, :second)
      key = {s.station_code, DateTime.to_date(start_dt), div(start_dt.hour, 4)}

      if !:ets.member(asos_seen, key) do
        :ets.insert(asos_seen, {key, true})
        station = Map.get(asos_station_map, s.station_code)

        if station && !Weather.has_surface_observations?(station.id, start_dt, end_dt) do
          %{
            "fetch_type" => "asos",
            "station_id" => station.id,
            "station_code" => s.station_code,
            "start_dt" => DateTime.to_iso8601(start_dt),
            "end_dt" => DateTime.to_iso8601(end_dt)
          }
          |> WeatherFetchWorker.new()
          |> Oban.insert()

          :counters.add(asos_enqueued, 1, 1)
        else
          :counters.add(skipped, 1, 1)
        end
      end
    end)

    # Find nearby sounding stations (within 300km)
    nearby_soundings = ImportHelpers.nearest_stations(lat, lon, sounding_station_list, 300)

    Enum.each(nearby_soundings, fn s ->
      sounding_times = ImportHelpers.sounding_times_around(qso.qso_timestamp)

      Enum.each(sounding_times, fn sounding_time ->
        key = {s.station_code, sounding_time}

        if !:ets.member(raob_seen, key) do
          :ets.insert(raob_seen, {key, true})
          station = Map.get(sounding_station_map, s.station_code)

          if station && !Weather.has_sounding?(station.id, sounding_time) do
            %{
              "fetch_type" => "raob",
              "station_id" => station.id,
              "station_code" => s.station_code,
              "sounding_time" => DateTime.to_iso8601(sounding_time)
            }
            |> WeatherFetchWorker.new()
            |> Oban.insert()

            :counters.add(raob_enqueued, 1, 1)
          else
            :counters.add(skipped, 1, 1)
          end
        end
      end)
    end)
  end
end)

:ets.delete(asos_seen)
:ets.delete(raob_seen)

# --- Summary ---

asos_count = Repo.aggregate(Microwaveprop.Weather.SurfaceObservation, :count)
sounding_count = Repo.aggregate(Microwaveprop.Weather.Sounding, :count)
station_count = Repo.aggregate(Microwaveprop.Weather.Station, :count)
solar_count = Repo.aggregate(Microwaveprop.Weather.SolarIndex, :count)
job_count = Oban.Job |> Repo.aggregate(:count)

IO.puts("""

Scan complete:
  Solar index days:      #{solar_count}
  Stations:              #{station_count}
  Surface observations:  #{asos_count} (existing)
  Soundings:             #{sounding_count} (existing)
  ASOS jobs enqueued:    #{:counters.get(asos_enqueued, 1)}
  RAOB jobs enqueued:    #{:counters.get(raob_enqueued, 1)}
  Total Oban jobs:       #{job_count}

Oban will now process the weather queue (concurrency: 3) with automatic retries.
""")