Graham McIntire
d61fbd346e
Enabled :error_handling, :unknown, :unmatched_returns, :extra_return, :missing_return in an earlier commit and landed a 129-warning baseline. Four parallel agents each fixed a directory slice: - Core contexts (29): Radio, Release, Weather, Beacons, Cache, Backtest.Features, Terrain.Srtm, Ionosphere.GiroClient, Propagation.RunTiming, Accounts.Scope, RepoListener. Fixes were (a) prefix side-effect calls (Task.start, Phoenix.PubSub, Logger, :ets.new) with _ = ; (b) tighten/widen specs that didn't match actual returns; (c) add missing @type t declarations; (d) drop dead parse_int(nil) clause. - Propagation + weather subdirs (15): FreshnessMonitor, NotifyListener, ScoreCache, ScoreCacheReconciler, Weather.FrontalAnalysis, Weather.Grib2.Extractor, Weather.Grib2.Wgrib2, GridCache, HrrrPointEnqueuer, NexradCache. Same patterns — mostly _ = on PubSub / :ets / Repo.insert_all; widened two specs (float -> number) where integer returns were reachable. - Workers (35): BackfillEnqueue, CanadianSoundingFetch, ContactImport, ContactWeatherEnqueue, GefsFetch, IemreFetch, NarrFetch, SolarIndex, TerrainProfile, WeatherFetch. Prefixed Repo.update_all / Radio.set_enrichment_status! / Weather.upsert_* side-effect calls. Fixed one :pattern_match in CanadianSoundingFetch.most_recent_sounding_time/1 where a tautological cond guard generated unreachable code. - Web + Mix tasks + lib_ml (46 of 50): controllers, LiveViews, UserAuth, and 11 mix tasks. Same prefix strategy. 4 remaining warnings originate in LiveTable.LiveResource dep macro expansion and can't be fixed without forking the dep — added .dialyzer_ignore.exs to suppress just those specific file:line pairs. Also wired ignore_warnings in mix.exs dialyzer config. mix dialyzer --format short | grep ^lib/ | wc -l -> 0 mix test: 2163 tests, 3 pre-existing flakes, 0 regressions.
72 lines
2.4 KiB
Elixir
72 lines
2.4 KiB
Elixir
defmodule Mix.Tasks.NexradBackfill do
|
|
@shortdoc "Enqueue NexradWorker jobs for the top-N hours by contact count"
|
|
@moduledoc """
|
|
Backfills the `nexrad_observations` table by enqueueing one
|
|
`NexradWorker` job per distinct `(year, month, day, hour, minute=0)`
|
|
where we have contacts, prioritized by contact count so the most
|
|
data-dense hours land first.
|
|
|
|
Each n0q frame is 2-4 MB, so a 200-hour backfill is ~400-800 MB.
|
|
|
|
mix nexrad_backfill --limit 200
|
|
|
|
Jobs are deduplicated by Oban's unique constraint on
|
|
`{year, month, day, hour, minute}` so running the task twice is safe.
|
|
"""
|
|
use Mix.Task
|
|
|
|
import Ecto.Query
|
|
|
|
alias Microwaveprop.Radio.Contact
|
|
alias Microwaveprop.Repo
|
|
alias Microwaveprop.Workers.NexradWorker
|
|
|
|
@impl Mix.Task
|
|
def run(argv) do
|
|
Mix.Task.run("app.start")
|
|
_ = Oban.pause_all_queues(Oban)
|
|
|
|
{opts, _, _} = OptionParser.parse(argv, switches: [limit: :integer])
|
|
limit = Keyword.get(opts, :limit, 200)
|
|
|
|
hours = top_hours_by_contact_count(limit)
|
|
Mix.shell().info("Enqueueing #{length(hours)} NexradWorker jobs")
|
|
|
|
Enum.each(hours, fn %{year: y, month: m, day: d, hour: h, contacts: n} ->
|
|
args = %{"year" => y, "month" => m, "day" => d, "hour" => h, "minute" => 0}
|
|
|
|
case Oban.insert(NexradWorker.new(args)) do
|
|
{:ok, _job} ->
|
|
Mix.shell().info(" #{y}-#{pad(m)}-#{pad(d)} #{pad(h)}Z (#{n} contacts)")
|
|
|
|
{:error, reason} ->
|
|
Mix.shell().error(" #{y}-#{pad(m)}-#{pad(d)} #{pad(h)}Z failed: #{inspect(reason)}")
|
|
end
|
|
end)
|
|
end
|
|
|
|
defp pad(n), do: n |> Integer.to_string() |> String.pad_leading(2, "0")
|
|
|
|
defp top_hours_by_contact_count(limit) do
|
|
Repo.all(
|
|
from(c in Contact,
|
|
where: not is_nil(c.pos1),
|
|
select: %{
|
|
year: fragment("EXTRACT(YEAR FROM ?)::int", c.qso_timestamp),
|
|
month: fragment("EXTRACT(MONTH FROM ?)::int", c.qso_timestamp),
|
|
day: fragment("EXTRACT(DAY FROM ?)::int", c.qso_timestamp),
|
|
hour: fragment("EXTRACT(HOUR FROM ?)::int", c.qso_timestamp),
|
|
contacts: count(c.id)
|
|
},
|
|
group_by: [
|
|
fragment("EXTRACT(YEAR FROM ?)::int", c.qso_timestamp),
|
|
fragment("EXTRACT(MONTH FROM ?)::int", c.qso_timestamp),
|
|
fragment("EXTRACT(DAY FROM ?)::int", c.qso_timestamp),
|
|
fragment("EXTRACT(HOUR FROM ?)::int", c.qso_timestamp)
|
|
],
|
|
order_by: [desc: count(c.id)],
|
|
limit: ^limit
|
|
)
|
|
)
|
|
end
|
|
end
|