diff --git a/lib/mix/tasks/hrrr_backfill.ex b/lib/mix/tasks/hrrr_backfill.ex index 04c9dbc9..c93368fc 100644 --- a/lib/mix/tasks/hrrr_backfill.ex +++ b/lib/mix/tasks/hrrr_backfill.ex @@ -58,21 +58,25 @@ defmodule Mix.Tasks.HrrrBackfill do |> Enum.map(fn {hour, points} -> {hour, Enum.uniq(points)} end) |> Enum.sort_by(fn {hour, _} -> hour end, {:desc, DateTime}) - # Filter to only hours needing backfill + # Filter out points that already have enough levels by_hour = if refetch_all? do by_hour else - Enum.filter(by_hour, fn {hour, points} -> - needs_backfill?(hour, points) + by_hour + |> Enum.map(fn {hour, points} -> + needed = filter_points_needing_backfill(hour, points) + {hour, needed} end) + |> Enum.reject(fn {_hour, points} -> points == [] end) end by_hour = if limit, do: Enum.take(by_hour, limit), else: by_hour total = length(by_hour) total_points = Enum.reduce(by_hour, 0, fn {_, pts}, acc -> acc + length(pts) end) - Logger.info("HRRR backfill: #{total} hours, #{total_points} points to process") + skipped = length(contacts) - total_points + Logger.info("HRRR backfill: #{total} hours, #{total_points} points to fetch (#{skipped} already up to date)") by_hour |> Enum.with_index(1) @@ -98,21 +102,27 @@ defmodule Mix.Tasks.HrrrBackfill do Logger.info("HRRR backfill complete") end - defp needs_backfill?(hour, points) do - # Check if any point for this hour is missing or has fewer than @min_levels - sample = hd(points) - {lat, lon} = sample + defp filter_points_needing_backfill(hour, points) do + # Query level counts for all points at this hour in one query + point_tuples = Enum.map(points, fn {lat, lon} -> {lat, lon} end) + lats = Enum.map(point_tuples, &elem(&1, 0)) + lons = Enum.map(point_tuples, &elem(&1, 1)) - case Repo.one( - from(h in HrrrProfile, - where: h.lat == ^lat and h.lon == ^lon and h.valid_time == ^hour, - select: fragment("array_length(profile, 1)") - ) - ) do - nil -> true - count when count < @min_levels -> true - _ -> false - end + existing = + from(h in HrrrProfile, + where: h.valid_time == ^hour and h.lat in ^lats and h.lon in ^lons, + select: {h.lat, h.lon, fragment("array_length(profile, 1)")} + ) + |> Repo.all() + |> Map.new(fn {lat, lon, count} -> {{lat, lon}, count || 0} end) + + Enum.filter(points, fn {lat, lon} -> + case Map.get(existing, {lat, lon}) do + nil -> true + count when count < @min_levels -> true + _ -> false + end + end) end defp store_profile(lat, lon, valid_time, data) do