defmodule Microwaveprop.Workers.Era5PollWorker do @moduledoc """ Second half of the split ERA5 backfill pipeline. Polls CDS for the two jobs submitted by `Era5SubmitWorker`; on completion, streams both GRIB2 files to disk, decodes them, bulk-inserts the profiles, deletes the `era5_cds_jobs` row, and asks CDS to delete the completed jobs so they don't count against our per-user quota. While either CDS job is still running the worker returns `{:snooze, N}` — Oban re-schedules the same job without counting an attempt and releases the worker slot immediately, so one pod can keep dozens of tile-months in flight without pinning workers to `Process.sleep`. `max_attempts` is high because each attempt models a human-scale failure (e.g. CDS reports the job failed), not a poll retry — snoozes don't consume attempts. """ use Oban.Worker, queue: :era5_poll, max_attempts: 10 alias Microwaveprop.Repo alias Microwaveprop.Weather.Era5BatchClient alias Microwaveprop.Weather.Era5CdsJob alias Microwaveprop.Weather.Era5Client require Logger # Re-check every 5 minutes. CDS month-tile jobs typically take 30-45 min, # so 5 minutes strikes a balance between responsiveness and noise. @snooze_seconds 5 * 60 @impl Oban.Worker def perform(%Oban.Job{args: %{"era5_cds_job_id" => id}}) do case Repo.get(Era5CdsJob, id) do nil -> Logger.debug("Era5Poll: row #{id} no longer exists — no-op") :ok %Era5CdsJob{} = row -> handle_row(row) end end defp handle_row(row) do single = Era5Client.check_status(row.single_job_id) pressure = Era5Client.check_status(row.pressure_job_id) case {single, pressure} do {{:done, single_source}, {:done, pressure_source}} -> handle_both_done(row, single_source, pressure_source) {{:failed, reason}, _} -> fail_row(row, "single-level job failed: #{reason}") {_, {:failed, reason}} -> fail_row(row, "pressure-level job failed: #{reason}") {{:error, reason}, _} -> {:error, "single-level check_status error: #{reason}"} {_, {:error, reason}} -> {:error, "pressure-level check_status error: #{reason}"} _both_running_or_mixed -> bump_poll_count(row) {:snooze, @snooze_seconds} end end defp handle_both_done(row, single_source, pressure_source) do tmp_dir = System.tmp_dir!() tag = "#{row.year}_#{pad(row.month)}_#{row.tile_lat}_#{row.tile_lon}_#{System.unique_integer([:positive])}" single_path = Path.join(tmp_dir, "era5_single_#{tag}.grib2") pressure_path = Path.join(tmp_dir, "era5_pressure_#{tag}.grib2") try do with :ok <- Era5Client.download_source_to_file(single_source, api_key(), single_path), :ok <- Era5Client.download_source_to_file(pressure_source, api_key(), pressure_path), {:ok, inserted} <- Era5BatchClient.decode_and_insert( row.year, row.month, row.tile_lat, row.tile_lon, single_path, pressure_path ) do Repo.delete!(row) delete_cds_jobs(row) Logger.info( "Era5Poll: stored #{inserted} profiles for #{row.year}-#{pad(row.month)} tile #{row.tile_lat},#{row.tile_lon}" ) {:ok, inserted} else {:error, reason} -> Logger.warning( "Era5Poll: download/decode failed for #{row.year}-#{pad(row.month)} tile #{row.tile_lat},#{row.tile_lon}: #{inspect(reason)}" ) {:error, reason} end after File.rm(single_path) File.rm(pressure_path) end end defp fail_row(row, reason) do Logger.warning( "Era5Poll: CDS failure for #{row.year}-#{pad(row.month)} tile #{row.tile_lat},#{row.tile_lon}: #{reason}" ) delete_cds_jobs(row) Repo.delete!(row) {:error, reason} end defp delete_cds_jobs(row) do # Best-effort — don't block progress on a CDS cleanup failure. for job_id <- [row.single_job_id, row.pressure_job_id] do case Era5Client.delete_job(job_id) do :ok -> :ok {:error, reason} -> Logger.warning("Era5Poll: failed to delete CDS job #{job_id}: #{reason}") :ok end end end defp bump_poll_count(row) do now = DateTime.truncate(DateTime.utc_now(), :second) row |> Era5CdsJob.changeset(%{poll_count: row.poll_count + 1, last_polled_at: now}) |> Repo.update!() end defp api_key, do: System.get_env("ERA5_CDS_API_KEY") defp pad(n), do: String.pad_leading(Integer.to_string(n), 2, "0") end