diff --git a/lib/towerops/pagerduty/client.ex b/lib/towerops/pagerduty/client.ex index fd61d8e5..274f8145 100644 --- a/lib/towerops/pagerduty/client.ex +++ b/lib/towerops/pagerduty/client.ex @@ -102,12 +102,17 @@ defmodule Towerops.PagerDuty.Client do {:error, {:bad_request, resp_body["message"] || "Invalid event"}} {:ok, %{status: 429} = response} -> - # Respect Retry-After header if present - retry_after = extract_retry_after(response, retries) + if retries >= @max_retries do + Logger.warning("PagerDuty rate limited after #{@max_retries} retries, giving up") + {:error, :rate_limited} + else + retry_after = extract_retry_after(response, retries) - Logger.info("PagerDuty rate limited, retrying in #{retry_after}ms (attempt #{retries + 1}/#{@max_retries})") - Process.sleep(retry_after) - send_event_with_retry(body, retries + 1) + Logger.info("PagerDuty rate limited, retrying in #{retry_after}ms (attempt #{retries + 1}/#{@max_retries})") + + Process.sleep(retry_after) + send_event_with_retry(body, retries + 1) + end {:ok, %{status: status}} -> {:error, {:unexpected_status, status}} diff --git a/test/towerops/uisp/config_snapshot_test.exs b/test/towerops/uisp/config_snapshot_test.exs index 52adbec3..cc95b6f2 100644 --- a/test/towerops/uisp/config_snapshot_test.exs +++ b/test/towerops/uisp/config_snapshot_test.exs @@ -222,8 +222,15 @@ defmodule Towerops.Uisp.ConfigSnapshotTest do test "returns computed diff of two stored snapshots", %{device: device} do ConfigSnapshot.store_if_changed(device, %{"a" => 1}, %{stored: 0, unchanged: 0}) - # Ensure distinct snapshot_at timestamps (second precision). - Process.sleep(1_100) + # Backdate the first snapshot so the second one has a strictly later + # snapshot_at, without the slow Process.sleep(1_100) a second-precision + # timestamp would otherwise require. + {1, _} = + Towerops.Repo.update_all( + Ecto.Query.from(s in "uisp_config_snapshots", where: s.device_id == type(^device.id, :binary_id)), + set: [snapshot_at: DateTime.add(Towerops.Time.now(), -5, :second)] + ) + ConfigSnapshot.store_if_changed(device, %{"a" => 2, "b" => 3}, %{stored: 0, unchanged: 0}) [newer, older] = ConfigSnapshot.list_snapshots(device.id)