diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 0e1be2c9..c200ed22 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,21 @@ +2026-03-09 +feat: add ErrorTrackerNotifier for email alerts on exceptions + - Added error_tracker_notifier dependency (~> 0.2) + - Configured email notifications in config/runtime.exs (production only) + - Sends exception alerts from alerts@towerops.net to graham@towerops.net + - Added to supervision tree in lib/towerops/application.ex + - Includes throttling to prevent duplicate error spam + - Gracefully shuts down in dev/test (no emails sent) + - Files: mix.exs, config/runtime.exs, lib/towerops/application.ex + +2026-03-09 +fix: AlertNotificationWorker failing when PagerDuty not configured + - Worker was treating {:error, :not_configured} as failure, causing retries + - Added graceful handling for :not_configured in all three actions (trigger, acknowledge, resolve) + - Now logs as debug and succeeds when PagerDuty integration not set up + - Prevents unnecessary error noise in ErrorTracker for organizations without PagerDuty + - Files: lib/towerops/workers/alert_notification_worker.ex + 2026-03-06 feat: improve Gaiia webhook setup instructions and descriptions - Updated Gaiia provider description to explain what it enables diff --git a/config/runtime.exs b/config/runtime.exs index fc36aee8..48546fca 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -382,6 +382,13 @@ if config_env() == :prod do This is the Stripe billing meter ID for usage reporting. """ + # Configure ErrorTrackerNotifier for email alerts on exceptions + config :error_tracker_notifier, + notification_type: :email, + from_email: "alerts@towerops.net", + to_email: System.get_env("ERROR_ALERT_EMAIL") || "graham@towerops.net", + mailer: Towerops.Mailer + config :towerops, stripe_secret_key: stripe_secret_key, stripe_webhook_secret: stripe_webhook_secret, diff --git a/lib/towerops/application.ex b/lib/towerops/application.ex index 13957e3f..be0b5dda 100644 --- a/lib/towerops/application.ex +++ b/lib/towerops/application.ex @@ -93,6 +93,8 @@ defmodule Towerops.Application do Towerops.Repo, # Encryption vault for sensitive data (passwords, API tokens) Towerops.Vault, + # Error notifications via email (production only, gracefully shuts down in dev/test) + {ErrorTrackerNotifier, []}, # Redis connection for brute force protection and rate limiting redis_spec(), # Rate limiting backend (ETS-based, per-node) diff --git a/lib/towerops/workers/alert_notification_worker.ex b/lib/towerops/workers/alert_notification_worker.ex index 93a40866..eda44cb7 100644 --- a/lib/towerops/workers/alert_notification_worker.ex +++ b/lib/towerops/workers/alert_notification_worker.ex @@ -30,6 +30,10 @@ defmodule Towerops.Workers.AlertNotificationWorker do Logger.warning("Device not found for alert #{alert_id}") :ok + {:error, :not_configured} -> + Logger.debug("PagerDuty not configured for alert #{alert_id}, skipping notification") + :ok + {:error, reason} -> Logger.error("Failed to send trigger notification for alert #{alert_id}: #{inspect(reason)}") {:error, reason} @@ -39,7 +43,14 @@ defmodule Towerops.Workers.AlertNotificationWorker do def perform(%Oban.Job{args: %{"action" => "acknowledge", "alert_id" => alert_id}}) do case fetch_alert(alert_id) do {:ok, alert} -> - Notifier.notify_acknowledge(alert) + case Notifier.notify_acknowledge(alert) do + {:error, :not_configured} -> + Logger.debug("PagerDuty not configured for alert #{alert_id}, skipping notification") + :ok + + result -> + result + end {:error, :alert_not_found} -> Logger.warning("Alert #{alert_id} not found for acknowledge notification") @@ -54,7 +65,14 @@ defmodule Towerops.Workers.AlertNotificationWorker do def perform(%Oban.Job{args: %{"action" => "resolve", "alert_id" => alert_id}}) do case fetch_alert(alert_id) do {:ok, alert} -> - Notifier.notify_resolve(alert) + case Notifier.notify_resolve(alert) do + {:error, :not_configured} -> + Logger.debug("PagerDuty not configured for alert #{alert_id}, skipping notification") + :ok + + result -> + result + end {:error, :alert_not_found} -> Logger.warning("Alert #{alert_id} not found for resolve notification") diff --git a/mix.exs b/mix.exs index 1b761ee5..75584ddc 100644 --- a/mix.exs +++ b/mix.exs @@ -81,6 +81,7 @@ defmodule Towerops.MixProject do {:absinthe_plug, "~> 1.5"}, {:honeybadger, "~> 0.24"}, {:error_tracker, "~> 0.7"}, + {:error_tracker_notifier, "~> 0.2"}, {:stream_data, "~> 1.1", only: :test}, {:tzdata, "~> 1.1"}, {:styler, "~> 1.10", only: [:dev, :test], runtime: false}, diff --git a/mix.lock b/mix.lock index 0ecc2e4b..323e2a22 100644 --- a/mix.lock +++ b/mix.lock @@ -23,6 +23,7 @@ "eqrcode": {:hex, :eqrcode, "0.2.1", "d12838813e8fc87b8940cc05f9baadb189031f6009facdc56ff074375ec73b6e", [:mix], [], "hexpm", "d5828a222b904c68360e7dc2a40c3ef33a1328b7c074583898040f389f928025"}, "erlex": {:hex, :erlex, "0.2.8", "cd8116f20f3c0afe376d1e8d1f0ae2452337729f68be016ea544a72f767d9c12", [:mix], [], "hexpm", "9d66ff9fedf69e49dc3fd12831e12a8a37b76f8651dd21cd45fcf5561a8a7590"}, "error_tracker": {:hex, :error_tracker, "0.8.0", "15774836205c1b4cd8275b594d99c10f4de168eb00573743526b6a919f2aca4a", [:mix], [{:ecto, "~> 3.13", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.13", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, ">= 0.0.0", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:igniter, "~> 0.5", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:myxql, ">= 0.0.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:phoenix_ecto, "~> 4.6", [hex: :phoenix_ecto, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "00ab8342932de7ad63be567ddd36ea238c840c122c7b2c5d1a42638ce6f39220"}, + "error_tracker_notifier": {:hex, :error_tracker_notifier, "0.2.1", "c13452605f4f27c9c9665136faab6c82fa626b7fadd59198ad948d61dc633715", [:mix], [{:error_tracker, "~> 0.5", [hex: :error_tracker, repo: "hexpm", optional: false]}, {:httpoison, "~> 2.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:swoosh, "~> 1.5", [hex: :swoosh, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "fda342098c28ca2df45463821de49be6a1402b30321c076aae689204c5e3fecc"}, "esbuild": {:hex, :esbuild, "0.10.0", "b0aa3388a1c23e727c5a3e7427c932d89ee791746b0081bbe56103e9ef3d291f", [:mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "468489cda427b974a7cc9f03ace55368a83e1a7be12fba7e30969af78e5f8c70"}, "expo": {:hex, :expo, "1.1.1", "4202e1d2ca6e2b3b63e02f69cfe0a404f77702b041d02b58597c00992b601db5", [:mix], [], "hexpm", "5fb308b9cb359ae200b7e23d37c76978673aa1b06e2b3075d814ce12c5811640"}, "file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"}, @@ -36,6 +37,7 @@ "heroicons": {:git, "https://github.com/tailwindlabs/heroicons.git", "0435d4ca364a608cc75e2f8683d374e55abbae26", [tag: "v2.2.0", sparse: "optimized", depth: 1]}, "honeybadger": {:hex, :honeybadger, "0.25.0", "18ccd1a6cc9efd7a723cb8db0b2a853eea2d8d4a77ede8fbe498d16dbd5350d7", [:mix], [{:ash, "~> 3.0", [hex: :ash, repo: "hexpm", optional: true]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:hackney, "~> 1.1", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.0.0 and < 2.0.0", [hex: :phoenix, repo: "hexpm", optional: true]}, {:plug, ">= 1.0.0 and < 2.0.0", [hex: :plug, repo: "hexpm", optional: true]}, {:process_tree, "~> 0.2.1", [hex: :process_tree, repo: "hexpm", optional: false]}, {:req, "~> 0.5.0", [hex: :req, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c39f02a1d86c34d7c915c4a188e9eaef953b3a1ec5cc02c4585f7697b523c6d5"}, "hpax": {:hex, :hpax, "1.0.3", "ed67ef51ad4df91e75cc6a1494f851850c0bd98ebc0be6e81b026e765ee535aa", [:mix], [], "hexpm", "8eab6e1cfa8d5918c2ce4ba43588e894af35dbd8e91e6e55c817bca5847df34a"}, + "httpoison": {:hex, :httpoison, "2.3.0", "10eef046405bc44ba77dc5b48957944df8952cc4966364b3cf6aa71dce6de587", [:mix], [{:hackney, "~> 1.21", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "d388ee70be56d31a901e333dbcdab3682d356f651f93cf492ba9f06056436a2c"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "inet_cidr": {:hex, :inet_cidr, "1.0.9", "e0ef72a2942529da78c8e4147d53f2ef5f6f5293335c3637b0fdf83c012cc816", [:mix], [], "hexpm", "172da15ff7cf635b1feaf14f5818be28c811b37cc5fb7c5f7c01058c1c1066cc"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, @@ -84,7 +86,7 @@ "swoosh": {:hex, :swoosh, "1.23.0", "a1b7f41705357ffb06457d177e734bf378022901ce53889a68bcc59d10a23c27", [:mix], [{:bandit, ">= 1.0.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:ex_aws, "~> 2.1", [hex: :ex_aws, repo: "hexpm", optional: true]}, {:finch, "~> 0.6", [hex: :finch, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:idna, "~> 6.0", [hex: :idna, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mua, "~> 0.2.3", [hex: :mua, repo: "hexpm", optional: true]}, {:multipart, "~> 0.4", [hex: :multipart, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:req, "~> 0.5.10 or ~> 0.6 or ~> 1.0", [hex: :req, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "97aaf04481ce8a351e2d15a3907778bdf3b1ea071cfff3eb8728b65943c77f6d"}, "table_rex": {:hex, :table_rex, "4.1.0", "fbaa8b1ce154c9772012bf445bfb86b587430fb96f3b12022d3f35ee4a68c918", [:mix], [], "hexpm", "95932701df195d43bc2d1c6531178fc8338aa8f38c80f098504d529c43bc2601"}, "tailwind": {:hex, :tailwind, "0.4.1", "e7bcc222fe96a1e55f948e76d13dd84a1a7653fb051d2a167135db3b4b08d3e9", [:mix], [], "hexpm", "6249d4f9819052911120dbdbe9e532e6bd64ea23476056adb7f730aa25c220d1"}, - "telemetry": {:hex, :telemetry, "1.4.0", "69aa83d53f152f93f05fd40b77ded6fab247de093b7a3c4ca2879e634144446e", [:rebar3], [], "hexpm", "d1ff426f988ac1092f9d684d34d08e51042a70567c16be793fbc8f399fd2e77d"}, + "telemetry": {:hex, :telemetry, "1.4.1", "ab6de178e2b29b58e8256b92b382ea3f590a47152ca3651ea857a6cae05ac423", [:rebar3], [], "hexpm", "2172e05a27531d3d31dd9782841065c50dd5c3c7699d95266b2edd54c2dafa1c"}, "telemetry_metrics": {:hex, :telemetry_metrics, "1.1.0", "5bd5f3b5637e0abea0426b947e3ce5dd304f8b3bc6617039e2b5a008adc02f8f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e7b79e8ddfde70adb6db8a6623d1778ec66401f366e9a8f5dd0955c56bc8ce67"}, "telemetry_poller": {:hex, :telemetry_poller, "1.3.0", "d5c46420126b5ac2d72bc6580fb4f537d35e851cc0f8dbd571acf6d6e10f5ec7", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "51f18bed7128544a50f75897db9974436ea9bfba560420b646af27a9a9b35211"}, "thousand_island": {:hex, :thousand_island, "1.4.3", "2158209580f633be38d43ec4e3ce0a01079592b9657afff9080d5d8ca149a3af", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6e4ce09b0fd761a58594d02814d40f77daff460c48a7354a15ab353bb998ea0b"}, diff --git a/priv/static/changelog.txt b/priv/static/changelog.txt index e65da2ce..31ccf068 100644 --- a/priv/static/changelog.txt +++ b/priv/static/changelog.txt @@ -1,3 +1,7 @@ +2026-03-09 — Reliability Improvements +* Improved handling of alerts when PagerDuty integration is not configured +* Reduced unnecessary error notifications for expected conditions + 2026-03-06 — Integration Improvements * Improved webhook setup instructions with detailed descriptions and step-by-step guidance * Added descriptions of what each webhook event type enables for real-time data sync