From c92dfff6c955be30a1e00ade1d91afabb1c1f9f1 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Tue, 17 Feb 2026 08:13:22 -0600 Subject: [PATCH] Add ErrorTracker, remove Honeybadger email notice filter Replace the HoneybadgerNoticeFilter that emailed raw stacktraces on every exception with ErrorTracker for self-hosted error tracking. Honeybadger itself is retained. ErrorTracker dashboard is mounted at /admin/errors behind the superuser auth wall. --- config/config.exs | 7 +- config/test.exs | 3 +- lib/towerops/honeybadger_notice_filter.ex | 98 ----------- lib/towerops_web/router.ex | 2 + mix.exs | 1 + mix.lock | 3 +- .../20260217140215_add_error_tracker.exs | 6 + .../honeybadger_notice_filter_test.exs | 157 ------------------ 8 files changed, 18 insertions(+), 259 deletions(-) delete mode 100644 lib/towerops/honeybadger_notice_filter.ex create mode 100644 priv/repo/migrations/20260217140215_add_error_tracker.exs delete mode 100644 test/towerops/honeybadger_notice_filter_test.exs diff --git a/config/config.exs b/config/config.exs index cf2597b6..a84764e2 100644 --- a/config/config.exs +++ b/config/config.exs @@ -10,6 +10,10 @@ import Config # Configure time zone database to use tzdata for IANA timezone support config :elixir, :time_zone_database, Tzdata.TimeZoneDatabase +config :error_tracker, + repo: Towerops.Repo, + otp_app: :towerops + # Configure esbuild (the version is required) config :esbuild, version: "0.25.4", @@ -25,8 +29,7 @@ config :honeybadger, environment_name: config_env(), insights_enabled: true, use_logger: true, - filter: Towerops.HoneybadgerFilter, - notice_filter: Towerops.HoneybadgerNoticeFilter + filter: Towerops.HoneybadgerFilter # Configure Elixir's Logger config :logger, :default_formatter, diff --git a/config/test.exs b/config/test.exs index 5a92a6a3..1bbfa887 100644 --- a/config/test.exs +++ b/config/test.exs @@ -5,6 +5,8 @@ config :argon2_elixir, t_cost: 1, m_cost: 8 +config :error_tracker, enabled: false + # Suppress log output during tests (tests that need logs use capture_log) config :logger, :default_handler, false config :logger, level: :error @@ -70,7 +72,6 @@ config :towerops, :agent_webhook_secret, "test-webhook-secret" # Set environment identifier for runtime checks config :towerops, :env, :test -# Use mocks for testing config :towerops, ping_module: Towerops.Monitoring.PingMock, poller_module: Towerops.Snmp.PollerMock, diff --git a/lib/towerops/honeybadger_notice_filter.ex b/lib/towerops/honeybadger_notice_filter.ex deleted file mode 100644 index 7bcd4ea5..00000000 --- a/lib/towerops/honeybadger_notice_filter.ex +++ /dev/null @@ -1,98 +0,0 @@ -defmodule Towerops.HoneybadgerNoticeFilter do - @moduledoc """ - Honeybadger notice filter that emails a raw stacktrace to the - ops contact whenever an exception is reported. - - Implements `Honeybadger.NoticeFilter` — receives the fully-built - notice, sends an email, and returns the notice unchanged so it - still reaches Honeybadger. - """ - - @behaviour Honeybadger.NoticeFilter - - import Swoosh.Email - - alias Towerops.Mailer - - require Logger - - @recipient "graham@mcintire.me" - - @impl Honeybadger.NoticeFilter - def filter(%Honeybadger.Notice{} = notice) do - if shutdown_error?(notice) do - nil - else - send_error_email(notice) - notice - end - end - - defp shutdown_error?(%Honeybadger.Notice{error: error}) do - msg = error.message || "" - - String.contains?(msg, "port_died") or - (String.contains?(msg, "write_failed") and String.contains?(msg, "epipe")) or - String.contains?(msg, "broken pipe (epipe)") - end - - defp send_error_email(%Honeybadger.Notice{error: error, server: server}) do - from_address = Application.get_env(:towerops, :mailer_from, {"Towerops", "hi@towerops.net"}) - subject = "[#{server.environment_name}] #{error.class}: #{truncate(error.message, 120)}" - - body = format_body(error, server) - - email = - new() - |> to(@recipient) - |> from(from_address) - |> subject(subject) - |> text_body(body) - - try do - case Mailer.deliver(email) do - {:ok, _} -> - :ok - - {:error, reason} -> - Logger.warning("Failed to send error email: #{inspect(reason)}") - :error - end - rescue - e -> - Logger.warning("Error email delivery crashed: #{Exception.message(e)}") - :error - end - end - - defp format_body(error, server) do - backtrace = format_backtrace(error.backtrace) - - """ - Exception: #{error.class} - Message: #{error.message} - - Environment: #{server.environment_name} - Hostname: #{server.hostname} - - Stacktrace: - #{backtrace} - """ - end - - defp format_backtrace(backtrace) when is_list(backtrace) do - Enum.map_join(backtrace, "\n", fn entry -> - " #{entry.file}:#{entry.number} in #{entry.method}" - end) - end - - defp format_backtrace(_), do: " (no stacktrace available)" - - defp truncate(nil, _max), do: "(no message)" - - defp truncate(message, max) when byte_size(message) > max do - String.slice(message, 0, max) <> "..." - end - - defp truncate(message, _max), do: message -end diff --git a/lib/towerops_web/router.ex b/lib/towerops_web/router.ex index 3de48bad..12f7998d 100644 --- a/lib/towerops_web/router.ex +++ b/lib/towerops_web/router.ex @@ -3,6 +3,7 @@ defmodule ToweropsWeb.Router do use ToweropsWeb, :router use Honeybadger.Plug + use ErrorTracker.Web, :router import Oban.Web.Router import Phoenix.LiveDashboard.Router @@ -216,6 +217,7 @@ defmodule ToweropsWeb.Router do pipe_through [:browser, :require_authenticated_user, :require_superuser] oban_dashboard("/oban") + error_tracker_dashboard("/errors") get "/headers", ToweropsWeb.DebugController, :headers end diff --git a/mix.exs b/mix.exs index 489cdc41..bbfd818c 100644 --- a/mix.exs +++ b/mix.exs @@ -80,6 +80,7 @@ defmodule Towerops.MixProject do {:absinthe, "~> 1.7"}, {:absinthe_plug, "~> 1.5"}, {:honeybadger, "~> 0.24"}, + {:error_tracker, "~> 0.7"}, {:stream_data, "~> 1.1", only: :test}, {:tzdata, "~> 1.1"}, {:styler, "~> 1.10", only: [:dev, :test], runtime: false}, diff --git a/mix.lock b/mix.lock index 881c5326..eab6c93c 100644 --- a/mix.lock +++ b/mix.lock @@ -21,6 +21,7 @@ "elixir_make": {:hex, :elixir_make, "0.9.0", "6484b3cd8c0cee58f09f05ecaf1a140a8c97670671a6a0e7ab4dc326c3109726", [:mix], [], "hexpm", "db23d4fd8b757462ad02f8aa73431a426fe6671c80b200d9710caf3d1dd0ffdb"}, "eqrcode": {:hex, :eqrcode, "0.2.1", "d12838813e8fc87b8940cc05f9baadb189031f6009facdc56ff074375ec73b6e", [:mix], [], "hexpm", "d5828a222b904c68360e7dc2a40c3ef33a1328b7c074583898040f389f928025"}, "erlex": {:hex, :erlex, "0.2.8", "cd8116f20f3c0afe376d1e8d1f0ae2452337729f68be016ea544a72f767d9c12", [:mix], [], "hexpm", "9d66ff9fedf69e49dc3fd12831e12a8a37b76f8651dd21cd45fcf5561a8a7590"}, + "error_tracker": {:hex, :error_tracker, "0.7.1", "074f10bfb01961bd0d51eb010c6ddbe13bf40eb4ddfbb25538680ddb520ab1aa", [:mix], [{:ecto, "~> 3.13", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.13", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, ">= 0.0.0", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:igniter, "~> 0.5", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:myxql, ">= 0.0.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:phoenix_ecto, "~> 4.6", [hex: :phoenix_ecto, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "cc77434c084d100e17dcc5ccea6bed05b15748186df5207e21fb761651bf4e0d"}, "esbuild": {:hex, :esbuild, "0.10.0", "b0aa3388a1c23e727c5a3e7427c932d89ee791746b0081bbe56103e9ef3d291f", [:mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "468489cda427b974a7cc9f03ace55368a83e1a7be12fba7e30969af78e5f8c70"}, "expo": {:hex, :expo, "1.1.1", "4202e1d2ca6e2b3b63e02f69cfe0a404f77702b041d02b58597c00992b601db5", [:mix], [], "hexpm", "5fb308b9cb359ae200b7e23d37c76978673aa1b06e2b3075d814ce12c5811640"}, "file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"}, @@ -62,7 +63,7 @@ "phoenix_html": {:hex, :phoenix_html, "4.3.0", "d3577a5df4b6954cd7890c84d955c470b5310bb49647f0a114a6eeecc850f7ad", [:mix], [], "hexpm", "3eaa290a78bab0f075f791a46a981bbe769d94bc776869f4f3063a14f30497ad"}, "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.8.7", "405880012cb4b706f26dd1c6349125bfc903fb9e44d1ea668adaf4e04d4884b7", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.5", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:ecto_sqlite3_extras, "~> 1.1.7 or ~> 1.2.0", [hex: :ecto_sqlite3_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.19 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "3a8625cab39ec261d48a13b7468dc619c0ede099601b084e343968309bd4d7d7"}, "phoenix_live_reload": {:hex, :phoenix_live_reload, "1.6.2", "b18b0773a1ba77f28c52decbb0f10fd1ac4d3ae5b8632399bbf6986e3b665f62", [:mix], [{:file_system, "~> 0.2.10 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "d1f89c18114c50d394721365ffb428cce24f1c13de0467ffa773e2ff4a30d5b9"}, - "phoenix_live_view": {:hex, :phoenix_live_view, "1.1.23", "5e41c63377d508dc9b22dcebcefa0a3a243695932f72eac6003831fd4ed96165", [:mix], [{:igniter, ">= 0.6.16 and < 1.0.0-0", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:lazy_html, "~> 0.1.0", [hex: :lazy_html, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0 or ~> 1.8.0-rc", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.15", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f5296ef67b41da7fe345e139ebaa621ceb5e7e84e37c89a12574cc0536a80604"}, + "phoenix_live_view": {:hex, :phoenix_live_view, "1.1.24", "1a000a048d5971b61a9efe29a3c4144ca955afd42224998d841c5011a5354838", [:mix], [{:igniter, ">= 0.6.16 and < 1.0.0-0", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:lazy_html, "~> 0.1.0", [hex: :lazy_html, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0 or ~> 1.8.0-rc", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.15", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "0c724e6c65f197841cac49d73be4e0f9b93a7711eaa52d2d4d1b9f859c329267"}, "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.2.0", "ff3a5616e1bed6804de7773b92cbccfc0b0f473faf1f63d7daf1206c7aeaaa6f", [:mix], [], "hexpm", "adc313a5bf7136039f63cfd9668fde73bba0765e0614cba80c06ac9460ff3e96"}, "phoenix_pubsub_redis": {:hex, :phoenix_pubsub_redis, "3.0.1", "d4d856b1e57a21358e448543e1d091e07e83403dde4383b8be04ed9d2c201cbc", [:mix], [{:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:poolboy, "~> 1.5.1 or ~> 1.6", [hex: :poolboy, repo: "hexpm", optional: false]}, {:redix, "~> 0.10.0 or ~> 1.0", [hex: :redix, repo: "hexpm", optional: false]}], "hexpm", "0b36a17ff6e9a56159f8df8933d62b5c1f0695eae995a02e0c86c035ace6a309"}, "phoenix_template": {:hex, :phoenix_template, "1.0.4", "e2092c132f3b5e5b2d49c96695342eb36d0ed514c5b252a77048d5969330d639", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "2c0c81f0e5c6753faf5cca2f229c9709919aba34fab866d3bc05060c9c444206"}, diff --git a/priv/repo/migrations/20260217140215_add_error_tracker.exs b/priv/repo/migrations/20260217140215_add_error_tracker.exs new file mode 100644 index 00000000..7624d564 --- /dev/null +++ b/priv/repo/migrations/20260217140215_add_error_tracker.exs @@ -0,0 +1,6 @@ +defmodule Towerops.Repo.Migrations.AddErrorTracker do + use Ecto.Migration + + def up, do: ErrorTracker.Migration.up(version: 5) + def down, do: ErrorTracker.Migration.down(version: 1) +end diff --git a/test/towerops/honeybadger_notice_filter_test.exs b/test/towerops/honeybadger_notice_filter_test.exs deleted file mode 100644 index af2b3bb8..00000000 --- a/test/towerops/honeybadger_notice_filter_test.exs +++ /dev/null @@ -1,157 +0,0 @@ -defmodule Towerops.HoneybadgerNoticeFilterTest do - use ExUnit.Case, async: true - - import Swoosh.TestAssertions - - alias Towerops.HoneybadgerNoticeFilter - - defp build_notice(attrs \\ %{}) do - default_error = %{ - class: "RuntimeError", - message: "something went wrong", - backtrace: [ - %{file: "lib/towerops_web/live/dashboard_live.ex", number: 42, method: "mount/3"}, - %{file: "lib/phoenix_live_view/channel.ex", number: 100, method: "handle_info/2"} - ], - tags: [], - fingerprint: "" - } - - default_server = %{ - environment_name: :test, - hostname: "test-host", - project_root: "/app", - revision: nil - } - - %Honeybadger.Notice{ - error: Map.merge(default_error, Map.get(attrs, :error, %{})), - server: Map.merge(default_server, Map.get(attrs, :server, %{})), - request: Map.get(attrs, :request, %{}), - notifier: %{name: "honeybadger-elixir", url: "", version: "0.0.0"}, - breadcrumbs: %{}, - correlation_context: %{} - } - end - - describe "filter/1" do - test "returns the notice unchanged so Honeybadger still receives it" do - notice = build_notice() - - result = HoneybadgerNoticeFilter.filter(notice) - - assert result == notice - end - - test "sends an email with the error class and message" do - notice = build_notice(%{error: %{class: "ArgumentError", message: "bad argument"}}) - - HoneybadgerNoticeFilter.filter(notice) - - assert_email_sent(fn email -> - assert email.to == [{"", "graham@mcintire.me"}] - assert email.subject =~ "ArgumentError" - assert email.subject =~ "bad argument" - end) - end - - test "email body contains the full stacktrace" do - notice = - build_notice(%{ - error: %{ - backtrace: [ - %{file: "lib/my_app/worker.ex", number: 10, method: "perform/1"}, - %{file: "lib/oban/queue/executor.ex", number: 55, method: "safe_call/1"} - ] - } - }) - - HoneybadgerNoticeFilter.filter(notice) - - assert_email_sent(fn email -> - body = email.text_body - assert body =~ "lib/my_app/worker.ex:10 in perform/1" - assert body =~ "lib/oban/queue/executor.ex:55 in safe_call/1" - end) - end - - test "email body includes server environment info" do - notice = - build_notice(%{ - server: %{environment_name: :prod, hostname: "web-1.towerops.net"} - }) - - HoneybadgerNoticeFilter.filter(notice) - - assert_email_sent(fn email -> - body = email.text_body - assert body =~ "prod" - assert body =~ "web-1.towerops.net" - end) - end - - test "email is sent from the configured mailer_from address" do - notice = build_notice() - - HoneybadgerNoticeFilter.filter(notice) - - assert_email_sent(fn email -> - assert email.from == {"Towerops", "hi@towerops.net"} - end) - end - - test "drops notice with port_died error message (no email, no report)" do - notice = - build_notice(%{ - error: %{ - class: "ErlangError", - message: "Erlang error: {:port_died, :normal}" - } - }) - - assert HoneybadgerNoticeFilter.filter(notice) == nil - assert_no_email_sent() - end - - test "drops notice with write_failed and epipe error message" do - notice = - build_notice(%{ - error: %{ - class: "ErlangError", - message: "{:write_failed, :standard_io, :enoent, {:no_translation, :unicode, :latin1}} {:device, :epipe}" - } - }) - - assert HoneybadgerNoticeFilter.filter(notice) == nil - assert_no_email_sent() - end - - test "does not drop write_failed without epipe" do - notice = - build_notice(%{ - error: %{ - class: "ErlangError", - message: "{:write_failed, :standard_io, :enoent}" - } - }) - - result = HoneybadgerNoticeFilter.filter(notice) - - assert result == notice - end - - test "does not drop unrelated errors" do - notice = - build_notice(%{ - error: %{ - class: "RuntimeError", - message: "something totally unrelated" - } - }) - - result = HoneybadgerNoticeFilter.filter(notice) - - assert result == notice - end - end -end