Add ErrorTracker, remove Honeybadger email notice filter

Replace the HoneybadgerNoticeFilter that emailed raw stacktraces on
every exception with ErrorTracker for self-hosted error tracking.
Honeybadger itself is retained. ErrorTracker dashboard is mounted at
/admin/errors behind the superuser auth wall.

config/config.exs

@@ -10,6 +10,10 @@ import Config
 # Configure time zone database to use tzdata for IANA timezone support
 config :elixir, :time_zone_database, Tzdata.TimeZoneDatabase
 
+config :error_tracker,
+  repo: Towerops.Repo,
+  otp_app: :towerops
+
 # Configure esbuild (the version is required)
 config :esbuild,
   version: "0.25.4",
@@ -25,8 +29,7 @@ config :honeybadger,
   environment_name: config_env(),
   insights_enabled: true,
   use_logger: true,
-  filter: Towerops.HoneybadgerFilter,
-  notice_filter: Towerops.HoneybadgerNoticeFilter
+  filter: Towerops.HoneybadgerFilter
 
 # Configure Elixir's Logger
 config :logger, :default_formatter,

config/test.exs

@@ -5,6 +5,8 @@ config :argon2_elixir,
   t_cost: 1,
   m_cost: 8
 
+config :error_tracker, enabled: false
+
 # Suppress log output during tests (tests that need logs use capture_log)
 config :logger, :default_handler, false
 config :logger, level: :error
@@ -70,7 +72,6 @@ config :towerops, :agent_webhook_secret, "test-webhook-secret"
 # Set environment identifier for runtime checks
 config :towerops, :env, :test
 
-# Use mocks for testing
 config :towerops,
   ping_module: Towerops.Monitoring.PingMock,
   poller_module: Towerops.Snmp.PollerMock,

lib/towerops/honeybadger_notice_filter.ex

@@ -1,98 +0,0 @@
-defmodule Towerops.HoneybadgerNoticeFilter do
-  @moduledoc """
-  Honeybadger notice filter that emails a raw stacktrace to the
-  ops contact whenever an exception is reported.
-
-  Implements `Honeybadger.NoticeFilter` — receives the fully-built
-  notice, sends an email, and returns the notice unchanged so it
-  still reaches Honeybadger.
-  """
-
-  @behaviour Honeybadger.NoticeFilter
-
-  import Swoosh.Email
-
-  alias Towerops.Mailer
-
-  require Logger
-
-  @recipient "graham@mcintire.me"
-
-  @impl Honeybadger.NoticeFilter
-  def filter(%Honeybadger.Notice{} = notice) do
-    if shutdown_error?(notice) do
-      nil
-    else
-      send_error_email(notice)
-      notice
-    end
-  end
-
-  defp shutdown_error?(%Honeybadger.Notice{error: error}) do
-    msg = error.message || ""
-
-    String.contains?(msg, "port_died") or
-      (String.contains?(msg, "write_failed") and String.contains?(msg, "epipe")) or
-      String.contains?(msg, "broken pipe (epipe)")
-  end
-
-  defp send_error_email(%Honeybadger.Notice{error: error, server: server}) do
-    from_address = Application.get_env(:towerops, :mailer_from, {"Towerops", "hi@towerops.net"})
-    subject = "[#{server.environment_name}] #{error.class}: #{truncate(error.message, 120)}"
-
-    body = format_body(error, server)
-
-    email =
-      new()
-      |> to(@recipient)
-      |> from(from_address)
-      |> subject(subject)
-      |> text_body(body)
-
-    try do
-      case Mailer.deliver(email) do
-        {:ok, _} ->
-          :ok
-
-        {:error, reason} ->
-          Logger.warning("Failed to send error email: #{inspect(reason)}")
-          :error
-      end
-    rescue
-      e ->
-        Logger.warning("Error email delivery crashed: #{Exception.message(e)}")
-        :error
-    end
-  end
-
-  defp format_body(error, server) do
-    backtrace = format_backtrace(error.backtrace)
-
-    """
-    Exception: #{error.class}
-    Message: #{error.message}
-
-    Environment: #{server.environment_name}
-    Hostname: #{server.hostname}
-
-    Stacktrace:
-    #{backtrace}
-    """
-  end
-
-  defp format_backtrace(backtrace) when is_list(backtrace) do
-    Enum.map_join(backtrace, "\n", fn entry ->
-      "  #{entry.file}:#{entry.number} in #{entry.method}"
-    end)
-  end
-
-  defp format_backtrace(_), do: "  (no stacktrace available)"
-
-  defp truncate(nil, _max), do: "(no message)"
-
-  defp truncate(message, max) when byte_size(message) > max do
-    String.slice(message, 0, max) <> "..."
-  end
-
-  defp truncate(message, _max), do: message
-end

lib/towerops_web/router.ex

@@ -3,6 +3,7 @@ defmodule ToweropsWeb.Router do
 
   use ToweropsWeb, :router
   use Honeybadger.Plug
+  use ErrorTracker.Web, :router
 
   import Oban.Web.Router
   import Phoenix.LiveDashboard.Router
@@ -216,6 +217,7 @@ defmodule ToweropsWeb.Router do
     pipe_through [:browser, :require_authenticated_user, :require_superuser]
 
     oban_dashboard("/oban")
+    error_tracker_dashboard("/errors")
     get "/headers", ToweropsWeb.DebugController, :headers
   end
 

mix.exs

@@ -80,6 +80,7 @@ defmodule Towerops.MixProject do
       {:absinthe, "~> 1.7"},
       {:absinthe_plug, "~> 1.5"},
       {:honeybadger, "~> 0.24"},
+      {:error_tracker, "~> 0.7"},
       {:stream_data, "~> 1.1", only: :test},
       {:tzdata, "~> 1.1"},
       {:styler, "~> 1.10", only: [:dev, :test], runtime: false},

mix.lock

@@ -21,6 +21,7 @@
   "elixir_make": {:hex, :elixir_make, "0.9.0", "6484b3cd8c0cee58f09f05ecaf1a140a8c97670671a6a0e7ab4dc326c3109726", [:mix], [], "hexpm", "db23d4fd8b757462ad02f8aa73431a426fe6671c80b200d9710caf3d1dd0ffdb"},
   "eqrcode": {:hex, :eqrcode, "0.2.1", "d12838813e8fc87b8940cc05f9baadb189031f6009facdc56ff074375ec73b6e", [:mix], [], "hexpm", "d5828a222b904c68360e7dc2a40c3ef33a1328b7c074583898040f389f928025"},
   "erlex": {:hex, :erlex, "0.2.8", "cd8116f20f3c0afe376d1e8d1f0ae2452337729f68be016ea544a72f767d9c12", [:mix], [], "hexpm", "9d66ff9fedf69e49dc3fd12831e12a8a37b76f8651dd21cd45fcf5561a8a7590"},
+  "error_tracker": {:hex, :error_tracker, "0.7.1", "074f10bfb01961bd0d51eb010c6ddbe13bf40eb4ddfbb25538680ddb520ab1aa", [:mix], [{:ecto, "~> 3.13", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.13", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, ">= 0.0.0", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:igniter, "~> 0.5", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:myxql, ">= 0.0.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:phoenix_ecto, "~> 4.6", [hex: :phoenix_ecto, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "cc77434c084d100e17dcc5ccea6bed05b15748186df5207e21fb761651bf4e0d"},
   "esbuild": {:hex, :esbuild, "0.10.0", "b0aa3388a1c23e727c5a3e7427c932d89ee791746b0081bbe56103e9ef3d291f", [:mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "468489cda427b974a7cc9f03ace55368a83e1a7be12fba7e30969af78e5f8c70"},
   "expo": {:hex, :expo, "1.1.1", "4202e1d2ca6e2b3b63e02f69cfe0a404f77702b041d02b58597c00992b601db5", [:mix], [], "hexpm", "5fb308b9cb359ae200b7e23d37c76978673aa1b06e2b3075d814ce12c5811640"},
   "file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"},
@@ -62,7 +63,7 @@
   "phoenix_html": {:hex, :phoenix_html, "4.3.0", "d3577a5df4b6954cd7890c84d955c470b5310bb49647f0a114a6eeecc850f7ad", [:mix], [], "hexpm", "3eaa290a78bab0f075f791a46a981bbe769d94bc776869f4f3063a14f30497ad"},
   "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.8.7", "405880012cb4b706f26dd1c6349125bfc903fb9e44d1ea668adaf4e04d4884b7", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.5", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:ecto_sqlite3_extras, "~> 1.1.7 or ~> 1.2.0", [hex: :ecto_sqlite3_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.19 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "3a8625cab39ec261d48a13b7468dc619c0ede099601b084e343968309bd4d7d7"},
   "phoenix_live_reload": {:hex, :phoenix_live_reload, "1.6.2", "b18b0773a1ba77f28c52decbb0f10fd1ac4d3ae5b8632399bbf6986e3b665f62", [:mix], [{:file_system, "~> 0.2.10 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "d1f89c18114c50d394721365ffb428cce24f1c13de0467ffa773e2ff4a30d5b9"},
-  "phoenix_live_view": {:hex, :phoenix_live_view, "1.1.23", "5e41c63377d508dc9b22dcebcefa0a3a243695932f72eac6003831fd4ed96165", [:mix], [{:igniter, ">= 0.6.16 and < 1.0.0-0", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:lazy_html, "~> 0.1.0", [hex: :lazy_html, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0 or ~> 1.8.0-rc", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.15", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f5296ef67b41da7fe345e139ebaa621ceb5e7e84e37c89a12574cc0536a80604"},
+  "phoenix_live_view": {:hex, :phoenix_live_view, "1.1.24", "1a000a048d5971b61a9efe29a3c4144ca955afd42224998d841c5011a5354838", [:mix], [{:igniter, ">= 0.6.16 and < 1.0.0-0", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:lazy_html, "~> 0.1.0", [hex: :lazy_html, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0 or ~> 1.8.0-rc", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.15", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "0c724e6c65f197841cac49d73be4e0f9b93a7711eaa52d2d4d1b9f859c329267"},
   "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.2.0", "ff3a5616e1bed6804de7773b92cbccfc0b0f473faf1f63d7daf1206c7aeaaa6f", [:mix], [], "hexpm", "adc313a5bf7136039f63cfd9668fde73bba0765e0614cba80c06ac9460ff3e96"},
   "phoenix_pubsub_redis": {:hex, :phoenix_pubsub_redis, "3.0.1", "d4d856b1e57a21358e448543e1d091e07e83403dde4383b8be04ed9d2c201cbc", [:mix], [{:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:poolboy, "~> 1.5.1 or ~> 1.6", [hex: :poolboy, repo: "hexpm", optional: false]}, {:redix, "~> 0.10.0 or ~> 1.0", [hex: :redix, repo: "hexpm", optional: false]}], "hexpm", "0b36a17ff6e9a56159f8df8933d62b5c1f0695eae995a02e0c86c035ace6a309"},
   "phoenix_template": {:hex, :phoenix_template, "1.0.4", "e2092c132f3b5e5b2d49c96695342eb36d0ed514c5b252a77048d5969330d639", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "2c0c81f0e5c6753faf5cca2f229c9709919aba34fab866d3bc05060c9c444206"},

priv/repo/migrations/20260217140215_add_error_tracker.exs

@@ -0,0 +1,6 @@
+defmodule Towerops.Repo.Migrations.AddErrorTracker do
+  use Ecto.Migration
+
+  def up, do: ErrorTracker.Migration.up(version: 5)
+  def down, do: ErrorTracker.Migration.down(version: 1)
+end

test/towerops/honeybadger_notice_filter_test.exs

@@ -1,157 +0,0 @@
-defmodule Towerops.HoneybadgerNoticeFilterTest do
-  use ExUnit.Case, async: true
-
-  import Swoosh.TestAssertions
-
-  alias Towerops.HoneybadgerNoticeFilter
-
-  defp build_notice(attrs \\ %{}) do
-    default_error = %{
-      class: "RuntimeError",
-      message: "something went wrong",
-      backtrace: [
-        %{file: "lib/towerops_web/live/dashboard_live.ex", number: 42, method: "mount/3"},
-        %{file: "lib/phoenix_live_view/channel.ex", number: 100, method: "handle_info/2"}
-      ],
-      tags: [],
-      fingerprint: ""
-    }
-
-    default_server = %{
-      environment_name: :test,
-      hostname: "test-host",
-      project_root: "/app",
-      revision: nil
-    }
-
-    %Honeybadger.Notice{
-      error: Map.merge(default_error, Map.get(attrs, :error, %{})),
-      server: Map.merge(default_server, Map.get(attrs, :server, %{})),
-      request: Map.get(attrs, :request, %{}),
-      notifier: %{name: "honeybadger-elixir", url: "", version: "0.0.0"},
-      breadcrumbs: %{},
-      correlation_context: %{}
-    }
-  end
-
-  describe "filter/1" do
-    test "returns the notice unchanged so Honeybadger still receives it" do
-      notice = build_notice()
-
-      result = HoneybadgerNoticeFilter.filter(notice)
-
-      assert result == notice
-    end
-
-    test "sends an email with the error class and message" do
-      notice = build_notice(%{error: %{class: "ArgumentError", message: "bad argument"}})
-
-      HoneybadgerNoticeFilter.filter(notice)
-
-      assert_email_sent(fn email ->
-        assert email.to == [{"", "graham@mcintire.me"}]
-        assert email.subject =~ "ArgumentError"
-        assert email.subject =~ "bad argument"
-      end)
-    end
-
-    test "email body contains the full stacktrace" do
-      notice =
-        build_notice(%{
-          error: %{
-            backtrace: [
-              %{file: "lib/my_app/worker.ex", number: 10, method: "perform/1"},
-              %{file: "lib/oban/queue/executor.ex", number: 55, method: "safe_call/1"}
-            ]
-          }
-        })
-
-      HoneybadgerNoticeFilter.filter(notice)
-
-      assert_email_sent(fn email ->
-        body = email.text_body
-        assert body =~ "lib/my_app/worker.ex:10 in perform/1"
-        assert body =~ "lib/oban/queue/executor.ex:55 in safe_call/1"
-      end)
-    end
-
-    test "email body includes server environment info" do
-      notice =
-        build_notice(%{
-          server: %{environment_name: :prod, hostname: "web-1.towerops.net"}
-        })
-
-      HoneybadgerNoticeFilter.filter(notice)
-
-      assert_email_sent(fn email ->
-        body = email.text_body
-        assert body =~ "prod"
-        assert body =~ "web-1.towerops.net"
-      end)
-    end
-
-    test "email is sent from the configured mailer_from address" do
-      notice = build_notice()
-
-      HoneybadgerNoticeFilter.filter(notice)
-
-      assert_email_sent(fn email ->
-        assert email.from == {"Towerops", "hi@towerops.net"}
-      end)
-    end
-
-    test "drops notice with port_died error message (no email, no report)" do
-      notice =
-        build_notice(%{
-          error: %{
-            class: "ErlangError",
-            message: "Erlang error: {:port_died, :normal}"
-          }
-        })
-
-      assert HoneybadgerNoticeFilter.filter(notice) == nil
-      assert_no_email_sent()
-    end
-
-    test "drops notice with write_failed and epipe error message" do
-      notice =
-        build_notice(%{
-          error: %{
-            class: "ErlangError",
-            message: "{:write_failed, :standard_io, :enoent, {:no_translation, :unicode, :latin1}} {:device, :epipe}"
-          }
-        })
-
-      assert HoneybadgerNoticeFilter.filter(notice) == nil
-      assert_no_email_sent()
-    end
-
-    test "does not drop write_failed without epipe" do
-      notice =
-        build_notice(%{
-          error: %{
-            class: "ErlangError",
-            message: "{:write_failed, :standard_io, :enoent}"
-          }
-        })
-
-      result = HoneybadgerNoticeFilter.filter(notice)
-
-      assert result == notice
-    end
-
-    test "does not drop unrelated errors" do
-      notice =
-        build_notice(%{
-          error: %{
-            class: "RuntimeError",
-            message: "something totally unrelated"
-          }
-        })
-
-      result = HoneybadgerNoticeFilter.filter(notice)
-
-      assert result == notice
-    end
-  end
-end