Add email notifications for production errors
Implements error email notifications that: - Send email only for first occurrence of each unique error - Only send in production environment (dev/test are silent) - Include error details, location, and link to ErrorTracker UI - Use existing Resend/Swoosh configuration for delivery - Email sent to graham@mcintire.me Integrates with ErrorTracker via telemetry events.
This commit is contained in:
parent
812163fa92
commit
0c9087b7f1
3 changed files with 323 additions and 0 deletions
|
|
@ -80,6 +80,9 @@ defmodule Aprsme.Application do
|
|||
opts = [strategy: :one_for_one, name: Aprsme.Supervisor]
|
||||
{:ok, sup} = Supervisor.start_link(children, opts)
|
||||
|
||||
# Attach error notification telemetry handler
|
||||
Aprsme.ErrorNotifier.attach()
|
||||
|
||||
# Now that the Repo is started, run the refresh in a background task
|
||||
# Skip in test environment to avoid DBConnection.OwnershipError
|
||||
env = Application.get_env(:aprsme, :env)
|
||||
|
|
|
|||
147
lib/aprsme/error_notifier.ex
Normal file
147
lib/aprsme/error_notifier.ex
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
defmodule Aprsme.ErrorNotifier do
|
||||
@moduledoc """
|
||||
Sends email notifications when errors occur in production.
|
||||
|
||||
Integrates with ErrorTracker to send emails only for the first occurrence
|
||||
of each unique error (based on error signature).
|
||||
"""
|
||||
|
||||
import Swoosh.Email
|
||||
|
||||
require Logger
|
||||
|
||||
@doc """
|
||||
Attaches telemetry handler for error occurrences.
|
||||
Called during application startup.
|
||||
"""
|
||||
def attach do
|
||||
:telemetry.attach(
|
||||
"aprsme-error-email-notifier",
|
||||
[:error_tracker, :occurrence, :created],
|
||||
&__MODULE__.handle_error_occurrence/4,
|
||||
nil
|
||||
)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Telemetry handler for error occurrences.
|
||||
Sends email if this is the first occurrence and we're in production.
|
||||
"""
|
||||
def handle_error_occurrence(_event, _measurements, metadata, _config) do
|
||||
occurrence = metadata.occurrence
|
||||
|
||||
# Only send email for first occurrence in production
|
||||
if should_send_email?(occurrence) do
|
||||
send_error_email(occurrence)
|
||||
end
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
defp should_send_email?(occurrence) do
|
||||
occurrence.occurrence_count == 1 && production?()
|
||||
end
|
||||
|
||||
defp production? do
|
||||
Application.get_env(:aprsme, :env) == :prod
|
||||
end
|
||||
|
||||
defp send_error_email(occurrence) do
|
||||
# Extract file and line information for the subject line
|
||||
first_line = get_first_stack_line(occurrence)
|
||||
file = if first_line, do: first_line.file, else: "unknown_file"
|
||||
line = if first_line, do: first_line.line, else: "?"
|
||||
error_name = String.slice(occurrence.reason, 0, 80) || "Unknown error"
|
||||
|
||||
subject = "[APRS.me] Error: #{error_name} - #{file} - #{line}"
|
||||
|
||||
email =
|
||||
new()
|
||||
|> to({"", "graham@mcintire.me"})
|
||||
|> from({"APRS.me Alerts", "alerts@aprs.me"})
|
||||
|> subject(subject)
|
||||
|> html_body(occurrence_email_html(occurrence))
|
||||
|
||||
case Aprsme.Mailer.deliver(email) do
|
||||
{:ok, _} ->
|
||||
Logger.info("Error notification email sent successfully for error #{occurrence.error_id}")
|
||||
{:ok, "Email sent successfully"}
|
||||
|
||||
{:error, reason} ->
|
||||
Logger.error("Failed to send error notification: #{inspect(reason)}")
|
||||
{:error, reason}
|
||||
end
|
||||
end
|
||||
|
||||
defp occurrence_email_html(occurrence) do
|
||||
# Extract the first line from the stacktrace for the error location
|
||||
first_line = get_first_stack_line(occurrence)
|
||||
|
||||
# Extract useful information from the stacktrace line
|
||||
error_location =
|
||||
if first_line do
|
||||
"#{first_line.module}.#{first_line.function} (#{first_line.file}:#{first_line.line})"
|
||||
else
|
||||
"Unknown location"
|
||||
end
|
||||
|
||||
# Extract useful context information
|
||||
view = occurrence.context["live_view.view"] || "Unknown view"
|
||||
path = occurrence.context["request.path"] || "Unknown path"
|
||||
|
||||
# Get the error URL
|
||||
error_url = "https://aprs.me/dev/error-tracker/errors/#{occurrence.error_id}"
|
||||
|
||||
# Escape all user-controlled fields to prevent XSS
|
||||
escaped_error_id = occurrence.error_id |> Phoenix.HTML.html_escape() |> Phoenix.HTML.safe_to_string()
|
||||
|
||||
escaped_reason =
|
||||
occurrence.reason |> String.slice(0..199) |> Phoenix.HTML.html_escape() |> Phoenix.HTML.safe_to_string()
|
||||
|
||||
escaped_location = error_location |> Phoenix.HTML.html_escape() |> Phoenix.HTML.safe_to_string()
|
||||
escaped_view = view |> Phoenix.HTML.html_escape() |> Phoenix.HTML.safe_to_string()
|
||||
escaped_path = path |> Phoenix.HTML.html_escape() |> Phoenix.HTML.safe_to_string()
|
||||
escaped_url = error_url |> Phoenix.HTML.html_escape() |> Phoenix.HTML.safe_to_string()
|
||||
|
||||
"""
|
||||
<div style="max-width: 600px; margin: 0 auto; padding: 20px; font-family: system-ui, -apple-system, sans-serif;">
|
||||
<div style="background-color: white; border-radius: 8px; padding: 24px; box-shadow: 0 1px 3px 0 rgb(0 0 0 / 0.1);">
|
||||
<h1 style="color: #dc2626; font-size: 24px; font-weight: bold; margin-bottom: 16px;">
|
||||
New Error Detected
|
||||
</h1>
|
||||
<p style="color: #374151; font-size: 16px; line-height: 24px; margin-bottom: 24px;">
|
||||
ErrorTracker has detected a new error in production:
|
||||
</p>
|
||||
|
||||
<div style="background-color: #f9fafb; border-radius: 6px; padding: 16px; margin-bottom: 24px;">
|
||||
<p><strong>Error ID:</strong> #{escaped_error_id}</p>
|
||||
<p><strong>Reason:</strong> #{escaped_reason}</p>
|
||||
<p><strong>Location:</strong> #{escaped_location}</p>
|
||||
<p><strong>View:</strong> #{escaped_view}</p>
|
||||
<p><strong>Request Path:</strong> #{escaped_path}</p>
|
||||
<p><strong>Time:</strong> #{format_time()}</p>
|
||||
</div>
|
||||
|
||||
<p style="margin-bottom: 24px;">
|
||||
<a href="#{escaped_url}"
|
||||
style="display: inline-block; background-color: #dc2626; color: white; font-weight: 500;
|
||||
padding: 8px 16px; border-radius: 4px; text-decoration: none;">
|
||||
View Error Details
|
||||
</a>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
end
|
||||
|
||||
defp get_first_stack_line(occurrence) do
|
||||
case occurrence.stacktrace do
|
||||
%{lines: [first | _]} -> first
|
||||
_ -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp format_time do
|
||||
DateTime.to_string(DateTime.utc_now())
|
||||
end
|
||||
end
|
||||
173
test/aprsme/error_notifier_test.exs
Normal file
173
test/aprsme/error_notifier_test.exs
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
defmodule Aprsme.ErrorNotifierTest do
|
||||
use Aprsme.DataCase, async: false
|
||||
|
||||
import Swoosh.TestAssertions
|
||||
|
||||
alias Aprsme.ErrorNotifier
|
||||
alias Swoosh.Adapters.Test
|
||||
|
||||
setup do
|
||||
# Store original config
|
||||
original_env = Application.get_env(:aprsme, :env)
|
||||
original_mailer = Application.get_env(:aprsme, Aprsme.Mailer)
|
||||
|
||||
on_exit(fn ->
|
||||
Application.put_env(:aprsme, :env, original_env)
|
||||
Application.put_env(:aprsme, Aprsme.Mailer, original_mailer)
|
||||
end)
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
describe "handle_error_occurrence/4" do
|
||||
test "sends email for first occurrence of an error in production" do
|
||||
# Set environment to production
|
||||
Application.put_env(:aprsme, :env, :prod)
|
||||
Application.put_env(:aprsme, Aprsme.Mailer, adapter: Test)
|
||||
|
||||
occurrence_data = %{
|
||||
error_id: "test-error-123",
|
||||
reason: "RuntimeError: Something went wrong",
|
||||
occurrence_count: 1,
|
||||
stacktrace: %{
|
||||
lines: [
|
||||
%{
|
||||
module: "Aprsme.SomeModule",
|
||||
function: "some_function/2",
|
||||
file: "lib/aprsme/some_module.ex",
|
||||
line: 42
|
||||
}
|
||||
]
|
||||
},
|
||||
context: %{
|
||||
"live_view.view" => "AprsmeWeb.MapLive.Index",
|
||||
"request.path" => "/map"
|
||||
}
|
||||
}
|
||||
|
||||
measurements = %{}
|
||||
metadata = %{occurrence: occurrence_data}
|
||||
|
||||
ErrorNotifier.handle_error_occurrence(
|
||||
[:error_tracker, :occurrence, :created],
|
||||
measurements,
|
||||
metadata,
|
||||
nil
|
||||
)
|
||||
|
||||
assert_email_sent(fn email ->
|
||||
assert email.to == [{"", "graham@mcintire.me"}]
|
||||
assert email.subject =~ "Something went wrong"
|
||||
assert email.subject =~ "lib/aprsme/some_module.ex"
|
||||
assert email.subject =~ "42"
|
||||
assert email.html_body =~ "test-error-123"
|
||||
assert email.html_body =~ "RuntimeError: Something went wrong"
|
||||
assert email.html_body =~ "Aprsme.SomeModule.some_function/2"
|
||||
assert email.html_body =~ "https://aprs.me/dev/error-tracker/errors/test-error-123"
|
||||
end)
|
||||
end
|
||||
|
||||
test "does not send email for subsequent occurrences" do
|
||||
Application.put_env(:aprsme, :env, :prod)
|
||||
Application.put_env(:aprsme, Aprsme.Mailer, adapter: Test)
|
||||
|
||||
occurrence_data = %{
|
||||
error_id: "test-error-123",
|
||||
reason: "RuntimeError: Something went wrong",
|
||||
occurrence_count: 5,
|
||||
stacktrace: %{lines: []},
|
||||
context: %{}
|
||||
}
|
||||
|
||||
measurements = %{}
|
||||
metadata = %{occurrence: occurrence_data}
|
||||
|
||||
ErrorNotifier.handle_error_occurrence(
|
||||
[:error_tracker, :occurrence, :created],
|
||||
measurements,
|
||||
metadata,
|
||||
nil
|
||||
)
|
||||
|
||||
assert_no_email_sent()
|
||||
end
|
||||
|
||||
test "does not send email in development environment" do
|
||||
Application.put_env(:aprsme, :env, :dev)
|
||||
Application.put_env(:aprsme, Aprsme.Mailer, adapter: Test)
|
||||
|
||||
occurrence_data = %{
|
||||
error_id: "test-error-123",
|
||||
reason: "RuntimeError: Something went wrong",
|
||||
occurrence_count: 1,
|
||||
stacktrace: %{lines: []},
|
||||
context: %{}
|
||||
}
|
||||
|
||||
measurements = %{}
|
||||
metadata = %{occurrence: occurrence_data}
|
||||
|
||||
ErrorNotifier.handle_error_occurrence(
|
||||
[:error_tracker, :occurrence, :created],
|
||||
measurements,
|
||||
metadata,
|
||||
nil
|
||||
)
|
||||
|
||||
assert_no_email_sent()
|
||||
end
|
||||
|
||||
test "does not send email in test environment" do
|
||||
Application.put_env(:aprsme, :env, :test)
|
||||
Application.put_env(:aprsme, Aprsme.Mailer, adapter: Test)
|
||||
|
||||
occurrence_data = %{
|
||||
error_id: "test-error-123",
|
||||
reason: "RuntimeError: Something went wrong",
|
||||
occurrence_count: 1,
|
||||
stacktrace: %{lines: []},
|
||||
context: %{}
|
||||
}
|
||||
|
||||
measurements = %{}
|
||||
metadata = %{occurrence: occurrence_data}
|
||||
|
||||
ErrorNotifier.handle_error_occurrence(
|
||||
[:error_tracker, :occurrence, :created],
|
||||
measurements,
|
||||
metadata,
|
||||
nil
|
||||
)
|
||||
|
||||
assert_no_email_sent()
|
||||
end
|
||||
|
||||
test "handles missing stacktrace gracefully" do
|
||||
Application.put_env(:aprsme, :env, :prod)
|
||||
Application.put_env(:aprsme, Mailer, adapter: Test)
|
||||
|
||||
occurrence_data = %{
|
||||
error_id: "test-error-123",
|
||||
reason: "RuntimeError: Something went wrong",
|
||||
occurrence_count: 1,
|
||||
stacktrace: %{lines: []},
|
||||
context: %{}
|
||||
}
|
||||
|
||||
measurements = %{}
|
||||
metadata = %{occurrence: occurrence_data}
|
||||
|
||||
ErrorNotifier.handle_error_occurrence(
|
||||
[:error_tracker, :occurrence, :created],
|
||||
measurements,
|
||||
metadata,
|
||||
nil
|
||||
)
|
||||
|
||||
assert_email_sent(fn email ->
|
||||
assert email.subject =~ "unknown_file"
|
||||
assert email.html_body =~ "Unknown location"
|
||||
end)
|
||||
end
|
||||
end
|
||||
end
|
||||
Loading…
Add table
Reference in a new issue