fix(otel): drop opentelemetry_ecto + disable OTLP on Elixir pods
Ecto auto-instrumentation created one span per query, so Oban jobs running thousands of DB ops produced traces with 8k+ child spans. Tempo's compactor couldn't ingest them — it stalled long enough to miss heartbeats, got auto-forgotten from the ring, and crash-looped. The Elixir grpcbox exporter then blew up trying to reconnect. Stops the cascade: OTLP endpoint removed from prop and prop-backfill Deployments, opentelemetry_ecto removed from deps, Phoenix/Oban/Bandit auto-instrumentation retained. Rust workers still export to Tempo.
This commit is contained in:
parent
5f4c02fb46
commit
1a1ba460e4
5 changed files with 13 additions and 17 deletions
|
|
@ -68,8 +68,7 @@ spec:
|
|||
value: "prop.w5isp.com"
|
||||
- name: HRRR_BASE_URL
|
||||
value: "http://skippy.w5isp.com:8080"
|
||||
- name: OTEL_EXPORTER_OTLP_ENDPOINT
|
||||
value: "http://otel-collector.observability.svc.cluster.local:4317"
|
||||
# OTLP disabled — see deployment.yaml for rationale.
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: prop-secrets
|
||||
|
|
|
|||
|
|
@ -61,11 +61,11 @@ spec:
|
|||
value: "5000"
|
||||
- name: HRRR_BASE_URL
|
||||
value: "http://skippy.w5isp.com:8080"
|
||||
# Elixir OTLP/gRPC trace export to the cluster OTel
|
||||
# Collector. See vntx-infra/observability/ and
|
||||
# config/runtime.exs for wiring. Unset disables exports.
|
||||
- name: OTEL_EXPORTER_OTLP_ENDPOINT
|
||||
value: "http://otel-collector.observability.svc.cluster.local:4317"
|
||||
# OTLP export disabled: opentelemetry_ecto emits one span per
|
||||
# query, and Oban jobs that run thousands of queries produce
|
||||
# traces Tempo can't ingest (8k+ spans, crashes compactor).
|
||||
# Rust workers still export. Re-enable after opentelemetry_ecto
|
||||
# is removed or filtered.
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: prop-secrets
|
||||
|
|
|
|||
|
|
@ -12,14 +12,11 @@ defmodule Microwaveprop.Application do
|
|||
topologies = Application.get_env(:libcluster, :topologies, [])
|
||||
|
||||
# Auto-attach OpenTelemetry instrumentation handlers for Phoenix,
|
||||
# Ecto, Oban, and Bandit. These wire into the telemetry events
|
||||
# those libraries already emit; no call-site changes required.
|
||||
# The OTLP exporter config lives in `config :opentelemetry` in
|
||||
# config/runtime.exs — when OTEL_EXPORTER_OTLP_ENDPOINT is unset
|
||||
# the exporter noops and these attaches are cheap.
|
||||
# Oban, and Bandit. Ecto instrumentation intentionally omitted —
|
||||
# one span per query produces Oban-job traces with thousands of
|
||||
# spans that overwhelm Tempo ingestion.
|
||||
OpentelemetryPhoenix.setup(adapter: :bandit)
|
||||
OpentelemetryBandit.setup()
|
||||
OpentelemetryEcto.setup([:microwaveprop, :repo], db_statement: :enabled)
|
||||
OpentelemetryOban.setup()
|
||||
|
||||
children = [
|
||||
|
|
|
|||
7
mix.exs
7
mix.exs
|
|
@ -69,13 +69,14 @@ defmodule Microwaveprop.MixProject do
|
|||
{:prom_ex, "~> 1.11"},
|
||||
# OpenTelemetry — traces only. Exporter ships via OTLP/gRPC to
|
||||
# the cluster Collector at OTEL_EXPORTER_OTLP_ENDPOINT. The
|
||||
# *_phoenix / *_ecto / *_oban helper packages auto-instrument
|
||||
# those libraries via their telemetry events.
|
||||
# *_phoenix / *_oban / *_bandit helper packages auto-instrument
|
||||
# those libraries via their telemetry events. opentelemetry_ecto
|
||||
# deliberately omitted: one span per query produced Oban-job
|
||||
# traces with thousands of child spans that crashed Tempo.
|
||||
{:opentelemetry, "~> 1.5"},
|
||||
{:opentelemetry_api, "~> 1.4"},
|
||||
{:opentelemetry_exporter, "~> 1.8"},
|
||||
{:opentelemetry_phoenix, "~> 2.0"},
|
||||
{:opentelemetry_ecto, "~> 1.2"},
|
||||
{:opentelemetry_oban, "~> 1.1"},
|
||||
{:opentelemetry_bandit, "~> 0.2"},
|
||||
{:gettext, "~> 0.26"},
|
||||
|
|
|
|||
1
mix.lock
1
mix.lock
|
|
@ -53,7 +53,6 @@
|
|||
"opentelemetry": {:hex, :opentelemetry, "1.7.0", "20d0f12d3d1c398d3670fd44fd1a7c495dd748ab3e5b692a7906662e2fb1a38a", [:rebar3], [{:opentelemetry_api, "~> 1.5.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}], "hexpm", "a9173b058c4549bf824cbc2f1d2fa2adc5cdedc22aa3f0f826951187bbd53131"},
|
||||
"opentelemetry_api": {:hex, :opentelemetry_api, "1.5.0", "1a676f3e3340cab81c763e939a42e11a70c22863f645aa06aafefc689b5550cf", [:mix, :rebar3], [], "hexpm", "f53ec8a1337ae4a487d43ac89da4bd3a3c99ddf576655d071deed8b56a2d5dda"},
|
||||
"opentelemetry_bandit": {:hex, :opentelemetry_bandit, "0.3.0", "2c242dfdaabd747c75f4d8331fc9c17cfc9fb1db0638309762a4fcfa6d49a147", [:mix], [{:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.3", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_semantic_conventions, "~> 1.27", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}, {:otel_http, "~> 0.2", [hex: :otel_http, repo: "hexpm", optional: false]}, {:plug, ">= 1.15.0", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.2", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "5aa12378f5ff7cc3368f02905693571833f9449df86211fd99f4d764720cff60"},
|
||||
"opentelemetry_ecto": {:hex, :opentelemetry_ecto, "1.2.0", "2382cb47ddc231f953d3b8263ed029d87fbf217915a1da82f49159d122b64865", [:mix], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_process_propagator, "~> 0.2", [hex: :opentelemetry_process_propagator, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "70dfa2e79932e86f209df00e36c980b17a32f82d175f0068bf7ef9a96cf080cf"},
|
||||
"opentelemetry_exporter": {:hex, :opentelemetry_exporter, "1.10.0", "972e142392dbfa679ec959914664adefea38399e4f56ceba5c473e1cabdbad79", [:rebar3], [{:grpcbox, ">= 0.0.0", [hex: :grpcbox, repo: "hexpm", optional: false]}, {:opentelemetry, "~> 1.7.0", [hex: :opentelemetry, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.5.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:tls_certificate_check, "~> 1.18", [hex: :tls_certificate_check, repo: "hexpm", optional: false]}], "hexpm", "33a116ed7304cb91783f779dec02478f887c87988077bfd72840f760b8d4b952"},
|
||||
"opentelemetry_oban": {:hex, :opentelemetry_oban, "1.2.0", "4cbdfaa533deda678b6b534ee806636359107b70b0687accd57074b1c440d383", [:mix], [{:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:oban, "~> 2.0", [hex: :oban, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.2", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_semantic_conventions, "~> 1.27", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}, {:opentelemetry_telemetry, "~> 1.1", [hex: :opentelemetry_telemetry, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "0c96790ad689ba6af6ca64ca2a148543c48090a2ae1c46cce2fd1b32d71b5f71"},
|
||||
"opentelemetry_phoenix": {:hex, :opentelemetry_phoenix, "2.0.1", "c664cdef205738cffcd409b33599439a4ffb2035ef6e21a77927ac1da90463cb", [:mix], [{:nimble_options, "~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.4", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_process_propagator, "~> 0.3", [hex: :opentelemetry_process_propagator, repo: "hexpm", optional: false]}, {:opentelemetry_semantic_conventions, "~> 1.27", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}, {:opentelemetry_telemetry, "~> 1.1", [hex: :opentelemetry_telemetry, repo: "hexpm", optional: false]}, {:otel_http, "~> 0.2", [hex: :otel_http, repo: "hexpm", optional: false]}, {:plug, ">= 1.11.0", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a24fdccdfa6b890c8892c6366beab4a15a27ec0c692b0f77ec2a862e7b235f6e"},
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue