diff --git a/k8s/deployment-backfill.yaml b/k8s/deployment-backfill.yaml index 5851d82d..58b97786 100644 --- a/k8s/deployment-backfill.yaml +++ b/k8s/deployment-backfill.yaml @@ -68,8 +68,7 @@ spec: value: "prop.w5isp.com" - name: HRRR_BASE_URL value: "http://skippy.w5isp.com:8080" - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: "http://otel-collector.observability.svc.cluster.local:4317" + # OTLP disabled — see deployment.yaml for rationale. envFrom: - secretRef: name: prop-secrets diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index c4f228f3..a4767c0f 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -61,11 +61,11 @@ spec: value: "5000" - name: HRRR_BASE_URL value: "http://skippy.w5isp.com:8080" - # Elixir OTLP/gRPC trace export to the cluster OTel - # Collector. See vntx-infra/observability/ and - # config/runtime.exs for wiring. Unset disables exports. - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: "http://otel-collector.observability.svc.cluster.local:4317" + # OTLP export disabled: opentelemetry_ecto emits one span per + # query, and Oban jobs that run thousands of queries produce + # traces Tempo can't ingest (8k+ spans, crashes compactor). + # Rust workers still export. Re-enable after opentelemetry_ecto + # is removed or filtered. envFrom: - secretRef: name: prop-secrets diff --git a/lib/microwaveprop/application.ex b/lib/microwaveprop/application.ex index 7338e29d..128b6d9f 100644 --- a/lib/microwaveprop/application.ex +++ b/lib/microwaveprop/application.ex @@ -12,14 +12,11 @@ defmodule Microwaveprop.Application do topologies = Application.get_env(:libcluster, :topologies, []) # Auto-attach OpenTelemetry instrumentation handlers for Phoenix, - # Ecto, Oban, and Bandit. These wire into the telemetry events - # those libraries already emit; no call-site changes required. - # The OTLP exporter config lives in `config :opentelemetry` in - # config/runtime.exs — when OTEL_EXPORTER_OTLP_ENDPOINT is unset - # the exporter noops and these attaches are cheap. + # Oban, and Bandit. Ecto instrumentation intentionally omitted — + # one span per query produces Oban-job traces with thousands of + # spans that overwhelm Tempo ingestion. OpentelemetryPhoenix.setup(adapter: :bandit) OpentelemetryBandit.setup() - OpentelemetryEcto.setup([:microwaveprop, :repo], db_statement: :enabled) OpentelemetryOban.setup() children = [ diff --git a/mix.exs b/mix.exs index 4c5f38da..3ecc79f5 100644 --- a/mix.exs +++ b/mix.exs @@ -69,13 +69,14 @@ defmodule Microwaveprop.MixProject do {:prom_ex, "~> 1.11"}, # OpenTelemetry — traces only. Exporter ships via OTLP/gRPC to # the cluster Collector at OTEL_EXPORTER_OTLP_ENDPOINT. The - # *_phoenix / *_ecto / *_oban helper packages auto-instrument - # those libraries via their telemetry events. + # *_phoenix / *_oban / *_bandit helper packages auto-instrument + # those libraries via their telemetry events. opentelemetry_ecto + # deliberately omitted: one span per query produced Oban-job + # traces with thousands of child spans that crashed Tempo. {:opentelemetry, "~> 1.5"}, {:opentelemetry_api, "~> 1.4"}, {:opentelemetry_exporter, "~> 1.8"}, {:opentelemetry_phoenix, "~> 2.0"}, - {:opentelemetry_ecto, "~> 1.2"}, {:opentelemetry_oban, "~> 1.1"}, {:opentelemetry_bandit, "~> 0.2"}, {:gettext, "~> 0.26"}, diff --git a/mix.lock b/mix.lock index f630a0d2..eed32f19 100644 --- a/mix.lock +++ b/mix.lock @@ -53,7 +53,6 @@ "opentelemetry": {:hex, :opentelemetry, "1.7.0", "20d0f12d3d1c398d3670fd44fd1a7c495dd748ab3e5b692a7906662e2fb1a38a", [:rebar3], [{:opentelemetry_api, "~> 1.5.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}], "hexpm", "a9173b058c4549bf824cbc2f1d2fa2adc5cdedc22aa3f0f826951187bbd53131"}, "opentelemetry_api": {:hex, :opentelemetry_api, "1.5.0", "1a676f3e3340cab81c763e939a42e11a70c22863f645aa06aafefc689b5550cf", [:mix, :rebar3], [], "hexpm", "f53ec8a1337ae4a487d43ac89da4bd3a3c99ddf576655d071deed8b56a2d5dda"}, "opentelemetry_bandit": {:hex, :opentelemetry_bandit, "0.3.0", "2c242dfdaabd747c75f4d8331fc9c17cfc9fb1db0638309762a4fcfa6d49a147", [:mix], [{:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.3", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_semantic_conventions, "~> 1.27", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}, {:otel_http, "~> 0.2", [hex: :otel_http, repo: "hexpm", optional: false]}, {:plug, ">= 1.15.0", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.2", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "5aa12378f5ff7cc3368f02905693571833f9449df86211fd99f4d764720cff60"}, - "opentelemetry_ecto": {:hex, :opentelemetry_ecto, "1.2.0", "2382cb47ddc231f953d3b8263ed029d87fbf217915a1da82f49159d122b64865", [:mix], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_process_propagator, "~> 0.2", [hex: :opentelemetry_process_propagator, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "70dfa2e79932e86f209df00e36c980b17a32f82d175f0068bf7ef9a96cf080cf"}, "opentelemetry_exporter": {:hex, :opentelemetry_exporter, "1.10.0", "972e142392dbfa679ec959914664adefea38399e4f56ceba5c473e1cabdbad79", [:rebar3], [{:grpcbox, ">= 0.0.0", [hex: :grpcbox, repo: "hexpm", optional: false]}, {:opentelemetry, "~> 1.7.0", [hex: :opentelemetry, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.5.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:tls_certificate_check, "~> 1.18", [hex: :tls_certificate_check, repo: "hexpm", optional: false]}], "hexpm", "33a116ed7304cb91783f779dec02478f887c87988077bfd72840f760b8d4b952"}, "opentelemetry_oban": {:hex, :opentelemetry_oban, "1.2.0", "4cbdfaa533deda678b6b534ee806636359107b70b0687accd57074b1c440d383", [:mix], [{:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:oban, "~> 2.0", [hex: :oban, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.2", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_semantic_conventions, "~> 1.27", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}, {:opentelemetry_telemetry, "~> 1.1", [hex: :opentelemetry_telemetry, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "0c96790ad689ba6af6ca64ca2a148543c48090a2ae1c46cce2fd1b32d71b5f71"}, "opentelemetry_phoenix": {:hex, :opentelemetry_phoenix, "2.0.1", "c664cdef205738cffcd409b33599439a4ffb2035ef6e21a77927ac1da90463cb", [:mix], [{:nimble_options, "~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.4", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_process_propagator, "~> 0.3", [hex: :opentelemetry_process_propagator, repo: "hexpm", optional: false]}, {:opentelemetry_semantic_conventions, "~> 1.27", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}, {:opentelemetry_telemetry, "~> 1.1", [hex: :opentelemetry_telemetry, repo: "hexpm", optional: false]}, {:otel_http, "~> 0.2", [hex: :otel_http, repo: "hexpm", optional: false]}, {:plug, ">= 1.11.0", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a24fdccdfa6b890c8892c6366beab4a15a27ec0c692b0f77ec2a862e7b235f6e"},