diff --git a/config/runtime.exs b/config/runtime.exs index 54cc7312..17fd6ff6 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -366,6 +366,19 @@ if config_env() == :prod do end if otlp_endpoint = System.get_env("OTEL_EXPORTER_OTLP_ENDPOINT") do + # OTLP/HTTP + protobuf over the collector's 4318 port. We were + # previously using the gRPC exporter on 4317, which is transported + # by `grpcbox` — that channel's terminate/3 callback races with + # gproc's ETS teardown on BEAM shutdown and dumps a pair of + # ArgumentError stack traces every pod restart. The HTTP/protobuf + # exporter has no channel abstraction to unwind, so shutdown is + # quiet. Spans/metrics payload shape is identical. + otlp_protocol = + case System.get_env("OTEL_EXPORTER_OTLP_PROTOCOL") do + "grpc" -> :grpc + _ -> :http_protobuf + end + config :opentelemetry, :resource, service: [ name: "microwaveprop", @@ -377,7 +390,7 @@ if otlp_endpoint = System.get_env("OTEL_EXPORTER_OTLP_ENDPOINT") do traces_exporter: :otlp config :opentelemetry_exporter, - otlp_protocol: :grpc, + otlp_protocol: otlp_protocol, otlp_endpoint: otlp_endpoint else config :opentelemetry, traces_exporter: :none diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index b434d283..ac050fc5 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -67,8 +67,15 @@ spec: # endpoint used by prop-grid-rs / prop-hrrr-point-rs so # Elixir-initiated traces link up with downstream Rust # worker spans in Tempo. + # Port 4318 = OTLP/HTTP. We intentionally bypass the + # 4317 gRPC path because the Elixir grpcbox transport's + # terminate/3 callback races with gproc's ETS teardown on + # pod shutdown and dumps a pair of ArgumentError stacks + # per restart — HTTP/protobuf has no channel abstraction + # to unwind and shuts down quietly. Collector accepts + # both. - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: "http://otel-collector.observability.svc.cluster.local:4317" + value: "http://otel-collector.observability.svc.cluster.local:4318" envFrom: - secretRef: name: prop-secrets