From c46e89a48f8106ac4a904859f65a7f6ee9ee9e43 Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Tue, 21 Apr 2026 17:54:43 -0500
Subject: [PATCH] fix(otel): switch Elixir exporter to OTLP/HTTP to avoid
 grpcbox shutdown noise
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Every prop pod restart was dumping a pair of ArgumentError stack
traces from grpcbox_channel / grpcbox_subchannel terminate/3:

    :ets.select(:gproc, …)
    * 1st argument: the table identifier does not refer to an
      existing ETS table

This is a known grpcbox shutdown race: the OTLP/gRPC transport's
terminate callback tries to look itself up in gproc's ETS just
after gproc's supervisor has already torn the table down. Noisy,
not harmful — but one ArgumentError per restart × N pods adds up.

The OTLP/HTTP transport (port 4318 on the same otel-collector
Service) has no gproc/ETS channel machinery, so shutdown is
quiet. Spans are identical — our collector already accepts both.

- `config/runtime.exs`: default `otlp_protocol: :http_protobuf`,
  with an `OTEL_EXPORTER_OTLP_PROTOCOL=grpc` escape hatch.
- `k8s/deployment.yaml`: point `OTEL_EXPORTER_OTLP_ENDPOINT` at
  :4318 instead of :4317. The Rust workers stay on :4317 — their
  native opentelemetry-otlp transport doesn't share the grpcbox
  bug.
---
 config/runtime.exs  | 15 ++++++++++++++-
 k8s/deployment.yaml |  9 ++++++++-
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/config/runtime.exs b/config/runtime.exs
index 54cc7312..17fd6ff6 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -366,6 +366,19 @@ if config_env() == :prod do
 end
 
 if otlp_endpoint = System.get_env("OTEL_EXPORTER_OTLP_ENDPOINT") do
+  # OTLP/HTTP + protobuf over the collector's 4318 port. We were
+  # previously using the gRPC exporter on 4317, which is transported
+  # by `grpcbox` — that channel's terminate/3 callback races with
+  # gproc's ETS teardown on BEAM shutdown and dumps a pair of
+  # ArgumentError stack traces every pod restart. The HTTP/protobuf
+  # exporter has no channel abstraction to unwind, so shutdown is
+  # quiet. Spans/metrics payload shape is identical.
+  otlp_protocol =
+    case System.get_env("OTEL_EXPORTER_OTLP_PROTOCOL") do
+      "grpc" -> :grpc
+      _ -> :http_protobuf
+    end
+
   config :opentelemetry, :resource,
     service: [
       name: "microwaveprop",
@@ -377,7 +390,7 @@ if otlp_endpoint = System.get_env("OTEL_EXPORTER_OTLP_ENDPOINT") do
     traces_exporter: :otlp
 
   config :opentelemetry_exporter,
-    otlp_protocol: :grpc,
+    otlp_protocol: otlp_protocol,
     otlp_endpoint: otlp_endpoint
 else
   config :opentelemetry, traces_exporter: :none
diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml
index b434d283..ac050fc5 100644
--- a/k8s/deployment.yaml
+++ b/k8s/deployment.yaml
@@ -67,8 +67,15 @@ spec:
             # endpoint used by prop-grid-rs / prop-hrrr-point-rs so
             # Elixir-initiated traces link up with downstream Rust
             # worker spans in Tempo.
+            # Port 4318 = OTLP/HTTP. We intentionally bypass the
+            # 4317 gRPC path because the Elixir grpcbox transport's
+            # terminate/3 callback races with gproc's ETS teardown on
+            # pod shutdown and dumps a pair of ArgumentError stacks
+            # per restart — HTTP/protobuf has no channel abstraction
+            # to unwind and shuts down quietly. Collector accepts
+            # both.
             - name: OTEL_EXPORTER_OTLP_ENDPOINT
-              value: "http://otel-collector.observability.svc.cluster.local:4317"
+              value: "http://otel-collector.observability.svc.cluster.local:4318"
           envFrom:
             - secretRef:
                 name: prop-secrets