chore(rust): drop OpenTelemetry stack to speed up cold builds

Removes the OTLP span exporter (tracing-opentelemetry, opentelemetry,
opentelemetry_sdk, opentelemetry-otlp) and its transitive tonic/prost/
h2 tree — 56 crates off the build graph, ~3-5 min off a cold build on
the forgejo runner.

The fmt layer (JSON to stdout via tracing-subscriber) is unchanged;
structured logs still reach the Loki collector through kubectl logs,
and /metrics on :9100 still serves Prometheus. Traces were a nice-to-
have that nothing on call was actually reading.

telemetry::init keeps its signature (service_name + TelemetryGuard)
so re-adding OTLP later is a single-module change. k8s manifests drop
the now-unused OTEL_EXPORTER_OTLP_ENDPOINT env var.
This commit is contained in:
Graham McIntire 2026-04-24 09:11:38 -05:00
parent a1ccbeb3e5
commit d7d865d566
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
5 changed files with 23 additions and 311 deletions

View file

@ -91,12 +91,6 @@ spec:
value: "3" value: "3"
- name: METRICS_ADDR - name: METRICS_ADDR
value: "0.0.0.0:9100" value: "0.0.0.0:9100"
# OTLP/gRPC endpoint for the cluster OTel Collector. Unset
# disables span export entirely (fmt layer still logs to
# stdout). See vntx-infra/observability/ for the collector
# + Tempo backend.
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: "http://otel-collector.observability.svc.cluster.local:4317"
envFrom: envFrom:
- secretRef: - secretRef:
name: prop-secrets name: prop-secrets

View file

@ -58,8 +58,6 @@ spec:
# chain-step idx fetch warms both workers. # chain-step idx fetch warms both workers.
- name: HRRR_IDX_CACHE_DIR - name: HRRR_IDX_CACHE_DIR
value: "/data/hrrr_idx" value: "/data/hrrr_idx"
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: "http://otel-collector.observability.svc.cluster.local:4317"
envFrom: envFrom:
- secretRef: - secretRef:
name: prop-secrets name: prop-secrets

View file

@ -622,25 +622,6 @@ dependencies = [
"wasip3", "wasip3",
] ]
[[package]]
name = "h2"
version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54"
dependencies = [
"atomic-waker",
"bytes",
"fnv",
"futures-core",
"futures-sink",
"http",
"indexmap",
"slab",
"tokio",
"tokio-util",
"tracing",
]
[[package]] [[package]]
name = "hashbrown" name = "hashbrown"
version = "0.15.5" version = "0.15.5"
@ -767,7 +748,6 @@ dependencies = [
"bytes", "bytes",
"futures-channel", "futures-channel",
"futures-core", "futures-core",
"h2",
"http", "http",
"http-body", "http-body",
"httparse", "httparse",
@ -795,19 +775,6 @@ dependencies = [
"webpki-roots 1.0.7", "webpki-roots 1.0.7",
] ]
[[package]]
name = "hyper-timeout"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0"
dependencies = [
"hyper",
"hyper-util",
"pin-project-lite",
"tokio",
"tower-service",
]
[[package]] [[package]]
name = "hyper-util" name = "hyper-util"
version = "0.1.20" version = "0.1.20"
@ -992,15 +959,6 @@ dependencies = [
"serde", "serde",
] ]
[[package]]
name = "itertools"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
dependencies = [
"either",
]
[[package]] [[package]]
name = "itoa" name = "itoa"
version = "1.0.18" version = "1.0.18"
@ -1230,66 +1188,6 @@ version = "1.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
[[package]]
name = "opentelemetry"
version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0"
dependencies = [
"futures-core",
"futures-sink",
"js-sys",
"pin-project-lite",
"thiserror 2.0.18",
"tracing",
]
[[package]]
name = "opentelemetry-otlp"
version = "0.31.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f"
dependencies = [
"http",
"opentelemetry",
"opentelemetry-proto",
"opentelemetry_sdk",
"prost",
"thiserror 2.0.18",
"tokio",
"tonic",
]
[[package]]
name = "opentelemetry-proto"
version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f"
dependencies = [
"opentelemetry",
"opentelemetry_sdk",
"prost",
"tonic",
"tonic-prost",
]
[[package]]
name = "opentelemetry_sdk"
version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd"
dependencies = [
"futures-channel",
"futures-executor",
"futures-util",
"opentelemetry",
"percent-encoding",
"rand 0.9.4",
"thiserror 2.0.18",
"tokio",
"tokio-stream",
]
[[package]] [[package]]
name = "parking" name = "parking"
version = "2.2.1" version = "2.2.1"
@ -1334,26 +1232,6 @@ version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
[[package]]
name = "pin-project"
version = "1.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517"
dependencies = [
"pin-project-internal",
]
[[package]]
name = "pin-project-internal"
version = "1.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]] [[package]]
name = "pin-project-lite" name = "pin-project-lite"
version = "0.2.17" version = "0.2.17"
@ -1480,9 +1358,6 @@ dependencies = [
"flate2", "flate2",
"futures", "futures",
"num_cpus", "num_cpus",
"opentelemetry",
"opentelemetry-otlp",
"opentelemetry_sdk",
"png", "png",
"pretty_assertions", "pretty_assertions",
"prometheus", "prometheus",
@ -1498,34 +1373,10 @@ dependencies = [
"tikv-jemallocator", "tikv-jemallocator",
"tokio", "tokio",
"tracing", "tracing",
"tracing-opentelemetry",
"tracing-subscriber", "tracing-subscriber",
"uuid", "uuid",
] ]
[[package]]
name = "prost"
version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568"
dependencies = [
"bytes",
"prost-derive",
]
[[package]]
name = "prost-derive"
version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
dependencies = [
"anyhow",
"itertools",
"proc-macro2",
"quote",
"syn",
]
[[package]] [[package]]
name = "quinn" name = "quinn"
version = "0.11.9" version = "0.11.9"
@ -2492,43 +2343,6 @@ dependencies = [
"tokio", "tokio",
] ]
[[package]]
name = "tonic"
version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec"
dependencies = [
"async-trait",
"base64",
"bytes",
"http",
"http-body",
"http-body-util",
"hyper",
"hyper-timeout",
"hyper-util",
"percent-encoding",
"pin-project",
"sync_wrapper",
"tokio",
"tokio-stream",
"tower",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "tonic-prost"
version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309"
dependencies = [
"bytes",
"prost",
"tonic",
]
[[package]] [[package]]
name = "tower" name = "tower"
version = "0.5.3" version = "0.5.3"
@ -2537,15 +2351,11 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"futures-util", "futures-util",
"indexmap",
"pin-project-lite", "pin-project-lite",
"slab",
"sync_wrapper", "sync_wrapper",
"tokio", "tokio",
"tokio-util",
"tower-layer", "tower-layer",
"tower-service", "tower-service",
"tracing",
] ]
[[package]] [[package]]
@ -2622,22 +2432,6 @@ dependencies = [
"tracing-core", "tracing-core",
] ]
[[package]]
name = "tracing-opentelemetry"
version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ac28f2d093c6c477eaa76b23525478f38de514fa9aeb1285738d4b97a9552fc"
dependencies = [
"js-sys",
"opentelemetry",
"smallvec",
"tracing",
"tracing-core",
"tracing-log",
"tracing-subscriber",
"web-time",
]
[[package]] [[package]]
name = "tracing-serde" name = "tracing-serde"
version = "0.2.0" version = "0.2.0"

View file

@ -25,10 +25,6 @@ anyhow = "1"
thiserror = "2" thiserror = "2"
tracing = "0.1" tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
tracing-opentelemetry = "0.32"
opentelemetry = "0.31"
opentelemetry_sdk = { version = "0.31", features = ["rt-tokio"] }
opentelemetry-otlp = { version = "0.31", default-features = false, features = ["grpc-tonic", "trace"] }
chrono = { version = "0.4", features = ["serde"] } chrono = { version = "0.4", features = ["serde"] }
uuid = { version = "1", features = ["v4", "serde"] } uuid = { version = "1", features = ["v4", "serde"] }
serde = { version = "1", features = ["derive"] } serde = { version = "1", features = ["derive"] }

View file

@ -1,108 +1,38 @@
//! Tracing init with optional OTLP export. //! Tracing init for the Rust workers.
//! //!
//! Both bin targets (`worker`, `hrrr_point_worker`) call `init(svc)` at //! Both bin targets (`worker`, `hrrr_point_worker`) call `init(svc)` at
//! startup. The local fmt layer (JSON to stdout) is always on so kubectl //! startup to install a JSON-to-stdout `tracing` subscriber so kubectl
//! logs keeps working. When `OTEL_EXPORTER_OTLP_ENDPOINT` is set we //! logs keep working.
//! additionally install a `tracing-opentelemetry` layer that ships
//! spans over OTLP/gRPC to the cluster's OTel Collector — see
//! `k8s/deployment-grid-rs.yaml` + `deployment-hrrr-point-rs.yaml`
//! for the env wiring.
//! //!
//! Returning the `TracerProvider` from `init` is deliberate: the caller //! OTLP span export was removed in 2026-04 — the opentelemetry stack
//! must hold it until shutdown and call `.shutdown()` so queued spans //! pulled ~80 transitive crates and added ~3-5 min to cold builds on
//! flush before the process exits. Dropping it mid-run cancels the //! the forgejo runner, and the cluster's trace consumption was
//! background exporter and loses recent spans. //! negligible compared to structured logs + /metrics. Re-add via
//! //! `tracing-opentelemetry` only if Tempo becomes load-bearing.
//! Service identity is set via OpenTelemetry resource attributes
//! (`service.name`, `service.namespace`) so Tempo groups spans per
//! binary. `service.namespace=prop` matches the k8s namespace.
use opentelemetry::{global, trace::TracerProvider as _, KeyValue};
use opentelemetry_otlp::WithExportConfig;
use opentelemetry_sdk::{trace::SdkTracerProvider, Resource};
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter}; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter};
/// Opaque handle the caller must hold until process shutdown; dropping /// Opaque handle the caller holds until shutdown. No-op today; kept
/// it stops the OTLP exporter. Returns `None` when OTLP is disabled /// so re-adding OTLP later doesn't require churn at every call site.
/// (no endpoint env var) so callers don't need to care which mode is pub struct TelemetryGuard;
/// active.
pub struct TelemetryGuard {
provider: Option<SdkTracerProvider>,
}
impl TelemetryGuard { impl TelemetryGuard {
/// Flush queued spans and shut down the exporter. Safe to call /// Flush/drop hook. Currently a no-op; the fmt layer flushes each
/// multiple times; subsequent calls are no-ops. /// line as it's written.
pub fn shutdown(mut self) { pub fn shutdown(self) {}
if let Some(provider) = self.provider.take() {
let _ = provider.shutdown();
}
}
} }
impl Drop for TelemetryGuard { /// Initialize tracing for a binary. `service_name` is accepted for API
fn drop(&mut self) { /// compatibility with the previous OTLP-capable init and is currently
if let Some(provider) = self.provider.take() { /// unused in the fmt-only path.
let _ = provider.shutdown(); pub fn init(_service_name: &'static str) -> TelemetryGuard {
}
}
}
/// Initialize tracing for a binary. `service_name` becomes the
/// `service.name` attribute in exported spans and the log target
/// prefix.
pub fn init(service_name: &'static str) -> TelemetryGuard {
let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
let fmt_layer = tracing_subscriber::fmt::layer().json(); let fmt_layer = tracing_subscriber::fmt::layer().json();
// If the collector endpoint isn't set, skip the OTLP layer entirely tracing_subscriber::registry()
// — useful for local cargo runs where there's no collector to ship .with(env_filter)
// to. The fmt layer alone keeps dev parity with prod log output. .with(fmt_layer)
let otlp_endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").ok(); .init();
match otlp_endpoint { TelemetryGuard
Some(endpoint) if !endpoint.is_empty() => {
let exporter = opentelemetry_otlp::SpanExporter::builder()
.with_tonic()
.with_endpoint(endpoint)
.build()
.expect("build OTLP span exporter");
let resource = Resource::builder()
.with_attributes([
KeyValue::new("service.name", service_name),
KeyValue::new("service.namespace", "prop"),
])
.build();
let provider = SdkTracerProvider::builder()
.with_batch_exporter(exporter)
.with_resource(resource)
.build();
let tracer = provider.tracer(service_name);
global::set_tracer_provider(provider.clone());
let otel_layer = tracing_opentelemetry::layer().with_tracer(tracer);
tracing_subscriber::registry()
.with(env_filter)
.with(fmt_layer)
.with(otel_layer)
.init();
TelemetryGuard {
provider: Some(provider),
}
}
_ => {
tracing_subscriber::registry()
.with(env_filter)
.with(fmt_layer)
.init();
TelemetryGuard { provider: None }
}
}
} }