From 641789449ae4b322f0faca61532498a48e173390 Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Wed, 22 Apr 2026 08:52:16 -0500
Subject: [PATCH] fix(prop-grid-rs): switch global allocator to jemalloc
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Pods were OOMKilled at :05 every hour at the 3 GiB cgroup limit.
Idle RSS climbed 750 MiB → 1.5 GiB across pod lifetime as the
hourly f00 analysis cycled surface + pressure + native-duct blobs
(50 MB / 400 MB / 530 MB) simultaneously during decode. glibc
malloc retains dirty pages after freeing those large transient
allocations, so RSS drifted upward with each cycle and the next
hourly peak eventually crossed the limit.

jemalloc's background decay returns pages to the kernel on a
short schedule, so RSS tracks the actual working set. Wired in
at lib.rs so both `worker` and `hrrr_point_worker` inherit it.
Skipped on MSVC targets (Windows CI if we ever add one); dev
builds on macOS still pick it up via the generic target_env
gate.
---
 rust/prop_grid_rs/Cargo.lock | 21 +++++++++++++++++++++
 rust/prop_grid_rs/Cargo.toml | 10 ++++++++++
 rust/prop_grid_rs/src/lib.rs |  8 ++++++++
 3 files changed, 39 insertions(+)

diff --git a/rust/prop_grid_rs/Cargo.lock b/rust/prop_grid_rs/Cargo.lock
index 812587c7..9555c179 100644
--- a/rust/prop_grid_rs/Cargo.lock
+++ b/rust/prop_grid_rs/Cargo.lock
@@ -1495,6 +1495,7 @@ dependencies = [
  "sqlx",
  "tempfile",
  "thiserror 2.0.18",
+ "tikv-jemallocator",
  "tokio",
  "tracing",
  "tracing-opentelemetry",
@@ -2384,6 +2385,26 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "tikv-jemalloc-sys"
+version = "0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd8aa5b2ab86a2cefa406d889139c162cbb230092f7d1d7cbc1716405d852a3b"
+dependencies = [
+ "cc",
+ "libc",
+]
+
+[[package]]
+name = "tikv-jemallocator"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0359b4327f954e0567e69fb191cf1436617748813819c94b8cd4a431422d053a"
+dependencies = [
+ "libc",
+ "tikv-jemalloc-sys",
+]
+
 [[package]]
 name = "tinystr"
 version = "0.8.3"
diff --git a/rust/prop_grid_rs/Cargo.toml b/rust/prop_grid_rs/Cargo.toml
index 8fb0229d..a8aa18b4 100644
--- a/rust/prop_grid_rs/Cargo.toml
+++ b/rust/prop_grid_rs/Cargo.toml
@@ -45,6 +45,16 @@ rmp-serde = "1.3"
 rmpv = { version = "1.3", features = ["with-serde"] }
 flate2 = "1"
 
+# glibc malloc retains most dirty pages after large transient
+# allocations, so RSS drifted from ~500 MiB at boot to ~1.5 GiB
+# after a few hourly f00 analysis cycles and tripped the 3 GiB
+# cgroup limit. jemalloc returns pages to the kernel on a short
+# decay schedule, so RSS follows actual working set and survives
+# the hourly peak (surface + pressure + native-duct blobs held
+# simultaneously during decode).
+[target.'cfg(not(target_env = "msvc"))'.dependencies]
+tikv-jemallocator = "0.6"
+
 [dev-dependencies]
 tokio = { version = "1", features = ["full", "test-util"] }
 pretty_assertions = "1"
diff --git a/rust/prop_grid_rs/src/lib.rs b/rust/prop_grid_rs/src/lib.rs
index 501f6463..1cfac736 100644
--- a/rust/prop_grid_rs/src/lib.rs
+++ b/rust/prop_grid_rs/src/lib.rs
@@ -19,6 +19,14 @@
 //! merge, no NEXRAD, no commercial-link boost. Those are f00-only in
 //! production and Elixir keeps handling f00.
 
+// jemalloc for all non-MSVC targets (Linux release builds land here).
+// Set as the global allocator at the library root so every binary in
+// the crate (worker, hrrr_point_worker) inherits it. See Cargo.toml
+// for the rationale behind switching off glibc malloc.
+#[cfg(not(target_env = "msvc"))]
+#[global_allocator]
+static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
+
 pub mod band_config;
 pub mod commercial;
 pub mod db;