Switch from Alpine (musl) to Debian slim (glibc) to fix SNMP SIGSEGV

musl libc has weak fork-safety guarantees in multi-threaded processes.
When Tokio's runtime is running and we fork() for SNMP operations, musl's
internal state can be inconsistent in the child, causing SIGSEGV on every
SNMP walk to Ubiquiti devices. glibc handles this via pthread_atfork.

- Dockerfile: Alpine → rust:1.93-bookworm build + debian:12-slim runtime
- CI: Remove ensure-netsnmp-base/manifest jobs, simplify build deps
- Delete Dockerfile.netsnmp and netsnmp-base.yml workflow
- Add NULL pointer guards in snmp_helper.c for fork-safety
- Add SNMP exception type handling (NoSuchObject/Instance/EndOfMibView)
- Add crash reproduction tests

.github/workflows/ci.yml

@@ -22,93 +22,10 @@ env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
   DOCKERHUB_IMAGE: gmcintire/towerops-agent
-  NETSNMP_IMAGE: ghcr.io/towerops-app/netsnmp-alpine:5.9.5.2
   CARGO_TERM_COLOR: always
   RUSTFLAGS: -C link-arg=-fuse-ld=lld
 
 jobs:
-  # Ensure the pre-built net-snmp base image exists in GHCR.
-  # First run: builds from source (~5 min per arch). After that: skips (~5s check).
-  ensure-netsnmp-base:
-    name: Ensure net-snmp base (${{ matrix.arch }})
-    runs-on: ${{ matrix.runner }}
-    permissions:
-      contents: read
-      packages: write
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - arch: amd64
-            runner: blacksmith-4vcpu-ubuntu-2404
-          - arch: arm64
-            runner: blacksmith-4vcpu-ubuntu-2404-arm
-    steps:
-      - name: Log in to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Check if base image exists
-        id: check
-        run: |
-          if docker buildx imagetools inspect ${{ env.NETSNMP_IMAGE }}-${{ matrix.arch }} > /dev/null 2>&1; then
-            echo "exists=true" >> $GITHUB_OUTPUT
-          else
-            echo "exists=false" >> $GITHUB_OUTPUT
-          fi
-
-      - name: Checkout
-        if: steps.check.outputs.exists == 'false'
-        uses: actions/checkout@v4
-
-      - name: Setup Blacksmith Builder
-        if: steps.check.outputs.exists == 'false'
-        uses: useblacksmith/setup-docker-builder@v1
-
-      - name: Build and push base image
-        if: steps.check.outputs.exists == 'false'
-        uses: useblacksmith/build-push-action@v2
-        with:
-          context: .
-          file: Dockerfile.netsnmp
-          push: true
-          provenance: false
-          tags: ${{ env.NETSNMP_IMAGE }}-${{ matrix.arch }}
-
-  ensure-netsnmp-manifest:
-    name: Ensure net-snmp manifest
-    needs: ensure-netsnmp-base
-    runs-on: blacksmith-4vcpu-ubuntu-2404
-    permissions:
-      packages: write
-    steps:
-      - name: Log in to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Check if manifest exists
-        id: check
-        run: |
-          if docker buildx imagetools inspect ${{ env.NETSNMP_IMAGE }} > /dev/null 2>&1; then
-            echo "exists=true" >> $GITHUB_OUTPUT
-          else
-            echo "exists=false" >> $GITHUB_OUTPUT
-          fi
-
-      - name: Create multi-arch manifest
-        if: steps.check.outputs.exists == 'false'
-        run: |
-          docker buildx imagetools create \
-            -t ${{ env.NETSNMP_IMAGE }} \
-            ${{ env.NETSNMP_IMAGE }}-amd64 \
-            ${{ env.NETSNMP_IMAGE }}-arm64
-
   test:
     name: Test
     runs-on: blacksmith-4vcpu-ubuntu-2404
@@ -167,7 +84,7 @@ jobs:
 
   build-branch:
     name: Build (Branch)
-    needs: [test, ensure-netsnmp-manifest]
+    needs: [test]
     runs-on: blacksmith-4vcpu-ubuntu-2404
     if: github.event_name == 'push' && github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/')
     permissions:
@@ -240,7 +157,7 @@ jobs:
   # Build architecture-specific images in parallel (native runners)
   build-release:
     name: Build (${{ matrix.arch }})
-    needs: [test, version, ensure-netsnmp-manifest]
+    needs: [test, version]
     runs-on: ${{ matrix.runner }}
     if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')
     permissions:

.github/workflows/netsnmp-base.yml

@@ -1,99 +0,0 @@
-# Builds the net-snmp base image used by the agent Dockerfile.
-# Only needs to run when the net-snmp version changes.
-#
-# Trigger manually or on changes to Dockerfile.netsnmp.
-
-name: Build net-snmp base image
-
-on:
-  push:
-    branches: [main]
-    paths: [Dockerfile.netsnmp]
-  workflow_dispatch:
-    inputs:
-      netsnmp_version:
-        description: "Net-SNMP version to build"
-        default: "5.9.5.2"
-        required: true
-
-env:
-  REGISTRY: ghcr.io
-  IMAGE_NAME: towerops-app/netsnmp-alpine
-
-jobs:
-  build:
-    name: Build (${{ matrix.arch }})
-    runs-on: ${{ matrix.runner }}
-    permissions:
-      contents: read
-      packages: write
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - platform: linux/amd64
-            arch: amd64
-            runner: blacksmith-4vcpu-ubuntu-2404
-          - platform: linux/arm64
-            arch: arm64
-            runner: blacksmith-4vcpu-ubuntu-2404-arm
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Setup Blacksmith Builder
-        uses: useblacksmith/setup-docker-builder@v1
-
-      - name: Log in to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build and push
-        uses: useblacksmith/build-push-action@v2
-        with:
-          context: .
-          file: Dockerfile.netsnmp
-          platforms: ${{ matrix.platform }}
-          push: true
-          provenance: false
-          build-args: |
-            NETSNMP_VERSION=${{ inputs.netsnmp_version || '5.9.5.2' }}
-          tags: |
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ inputs.netsnmp_version || '5.9.5.2' }}-${{ matrix.arch }}
-
-  manifest:
-    name: Create Manifest
-    needs: build
-    runs-on: blacksmith-4vcpu-ubuntu-2404
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Log in to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Create multi-arch manifest
-        run: |
-          VERSION="${{ inputs.netsnmp_version || '5.9.5.2' }}"
-          IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
-
-          docker buildx imagetools create -t ${IMAGE}:${VERSION} \
-            ${IMAGE}:${VERSION}-amd64 \
-            ${IMAGE}:${VERSION}-arm64
-
-          docker buildx imagetools create -t ${IMAGE}:latest \
-            ${IMAGE}:${VERSION}-amd64 \
-            ${IMAGE}:${VERSION}-arm64
-
-      - name: Summary
-        run: |
-          echo "### net-snmp base image built" >> $GITHUB_STEP_SUMMARY
-          echo "Version: ${{ inputs.netsnmp_version || '5.9.5.2' }}" >> $GITHUB_STEP_SUMMARY
-          echo "Image: \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ inputs.netsnmp_version || '5.9.5.2' }}\`" >> $GITHUB_STEP_SUMMARY

Dockerfile

@@ -1,10 +1,6 @@
 # syntax=docker/dockerfile:1.4
-# Pre-built net-snmp 5.9.5.2 (fixes segfaults in Alpine's 5.9.4)
+# Build stage - Debian bookworm with glibc (fixes musl fork-safety SIGSEGV)
-# Built from Dockerfile.netsnmp via .github/workflows/netsnmp-base.yml
+FROM rust:1.93-bookworm AS builder
-FROM ghcr.io/towerops-app/netsnmp-alpine:5.9.5.2 AS netsnmp
-
-# Build stage
-FROM rust:1.93-alpine AS builder
 
 # Build arguments provided by Docker buildx
 ARG TARGETPLATFORM
@@ -13,20 +9,23 @@ ARG VERSION=0.1.0-unknown
 
 WORKDIR /app
 
-# Install build dependencies (net-snmp headers/libs from pre-built image)
+# Install build dependencies
-RUN apk add --no-cache musl-dev protobuf-dev openssl-dev openssl-libs-static cmake perl g++
+RUN apt-get update && apt-get install -y --no-install-recommends \
-COPY --from=netsnmp /usr/include/net-snmp /usr/include/net-snmp
+    protobuf-compiler \
-COPY --from=netsnmp /usr/lib/libnetsnmp* /usr/lib/
+    libsnmp-dev \
-COPY --from=netsnmp /usr/lib/libsnmp* /usr/lib/
+    cmake \
+    g++ \
+    pkg-config \
+    libssl-dev \
+    && rm -rf /var/lib/apt/lists/*
 
-# Determine Rust target based on platform and add it
+# Determine Rust target based on platform
 RUN case "$TARGETPLATFORM" in \
-    "linux/amd64") RUST_TARGET="x86_64-unknown-linux-musl" ;; \
+    "linux/amd64") RUST_TARGET="x86_64-unknown-linux-gnu" ;; \
-    "linux/arm64") RUST_TARGET="aarch64-unknown-linux-musl" ;; \
+    "linux/arm64") RUST_TARGET="aarch64-unknown-linux-gnu" ;; \
     *) echo "Unsupported platform: $TARGETPLATFORM" && exit 1 ;; \
     esac && \
-    echo "$RUST_TARGET" > /tmp/rust-target && \
+    echo "$RUST_TARGET" > /tmp/rust-target
-    rustup target add "$RUST_TARGET"
 
 # Copy manifests and build files
 COPY Cargo.toml Cargo.lock build.rs ./
@@ -59,23 +58,24 @@ RUN --mount=type=cache,id=cargo-registry-${TARGETARCH},target=/usr/local/cargo/r
     BUILD_VERSION="$VERSION" cargo build --release --target "$RUST_TARGET" && \
     cp "target/$RUST_TARGET/release/towerops-agent" /tmp/towerops-agent
 
-# Runtime stage - must match builder's Alpine version for ABI compatibility
+# Runtime stage - Debian slim with glibc
-# (rust:1.93-alpine uses Alpine 3.23)
+FROM debian:12-slim
-FROM alpine:3.23
 
 # Install runtime dependencies
-# iputils provides ping with setuid root (doesn't require CAP_NET_RAW)
+# iputils-ping provides ping with setuid root (doesn't require CAP_NET_RAW)
-RUN apk add --no-cache ca-certificates iputils openssl
+RUN apt-get update && apt-get install -y --no-install-recommends \
-
+    ca-certificates \
-# Copy net-snmp 5.9.5.2 shared libraries from pre-built image
+    iputils-ping \
-COPY --from=netsnmp /usr/lib/libnetsnmp.so* /usr/lib/
+    libsnmp40 \
+    openssl \
+    && rm -rf /var/lib/apt/lists/*
 
 # Copy binary from builder
 COPY --from=builder /tmp/towerops-agent /usr/local/bin/towerops-agent
 
 # Create non-root user
-RUN addgroup -g 1000 towerops && \
+RUN groupadd -g 1000 towerops && \
-    adduser -D -u 1000 -G towerops towerops
+    useradd -u 1000 -g towerops -s /bin/false towerops
 
 # Allow non-root user to overwrite binary during self-update
 RUN chown towerops /usr/local/bin/towerops-agent

Dockerfile.netsnmp

@@ -1,68 +0,0 @@
-# syntax=docker/dockerfile:1.4
-# Base image with net-snmp 5.9.5.2 compiled from source
-# Alpine 3.23 ships 5.9.4 which has segfault bugs in varbind construction
-# and buffer overflows in octet string handling.
-#
-# Build and push with:
-#   docker buildx build -f Dockerfile.netsnmp --platform linux/amd64,linux/arm64 \
-#     -t ghcr.io/towerops-app/netsnmp-alpine:5.9.5.2 --push .
-#
-# Or use the GitHub Actions workflow: .github/workflows/netsnmp-base.yml
-
-FROM alpine:3.23 AS builder
-
-ARG NETSNMP_VERSION=5.9.5.2
-
-# Install build dependencies
-RUN apk add --no-cache \
-    gcc \
-    g++ \
-    make \
-    musl-dev \
-    openssl-dev \
-    linux-headers \
-    perl \
-    curl \
-    file
-
-# Download and extract net-snmp source
-RUN curl -fsSL "https://downloads.sourceforge.net/project/net-snmp/net-snmp/${NETSNMP_VERSION}/net-snmp-${NETSNMP_VERSION}.tar.gz" \
-    -o /tmp/net-snmp.tar.gz && \
-    tar xzf /tmp/net-snmp.tar.gz -C /tmp && \
-    rm /tmp/net-snmp.tar.gz
-
-WORKDIR /tmp/net-snmp-${NETSNMP_VERSION}
-
-# Configure with minimal features (we only need libnetsnmp)
-RUN ./configure \
-    --prefix=/usr \
-    --sysconfdir=/etc \
-    --with-defaults \
-    --with-openssl \
-    --enable-shared \
-    --enable-static \
-    --disable-applications \
-    --disable-manuals \
-    --disable-scripts \
-    --without-perl-modules \
-    --without-python-modules \
-    --disable-embedded-perl \
-    --with-default-snmp-version=3
-
-# Build and install
-RUN make -j$(nproc) && \
-    make install DESTDIR=/netsnmp-install
-
-# Final stage: slim image with just the compiled artifacts
-FROM alpine:3.23
-
-# Install runtime dependency (OpenSSL) that net-snmp links against
-RUN apk add --no-cache openssl-dev musl-dev
-
-# Copy compiled net-snmp (headers, static libs, shared libs)
-COPY --from=builder /netsnmp-install/usr/include/net-snmp /usr/include/net-snmp
-COPY --from=builder /netsnmp-install/usr/lib/libnetsnmp* /usr/lib/
-COPY --from=builder /netsnmp-install/usr/lib/libsnmp* /usr/lib/
-
-# Ensure shared library cache is updated
-RUN ldconfig /usr/lib 2>/dev/null || true

native/snmp_helper.c

@@ -282,13 +282,30 @@ int snmp_get(
         struct variable_list *var = response->variables;
         *value_type = var->type;
 
+        // Handle SNMP exception types (NoSuchObject, NoSuchInstance, EndOfMibView)
+        if (var->type == SNMP_NOSUCHOBJECT ||
+            var->type == SNMP_NOSUCHINSTANCE ||
+            var->type == SNMP_ENDOFMIBVIEW) {
+            if (error_buf && error_buf_len > 0) {
+                const char *label = var->type == SNMP_NOSUCHOBJECT ? "noSuchObject" :
+                                    var->type == SNMP_NOSUCHINSTANCE ? "noSuchInstance" :
+                                    "endOfMibView";
+                snprintf(error_buf, error_buf_len, "%s", label);
+            }
+            snmp_free_pdu(response);
+            return -1;
+        }
+
         switch (var->type) {
             case ASN_OCTET_STR:
             case ASN_OPAQUE:
             case ASN_IPADDRESS:
-                if (var->val_len <= value_buf_len) {
+                if (var->val.string && var->val_len > 0 &&
+                    var->val_len <= value_buf_len) {
                     memcpy(value_buf, var->val.string, var->val_len);
                     result = (int)var->val_len;
+                } else if (!var->val.string || var->val_len == 0) {
+                    result = 0; // Empty string
                 } else {
                     if (error_buf && error_buf_len > 0) {
                         snprintf(error_buf, error_buf_len, "Buffer too small");
@@ -301,31 +318,33 @@ int snmp_get(
             case ASN_GAUGE:
             case ASN_TIMETICKS:
             case ASN_UINTEGER:
-                if (sizeof(long) <= value_buf_len) {
+                if (var->val.integer && sizeof(long) <= value_buf_len) {
                     *((long*)value_buf) = *var->val.integer;
                     result = sizeof(long);
                 }
                 break;
 
             case ASN_COUNTER64:
-                if (sizeof(struct counter64) <= value_buf_len) {
+                if (var->val.counter64 &&
+                    sizeof(struct counter64) <= value_buf_len) {
                     memcpy(value_buf, var->val.counter64, sizeof(struct counter64));
                     result = sizeof(struct counter64);
                 }
                 break;
 
             case ASN_OBJECT_ID:
-                // Convert OID to string representation
+                if (var->val.objid && var->val_len > 0) {
-                {
                     char oid_buf[256];
-                    snprint_objid(oid_buf, sizeof(oid_buf), var->val.objid, var->val_len / sizeof(oid));
+                    snprint_objid(oid_buf, sizeof(oid_buf), var->val.objid,
+                                  var->val_len / sizeof(oid));
                     size_t oid_str_len = strlen(oid_buf);
                     if (oid_str_len <= value_buf_len) {
                         memcpy(value_buf, oid_buf, oid_str_len);
                         result = (int)oid_str_len;
                     } else {
                         if (error_buf && error_buf_len > 0) {
-                            snprintf(error_buf, error_buf_len, "Buffer too small for OID string");
+                            snprintf(error_buf, error_buf_len,
+                                     "Buffer too small for OID string");
                         }
                     }
                 }
@@ -339,7 +358,8 @@ int snmp_get(
             default:
                 // Unknown type
                 if (error_buf && error_buf_len > 0) {
-                    snprintf(error_buf, error_buf_len, "Unsupported type: %d", var->type);
+                    snprintf(error_buf, error_buf_len,
+                             "Unsupported type: %d", var->type);
                 }
                 break;
         }
@@ -412,13 +432,39 @@ int snmp_walk(
             break;
         }
 
+        // Handle SNMP exception types that indicate end-of-data or missing values.
+        // These have type 0x80 (NoSuchObject), 0x81 (NoSuchInstance),
+        // 0x82 (EndOfMibView) and their val pointers may be NULL.
+        if (var->type == SNMP_NOSUCHOBJECT ||
+            var->type == SNMP_NOSUCHINSTANCE) {
+            // Object/instance doesn't exist at this index - skip and continue walk
+            // Update OID for next iteration
+            if (var->name_length <= MAX_OID_LEN) {
+                memcpy(name, var->name, var->name_length * sizeof(oid));
+                name_length = var->name_length;
+            } else {
+                running = 0;
+            }
+            snmp_free_pdu(response);
+            continue;
+        }
+
+        if (var->type == SNMP_ENDOFMIBVIEW) {
+            // No more data available - terminate the walk
+            snmp_free_pdu(response);
+            break;
+        }
+
         // Store result
         snmp_walk_result_t *res = &results[*num_results];
 
         // Convert OID to string
         snprint_objid(res->oid, sizeof(res->oid), var->name, var->name_length);
 
-        // Store value
+        // Store value - check for NULL val pointers before every dereference.
+        // After fork() from a multi-threaded process, net-snmp's internal state
+        // can be inconsistent, potentially leaving val pointers NULL even for
+        // standard types.
         res->value_type = var->type;
         res->value_len = 0;
 
@@ -426,17 +472,18 @@ int snmp_walk(
             case ASN_OCTET_STR:
             case ASN_OPAQUE:
             case ASN_IPADDRESS:
-                if (var->val_len <= sizeof(res->value)) {
+                if (var->val.string && var->val_len > 0 &&
+                    var->val_len <= sizeof(res->value)) {
                     memcpy(res->value, var->val.string, var->val_len);
                     res->value_len = var->val_len;
                 }
                 break;
 
             case ASN_OBJECT_ID:
-                // Convert OID to string representation
+                if (var->val.objid && var->val_len > 0) {
-                {
                     char oid_buf[256];
-                    snprint_objid(oid_buf, sizeof(oid_buf), var->val.objid, var->val_len / sizeof(oid));
+                    snprint_objid(oid_buf, sizeof(oid_buf), var->val.objid,
+                                  var->val_len / sizeof(oid));
                     size_t oid_str_len = strlen(oid_buf);
                     if (oid_str_len < sizeof(res->value)) {
                         memcpy(res->value, oid_buf, oid_str_len);
@@ -450,18 +497,27 @@ int snmp_walk(
             case ASN_GAUGE:
             case ASN_TIMETICKS:
             case ASN_UINTEGER:
-                if (sizeof(long) <= sizeof(res->value)) {
+                if (var->val.integer && sizeof(long) <= sizeof(res->value)) {
                     *((long*)res->value) = *var->val.integer;
                     res->value_len = sizeof(long);
                 }
                 break;
 
             case ASN_COUNTER64:
-                if (sizeof(struct counter64) <= sizeof(res->value)) {
+                if (var->val.counter64 &&
+                    sizeof(struct counter64) <= sizeof(res->value)) {
                     memcpy(res->value, var->val.counter64, sizeof(struct counter64));
                     res->value_len = sizeof(struct counter64);
                 }
                 break;
+
+            case ASN_NULL:
+                // NULL values are valid but contain no data - skip
+                break;
+
+            default:
+                // Unknown or unsupported type - skip silently
+                break;
         }
 
         if (res->value_len > 0) {