graham/towerops-agent
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Switch from Alpine (musl) to Debian slim (glibc) to fix SNMP SIGSEGV

Browse source 
musl libc has weak fork-safety guarantees in multi-threaded processes.
When Tokio's runtime is running and we fork() for SNMP operations, musl's
internal state can be inconsistent in the child, causing SIGSEGV on every
SNMP walk to Ubiquiti devices. glibc handles this via pthread_atfork.

- Dockerfile: Alpine → rust:1.93-bookworm build + debian:12-slim runtime
- CI: Remove ensure-netsnmp-base/manifest jobs, simplify build deps
- Delete Dockerfile.netsnmp and netsnmp-base.yml workflow
- Add NULL pointer guards in snmp_helper.c for fork-safety
- Add SNMP exception type handling (NoSuchObject/Instance/EndOfMibView)
- Add crash reproduction tests
This commit is contained in:
Graham McIntire 2026-02-11 08:41:06 -06:00
parent fc14fa7798
commit a9bf0b0f8b
No known key found for this signature in database
6 changed files with 1108 additions and 293 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

87
.github/workflows/ci.yml vendored
View file

@ -22,93 +22,10 @@ env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}
DOCKERHUB_IMAGE: gmcintire/towerops-agent
NETSNMP_IMAGE: ghcr.io/towerops-app/netsnmp-alpine:5.9.5.2
CARGO_TERM_COLOR: always
RUSTFLAGS: -C link-arg=-fuse-ld=lld
jobs:
# Ensure the pre-built net-snmp base image exists in GHCR.
# First run: builds from source (~5 min per arch). After that: skips (~5s check).
ensure-netsnmp-base:
name: Ensure net-snmp base (${{ matrix.arch }})
runs-on: ${{ matrix.runner }}
permissions:
contents: read
packages: write
strategy:
fail-fast: false
matrix:
include:
- arch: amd64
runner: blacksmith-4vcpu-ubuntu-2404
- arch: arm64
runner: blacksmith-4vcpu-ubuntu-2404-arm
steps:
- name: Log in to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Check if base image exists
id: check
run: |
if docker buildx imagetools inspect ${{ env.NETSNMP_IMAGE }}-${{ matrix.arch }} > /dev/null 2>&1; then
echo "exists=true" >> $GITHUB_OUTPUT
else
echo "exists=false" >> $GITHUB_OUTPUT
fi
- name: Checkout
if: steps.check.outputs.exists == 'false'
uses: actions/checkout@v4
- name: Setup Blacksmith Builder
if: steps.check.outputs.exists == 'false'
uses: useblacksmith/setup-docker-builder@v1
- name: Build and push base image
if: steps.check.outputs.exists == 'false'
uses: useblacksmith/build-push-action@v2
with:
context: .
file: Dockerfile.netsnmp
push: true
provenance: false
tags: ${{ env.NETSNMP_IMAGE }}-${{ matrix.arch }}
ensure-netsnmp-manifest:
name: Ensure net-snmp manifest
needs: ensure-netsnmp-base
runs-on: blacksmith-4vcpu-ubuntu-2404
permissions:
packages: write
steps:
- name: Log in to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Check if manifest exists
id: check
run: |
if docker buildx imagetools inspect ${{ env.NETSNMP_IMAGE }} > /dev/null 2>&1; then
echo "exists=true" >> $GITHUB_OUTPUT
else
echo "exists=false" >> $GITHUB_OUTPUT
fi
- name: Create multi-arch manifest
if: steps.check.outputs.exists == 'false'
run: |
docker buildx imagetools create \
-t ${{ env.NETSNMP_IMAGE }} \
${{ env.NETSNMP_IMAGE }}-amd64 \
${{ env.NETSNMP_IMAGE }}-arm64
test:
name: Test
runs-on: blacksmith-4vcpu-ubuntu-2404
@ -167,7 +84,7 @@ jobs:
build-branch:
name: Build (Branch)
needs: [test, ensure-netsnmp-manifest]
needs: [test]
runs-on: blacksmith-4vcpu-ubuntu-2404
if: github.event_name == 'push' && github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/')
permissions:
@ -240,7 +157,7 @@ jobs:
# Build architecture-specific images in parallel (native runners)
build-release:
name: Build (${{ matrix.arch }})
needs: [test, version, ensure-netsnmp-manifest]
needs: [test, version]
runs-on: ${{ matrix.runner }}
if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')
permissions:

99
.github/workflows/netsnmp-base.yml vendored
View file

@ -1,99 +0,0 @@
# Builds the net-snmp base image used by the agent Dockerfile.
# Only needs to run when the net-snmp version changes.
#
# Trigger manually or on changes to Dockerfile.netsnmp.
name: Build net-snmp base image
on:
push:
branches: [main]
paths: [Dockerfile.netsnmp]
workflow_dispatch:
inputs:
netsnmp_version:
description: "Net-SNMP version to build"
default: "5.9.5.2"
required: true
env:
REGISTRY: ghcr.io
IMAGE_NAME: towerops-app/netsnmp-alpine
jobs:
build:
name: Build (${{ matrix.arch }})
runs-on: ${{ matrix.runner }}
permissions:
contents: read
packages: write
strategy:
fail-fast: false
matrix:
include:
- platform: linux/amd64
arch: amd64
runner: blacksmith-4vcpu-ubuntu-2404
- platform: linux/arm64
arch: arm64
runner: blacksmith-4vcpu-ubuntu-2404-arm
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Blacksmith Builder
uses: useblacksmith/setup-docker-builder@v1
- name: Log in to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push
uses: useblacksmith/build-push-action@v2
with:
context: .
file: Dockerfile.netsnmp
platforms: ${{ matrix.platform }}
push: true
provenance: false
build-args: |
NETSNMP_VERSION=${{ inputs.netsnmp_version || '5.9.5.2' }}
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ inputs.netsnmp_version || '5.9.5.2' }}-${{ matrix.arch }}
manifest:
name: Create Manifest
needs: build
runs-on: blacksmith-4vcpu-ubuntu-2404
permissions:
contents: read
packages: write
steps:
- name: Log in to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Create multi-arch manifest
run: |
VERSION="${{ inputs.netsnmp_version || '5.9.5.2' }}"
IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
docker buildx imagetools create -t ${IMAGE}:${VERSION} \
${IMAGE}:${VERSION}-amd64 \
${IMAGE}:${VERSION}-arm64
docker buildx imagetools create -t ${IMAGE}:latest \
${IMAGE}:${VERSION}-amd64 \
${IMAGE}:${VERSION}-arm64
- name: Summary
run: |
echo "### net-snmp base image built" >> $GITHUB_STEP_SUMMARY
echo "Version: ${{ inputs.netsnmp_version || '5.9.5.2' }}" >> $GITHUB_STEP_SUMMARY
echo "Image: \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ inputs.netsnmp_version || '5.9.5.2' }}\`" >> $GITHUB_STEP_SUMMARY

52
Dockerfile
View file

@ -1,10 +1,6 @@
# syntax=docker/dockerfile:1.4
# Pre-built net-snmp 5.9.5.2 (fixes segfaults in Alpine's 5.9.4)
# Built from Dockerfile.netsnmp via .github/workflows/netsnmp-base.yml
FROM ghcr.io/towerops-app/netsnmp-alpine:5.9.5.2 AS netsnmp
# Build stage
FROM rust:1.93-alpine AS builder
# Build stage - Debian bookworm with glibc (fixes musl fork-safety SIGSEGV)
FROM rust:1.93-bookworm AS builder
# Build arguments provided by Docker buildx
ARG TARGETPLATFORM
@ -13,20 +9,23 @@ ARG VERSION=0.1.0-unknown
WORKDIR /app
# Install build dependencies (net-snmp headers/libs from pre-built image)
RUN apk add --no-cache musl-dev protobuf-dev openssl-dev openssl-libs-static cmake perl g++
COPY --from=netsnmp /usr/include/net-snmp /usr/include/net-snmp
COPY --from=netsnmp /usr/lib/libnetsnmp* /usr/lib/
COPY --from=netsnmp /usr/lib/libsnmp* /usr/lib/
# Install build dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
protobuf-compiler \
libsnmp-dev \
cmake \
g++ \
pkg-config \
libssl-dev \
&& rm -rf /var/lib/apt/lists/*
# Determine Rust target based on platform and add it
# Determine Rust target based on platform
RUN case "$TARGETPLATFORM" in \
"linux/amd64") RUST_TARGET="x86_64-unknown-linux-musl" ;; \
"linux/arm64") RUST_TARGET="aarch64-unknown-linux-musl" ;; \
"linux/amd64") RUST_TARGET="x86_64-unknown-linux-gnu" ;; \
"linux/arm64") RUST_TARGET="aarch64-unknown-linux-gnu" ;; \
*) echo "Unsupported platform: $TARGETPLATFORM" && exit 1 ;; \
esac && \
echo "$RUST_TARGET" > /tmp/rust-target && \
rustup target add "$RUST_TARGET"
echo "$RUST_TARGET" > /tmp/rust-target
# Copy manifests and build files
COPY Cargo.toml Cargo.lock build.rs ./
@ -59,23 +58,24 @@ RUN --mount=type=cache,id=cargo-registry-${TARGETARCH},target=/usr/local/cargo/r
BUILD_VERSION="$VERSION" cargo build --release --target "$RUST_TARGET" && \
cp "target/$RUST_TARGET/release/towerops-agent" /tmp/towerops-agent
# Runtime stage - must match builder's Alpine version for ABI compatibility
# (rust:1.93-alpine uses Alpine 3.23)
FROM alpine:3.23
# Runtime stage - Debian slim with glibc
FROM debian:12-slim
# Install runtime dependencies
# iputils provides ping with setuid root (doesn't require CAP_NET_RAW)
RUN apk add --no-cache ca-certificates iputils openssl
# Copy net-snmp 5.9.5.2 shared libraries from pre-built image
COPY --from=netsnmp /usr/lib/libnetsnmp.so* /usr/lib/
# iputils-ping provides ping with setuid root (doesn't require CAP_NET_RAW)
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates \
iputils-ping \
libsnmp40 \
openssl \
&& rm -rf /var/lib/apt/lists/*
# Copy binary from builder
COPY --from=builder /tmp/towerops-agent /usr/local/bin/towerops-agent
# Create non-root user
RUN addgroup -g 1000 towerops && \
adduser -D -u 1000 -G towerops towerops
RUN groupadd -g 1000 towerops && \
useradd -u 1000 -g towerops -s /bin/false towerops
# Allow non-root user to overwrite binary during self-update
RUN chown towerops /usr/local/bin/towerops-agent

68
Dockerfile.netsnmp
View file

@ -1,68 +0,0 @@
# syntax=docker/dockerfile:1.4
# Base image with net-snmp 5.9.5.2 compiled from source
# Alpine 3.23 ships 5.9.4 which has segfault bugs in varbind construction
# and buffer overflows in octet string handling.
#
# Build and push with:
# docker buildx build -f Dockerfile.netsnmp --platform linux/amd64,linux/arm64 \
# -t ghcr.io/towerops-app/netsnmp-alpine:5.9.5.2 --push .
#
# Or use the GitHub Actions workflow: .github/workflows/netsnmp-base.yml
FROM alpine:3.23 AS builder
ARG NETSNMP_VERSION=5.9.5.2
# Install build dependencies
RUN apk add --no-cache \
gcc \
g++ \
make \
musl-dev \
openssl-dev \
linux-headers \
perl \
curl \
file
# Download and extract net-snmp source
RUN curl -fsSL "https://downloads.sourceforge.net/project/net-snmp/net-snmp/${NETSNMP_VERSION}/net-snmp-${NETSNMP_VERSION}.tar.gz" \
-o /tmp/net-snmp.tar.gz && \
tar xzf /tmp/net-snmp.tar.gz -C /tmp && \
rm /tmp/net-snmp.tar.gz
WORKDIR /tmp/net-snmp-${NETSNMP_VERSION}
# Configure with minimal features (we only need libnetsnmp)
RUN ./configure \
--prefix=/usr \
--sysconfdir=/etc \
--with-defaults \
--with-openssl \
--enable-shared \
--enable-static \
--disable-applications \
--disable-manuals \
--disable-scripts \
--without-perl-modules \
--without-python-modules \
--disable-embedded-perl \
--with-default-snmp-version=3
# Build and install
RUN make -j$(nproc) && \
make install DESTDIR=/netsnmp-install
# Final stage: slim image with just the compiled artifacts
FROM alpine:3.23
# Install runtime dependency (OpenSSL) that net-snmp links against
RUN apk add --no-cache openssl-dev musl-dev
# Copy compiled net-snmp (headers, static libs, shared libs)
COPY --from=builder /netsnmp-install/usr/include/net-snmp /usr/include/net-snmp
COPY --from=builder /netsnmp-install/usr/lib/libnetsnmp* /usr/lib/
COPY --from=builder /netsnmp-install/usr/lib/libsnmp* /usr/lib/
# Ensure shared library cache is updated
RUN ldconfig /usr/lib 2>/dev/null || true

86
native/snmp_helper.c
View file

@ -282,13 +282,30 @@ int snmp_get(
struct variable_list *var = response->variables;
*value_type = var->type;
// Handle SNMP exception types (NoSuchObject, NoSuchInstance, EndOfMibView)
if (var->type == SNMP_NOSUCHOBJECT ||
var->type == SNMP_NOSUCHINSTANCE ||
var->type == SNMP_ENDOFMIBVIEW) {
if (error_buf && error_buf_len > 0) {
const char *label = var->type == SNMP_NOSUCHOBJECT ? "noSuchObject" :
var->type == SNMP_NOSUCHINSTANCE ? "noSuchInstance" :
"endOfMibView";
snprintf(error_buf, error_buf_len, "%s", label);
}
snmp_free_pdu(response);
return -1;
}
switch (var->type) {
case ASN_OCTET_STR:
case ASN_OPAQUE:
case ASN_IPADDRESS:
if (var->val_len <= value_buf_len) {
if (var->val.string && var->val_len > 0 &&
var->val_len <= value_buf_len) {
memcpy(value_buf, var->val.string, var->val_len);
result = (int)var->val_len;
} else if (!var->val.string || var->val_len == 0) {
result = 0; // Empty string
} else {
if (error_buf && error_buf_len > 0) {
snprintf(error_buf, error_buf_len, "Buffer too small");
@ -301,31 +318,33 @@ int snmp_get(
case ASN_GAUGE:
case ASN_TIMETICKS:
case ASN_UINTEGER:
if (sizeof(long) <= value_buf_len) {
if (var->val.integer && sizeof(long) <= value_buf_len) {
*((long*)value_buf) = *var->val.integer;
result = sizeof(long);
}
break;
case ASN_COUNTER64:
if (sizeof(struct counter64) <= value_buf_len) {
if (var->val.counter64 &&
sizeof(struct counter64) <= value_buf_len) {
memcpy(value_buf, var->val.counter64, sizeof(struct counter64));
result = sizeof(struct counter64);
}
break;
case ASN_OBJECT_ID:
// Convert OID to string representation
{
if (var->val.objid && var->val_len > 0) {
char oid_buf[256];
snprint_objid(oid_buf, sizeof(oid_buf), var->val.objid, var->val_len / sizeof(oid));
snprint_objid(oid_buf, sizeof(oid_buf), var->val.objid,
var->val_len / sizeof(oid));
size_t oid_str_len = strlen(oid_buf);
if (oid_str_len <= value_buf_len) {
memcpy(value_buf, oid_buf, oid_str_len);
result = (int)oid_str_len;
} else {
if (error_buf && error_buf_len > 0) {
snprintf(error_buf, error_buf_len, "Buffer too small for OID string");
snprintf(error_buf, error_buf_len,
"Buffer too small for OID string");
}
}
}
@ -339,7 +358,8 @@ int snmp_get(
default:
// Unknown type
if (error_buf && error_buf_len > 0) {
snprintf(error_buf, error_buf_len, "Unsupported type: %d", var->type);
snprintf(error_buf, error_buf_len,
"Unsupported type: %d", var->type);
}
break;
}
@ -412,13 +432,39 @@ int snmp_walk(
break;
}
// Handle SNMP exception types that indicate end-of-data or missing values.
// These have type 0x80 (NoSuchObject), 0x81 (NoSuchInstance),
// 0x82 (EndOfMibView) and their val pointers may be NULL.
if (var->type == SNMP_NOSUCHOBJECT ||
var->type == SNMP_NOSUCHINSTANCE) {
// Object/instance doesn't exist at this index - skip and continue walk
// Update OID for next iteration
if (var->name_length <= MAX_OID_LEN) {
memcpy(name, var->name, var->name_length * sizeof(oid));
name_length = var->name_length;
} else {
running = 0;
}
snmp_free_pdu(response);
continue;
}
if (var->type == SNMP_ENDOFMIBVIEW) {
// No more data available - terminate the walk
snmp_free_pdu(response);
break;
}
// Store result
snmp_walk_result_t *res = &results[*num_results];
// Convert OID to string
snprint_objid(res->oid, sizeof(res->oid), var->name, var->name_length);
// Store value
// Store value - check for NULL val pointers before every dereference.
// After fork() from a multi-threaded process, net-snmp's internal state
// can be inconsistent, potentially leaving val pointers NULL even for
// standard types.
res->value_type = var->type;
res->value_len = 0;
@ -426,17 +472,18 @@ int snmp_walk(
case ASN_OCTET_STR:
case ASN_OPAQUE:
case ASN_IPADDRESS:
if (var->val_len <= sizeof(res->value)) {
if (var->val.string && var->val_len > 0 &&
var->val_len <= sizeof(res->value)) {
memcpy(res->value, var->val.string, var->val_len);
res->value_len = var->val_len;
}
break;
case ASN_OBJECT_ID:
// Convert OID to string representation
{
if (var->val.objid && var->val_len > 0) {
char oid_buf[256];
snprint_objid(oid_buf, sizeof(oid_buf), var->val.objid, var->val_len / sizeof(oid));
snprint_objid(oid_buf, sizeof(oid_buf), var->val.objid,
var->val_len / sizeof(oid));
size_t oid_str_len = strlen(oid_buf);
if (oid_str_len < sizeof(res->value)) {
memcpy(res->value, oid_buf, oid_str_len);
@ -450,18 +497,27 @@ int snmp_walk(
case ASN_GAUGE:
case ASN_TIMETICKS:
case ASN_UINTEGER:
if (sizeof(long) <= sizeof(res->value)) {
if (var->val.integer && sizeof(long) <= sizeof(res->value)) {
*((long*)res->value) = *var->val.integer;
res->value_len = sizeof(long);
}
break;
case ASN_COUNTER64:
if (sizeof(struct counter64) <= sizeof(res->value)) {
if (var->val.counter64 &&
sizeof(struct counter64) <= sizeof(res->value)) {
memcpy(res->value, var->val.counter64, sizeof(struct counter64));
res->value_len = sizeof(struct counter64);
}
break;
case ASN_NULL:
// NULL values are valid but contain no data - skip
break;
default:
// Unknown or unsupported type - skip silently
break;
}
if (res->value_len > 0) {

1009
tests/snmp_crash_test.rs Normal file
View file

File diff suppressed because it is too large Load diff