fix: use session-scoped context to prevent stuck reconnects

When the WebSocket breaks (broken pipe), the main loop can get stuck
in handleMessage -> dispatchJob -> submit if the worker pool is full.
submit blocks on ctx.Done() but ctx is the parent context which only
cancels on SIGINT/SIGTERM, not on connection errors.

Create a session-scoped context that cancels when read/write errors
occur. Pass it to handleMessage so blocked submit calls unblock
immediately, allowing the main loop to receive the write error and
reconnect.
This commit is contained in:
Graham McIntire 2026-03-23 13:47:08 -05:00
parent 3d98105853
commit 1b12a1bdd8
No known key found for this signature in database

View file

@ -100,6 +100,11 @@ func runSession(ctx context.Context, baseURL, token string) error {
slog.Info("connected", "agent_id", agentID)
// Session-scoped context: cancelled on write/read errors so blocked
// pool submits unblock immediately instead of waiting for workers to finish.
sessionCtx, sessionCancel := context.WithCancel(ctx)
defer sessionCancel()
// Channel for serializing WebSocket writes
writeCh := make(chan []byte, 10000)
@ -180,6 +185,7 @@ func runSession(ctx context.Context, baseURL, token string) error {
data, _, err := ws.ReadMessage()
if err != nil {
errCh <- err
sessionCancel() // Unblock any stuck pool submits
return
}
msgCh <- data
@ -236,6 +242,7 @@ func runSession(ctx context.Context, baseURL, token string) error {
case writeErrCh <- err:
default:
}
sessionCancel() // Unblock any stuck pool submits
return
}
}
@ -301,7 +308,7 @@ func runSession(ctx context.Context, baseURL, token string) error {
slog.Warn("invalid message", "error", err)
continue
}
if handleMessage(ctx, msg, pools, snmpResultCh, mikrotikResultCh, credTestResultCh, monitoringCheckCh, checkResultCh, lldpTopologyResultCh) {
if handleMessage(sessionCtx, msg, pools, snmpResultCh, mikrotikResultCh, credTestResultCh, monitoringCheckCh, checkResultCh, lldpTopologyResultCh) {
flushSnmpBatch()
return errRestartRequested
}