fix: add timeout to worker pool shutdown to prevent blocking reconnection
When the WebSocket connection drops, session cleanup waits for all worker pools to drain. SNMP/MikroTik operations against unresponsive devices can block for 30+ seconds per worker, preventing the agent from reconnecting during that time. Add stopWithTimeout(5s) so pool shutdown is bounded. If workers don't finish within the timeout, they're abandoned and reconnection proceeds immediately.
This commit is contained in:
parent
1be6392e24
commit
51e9345f1f
3 changed files with 52 additions and 4 deletions
17
agent.go
17
agent.go
|
|
@ -251,10 +251,19 @@ func runSession(ctx context.Context, baseURL, token string) error {
|
|||
startTime := time.Now()
|
||||
|
||||
defer func() {
|
||||
pools.snmp.stop()
|
||||
pools.mikrotik.stop()
|
||||
pools.ping.stop()
|
||||
pools.checks.stop()
|
||||
const poolShutdownTimeout = 5 * time.Second
|
||||
if !pools.snmp.stopWithTimeout(poolShutdownTimeout) {
|
||||
slog.Warn("snmp pool shutdown timed out, abandoning in-flight jobs")
|
||||
}
|
||||
if !pools.mikrotik.stopWithTimeout(poolShutdownTimeout) {
|
||||
slog.Warn("mikrotik pool shutdown timed out, abandoning in-flight jobs")
|
||||
}
|
||||
if !pools.ping.stopWithTimeout(poolShutdownTimeout) {
|
||||
slog.Warn("ping pool shutdown timed out, abandoning in-flight jobs")
|
||||
}
|
||||
if !pools.checks.stopWithTimeout(poolShutdownTimeout) {
|
||||
slog.Warn("checks pool shutdown timed out, abandoning in-flight jobs")
|
||||
}
|
||||
close(writeCh)
|
||||
writerWg.Wait()
|
||||
}()
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"log/slog"
|
||||
"runtime/debug"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// workerPool is a fixed-size goroutine pool for executing tasks.
|
||||
|
|
@ -55,3 +56,20 @@ func (p *workerPool) stop() {
|
|||
p.wg.Wait()
|
||||
})
|
||||
}
|
||||
|
||||
// stopWithTimeout closes the task channel and waits up to timeout for workers
|
||||
// to finish. Returns true if all workers completed, false if the timeout was
|
||||
// reached and some workers were abandoned.
|
||||
func (p *workerPool) stopWithTimeout(timeout time.Duration) bool {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
p.stop()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
return true
|
||||
case <-time.After(timeout):
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -57,6 +57,27 @@ func TestWorkerPool(t *testing.T) {
|
|||
pool.stop()
|
||||
pool.stop() // should not panic
|
||||
})
|
||||
|
||||
t.Run("stopWithTimeout returns true when workers finish in time", func(t *testing.T) {
|
||||
pool := newWorkerPool(2)
|
||||
pool.submit(context.Background(), func() {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
})
|
||||
if !pool.stopWithTimeout(time.Second) {
|
||||
t.Error("expected stopWithTimeout to return true")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("stopWithTimeout returns false when workers exceed timeout", func(t *testing.T) {
|
||||
pool := newWorkerPool(1)
|
||||
blocker := make(chan struct{})
|
||||
pool.submit(context.Background(), func() { <-blocker })
|
||||
|
||||
if pool.stopWithTimeout(50 * time.Millisecond) {
|
||||
t.Error("expected stopWithTimeout to return false")
|
||||
}
|
||||
close(blocker) // unblock for cleanup
|
||||
})
|
||||
}
|
||||
|
||||
func TestWorkerPoolRecoversPanic(t *testing.T) {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue