Add exponential backoff retry logic for WebSocket reconnection

- Starts with 1 second delay, doubles each retry
- Caps at 60 seconds between attempts
- Resets delay counter on successful connection
- Prevents constant reconnection hammering
This commit is contained in:
Graham McIntire 2026-01-16 18:28:42 -06:00
parent 6223638acf
commit 451b530641
No known key found for this signature in database

View file

@ -6,8 +6,10 @@ mod websocket_client;
use chrono::Local; use chrono::Local;
use clap::Parser; use clap::Parser;
use log::{info, warn, LevelFilter, Metadata, Record}; use log::{error, info, warn, LevelFilter, Metadata, Record};
use std::env; use std::env;
use std::time::Duration;
use tokio::time::sleep;
use websocket_client::AgentClient; use websocket_client::AgentClient;
/// Minimal logger that writes to stderr with timestamps /// Minimal logger that writes to stderr with timestamps
@ -88,20 +90,45 @@ async fn main() {
} }
}); });
// Connect to Towerops server via WebSocket // Retry loop with exponential backoff
let mut client = match AgentClient::connect(&ws_url, &args.token).await { let mut retry_delay = Duration::from_secs(1);
Ok(client) => client, let max_retry_delay = Duration::from_secs(60);
Err(e) => { let mut attempt = 0;
eprintln!("Failed to connect to server: {}", e);
std::process::exit(1); loop {
attempt += 1;
if attempt > 1 {
info!(
"Retry attempt {} - waiting {} seconds before reconnecting",
attempt,
retry_delay.as_secs()
);
sleep(retry_delay).await;
// Exponential backoff: double the delay, capped at max
retry_delay = std::cmp::min(retry_delay * 2, max_retry_delay);
} }
};
// Run the agent event loop // Connect to Towerops server via WebSocket
if let Err(e) = client.run().await { let mut client = match AgentClient::connect(&ws_url, &args.token).await {
eprintln!("Agent error: {}", e); Ok(client) => {
std::process::exit(1); info!("Successfully connected to server");
// Reset retry delay on successful connection
retry_delay = Duration::from_secs(1);
attempt = 0;
client
}
Err(e) => {
error!("Failed to connect to server: {}", e);
continue;
}
};
// Run the agent event loop
if let Err(e) = client.run().await {
error!("Agent disconnected: {}", e);
// Loop will retry with backoff
}
} }
info!("Agent shutting down");
} }