Add exponential backoff retry logic for WebSocket reconnection
- Starts with 1 second delay, doubles each retry - Caps at 60 seconds between attempts - Resets delay counter on successful connection - Prevents constant reconnection hammering
This commit is contained in:
parent
6223638acf
commit
451b530641
1 changed files with 41 additions and 14 deletions
55
src/main.rs
55
src/main.rs
|
|
@ -6,8 +6,10 @@ mod websocket_client;
|
||||||
|
|
||||||
use chrono::Local;
|
use chrono::Local;
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
use log::{info, warn, LevelFilter, Metadata, Record};
|
use log::{error, info, warn, LevelFilter, Metadata, Record};
|
||||||
use std::env;
|
use std::env;
|
||||||
|
use std::time::Duration;
|
||||||
|
use tokio::time::sleep;
|
||||||
use websocket_client::AgentClient;
|
use websocket_client::AgentClient;
|
||||||
|
|
||||||
/// Minimal logger that writes to stderr with timestamps
|
/// Minimal logger that writes to stderr with timestamps
|
||||||
|
|
@ -88,20 +90,45 @@ async fn main() {
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// Connect to Towerops server via WebSocket
|
// Retry loop with exponential backoff
|
||||||
let mut client = match AgentClient::connect(&ws_url, &args.token).await {
|
let mut retry_delay = Duration::from_secs(1);
|
||||||
Ok(client) => client,
|
let max_retry_delay = Duration::from_secs(60);
|
||||||
Err(e) => {
|
let mut attempt = 0;
|
||||||
eprintln!("Failed to connect to server: {}", e);
|
|
||||||
std::process::exit(1);
|
loop {
|
||||||
|
attempt += 1;
|
||||||
|
|
||||||
|
if attempt > 1 {
|
||||||
|
info!(
|
||||||
|
"Retry attempt {} - waiting {} seconds before reconnecting",
|
||||||
|
attempt,
|
||||||
|
retry_delay.as_secs()
|
||||||
|
);
|
||||||
|
sleep(retry_delay).await;
|
||||||
|
|
||||||
|
// Exponential backoff: double the delay, capped at max
|
||||||
|
retry_delay = std::cmp::min(retry_delay * 2, max_retry_delay);
|
||||||
}
|
}
|
||||||
};
|
|
||||||
|
|
||||||
// Run the agent event loop
|
// Connect to Towerops server via WebSocket
|
||||||
if let Err(e) = client.run().await {
|
let mut client = match AgentClient::connect(&ws_url, &args.token).await {
|
||||||
eprintln!("Agent error: {}", e);
|
Ok(client) => {
|
||||||
std::process::exit(1);
|
info!("Successfully connected to server");
|
||||||
|
// Reset retry delay on successful connection
|
||||||
|
retry_delay = Duration::from_secs(1);
|
||||||
|
attempt = 0;
|
||||||
|
client
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("Failed to connect to server: {}", e);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Run the agent event loop
|
||||||
|
if let Err(e) = client.run().await {
|
||||||
|
error!("Agent disconnected: {}", e);
|
||||||
|
// Loop will retry with backoff
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
info!("Agent shutting down");
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue