Add CI/CD monitoring script
Features: - Auto-detects project from git remote - Shows real-time pipeline progress - Color-coded job statuses - Links to failed job logs - For main app: monitors Kubernetes rollout - Auto-refreshes every 5 seconds Usage: export GITLAB_TOKEN=<token> ./scripts/monitor-deploy.sh [app|agent] Or just run from repo directory for auto-detection
This commit is contained in:
parent
b70ddfe166
commit
c1d944a612
2 changed files with 472 additions and 0 deletions
134
scripts/README.md
Normal file
134
scripts/README.md
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
# Deployment Monitoring Scripts
|
||||
|
||||
## monitor-deploy.sh
|
||||
|
||||
Real-time monitoring of GitLab CI/CD pipelines and Kubernetes deployments.
|
||||
|
||||
### Features
|
||||
|
||||
- Auto-detects project (main app vs agent) from git remote
|
||||
- Shows live pipeline progress with job statuses
|
||||
- Color-coded output with icons
|
||||
- Auto-refreshes every 5 seconds
|
||||
- Shows failed job logs
|
||||
- For main app: automatically monitors Kubernetes rollout after successful build
|
||||
|
||||
### Setup
|
||||
|
||||
1. **Get GitLab Personal Access Token**
|
||||
|
||||
Visit: https://gitlab.com/-/profile/personal_access_tokens
|
||||
|
||||
Create a token with scope: `read_api`
|
||||
|
||||
2. **Export token**
|
||||
|
||||
```bash
|
||||
export GITLAB_TOKEN=<your-token>
|
||||
```
|
||||
|
||||
Add to your `~/.zshrc` or `~/.bashrc`:
|
||||
```bash
|
||||
export GITLAB_TOKEN=glpat-xxxxxxxxxxxxxxxxxxxx
|
||||
```
|
||||
|
||||
3. **Install dependencies**
|
||||
|
||||
```bash
|
||||
brew install jq
|
||||
```
|
||||
|
||||
### Usage
|
||||
|
||||
**From main towerops directory:**
|
||||
```bash
|
||||
# Auto-detect project and monitor
|
||||
./scripts/monitor-deploy.sh
|
||||
|
||||
# Or explicitly specify
|
||||
./scripts/monitor-deploy.sh app
|
||||
```
|
||||
|
||||
**From towerops-agent directory:**
|
||||
```bash
|
||||
# Auto-detect and monitor agent
|
||||
../scripts/monitor-deploy.sh
|
||||
|
||||
# Or explicitly specify
|
||||
../scripts/monitor-deploy.sh agent
|
||||
```
|
||||
|
||||
**After git push:**
|
||||
```bash
|
||||
git push && ./scripts/monitor-deploy.sh
|
||||
```
|
||||
|
||||
### What it shows
|
||||
|
||||
**Pipeline Status:**
|
||||
- ✓ Success (green)
|
||||
- ✗ Failed (red)
|
||||
- → Running (blue)
|
||||
- ⏳ Pending (yellow)
|
||||
|
||||
**For each job:**
|
||||
- Stage (test, build, release)
|
||||
- Status icon
|
||||
- Job name
|
||||
- Duration
|
||||
|
||||
**For main app on main branch:**
|
||||
- Triggers Kubernetes rollout restart
|
||||
- Monitors deployment progress
|
||||
- Shows pod status
|
||||
- Displays current image version
|
||||
|
||||
### Example Output
|
||||
|
||||
```
|
||||
╔════════════════════════════════════════════════════════════════╗
|
||||
║ Towerops - CI/CD Monitor ║
|
||||
╚════════════════════════════════════════════════════════════════╝
|
||||
|
||||
Branch: main
|
||||
Pipeline: #12345
|
||||
Status: → running
|
||||
|
||||
Jobs:
|
||||
[test] ✓ test 45s
|
||||
[build] → build:main -
|
||||
[release] ⏳ release:multiarch -
|
||||
|
||||
Refreshing in 5s... (Ctrl+C to stop)
|
||||
```
|
||||
|
||||
### Troubleshooting
|
||||
|
||||
**"GITLAB_TOKEN not set"**
|
||||
- Export your GitLab personal access token
|
||||
|
||||
**"jq is required"**
|
||||
- Install: `brew install jq`
|
||||
|
||||
**"No pipeline found"**
|
||||
- Push to trigger a new pipeline
|
||||
- Check that you're on the correct branch
|
||||
|
||||
**"Deployment not found"**
|
||||
- Only applies to main app on main branch
|
||||
- Verify kubectl context: `/opt/homebrew/bin/kubectl config current-context`
|
||||
|
||||
### Project Detection
|
||||
|
||||
The script auto-detects the project from git remote URL:
|
||||
- URL contains "towerops-agent" → Agent
|
||||
- URL contains "towerops" → Main app
|
||||
|
||||
Manual override: `./scripts/monitor-deploy.sh [app|agent]`
|
||||
|
||||
### GitLab Project IDs
|
||||
|
||||
- Main app: `graham/towerops`
|
||||
- Agent: `towerops/towerops-agent`
|
||||
|
||||
Update these in the script if your project paths differ.
|
||||
338
scripts/monitor-deploy.sh
Executable file
338
scripts/monitor-deploy.sh
Executable file
|
|
@ -0,0 +1,338 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Monitor GitLab CI/CD pipeline and Kubernetes deployment progress
|
||||
# Works for both towerops main app and towerops-agent
|
||||
|
||||
set -e
|
||||
|
||||
# Colors
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
CYAN='\033[0;36m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Icons
|
||||
CHECK="✓"
|
||||
CROSS="✗"
|
||||
ARROW="→"
|
||||
CLOCK="⏳"
|
||||
|
||||
# Config
|
||||
KUBECTL="/opt/homebrew/bin/kubectl"
|
||||
REFRESH_INTERVAL=5
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 [app|agent]"
|
||||
echo ""
|
||||
echo "Monitor CI/CD pipeline and deployment progress."
|
||||
echo "Auto-detects project if run from git repository."
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " app - Monitor towerops main application"
|
||||
echo " agent - Monitor towerops-agent"
|
||||
echo ""
|
||||
echo "Environment variables:"
|
||||
echo " GITLAB_TOKEN - Required for API access"
|
||||
echo " Get from: https://gitlab.com/-/profile/personal_access_tokens"
|
||||
echo " Scopes needed: read_api"
|
||||
exit 1
|
||||
}
|
||||
|
||||
check_dependencies() {
|
||||
if [ -z "$GITLAB_TOKEN" ]; then
|
||||
echo -e "${RED}Error: GITLAB_TOKEN environment variable not set${NC}"
|
||||
echo "Get a token from: https://gitlab.com/-/profile/personal_access_tokens"
|
||||
echo "Then export GITLAB_TOKEN=<your-token>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v jq &> /dev/null; then
|
||||
echo -e "${RED}Error: jq is required but not installed${NC}"
|
||||
echo "Install with: brew install jq"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v curl &> /dev/null; then
|
||||
echo -e "${RED}Error: curl is required but not installed${NC}"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
detect_project() {
|
||||
# Try to detect from git remote
|
||||
if git remote get-url origin &>/dev/null; then
|
||||
local remote_url=$(git remote get-url origin)
|
||||
|
||||
if echo "$remote_url" | grep -q "towerops-agent"; then
|
||||
echo "agent"
|
||||
elif echo "$remote_url" | grep -q "towerops"; then
|
||||
echo "app"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
get_project_info() {
|
||||
local project=$1
|
||||
|
||||
case $project in
|
||||
app)
|
||||
PROJECT_ID="graham%2Ftowerops" # URL-encoded graham/towerops
|
||||
PROJECT_NAME="Towerops"
|
||||
K8S_NAMESPACE="towerops"
|
||||
K8S_DEPLOYMENT="towerops"
|
||||
HAS_K8S=true
|
||||
;;
|
||||
agent)
|
||||
PROJECT_ID="towerops%2Ftowerops-agent" # URL-encoded towerops/towerops-agent
|
||||
PROJECT_NAME="Towerops Agent"
|
||||
HAS_K8S=false
|
||||
;;
|
||||
*)
|
||||
echo -e "${RED}Unknown project: $project${NC}"
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
gitlab_api() {
|
||||
local endpoint=$1
|
||||
curl -s -H "PRIVATE-TOKEN: $GITLAB_TOKEN" \
|
||||
"https://gitlab.com/api/v4${endpoint}"
|
||||
}
|
||||
|
||||
get_current_branch() {
|
||||
git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "main"
|
||||
}
|
||||
|
||||
get_latest_pipeline() {
|
||||
local branch=$1
|
||||
gitlab_api "/projects/${PROJECT_ID}/pipelines?ref=${branch}&per_page=1" | \
|
||||
jq -r '.[0] // empty'
|
||||
}
|
||||
|
||||
get_pipeline_jobs() {
|
||||
local pipeline_id=$1
|
||||
gitlab_api "/projects/${PROJECT_ID}/pipelines/${pipeline_id}/jobs"
|
||||
}
|
||||
|
||||
get_job_log() {
|
||||
local job_id=$1
|
||||
gitlab_api "/projects/${PROJECT_ID}/jobs/${job_id}/trace"
|
||||
}
|
||||
|
||||
status_icon() {
|
||||
local status=$1
|
||||
case $status in
|
||||
success|passed)
|
||||
echo -e "${GREEN}${CHECK}${NC}"
|
||||
;;
|
||||
failed)
|
||||
echo -e "${RED}${CROSS}${NC}"
|
||||
;;
|
||||
running)
|
||||
echo -e "${BLUE}${ARROW}${NC}"
|
||||
;;
|
||||
pending|created)
|
||||
echo -e "${YELLOW}${CLOCK}${NC}"
|
||||
;;
|
||||
canceled|skipped)
|
||||
echo -e "${YELLOW}-${NC}"
|
||||
;;
|
||||
*)
|
||||
echo -e "${CYAN}?${NC}"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
format_duration() {
|
||||
local seconds=$1
|
||||
if [ -z "$seconds" ] || [ "$seconds" = "null" ]; then
|
||||
echo "-"
|
||||
return
|
||||
fi
|
||||
|
||||
local mins=$((seconds / 60))
|
||||
local secs=$((seconds % 60))
|
||||
|
||||
if [ $mins -gt 0 ]; then
|
||||
echo "${mins}m ${secs}s"
|
||||
else
|
||||
echo "${secs}s"
|
||||
fi
|
||||
}
|
||||
|
||||
print_header() {
|
||||
clear
|
||||
echo -e "${CYAN}╔════════════════════════════════════════════════════════════════╗${NC}"
|
||||
echo -e "${CYAN}║${NC} ${BLUE}${PROJECT_NAME} - CI/CD Monitor${NC} ${CYAN}║${NC}"
|
||||
echo -e "${CYAN}╚════════════════════════════════════════════════════════════════╝${NC}"
|
||||
echo ""
|
||||
}
|
||||
|
||||
monitor_pipeline() {
|
||||
local branch=$(get_current_branch)
|
||||
|
||||
echo -e "${BLUE}Branch:${NC} $branch"
|
||||
echo -e "${BLUE}Fetching latest pipeline...${NC}"
|
||||
echo ""
|
||||
|
||||
local pipeline=$(get_latest_pipeline "$branch")
|
||||
|
||||
if [ -z "$pipeline" ] || [ "$pipeline" = "null" ]; then
|
||||
echo -e "${YELLOW}No pipeline found for branch: $branch${NC}"
|
||||
echo "Push to trigger a new pipeline"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
local pipeline_id=$(echo "$pipeline" | jq -r '.id')
|
||||
local pipeline_status=$(echo "$pipeline" | jq -r '.status')
|
||||
local pipeline_web_url=$(echo "$pipeline" | jq -r '.web_url')
|
||||
|
||||
echo -e "${BLUE}Pipeline:${NC} #$pipeline_id"
|
||||
echo -e "${BLUE}Status:${NC} $(status_icon $pipeline_status) $pipeline_status"
|
||||
echo -e "${BLUE}URL:${NC} $pipeline_web_url"
|
||||
echo ""
|
||||
|
||||
local start_time=$(date +%s)
|
||||
|
||||
# Monitor until complete
|
||||
while true; do
|
||||
print_header
|
||||
|
||||
echo -e "${BLUE}Branch:${NC} $branch"
|
||||
echo -e "${BLUE}Pipeline:${NC} #$pipeline_id"
|
||||
echo -e "${BLUE}Status:${NC} $(status_icon $pipeline_status) $pipeline_status"
|
||||
echo ""
|
||||
|
||||
# Get and display jobs
|
||||
local jobs=$(get_pipeline_jobs "$pipeline_id")
|
||||
|
||||
echo -e "${CYAN}Jobs:${NC}"
|
||||
echo "$jobs" | jq -r '.[] | "\(.stage)|\(.name)|\(.status)|\(.duration)"' | \
|
||||
while IFS='|' read -r stage name status duration; do
|
||||
printf " %-12s $(status_icon $status) %-30s %s\n" \
|
||||
"[$stage]" "$name" "$(format_duration $duration)"
|
||||
done
|
||||
|
||||
echo ""
|
||||
|
||||
# Check for failures
|
||||
local failed_jobs=$(echo "$jobs" | jq -r '.[] | select(.status == "failed") | .id')
|
||||
if [ -n "$failed_jobs" ]; then
|
||||
echo -e "${RED}Failed Jobs:${NC}"
|
||||
for job_id in $failed_jobs; do
|
||||
local job_name=$(echo "$jobs" | jq -r ".[] | select(.id == $job_id) | .name")
|
||||
echo -e " ${RED}${CROSS}${NC} $job_name (Job #$job_id)"
|
||||
echo -e " ${BLUE}View logs:${NC} https://gitlab.com/$(echo $PROJECT_ID | sed 's/%2F/\//g')/-/jobs/$job_id"
|
||||
done
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Update pipeline status
|
||||
pipeline=$(get_latest_pipeline "$branch")
|
||||
local new_status=$(echo "$pipeline" | jq -r '.status')
|
||||
|
||||
if [ "$new_status" != "$pipeline_status" ]; then
|
||||
pipeline_status=$new_status
|
||||
fi
|
||||
|
||||
# Check if complete
|
||||
if [[ "$pipeline_status" == "success" ]] || [[ "$pipeline_status" == "failed" ]] || \
|
||||
[[ "$pipeline_status" == "canceled" ]] || [[ "$pipeline_status" == "skipped" ]]; then
|
||||
|
||||
local elapsed=$(($(date +%s) - start_time))
|
||||
echo -e "${BLUE}Pipeline completed in:${NC} $(format_duration $elapsed)"
|
||||
echo ""
|
||||
|
||||
if [ "$pipeline_status" = "success" ]; then
|
||||
echo -e "${GREEN}${CHECK} Pipeline succeeded!${NC}"
|
||||
|
||||
# Monitor Kubernetes deployment if applicable
|
||||
if [ "$HAS_K8S" = true ] && [ "$branch" = "main" ]; then
|
||||
echo ""
|
||||
monitor_k8s_deployment
|
||||
fi
|
||||
else
|
||||
echo -e "${RED}${CROSS} Pipeline $pipeline_status${NC}"
|
||||
fi
|
||||
|
||||
break
|
||||
fi
|
||||
|
||||
echo -e "${YELLOW}Refreshing in ${REFRESH_INTERVAL}s... (Ctrl+C to stop)${NC}"
|
||||
sleep $REFRESH_INTERVAL
|
||||
done
|
||||
}
|
||||
|
||||
monitor_k8s_deployment() {
|
||||
echo -e "${CYAN}╔════════════════════════════════════════════════════════════════╗${NC}"
|
||||
echo -e "${CYAN}║${NC} ${BLUE}Kubernetes Deployment${NC} ${CYAN}║${NC}"
|
||||
echo -e "${CYAN}╚════════════════════════════════════════════════════════════════╝${NC}"
|
||||
echo ""
|
||||
|
||||
if ! $KUBECTL get deployment -n $K8S_NAMESPACE $K8S_DEPLOYMENT &>/dev/null; then
|
||||
echo -e "${YELLOW}Warning: Deployment $K8S_DEPLOYMENT not found in namespace $K8S_NAMESPACE${NC}"
|
||||
return
|
||||
fi
|
||||
|
||||
echo -e "${BLUE}Namespace:${NC} $K8S_NAMESPACE"
|
||||
echo -e "${BLUE}Deployment:${NC} $K8S_DEPLOYMENT"
|
||||
echo ""
|
||||
|
||||
# Trigger rollout restart
|
||||
echo -e "${BLUE}Restarting deployment...${NC}"
|
||||
$KUBECTL -n $K8S_NAMESPACE rollout restart deployment/$K8S_DEPLOYMENT
|
||||
echo ""
|
||||
|
||||
# Monitor rollout status
|
||||
echo -e "${BLUE}Waiting for rollout to complete...${NC}"
|
||||
if $KUBECTL -n $K8S_NAMESPACE rollout status deployment/$K8S_DEPLOYMENT --timeout=300s; then
|
||||
echo ""
|
||||
echo -e "${GREEN}${CHECK} Deployment successful!${NC}"
|
||||
echo ""
|
||||
|
||||
# Show pod status
|
||||
echo -e "${BLUE}Pod status:${NC}"
|
||||
$KUBECTL -n $K8S_NAMESPACE get pods -l app=$K8S_DEPLOYMENT
|
||||
echo ""
|
||||
|
||||
# Get image version
|
||||
local image=$($KUBECTL -n $K8S_NAMESPACE get deployment $K8S_DEPLOYMENT -o jsonpath='{.spec.template.spec.containers[0].image}')
|
||||
echo -e "${BLUE}Current image:${NC} $image"
|
||||
else
|
||||
echo ""
|
||||
echo -e "${RED}${CROSS} Deployment failed or timed out${NC}"
|
||||
echo ""
|
||||
echo -e "${BLUE}Recent events:${NC}"
|
||||
$KUBECTL -n $K8S_NAMESPACE get events --sort-by=.lastTimestamp | tail -10
|
||||
fi
|
||||
}
|
||||
|
||||
# Main
|
||||
main() {
|
||||
check_dependencies
|
||||
|
||||
local project=""
|
||||
|
||||
# Detect or use argument
|
||||
if [ $# -eq 0 ]; then
|
||||
project=$(detect_project)
|
||||
if [ -z "$project" ]; then
|
||||
echo -e "${YELLOW}Could not auto-detect project${NC}"
|
||||
usage
|
||||
fi
|
||||
echo -e "${BLUE}Auto-detected project:${NC} $project"
|
||||
echo ""
|
||||
else
|
||||
project=$1
|
||||
fi
|
||||
|
||||
get_project_info "$project"
|
||||
|
||||
print_header
|
||||
monitor_pipeline
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Loading…
Add table
Reference in a new issue