Graham McIntire
c1d944a612
Features: - Auto-detects project from git remote - Shows real-time pipeline progress - Color-coded job statuses - Links to failed job logs - For main app: monitors Kubernetes rollout - Auto-refreshes every 5 seconds Usage: export GITLAB_TOKEN=<token> ./scripts/monitor-deploy.sh [app|agent] Or just run from repo directory for auto-detection
338 lines
10 KiB
Bash
Executable file
338 lines
10 KiB
Bash
Executable file
#!/bin/bash
|
|
|
|
# Monitor GitLab CI/CD pipeline and Kubernetes deployment progress
|
|
# Works for both towerops main app and towerops-agent
|
|
|
|
set -e
|
|
|
|
# Colors
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
CYAN='\033[0;36m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Icons
|
|
CHECK="✓"
|
|
CROSS="✗"
|
|
ARROW="→"
|
|
CLOCK="⏳"
|
|
|
|
# Config
|
|
KUBECTL="/opt/homebrew/bin/kubectl"
|
|
REFRESH_INTERVAL=5
|
|
|
|
usage() {
|
|
echo "Usage: $0 [app|agent]"
|
|
echo ""
|
|
echo "Monitor CI/CD pipeline and deployment progress."
|
|
echo "Auto-detects project if run from git repository."
|
|
echo ""
|
|
echo "Options:"
|
|
echo " app - Monitor towerops main application"
|
|
echo " agent - Monitor towerops-agent"
|
|
echo ""
|
|
echo "Environment variables:"
|
|
echo " GITLAB_TOKEN - Required for API access"
|
|
echo " Get from: https://gitlab.com/-/profile/personal_access_tokens"
|
|
echo " Scopes needed: read_api"
|
|
exit 1
|
|
}
|
|
|
|
check_dependencies() {
|
|
if [ -z "$GITLAB_TOKEN" ]; then
|
|
echo -e "${RED}Error: GITLAB_TOKEN environment variable not set${NC}"
|
|
echo "Get a token from: https://gitlab.com/-/profile/personal_access_tokens"
|
|
echo "Then export GITLAB_TOKEN=<your-token>"
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v jq &> /dev/null; then
|
|
echo -e "${RED}Error: jq is required but not installed${NC}"
|
|
echo "Install with: brew install jq"
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v curl &> /dev/null; then
|
|
echo -e "${RED}Error: curl is required but not installed${NC}"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
detect_project() {
|
|
# Try to detect from git remote
|
|
if git remote get-url origin &>/dev/null; then
|
|
local remote_url=$(git remote get-url origin)
|
|
|
|
if echo "$remote_url" | grep -q "towerops-agent"; then
|
|
echo "agent"
|
|
elif echo "$remote_url" | grep -q "towerops"; then
|
|
echo "app"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
get_project_info() {
|
|
local project=$1
|
|
|
|
case $project in
|
|
app)
|
|
PROJECT_ID="graham%2Ftowerops" # URL-encoded graham/towerops
|
|
PROJECT_NAME="Towerops"
|
|
K8S_NAMESPACE="towerops"
|
|
K8S_DEPLOYMENT="towerops"
|
|
HAS_K8S=true
|
|
;;
|
|
agent)
|
|
PROJECT_ID="towerops%2Ftowerops-agent" # URL-encoded towerops/towerops-agent
|
|
PROJECT_NAME="Towerops Agent"
|
|
HAS_K8S=false
|
|
;;
|
|
*)
|
|
echo -e "${RED}Unknown project: $project${NC}"
|
|
usage
|
|
;;
|
|
esac
|
|
}
|
|
|
|
gitlab_api() {
|
|
local endpoint=$1
|
|
curl -s -H "PRIVATE-TOKEN: $GITLAB_TOKEN" \
|
|
"https://gitlab.com/api/v4${endpoint}"
|
|
}
|
|
|
|
get_current_branch() {
|
|
git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "main"
|
|
}
|
|
|
|
get_latest_pipeline() {
|
|
local branch=$1
|
|
gitlab_api "/projects/${PROJECT_ID}/pipelines?ref=${branch}&per_page=1" | \
|
|
jq -r '.[0] // empty'
|
|
}
|
|
|
|
get_pipeline_jobs() {
|
|
local pipeline_id=$1
|
|
gitlab_api "/projects/${PROJECT_ID}/pipelines/${pipeline_id}/jobs"
|
|
}
|
|
|
|
get_job_log() {
|
|
local job_id=$1
|
|
gitlab_api "/projects/${PROJECT_ID}/jobs/${job_id}/trace"
|
|
}
|
|
|
|
status_icon() {
|
|
local status=$1
|
|
case $status in
|
|
success|passed)
|
|
echo -e "${GREEN}${CHECK}${NC}"
|
|
;;
|
|
failed)
|
|
echo -e "${RED}${CROSS}${NC}"
|
|
;;
|
|
running)
|
|
echo -e "${BLUE}${ARROW}${NC}"
|
|
;;
|
|
pending|created)
|
|
echo -e "${YELLOW}${CLOCK}${NC}"
|
|
;;
|
|
canceled|skipped)
|
|
echo -e "${YELLOW}-${NC}"
|
|
;;
|
|
*)
|
|
echo -e "${CYAN}?${NC}"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
format_duration() {
|
|
local seconds=$1
|
|
if [ -z "$seconds" ] || [ "$seconds" = "null" ]; then
|
|
echo "-"
|
|
return
|
|
fi
|
|
|
|
local mins=$((seconds / 60))
|
|
local secs=$((seconds % 60))
|
|
|
|
if [ $mins -gt 0 ]; then
|
|
echo "${mins}m ${secs}s"
|
|
else
|
|
echo "${secs}s"
|
|
fi
|
|
}
|
|
|
|
print_header() {
|
|
clear
|
|
echo -e "${CYAN}╔════════════════════════════════════════════════════════════════╗${NC}"
|
|
echo -e "${CYAN}║${NC} ${BLUE}${PROJECT_NAME} - CI/CD Monitor${NC} ${CYAN}║${NC}"
|
|
echo -e "${CYAN}╚════════════════════════════════════════════════════════════════╝${NC}"
|
|
echo ""
|
|
}
|
|
|
|
monitor_pipeline() {
|
|
local branch=$(get_current_branch)
|
|
|
|
echo -e "${BLUE}Branch:${NC} $branch"
|
|
echo -e "${BLUE}Fetching latest pipeline...${NC}"
|
|
echo ""
|
|
|
|
local pipeline=$(get_latest_pipeline "$branch")
|
|
|
|
if [ -z "$pipeline" ] || [ "$pipeline" = "null" ]; then
|
|
echo -e "${YELLOW}No pipeline found for branch: $branch${NC}"
|
|
echo "Push to trigger a new pipeline"
|
|
exit 0
|
|
fi
|
|
|
|
local pipeline_id=$(echo "$pipeline" | jq -r '.id')
|
|
local pipeline_status=$(echo "$pipeline" | jq -r '.status')
|
|
local pipeline_web_url=$(echo "$pipeline" | jq -r '.web_url')
|
|
|
|
echo -e "${BLUE}Pipeline:${NC} #$pipeline_id"
|
|
echo -e "${BLUE}Status:${NC} $(status_icon $pipeline_status) $pipeline_status"
|
|
echo -e "${BLUE}URL:${NC} $pipeline_web_url"
|
|
echo ""
|
|
|
|
local start_time=$(date +%s)
|
|
|
|
# Monitor until complete
|
|
while true; do
|
|
print_header
|
|
|
|
echo -e "${BLUE}Branch:${NC} $branch"
|
|
echo -e "${BLUE}Pipeline:${NC} #$pipeline_id"
|
|
echo -e "${BLUE}Status:${NC} $(status_icon $pipeline_status) $pipeline_status"
|
|
echo ""
|
|
|
|
# Get and display jobs
|
|
local jobs=$(get_pipeline_jobs "$pipeline_id")
|
|
|
|
echo -e "${CYAN}Jobs:${NC}"
|
|
echo "$jobs" | jq -r '.[] | "\(.stage)|\(.name)|\(.status)|\(.duration)"' | \
|
|
while IFS='|' read -r stage name status duration; do
|
|
printf " %-12s $(status_icon $status) %-30s %s\n" \
|
|
"[$stage]" "$name" "$(format_duration $duration)"
|
|
done
|
|
|
|
echo ""
|
|
|
|
# Check for failures
|
|
local failed_jobs=$(echo "$jobs" | jq -r '.[] | select(.status == "failed") | .id')
|
|
if [ -n "$failed_jobs" ]; then
|
|
echo -e "${RED}Failed Jobs:${NC}"
|
|
for job_id in $failed_jobs; do
|
|
local job_name=$(echo "$jobs" | jq -r ".[] | select(.id == $job_id) | .name")
|
|
echo -e " ${RED}${CROSS}${NC} $job_name (Job #$job_id)"
|
|
echo -e " ${BLUE}View logs:${NC} https://gitlab.com/$(echo $PROJECT_ID | sed 's/%2F/\//g')/-/jobs/$job_id"
|
|
done
|
|
echo ""
|
|
fi
|
|
|
|
# Update pipeline status
|
|
pipeline=$(get_latest_pipeline "$branch")
|
|
local new_status=$(echo "$pipeline" | jq -r '.status')
|
|
|
|
if [ "$new_status" != "$pipeline_status" ]; then
|
|
pipeline_status=$new_status
|
|
fi
|
|
|
|
# Check if complete
|
|
if [[ "$pipeline_status" == "success" ]] || [[ "$pipeline_status" == "failed" ]] || \
|
|
[[ "$pipeline_status" == "canceled" ]] || [[ "$pipeline_status" == "skipped" ]]; then
|
|
|
|
local elapsed=$(($(date +%s) - start_time))
|
|
echo -e "${BLUE}Pipeline completed in:${NC} $(format_duration $elapsed)"
|
|
echo ""
|
|
|
|
if [ "$pipeline_status" = "success" ]; then
|
|
echo -e "${GREEN}${CHECK} Pipeline succeeded!${NC}"
|
|
|
|
# Monitor Kubernetes deployment if applicable
|
|
if [ "$HAS_K8S" = true ] && [ "$branch" = "main" ]; then
|
|
echo ""
|
|
monitor_k8s_deployment
|
|
fi
|
|
else
|
|
echo -e "${RED}${CROSS} Pipeline $pipeline_status${NC}"
|
|
fi
|
|
|
|
break
|
|
fi
|
|
|
|
echo -e "${YELLOW}Refreshing in ${REFRESH_INTERVAL}s... (Ctrl+C to stop)${NC}"
|
|
sleep $REFRESH_INTERVAL
|
|
done
|
|
}
|
|
|
|
monitor_k8s_deployment() {
|
|
echo -e "${CYAN}╔════════════════════════════════════════════════════════════════╗${NC}"
|
|
echo -e "${CYAN}║${NC} ${BLUE}Kubernetes Deployment${NC} ${CYAN}║${NC}"
|
|
echo -e "${CYAN}╚════════════════════════════════════════════════════════════════╝${NC}"
|
|
echo ""
|
|
|
|
if ! $KUBECTL get deployment -n $K8S_NAMESPACE $K8S_DEPLOYMENT &>/dev/null; then
|
|
echo -e "${YELLOW}Warning: Deployment $K8S_DEPLOYMENT not found in namespace $K8S_NAMESPACE${NC}"
|
|
return
|
|
fi
|
|
|
|
echo -e "${BLUE}Namespace:${NC} $K8S_NAMESPACE"
|
|
echo -e "${BLUE}Deployment:${NC} $K8S_DEPLOYMENT"
|
|
echo ""
|
|
|
|
# Trigger rollout restart
|
|
echo -e "${BLUE}Restarting deployment...${NC}"
|
|
$KUBECTL -n $K8S_NAMESPACE rollout restart deployment/$K8S_DEPLOYMENT
|
|
echo ""
|
|
|
|
# Monitor rollout status
|
|
echo -e "${BLUE}Waiting for rollout to complete...${NC}"
|
|
if $KUBECTL -n $K8S_NAMESPACE rollout status deployment/$K8S_DEPLOYMENT --timeout=300s; then
|
|
echo ""
|
|
echo -e "${GREEN}${CHECK} Deployment successful!${NC}"
|
|
echo ""
|
|
|
|
# Show pod status
|
|
echo -e "${BLUE}Pod status:${NC}"
|
|
$KUBECTL -n $K8S_NAMESPACE get pods -l app=$K8S_DEPLOYMENT
|
|
echo ""
|
|
|
|
# Get image version
|
|
local image=$($KUBECTL -n $K8S_NAMESPACE get deployment $K8S_DEPLOYMENT -o jsonpath='{.spec.template.spec.containers[0].image}')
|
|
echo -e "${BLUE}Current image:${NC} $image"
|
|
else
|
|
echo ""
|
|
echo -e "${RED}${CROSS} Deployment failed or timed out${NC}"
|
|
echo ""
|
|
echo -e "${BLUE}Recent events:${NC}"
|
|
$KUBECTL -n $K8S_NAMESPACE get events --sort-by=.lastTimestamp | tail -10
|
|
fi
|
|
}
|
|
|
|
# Main
|
|
main() {
|
|
check_dependencies
|
|
|
|
local project=""
|
|
|
|
# Detect or use argument
|
|
if [ $# -eq 0 ]; then
|
|
project=$(detect_project)
|
|
if [ -z "$project" ]; then
|
|
echo -e "${YELLOW}Could not auto-detect project${NC}"
|
|
usage
|
|
fi
|
|
echo -e "${BLUE}Auto-detected project:${NC} $project"
|
|
echo ""
|
|
else
|
|
project=$1
|
|
fi
|
|
|
|
get_project_info "$project"
|
|
|
|
print_header
|
|
monitor_pipeline
|
|
}
|
|
|
|
main "$@"
|