#!/bin/bash # Monitor GitLab CI/CD pipeline and Kubernetes deployment progress # Works for both towerops main app and towerops-agent set -e # Colors RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' NC='\033[0m' # No Color # Icons CHECK="✓" CROSS="✗" ARROW="→" CLOCK="⏳" # Config KUBECTL="/opt/homebrew/bin/kubectl" REFRESH_INTERVAL=5 usage() { echo "Usage: $0 [app|agent]" echo "" echo "Monitor CI/CD pipeline and deployment progress." echo "Auto-detects project if run from git repository." echo "" echo "Options:" echo " app - Monitor towerops main application" echo " agent - Monitor towerops-agent" echo "" echo "Environment variables:" echo " GITLAB_ACCESS_TOKEN - Required for API access" echo " Get from: https://gitlab.com/-/profile/personal_access_tokens" echo " Scopes needed: read_api" exit 1 } check_dependencies() { if [ -z "$GITLAB_ACCESS_TOKEN" ]; then echo -e "${RED}Error: GITLAB_ACCESS_TOKEN environment variable not set${NC}" echo "Get a token from: https://gitlab.com/-/profile/personal_access_tokens" echo "Then export GITLAB_ACCESS_TOKEN=" exit 1 fi if ! command -v jq &> /dev/null; then echo -e "${RED}Error: jq is required but not installed${NC}" echo "Install with: brew install jq" exit 1 fi if ! command -v curl &> /dev/null; then echo -e "${RED}Error: curl is required but not installed${NC}" exit 1 fi } detect_project() { # Try to detect from git remote if git remote get-url origin &>/dev/null; then local remote_url=$(git remote get-url origin) if echo "$remote_url" | grep -q "towerops-agent"; then echo "agent" elif echo "$remote_url" | grep -q "towerops"; then echo "app" fi fi } get_project_info() { local project=$1 case $project in app) PROJECT_ID="77257437" # Numeric ID for API calls PROJECT_PATH="towerops/towerops" # Path for web URLs PROJECT_NAME="Towerops" K8S_NAMESPACE="towerops" K8S_DEPLOYMENT="towerops" HAS_K8S=true ;; agent) PROJECT_ID="63766041" # Numeric ID for API calls PROJECT_PATH="towerops/towerops-agent" # Path for web URLs PROJECT_NAME="Towerops Agent" HAS_K8S=false ;; *) echo -e "${RED}Unknown project: $project${NC}" usage ;; esac } gitlab_api() { local endpoint=$1 local response=$(curl -sL -w "\n%{http_code}" -H "PRIVATE-TOKEN: $GITLAB_ACCESS_TOKEN" \ "https://gitlab.com/api/v4${endpoint}") local http_code=$(echo "$response" | tail -n1) local body=$(echo "$response" | sed '$d') if [ "$http_code" != "200" ]; then echo -e "${RED}API Error (HTTP $http_code):${NC}" >&2 echo "$body" >&2 return 1 fi echo "$body" } get_current_branch() { git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "main" } get_latest_pipeline() { local branch=$1 local response=$(gitlab_api "/projects/${PROJECT_ID}/pipelines?ref=${branch}&per_page=1") if [ $? -ne 0 ] || [ -z "$response" ]; then return 1 fi echo "$response" | jq -r '.[0] // empty' } get_pipeline_jobs() { local pipeline_id=$1 gitlab_api "/projects/${PROJECT_ID}/pipelines/${pipeline_id}/jobs" } get_job_log() { local job_id=$1 gitlab_api "/projects/${PROJECT_ID}/jobs/${job_id}/trace" } status_icon() { local status=$1 case $status in success|passed) echo -e "${GREEN}${CHECK}${NC}" ;; failed) echo -e "${RED}${CROSS}${NC}" ;; running) echo -e "${BLUE}${ARROW}${NC}" ;; pending|created) echo -e "${YELLOW}${CLOCK}${NC}" ;; canceled|skipped) echo -e "${YELLOW}-${NC}" ;; *) echo -e "${CYAN}?${NC}" ;; esac } format_duration() { local seconds=$1 if [ -z "$seconds" ] || [ "$seconds" = "null" ]; then echo "-" return fi local mins=$((seconds / 60)) local secs=$((seconds % 60)) if [ $mins -gt 0 ]; then echo "${mins}m ${secs}s" else echo "${secs}s" fi } print_header() { clear echo -e "${CYAN}╔════════════════════════════════════════════════════════════════╗${NC}" echo -e "${CYAN}║${NC} ${BLUE}${PROJECT_NAME} - CI/CD Monitor${NC} ${CYAN}║${NC}" echo -e "${CYAN}╚════════════════════════════════════════════════════════════════╝${NC}" echo "" } monitor_pipeline() { local branch=$(get_current_branch) echo -e "${BLUE}Branch:${NC} $branch" echo -e "${BLUE}Fetching latest pipeline...${NC}" echo "" local pipeline=$(get_latest_pipeline "$branch") if [ -z "$pipeline" ] || [ "$pipeline" = "null" ]; then echo -e "${YELLOW}No pipeline found for branch: $branch${NC}" echo "Push to trigger a new pipeline" exit 0 fi local pipeline_id=$(echo "$pipeline" | jq -r '.id') local pipeline_status=$(echo "$pipeline" | jq -r '.status') local pipeline_web_url=$(echo "$pipeline" | jq -r '.web_url') echo -e "${BLUE}Pipeline:${NC} #$pipeline_id" echo -e "${BLUE}Status:${NC} $(status_icon $pipeline_status) $pipeline_status" echo -e "${BLUE}URL:${NC} $pipeline_web_url" echo "" local start_time=$(date +%s) # Monitor until complete while true; do print_header echo -e "${BLUE}Branch:${NC} $branch" echo -e "${BLUE}Pipeline:${NC} #$pipeline_id" echo -e "${BLUE}Status:${NC} $(status_icon $pipeline_status) $pipeline_status" echo "" # Get and display jobs local jobs=$(get_pipeline_jobs "$pipeline_id") echo -e "${CYAN}Jobs:${NC}" echo "$jobs" | jq -r '.[] | "\(.stage)|\(.name)|\(.status)|\(.duration)"' | \ while IFS='|' read -r stage name status duration; do printf " %-12s $(status_icon $status) %-30s %s\n" \ "[$stage]" "$name" "$(format_duration $duration)" done echo "" # Check for failures local failed_jobs=$(echo "$jobs" | jq -r '.[] | select(.status == "failed") | .id') if [ -n "$failed_jobs" ]; then echo -e "${RED}Failed Jobs:${NC}" for job_id in $failed_jobs; do local job_name=$(echo "$jobs" | jq -r ".[] | select(.id == $job_id) | .name") echo -e " ${RED}${CROSS}${NC} $job_name (Job #$job_id)" echo -e " ${BLUE}View logs:${NC} https://gitlab.com/${PROJECT_PATH}/-/jobs/$job_id" done echo "" fi # Update pipeline status pipeline=$(get_latest_pipeline "$branch") local new_status=$(echo "$pipeline" | jq -r '.status') if [ "$new_status" != "$pipeline_status" ]; then pipeline_status=$new_status fi # Check if complete if [[ "$pipeline_status" == "success" ]] || [[ "$pipeline_status" == "failed" ]] || \ [[ "$pipeline_status" == "canceled" ]] || [[ "$pipeline_status" == "skipped" ]]; then local elapsed=$(($(date +%s) - start_time)) echo -e "${BLUE}Pipeline completed in:${NC} $(format_duration $elapsed)" echo "" if [ "$pipeline_status" = "success" ]; then echo -e "${GREEN}${CHECK} Pipeline succeeded!${NC}" # Monitor Kubernetes deployment if applicable if [ "$HAS_K8S" = true ] && [ "$branch" = "main" ]; then echo "" monitor_k8s_deployment fi else echo -e "${RED}${CROSS} Pipeline $pipeline_status${NC}" fi break fi echo -e "${YELLOW}Refreshing in ${REFRESH_INTERVAL}s... (Ctrl+C to stop)${NC}" sleep $REFRESH_INTERVAL done } monitor_k8s_deployment() { echo -e "${CYAN}╔════════════════════════════════════════════════════════════════╗${NC}" echo -e "${CYAN}║${NC} ${BLUE}Kubernetes Deployment${NC} ${CYAN}║${NC}" echo -e "${CYAN}╚════════════════════════════════════════════════════════════════╝${NC}" echo "" if ! $KUBECTL get deployment -n $K8S_NAMESPACE $K8S_DEPLOYMENT &>/dev/null; then echo -e "${YELLOW}Warning: Deployment $K8S_DEPLOYMENT not found in namespace $K8S_NAMESPACE${NC}" return fi echo -e "${BLUE}Namespace:${NC} $K8S_NAMESPACE" echo -e "${BLUE}Deployment:${NC} $K8S_DEPLOYMENT" echo "" # Trigger rollout restart echo -e "${BLUE}Restarting deployment...${NC}" $KUBECTL -n $K8S_NAMESPACE rollout restart deployment/$K8S_DEPLOYMENT echo "" # Monitor rollout status echo -e "${BLUE}Waiting for rollout to complete...${NC}" if $KUBECTL -n $K8S_NAMESPACE rollout status deployment/$K8S_DEPLOYMENT --timeout=300s; then echo "" echo -e "${GREEN}${CHECK} Deployment successful!${NC}" echo "" # Show pod status echo -e "${BLUE}Pod status:${NC}" $KUBECTL -n $K8S_NAMESPACE get pods -l app=$K8S_DEPLOYMENT echo "" # Get image version local image=$($KUBECTL -n $K8S_NAMESPACE get deployment $K8S_DEPLOYMENT -o jsonpath='{.spec.template.spec.containers[0].image}') echo -e "${BLUE}Current image:${NC} $image" else echo "" echo -e "${RED}${CROSS} Deployment failed or timed out${NC}" echo "" echo -e "${BLUE}Recent events:${NC}" $KUBECTL -n $K8S_NAMESPACE get events --sort-by=.lastTimestamp | tail -10 fi } # Main main() { check_dependencies local project="" # Detect or use argument if [ $# -eq 0 ]; then project=$(detect_project) if [ -z "$project" ]; then echo -e "${YELLOW}Could not auto-detect project${NC}" usage fi echo -e "${BLUE}Auto-detected project:${NC} $project" echo "" else project=$1 fi get_project_info "$project" print_header monitor_pipeline } main "$@"