#!/bin/bash # Monitor GitLab CI/CD pipeline and Kubernetes deployment progress # Works for both towerops main app and towerops-agent set -e # Colors RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' NC='\033[0m' # No Color # Icons CHECK="✓" CROSS="✗" ARROW="→" CLOCK="⏳" # Config KUBECTL="/opt/homebrew/bin/kubectl" REFRESH_INTERVAL=5 usage() { echo "Usage: $0 [app|agent]" echo "" echo "Monitor CI/CD pipeline and deployment progress." echo "Auto-detects project if run from git repository." echo "" echo "Options:" echo " app - Monitor towerops main application" echo " agent - Monitor towerops-agent" echo "" echo "Environment variables:" echo " GITLAB_ACCESS_TOKEN - Required for API access" echo " Get from: https://gitlab.com/-/profile/personal_access_tokens" echo " Scopes needed: read_api" exit 1 } check_dependencies() { if [ -z "$GITLAB_ACCESS_TOKEN" ]; then echo -e "${RED}Error: GITLAB_ACCESS_TOKEN environment variable not set${NC}" echo "Get a token from: https://gitlab.com/-/profile/personal_access_tokens" echo "Then export GITLAB_ACCESS_TOKEN=" exit 1 fi if ! command -v jq &> /dev/null; then echo -e "${RED}Error: jq is required but not installed${NC}" echo "Install with: brew install jq" exit 1 fi if ! command -v curl &> /dev/null; then echo -e "${RED}Error: curl is required but not installed${NC}" exit 1 fi } detect_project() { # Try to detect from git remote if git remote get-url origin &>/dev/null; then local remote_url=$(git remote get-url origin) if echo "$remote_url" | grep -q "towerops-agent"; then echo "agent" elif echo "$remote_url" | grep -q "towerops"; then echo "app" fi fi } get_project_info() { local project=$1 case $project in app) PROJECT_ID="77618300" # Numeric ID for towerops/towerops PROJECT_PATH="towerops/towerops" # Path for web URLs PROJECT_NAME="Towerops" K8S_NAMESPACE="towerops" K8S_DEPLOYMENT="towerops" HAS_K8S=true ;; agent) PROJECT_ID="77595235" # Numeric ID for towerops/towerops-agent PROJECT_PATH="towerops/towerops-agent" # Path for web URLs PROJECT_NAME="Towerops Agent" HAS_K8S=false ;; *) echo -e "${RED}Unknown project: $project${NC}" usage ;; esac } gitlab_api() { local endpoint=$1 local response=$(curl -sL -w "\n%{http_code}" -H "PRIVATE-TOKEN: $GITLAB_ACCESS_TOKEN" \ "https://gitlab.com/api/v4${endpoint}") local http_code=$(echo "$response" | tail -n1) local body=$(echo "$response" | sed '$d') if [ "$http_code" != "200" ]; then echo -e "${RED}API Error (HTTP $http_code):${NC}" >&2 echo "$body" >&2 return 1 fi echo "$body" } get_current_branch() { git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "main" } get_latest_pipeline() { local branch=$1 # Sort by id DESC to get the most recent pipeline (higher ID = newer) local response=$(gitlab_api "/projects/${PROJECT_ID}/pipelines?ref=${branch}&per_page=1&order_by=id&sort=desc") if [ $? -ne 0 ] || [ -z "$response" ]; then return 1 fi echo "$response" | jq -r '.[0] // empty' } get_pipeline_jobs() { local pipeline_id=$1 gitlab_api "/projects/${PROJECT_ID}/pipelines/${pipeline_id}/jobs" } get_job_log() { local job_id=$1 gitlab_api "/projects/${PROJECT_ID}/jobs/${job_id}/trace" } status_icon() { local status=$1 case $status in success|passed) echo -e "${GREEN}${CHECK}${NC}" ;; failed) echo -e "${RED}${CROSS}${NC}" ;; running) echo -e "${BLUE}${ARROW}${NC}" ;; pending|created) echo -e "${YELLOW}${CLOCK}${NC}" ;; canceled|skipped) echo -e "${YELLOW}-${NC}" ;; *) echo -e "${CYAN}?${NC}" ;; esac } format_duration() { local seconds=$1 if [ -z "$seconds" ] || [ "$seconds" = "null" ]; then echo "-" return fi # Convert to integer (GitLab returns decimals like 51.952868) seconds=${seconds%.*} local mins=$((seconds / 60)) local secs=$((seconds % 60)) if [ $mins -gt 0 ]; then echo "${mins}m ${secs}s" else echo "${secs}s" fi } format_time_ago() { local timestamp=$1 if [ -z "$timestamp" ] || [ "$timestamp" = "null" ]; then echo "Unknown" return fi # Convert ISO 8601 timestamp to Unix epoch # Handle both GNU date and BSD date (macOS) local created_epoch if date --version >/dev/null 2>&1; then # GNU date created_epoch=$(date -d "$timestamp" +%s 2>/dev/null) else # BSD date (macOS) - strip milliseconds and timezone, then parse as UTC local clean_timestamp=$(echo "$timestamp" | sed 's/\.[0-9]*Z$//') created_epoch=$(date -j -u -f "%Y-%m-%dT%H:%M:%S" "$clean_timestamp" +%s 2>/dev/null) fi if [ -z "$created_epoch" ]; then echo "Unknown" return fi local now=$(date +%s) local seconds_ago=$((now - created_epoch)) # Handle negative values (future timestamps) if [ $seconds_ago -lt 0 ]; then seconds_ago=$((seconds_ago * -1)) echo "in ${seconds_ago}s" return fi if [ $seconds_ago -lt 60 ]; then echo "${seconds_ago}s ago" elif [ $seconds_ago -lt 3600 ]; then echo "$((seconds_ago / 60))m ago" elif [ $seconds_ago -lt 86400 ]; then echo "$((seconds_ago / 3600))h ago" else echo "$((seconds_ago / 86400))d ago" fi } print_header() { # Only clear on first call if [ -z "$HEADER_PRINTED" ]; then clear echo -e "${CYAN}╔════════════════════════════════════════════════════════════════╗${NC}" echo -e "${CYAN}║${NC} ${BLUE}${PROJECT_NAME} - CI/CD Monitor${NC} ${CYAN}║${NC}" echo -e "${CYAN}╚════════════════════════════════════════════════════════════════╝${NC}" echo "" # Save cursor position after header tput sc HEADER_PRINTED=1 else # Restore cursor to saved position and clear from there to end tput rc tput ed fi } cleanup() { # Reset cursor visibility and show cursor tput cnorm echo "" } monitor_pipeline() { # Trap exit to cleanup terminal state trap cleanup EXIT INT TERM # Hide cursor for cleaner display tput civis local branch=$(get_current_branch) local local_sha=$(git rev-parse HEAD 2>/dev/null | cut -c1-8) local pipeline=$(get_latest_pipeline "$branch") if [ -z "$pipeline" ] || [ "$pipeline" = "null" ]; then echo -e "${YELLOW}No pipeline found for branch: $branch${NC}" echo "Push to trigger a new pipeline" exit 0 fi local pipeline_id=$(echo "$pipeline" | jq -r '.id') local pipeline_status=$(echo "$pipeline" | jq -r '.status') local pipeline_web_url=$(echo "$pipeline" | jq -r '.web_url') local pipeline_sha=$(echo "$pipeline" | jq -r '.sha') local pipeline_created=$(echo "$pipeline" | jq -r '.created_at') local start_time=$(date +%s) # Initialize header flag HEADER_PRINTED="" # Monitor until complete while true; do # Fetch fresh data first (before updating display) local jobs=$(get_pipeline_jobs "$pipeline_id") local job_count=$(echo "$jobs" | jq '. | length') local failed_jobs=$(echo "$jobs" | jq -r '.[] | select(.status == "failed") | .id') # Update pipeline status pipeline=$(get_latest_pipeline "$branch") local new_status=$(echo "$pipeline" | jq -r '.status') # Now update the display with all the fetched data print_header echo -e "${BLUE}Branch:${NC} $branch" echo -e "${BLUE}Local HEAD:${NC} $local_sha" echo -e "${BLUE}Pipeline:${NC} #$pipeline_id" echo -e "${BLUE}Created:${NC} $(format_time_ago "$pipeline_created")" echo -e "${BLUE}Commit:${NC} ${pipeline_sha:0:8}" # Warn if local commit doesn't match pipeline commit if [ "$local_sha" != "${pipeline_sha:0:8}" ]; then echo -e "${YELLOW}⚠️ Warning: Pipeline is for a different commit${NC}" fi echo -e "${BLUE}Status:${NC} $(status_icon $pipeline_status) $pipeline_status" echo -e "${BLUE}URL:${NC} $pipeline_web_url" echo "" # Display jobs echo -e "${CYAN}Jobs (${job_count} total):${NC}" echo "$jobs" | jq -r '.[] | "\(.stage)|\(.name)|\(.status)|\(.duration)|\(.id)"' | \ while IFS='|' read -r stage name status duration job_id; do printf " %-12s $(status_icon $status) %-30s %-10s [#%s]\n" \ "[$stage]" "$name" "$(format_duration $duration)" "$job_id" done echo "" # Display failures if [ -n "$failed_jobs" ]; then echo -e "${RED}Failed Jobs:${NC}" for job_id in $failed_jobs; do local job_name=$(echo "$jobs" | jq -r ".[] | select(.id == $job_id) | .name") echo -e " ${RED}${CROSS}${NC} $job_name (Job #$job_id)" echo -e " ${BLUE}View logs:${NC} https://gitlab.com/${PROJECT_PATH}/-/jobs/$job_id" done echo "" fi if [ "$new_status" != "$pipeline_status" ]; then pipeline_status=$new_status fi # Check if complete if [[ "$pipeline_status" == "success" ]] || [[ "$pipeline_status" == "failed" ]] || \ [[ "$pipeline_status" == "canceled" ]] || [[ "$pipeline_status" == "skipped" ]]; then local elapsed=$(($(date +%s) - start_time)) echo -e "${BLUE}Pipeline completed in:${NC} $(format_duration $elapsed)" echo "" if [ "$pipeline_status" = "success" ]; then echo -e "${GREEN}${CHECK} Pipeline succeeded!${NC}" # Monitor Kubernetes deployment if applicable if [ "$HAS_K8S" = true ] && [ "$branch" = "main" ]; then echo "" monitor_k8s_deployment fi else echo -e "${RED}${CROSS} Pipeline $pipeline_status${NC}" fi break fi echo -e "${YELLOW}Refreshing in ${REFRESH_INTERVAL}s... (Ctrl+C to stop)${NC}" sleep $REFRESH_INTERVAL done } monitor_k8s_deployment() { echo -e "${CYAN}╔════════════════════════════════════════════════════════════════╗${NC}" echo -e "${CYAN}║${NC} ${BLUE}Kubernetes Deployment${NC} ${CYAN}║${NC}" echo -e "${CYAN}╚════════════════════════════════════════════════════════════════╝${NC}" echo "" if ! $KUBECTL get deployment -n $K8S_NAMESPACE $K8S_DEPLOYMENT &>/dev/null; then echo -e "${YELLOW}Warning: Deployment $K8S_DEPLOYMENT not found in namespace $K8S_NAMESPACE${NC}" return fi echo -e "${BLUE}Namespace:${NC} $K8S_NAMESPACE" echo -e "${BLUE}Deployment:${NC} $K8S_DEPLOYMENT" echo "" # Trigger rollout restart echo -e "${BLUE}Restarting deployment...${NC}" $KUBECTL -n $K8S_NAMESPACE rollout restart deployment/$K8S_DEPLOYMENT echo "" # Monitor rollout status echo -e "${BLUE}Waiting for rollout to complete...${NC}" if $KUBECTL -n $K8S_NAMESPACE rollout status deployment/$K8S_DEPLOYMENT --timeout=300s; then echo "" echo -e "${GREEN}${CHECK} Deployment successful!${NC}" echo "" # Show pod status echo -e "${BLUE}Pod status:${NC}" $KUBECTL -n $K8S_NAMESPACE get pods -l app=$K8S_DEPLOYMENT echo "" # Get image version local image=$($KUBECTL -n $K8S_NAMESPACE get deployment $K8S_DEPLOYMENT -o jsonpath='{.spec.template.spec.containers[0].image}') echo -e "${BLUE}Current image:${NC} $image" else echo "" echo -e "${RED}${CROSS} Deployment failed or timed out${NC}" echo "" echo -e "${BLUE}Recent events:${NC}" $KUBECTL -n $K8S_NAMESPACE get events --sort-by=.lastTimestamp | tail -10 fi } # Main main() { check_dependencies local project="" # Detect or use argument if [ $# -eq 0 ]; then project=$(detect_project) if [ -z "$project" ]; then echo -e "${YELLOW}Could not auto-detect project${NC}" usage fi echo -e "${BLUE}Auto-detected project:${NC} $project" echo "" else project=$1 fi get_project_info "$project" print_header monitor_pipeline } main "$@"