Graham McIntire
82413df58b
Instead of clearing the entire screen every 5 seconds, the script now: - Prints the header once and saves cursor position - Fetches all API data before updating display (no blank period) - Restores cursor position and updates only the content area - Hides cursor during display for cleaner look - Shows cursor on exit via cleanup trap Uses tput commands: - sc/rc: Save/restore cursor position - ed: Clear from cursor to end of screen - civis/cnorm: Hide/show cursor
437 lines
13 KiB
Bash
Executable file
437 lines
13 KiB
Bash
Executable file
#!/bin/bash
|
|
|
|
# Monitor GitLab CI/CD pipeline and Kubernetes deployment progress
|
|
# Works for both towerops main app and towerops-agent
|
|
|
|
set -e
|
|
|
|
# Colors
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
CYAN='\033[0;36m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Icons
|
|
CHECK="✓"
|
|
CROSS="✗"
|
|
ARROW="→"
|
|
CLOCK="⏳"
|
|
|
|
# Config
|
|
KUBECTL="/opt/homebrew/bin/kubectl"
|
|
REFRESH_INTERVAL=5
|
|
|
|
usage() {
|
|
echo "Usage: $0 [app|agent]"
|
|
echo ""
|
|
echo "Monitor CI/CD pipeline and deployment progress."
|
|
echo "Auto-detects project if run from git repository."
|
|
echo ""
|
|
echo "Options:"
|
|
echo " app - Monitor towerops main application"
|
|
echo " agent - Monitor towerops-agent"
|
|
echo ""
|
|
echo "Environment variables:"
|
|
echo " GITLAB_ACCESS_TOKEN - Required for API access"
|
|
echo " Get from: https://gitlab.com/-/profile/personal_access_tokens"
|
|
echo " Scopes needed: read_api"
|
|
exit 1
|
|
}
|
|
|
|
check_dependencies() {
|
|
if [ -z "$GITLAB_ACCESS_TOKEN" ]; then
|
|
echo -e "${RED}Error: GITLAB_ACCESS_TOKEN environment variable not set${NC}"
|
|
echo "Get a token from: https://gitlab.com/-/profile/personal_access_tokens"
|
|
echo "Then export GITLAB_ACCESS_TOKEN=<your-token>"
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v jq &> /dev/null; then
|
|
echo -e "${RED}Error: jq is required but not installed${NC}"
|
|
echo "Install with: brew install jq"
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v curl &> /dev/null; then
|
|
echo -e "${RED}Error: curl is required but not installed${NC}"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
detect_project() {
|
|
# Try to detect from git remote
|
|
if git remote get-url origin &>/dev/null; then
|
|
local remote_url=$(git remote get-url origin)
|
|
|
|
if echo "$remote_url" | grep -q "towerops-agent"; then
|
|
echo "agent"
|
|
elif echo "$remote_url" | grep -q "towerops"; then
|
|
echo "app"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
get_project_info() {
|
|
local project=$1
|
|
|
|
case $project in
|
|
app)
|
|
PROJECT_ID="77618300" # Numeric ID for towerops/towerops
|
|
PROJECT_PATH="towerops/towerops" # Path for web URLs
|
|
PROJECT_NAME="Towerops"
|
|
K8S_NAMESPACE="towerops"
|
|
K8S_DEPLOYMENT="towerops"
|
|
HAS_K8S=true
|
|
;;
|
|
agent)
|
|
PROJECT_ID="77595235" # Numeric ID for towerops/towerops-agent
|
|
PROJECT_PATH="towerops/towerops-agent" # Path for web URLs
|
|
PROJECT_NAME="Towerops Agent"
|
|
HAS_K8S=false
|
|
;;
|
|
*)
|
|
echo -e "${RED}Unknown project: $project${NC}"
|
|
usage
|
|
;;
|
|
esac
|
|
}
|
|
|
|
gitlab_api() {
|
|
local endpoint=$1
|
|
local response=$(curl -sL -w "\n%{http_code}" -H "PRIVATE-TOKEN: $GITLAB_ACCESS_TOKEN" \
|
|
"https://gitlab.com/api/v4${endpoint}")
|
|
|
|
local http_code=$(echo "$response" | tail -n1)
|
|
local body=$(echo "$response" | sed '$d')
|
|
|
|
if [ "$http_code" != "200" ]; then
|
|
echo -e "${RED}API Error (HTTP $http_code):${NC}" >&2
|
|
echo "$body" >&2
|
|
return 1
|
|
fi
|
|
|
|
echo "$body"
|
|
}
|
|
|
|
get_current_branch() {
|
|
git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "main"
|
|
}
|
|
|
|
get_latest_pipeline() {
|
|
local branch=$1
|
|
# Sort by id DESC to get the most recent pipeline (higher ID = newer)
|
|
local response=$(gitlab_api "/projects/${PROJECT_ID}/pipelines?ref=${branch}&per_page=1&order_by=id&sort=desc")
|
|
|
|
if [ $? -ne 0 ] || [ -z "$response" ]; then
|
|
return 1
|
|
fi
|
|
|
|
echo "$response" | jq -r '.[0] // empty'
|
|
}
|
|
|
|
get_pipeline_jobs() {
|
|
local pipeline_id=$1
|
|
gitlab_api "/projects/${PROJECT_ID}/pipelines/${pipeline_id}/jobs"
|
|
}
|
|
|
|
get_job_log() {
|
|
local job_id=$1
|
|
gitlab_api "/projects/${PROJECT_ID}/jobs/${job_id}/trace"
|
|
}
|
|
|
|
status_icon() {
|
|
local status=$1
|
|
case $status in
|
|
success|passed)
|
|
echo -e "${GREEN}${CHECK}${NC}"
|
|
;;
|
|
failed)
|
|
echo -e "${RED}${CROSS}${NC}"
|
|
;;
|
|
running)
|
|
echo -e "${BLUE}${ARROW}${NC}"
|
|
;;
|
|
pending|created)
|
|
echo -e "${YELLOW}${CLOCK}${NC}"
|
|
;;
|
|
canceled|skipped)
|
|
echo -e "${YELLOW}-${NC}"
|
|
;;
|
|
*)
|
|
echo -e "${CYAN}?${NC}"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
format_duration() {
|
|
local seconds=$1
|
|
if [ -z "$seconds" ] || [ "$seconds" = "null" ]; then
|
|
echo "-"
|
|
return
|
|
fi
|
|
|
|
# Convert to integer (GitLab returns decimals like 51.952868)
|
|
seconds=${seconds%.*}
|
|
|
|
local mins=$((seconds / 60))
|
|
local secs=$((seconds % 60))
|
|
|
|
if [ $mins -gt 0 ]; then
|
|
echo "${mins}m ${secs}s"
|
|
else
|
|
echo "${secs}s"
|
|
fi
|
|
}
|
|
|
|
format_time_ago() {
|
|
local timestamp=$1
|
|
if [ -z "$timestamp" ] || [ "$timestamp" = "null" ]; then
|
|
echo "Unknown"
|
|
return
|
|
fi
|
|
|
|
# Convert ISO 8601 timestamp to Unix epoch
|
|
# Handle both GNU date and BSD date (macOS)
|
|
local created_epoch
|
|
if date --version >/dev/null 2>&1; then
|
|
# GNU date
|
|
created_epoch=$(date -d "$timestamp" +%s 2>/dev/null)
|
|
else
|
|
# BSD date (macOS) - strip milliseconds and timezone, then parse as UTC
|
|
local clean_timestamp=$(echo "$timestamp" | sed 's/\.[0-9]*Z$//')
|
|
created_epoch=$(date -j -u -f "%Y-%m-%dT%H:%M:%S" "$clean_timestamp" +%s 2>/dev/null)
|
|
fi
|
|
|
|
if [ -z "$created_epoch" ]; then
|
|
echo "Unknown"
|
|
return
|
|
fi
|
|
|
|
local now=$(date +%s)
|
|
local seconds_ago=$((now - created_epoch))
|
|
|
|
# Handle negative values (future timestamps)
|
|
if [ $seconds_ago -lt 0 ]; then
|
|
seconds_ago=$((seconds_ago * -1))
|
|
echo "in ${seconds_ago}s"
|
|
return
|
|
fi
|
|
|
|
if [ $seconds_ago -lt 60 ]; then
|
|
echo "${seconds_ago}s ago"
|
|
elif [ $seconds_ago -lt 3600 ]; then
|
|
echo "$((seconds_ago / 60))m ago"
|
|
elif [ $seconds_ago -lt 86400 ]; then
|
|
echo "$((seconds_ago / 3600))h ago"
|
|
else
|
|
echo "$((seconds_ago / 86400))d ago"
|
|
fi
|
|
}
|
|
|
|
print_header() {
|
|
# Only clear on first call
|
|
if [ -z "$HEADER_PRINTED" ]; then
|
|
clear
|
|
echo -e "${CYAN}╔════════════════════════════════════════════════════════════════╗${NC}"
|
|
echo -e "${CYAN}║${NC} ${BLUE}${PROJECT_NAME} - CI/CD Monitor${NC} ${CYAN}║${NC}"
|
|
echo -e "${CYAN}╚════════════════════════════════════════════════════════════════╝${NC}"
|
|
echo ""
|
|
# Save cursor position after header
|
|
tput sc
|
|
HEADER_PRINTED=1
|
|
else
|
|
# Restore cursor to saved position and clear from there to end
|
|
tput rc
|
|
tput ed
|
|
fi
|
|
}
|
|
|
|
cleanup() {
|
|
# Reset cursor visibility and show cursor
|
|
tput cnorm
|
|
echo ""
|
|
}
|
|
|
|
monitor_pipeline() {
|
|
# Trap exit to cleanup terminal state
|
|
trap cleanup EXIT INT TERM
|
|
|
|
# Hide cursor for cleaner display
|
|
tput civis
|
|
|
|
local branch=$(get_current_branch)
|
|
local local_sha=$(git rev-parse HEAD 2>/dev/null | cut -c1-8)
|
|
|
|
local pipeline=$(get_latest_pipeline "$branch")
|
|
|
|
if [ -z "$pipeline" ] || [ "$pipeline" = "null" ]; then
|
|
echo -e "${YELLOW}No pipeline found for branch: $branch${NC}"
|
|
echo "Push to trigger a new pipeline"
|
|
exit 0
|
|
fi
|
|
|
|
local pipeline_id=$(echo "$pipeline" | jq -r '.id')
|
|
local pipeline_status=$(echo "$pipeline" | jq -r '.status')
|
|
local pipeline_web_url=$(echo "$pipeline" | jq -r '.web_url')
|
|
local pipeline_sha=$(echo "$pipeline" | jq -r '.sha')
|
|
local pipeline_created=$(echo "$pipeline" | jq -r '.created_at')
|
|
|
|
local start_time=$(date +%s)
|
|
|
|
# Initialize header flag
|
|
HEADER_PRINTED=""
|
|
|
|
# Monitor until complete
|
|
while true; do
|
|
# Fetch fresh data first (before updating display)
|
|
local jobs=$(get_pipeline_jobs "$pipeline_id")
|
|
local job_count=$(echo "$jobs" | jq '. | length')
|
|
local failed_jobs=$(echo "$jobs" | jq -r '.[] | select(.status == "failed") | .id')
|
|
|
|
# Update pipeline status
|
|
pipeline=$(get_latest_pipeline "$branch")
|
|
local new_status=$(echo "$pipeline" | jq -r '.status')
|
|
|
|
# Now update the display with all the fetched data
|
|
print_header
|
|
|
|
echo -e "${BLUE}Branch:${NC} $branch"
|
|
echo -e "${BLUE}Local HEAD:${NC} $local_sha"
|
|
echo -e "${BLUE}Pipeline:${NC} #$pipeline_id"
|
|
echo -e "${BLUE}Created:${NC} $(format_time_ago "$pipeline_created")"
|
|
echo -e "${BLUE}Commit:${NC} ${pipeline_sha:0:8}"
|
|
|
|
# Warn if local commit doesn't match pipeline commit
|
|
if [ "$local_sha" != "${pipeline_sha:0:8}" ]; then
|
|
echo -e "${YELLOW}⚠️ Warning: Pipeline is for a different commit${NC}"
|
|
fi
|
|
|
|
echo -e "${BLUE}Status:${NC} $(status_icon $pipeline_status) $pipeline_status"
|
|
echo -e "${BLUE}URL:${NC} $pipeline_web_url"
|
|
echo ""
|
|
|
|
# Display jobs
|
|
echo -e "${CYAN}Jobs (${job_count} total):${NC}"
|
|
echo "$jobs" | jq -r '.[] | "\(.stage)|\(.name)|\(.status)|\(.duration)|\(.id)"' | \
|
|
while IFS='|' read -r stage name status duration job_id; do
|
|
printf " %-12s $(status_icon $status) %-30s %-10s [#%s]\n" \
|
|
"[$stage]" "$name" "$(format_duration $duration)" "$job_id"
|
|
done
|
|
|
|
echo ""
|
|
|
|
# Display failures
|
|
if [ -n "$failed_jobs" ]; then
|
|
echo -e "${RED}Failed Jobs:${NC}"
|
|
for job_id in $failed_jobs; do
|
|
local job_name=$(echo "$jobs" | jq -r ".[] | select(.id == $job_id) | .name")
|
|
echo -e " ${RED}${CROSS}${NC} $job_name (Job #$job_id)"
|
|
echo -e " ${BLUE}View logs:${NC} https://gitlab.com/${PROJECT_PATH}/-/jobs/$job_id"
|
|
done
|
|
echo ""
|
|
fi
|
|
|
|
if [ "$new_status" != "$pipeline_status" ]; then
|
|
pipeline_status=$new_status
|
|
fi
|
|
|
|
# Check if complete
|
|
if [[ "$pipeline_status" == "success" ]] || [[ "$pipeline_status" == "failed" ]] || \
|
|
[[ "$pipeline_status" == "canceled" ]] || [[ "$pipeline_status" == "skipped" ]]; then
|
|
|
|
local elapsed=$(($(date +%s) - start_time))
|
|
echo -e "${BLUE}Pipeline completed in:${NC} $(format_duration $elapsed)"
|
|
echo ""
|
|
|
|
if [ "$pipeline_status" = "success" ]; then
|
|
echo -e "${GREEN}${CHECK} Pipeline succeeded!${NC}"
|
|
|
|
# Monitor Kubernetes deployment if applicable
|
|
if [ "$HAS_K8S" = true ] && [ "$branch" = "main" ]; then
|
|
echo ""
|
|
monitor_k8s_deployment
|
|
fi
|
|
else
|
|
echo -e "${RED}${CROSS} Pipeline $pipeline_status${NC}"
|
|
fi
|
|
|
|
break
|
|
fi
|
|
|
|
echo -e "${YELLOW}Refreshing in ${REFRESH_INTERVAL}s... (Ctrl+C to stop)${NC}"
|
|
sleep $REFRESH_INTERVAL
|
|
done
|
|
}
|
|
|
|
monitor_k8s_deployment() {
|
|
echo -e "${CYAN}╔════════════════════════════════════════════════════════════════╗${NC}"
|
|
echo -e "${CYAN}║${NC} ${BLUE}Kubernetes Deployment${NC} ${CYAN}║${NC}"
|
|
echo -e "${CYAN}╚════════════════════════════════════════════════════════════════╝${NC}"
|
|
echo ""
|
|
|
|
if ! $KUBECTL get deployment -n $K8S_NAMESPACE $K8S_DEPLOYMENT &>/dev/null; then
|
|
echo -e "${YELLOW}Warning: Deployment $K8S_DEPLOYMENT not found in namespace $K8S_NAMESPACE${NC}"
|
|
return
|
|
fi
|
|
|
|
echo -e "${BLUE}Namespace:${NC} $K8S_NAMESPACE"
|
|
echo -e "${BLUE}Deployment:${NC} $K8S_DEPLOYMENT"
|
|
echo ""
|
|
|
|
# Trigger rollout restart
|
|
echo -e "${BLUE}Restarting deployment...${NC}"
|
|
$KUBECTL -n $K8S_NAMESPACE rollout restart deployment/$K8S_DEPLOYMENT
|
|
echo ""
|
|
|
|
# Monitor rollout status
|
|
echo -e "${BLUE}Waiting for rollout to complete...${NC}"
|
|
if $KUBECTL -n $K8S_NAMESPACE rollout status deployment/$K8S_DEPLOYMENT --timeout=300s; then
|
|
echo ""
|
|
echo -e "${GREEN}${CHECK} Deployment successful!${NC}"
|
|
echo ""
|
|
|
|
# Show pod status
|
|
echo -e "${BLUE}Pod status:${NC}"
|
|
$KUBECTL -n $K8S_NAMESPACE get pods -l app=$K8S_DEPLOYMENT
|
|
echo ""
|
|
|
|
# Get image version
|
|
local image=$($KUBECTL -n $K8S_NAMESPACE get deployment $K8S_DEPLOYMENT -o jsonpath='{.spec.template.spec.containers[0].image}')
|
|
echo -e "${BLUE}Current image:${NC} $image"
|
|
else
|
|
echo ""
|
|
echo -e "${RED}${CROSS} Deployment failed or timed out${NC}"
|
|
echo ""
|
|
echo -e "${BLUE}Recent events:${NC}"
|
|
$KUBECTL -n $K8S_NAMESPACE get events --sort-by=.lastTimestamp | tail -10
|
|
fi
|
|
}
|
|
|
|
# Main
|
|
main() {
|
|
check_dependencies
|
|
|
|
local project=""
|
|
|
|
# Detect or use argument
|
|
if [ $# -eq 0 ]; then
|
|
project=$(detect_project)
|
|
if [ -z "$project" ]; then
|
|
echo -e "${YELLOW}Could not auto-detect project${NC}"
|
|
usage
|
|
fi
|
|
echo -e "${BLUE}Auto-detected project:${NC} $project"
|
|
echo ""
|
|
else
|
|
project=$1
|
|
fi
|
|
|
|
get_project_info "$project"
|
|
|
|
print_header
|
|
monitor_pipeline
|
|
}
|
|
|
|
main "$@"
|