diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000..9d3a7c63 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,134 @@ +# Deployment Monitoring Scripts + +## monitor-deploy.sh + +Real-time monitoring of GitLab CI/CD pipelines and Kubernetes deployments. + +### Features + +- Auto-detects project (main app vs agent) from git remote +- Shows live pipeline progress with job statuses +- Color-coded output with icons +- Auto-refreshes every 5 seconds +- Shows failed job logs +- For main app: automatically monitors Kubernetes rollout after successful build + +### Setup + +1. **Get GitLab Personal Access Token** + + Visit: https://gitlab.com/-/profile/personal_access_tokens + + Create a token with scope: `read_api` + +2. **Export token** + + ```bash + export GITLAB_TOKEN= + ``` + + Add to your `~/.zshrc` or `~/.bashrc`: + ```bash + export GITLAB_TOKEN=glpat-xxxxxxxxxxxxxxxxxxxx + ``` + +3. **Install dependencies** + + ```bash + brew install jq + ``` + +### Usage + +**From main towerops directory:** +```bash +# Auto-detect project and monitor +./scripts/monitor-deploy.sh + +# Or explicitly specify +./scripts/monitor-deploy.sh app +``` + +**From towerops-agent directory:** +```bash +# Auto-detect and monitor agent +../scripts/monitor-deploy.sh + +# Or explicitly specify +../scripts/monitor-deploy.sh agent +``` + +**After git push:** +```bash +git push && ./scripts/monitor-deploy.sh +``` + +### What it shows + +**Pipeline Status:** +- ✓ Success (green) +- ✗ Failed (red) +- → Running (blue) +- ⏳ Pending (yellow) + +**For each job:** +- Stage (test, build, release) +- Status icon +- Job name +- Duration + +**For main app on main branch:** +- Triggers Kubernetes rollout restart +- Monitors deployment progress +- Shows pod status +- Displays current image version + +### Example Output + +``` +╔════════════════════════════════════════════════════════════════╗ +║ Towerops - CI/CD Monitor ║ +╚════════════════════════════════════════════════════════════════╝ + +Branch: main +Pipeline: #12345 +Status: → running + +Jobs: + [test] ✓ test 45s + [build] → build:main - + [release] ⏳ release:multiarch - + +Refreshing in 5s... (Ctrl+C to stop) +``` + +### Troubleshooting + +**"GITLAB_TOKEN not set"** +- Export your GitLab personal access token + +**"jq is required"** +- Install: `brew install jq` + +**"No pipeline found"** +- Push to trigger a new pipeline +- Check that you're on the correct branch + +**"Deployment not found"** +- Only applies to main app on main branch +- Verify kubectl context: `/opt/homebrew/bin/kubectl config current-context` + +### Project Detection + +The script auto-detects the project from git remote URL: +- URL contains "towerops-agent" → Agent +- URL contains "towerops" → Main app + +Manual override: `./scripts/monitor-deploy.sh [app|agent]` + +### GitLab Project IDs + +- Main app: `graham/towerops` +- Agent: `towerops/towerops-agent` + +Update these in the script if your project paths differ. diff --git a/scripts/monitor-deploy.sh b/scripts/monitor-deploy.sh new file mode 100755 index 00000000..54941d39 --- /dev/null +++ b/scripts/monitor-deploy.sh @@ -0,0 +1,338 @@ +#!/bin/bash + +# Monitor GitLab CI/CD pipeline and Kubernetes deployment progress +# Works for both towerops main app and towerops-agent + +set -e + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +NC='\033[0m' # No Color + +# Icons +CHECK="✓" +CROSS="✗" +ARROW="→" +CLOCK="⏳" + +# Config +KUBECTL="/opt/homebrew/bin/kubectl" +REFRESH_INTERVAL=5 + +usage() { + echo "Usage: $0 [app|agent]" + echo "" + echo "Monitor CI/CD pipeline and deployment progress." + echo "Auto-detects project if run from git repository." + echo "" + echo "Options:" + echo " app - Monitor towerops main application" + echo " agent - Monitor towerops-agent" + echo "" + echo "Environment variables:" + echo " GITLAB_TOKEN - Required for API access" + echo " Get from: https://gitlab.com/-/profile/personal_access_tokens" + echo " Scopes needed: read_api" + exit 1 +} + +check_dependencies() { + if [ -z "$GITLAB_TOKEN" ]; then + echo -e "${RED}Error: GITLAB_TOKEN environment variable not set${NC}" + echo "Get a token from: https://gitlab.com/-/profile/personal_access_tokens" + echo "Then export GITLAB_TOKEN=" + exit 1 + fi + + if ! command -v jq &> /dev/null; then + echo -e "${RED}Error: jq is required but not installed${NC}" + echo "Install with: brew install jq" + exit 1 + fi + + if ! command -v curl &> /dev/null; then + echo -e "${RED}Error: curl is required but not installed${NC}" + exit 1 + fi +} + +detect_project() { + # Try to detect from git remote + if git remote get-url origin &>/dev/null; then + local remote_url=$(git remote get-url origin) + + if echo "$remote_url" | grep -q "towerops-agent"; then + echo "agent" + elif echo "$remote_url" | grep -q "towerops"; then + echo "app" + fi + fi +} + +get_project_info() { + local project=$1 + + case $project in + app) + PROJECT_ID="graham%2Ftowerops" # URL-encoded graham/towerops + PROJECT_NAME="Towerops" + K8S_NAMESPACE="towerops" + K8S_DEPLOYMENT="towerops" + HAS_K8S=true + ;; + agent) + PROJECT_ID="towerops%2Ftowerops-agent" # URL-encoded towerops/towerops-agent + PROJECT_NAME="Towerops Agent" + HAS_K8S=false + ;; + *) + echo -e "${RED}Unknown project: $project${NC}" + usage + ;; + esac +} + +gitlab_api() { + local endpoint=$1 + curl -s -H "PRIVATE-TOKEN: $GITLAB_TOKEN" \ + "https://gitlab.com/api/v4${endpoint}" +} + +get_current_branch() { + git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "main" +} + +get_latest_pipeline() { + local branch=$1 + gitlab_api "/projects/${PROJECT_ID}/pipelines?ref=${branch}&per_page=1" | \ + jq -r '.[0] // empty' +} + +get_pipeline_jobs() { + local pipeline_id=$1 + gitlab_api "/projects/${PROJECT_ID}/pipelines/${pipeline_id}/jobs" +} + +get_job_log() { + local job_id=$1 + gitlab_api "/projects/${PROJECT_ID}/jobs/${job_id}/trace" +} + +status_icon() { + local status=$1 + case $status in + success|passed) + echo -e "${GREEN}${CHECK}${NC}" + ;; + failed) + echo -e "${RED}${CROSS}${NC}" + ;; + running) + echo -e "${BLUE}${ARROW}${NC}" + ;; + pending|created) + echo -e "${YELLOW}${CLOCK}${NC}" + ;; + canceled|skipped) + echo -e "${YELLOW}-${NC}" + ;; + *) + echo -e "${CYAN}?${NC}" + ;; + esac +} + +format_duration() { + local seconds=$1 + if [ -z "$seconds" ] || [ "$seconds" = "null" ]; then + echo "-" + return + fi + + local mins=$((seconds / 60)) + local secs=$((seconds % 60)) + + if [ $mins -gt 0 ]; then + echo "${mins}m ${secs}s" + else + echo "${secs}s" + fi +} + +print_header() { + clear + echo -e "${CYAN}╔════════════════════════════════════════════════════════════════╗${NC}" + echo -e "${CYAN}║${NC} ${BLUE}${PROJECT_NAME} - CI/CD Monitor${NC} ${CYAN}║${NC}" + echo -e "${CYAN}╚════════════════════════════════════════════════════════════════╝${NC}" + echo "" +} + +monitor_pipeline() { + local branch=$(get_current_branch) + + echo -e "${BLUE}Branch:${NC} $branch" + echo -e "${BLUE}Fetching latest pipeline...${NC}" + echo "" + + local pipeline=$(get_latest_pipeline "$branch") + + if [ -z "$pipeline" ] || [ "$pipeline" = "null" ]; then + echo -e "${YELLOW}No pipeline found for branch: $branch${NC}" + echo "Push to trigger a new pipeline" + exit 0 + fi + + local pipeline_id=$(echo "$pipeline" | jq -r '.id') + local pipeline_status=$(echo "$pipeline" | jq -r '.status') + local pipeline_web_url=$(echo "$pipeline" | jq -r '.web_url') + + echo -e "${BLUE}Pipeline:${NC} #$pipeline_id" + echo -e "${BLUE}Status:${NC} $(status_icon $pipeline_status) $pipeline_status" + echo -e "${BLUE}URL:${NC} $pipeline_web_url" + echo "" + + local start_time=$(date +%s) + + # Monitor until complete + while true; do + print_header + + echo -e "${BLUE}Branch:${NC} $branch" + echo -e "${BLUE}Pipeline:${NC} #$pipeline_id" + echo -e "${BLUE}Status:${NC} $(status_icon $pipeline_status) $pipeline_status" + echo "" + + # Get and display jobs + local jobs=$(get_pipeline_jobs "$pipeline_id") + + echo -e "${CYAN}Jobs:${NC}" + echo "$jobs" | jq -r '.[] | "\(.stage)|\(.name)|\(.status)|\(.duration)"' | \ + while IFS='|' read -r stage name status duration; do + printf " %-12s $(status_icon $status) %-30s %s\n" \ + "[$stage]" "$name" "$(format_duration $duration)" + done + + echo "" + + # Check for failures + local failed_jobs=$(echo "$jobs" | jq -r '.[] | select(.status == "failed") | .id') + if [ -n "$failed_jobs" ]; then + echo -e "${RED}Failed Jobs:${NC}" + for job_id in $failed_jobs; do + local job_name=$(echo "$jobs" | jq -r ".[] | select(.id == $job_id) | .name") + echo -e " ${RED}${CROSS}${NC} $job_name (Job #$job_id)" + echo -e " ${BLUE}View logs:${NC} https://gitlab.com/$(echo $PROJECT_ID | sed 's/%2F/\//g')/-/jobs/$job_id" + done + echo "" + fi + + # Update pipeline status + pipeline=$(get_latest_pipeline "$branch") + local new_status=$(echo "$pipeline" | jq -r '.status') + + if [ "$new_status" != "$pipeline_status" ]; then + pipeline_status=$new_status + fi + + # Check if complete + if [[ "$pipeline_status" == "success" ]] || [[ "$pipeline_status" == "failed" ]] || \ + [[ "$pipeline_status" == "canceled" ]] || [[ "$pipeline_status" == "skipped" ]]; then + + local elapsed=$(($(date +%s) - start_time)) + echo -e "${BLUE}Pipeline completed in:${NC} $(format_duration $elapsed)" + echo "" + + if [ "$pipeline_status" = "success" ]; then + echo -e "${GREEN}${CHECK} Pipeline succeeded!${NC}" + + # Monitor Kubernetes deployment if applicable + if [ "$HAS_K8S" = true ] && [ "$branch" = "main" ]; then + echo "" + monitor_k8s_deployment + fi + else + echo -e "${RED}${CROSS} Pipeline $pipeline_status${NC}" + fi + + break + fi + + echo -e "${YELLOW}Refreshing in ${REFRESH_INTERVAL}s... (Ctrl+C to stop)${NC}" + sleep $REFRESH_INTERVAL + done +} + +monitor_k8s_deployment() { + echo -e "${CYAN}╔════════════════════════════════════════════════════════════════╗${NC}" + echo -e "${CYAN}║${NC} ${BLUE}Kubernetes Deployment${NC} ${CYAN}║${NC}" + echo -e "${CYAN}╚════════════════════════════════════════════════════════════════╝${NC}" + echo "" + + if ! $KUBECTL get deployment -n $K8S_NAMESPACE $K8S_DEPLOYMENT &>/dev/null; then + echo -e "${YELLOW}Warning: Deployment $K8S_DEPLOYMENT not found in namespace $K8S_NAMESPACE${NC}" + return + fi + + echo -e "${BLUE}Namespace:${NC} $K8S_NAMESPACE" + echo -e "${BLUE}Deployment:${NC} $K8S_DEPLOYMENT" + echo "" + + # Trigger rollout restart + echo -e "${BLUE}Restarting deployment...${NC}" + $KUBECTL -n $K8S_NAMESPACE rollout restart deployment/$K8S_DEPLOYMENT + echo "" + + # Monitor rollout status + echo -e "${BLUE}Waiting for rollout to complete...${NC}" + if $KUBECTL -n $K8S_NAMESPACE rollout status deployment/$K8S_DEPLOYMENT --timeout=300s; then + echo "" + echo -e "${GREEN}${CHECK} Deployment successful!${NC}" + echo "" + + # Show pod status + echo -e "${BLUE}Pod status:${NC}" + $KUBECTL -n $K8S_NAMESPACE get pods -l app=$K8S_DEPLOYMENT + echo "" + + # Get image version + local image=$($KUBECTL -n $K8S_NAMESPACE get deployment $K8S_DEPLOYMENT -o jsonpath='{.spec.template.spec.containers[0].image}') + echo -e "${BLUE}Current image:${NC} $image" + else + echo "" + echo -e "${RED}${CROSS} Deployment failed or timed out${NC}" + echo "" + echo -e "${BLUE}Recent events:${NC}" + $KUBECTL -n $K8S_NAMESPACE get events --sort-by=.lastTimestamp | tail -10 + fi +} + +# Main +main() { + check_dependencies + + local project="" + + # Detect or use argument + if [ $# -eq 0 ]; then + project=$(detect_project) + if [ -z "$project" ]; then + echo -e "${YELLOW}Could not auto-detect project${NC}" + usage + fi + echo -e "${BLUE}Auto-detected project:${NC} $project" + echo "" + else + project=$1 + fi + + get_project_info "$project" + + print_header + monitor_pipeline +} + +main "$@"