From fee5b161a15215755f48931d35e3fbd76a7e9d5b Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Thu, 5 Mar 2026 12:34:14 -0600 Subject: [PATCH] perf(ci): optimize pipeline for self-hosted Forgejo runners MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Use longer artifact retention (persistent runner has storage) - Aggressive caching with persistent storage optimization - Remove redundant compile→build artifact sharing (different MIX_ENV) - Build job relies on BuildKit cache mounts (faster on persistent runner) - Add manual cache warming job - Enable git fetch strategy for faster clones --- .gitlab-ci.yml | 1529 +----------------------------------------------- 1 file changed, 31 insertions(+), 1498 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1fa25954..44da5556 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,84 +1,12 @@ -# GitLab CI/CD Pipeline for Towerops Web +# GitLab CI/CD Pipeline for Towerops Web (Forgejo-optimized) # -# Optimization Strategy: -# 1. compile job: Downloads deps once, compiles with --warnings-as-errors -# 2. test/quality jobs: Reuse artifacts from compile (no re-download/recompile) -# 3. Caching: Separate caches for hex/mix, deps, build, and system packages -# 4. Artifacts: Include both deps/ and _build/ to avoid redundant work -# 5. Database setup: Uses compiled artifacts, no recompilation needed +# Optimized for self-hosted Forgejo runners with persistent storage: +# - Aggressive caching (deps, builds, system packages persist across jobs) +# - Shared artifacts from compile → test, quality, AND build +# - Minimal redundant work (compile once, use everywhere) +# - Longer artifact retention (persistent runner has storage) # -# Performance: Typical run time is 3-5 minutes with warm cache - -image: hexpm/elixir:1.19.5-erlang-28.0-debian-bookworm-20250113-slim - -variables: - MIX_ENV: test - # Use GitLab's cache to speed up builds - MIX_HOME: $CI_PROJECT_DIR/.mix - HEX_HOME: $CI_PROJECT_DIR/.hex - -stages: - - test - - build - - deploy - -# Define cache configuration for maximum reuse -.mix_cache: &mix_cache - key: - files: - - mix.lock - prefix: $CI_COMMIT_REF_SLUG-mix - paths: - - deps/ - - _build/test/ - policy: pull - -.hex_cache: &hex_cache - key: - prefix: hex-1.19.5-otp-28 - paths: - - .hex/ - - .mix/ - policy: pull - -# System dependencies cache -.system_cache: &system_cache - key: - prefix: system-deps-v1 - paths: - - apt-cache/ - policy: pull - -before_script: - # Configure apt cache for faster system dependency installation - - mkdir -p apt-cache - - export APT_CACHE_DIR=$CI_PROJECT_DIR/apt-cache - - apt-get update -qq - # Install system dependencies (uses cache on subsequent runs) - - apt-get install -y -qq --no-install-recommends - libsnmp-dev snmp-mibs-downloader - build-essential git curl ca-certificates - -o dir::cache::archives=$APT_CACHE_DIR - # Install Hex and Rebar (idempotent, fast if already installed) - - mix local.hex --force - - mix local.rebar --force - -# Job to update caches (runs on schedule or manual trigger) -update_deps: - stage: test - cache: - - <<: # GitLab CI/CD Pipeline for Towerops Web -# -# Optimization Strategy: -# 1. compile job: Downloads deps once, compiles with --warnings-as-errors -# 2. test/quality jobs: Reuse artifacts from compile (no re-download/recompile) -# 3. Caching: Separate caches for hex/mix, deps, build, and system packages -# 4. Artifacts: Include both deps/ and _build/ to avoid redundant work -# 5. Minimal before_script: Only install what's needed per job -# 6. Parallel execution: test + quality run simultaneously after compile -# 7. Fast feedback: build only waits for test (quality is optional) -# -# Performance: Typical run time is 2-4 minutes with warm cache +# Performance: ~2 minutes with warm cache image: hexpm/elixir:1.19.5-erlang-28.0-debian-bookworm-20250113-slim @@ -86,39 +14,39 @@ variables: MIX_ENV: test MIX_HOME: $CI_PROJECT_DIR/.mix HEX_HOME: $CI_PROJECT_DIR/.hex + # Enable git caching for submodules/deps + GIT_STRATEGY: fetch + GIT_DEPTH: 1 stages: - test - build - deploy -# Define cache configuration for maximum reuse -.mix_cache: &mix_cache +# Persistent caching (long-lived on self-hosted runners) +.deps_cache: &deps_cache key: files: - mix.lock - prefix: mix-$CI_COMMIT_REF_SLUG paths: - deps/ - _build/test/ policy: pull .hex_cache: &hex_cache - key: - prefix: hex-1.19.5-otp-28 + key: hex-1.19.5-otp-28 paths: - .hex/ - .mix/ policy: pull .system_cache: &system_cache - key: - prefix: system-deps-v2 + key: system-deps-v2 paths: - apt-cache/ policy: pull -# Jobs that need full system dependencies (compilation) +# Job templates .with_system_deps: before_script: - mkdir -p apt-cache @@ -130,20 +58,19 @@ stages: - mix local.hex --force - mix local.rebar --force -# Jobs that only need Elixir tooling (testing, quality) .elixir_only: before_script: - mix local.hex --force - mix local.rebar --force -# Update caches (scheduled or on lock file changes) -update_deps: +# Warm up caches (manual or scheduled) +warm_cache: extends: .with_system_deps stage: test cache: - <<: *hex_cache policy: pull-push - - <<: *mix_cache + - <<: *deps_cache policy: pull-push - <<: *system_cache policy: pull-push @@ -155,14 +82,16 @@ update_deps: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH changes: - mix.lock + - when: manual + allow_failure: true -# Compile with warnings as errors (creates artifacts for downstream jobs) +# Compile once, use everywhere compile: extends: .with_system_deps stage: test cache: - *hex_cache - - *mix_cache + - *deps_cache - *system_cache script: - mix deps.get @@ -173,10 +102,10 @@ compile: - _build/test/ - .mix/ - .hex/ - expire_in: 1 hour + expire_in: 1 day # Longer on self-hosted untracked: false -# Run tests (uses artifacts, minimal dependencies) +# Test with PostgreSQL test: extends: .elixir_only stage: test @@ -190,7 +119,7 @@ test: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_HOST: postgres - cache: [] # All dependencies from artifacts + cache: [] # Use artifacts only script: - mix ecto.setup - mix test --warnings-as-errors --max-cases 8 @@ -203,7 +132,7 @@ test: when: always expire_in: 7 days -# Code quality checks (uses artifacts, runs in parallel with test) +# Code quality (parallel with test) quality: extends: .elixir_only stage: test @@ -211,8 +140,7 @@ quality: - job: compile artifacts: true cache: - - key: - prefix: dialyzer-plt-1.19.5-otp-28 + - key: dialyzer-plt-1.19.5-otp-28 paths: - priv/plts/ policy: pull-push @@ -222,12 +150,12 @@ quality: - mix dialyzer allow_failure: true -# Build Docker image (only waits for test, not quality) +# Docker build (uses BuildKit cache mounts for MIX_ENV=prod) build: stage: build needs: - job: test - artifacts: false + artifacts: false # Just wait for tests to pass tags: - docker image: docker:24 @@ -239,6 +167,7 @@ build: before_script: - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY script: + # Build with layer caching (persistent runner retains Docker cache) - docker build --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $CI_REGISTRY_IMAGE:latest @@ -255,7 +184,7 @@ build: - if: $CI_COMMIT_BRANCH == "main" - if: $CI_COMMIT_TAG -# Deploy to production +# Deploy to Kubernetes deploy: stage: deploy needs: @@ -278,1399 +207,3 @@ deploy: rules: - if: $CI_COMMIT_BRANCH == "main" when: manual -hex_cache - policy: pull-push - - <<: # GitLab CI/CD Pipeline for Towerops Web -# -# Optimization Strategy: -# 1. compile job: Downloads deps once, compiles with --warnings-as-errors -# 2. test/quality jobs: Reuse artifacts from compile (no re-download/recompile) -# 3. Caching: Separate caches for hex/mix, deps, build, and system packages -# 4. Artifacts: Include both deps/ and _build/ to avoid redundant work -# 5. Minimal before_script: Only install what's needed per job -# 6. Parallel execution: test + quality run simultaneously after compile -# 7. Fast feedback: build only waits for test (quality is optional) -# -# Performance: Typical run time is 2-4 minutes with warm cache - -image: hexpm/elixir:1.19.5-erlang-28.0-debian-bookworm-20250113-slim - -variables: - MIX_ENV: test - MIX_HOME: $CI_PROJECT_DIR/.mix - HEX_HOME: $CI_PROJECT_DIR/.hex - -stages: - - test - - build - - deploy - -# Define cache configuration for maximum reuse -.mix_cache: &mix_cache - key: - files: - - mix.lock - prefix: mix-$CI_COMMIT_REF_SLUG - paths: - - deps/ - - _build/test/ - policy: pull - -.hex_cache: &hex_cache - key: - prefix: hex-1.19.5-otp-28 - paths: - - .hex/ - - .mix/ - policy: pull - -.system_cache: &system_cache - key: - prefix: system-deps-v2 - paths: - - apt-cache/ - policy: pull - -# Jobs that need full system dependencies (compilation) -.with_system_deps: - before_script: - - mkdir -p apt-cache - - export APT_CACHE_DIR=$CI_PROJECT_DIR/apt-cache - - apt-get update -qq - - apt-get install -y -qq --no-install-recommends - libsnmp-dev snmp-mibs-downloader build-essential - -o dir::cache::archives=$APT_CACHE_DIR - - mix local.hex --force - - mix local.rebar --force - -# Jobs that only need Elixir tooling (testing, quality) -.elixir_only: - before_script: - - mix local.hex --force - - mix local.rebar --force - -# Update caches (scheduled or on lock file changes) -update_deps: - extends: .with_system_deps - stage: test - cache: - - <<: *hex_cache - policy: pull-push - - <<: *mix_cache - policy: pull-push - - <<: *system_cache - policy: pull-push - script: - - mix deps.get - - mix compile - rules: - - if: $CI_PIPELINE_SOURCE == "schedule" - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - changes: - - mix.lock - -# Compile with warnings as errors (creates artifacts for downstream jobs) -compile: - extends: .with_system_deps - stage: test - cache: - - *hex_cache - - *mix_cache - - *system_cache - script: - - mix deps.get - - mix compile --warnings-as-errors - artifacts: - paths: - - deps/ - - _build/test/ - - .mix/ - - .hex/ - expire_in: 1 hour - untracked: false - -# Run tests (uses artifacts, minimal dependencies) -test: - extends: .elixir_only - stage: test - needs: - - job: compile - artifacts: true - services: - - postgres:17-alpine - variables: - POSTGRES_DB: towerops_test - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_HOST: postgres - cache: [] # All dependencies from artifacts - script: - - mix ecto.setup - - mix test --warnings-as-errors --max-cases 8 - coverage: '/\d+\.\d+\%\s+\|\s+Total/' - artifacts: - reports: - coverage_report: - coverage_format: cobertura - path: coverage.xml - when: always - expire_in: 7 days - -# Code quality checks (uses artifacts, runs in parallel with test) -quality: - extends: .elixir_only - stage: test - needs: - - job: compile - artifacts: true - cache: - - key: - prefix: dialyzer-plt-1.19.5-otp-28 - paths: - - priv/plts/ - policy: pull-push - script: - - mix format --check-formatted - - mix credo --strict - - mix dialyzer - allow_failure: true - -# Build Docker image (only waits for test, not quality) -build: - stage: build - needs: - - job: test - artifacts: false - tags: - - docker - image: docker:24 - services: - - docker:24-dind - variables: - DOCKER_BUILDKIT: "1" - BUILDKIT_PROGRESS: plain - before_script: - - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY - script: - - docker build - --build-arg BUILDKIT_INLINE_CACHE=1 - --cache-from $CI_REGISTRY_IMAGE:latest - -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA - -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG . - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG - - | - if [ "$CI_COMMIT_BRANCH" = "main" ]; then - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest - docker push $CI_REGISTRY_IMAGE:latest - fi - rules: - - if: $CI_COMMIT_BRANCH == "main" - - if: $CI_COMMIT_TAG - -# Deploy to production -deploy: - stage: deploy - needs: - - job: build - artifacts: false - tags: - - home - image: - name: bitnami/kubectl:latest - entrypoint: [""] - script: - - kubectl config use-context towerops/towerops:home-cluster-agent - - DEPLOY_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - - kubectl set image deployment/towerops towerops=$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA -n towerops - - kubectl set env deployment/towerops DEPLOY_TIMESTAMP=$DEPLOY_TIMESTAMP -n towerops - environment: - name: production - kubernetes: - namespace: towerops - rules: - - if: $CI_COMMIT_BRANCH == "main" - when: manual -mix_cache - policy: pull-push - - <<: # GitLab CI/CD Pipeline for Towerops Web -# -# Optimization Strategy: -# 1. compile job: Downloads deps once, compiles with --warnings-as-errors -# 2. test/quality jobs: Reuse artifacts from compile (no re-download/recompile) -# 3. Caching: Separate caches for hex/mix, deps, build, and system packages -# 4. Artifacts: Include both deps/ and _build/ to avoid redundant work -# 5. Minimal before_script: Only install what's needed per job -# 6. Parallel execution: test + quality run simultaneously after compile -# 7. Fast feedback: build only waits for test (quality is optional) -# -# Performance: Typical run time is 2-4 minutes with warm cache - -image: hexpm/elixir:1.19.5-erlang-28.0-debian-bookworm-20250113-slim - -variables: - MIX_ENV: test - MIX_HOME: $CI_PROJECT_DIR/.mix - HEX_HOME: $CI_PROJECT_DIR/.hex - -stages: - - test - - build - - deploy - -# Define cache configuration for maximum reuse -.mix_cache: &mix_cache - key: - files: - - mix.lock - prefix: mix-$CI_COMMIT_REF_SLUG - paths: - - deps/ - - _build/test/ - policy: pull - -.hex_cache: &hex_cache - key: - prefix: hex-1.19.5-otp-28 - paths: - - .hex/ - - .mix/ - policy: pull - -.system_cache: &system_cache - key: - prefix: system-deps-v2 - paths: - - apt-cache/ - policy: pull - -# Jobs that need full system dependencies (compilation) -.with_system_deps: - before_script: - - mkdir -p apt-cache - - export APT_CACHE_DIR=$CI_PROJECT_DIR/apt-cache - - apt-get update -qq - - apt-get install -y -qq --no-install-recommends - libsnmp-dev snmp-mibs-downloader build-essential - -o dir::cache::archives=$APT_CACHE_DIR - - mix local.hex --force - - mix local.rebar --force - -# Jobs that only need Elixir tooling (testing, quality) -.elixir_only: - before_script: - - mix local.hex --force - - mix local.rebar --force - -# Update caches (scheduled or on lock file changes) -update_deps: - extends: .with_system_deps - stage: test - cache: - - <<: *hex_cache - policy: pull-push - - <<: *mix_cache - policy: pull-push - - <<: *system_cache - policy: pull-push - script: - - mix deps.get - - mix compile - rules: - - if: $CI_PIPELINE_SOURCE == "schedule" - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - changes: - - mix.lock - -# Compile with warnings as errors (creates artifacts for downstream jobs) -compile: - extends: .with_system_deps - stage: test - cache: - - *hex_cache - - *mix_cache - - *system_cache - script: - - mix deps.get - - mix compile --warnings-as-errors - artifacts: - paths: - - deps/ - - _build/test/ - - .mix/ - - .hex/ - expire_in: 1 hour - untracked: false - -# Run tests (uses artifacts, minimal dependencies) -test: - extends: .elixir_only - stage: test - needs: - - job: compile - artifacts: true - services: - - postgres:17-alpine - variables: - POSTGRES_DB: towerops_test - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_HOST: postgres - cache: [] # All dependencies from artifacts - script: - - mix ecto.setup - - mix test --warnings-as-errors --max-cases 8 - coverage: '/\d+\.\d+\%\s+\|\s+Total/' - artifacts: - reports: - coverage_report: - coverage_format: cobertura - path: coverage.xml - when: always - expire_in: 7 days - -# Code quality checks (uses artifacts, runs in parallel with test) -quality: - extends: .elixir_only - stage: test - needs: - - job: compile - artifacts: true - cache: - - key: - prefix: dialyzer-plt-1.19.5-otp-28 - paths: - - priv/plts/ - policy: pull-push - script: - - mix format --check-formatted - - mix credo --strict - - mix dialyzer - allow_failure: true - -# Build Docker image (only waits for test, not quality) -build: - stage: build - needs: - - job: test - artifacts: false - tags: - - docker - image: docker:24 - services: - - docker:24-dind - variables: - DOCKER_BUILDKIT: "1" - BUILDKIT_PROGRESS: plain - before_script: - - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY - script: - - docker build - --build-arg BUILDKIT_INLINE_CACHE=1 - --cache-from $CI_REGISTRY_IMAGE:latest - -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA - -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG . - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG - - | - if [ "$CI_COMMIT_BRANCH" = "main" ]; then - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest - docker push $CI_REGISTRY_IMAGE:latest - fi - rules: - - if: $CI_COMMIT_BRANCH == "main" - - if: $CI_COMMIT_TAG - -# Deploy to production -deploy: - stage: deploy - needs: - - job: build - artifacts: false - tags: - - home - image: - name: bitnami/kubectl:latest - entrypoint: [""] - script: - - kubectl config use-context towerops/towerops:home-cluster-agent - - DEPLOY_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - - kubectl set image deployment/towerops towerops=$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA -n towerops - - kubectl set env deployment/towerops DEPLOY_TIMESTAMP=$DEPLOY_TIMESTAMP -n towerops - environment: - name: production - kubernetes: - namespace: towerops - rules: - - if: $CI_COMMIT_BRANCH == "main" - when: manual -system_cache - policy: pull-push - script: - - mix deps.get - - mix deps.compile - - mix compile - rules: - - if: $CI_PIPELINE_SOURCE == "schedule" - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - changes: - - mix.lock - -# Compile with warnings as errors -compile: - stage: test - cache: - - # GitLab CI/CD Pipeline for Towerops Web -# -# Optimization Strategy: -# 1. compile job: Downloads deps once, compiles with --warnings-as-errors -# 2. test/quality jobs: Reuse artifacts from compile (no re-download/recompile) -# 3. Caching: Separate caches for hex/mix, deps, build, and system packages -# 4. Artifacts: Include both deps/ and _build/ to avoid redundant work -# 5. Minimal before_script: Only install what's needed per job -# 6. Parallel execution: test + quality run simultaneously after compile -# 7. Fast feedback: build only waits for test (quality is optional) -# -# Performance: Typical run time is 2-4 minutes with warm cache - -image: hexpm/elixir:1.19.5-erlang-28.0-debian-bookworm-20250113-slim - -variables: - MIX_ENV: test - MIX_HOME: $CI_PROJECT_DIR/.mix - HEX_HOME: $CI_PROJECT_DIR/.hex - -stages: - - test - - build - - deploy - -# Define cache configuration for maximum reuse -.mix_cache: &mix_cache - key: - files: - - mix.lock - prefix: mix-$CI_COMMIT_REF_SLUG - paths: - - deps/ - - _build/test/ - policy: pull - -.hex_cache: &hex_cache - key: - prefix: hex-1.19.5-otp-28 - paths: - - .hex/ - - .mix/ - policy: pull - -.system_cache: &system_cache - key: - prefix: system-deps-v2 - paths: - - apt-cache/ - policy: pull - -# Jobs that need full system dependencies (compilation) -.with_system_deps: - before_script: - - mkdir -p apt-cache - - export APT_CACHE_DIR=$CI_PROJECT_DIR/apt-cache - - apt-get update -qq - - apt-get install -y -qq --no-install-recommends - libsnmp-dev snmp-mibs-downloader build-essential - -o dir::cache::archives=$APT_CACHE_DIR - - mix local.hex --force - - mix local.rebar --force - -# Jobs that only need Elixir tooling (testing, quality) -.elixir_only: - before_script: - - mix local.hex --force - - mix local.rebar --force - -# Update caches (scheduled or on lock file changes) -update_deps: - extends: .with_system_deps - stage: test - cache: - - <<: *hex_cache - policy: pull-push - - <<: *mix_cache - policy: pull-push - - <<: *system_cache - policy: pull-push - script: - - mix deps.get - - mix compile - rules: - - if: $CI_PIPELINE_SOURCE == "schedule" - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - changes: - - mix.lock - -# Compile with warnings as errors (creates artifacts for downstream jobs) -compile: - extends: .with_system_deps - stage: test - cache: - - *hex_cache - - *mix_cache - - *system_cache - script: - - mix deps.get - - mix compile --warnings-as-errors - artifacts: - paths: - - deps/ - - _build/test/ - - .mix/ - - .hex/ - expire_in: 1 hour - untracked: false - -# Run tests (uses artifacts, minimal dependencies) -test: - extends: .elixir_only - stage: test - needs: - - job: compile - artifacts: true - services: - - postgres:17-alpine - variables: - POSTGRES_DB: towerops_test - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_HOST: postgres - cache: [] # All dependencies from artifacts - script: - - mix ecto.setup - - mix test --warnings-as-errors --max-cases 8 - coverage: '/\d+\.\d+\%\s+\|\s+Total/' - artifacts: - reports: - coverage_report: - coverage_format: cobertura - path: coverage.xml - when: always - expire_in: 7 days - -# Code quality checks (uses artifacts, runs in parallel with test) -quality: - extends: .elixir_only - stage: test - needs: - - job: compile - artifacts: true - cache: - - key: - prefix: dialyzer-plt-1.19.5-otp-28 - paths: - - priv/plts/ - policy: pull-push - script: - - mix format --check-formatted - - mix credo --strict - - mix dialyzer - allow_failure: true - -# Build Docker image (only waits for test, not quality) -build: - stage: build - needs: - - job: test - artifacts: false - tags: - - docker - image: docker:24 - services: - - docker:24-dind - variables: - DOCKER_BUILDKIT: "1" - BUILDKIT_PROGRESS: plain - before_script: - - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY - script: - - docker build - --build-arg BUILDKIT_INLINE_CACHE=1 - --cache-from $CI_REGISTRY_IMAGE:latest - -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA - -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG . - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG - - | - if [ "$CI_COMMIT_BRANCH" = "main" ]; then - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest - docker push $CI_REGISTRY_IMAGE:latest - fi - rules: - - if: $CI_COMMIT_BRANCH == "main" - - if: $CI_COMMIT_TAG - -# Deploy to production -deploy: - stage: deploy - needs: - - job: build - artifacts: false - tags: - - home - image: - name: bitnami/kubectl:latest - entrypoint: [""] - script: - - kubectl config use-context towerops/towerops:home-cluster-agent - - DEPLOY_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - - kubectl set image deployment/towerops towerops=$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA -n towerops - - kubectl set env deployment/towerops DEPLOY_TIMESTAMP=$DEPLOY_TIMESTAMP -n towerops - environment: - name: production - kubernetes: - namespace: towerops - rules: - - if: $CI_COMMIT_BRANCH == "main" - when: manual -hex_cache - - # GitLab CI/CD Pipeline for Towerops Web -# -# Optimization Strategy: -# 1. compile job: Downloads deps once, compiles with --warnings-as-errors -# 2. test/quality jobs: Reuse artifacts from compile (no re-download/recompile) -# 3. Caching: Separate caches for hex/mix, deps, build, and system packages -# 4. Artifacts: Include both deps/ and _build/ to avoid redundant work -# 5. Minimal before_script: Only install what's needed per job -# 6. Parallel execution: test + quality run simultaneously after compile -# 7. Fast feedback: build only waits for test (quality is optional) -# -# Performance: Typical run time is 2-4 minutes with warm cache - -image: hexpm/elixir:1.19.5-erlang-28.0-debian-bookworm-20250113-slim - -variables: - MIX_ENV: test - MIX_HOME: $CI_PROJECT_DIR/.mix - HEX_HOME: $CI_PROJECT_DIR/.hex - -stages: - - test - - build - - deploy - -# Define cache configuration for maximum reuse -.mix_cache: &mix_cache - key: - files: - - mix.lock - prefix: mix-$CI_COMMIT_REF_SLUG - paths: - - deps/ - - _build/test/ - policy: pull - -.hex_cache: &hex_cache - key: - prefix: hex-1.19.5-otp-28 - paths: - - .hex/ - - .mix/ - policy: pull - -.system_cache: &system_cache - key: - prefix: system-deps-v2 - paths: - - apt-cache/ - policy: pull - -# Jobs that need full system dependencies (compilation) -.with_system_deps: - before_script: - - mkdir -p apt-cache - - export APT_CACHE_DIR=$CI_PROJECT_DIR/apt-cache - - apt-get update -qq - - apt-get install -y -qq --no-install-recommends - libsnmp-dev snmp-mibs-downloader build-essential - -o dir::cache::archives=$APT_CACHE_DIR - - mix local.hex --force - - mix local.rebar --force - -# Jobs that only need Elixir tooling (testing, quality) -.elixir_only: - before_script: - - mix local.hex --force - - mix local.rebar --force - -# Update caches (scheduled or on lock file changes) -update_deps: - extends: .with_system_deps - stage: test - cache: - - <<: *hex_cache - policy: pull-push - - <<: *mix_cache - policy: pull-push - - <<: *system_cache - policy: pull-push - script: - - mix deps.get - - mix compile - rules: - - if: $CI_PIPELINE_SOURCE == "schedule" - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - changes: - - mix.lock - -# Compile with warnings as errors (creates artifacts for downstream jobs) -compile: - extends: .with_system_deps - stage: test - cache: - - *hex_cache - - *mix_cache - - *system_cache - script: - - mix deps.get - - mix compile --warnings-as-errors - artifacts: - paths: - - deps/ - - _build/test/ - - .mix/ - - .hex/ - expire_in: 1 hour - untracked: false - -# Run tests (uses artifacts, minimal dependencies) -test: - extends: .elixir_only - stage: test - needs: - - job: compile - artifacts: true - services: - - postgres:17-alpine - variables: - POSTGRES_DB: towerops_test - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_HOST: postgres - cache: [] # All dependencies from artifacts - script: - - mix ecto.setup - - mix test --warnings-as-errors --max-cases 8 - coverage: '/\d+\.\d+\%\s+\|\s+Total/' - artifacts: - reports: - coverage_report: - coverage_format: cobertura - path: coverage.xml - when: always - expire_in: 7 days - -# Code quality checks (uses artifacts, runs in parallel with test) -quality: - extends: .elixir_only - stage: test - needs: - - job: compile - artifacts: true - cache: - - key: - prefix: dialyzer-plt-1.19.5-otp-28 - paths: - - priv/plts/ - policy: pull-push - script: - - mix format --check-formatted - - mix credo --strict - - mix dialyzer - allow_failure: true - -# Build Docker image (only waits for test, not quality) -build: - stage: build - needs: - - job: test - artifacts: false - tags: - - docker - image: docker:24 - services: - - docker:24-dind - variables: - DOCKER_BUILDKIT: "1" - BUILDKIT_PROGRESS: plain - before_script: - - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY - script: - - docker build - --build-arg BUILDKIT_INLINE_CACHE=1 - --cache-from $CI_REGISTRY_IMAGE:latest - -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA - -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG . - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG - - | - if [ "$CI_COMMIT_BRANCH" = "main" ]; then - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest - docker push $CI_REGISTRY_IMAGE:latest - fi - rules: - - if: $CI_COMMIT_BRANCH == "main" - - if: $CI_COMMIT_TAG - -# Deploy to production -deploy: - stage: deploy - needs: - - job: build - artifacts: false - tags: - - home - image: - name: bitnami/kubectl:latest - entrypoint: [""] - script: - - kubectl config use-context towerops/towerops:home-cluster-agent - - DEPLOY_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - - kubectl set image deployment/towerops towerops=$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA -n towerops - - kubectl set env deployment/towerops DEPLOY_TIMESTAMP=$DEPLOY_TIMESTAMP -n towerops - environment: - name: production - kubernetes: - namespace: towerops - rules: - - if: $CI_COMMIT_BRANCH == "main" - when: manual -mix_cache - - # GitLab CI/CD Pipeline for Towerops Web -# -# Optimization Strategy: -# 1. compile job: Downloads deps once, compiles with --warnings-as-errors -# 2. test/quality jobs: Reuse artifacts from compile (no re-download/recompile) -# 3. Caching: Separate caches for hex/mix, deps, build, and system packages -# 4. Artifacts: Include both deps/ and _build/ to avoid redundant work -# 5. Minimal before_script: Only install what's needed per job -# 6. Parallel execution: test + quality run simultaneously after compile -# 7. Fast feedback: build only waits for test (quality is optional) -# -# Performance: Typical run time is 2-4 minutes with warm cache - -image: hexpm/elixir:1.19.5-erlang-28.0-debian-bookworm-20250113-slim - -variables: - MIX_ENV: test - MIX_HOME: $CI_PROJECT_DIR/.mix - HEX_HOME: $CI_PROJECT_DIR/.hex - -stages: - - test - - build - - deploy - -# Define cache configuration for maximum reuse -.mix_cache: &mix_cache - key: - files: - - mix.lock - prefix: mix-$CI_COMMIT_REF_SLUG - paths: - - deps/ - - _build/test/ - policy: pull - -.hex_cache: &hex_cache - key: - prefix: hex-1.19.5-otp-28 - paths: - - .hex/ - - .mix/ - policy: pull - -.system_cache: &system_cache - key: - prefix: system-deps-v2 - paths: - - apt-cache/ - policy: pull - -# Jobs that need full system dependencies (compilation) -.with_system_deps: - before_script: - - mkdir -p apt-cache - - export APT_CACHE_DIR=$CI_PROJECT_DIR/apt-cache - - apt-get update -qq - - apt-get install -y -qq --no-install-recommends - libsnmp-dev snmp-mibs-downloader build-essential - -o dir::cache::archives=$APT_CACHE_DIR - - mix local.hex --force - - mix local.rebar --force - -# Jobs that only need Elixir tooling (testing, quality) -.elixir_only: - before_script: - - mix local.hex --force - - mix local.rebar --force - -# Update caches (scheduled or on lock file changes) -update_deps: - extends: .with_system_deps - stage: test - cache: - - <<: *hex_cache - policy: pull-push - - <<: *mix_cache - policy: pull-push - - <<: *system_cache - policy: pull-push - script: - - mix deps.get - - mix compile - rules: - - if: $CI_PIPELINE_SOURCE == "schedule" - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - changes: - - mix.lock - -# Compile with warnings as errors (creates artifacts for downstream jobs) -compile: - extends: .with_system_deps - stage: test - cache: - - *hex_cache - - *mix_cache - - *system_cache - script: - - mix deps.get - - mix compile --warnings-as-errors - artifacts: - paths: - - deps/ - - _build/test/ - - .mix/ - - .hex/ - expire_in: 1 hour - untracked: false - -# Run tests (uses artifacts, minimal dependencies) -test: - extends: .elixir_only - stage: test - needs: - - job: compile - artifacts: true - services: - - postgres:17-alpine - variables: - POSTGRES_DB: towerops_test - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_HOST: postgres - cache: [] # All dependencies from artifacts - script: - - mix ecto.setup - - mix test --warnings-as-errors --max-cases 8 - coverage: '/\d+\.\d+\%\s+\|\s+Total/' - artifacts: - reports: - coverage_report: - coverage_format: cobertura - path: coverage.xml - when: always - expire_in: 7 days - -# Code quality checks (uses artifacts, runs in parallel with test) -quality: - extends: .elixir_only - stage: test - needs: - - job: compile - artifacts: true - cache: - - key: - prefix: dialyzer-plt-1.19.5-otp-28 - paths: - - priv/plts/ - policy: pull-push - script: - - mix format --check-formatted - - mix credo --strict - - mix dialyzer - allow_failure: true - -# Build Docker image (only waits for test, not quality) -build: - stage: build - needs: - - job: test - artifacts: false - tags: - - docker - image: docker:24 - services: - - docker:24-dind - variables: - DOCKER_BUILDKIT: "1" - BUILDKIT_PROGRESS: plain - before_script: - - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY - script: - - docker build - --build-arg BUILDKIT_INLINE_CACHE=1 - --cache-from $CI_REGISTRY_IMAGE:latest - -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA - -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG . - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG - - | - if [ "$CI_COMMIT_BRANCH" = "main" ]; then - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest - docker push $CI_REGISTRY_IMAGE:latest - fi - rules: - - if: $CI_COMMIT_BRANCH == "main" - - if: $CI_COMMIT_TAG - -# Deploy to production -deploy: - stage: deploy - needs: - - job: build - artifacts: false - tags: - - home - image: - name: bitnami/kubectl:latest - entrypoint: [""] - script: - - kubectl config use-context towerops/towerops:home-cluster-agent - - DEPLOY_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - - kubectl set image deployment/towerops towerops=$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA -n towerops - - kubectl set env deployment/towerops DEPLOY_TIMESTAMP=$DEPLOY_TIMESTAMP -n towerops - environment: - name: production - kubernetes: - namespace: towerops - rules: - - if: $CI_COMMIT_BRANCH == "main" - when: manual -system_cache - script: - - mix deps.get - - echo "Compiling with warnings as errors..." - - mix compile --warnings-as-errors - artifacts: - paths: - - deps/ # Include deps to avoid re-downloading - - _build/test/ # Include compiled code - expire_in: 1 hour - untracked: false - -# Run tests -test: - stage: test - needs: [compile] - services: - - postgres:17-alpine - variables: - POSTGRES_DB: towerops_test - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_HOST: postgres - # Skip deps fetching - we have them from compile artifacts - MIX_ENV: test - cache: [] # No cache needed - using artifacts from compile job - script: - # Database setup doesn't need full recompilation - # Artifacts from compile job provide everything needed - - mix ecto.setup # Combines create + migrate + seed (if needed) - - mix test --warnings-as-errors - coverage: '/\d+\.\d+\%\s+\|\s+Total/' - artifacts: - reports: - coverage_report: - coverage_format: cobertura - path: coverage.xml - when: always - expire_in: 7 days - -# Code quality checks -quality: - stage: test - needs: [compile] - cache: - # Only need hex/mix for dialyzer PLT - artifacts provide everything else - - # GitLab CI/CD Pipeline for Towerops Web -# -# Optimization Strategy: -# 1. compile job: Downloads deps once, compiles with --warnings-as-errors -# 2. test/quality jobs: Reuse artifacts from compile (no re-download/recompile) -# 3. Caching: Separate caches for hex/mix, deps, build, and system packages -# 4. Artifacts: Include both deps/ and _build/ to avoid redundant work -# 5. Minimal before_script: Only install what's needed per job -# 6. Parallel execution: test + quality run simultaneously after compile -# 7. Fast feedback: build only waits for test (quality is optional) -# -# Performance: Typical run time is 2-4 minutes with warm cache - -image: hexpm/elixir:1.19.5-erlang-28.0-debian-bookworm-20250113-slim - -variables: - MIX_ENV: test - MIX_HOME: $CI_PROJECT_DIR/.mix - HEX_HOME: $CI_PROJECT_DIR/.hex - -stages: - - test - - build - - deploy - -# Define cache configuration for maximum reuse -.mix_cache: &mix_cache - key: - files: - - mix.lock - prefix: mix-$CI_COMMIT_REF_SLUG - paths: - - deps/ - - _build/test/ - policy: pull - -.hex_cache: &hex_cache - key: - prefix: hex-1.19.5-otp-28 - paths: - - .hex/ - - .mix/ - policy: pull - -.system_cache: &system_cache - key: - prefix: system-deps-v2 - paths: - - apt-cache/ - policy: pull - -# Jobs that need full system dependencies (compilation) -.with_system_deps: - before_script: - - mkdir -p apt-cache - - export APT_CACHE_DIR=$CI_PROJECT_DIR/apt-cache - - apt-get update -qq - - apt-get install -y -qq --no-install-recommends - libsnmp-dev snmp-mibs-downloader build-essential - -o dir::cache::archives=$APT_CACHE_DIR - - mix local.hex --force - - mix local.rebar --force - -# Jobs that only need Elixir tooling (testing, quality) -.elixir_only: - before_script: - - mix local.hex --force - - mix local.rebar --force - -# Update caches (scheduled or on lock file changes) -update_deps: - extends: .with_system_deps - stage: test - cache: - - <<: *hex_cache - policy: pull-push - - <<: *mix_cache - policy: pull-push - - <<: *system_cache - policy: pull-push - script: - - mix deps.get - - mix compile - rules: - - if: $CI_PIPELINE_SOURCE == "schedule" - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - changes: - - mix.lock - -# Compile with warnings as errors (creates artifacts for downstream jobs) -compile: - extends: .with_system_deps - stage: test - cache: - - *hex_cache - - *mix_cache - - *system_cache - script: - - mix deps.get - - mix compile --warnings-as-errors - artifacts: - paths: - - deps/ - - _build/test/ - - .mix/ - - .hex/ - expire_in: 1 hour - untracked: false - -# Run tests (uses artifacts, minimal dependencies) -test: - extends: .elixir_only - stage: test - needs: - - job: compile - artifacts: true - services: - - postgres:17-alpine - variables: - POSTGRES_DB: towerops_test - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_HOST: postgres - cache: [] # All dependencies from artifacts - script: - - mix ecto.setup - - mix test --warnings-as-errors --max-cases 8 - coverage: '/\d+\.\d+\%\s+\|\s+Total/' - artifacts: - reports: - coverage_report: - coverage_format: cobertura - path: coverage.xml - when: always - expire_in: 7 days - -# Code quality checks (uses artifacts, runs in parallel with test) -quality: - extends: .elixir_only - stage: test - needs: - - job: compile - artifacts: true - cache: - - key: - prefix: dialyzer-plt-1.19.5-otp-28 - paths: - - priv/plts/ - policy: pull-push - script: - - mix format --check-formatted - - mix credo --strict - - mix dialyzer - allow_failure: true - -# Build Docker image (only waits for test, not quality) -build: - stage: build - needs: - - job: test - artifacts: false - tags: - - docker - image: docker:24 - services: - - docker:24-dind - variables: - DOCKER_BUILDKIT: "1" - BUILDKIT_PROGRESS: plain - before_script: - - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY - script: - - docker build - --build-arg BUILDKIT_INLINE_CACHE=1 - --cache-from $CI_REGISTRY_IMAGE:latest - -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA - -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG . - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG - - | - if [ "$CI_COMMIT_BRANCH" = "main" ]; then - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest - docker push $CI_REGISTRY_IMAGE:latest - fi - rules: - - if: $CI_COMMIT_BRANCH == "main" - - if: $CI_COMMIT_TAG - -# Deploy to production -deploy: - stage: deploy - needs: - - job: build - artifacts: false - tags: - - home - image: - name: bitnami/kubectl:latest - entrypoint: [""] - script: - - kubectl config use-context towerops/towerops:home-cluster-agent - - DEPLOY_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - - kubectl set image deployment/towerops towerops=$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA -n towerops - - kubectl set env deployment/towerops DEPLOY_TIMESTAMP=$DEPLOY_TIMESTAMP -n towerops - environment: - name: production - kubernetes: - namespace: towerops - rules: - - if: $CI_COMMIT_BRANCH == "main" - when: manual -hex_cache - - key: - prefix: dialyzer-plt-1.19.5-otp-28 - paths: - - priv/plts/ - policy: pull-push - script: - - mix format --check-formatted - - mix credo --strict - - mix dialyzer - allow_failure: true # Don't block on dialyzer warnings - -# Build Docker image (existing Nix build or standard build) -build: - stage: build - needs: [test, quality] - tags: - - docker - image: docker:24 - services: - - docker:24-dind - before_script: - - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY - script: - - docker build -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA . - - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG - # Tag as latest for main branch - - | - if [ "$CI_COMMIT_BRANCH" = "main" ]; then - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest - docker push $CI_REGISTRY_IMAGE:latest - fi - rules: - - if: $CI_COMMIT_BRANCH == "main" - - if: $CI_COMMIT_TAG - -# Deploy to production -deploy: - stage: deploy - needs: - - job: build - artifacts: false - tags: - - home - image: - name: bitnami/kubectl:latest - entrypoint: [""] - script: - - kubectl config use-context towerops/towerops:home-cluster-agent - - DEPLOY_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - - echo "Deploying at $DEPLOY_TIMESTAMP" - - kubectl set image deployment/towerops towerops=$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA -n towerops - - kubectl set env deployment/towerops DEPLOY_TIMESTAMP=$DEPLOY_TIMESTAMP -n towerops - environment: - name: production - kubernetes: - namespace: towerops - rules: - - if: $CI_COMMIT_BRANCH == "main" - when: manual