diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6265e237..1fa25954 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -65,6 +65,80 @@ before_script: # Job to update caches (runs on schedule or manual trigger) update_deps: + stage: test + cache: + - <<: # GitLab CI/CD Pipeline for Towerops Web +# +# Optimization Strategy: +# 1. compile job: Downloads deps once, compiles with --warnings-as-errors +# 2. test/quality jobs: Reuse artifacts from compile (no re-download/recompile) +# 3. Caching: Separate caches for hex/mix, deps, build, and system packages +# 4. Artifacts: Include both deps/ and _build/ to avoid redundant work +# 5. Minimal before_script: Only install what's needed per job +# 6. Parallel execution: test + quality run simultaneously after compile +# 7. Fast feedback: build only waits for test (quality is optional) +# +# Performance: Typical run time is 2-4 minutes with warm cache + +image: hexpm/elixir:1.19.5-erlang-28.0-debian-bookworm-20250113-slim + +variables: + MIX_ENV: test + MIX_HOME: $CI_PROJECT_DIR/.mix + HEX_HOME: $CI_PROJECT_DIR/.hex + +stages: + - test + - build + - deploy + +# Define cache configuration for maximum reuse +.mix_cache: &mix_cache + key: + files: + - mix.lock + prefix: mix-$CI_COMMIT_REF_SLUG + paths: + - deps/ + - _build/test/ + policy: pull + +.hex_cache: &hex_cache + key: + prefix: hex-1.19.5-otp-28 + paths: + - .hex/ + - .mix/ + policy: pull + +.system_cache: &system_cache + key: + prefix: system-deps-v2 + paths: + - apt-cache/ + policy: pull + +# Jobs that need full system dependencies (compilation) +.with_system_deps: + before_script: + - mkdir -p apt-cache + - export APT_CACHE_DIR=$CI_PROJECT_DIR/apt-cache + - apt-get update -qq + - apt-get install -y -qq --no-install-recommends + libsnmp-dev snmp-mibs-downloader build-essential + -o dir::cache::archives=$APT_CACHE_DIR + - mix local.hex --force + - mix local.rebar --force + +# Jobs that only need Elixir tooling (testing, quality) +.elixir_only: + before_script: + - mix local.hex --force + - mix local.rebar --force + +# Update caches (scheduled or on lock file changes) +update_deps: + extends: .with_system_deps stage: test cache: - <<: *hex_cache @@ -73,6 +147,565 @@ update_deps: policy: pull-push - <<: *system_cache policy: pull-push + script: + - mix deps.get + - mix compile + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + changes: + - mix.lock + +# Compile with warnings as errors (creates artifacts for downstream jobs) +compile: + extends: .with_system_deps + stage: test + cache: + - *hex_cache + - *mix_cache + - *system_cache + script: + - mix deps.get + - mix compile --warnings-as-errors + artifacts: + paths: + - deps/ + - _build/test/ + - .mix/ + - .hex/ + expire_in: 1 hour + untracked: false + +# Run tests (uses artifacts, minimal dependencies) +test: + extends: .elixir_only + stage: test + needs: + - job: compile + artifacts: true + services: + - postgres:17-alpine + variables: + POSTGRES_DB: towerops_test + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_HOST: postgres + cache: [] # All dependencies from artifacts + script: + - mix ecto.setup + - mix test --warnings-as-errors --max-cases 8 + coverage: '/\d+\.\d+\%\s+\|\s+Total/' + artifacts: + reports: + coverage_report: + coverage_format: cobertura + path: coverage.xml + when: always + expire_in: 7 days + +# Code quality checks (uses artifacts, runs in parallel with test) +quality: + extends: .elixir_only + stage: test + needs: + - job: compile + artifacts: true + cache: + - key: + prefix: dialyzer-plt-1.19.5-otp-28 + paths: + - priv/plts/ + policy: pull-push + script: + - mix format --check-formatted + - mix credo --strict + - mix dialyzer + allow_failure: true + +# Build Docker image (only waits for test, not quality) +build: + stage: build + needs: + - job: test + artifacts: false + tags: + - docker + image: docker:24 + services: + - docker:24-dind + variables: + DOCKER_BUILDKIT: "1" + BUILDKIT_PROGRESS: plain + before_script: + - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY + script: + - docker build + --build-arg BUILDKIT_INLINE_CACHE=1 + --cache-from $CI_REGISTRY_IMAGE:latest + -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA + -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG . + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + - | + if [ "$CI_COMMIT_BRANCH" = "main" ]; then + docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest + docker push $CI_REGISTRY_IMAGE:latest + fi + rules: + - if: $CI_COMMIT_BRANCH == "main" + - if: $CI_COMMIT_TAG + +# Deploy to production +deploy: + stage: deploy + needs: + - job: build + artifacts: false + tags: + - home + image: + name: bitnami/kubectl:latest + entrypoint: [""] + script: + - kubectl config use-context towerops/towerops:home-cluster-agent + - DEPLOY_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + - kubectl set image deployment/towerops towerops=$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA -n towerops + - kubectl set env deployment/towerops DEPLOY_TIMESTAMP=$DEPLOY_TIMESTAMP -n towerops + environment: + name: production + kubernetes: + namespace: towerops + rules: + - if: $CI_COMMIT_BRANCH == "main" + when: manual +hex_cache + policy: pull-push + - <<: # GitLab CI/CD Pipeline for Towerops Web +# +# Optimization Strategy: +# 1. compile job: Downloads deps once, compiles with --warnings-as-errors +# 2. test/quality jobs: Reuse artifacts from compile (no re-download/recompile) +# 3. Caching: Separate caches for hex/mix, deps, build, and system packages +# 4. Artifacts: Include both deps/ and _build/ to avoid redundant work +# 5. Minimal before_script: Only install what's needed per job +# 6. Parallel execution: test + quality run simultaneously after compile +# 7. Fast feedback: build only waits for test (quality is optional) +# +# Performance: Typical run time is 2-4 minutes with warm cache + +image: hexpm/elixir:1.19.5-erlang-28.0-debian-bookworm-20250113-slim + +variables: + MIX_ENV: test + MIX_HOME: $CI_PROJECT_DIR/.mix + HEX_HOME: $CI_PROJECT_DIR/.hex + +stages: + - test + - build + - deploy + +# Define cache configuration for maximum reuse +.mix_cache: &mix_cache + key: + files: + - mix.lock + prefix: mix-$CI_COMMIT_REF_SLUG + paths: + - deps/ + - _build/test/ + policy: pull + +.hex_cache: &hex_cache + key: + prefix: hex-1.19.5-otp-28 + paths: + - .hex/ + - .mix/ + policy: pull + +.system_cache: &system_cache + key: + prefix: system-deps-v2 + paths: + - apt-cache/ + policy: pull + +# Jobs that need full system dependencies (compilation) +.with_system_deps: + before_script: + - mkdir -p apt-cache + - export APT_CACHE_DIR=$CI_PROJECT_DIR/apt-cache + - apt-get update -qq + - apt-get install -y -qq --no-install-recommends + libsnmp-dev snmp-mibs-downloader build-essential + -o dir::cache::archives=$APT_CACHE_DIR + - mix local.hex --force + - mix local.rebar --force + +# Jobs that only need Elixir tooling (testing, quality) +.elixir_only: + before_script: + - mix local.hex --force + - mix local.rebar --force + +# Update caches (scheduled or on lock file changes) +update_deps: + extends: .with_system_deps + stage: test + cache: + - <<: *hex_cache + policy: pull-push + - <<: *mix_cache + policy: pull-push + - <<: *system_cache + policy: pull-push + script: + - mix deps.get + - mix compile + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + changes: + - mix.lock + +# Compile with warnings as errors (creates artifacts for downstream jobs) +compile: + extends: .with_system_deps + stage: test + cache: + - *hex_cache + - *mix_cache + - *system_cache + script: + - mix deps.get + - mix compile --warnings-as-errors + artifacts: + paths: + - deps/ + - _build/test/ + - .mix/ + - .hex/ + expire_in: 1 hour + untracked: false + +# Run tests (uses artifacts, minimal dependencies) +test: + extends: .elixir_only + stage: test + needs: + - job: compile + artifacts: true + services: + - postgres:17-alpine + variables: + POSTGRES_DB: towerops_test + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_HOST: postgres + cache: [] # All dependencies from artifacts + script: + - mix ecto.setup + - mix test --warnings-as-errors --max-cases 8 + coverage: '/\d+\.\d+\%\s+\|\s+Total/' + artifacts: + reports: + coverage_report: + coverage_format: cobertura + path: coverage.xml + when: always + expire_in: 7 days + +# Code quality checks (uses artifacts, runs in parallel with test) +quality: + extends: .elixir_only + stage: test + needs: + - job: compile + artifacts: true + cache: + - key: + prefix: dialyzer-plt-1.19.5-otp-28 + paths: + - priv/plts/ + policy: pull-push + script: + - mix format --check-formatted + - mix credo --strict + - mix dialyzer + allow_failure: true + +# Build Docker image (only waits for test, not quality) +build: + stage: build + needs: + - job: test + artifacts: false + tags: + - docker + image: docker:24 + services: + - docker:24-dind + variables: + DOCKER_BUILDKIT: "1" + BUILDKIT_PROGRESS: plain + before_script: + - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY + script: + - docker build + --build-arg BUILDKIT_INLINE_CACHE=1 + --cache-from $CI_REGISTRY_IMAGE:latest + -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA + -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG . + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + - | + if [ "$CI_COMMIT_BRANCH" = "main" ]; then + docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest + docker push $CI_REGISTRY_IMAGE:latest + fi + rules: + - if: $CI_COMMIT_BRANCH == "main" + - if: $CI_COMMIT_TAG + +# Deploy to production +deploy: + stage: deploy + needs: + - job: build + artifacts: false + tags: + - home + image: + name: bitnami/kubectl:latest + entrypoint: [""] + script: + - kubectl config use-context towerops/towerops:home-cluster-agent + - DEPLOY_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + - kubectl set image deployment/towerops towerops=$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA -n towerops + - kubectl set env deployment/towerops DEPLOY_TIMESTAMP=$DEPLOY_TIMESTAMP -n towerops + environment: + name: production + kubernetes: + namespace: towerops + rules: + - if: $CI_COMMIT_BRANCH == "main" + when: manual +mix_cache + policy: pull-push + - <<: # GitLab CI/CD Pipeline for Towerops Web +# +# Optimization Strategy: +# 1. compile job: Downloads deps once, compiles with --warnings-as-errors +# 2. test/quality jobs: Reuse artifacts from compile (no re-download/recompile) +# 3. Caching: Separate caches for hex/mix, deps, build, and system packages +# 4. Artifacts: Include both deps/ and _build/ to avoid redundant work +# 5. Minimal before_script: Only install what's needed per job +# 6. Parallel execution: test + quality run simultaneously after compile +# 7. Fast feedback: build only waits for test (quality is optional) +# +# Performance: Typical run time is 2-4 minutes with warm cache + +image: hexpm/elixir:1.19.5-erlang-28.0-debian-bookworm-20250113-slim + +variables: + MIX_ENV: test + MIX_HOME: $CI_PROJECT_DIR/.mix + HEX_HOME: $CI_PROJECT_DIR/.hex + +stages: + - test + - build + - deploy + +# Define cache configuration for maximum reuse +.mix_cache: &mix_cache + key: + files: + - mix.lock + prefix: mix-$CI_COMMIT_REF_SLUG + paths: + - deps/ + - _build/test/ + policy: pull + +.hex_cache: &hex_cache + key: + prefix: hex-1.19.5-otp-28 + paths: + - .hex/ + - .mix/ + policy: pull + +.system_cache: &system_cache + key: + prefix: system-deps-v2 + paths: + - apt-cache/ + policy: pull + +# Jobs that need full system dependencies (compilation) +.with_system_deps: + before_script: + - mkdir -p apt-cache + - export APT_CACHE_DIR=$CI_PROJECT_DIR/apt-cache + - apt-get update -qq + - apt-get install -y -qq --no-install-recommends + libsnmp-dev snmp-mibs-downloader build-essential + -o dir::cache::archives=$APT_CACHE_DIR + - mix local.hex --force + - mix local.rebar --force + +# Jobs that only need Elixir tooling (testing, quality) +.elixir_only: + before_script: + - mix local.hex --force + - mix local.rebar --force + +# Update caches (scheduled or on lock file changes) +update_deps: + extends: .with_system_deps + stage: test + cache: + - <<: *hex_cache + policy: pull-push + - <<: *mix_cache + policy: pull-push + - <<: *system_cache + policy: pull-push + script: + - mix deps.get + - mix compile + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + changes: + - mix.lock + +# Compile with warnings as errors (creates artifacts for downstream jobs) +compile: + extends: .with_system_deps + stage: test + cache: + - *hex_cache + - *mix_cache + - *system_cache + script: + - mix deps.get + - mix compile --warnings-as-errors + artifacts: + paths: + - deps/ + - _build/test/ + - .mix/ + - .hex/ + expire_in: 1 hour + untracked: false + +# Run tests (uses artifacts, minimal dependencies) +test: + extends: .elixir_only + stage: test + needs: + - job: compile + artifacts: true + services: + - postgres:17-alpine + variables: + POSTGRES_DB: towerops_test + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_HOST: postgres + cache: [] # All dependencies from artifacts + script: + - mix ecto.setup + - mix test --warnings-as-errors --max-cases 8 + coverage: '/\d+\.\d+\%\s+\|\s+Total/' + artifacts: + reports: + coverage_report: + coverage_format: cobertura + path: coverage.xml + when: always + expire_in: 7 days + +# Code quality checks (uses artifacts, runs in parallel with test) +quality: + extends: .elixir_only + stage: test + needs: + - job: compile + artifacts: true + cache: + - key: + prefix: dialyzer-plt-1.19.5-otp-28 + paths: + - priv/plts/ + policy: pull-push + script: + - mix format --check-formatted + - mix credo --strict + - mix dialyzer + allow_failure: true + +# Build Docker image (only waits for test, not quality) +build: + stage: build + needs: + - job: test + artifacts: false + tags: + - docker + image: docker:24 + services: + - docker:24-dind + variables: + DOCKER_BUILDKIT: "1" + BUILDKIT_PROGRESS: plain + before_script: + - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY + script: + - docker build + --build-arg BUILDKIT_INLINE_CACHE=1 + --cache-from $CI_REGISTRY_IMAGE:latest + -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA + -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG . + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + - | + if [ "$CI_COMMIT_BRANCH" = "main" ]; then + docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest + docker push $CI_REGISTRY_IMAGE:latest + fi + rules: + - if: $CI_COMMIT_BRANCH == "main" + - if: $CI_COMMIT_TAG + +# Deploy to production +deploy: + stage: deploy + needs: + - job: build + artifacts: false + tags: + - home + image: + name: bitnami/kubectl:latest + entrypoint: [""] + script: + - kubectl config use-context towerops/towerops:home-cluster-agent + - DEPLOY_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + - kubectl set image deployment/towerops towerops=$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA -n towerops + - kubectl set env deployment/towerops DEPLOY_TIMESTAMP=$DEPLOY_TIMESTAMP -n towerops + environment: + name: production + kubernetes: + namespace: towerops + rules: + - if: $CI_COMMIT_BRANCH == "main" + when: manual +system_cache + policy: pull-push script: - mix deps.get - mix deps.compile @@ -85,11 +718,644 @@ update_deps: # Compile with warnings as errors compile: + stage: test + cache: + - # GitLab CI/CD Pipeline for Towerops Web +# +# Optimization Strategy: +# 1. compile job: Downloads deps once, compiles with --warnings-as-errors +# 2. test/quality jobs: Reuse artifacts from compile (no re-download/recompile) +# 3. Caching: Separate caches for hex/mix, deps, build, and system packages +# 4. Artifacts: Include both deps/ and _build/ to avoid redundant work +# 5. Minimal before_script: Only install what's needed per job +# 6. Parallel execution: test + quality run simultaneously after compile +# 7. Fast feedback: build only waits for test (quality is optional) +# +# Performance: Typical run time is 2-4 minutes with warm cache + +image: hexpm/elixir:1.19.5-erlang-28.0-debian-bookworm-20250113-slim + +variables: + MIX_ENV: test + MIX_HOME: $CI_PROJECT_DIR/.mix + HEX_HOME: $CI_PROJECT_DIR/.hex + +stages: + - test + - build + - deploy + +# Define cache configuration for maximum reuse +.mix_cache: &mix_cache + key: + files: + - mix.lock + prefix: mix-$CI_COMMIT_REF_SLUG + paths: + - deps/ + - _build/test/ + policy: pull + +.hex_cache: &hex_cache + key: + prefix: hex-1.19.5-otp-28 + paths: + - .hex/ + - .mix/ + policy: pull + +.system_cache: &system_cache + key: + prefix: system-deps-v2 + paths: + - apt-cache/ + policy: pull + +# Jobs that need full system dependencies (compilation) +.with_system_deps: + before_script: + - mkdir -p apt-cache + - export APT_CACHE_DIR=$CI_PROJECT_DIR/apt-cache + - apt-get update -qq + - apt-get install -y -qq --no-install-recommends + libsnmp-dev snmp-mibs-downloader build-essential + -o dir::cache::archives=$APT_CACHE_DIR + - mix local.hex --force + - mix local.rebar --force + +# Jobs that only need Elixir tooling (testing, quality) +.elixir_only: + before_script: + - mix local.hex --force + - mix local.rebar --force + +# Update caches (scheduled or on lock file changes) +update_deps: + extends: .with_system_deps + stage: test + cache: + - <<: *hex_cache + policy: pull-push + - <<: *mix_cache + policy: pull-push + - <<: *system_cache + policy: pull-push + script: + - mix deps.get + - mix compile + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + changes: + - mix.lock + +# Compile with warnings as errors (creates artifacts for downstream jobs) +compile: + extends: .with_system_deps stage: test cache: - *hex_cache - *mix_cache - *system_cache + script: + - mix deps.get + - mix compile --warnings-as-errors + artifacts: + paths: + - deps/ + - _build/test/ + - .mix/ + - .hex/ + expire_in: 1 hour + untracked: false + +# Run tests (uses artifacts, minimal dependencies) +test: + extends: .elixir_only + stage: test + needs: + - job: compile + artifacts: true + services: + - postgres:17-alpine + variables: + POSTGRES_DB: towerops_test + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_HOST: postgres + cache: [] # All dependencies from artifacts + script: + - mix ecto.setup + - mix test --warnings-as-errors --max-cases 8 + coverage: '/\d+\.\d+\%\s+\|\s+Total/' + artifacts: + reports: + coverage_report: + coverage_format: cobertura + path: coverage.xml + when: always + expire_in: 7 days + +# Code quality checks (uses artifacts, runs in parallel with test) +quality: + extends: .elixir_only + stage: test + needs: + - job: compile + artifacts: true + cache: + - key: + prefix: dialyzer-plt-1.19.5-otp-28 + paths: + - priv/plts/ + policy: pull-push + script: + - mix format --check-formatted + - mix credo --strict + - mix dialyzer + allow_failure: true + +# Build Docker image (only waits for test, not quality) +build: + stage: build + needs: + - job: test + artifacts: false + tags: + - docker + image: docker:24 + services: + - docker:24-dind + variables: + DOCKER_BUILDKIT: "1" + BUILDKIT_PROGRESS: plain + before_script: + - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY + script: + - docker build + --build-arg BUILDKIT_INLINE_CACHE=1 + --cache-from $CI_REGISTRY_IMAGE:latest + -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA + -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG . + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + - | + if [ "$CI_COMMIT_BRANCH" = "main" ]; then + docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest + docker push $CI_REGISTRY_IMAGE:latest + fi + rules: + - if: $CI_COMMIT_BRANCH == "main" + - if: $CI_COMMIT_TAG + +# Deploy to production +deploy: + stage: deploy + needs: + - job: build + artifacts: false + tags: + - home + image: + name: bitnami/kubectl:latest + entrypoint: [""] + script: + - kubectl config use-context towerops/towerops:home-cluster-agent + - DEPLOY_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + - kubectl set image deployment/towerops towerops=$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA -n towerops + - kubectl set env deployment/towerops DEPLOY_TIMESTAMP=$DEPLOY_TIMESTAMP -n towerops + environment: + name: production + kubernetes: + namespace: towerops + rules: + - if: $CI_COMMIT_BRANCH == "main" + when: manual +hex_cache + - # GitLab CI/CD Pipeline for Towerops Web +# +# Optimization Strategy: +# 1. compile job: Downloads deps once, compiles with --warnings-as-errors +# 2. test/quality jobs: Reuse artifacts from compile (no re-download/recompile) +# 3. Caching: Separate caches for hex/mix, deps, build, and system packages +# 4. Artifacts: Include both deps/ and _build/ to avoid redundant work +# 5. Minimal before_script: Only install what's needed per job +# 6. Parallel execution: test + quality run simultaneously after compile +# 7. Fast feedback: build only waits for test (quality is optional) +# +# Performance: Typical run time is 2-4 minutes with warm cache + +image: hexpm/elixir:1.19.5-erlang-28.0-debian-bookworm-20250113-slim + +variables: + MIX_ENV: test + MIX_HOME: $CI_PROJECT_DIR/.mix + HEX_HOME: $CI_PROJECT_DIR/.hex + +stages: + - test + - build + - deploy + +# Define cache configuration for maximum reuse +.mix_cache: &mix_cache + key: + files: + - mix.lock + prefix: mix-$CI_COMMIT_REF_SLUG + paths: + - deps/ + - _build/test/ + policy: pull + +.hex_cache: &hex_cache + key: + prefix: hex-1.19.5-otp-28 + paths: + - .hex/ + - .mix/ + policy: pull + +.system_cache: &system_cache + key: + prefix: system-deps-v2 + paths: + - apt-cache/ + policy: pull + +# Jobs that need full system dependencies (compilation) +.with_system_deps: + before_script: + - mkdir -p apt-cache + - export APT_CACHE_DIR=$CI_PROJECT_DIR/apt-cache + - apt-get update -qq + - apt-get install -y -qq --no-install-recommends + libsnmp-dev snmp-mibs-downloader build-essential + -o dir::cache::archives=$APT_CACHE_DIR + - mix local.hex --force + - mix local.rebar --force + +# Jobs that only need Elixir tooling (testing, quality) +.elixir_only: + before_script: + - mix local.hex --force + - mix local.rebar --force + +# Update caches (scheduled or on lock file changes) +update_deps: + extends: .with_system_deps + stage: test + cache: + - <<: *hex_cache + policy: pull-push + - <<: *mix_cache + policy: pull-push + - <<: *system_cache + policy: pull-push + script: + - mix deps.get + - mix compile + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + changes: + - mix.lock + +# Compile with warnings as errors (creates artifacts for downstream jobs) +compile: + extends: .with_system_deps + stage: test + cache: + - *hex_cache + - *mix_cache + - *system_cache + script: + - mix deps.get + - mix compile --warnings-as-errors + artifacts: + paths: + - deps/ + - _build/test/ + - .mix/ + - .hex/ + expire_in: 1 hour + untracked: false + +# Run tests (uses artifacts, minimal dependencies) +test: + extends: .elixir_only + stage: test + needs: + - job: compile + artifacts: true + services: + - postgres:17-alpine + variables: + POSTGRES_DB: towerops_test + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_HOST: postgres + cache: [] # All dependencies from artifacts + script: + - mix ecto.setup + - mix test --warnings-as-errors --max-cases 8 + coverage: '/\d+\.\d+\%\s+\|\s+Total/' + artifacts: + reports: + coverage_report: + coverage_format: cobertura + path: coverage.xml + when: always + expire_in: 7 days + +# Code quality checks (uses artifacts, runs in parallel with test) +quality: + extends: .elixir_only + stage: test + needs: + - job: compile + artifacts: true + cache: + - key: + prefix: dialyzer-plt-1.19.5-otp-28 + paths: + - priv/plts/ + policy: pull-push + script: + - mix format --check-formatted + - mix credo --strict + - mix dialyzer + allow_failure: true + +# Build Docker image (only waits for test, not quality) +build: + stage: build + needs: + - job: test + artifacts: false + tags: + - docker + image: docker:24 + services: + - docker:24-dind + variables: + DOCKER_BUILDKIT: "1" + BUILDKIT_PROGRESS: plain + before_script: + - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY + script: + - docker build + --build-arg BUILDKIT_INLINE_CACHE=1 + --cache-from $CI_REGISTRY_IMAGE:latest + -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA + -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG . + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + - | + if [ "$CI_COMMIT_BRANCH" = "main" ]; then + docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest + docker push $CI_REGISTRY_IMAGE:latest + fi + rules: + - if: $CI_COMMIT_BRANCH == "main" + - if: $CI_COMMIT_TAG + +# Deploy to production +deploy: + stage: deploy + needs: + - job: build + artifacts: false + tags: + - home + image: + name: bitnami/kubectl:latest + entrypoint: [""] + script: + - kubectl config use-context towerops/towerops:home-cluster-agent + - DEPLOY_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + - kubectl set image deployment/towerops towerops=$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA -n towerops + - kubectl set env deployment/towerops DEPLOY_TIMESTAMP=$DEPLOY_TIMESTAMP -n towerops + environment: + name: production + kubernetes: + namespace: towerops + rules: + - if: $CI_COMMIT_BRANCH == "main" + when: manual +mix_cache + - # GitLab CI/CD Pipeline for Towerops Web +# +# Optimization Strategy: +# 1. compile job: Downloads deps once, compiles with --warnings-as-errors +# 2. test/quality jobs: Reuse artifacts from compile (no re-download/recompile) +# 3. Caching: Separate caches for hex/mix, deps, build, and system packages +# 4. Artifacts: Include both deps/ and _build/ to avoid redundant work +# 5. Minimal before_script: Only install what's needed per job +# 6. Parallel execution: test + quality run simultaneously after compile +# 7. Fast feedback: build only waits for test (quality is optional) +# +# Performance: Typical run time is 2-4 minutes with warm cache + +image: hexpm/elixir:1.19.5-erlang-28.0-debian-bookworm-20250113-slim + +variables: + MIX_ENV: test + MIX_HOME: $CI_PROJECT_DIR/.mix + HEX_HOME: $CI_PROJECT_DIR/.hex + +stages: + - test + - build + - deploy + +# Define cache configuration for maximum reuse +.mix_cache: &mix_cache + key: + files: + - mix.lock + prefix: mix-$CI_COMMIT_REF_SLUG + paths: + - deps/ + - _build/test/ + policy: pull + +.hex_cache: &hex_cache + key: + prefix: hex-1.19.5-otp-28 + paths: + - .hex/ + - .mix/ + policy: pull + +.system_cache: &system_cache + key: + prefix: system-deps-v2 + paths: + - apt-cache/ + policy: pull + +# Jobs that need full system dependencies (compilation) +.with_system_deps: + before_script: + - mkdir -p apt-cache + - export APT_CACHE_DIR=$CI_PROJECT_DIR/apt-cache + - apt-get update -qq + - apt-get install -y -qq --no-install-recommends + libsnmp-dev snmp-mibs-downloader build-essential + -o dir::cache::archives=$APT_CACHE_DIR + - mix local.hex --force + - mix local.rebar --force + +# Jobs that only need Elixir tooling (testing, quality) +.elixir_only: + before_script: + - mix local.hex --force + - mix local.rebar --force + +# Update caches (scheduled or on lock file changes) +update_deps: + extends: .with_system_deps + stage: test + cache: + - <<: *hex_cache + policy: pull-push + - <<: *mix_cache + policy: pull-push + - <<: *system_cache + policy: pull-push + script: + - mix deps.get + - mix compile + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + changes: + - mix.lock + +# Compile with warnings as errors (creates artifacts for downstream jobs) +compile: + extends: .with_system_deps + stage: test + cache: + - *hex_cache + - *mix_cache + - *system_cache + script: + - mix deps.get + - mix compile --warnings-as-errors + artifacts: + paths: + - deps/ + - _build/test/ + - .mix/ + - .hex/ + expire_in: 1 hour + untracked: false + +# Run tests (uses artifacts, minimal dependencies) +test: + extends: .elixir_only + stage: test + needs: + - job: compile + artifacts: true + services: + - postgres:17-alpine + variables: + POSTGRES_DB: towerops_test + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_HOST: postgres + cache: [] # All dependencies from artifacts + script: + - mix ecto.setup + - mix test --warnings-as-errors --max-cases 8 + coverage: '/\d+\.\d+\%\s+\|\s+Total/' + artifacts: + reports: + coverage_report: + coverage_format: cobertura + path: coverage.xml + when: always + expire_in: 7 days + +# Code quality checks (uses artifacts, runs in parallel with test) +quality: + extends: .elixir_only + stage: test + needs: + - job: compile + artifacts: true + cache: + - key: + prefix: dialyzer-plt-1.19.5-otp-28 + paths: + - priv/plts/ + policy: pull-push + script: + - mix format --check-formatted + - mix credo --strict + - mix dialyzer + allow_failure: true + +# Build Docker image (only waits for test, not quality) +build: + stage: build + needs: + - job: test + artifacts: false + tags: + - docker + image: docker:24 + services: + - docker:24-dind + variables: + DOCKER_BUILDKIT: "1" + BUILDKIT_PROGRESS: plain + before_script: + - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY + script: + - docker build + --build-arg BUILDKIT_INLINE_CACHE=1 + --cache-from $CI_REGISTRY_IMAGE:latest + -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA + -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG . + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + - | + if [ "$CI_COMMIT_BRANCH" = "main" ]; then + docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest + docker push $CI_REGISTRY_IMAGE:latest + fi + rules: + - if: $CI_COMMIT_BRANCH == "main" + - if: $CI_COMMIT_TAG + +# Deploy to production +deploy: + stage: deploy + needs: + - job: build + artifacts: false + tags: + - home + image: + name: bitnami/kubectl:latest + entrypoint: [""] + script: + - kubectl config use-context towerops/towerops:home-cluster-agent + - DEPLOY_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + - kubectl set image deployment/towerops towerops=$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA -n towerops + - kubectl set env deployment/towerops DEPLOY_TIMESTAMP=$DEPLOY_TIMESTAMP -n towerops + environment: + name: production + kubernetes: + namespace: towerops + rules: + - if: $CI_COMMIT_BRANCH == "main" + when: manual +system_cache script: - mix deps.get - echo "Compiling with warnings as errors..." @@ -135,7 +1401,218 @@ quality: needs: [compile] cache: # Only need hex/mix for dialyzer PLT - artifacts provide everything else + - # GitLab CI/CD Pipeline for Towerops Web +# +# Optimization Strategy: +# 1. compile job: Downloads deps once, compiles with --warnings-as-errors +# 2. test/quality jobs: Reuse artifacts from compile (no re-download/recompile) +# 3. Caching: Separate caches for hex/mix, deps, build, and system packages +# 4. Artifacts: Include both deps/ and _build/ to avoid redundant work +# 5. Minimal before_script: Only install what's needed per job +# 6. Parallel execution: test + quality run simultaneously after compile +# 7. Fast feedback: build only waits for test (quality is optional) +# +# Performance: Typical run time is 2-4 minutes with warm cache + +image: hexpm/elixir:1.19.5-erlang-28.0-debian-bookworm-20250113-slim + +variables: + MIX_ENV: test + MIX_HOME: $CI_PROJECT_DIR/.mix + HEX_HOME: $CI_PROJECT_DIR/.hex + +stages: + - test + - build + - deploy + +# Define cache configuration for maximum reuse +.mix_cache: &mix_cache + key: + files: + - mix.lock + prefix: mix-$CI_COMMIT_REF_SLUG + paths: + - deps/ + - _build/test/ + policy: pull + +.hex_cache: &hex_cache + key: + prefix: hex-1.19.5-otp-28 + paths: + - .hex/ + - .mix/ + policy: pull + +.system_cache: &system_cache + key: + prefix: system-deps-v2 + paths: + - apt-cache/ + policy: pull + +# Jobs that need full system dependencies (compilation) +.with_system_deps: + before_script: + - mkdir -p apt-cache + - export APT_CACHE_DIR=$CI_PROJECT_DIR/apt-cache + - apt-get update -qq + - apt-get install -y -qq --no-install-recommends + libsnmp-dev snmp-mibs-downloader build-essential + -o dir::cache::archives=$APT_CACHE_DIR + - mix local.hex --force + - mix local.rebar --force + +# Jobs that only need Elixir tooling (testing, quality) +.elixir_only: + before_script: + - mix local.hex --force + - mix local.rebar --force + +# Update caches (scheduled or on lock file changes) +update_deps: + extends: .with_system_deps + stage: test + cache: + - <<: *hex_cache + policy: pull-push + - <<: *mix_cache + policy: pull-push + - <<: *system_cache + policy: pull-push + script: + - mix deps.get + - mix compile + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + changes: + - mix.lock + +# Compile with warnings as errors (creates artifacts for downstream jobs) +compile: + extends: .with_system_deps + stage: test + cache: - *hex_cache + - *mix_cache + - *system_cache + script: + - mix deps.get + - mix compile --warnings-as-errors + artifacts: + paths: + - deps/ + - _build/test/ + - .mix/ + - .hex/ + expire_in: 1 hour + untracked: false + +# Run tests (uses artifacts, minimal dependencies) +test: + extends: .elixir_only + stage: test + needs: + - job: compile + artifacts: true + services: + - postgres:17-alpine + variables: + POSTGRES_DB: towerops_test + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_HOST: postgres + cache: [] # All dependencies from artifacts + script: + - mix ecto.setup + - mix test --warnings-as-errors --max-cases 8 + coverage: '/\d+\.\d+\%\s+\|\s+Total/' + artifacts: + reports: + coverage_report: + coverage_format: cobertura + path: coverage.xml + when: always + expire_in: 7 days + +# Code quality checks (uses artifacts, runs in parallel with test) +quality: + extends: .elixir_only + stage: test + needs: + - job: compile + artifacts: true + cache: + - key: + prefix: dialyzer-plt-1.19.5-otp-28 + paths: + - priv/plts/ + policy: pull-push + script: + - mix format --check-formatted + - mix credo --strict + - mix dialyzer + allow_failure: true + +# Build Docker image (only waits for test, not quality) +build: + stage: build + needs: + - job: test + artifacts: false + tags: + - docker + image: docker:24 + services: + - docker:24-dind + variables: + DOCKER_BUILDKIT: "1" + BUILDKIT_PROGRESS: plain + before_script: + - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY + script: + - docker build + --build-arg BUILDKIT_INLINE_CACHE=1 + --cache-from $CI_REGISTRY_IMAGE:latest + -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA + -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG . + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + - | + if [ "$CI_COMMIT_BRANCH" = "main" ]; then + docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest + docker push $CI_REGISTRY_IMAGE:latest + fi + rules: + - if: $CI_COMMIT_BRANCH == "main" + - if: $CI_COMMIT_TAG + +# Deploy to production +deploy: + stage: deploy + needs: + - job: build + artifacts: false + tags: + - home + image: + name: bitnami/kubectl:latest + entrypoint: [""] + script: + - kubectl config use-context towerops/towerops:home-cluster-agent + - DEPLOY_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + - kubectl set image deployment/towerops towerops=$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA -n towerops + - kubectl set env deployment/towerops DEPLOY_TIMESTAMP=$DEPLOY_TIMESTAMP -n towerops + environment: + name: production + kubernetes: + namespace: towerops + rules: + - if: $CI_COMMIT_BRANCH == "main" + when: manual +hex_cache - key: prefix: dialyzer-plt-1.19.5-otp-28 paths: