diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dc2cd352..6a00b1b8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,9 +3,12 @@ stages: - deploy variables: - DOCKER_BUILDKIT: 1 - CI_BUILDX_ARCHS: "linux/amd64" - DOCKER_TLS_CERTDIR: "" + # Nix configuration + NIX_CONFIG: | + experimental-features = nix-command flakes + substituters = https://cache.nixos.org https://towerops.cachix.org + trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= towerops.cachix.org-1:YOUR_PUBLIC_KEY_HERE + CACHIX_CACHE: "towerops" workflow: auto_cancel: @@ -15,23 +18,42 @@ build: stage: build interruptible: true tags: - - home - image: docker:27 - services: - - docker:27-dind + - nix # Requires GitLab Runner with Nix installed + image: nixos/nix:latest before_script: - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + # Install Docker for loading and pushing images + - nix-env -iA nixpkgs.docker + # Install Cachix if auth token is available + - | + if [ -n "$CACHIX_AUTH_TOKEN" ]; then + nix-env -iA cachix -f https://cachix.org/api/v1/install + echo "$CACHIX_AUTH_TOKEN" | cachix authtoken + fi script: - # Pull latest image for layer caching (ignore failures if not exists) - - docker pull $CI_REGISTRY_IMAGE:latest || true - # Build with cache-from for faster builds and inline cache for future builds - - docker build - --cache-from $CI_REGISTRY_IMAGE:latest - --build-arg BUILDKIT_INLINE_CACHE=1 - -f k8s/Dockerfile - -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA - -t $CI_REGISTRY_IMAGE:latest - . + # Build Docker image with Nix + - echo "Building Docker image with Nix..." + - nix build .#dockerImage --print-build-logs + + # Push build artifacts to Cachix for faster future builds + - | + if [ -n "$CACHIX_AUTH_TOKEN" ]; then + echo "Pushing build artifacts to Cachix..." + nix path-info --json .#dockerImage | jq -r '.[].path' | cachix push $CACHIX_CACHE + fi + + # Load image into Docker + - echo "Loading Docker image..." + - docker load < result + + # Tag image with commit SHA and latest + - docker tag $(docker images -q | head -1) $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA + - docker tag $(docker images -q | head -1) $CI_REGISTRY_IMAGE:latest + + # Login to GitLab Container Registry + - echo "$CI_REGISTRY_PASSWORD" | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY + + # Push to registry + - echo "Pushing to GitLab Container Registry..." - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA - docker push $CI_REGISTRY_IMAGE:latest rules: diff --git a/docs/gitlab-runner-nix-setup.md b/docs/gitlab-runner-nix-setup.md new file mode 100644 index 00000000..b8bcc39d --- /dev/null +++ b/docs/gitlab-runner-nix-setup.md @@ -0,0 +1,314 @@ +# GitLab Runner with Nix Support + +This guide explains how to set up a GitLab Runner with Nix support for building towerops Docker images using Nix flakes. + +## Prerequisites + +- A server with Nix installed (can be NixOS or any Linux with Nix package manager) +- GitLab Runner installed +- Docker installed (for loading and pushing images) +- Network access to GitLab and Docker registries + +## Option 1: NixOS GitLab Runner (Recommended) + +If you're running NixOS, this is the cleanest approach. + +### 1. Configure GitLab Runner in NixOS + +Add to `/etc/nixos/configuration.nix`: + +```nix +{ config, pkgs, ... }: + +{ + # Enable Nix flakes + nix.settings.experimental-features = [ "nix-command" "flakes" ]; + + # Install Docker + virtualisation.docker.enable = true; + + # Configure GitLab Runner + services.gitlab-runner = { + enable = true; + services = { + # Nix builder + nix-builder = { + registrationConfigFile = "/etc/gitlab-runner/nix-builder-registration"; + dockerImage = "nixos/nix:latest"; + dockerPrivileged = true; # Required for docker-in-docker + dockerVolumes = [ + "/nix/store:/nix/store:ro" + "/nix/var/nix/db:/nix/var/nix/db:ro" + "/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket:ro" + "/var/run/docker.sock:/var/run/docker.sock" # For docker load/push + ]; + tagList = [ "nix" ]; + }; + }; + }; + + # Add gitlab-runner user to docker group + users.users.gitlab-runner.extraGroups = [ "docker" ]; +} +``` + +### 2. Create Registration Config + +Create `/etc/gitlab-runner/nix-builder-registration`: + +```toml +[[runners]] + url = "https://gitlab.com/" + token = "YOUR_RUNNER_REGISTRATION_TOKEN" + executor = "docker" +``` + +Get the registration token from: GitLab Project → Settings → CI/CD → Runners → New project runner + +### 3. Apply Configuration + +```bash +sudo nixos-rebuild switch +``` + +## Option 2: Standard Linux with Nix + +If you're using a standard Linux distribution with Nix installed: + +### 1. Install Prerequisites + +```bash +# Install Nix (if not already installed) +curl -L https://nixos.org/nix/install | sh -s -- --daemon + +# Enable flakes +mkdir -p ~/.config/nix +echo "experimental-features = nix-command flakes" >> ~/.config/nix/nix.conf + +# Install Docker +# (distribution-specific - see Docker docs) + +# Install GitLab Runner +curl -L "https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh" | sudo bash +sudo apt-get install gitlab-runner +``` + +### 2. Register Runner + +```bash +sudo gitlab-runner register \ + --url https://gitlab.com/ \ + --token YOUR_RUNNER_REGISTRATION_TOKEN \ + --executor docker \ + --docker-image nixos/nix:latest \ + --docker-privileged \ + --docker-volumes /nix/store:/nix/store:ro \ + --docker-volumes /nix/var/nix/db:/nix/var/nix/db:ro \ + --docker-volumes /var/run/docker.sock:/var/run/docker.sock \ + --tag-list nix \ + --run-untagged=false +``` + +### 3. Add gitlab-runner to Docker Group + +```bash +sudo usermod -aG docker gitlab-runner +sudo systemctl restart gitlab-runner +``` + +## Setting Up Cachix (Optional but Recommended) + +Cachix provides binary caching to speed up builds dramatically. + +### 1. Create Cachix Account and Cache + +```bash +# Install cachix +nix-env -iA cachix -f https://cachix.org/api/v1/install + +# Login to cachix +cachix authtoken + +# Create cache +cachix create towerops + +# Generate keypair +cachix generate-keypair towerops +``` + +### 2. Get Public Key + +```bash +cachix get towerops +``` + +This will output something like: +``` +towerops.cachix.org-1:AbCdEfGhIjKlMnOpQrStUvWxYz1234567890ABCDEFG= +``` + +### 3. Update Configuration + +**In flake.nix**, replace the placeholder: + +```nix +extra-trusted-public-keys = [ + "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" + "towerops.cachix.org-1:YOUR_ACTUAL_PUBLIC_KEY_HERE" # Replace this +]; +``` + +**In .gitlab-ci.yml**, replace: + +```yaml +trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= towerops.cachix.org-1:YOUR_ACTUAL_PUBLIC_KEY_HERE +``` + +### 4. Add Auth Token to GitLab + +Get your auth token: + +```bash +cachix authtoken +``` + +Add to GitLab: +1. Go to: Project → Settings → CI/CD → Variables +2. Add variable: + - **Key**: `CACHIX_AUTH_TOKEN` + - **Value**: (paste token from above) + - **Type**: Variable + - **Protected**: Yes + - **Masked**: Yes + - **Expand variable reference**: No + +## Verifying the Setup + +### Test Runner Connection + +Check that the runner is connected: + +```bash +# On the runner host +sudo gitlab-runner verify + +# In GitLab UI +# Go to: Settings → CI/CD → Runners +# You should see your runner with the "nix" tag listed +``` + +### Test Build + +Push a commit to the `main` branch and watch the pipeline: + +1. Go to: CI/CD → Pipelines +2. Click on the latest pipeline +3. Watch the build job + +Expected output: +``` +Building Docker image with Nix... +[Building all dependencies...] +Creating layer 1 from paths: [...] +Creating layer 2 from paths: [...] +... +Done. +Loading Docker image... +Pushing to GitLab Container Registry... +``` + +## Troubleshooting + +### Runner shows offline in GitLab + +**Check runner status**: +```bash +sudo gitlab-runner status +sudo systemctl status gitlab-runner +``` + +**Check logs**: +```bash +sudo journalctl -u gitlab-runner -f +``` + +### Build fails with "experimental feature 'flakes' is not enabled" + +**Fix**: Ensure NIX_CONFIG in `.gitlab-ci.yml` includes experimental features: +```yaml +NIX_CONFIG: | + experimental-features = nix-command flakes +``` + +### Docker load fails with permission denied + +**Fix**: Ensure gitlab-runner user is in docker group: +```bash +sudo usermod -aG docker gitlab-runner +sudo systemctl restart gitlab-runner +``` + +### Cachix push fails + +**Check auth token**: +- Verify `CACHIX_AUTH_TOKEN` is set in GitLab CI/CD variables +- Verify token is correct: run `cachix authtoken` locally + +**Check network**: +- Ensure runner can reach cachix.org +- Check firewall rules + +### Build is very slow + +**Without Cachix**: First build will compile everything from source (~20-30 minutes) + +**With Cachix**: Subsequent builds should be much faster (~2-5 minutes) as dependencies are cached + +**Improve speed**: +1. Set up Cachix (see above) +2. Use a more powerful runner instance +3. Increase runner's `concurrent` setting in `/etc/gitlab-runner/config.toml` + +## Rollback to Docker Builds + +If you need to roll back to the old Docker-based builds: + +```bash +# The old config is in git history +git log --all -- .gitlab-ci.yml + +# Restore the old version (find the commit hash from log) +git show COMMIT_HASH:.gitlab-ci.yml > .gitlab-ci.yml + +# Commit and push +git add .gitlab-ci.yml +git commit -m "Revert to Docker-based builds" +git push +``` + +The old Dockerfile is still present at `k8s/Dockerfile` and can be used. + +## Performance Comparison + +### Docker-based builds (before Nix): +- First build: ~15 minutes +- Incremental: ~10 minutes (with layer caching) +- Image size: ~500 MB + +### Nix-based builds (with Cachix): +- First build: ~25 minutes (everything from source) +- Subsequent builds: ~2-5 minutes (binary cache hits) +- Image size: ~507 MB compressed (~960 MB uncompressed) + +### Benefits of Nix: +- **Reproducible**: Identical builds across all environments +- **Cacheable**: Binary caching eliminates redundant compilation +- **Declarative**: All dependencies pinned in flake.lock +- **Fast incremental builds**: Only changed layers rebuild + +## Additional Resources + +- [Nix Flakes Manual](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-flake.html) +- [GitLab Runner Docker Executor](https://docs.gitlab.com/runner/executors/docker.html) +- [Cachix Documentation](https://docs.cachix.org/) +- [NixOS GitLab Runner](https://nixos.wiki/wiki/Gitlab_runner)