From 31cbd18128d5dbde82b5a31940e973f2f14fc3dc Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Tue, 5 May 2026 11:19:49 -0500 Subject: [PATCH] ci: cache slow runtime apt deps in a prebuilt base image Splits the runtime apt installs (gdal-bin, snmp, libsnmp40, locales, BEAM runtime libs) into k8s/Dockerfile.base, hosted at codeberg.org/gmcintire/towerops-base:latest. The app Dockerfile now does FROM that base instead of re-installing gdal (~500 MB) on every push. The new build-base workflow rebuilds the base image only when k8s/Dockerfile.base or the workflow itself changes, weekly via cron (Sundays 06:00 UTC, with CACHE_BUST= to force apt-get update on a week boundary), or via workflow_dispatch. Production workflow now uses buildx + does docker login before the build so it can pull the private base image. --- .forgejo/workflows/build-base.yaml | 160 +++++++++++++++++++++++++++++ .forgejo/workflows/production.yaml | 95 +++++++++++++---- CLAUDE.md | 29 +++++- k8s/Dockerfile | 37 +++---- k8s/Dockerfile.base | 62 +++++++++++ 5 files changed, 341 insertions(+), 42 deletions(-) create mode 100644 .forgejo/workflows/build-base.yaml create mode 100644 k8s/Dockerfile.base diff --git a/.forgejo/workflows/build-base.yaml b/.forgejo/workflows/build-base.yaml new file mode 100644 index 00000000..5ed1ae8b --- /dev/null +++ b/.forgejo/workflows/build-base.yaml @@ -0,0 +1,160 @@ +name: Build base image + +# Path-filtered so it fires when the base Dockerfile (or this workflow) +# changes, weekly via cron so Debian security updates land without us +# remembering to bump anything, and on-demand via workflow_dispatch. +# +# The weekly cron passes CACHE_BUST= as a build-arg so the +# apt layers in Dockerfile.base re-execute (otherwise BuildKit's layer +# cache would short-circuit `apt-get update` on a deterministic input +# and silently skip security patches). +on: + push: + branches: + - main + paths: + - 'k8s/Dockerfile.base' + - '.forgejo/workflows/build-base.yaml' + schedule: + # Sundays 06:00 UTC — runs on a quiet day-of-week so a long base + # rebuild doesn't collide with mid-week deploys, and matches the + # cadence that Debian security advisories typically aggregate at. + - cron: '0 6 * * 0' + workflow_dispatch: + +# Image hosted on Codeberg's container registry. The shared +# REGISTRY_USER / REGISTRY_PASSWORD secrets carry Codeberg credentials; +# login URL is the hardcoded env.REGISTRY rather than secrets.REGISTRY_URL +# so a stale URL secret can't push to the wrong registry. +env: + REGISTRY: codeberg.org + IMAGE_NAME: gmcintire/towerops-base + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-and-push: + name: Build and push base image + runs-on: ubuntu-22.04 + + steps: + - name: Checkout code + uses: https://github.com/actions/checkout@v4 + + # Static docker + buildx download — same dance as production.yaml. + # The runner image's bookworm apt repos have broken GPG signatures, + # so we bypass apt entirely. + - name: Install Docker CLI and buildx + run: | + curl -fsSL -o /tmp/docker.tgz \ + https://download.docker.com/linux/static/stable/x86_64/docker-27.5.1.tgz + tar xzf /tmp/docker.tgz -C /tmp + install -m 0755 /tmp/docker/docker /usr/local/bin/docker + rm -rf /tmp/docker /tmp/docker.tgz + + mkdir -p /usr/libexec/docker/cli-plugins + curl -fsSL -o /usr/libexec/docker/cli-plugins/docker-buildx \ + https://github.com/docker/buildx/releases/download/v0.19.3/buildx-v0.19.3.linux-amd64 + chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + + docker --version + docker buildx version + + - name: Wait for Docker daemon + run: | + sock="${DOCKER_HOST:-unix:///var/run/docker.sock}" + echo "Using DOCKER_HOST=${sock}" + + for i in $(seq 1 30); do + if docker info >/dev/null 2>&1; then + echo "Docker daemon reachable after ${i}s" + exit 0 + fi + echo "Waiting for Docker daemon... ($i/30)" + sleep 1 + done + + echo "Docker daemon never became reachable — diagnostics:" >&2 + ls -l /var/run/docker.sock 2>&1 || true + env | grep -i docker 2>&1 || true + docker info 2>&1 || true + exit 1 + + - name: Generate image tag + id: tag + run: | + # Embed the Debian release in the tag so prior bases remain + # pullable for hotfix rollbacks if a debian bump regresses + # something at runtime. + DEBIAN=$(grep -E '^ARG DEBIAN_VERSION=' k8s/Dockerfile.base | head -1 | cut -d= -f2) + TIMESTAMP=$(date +%s) + TAG="debian-${DEBIAN}-${TIMESTAMP}" + echo "tag=${TAG}" >> $GITHUB_OUTPUT + + - name: Log in to Codeberg container registry + run: | + # Reuses the shared REGISTRY_USER / REGISTRY_PASSWORD + # Forgejo Actions secrets — already set with Codeberg + # credentials (token must have `write:package` scope). + attempt=1 + max_attempts=3 + while : ; do + if echo "${{ secrets.REGISTRY_PASSWORD }}" | \ + docker login "${{ env.REGISTRY }}" \ + -u "${{ secrets.REGISTRY_USER }}" \ + --password-stdin; then + exit 0 + fi + if [ "$attempt" -ge "$max_attempts" ]; then + echo "docker login failed after $attempt attempts" >&2 + exit 1 + fi + delay=$((2 ** attempt)) + echo "docker login attempt $attempt failed; retrying in ${delay}s" + sleep "$delay" + attempt=$((attempt + 1)) + done + + - name: Build and push base image + run: | + IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}" + TAG="${{ steps.tag.outputs.tag }}" + # ISO year+week (e.g. 2026-W18). Same value all week, so a + # cron-triggered rebuild on day 1 vs day 7 reuses the layer + # cache; the next ISO week boundary forces apt-get update + + # install to re-execute and pull fresh security patches. + CACHE_BUST=$(date -u +%G-W%V) + echo "CACHE_BUST=${CACHE_BUST}" + + attempt=1 + max_attempts=3 + while : ; do + if docker buildx build \ + -f k8s/Dockerfile.base \ + --build-arg CACHE_BUST="${CACHE_BUST}" \ + -t "${IMAGE}:${TAG}" \ + -t "${IMAGE}:latest" \ + --push \ + .; then + exit 0 + fi + if [ "$attempt" -ge "$max_attempts" ]; then + echo "docker buildx build failed after $attempt attempts" >&2 + exit 1 + fi + delay=$((2 ** attempt)) + echo "docker buildx build attempt $attempt failed; retrying in ${delay}s" + sleep "$delay" + attempt=$((attempt + 1)) + done + + - name: Build summary + run: | + echo "### ✅ Base Image Built and Pushed" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Image:** \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}\`" >> $GITHUB_STEP_SUMMARY + echo "**Also tagged:** \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Next push to \`main\` will pick up this base in the app build." >> $GITHUB_STEP_SUMMARY diff --git a/.forgejo/workflows/production.yaml b/.forgejo/workflows/production.yaml index 1875b48a..88f2f8f1 100644 --- a/.forgejo/workflows/production.yaml +++ b/.forgejo/workflows/production.yaml @@ -112,29 +112,88 @@ jobs: echo "tag=${TAG}" >> $GITHUB_OUTPUT echo "Full image tag: ${TAG}" - - name: Install Docker CLI + # Static docker + buildx download. The runner image's bookworm + # apt repos have broken GPG signatures, so we bypass apt entirely. + # buildx is needed because k8s/Dockerfile uses `# syntax=` and + # pulls FROM the prebuilt towerops-base — buildx authenticates + # against Codeberg via the docker login below. + - name: Install Docker CLI and buildx run: | - apt-get update && apt-get install -y docker.io + curl -fsSL -o /tmp/docker.tgz \ + https://download.docker.com/linux/static/stable/x86_64/docker-27.5.1.tgz + tar xzf /tmp/docker.tgz -C /tmp + install -m 0755 /tmp/docker/docker /usr/local/bin/docker + rm -rf /tmp/docker /tmp/docker.tgz + + mkdir -p /usr/libexec/docker/cli-plugins + curl -fsSL -o /usr/libexec/docker/cli-plugins/docker-buildx \ + https://github.com/docker/buildx/releases/download/v0.19.3/buildx-v0.19.3.linux-amd64 + chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + + docker --version + docker buildx version + + - name: Wait for Docker daemon + run: | + for i in $(seq 1 30); do + if docker info >/dev/null 2>&1; then + echo "Docker daemon reachable after ${i}s" + exit 0 + fi + echo "Waiting for Docker daemon... ($i/30)" + sleep 1 + done + echo "Docker daemon never became reachable" >&2 + exit 1 + + # Login BEFORE the build so buildx can pull the private base image. + - name: Log in to Codeberg container registry + run: | + attempt=1 + max_attempts=3 + while : ; do + if echo "${{ secrets.REGISTRY_PASSWORD }}" | \ + docker login "${{ env.REGISTRY }}" \ + -u "${{ secrets.REGISTRY_USER }}" \ + --password-stdin; then + exit 0 + fi + if [ "$attempt" -ge "$max_attempts" ]; then + echo "docker login failed after $attempt attempts" >&2 + exit 1 + fi + delay=$((2 ** attempt)) + echo "docker login attempt $attempt failed; retrying in ${delay}s" + sleep "$delay" + attempt=$((attempt + 1)) + done - name: Build and push Docker image run: | - IMAGE_TAG="${{ steps.tag.outputs.tag }}" + IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}" + TAG="${{ steps.tag.outputs.tag }}" - docker build \ - --build-arg MIX_ENV=prod \ - --file k8s/Dockerfile \ - --tag "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${IMAGE_TAG}" \ - --tag "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:production" \ - . - - # Login and push both tags - echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login \ - --username "${{ secrets.REGISTRY_USER }}" \ - --password-stdin \ - "${{ env.REGISTRY }}" - - docker push "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${IMAGE_TAG}" - docker push "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:production" + attempt=1 + max_attempts=3 + while : ; do + if docker buildx build \ + --build-arg MIX_ENV=prod \ + --file k8s/Dockerfile \ + -t "${IMAGE}:${TAG}" \ + -t "${IMAGE}:production" \ + --push \ + .; then + exit 0 + fi + if [ "$attempt" -ge "$max_attempts" ]; then + echo "docker buildx build failed after $attempt attempts" >&2 + exit 1 + fi + delay=$((2 ** attempt)) + echo "docker buildx build attempt $attempt failed; retrying in ${delay}s" + sleep "$delay" + attempt=$((attempt + 1)) + done - name: Deployment summary run: | diff --git a/CLAUDE.md b/CLAUDE.md index d78cf58b..85c7d64f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -293,8 +293,8 @@ Towerops uses Forgejo CI/CD with two trigger paths (see `.forgejo/workflows/`): - Push to `main` triggers `production.yaml` - **Test gate:** ExUnit suite must pass before any image is built - After tests pass: - - Builds Docker image from `k8s/Dockerfile` - - Pushes to `git.mcintire.me/graham/towerops-web` (tagged `main--` and `production`) + - Builds Docker image from `k8s/Dockerfile` — `FROM codeberg.org/gmcintire/towerops-base:latest` + - Pushes to `codeberg.org/gmcintire/towerops` (tagged `main--` and `production`) - `argocd-image-updater` picks up the new tag (~2 min), writes it to the ArgoCD Application's `spec.source.kustomize.images`, and ArgoCD rolls the Deployment. No commit to `k8s/deployment.yaml`. @@ -302,6 +302,31 @@ Towerops uses Forgejo CI/CD with two trigger paths (see `.forgejo/workflows/`): There is no `production` branch — `main` IS production. Pushing to `main` will ship to prod once tests pass. +### Base image (`k8s/Dockerfile.base`) + +The slow runtime apt installs (`gdal-bin`, `snmp`, `libsnmp40`, `locales`, +BEAM runtime libs) live in a prebuilt base image — +`codeberg.org/gmcintire/towerops-base:latest` — so day-to-day app builds +skip ~500 MB of GIS dep installation. + +- **Source**: `k8s/Dockerfile.base` +- **Built by**: `.forgejo/workflows/build-base.yaml` +- **Triggers**: changes to `k8s/Dockerfile.base` or the workflow itself, + weekly cron (Sundays 06:00 UTC for Debian security updates), + or `workflow_dispatch`. +- **Cache busting**: weekly cron passes `CACHE_BUST=` so + apt layers re-execute on a week boundary; same-week reruns reuse the + layer cache. + +Bumping the Debian release: edit `ARG DEBIAN_VERSION=` in +`k8s/Dockerfile.base` and push. The base workflow rebuilds and re-tags +`:latest`; the next app push picks it up automatically. + +If the registry GCs base layers and the next app build fails with +"could not fetch content descriptor … not found", trigger +`Build base image` via `workflow_dispatch` (or push any change to +`Dockerfile.base`) to re-upload the layers. + ### Deploying Changes **To deploy:** diff --git a/k8s/Dockerfile b/k8s/Dockerfile index 337c10b0..6dcddd84 100644 --- a/k8s/Dockerfile +++ b/k8s/Dockerfile @@ -1,22 +1,26 @@ -# Find eligible builder and runner images on Docker Hub. We use Ubuntu/Debian +# syntax=docker/dockerfile:1.6 +# Find eligible builder images on Docker Hub. We use Ubuntu/Debian # instead of Alpine to avoid DNS resolution issues in production. # # https://hub.docker.com/r/hexpm/elixir/tags?name=ubuntu -# https://hub.docker.com/_/ubuntu/tags # # This file is based on these images: # # - https://hub.docker.com/r/hexpm/elixir/tags - for the build image -# - https://hub.docker.com/_/debian/tags?name=trixie-20251229-slim - for the release image -# - https://pkgs.org/ - resource for finding needed packages -# - Ex: docker.io/hexpm/elixir:1.19.4-erlang-28.3-debian-trixie-20251229-slim +# - codeberg.org/gmcintire/towerops-base - prebuilt runtime base (see Dockerfile.base) +# - Ex: docker.io/hexpm/elixir:1.19.5-erlang-28.4.1-debian-trixie-20260316-slim # ARG ELIXIR_VERSION=1.19.5 ARG OTP_VERSION=28.4.1 ARG DEBIAN_VERSION=trixie-20260316-slim ARG BUILDER_IMAGE="docker.io/hexpm/elixir:${ELIXIR_VERSION}-erlang-${OTP_VERSION}-debian-${DEBIAN_VERSION}" -ARG RUNNER_IMAGE="docker.io/debian:${DEBIAN_VERSION}" +# Prebuilt runtime base (see Dockerfile.base) ships with gdal-bin + +# snmp + libsnmp40 + locale + BEAM runtime apt deps already installed. +# Built by .forgejo/workflows/build-base.yaml only when Dockerfile.base +# changes, so this Dockerfile pulls a ready-made layer instead of +# reinstalling gdal (~500 MB) on every push. +ARG BASE_IMAGE="codeberg.org/gmcintire/towerops-base:latest" FROM ${BUILDER_IMAGE} AS builder ENV MIX_OS_DEPS_COMPILE_PARTITION_COUNT=6 @@ -76,22 +80,11 @@ COPY config/runtime.exs config/ COPY rel rel RUN mix release -# start a new build stage so that the final image will only contain -# the compiled release and other runtime necessities -FROM ${RUNNER_IMAGE} AS final - -RUN apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends libstdc++6 openssl libncurses6 locales ca-certificates iputils-ping snmp libsnmp40 gdal-bin \ - && rm -rf /var/lib/apt/lists/* - -# Set the locale -RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen \ - && locale-gen - -ENV LANG=en_US.UTF-8 -ENV LANGUAGE=en_US:en -ENV LC_ALL=en_US.UTF-8 +# Final runtime stage. Pulls the prebuilt towerops-base image +# (Dockerfile.base) which already carries gdal-bin + snmp + libsnmp40 + +# locale + BEAM runtime apt deps. We just COPY the Elixir release on +# top. +FROM ${BASE_IMAGE} AS final WORKDIR "/app" RUN chown nobody /app diff --git a/k8s/Dockerfile.base b/k8s/Dockerfile.base new file mode 100644 index 00000000..f92ff05b --- /dev/null +++ b/k8s/Dockerfile.base @@ -0,0 +1,62 @@ +# syntax=docker/dockerfile:1.6 +# +# Pre-built runtime base image for the Towerops Phoenix app. Contains: +# * Debian trixie-slim with locale set to en_US.UTF-8 +# * Runtime apt deps for the BEAM + Erlang ssl + snmp +# * `gdal-bin` (~500 MB dep tree, the slowest install) for the LIDAR +# elevation reader +# +# Built by `.forgejo/workflows/build-base.yaml` only when this file or +# the workflow changes. The app `Dockerfile` does +# `FROM codeberg.org/gmcintire/towerops-base:latest` so everyday code +# pushes skip the gdal + snmp apt install entirely. +# +# Bumping the Debian release: change the ARG below and push — the base +# workflow rebuilds and re-tags `:latest`. The app Dockerfile follows +# `:latest`, so the next app push picks it up automatically. + +ARG DEBIAN_VERSION=trixie-20260316-slim +ARG RUNNER_IMAGE="docker.io/debian:${DEBIAN_VERSION}" + +FROM ${RUNNER_IMAGE} AS base + +# Weekly cron passes CACHE_BUST= so the apt layers below +# re-execute and pick up Debian security patches. Source-controlled +# pushes leave the value at its `none` default, so day-to-day base +# rebuilds (Dockerfile.base edits) don't pay the apt-update cost +# unless the week has rolled over. +# +# Note: the registry occasionally GCs base-image blobs that the +# manifest still references. When that happens, push any change to +# this file (or run the workflow via workflow_dispatch) to re-trigger +# the build-base workflow and re-upload the layers. +ARG CACHE_BUST=none + +# Runtime apt deps for the BEAM + ICU/locale + ping + SNMP + GDAL. +# gdal-bin is split into a second RUN so a transient gdal install +# failure doesn't invalidate the smaller base layer above. Referencing +# `${CACHE_BUST}` in the RUN line changes the layer hash week-to-week +# without altering the install set. +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + echo "cache-bust=${CACHE_BUST}" \ + && rm -f /etc/apt/apt.conf.d/docker-clean \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + libstdc++6 openssl libncurses6 locales ca-certificates iputils-ping \ + snmp libsnmp40 + +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + echo "cache-bust=${CACHE_BUST}" \ + && rm -f /etc/apt/apt.conf.d/docker-clean \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gdal-bin + +RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen \ + && locale-gen + +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US:en +ENV LC_ALL=en_US.UTF-8