ci: cache slow runtime apt deps in a prebuilt base image

Splits the runtime apt installs (gdal-bin, snmp, libsnmp40, locales,
BEAM runtime libs) into k8s/Dockerfile.base, hosted at
codeberg.org/gmcintire/towerops-base:latest. The app Dockerfile now
does FROM that base instead of re-installing gdal (~500 MB) on every
push.

The new build-base workflow rebuilds the base image only when
k8s/Dockerfile.base or the workflow itself changes, weekly via cron
(Sundays 06:00 UTC, with CACHE_BUST=<ISO week> to force apt-get update
on a week boundary), or via workflow_dispatch.

Production workflow now uses buildx + does docker login before the
build so it can pull the private base image.
This commit is contained in:
Graham McIntire 2026-05-05 11:19:49 -05:00
parent f99c041e84
commit 31cbd18128
5 changed files with 341 additions and 42 deletions

View file

@ -0,0 +1,160 @@
name: Build base image
# Path-filtered so it fires when the base Dockerfile (or this workflow)
# changes, weekly via cron so Debian security updates land without us
# remembering to bump anything, and on-demand via workflow_dispatch.
#
# The weekly cron passes CACHE_BUST=<ISO week> as a build-arg so the
# apt layers in Dockerfile.base re-execute (otherwise BuildKit's layer
# cache would short-circuit `apt-get update` on a deterministic input
# and silently skip security patches).
on:
push:
branches:
- main
paths:
- 'k8s/Dockerfile.base'
- '.forgejo/workflows/build-base.yaml'
schedule:
# Sundays 06:00 UTC — runs on a quiet day-of-week so a long base
# rebuild doesn't collide with mid-week deploys, and matches the
# cadence that Debian security advisories typically aggregate at.
- cron: '0 6 * * 0'
workflow_dispatch:
# Image hosted on Codeberg's container registry. The shared
# REGISTRY_USER / REGISTRY_PASSWORD secrets carry Codeberg credentials;
# login URL is the hardcoded env.REGISTRY rather than secrets.REGISTRY_URL
# so a stale URL secret can't push to the wrong registry.
env:
REGISTRY: codeberg.org
IMAGE_NAME: gmcintire/towerops-base
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
build-and-push:
name: Build and push base image
runs-on: ubuntu-22.04
steps:
- name: Checkout code
uses: https://github.com/actions/checkout@v4
# Static docker + buildx download — same dance as production.yaml.
# The runner image's bookworm apt repos have broken GPG signatures,
# so we bypass apt entirely.
- name: Install Docker CLI and buildx
run: |
curl -fsSL -o /tmp/docker.tgz \
https://download.docker.com/linux/static/stable/x86_64/docker-27.5.1.tgz
tar xzf /tmp/docker.tgz -C /tmp
install -m 0755 /tmp/docker/docker /usr/local/bin/docker
rm -rf /tmp/docker /tmp/docker.tgz
mkdir -p /usr/libexec/docker/cli-plugins
curl -fsSL -o /usr/libexec/docker/cli-plugins/docker-buildx \
https://github.com/docker/buildx/releases/download/v0.19.3/buildx-v0.19.3.linux-amd64
chmod +x /usr/libexec/docker/cli-plugins/docker-buildx
docker --version
docker buildx version
- name: Wait for Docker daemon
run: |
sock="${DOCKER_HOST:-unix:///var/run/docker.sock}"
echo "Using DOCKER_HOST=${sock}"
for i in $(seq 1 30); do
if docker info >/dev/null 2>&1; then
echo "Docker daemon reachable after ${i}s"
exit 0
fi
echo "Waiting for Docker daemon... ($i/30)"
sleep 1
done
echo "Docker daemon never became reachable — diagnostics:" >&2
ls -l /var/run/docker.sock 2>&1 || true
env | grep -i docker 2>&1 || true
docker info 2>&1 || true
exit 1
- name: Generate image tag
id: tag
run: |
# Embed the Debian release in the tag so prior bases remain
# pullable for hotfix rollbacks if a debian bump regresses
# something at runtime.
DEBIAN=$(grep -E '^ARG DEBIAN_VERSION=' k8s/Dockerfile.base | head -1 | cut -d= -f2)
TIMESTAMP=$(date +%s)
TAG="debian-${DEBIAN}-${TIMESTAMP}"
echo "tag=${TAG}" >> $GITHUB_OUTPUT
- name: Log in to Codeberg container registry
run: |
# Reuses the shared REGISTRY_USER / REGISTRY_PASSWORD
# Forgejo Actions secrets — already set with Codeberg
# credentials (token must have `write:package` scope).
attempt=1
max_attempts=3
while : ; do
if echo "${{ secrets.REGISTRY_PASSWORD }}" | \
docker login "${{ env.REGISTRY }}" \
-u "${{ secrets.REGISTRY_USER }}" \
--password-stdin; then
exit 0
fi
if [ "$attempt" -ge "$max_attempts" ]; then
echo "docker login failed after $attempt attempts" >&2
exit 1
fi
delay=$((2 ** attempt))
echo "docker login attempt $attempt failed; retrying in ${delay}s"
sleep "$delay"
attempt=$((attempt + 1))
done
- name: Build and push base image
run: |
IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
TAG="${{ steps.tag.outputs.tag }}"
# ISO year+week (e.g. 2026-W18). Same value all week, so a
# cron-triggered rebuild on day 1 vs day 7 reuses the layer
# cache; the next ISO week boundary forces apt-get update +
# install to re-execute and pull fresh security patches.
CACHE_BUST=$(date -u +%G-W%V)
echo "CACHE_BUST=${CACHE_BUST}"
attempt=1
max_attempts=3
while : ; do
if docker buildx build \
-f k8s/Dockerfile.base \
--build-arg CACHE_BUST="${CACHE_BUST}" \
-t "${IMAGE}:${TAG}" \
-t "${IMAGE}:latest" \
--push \
.; then
exit 0
fi
if [ "$attempt" -ge "$max_attempts" ]; then
echo "docker buildx build failed after $attempt attempts" >&2
exit 1
fi
delay=$((2 ** attempt))
echo "docker buildx build attempt $attempt failed; retrying in ${delay}s"
sleep "$delay"
attempt=$((attempt + 1))
done
- name: Build summary
run: |
echo "### ✅ Base Image Built and Pushed" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "**Image:** \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}\`" >> $GITHUB_STEP_SUMMARY
echo "**Also tagged:** \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest\`" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "Next push to \`main\` will pick up this base in the app build." >> $GITHUB_STEP_SUMMARY

View file

@ -112,29 +112,88 @@ jobs:
echo "tag=${TAG}" >> $GITHUB_OUTPUT
echo "Full image tag: ${TAG}"
- name: Install Docker CLI
# Static docker + buildx download. The runner image's bookworm
# apt repos have broken GPG signatures, so we bypass apt entirely.
# buildx is needed because k8s/Dockerfile uses `# syntax=` and
# pulls FROM the prebuilt towerops-base — buildx authenticates
# against Codeberg via the docker login below.
- name: Install Docker CLI and buildx
run: |
apt-get update && apt-get install -y docker.io
curl -fsSL -o /tmp/docker.tgz \
https://download.docker.com/linux/static/stable/x86_64/docker-27.5.1.tgz
tar xzf /tmp/docker.tgz -C /tmp
install -m 0755 /tmp/docker/docker /usr/local/bin/docker
rm -rf /tmp/docker /tmp/docker.tgz
mkdir -p /usr/libexec/docker/cli-plugins
curl -fsSL -o /usr/libexec/docker/cli-plugins/docker-buildx \
https://github.com/docker/buildx/releases/download/v0.19.3/buildx-v0.19.3.linux-amd64
chmod +x /usr/libexec/docker/cli-plugins/docker-buildx
docker --version
docker buildx version
- name: Wait for Docker daemon
run: |
for i in $(seq 1 30); do
if docker info >/dev/null 2>&1; then
echo "Docker daemon reachable after ${i}s"
exit 0
fi
echo "Waiting for Docker daemon... ($i/30)"
sleep 1
done
echo "Docker daemon never became reachable" >&2
exit 1
# Login BEFORE the build so buildx can pull the private base image.
- name: Log in to Codeberg container registry
run: |
attempt=1
max_attempts=3
while : ; do
if echo "${{ secrets.REGISTRY_PASSWORD }}" | \
docker login "${{ env.REGISTRY }}" \
-u "${{ secrets.REGISTRY_USER }}" \
--password-stdin; then
exit 0
fi
if [ "$attempt" -ge "$max_attempts" ]; then
echo "docker login failed after $attempt attempts" >&2
exit 1
fi
delay=$((2 ** attempt))
echo "docker login attempt $attempt failed; retrying in ${delay}s"
sleep "$delay"
attempt=$((attempt + 1))
done
- name: Build and push Docker image
run: |
IMAGE_TAG="${{ steps.tag.outputs.tag }}"
IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
TAG="${{ steps.tag.outputs.tag }}"
docker build \
--build-arg MIX_ENV=prod \
--file k8s/Dockerfile \
--tag "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${IMAGE_TAG}" \
--tag "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:production" \
.
# Login and push both tags
echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login \
--username "${{ secrets.REGISTRY_USER }}" \
--password-stdin \
"${{ env.REGISTRY }}"
docker push "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${IMAGE_TAG}"
docker push "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:production"
attempt=1
max_attempts=3
while : ; do
if docker buildx build \
--build-arg MIX_ENV=prod \
--file k8s/Dockerfile \
-t "${IMAGE}:${TAG}" \
-t "${IMAGE}:production" \
--push \
.; then
exit 0
fi
if [ "$attempt" -ge "$max_attempts" ]; then
echo "docker buildx build failed after $attempt attempts" >&2
exit 1
fi
delay=$((2 ** attempt))
echo "docker buildx build attempt $attempt failed; retrying in ${delay}s"
sleep "$delay"
attempt=$((attempt + 1))
done
- name: Deployment summary
run: |

View file

@ -293,8 +293,8 @@ Towerops uses Forgejo CI/CD with two trigger paths (see `.forgejo/workflows/`):
- Push to `main` triggers `production.yaml`
- **Test gate:** ExUnit suite must pass before any image is built
- After tests pass:
- Builds Docker image from `k8s/Dockerfile`
- Pushes to `git.mcintire.me/graham/towerops-web` (tagged `main-<ts>-<sha>` and `production`)
- Builds Docker image from `k8s/Dockerfile``FROM codeberg.org/gmcintire/towerops-base:latest`
- Pushes to `codeberg.org/gmcintire/towerops` (tagged `main-<ts>-<sha>` and `production`)
- `argocd-image-updater` picks up the new tag (~2 min), writes it to the
ArgoCD Application's `spec.source.kustomize.images`, and ArgoCD rolls the
Deployment. No commit to `k8s/deployment.yaml`.
@ -302,6 +302,31 @@ Towerops uses Forgejo CI/CD with two trigger paths (see `.forgejo/workflows/`):
There is no `production` branch — `main` IS production. Pushing to `main` will
ship to prod once tests pass.
### Base image (`k8s/Dockerfile.base`)
The slow runtime apt installs (`gdal-bin`, `snmp`, `libsnmp40`, `locales`,
BEAM runtime libs) live in a prebuilt base image —
`codeberg.org/gmcintire/towerops-base:latest` — so day-to-day app builds
skip ~500 MB of GIS dep installation.
- **Source**: `k8s/Dockerfile.base`
- **Built by**: `.forgejo/workflows/build-base.yaml`
- **Triggers**: changes to `k8s/Dockerfile.base` or the workflow itself,
weekly cron (Sundays 06:00 UTC for Debian security updates),
or `workflow_dispatch`.
- **Cache busting**: weekly cron passes `CACHE_BUST=<ISO year+week>` so
apt layers re-execute on a week boundary; same-week reruns reuse the
layer cache.
Bumping the Debian release: edit `ARG DEBIAN_VERSION=` in
`k8s/Dockerfile.base` and push. The base workflow rebuilds and re-tags
`:latest`; the next app push picks it up automatically.
If the registry GCs base layers and the next app build fails with
"could not fetch content descriptor … not found", trigger
`Build base image` via `workflow_dispatch` (or push any change to
`Dockerfile.base`) to re-upload the layers.
### Deploying Changes
**To deploy:**

View file

@ -1,22 +1,26 @@
# Find eligible builder and runner images on Docker Hub. We use Ubuntu/Debian
# syntax=docker/dockerfile:1.6
# Find eligible builder images on Docker Hub. We use Ubuntu/Debian
# instead of Alpine to avoid DNS resolution issues in production.
#
# https://hub.docker.com/r/hexpm/elixir/tags?name=ubuntu
# https://hub.docker.com/_/ubuntu/tags
#
# This file is based on these images:
#
# - https://hub.docker.com/r/hexpm/elixir/tags - for the build image
# - https://hub.docker.com/_/debian/tags?name=trixie-20251229-slim - for the release image
# - https://pkgs.org/ - resource for finding needed packages
# - Ex: docker.io/hexpm/elixir:1.19.4-erlang-28.3-debian-trixie-20251229-slim
# - codeberg.org/gmcintire/towerops-base - prebuilt runtime base (see Dockerfile.base)
# - Ex: docker.io/hexpm/elixir:1.19.5-erlang-28.4.1-debian-trixie-20260316-slim
#
ARG ELIXIR_VERSION=1.19.5
ARG OTP_VERSION=28.4.1
ARG DEBIAN_VERSION=trixie-20260316-slim
ARG BUILDER_IMAGE="docker.io/hexpm/elixir:${ELIXIR_VERSION}-erlang-${OTP_VERSION}-debian-${DEBIAN_VERSION}"
ARG RUNNER_IMAGE="docker.io/debian:${DEBIAN_VERSION}"
# Prebuilt runtime base (see Dockerfile.base) ships with gdal-bin +
# snmp + libsnmp40 + locale + BEAM runtime apt deps already installed.
# Built by .forgejo/workflows/build-base.yaml only when Dockerfile.base
# changes, so this Dockerfile pulls a ready-made layer instead of
# reinstalling gdal (~500 MB) on every push.
ARG BASE_IMAGE="codeberg.org/gmcintire/towerops-base:latest"
FROM ${BUILDER_IMAGE} AS builder
ENV MIX_OS_DEPS_COMPILE_PARTITION_COUNT=6
@ -76,22 +80,11 @@ COPY config/runtime.exs config/
COPY rel rel
RUN mix release
# start a new build stage so that the final image will only contain
# the compiled release and other runtime necessities
FROM ${RUNNER_IMAGE} AS final
RUN apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends libstdc++6 openssl libncurses6 locales ca-certificates iputils-ping snmp libsnmp40 gdal-bin \
&& rm -rf /var/lib/apt/lists/*
# Set the locale
RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen \
&& locale-gen
ENV LANG=en_US.UTF-8
ENV LANGUAGE=en_US:en
ENV LC_ALL=en_US.UTF-8
# Final runtime stage. Pulls the prebuilt towerops-base image
# (Dockerfile.base) which already carries gdal-bin + snmp + libsnmp40 +
# locale + BEAM runtime apt deps. We just COPY the Elixir release on
# top.
FROM ${BASE_IMAGE} AS final
WORKDIR "/app"
RUN chown nobody /app

62
k8s/Dockerfile.base Normal file
View file

@ -0,0 +1,62 @@
# syntax=docker/dockerfile:1.6
#
# Pre-built runtime base image for the Towerops Phoenix app. Contains:
# * Debian trixie-slim with locale set to en_US.UTF-8
# * Runtime apt deps for the BEAM + Erlang ssl + snmp
# * `gdal-bin` (~500 MB dep tree, the slowest install) for the LIDAR
# elevation reader
#
# Built by `.forgejo/workflows/build-base.yaml` only when this file or
# the workflow changes. The app `Dockerfile` does
# `FROM codeberg.org/gmcintire/towerops-base:latest` so everyday code
# pushes skip the gdal + snmp apt install entirely.
#
# Bumping the Debian release: change the ARG below and push — the base
# workflow rebuilds and re-tags `:latest`. The app Dockerfile follows
# `:latest`, so the next app push picks it up automatically.
ARG DEBIAN_VERSION=trixie-20260316-slim
ARG RUNNER_IMAGE="docker.io/debian:${DEBIAN_VERSION}"
FROM ${RUNNER_IMAGE} AS base
# Weekly cron passes CACHE_BUST=<iso year+week> so the apt layers below
# re-execute and pick up Debian security patches. Source-controlled
# pushes leave the value at its `none` default, so day-to-day base
# rebuilds (Dockerfile.base edits) don't pay the apt-update cost
# unless the week has rolled over.
#
# Note: the registry occasionally GCs base-image blobs that the
# manifest still references. When that happens, push any change to
# this file (or run the workflow via workflow_dispatch) to re-trigger
# the build-base workflow and re-upload the layers.
ARG CACHE_BUST=none
# Runtime apt deps for the BEAM + ICU/locale + ping + SNMP + GDAL.
# gdal-bin is split into a second RUN so a transient gdal install
# failure doesn't invalidate the smaller base layer above. Referencing
# `${CACHE_BUST}` in the RUN line changes the layer hash week-to-week
# without altering the install set.
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/lib/apt,sharing=locked \
echo "cache-bust=${CACHE_BUST}" \
&& rm -f /etc/apt/apt.conf.d/docker-clean \
&& apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
libstdc++6 openssl libncurses6 locales ca-certificates iputils-ping \
snmp libsnmp40
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/lib/apt,sharing=locked \
echo "cache-bust=${CACHE_BUST}" \
&& rm -f /etc/apt/apt.conf.d/docker-clean \
&& apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gdal-bin
RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen \
&& locale-gen
ENV LANG=en_US.UTF-8
ENV LANGUAGE=en_US:en
ENV LC_ALL=en_US.UTF-8