ci: cache slow runtime apt deps in a prebuilt base image

Splits the runtime apt installs (gdal-bin, snmp, libsnmp40, locales,
BEAM runtime libs) into k8s/Dockerfile.base, hosted at
codeberg.org/gmcintire/towerops-base:latest. The app Dockerfile now
does FROM that base instead of re-installing gdal (~500 MB) on every
push.

The new build-base workflow rebuilds the base image only when
k8s/Dockerfile.base or the workflow itself changes, weekly via cron
(Sundays 06:00 UTC, with CACHE_BUST=<ISO week> to force apt-get update
on a week boundary), or via workflow_dispatch.

Production workflow now uses buildx + does docker login before the
build so it can pull the private base image.

.forgejo/workflows/build-base.yaml

@@ -0,0 +1,160 @@
+name: Build base image
+
+# Path-filtered so it fires when the base Dockerfile (or this workflow)
+# changes, weekly via cron so Debian security updates land without us
+# remembering to bump anything, and on-demand via workflow_dispatch.
+#
+# The weekly cron passes CACHE_BUST=<ISO week> as a build-arg so the
+# apt layers in Dockerfile.base re-execute (otherwise BuildKit's layer
+# cache would short-circuit `apt-get update` on a deterministic input
+# and silently skip security patches).
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'k8s/Dockerfile.base'
+      - '.forgejo/workflows/build-base.yaml'
+  schedule:
+    # Sundays 06:00 UTC — runs on a quiet day-of-week so a long base
+    # rebuild doesn't collide with mid-week deploys, and matches the
+    # cadence that Debian security advisories typically aggregate at.
+    - cron: '0 6 * * 0'
+  workflow_dispatch:
+
+# Image hosted on Codeberg's container registry. The shared
+# REGISTRY_USER / REGISTRY_PASSWORD secrets carry Codeberg credentials;
+# login URL is the hardcoded env.REGISTRY rather than secrets.REGISTRY_URL
+# so a stale URL secret can't push to the wrong registry.
+env:
+  REGISTRY: codeberg.org
+  IMAGE_NAME: gmcintire/towerops-base
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-and-push:
+    name: Build and push base image
+    runs-on: ubuntu-22.04
+
+    steps:
+      - name: Checkout code
+        uses: https://github.com/actions/checkout@v4
+
+      # Static docker + buildx download — same dance as production.yaml.
+      # The runner image's bookworm apt repos have broken GPG signatures,
+      # so we bypass apt entirely.
+      - name: Install Docker CLI and buildx
+        run: |
+          curl -fsSL -o /tmp/docker.tgz \
+            https://download.docker.com/linux/static/stable/x86_64/docker-27.5.1.tgz
+          tar xzf /tmp/docker.tgz -C /tmp
+          install -m 0755 /tmp/docker/docker /usr/local/bin/docker
+          rm -rf /tmp/docker /tmp/docker.tgz
+
+          mkdir -p /usr/libexec/docker/cli-plugins
+          curl -fsSL -o /usr/libexec/docker/cli-plugins/docker-buildx \
+            https://github.com/docker/buildx/releases/download/v0.19.3/buildx-v0.19.3.linux-amd64
+          chmod +x /usr/libexec/docker/cli-plugins/docker-buildx
+
+          docker --version
+          docker buildx version
+
+      - name: Wait for Docker daemon
+        run: |
+          sock="${DOCKER_HOST:-unix:///var/run/docker.sock}"
+          echo "Using DOCKER_HOST=${sock}"
+
+          for i in $(seq 1 30); do
+            if docker info >/dev/null 2>&1; then
+              echo "Docker daemon reachable after ${i}s"
+              exit 0
+            fi
+            echo "Waiting for Docker daemon... ($i/30)"
+            sleep 1
+          done
+
+          echo "Docker daemon never became reachable — diagnostics:" >&2
+          ls -l /var/run/docker.sock 2>&1 || true
+          env | grep -i docker 2>&1 || true
+          docker info 2>&1 || true
+          exit 1
+
+      - name: Generate image tag
+        id: tag
+        run: |
+          # Embed the Debian release in the tag so prior bases remain
+          # pullable for hotfix rollbacks if a debian bump regresses
+          # something at runtime.
+          DEBIAN=$(grep -E '^ARG DEBIAN_VERSION=' k8s/Dockerfile.base | head -1 | cut -d= -f2)
+          TIMESTAMP=$(date +%s)
+          TAG="debian-${DEBIAN}-${TIMESTAMP}"
+          echo "tag=${TAG}" >> $GITHUB_OUTPUT
+
+      - name: Log in to Codeberg container registry
+        run: |
+          # Reuses the shared REGISTRY_USER / REGISTRY_PASSWORD
+          # Forgejo Actions secrets — already set with Codeberg
+          # credentials (token must have `write:package` scope).
+          attempt=1
+          max_attempts=3
+          while : ; do
+            if echo "${{ secrets.REGISTRY_PASSWORD }}" | \
+                docker login "${{ env.REGISTRY }}" \
+                  -u "${{ secrets.REGISTRY_USER }}" \
+                  --password-stdin; then
+              exit 0
+            fi
+            if [ "$attempt" -ge "$max_attempts" ]; then
+              echo "docker login failed after $attempt attempts" >&2
+              exit 1
+            fi
+            delay=$((2 ** attempt))
+            echo "docker login attempt $attempt failed; retrying in ${delay}s"
+            sleep "$delay"
+            attempt=$((attempt + 1))
+          done
+
+      - name: Build and push base image
+        run: |
+          IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
+          TAG="${{ steps.tag.outputs.tag }}"
+          # ISO year+week (e.g. 2026-W18). Same value all week, so a
+          # cron-triggered rebuild on day 1 vs day 7 reuses the layer
+          # cache; the next ISO week boundary forces apt-get update +
+          # install to re-execute and pull fresh security patches.
+          CACHE_BUST=$(date -u +%G-W%V)
+          echo "CACHE_BUST=${CACHE_BUST}"
+
+          attempt=1
+          max_attempts=3
+          while : ; do
+            if docker buildx build \
+                -f k8s/Dockerfile.base \
+                --build-arg CACHE_BUST="${CACHE_BUST}" \
+                -t "${IMAGE}:${TAG}" \
+                -t "${IMAGE}:latest" \
+                --push \
+                .; then
+              exit 0
+            fi
+            if [ "$attempt" -ge "$max_attempts" ]; then
+              echo "docker buildx build failed after $attempt attempts" >&2
+              exit 1
+            fi
+            delay=$((2 ** attempt))
+            echo "docker buildx build attempt $attempt failed; retrying in ${delay}s"
+            sleep "$delay"
+            attempt=$((attempt + 1))
+          done
+
+      - name: Build summary
+        run: |
+          echo "### ✅ Base Image Built and Pushed" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "**Image:** \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "**Also tagged:** \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest\`" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "Next push to \`main\` will pick up this base in the app build." >> $GITHUB_STEP_SUMMARY

.forgejo/workflows/production.yaml

@@ -112,29 +112,88 @@ jobs:
           echo "tag=${TAG}" >> $GITHUB_OUTPUT
           echo "Full image tag: ${TAG}"
 
-      - name: Install Docker CLI
+      # Static docker + buildx download. The runner image's bookworm
+      # apt repos have broken GPG signatures, so we bypass apt entirely.
+      # buildx is needed because k8s/Dockerfile uses `# syntax=` and
+      # pulls FROM the prebuilt towerops-base — buildx authenticates
+      # against Codeberg via the docker login below.
+      - name: Install Docker CLI and buildx
         run: |
-          apt-get update && apt-get install -y docker.io
+          curl -fsSL -o /tmp/docker.tgz \
+            https://download.docker.com/linux/static/stable/x86_64/docker-27.5.1.tgz
+          tar xzf /tmp/docker.tgz -C /tmp
+          install -m 0755 /tmp/docker/docker /usr/local/bin/docker
+          rm -rf /tmp/docker /tmp/docker.tgz
+
+          mkdir -p /usr/libexec/docker/cli-plugins
+          curl -fsSL -o /usr/libexec/docker/cli-plugins/docker-buildx \
+            https://github.com/docker/buildx/releases/download/v0.19.3/buildx-v0.19.3.linux-amd64
+          chmod +x /usr/libexec/docker/cli-plugins/docker-buildx
+
+          docker --version
+          docker buildx version
+
+      - name: Wait for Docker daemon
+        run: |
+          for i in $(seq 1 30); do
+            if docker info >/dev/null 2>&1; then
+              echo "Docker daemon reachable after ${i}s"
+              exit 0
+            fi
+            echo "Waiting for Docker daemon... ($i/30)"
+            sleep 1
+          done
+          echo "Docker daemon never became reachable" >&2
+          exit 1
+
+      # Login BEFORE the build so buildx can pull the private base image.
+      - name: Log in to Codeberg container registry
+        run: |
+          attempt=1
+          max_attempts=3
+          while : ; do
+            if echo "${{ secrets.REGISTRY_PASSWORD }}" | \
+                docker login "${{ env.REGISTRY }}" \
+                  -u "${{ secrets.REGISTRY_USER }}" \
+                  --password-stdin; then
+              exit 0
+            fi
+            if [ "$attempt" -ge "$max_attempts" ]; then
+              echo "docker login failed after $attempt attempts" >&2
+              exit 1
+            fi
+            delay=$((2 ** attempt))
+            echo "docker login attempt $attempt failed; retrying in ${delay}s"
+            sleep "$delay"
+            attempt=$((attempt + 1))
+          done
 
       - name: Build and push Docker image
         run: |
-          IMAGE_TAG="${{ steps.tag.outputs.tag }}"
+          IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
+          TAG="${{ steps.tag.outputs.tag }}"
 
-          docker build \
+          attempt=1
-            --build-arg MIX_ENV=prod \
+          max_attempts=3
-            --file k8s/Dockerfile \
+          while : ; do
-            --tag "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${IMAGE_TAG}" \
+            if docker buildx build \
-            --tag "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:production" \
+                --build-arg MIX_ENV=prod \
-            .
+                --file k8s/Dockerfile \
-
+                -t "${IMAGE}:${TAG}" \
-          # Login and push both tags
+                -t "${IMAGE}:production" \
-          echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login \
+                --push \
-            --username "${{ secrets.REGISTRY_USER }}" \
+                .; then
-            --password-stdin \
+              exit 0
-            "${{ env.REGISTRY }}"
+            fi
-
+            if [ "$attempt" -ge "$max_attempts" ]; then
-          docker push "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${IMAGE_TAG}"
+              echo "docker buildx build failed after $attempt attempts" >&2
-          docker push "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:production"
+              exit 1
+            fi
+            delay=$((2 ** attempt))
+            echo "docker buildx build attempt $attempt failed; retrying in ${delay}s"
+            sleep "$delay"
+            attempt=$((attempt + 1))
+          done
 
       - name: Deployment summary
         run: |

CLAUDE.md

@@ -293,8 +293,8 @@ Towerops uses Forgejo CI/CD with two trigger paths (see `.forgejo/workflows/`):
 - Push to `main` triggers `production.yaml`
 - **Test gate:** ExUnit suite must pass before any image is built
 - After tests pass:
-  - Builds Docker image from `k8s/Dockerfile`
+  - Builds Docker image from `k8s/Dockerfile` — `FROM codeberg.org/gmcintire/towerops-base:latest`
-  - Pushes to `git.mcintire.me/graham/towerops-web` (tagged `main-<ts>-<sha>` and `production`)
+  - Pushes to `codeberg.org/gmcintire/towerops` (tagged `main-<ts>-<sha>` and `production`)
   - `argocd-image-updater` picks up the new tag (~2 min), writes it to the
     ArgoCD Application's `spec.source.kustomize.images`, and ArgoCD rolls the
     Deployment. No commit to `k8s/deployment.yaml`.
@@ -302,6 +302,31 @@ Towerops uses Forgejo CI/CD with two trigger paths (see `.forgejo/workflows/`):
 There is no `production` branch — `main` IS production. Pushing to `main` will
 ship to prod once tests pass.
 
+### Base image (`k8s/Dockerfile.base`)
+
+The slow runtime apt installs (`gdal-bin`, `snmp`, `libsnmp40`, `locales`,
+BEAM runtime libs) live in a prebuilt base image —
+`codeberg.org/gmcintire/towerops-base:latest` — so day-to-day app builds
+skip ~500 MB of GIS dep installation.
+
+- **Source**: `k8s/Dockerfile.base`
+- **Built by**: `.forgejo/workflows/build-base.yaml`
+- **Triggers**: changes to `k8s/Dockerfile.base` or the workflow itself,
+  weekly cron (Sundays 06:00 UTC for Debian security updates),
+  or `workflow_dispatch`.
+- **Cache busting**: weekly cron passes `CACHE_BUST=<ISO year+week>` so
+  apt layers re-execute on a week boundary; same-week reruns reuse the
+  layer cache.
+
+Bumping the Debian release: edit `ARG DEBIAN_VERSION=` in
+`k8s/Dockerfile.base` and push. The base workflow rebuilds and re-tags
+`:latest`; the next app push picks it up automatically.
+
+If the registry GCs base layers and the next app build fails with
+"could not fetch content descriptor … not found", trigger
+`Build base image` via `workflow_dispatch` (or push any change to
+`Dockerfile.base`) to re-upload the layers.
+
 ### Deploying Changes
 
 **To deploy:**

k8s/Dockerfile

@@ -1,22 +1,26 @@
-# Find eligible builder and runner images on Docker Hub. We use Ubuntu/Debian
+# syntax=docker/dockerfile:1.6
+# Find eligible builder images on Docker Hub. We use Ubuntu/Debian
 # instead of Alpine to avoid DNS resolution issues in production.
 #
 # https://hub.docker.com/r/hexpm/elixir/tags?name=ubuntu
-# https://hub.docker.com/_/ubuntu/tags
 #
 # This file is based on these images:
 #
 #   - https://hub.docker.com/r/hexpm/elixir/tags - for the build image
-#   - https://hub.docker.com/_/debian/tags?name=trixie-20251229-slim - for the release image
+#   - codeberg.org/gmcintire/towerops-base - prebuilt runtime base (see Dockerfile.base)
-#   - https://pkgs.org/ - resource for finding needed packages
+#   - Ex: docker.io/hexpm/elixir:1.19.5-erlang-28.4.1-debian-trixie-20260316-slim
-#   - Ex: docker.io/hexpm/elixir:1.19.4-erlang-28.3-debian-trixie-20251229-slim
 #
 ARG ELIXIR_VERSION=1.19.5
 ARG OTP_VERSION=28.4.1
 ARG DEBIAN_VERSION=trixie-20260316-slim
 
 ARG BUILDER_IMAGE="docker.io/hexpm/elixir:${ELIXIR_VERSION}-erlang-${OTP_VERSION}-debian-${DEBIAN_VERSION}"
-ARG RUNNER_IMAGE="docker.io/debian:${DEBIAN_VERSION}"
+# Prebuilt runtime base (see Dockerfile.base) ships with gdal-bin +
+# snmp + libsnmp40 + locale + BEAM runtime apt deps already installed.
+# Built by .forgejo/workflows/build-base.yaml only when Dockerfile.base
+# changes, so this Dockerfile pulls a ready-made layer instead of
+# reinstalling gdal (~500 MB) on every push.
+ARG BASE_IMAGE="codeberg.org/gmcintire/towerops-base:latest"
 
 FROM ${BUILDER_IMAGE} AS builder
 ENV MIX_OS_DEPS_COMPILE_PARTITION_COUNT=6
@@ -76,22 +80,11 @@ COPY config/runtime.exs config/
 COPY rel rel
 RUN mix release
 
-# start a new build stage so that the final image will only contain
+# Final runtime stage. Pulls the prebuilt towerops-base image
-# the compiled release and other runtime necessities
+# (Dockerfile.base) which already carries gdal-bin + snmp + libsnmp40 +
-FROM ${RUNNER_IMAGE} AS final
+# locale + BEAM runtime apt deps. We just COPY the Elixir release on
-
+# top.
-RUN apt-get update \
+FROM ${BASE_IMAGE} AS final
-    && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends libstdc++6 openssl libncurses6 locales ca-certificates iputils-ping snmp libsnmp40 gdal-bin \
-    && rm -rf /var/lib/apt/lists/*
-
-# Set the locale
-RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen \
-    && locale-gen
-
-ENV LANG=en_US.UTF-8
-ENV LANGUAGE=en_US:en
-ENV LC_ALL=en_US.UTF-8
 
 WORKDIR "/app"
 RUN chown nobody /app

k8s/Dockerfile.base

@@ -0,0 +1,62 @@
+# syntax=docker/dockerfile:1.6
+#
+# Pre-built runtime base image for the Towerops Phoenix app. Contains:
+#   * Debian trixie-slim with locale set to en_US.UTF-8
+#   * Runtime apt deps for the BEAM + Erlang ssl + snmp
+#   * `gdal-bin` (~500 MB dep tree, the slowest install) for the LIDAR
+#     elevation reader
+#
+# Built by `.forgejo/workflows/build-base.yaml` only when this file or
+# the workflow changes. The app `Dockerfile` does
+# `FROM codeberg.org/gmcintire/towerops-base:latest` so everyday code
+# pushes skip the gdal + snmp apt install entirely.
+#
+# Bumping the Debian release: change the ARG below and push — the base
+# workflow rebuilds and re-tags `:latest`. The app Dockerfile follows
+# `:latest`, so the next app push picks it up automatically.
+
+ARG DEBIAN_VERSION=trixie-20260316-slim
+ARG RUNNER_IMAGE="docker.io/debian:${DEBIAN_VERSION}"
+
+FROM ${RUNNER_IMAGE} AS base
+
+# Weekly cron passes CACHE_BUST=<iso year+week> so the apt layers below
+# re-execute and pick up Debian security patches. Source-controlled
+# pushes leave the value at its `none` default, so day-to-day base
+# rebuilds (Dockerfile.base edits) don't pay the apt-update cost
+# unless the week has rolled over.
+#
+# Note: the registry occasionally GCs base-image blobs that the
+# manifest still references. When that happens, push any change to
+# this file (or run the workflow via workflow_dispatch) to re-trigger
+# the build-base workflow and re-upload the layers.
+ARG CACHE_BUST=none
+
+# Runtime apt deps for the BEAM + ICU/locale + ping + SNMP + GDAL.
+# gdal-bin is split into a second RUN so a transient gdal install
+# failure doesn't invalidate the smaller base layer above. Referencing
+# `${CACHE_BUST}` in the RUN line changes the layer hash week-to-week
+# without altering the install set.
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    echo "cache-bust=${CACHE_BUST}" \
+    && rm -f /etc/apt/apt.conf.d/docker-clean \
+    && apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+       libstdc++6 openssl libncurses6 locales ca-certificates iputils-ping \
+       snmp libsnmp40
+
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    echo "cache-bust=${CACHE_BUST}" \
+    && rm -f /etc/apt/apt.conf.d/docker-clean \
+    && apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gdal-bin
+
+RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen \
+    && locale-gen
+
+ENV LANG=en_US.UTF-8
+ENV LANGUAGE=en_US:en
+ENV LC_ALL=en_US.UTF-8