feat(infra): add prop-backfill deployment for T620 nodes

PROP_ROLE=backfill drops the hot-path propagation/commercial/solar
queues and the Cron plugin. Paired with a dedicated Deployment
(nodeSelector: prop-backfill=true, 4Gi memory, no web) this lets 8GB
T620 workers drain the hrrr/weather/terrain/iemre/narr backfill
queues without competing with the hourly grid chain.

config/runtime.exs

@@ -108,61 +108,19 @@ if config_env() == :prod do
   # handshake fails with an "unexpected_message" TLS alert.
   email_server = System.get_env("EMAIL_SERVER")
 
-  config :libcluster,
-    topologies: [
-      k8s: [
-        strategy: Cluster.Strategy.Kubernetes,
-        config: [
-          mode: :ip,
-          kubernetes_ip_lookup_mode: :pods,
-          kubernetes_node_basename: "microwaveprop",
-          kubernetes_selector: "app=prop",
-          kubernetes_namespace: "prop"
-        ]
-      ]
-    ]
-
-  config :microwaveprop, Microwaveprop.Mailer,
-    adapter: Swoosh.Adapters.SMTP,
-    relay: email_server,
-    port: 587,
-    username: System.get_env("EMAIL_USERNAME"),
-    password: System.get_env("EMAIL_PASSWORD"),
-    tls: :always,
-    tls_options: [
-      server_name_indication: String.to_charlist(email_server || ""),
-      versions: [:"tlsv1.2", :"tlsv1.3"],
-      verify: :verify_none
-    ],
-    auth: :always
-
-  config :microwaveprop, Microwaveprop.Repo,
-    # ssl: true,
-    url: database_url,
-    pool_size: String.to_integer(System.get_env("POOL_SIZE") || "20"),
-    # For machines with several cores, consider starting multiple pools of `pool_size`
-    # pool_count: 4,
-    socket_options: maybe_ipv6
-
-  config :microwaveprop, MicrowavepropWeb.Endpoint,
-    # SSL terminated by Cloudflare tunnel; generated URLs still use https
-    url: [host: host, port: 443, scheme: "https"],
-    http: [
-      # Enable IPv6 and bind on all interfaces.
-      # Set it to  {0, 0, 0, 0, 0, 0, 0, 1} for local network only access.
-      # See the documentation on https://hexdocs.pm/bandit/Bandit.html#t:options/0
-      # for details about using IPv6 vs IPv4 and loopback vs public addresses.
-      ip: {0, 0, 0, 0, 0, 0, 0, 0}
-    ],
-    secret_key_base: secret_key_base
-
   # Production Oban: live scoring, polling, and on-demand QSO enrichment.
   # Runs on the Oban Pro Smart engine so we can use global_limit / rate_limit
   # to protect rate-limited upstreams (Copernicus CDS in particular).
-  config :microwaveprop, Oban,
+  #
-    engine: Oban.Pro.Engines.Smart,
+  # PROP_ROLE controls which workloads this pod picks up:
-    # Per-pod concurrency (×3 pods = effective cluster total)
+  #   * "hot" (default) — full hot-path: web, crons, all queues.
-    queues: [
+  #   * "backfill" — no web server (unset PHX_SERVER), no crons, and only
+  #     backfill-style queues. Intended for smaller nodes dedicated to
+  #     catching up historical enrichment without competing with the
+  #     hourly propagation grid chain.
+  prop_role = System.get_env("PROP_ROLE", "hot")
+
+  hot_only_queues = [
     # 1 slot per pod. Two concurrent forecast-hour steps in a single
     # pod stack HRRR grid + native duct grid + scored band map into
     # ~5-6 GiB RSS, hitting the 6 GiB limit and OOM-killing the pod
@@ -174,7 +132,10 @@ if config_env() == :prod do
     # the next hourly fire.
     propagation: 1,
     commercial: 1,
-      solar: 1,
+    solar: 1
+  ]
+
+  shared_queues = [
     # 1 slot per pod (3 cluster-wide). Any higher than this and the
     # ASOS backfill hammers IEM hard enough to get HTTP 429s, which
     # fill the retryable queue and thrash pod CPU without making
@@ -201,18 +162,15 @@ if config_env() == :prod do
     mechanism: 4,
     exports: 1,
     contact_import: 4
-    ],
+  ]
-    plugins: [
+
-      {Oban.Plugins.Pruner, max_age: 3600 * 24},
+  oban_queues =
-      # DynamicLifeline uses producer records (heartbeats from each
+    case prop_role do
-      # live node) to rescue orphans, not a timer. When a pod dies
+      "backfill" -> shared_queues
-      # mid-deploy its Producer row disappears and any in-flight job
+      _ -> hot_only_queues ++ shared_queues
-      # with that producer in `attempted_by` gets moved back to
+    end
-      # `available` on the next rescue cycle — within ~30s instead of
+
-      # waiting for a timer-based `rescue_after` to expire. Critical
+  cron_plugin =
-      # for PropagationGridWorker chain steps to recover from rolling
-      # deploys before the next hourly cron fire.
-      {Oban.Pro.Plugins.DynamicLifeline, rescue_interval: to_timeout(second: 30)},
     {Oban.Plugins.Cron,
      crontab: [
        {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker},
@@ -262,8 +220,83 @@ if config_env() == :prod do
        # 5-min poll balances freshness vs request volume.
        {"*/5 * * * *", Microwaveprop.Workers.SpaceWeatherFetchWorker}
      ]}
+
+  base_plugins = [
+    {Oban.Plugins.Pruner, max_age: 3600 * 24},
+    # DynamicLifeline uses producer records (heartbeats from each
+    # live node) to rescue orphans, not a timer. When a pod dies
+    # mid-deploy its Producer row disappears and any in-flight job
+    # with that producer in `attempted_by` gets moved back to
+    # `available` on the next rescue cycle — within ~30s instead of
+    # waiting for a timer-based `rescue_after` to expire. Critical
+    # for PropagationGridWorker chain steps to recover from rolling
+    # deploys before the next hourly cron fire.
+    {Oban.Pro.Plugins.DynamicLifeline, rescue_interval: to_timeout(second: 30)}
   ]
 
+  # Backfill pods skip the cron plugin entirely — no risk of a
+  # backfill node becoming cron leader and enqueueing work targeted
+  # at queues it doesn't even run (propagation, commercial). Hot pods
+  # own scheduling; backfill pods just drain the queues.
+  oban_plugins =
+    case prop_role do
+      "backfill" -> base_plugins
+      _ -> base_plugins ++ [cron_plugin]
+    end
+
+  config :libcluster,
+    topologies: [
+      k8s: [
+        strategy: Cluster.Strategy.Kubernetes,
+        config: [
+          mode: :ip,
+          kubernetes_ip_lookup_mode: :pods,
+          kubernetes_node_basename: "microwaveprop",
+          kubernetes_selector: "app=prop",
+          kubernetes_namespace: "prop"
+        ]
+      ]
+    ]
+
+  config :microwaveprop, Microwaveprop.Mailer,
+    adapter: Swoosh.Adapters.SMTP,
+    relay: email_server,
+    port: 587,
+    username: System.get_env("EMAIL_USERNAME"),
+    password: System.get_env("EMAIL_PASSWORD"),
+    tls: :always,
+    tls_options: [
+      server_name_indication: String.to_charlist(email_server || ""),
+      versions: [:"tlsv1.2", :"tlsv1.3"],
+      verify: :verify_none
+    ],
+    auth: :always
+
+  config :microwaveprop, Microwaveprop.Repo,
+    # ssl: true,
+    url: database_url,
+    pool_size: String.to_integer(System.get_env("POOL_SIZE") || "20"),
+    # For machines with several cores, consider starting multiple pools of `pool_size`
+    # pool_count: 4,
+    socket_options: maybe_ipv6
+
+  config :microwaveprop, MicrowavepropWeb.Endpoint,
+    # SSL terminated by Cloudflare tunnel; generated URLs still use https
+    url: [host: host, port: 443, scheme: "https"],
+    http: [
+      # Enable IPv6 and bind on all interfaces.
+      # Set it to  {0, 0, 0, 0, 0, 0, 0, 1} for local network only access.
+      # See the documentation on https://hexdocs.pm/bandit/Bandit.html#t:options/0
+      # for details about using IPv6 vs IPv4 and loopback vs public addresses.
+      ip: {0, 0, 0, 0, 0, 0, 0, 0}
+    ],
+    secret_key_base: secret_key_base
+
+  config :microwaveprop, Oban,
+    engine: Oban.Pro.Engines.Smart,
+    queues: oban_queues
+
+  config :microwaveprop, Oban, plugins: oban_plugins
   config :microwaveprop, :email_from, {"NTMS Propagation", "prop@w5isp.com"}
   config :microwaveprop, hrrr_base_url: System.get_env("HRRR_BASE_URL", "http://skippy.w5isp.com:8080")
   config :microwaveprop, srtm_tiles_dir: "/data/srtm"

k8s/deployment-backfill.yaml

@@ -0,0 +1,113 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prop-backfill
+  namespace: prop
+spec:
+  # Start with 1. Scale up as additional prop-backfill=true nodes come online
+  # (podAntiAffinity keys on physical-host, same as prop, so replicas stay
+  # spread across distinct physical hosts).
+  replicas: 1
+  minReadySeconds: 5
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+  selector:
+    matchLabels:
+      app: prop
+      tier: backfill
+  template:
+    metadata:
+      labels:
+        # app=prop matches the libcluster selector in runtime.exs, so these
+        # pods join the same Erlang cluster as the hot-path prop pods and
+        # share Oban's queue ownership / leader election.
+        app: prop
+        tier: backfill
+    spec:
+      nodeSelector:
+        prop-backfill: "true"
+      tolerations:
+        # Matches the taint recommended for prop-backfill=true nodes. Remove
+        # this block if you haven't tainted the nodes.
+        - key: workload
+          operator: Equal
+          value: backfill
+          effect: NoSchedule
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchLabels:
+                  app: prop
+              topologyKey: topology.kubernetes.io/physical-host
+      serviceAccountName: prop
+      imagePullSecrets:
+        - name: forgejo-registry
+      securityContext:
+        runAsUser: 65534
+        runAsNonRoot: true
+        fsGroup: 65534
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: prop-backfill
+          image: git.mcintire.me/graham/prop:main-1776620433-6f63f3b # {"$imagepolicy": "flux-system:prop"}
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            # No PHX_SERVER — endpoint stays inert. No PORT either.
+            - name: PROP_ROLE
+              value: "backfill"
+            - name: PHX_HOST
+              value: "prop.w5isp.com"
+            - name: HRRR_BASE_URL
+              value: "http://skippy.w5isp.com:8080"
+          envFrom:
+            - secretRef:
+                name: prop-secrets
+          # No HTTP probes (no web server). Use exec probe against the
+          # running BEAM to confirm the VM is responsive.
+          startupProbe:
+            exec:
+              command: ["/bin/sh", "-c", "/app/bin/microwaveprop rpc ':ok'"]
+            initialDelaySeconds: 10
+            periodSeconds: 5
+            failureThreshold: 24
+            timeoutSeconds: 5
+          livenessProbe:
+            exec:
+              command: ["/bin/sh", "-c", "/app/bin/microwaveprop rpc ':ok'"]
+            periodSeconds: 30
+            failureThreshold: 3
+            timeoutSeconds: 5
+          resources:
+            # 8GB T620 box — leave ~3GB for kubelet/containerd/OS.
+            requests:
+              cpu: 100m
+              memory: 512Mi
+            limits:
+              cpu: "2"
+              memory: 4Gi
+          securityContext:
+            allowPrivilegeEscalation: false
+            runAsNonRoot: true
+            runAsUser: 65534
+            capabilities:
+              drop:
+                - ALL
+            seccompProfile:
+              type: RuntimeDefault
+          volumeMounts:
+            - name: data
+              mountPath: /data
+      volumes:
+        - name: data
+          nfs:
+            server: 10.0.15.103
+            path: /data

k8s/kustomization.yaml

@@ -7,5 +7,6 @@ resources:
   - namespace.yaml
   - rbac.yaml
   - deployment.yaml
+  - deployment-backfill.yaml
   - service.yaml
   - metrics-service.yaml