feat(infra): add prop-backfill deployment for T620 nodes

PROP_ROLE=backfill drops the hot-path propagation/commercial/solar
queues and the Cron plugin. Paired with a dedicated Deployment
(nodeSelector: prop-backfill=true, 4Gi memory, no web) this lets 8GB
T620 workers drain the hrrr/weather/terrain/iemre/narr backfill
queues without competing with the hourly grid chain.

config/runtime.exs

@@ -108,6 +108,142 @@ if config_env() == :prod do
   # handshake fails with an "unexpected_message" TLS alert.
   email_server = System.get_env("EMAIL_SERVER")
 
+  # Production Oban: live scoring, polling, and on-demand QSO enrichment.
+  # Runs on the Oban Pro Smart engine so we can use global_limit / rate_limit
+  # to protect rate-limited upstreams (Copernicus CDS in particular).
+  #
+  # PROP_ROLE controls which workloads this pod picks up:
+  #   * "hot" (default) — full hot-path: web, crons, all queues.
+  #   * "backfill" — no web server (unset PHX_SERVER), no crons, and only
+  #     backfill-style queues. Intended for smaller nodes dedicated to
+  #     catching up historical enrichment without competing with the
+  #     hourly propagation grid chain.
+  prop_role = System.get_env("PROP_ROLE", "hot")
+
+  hot_only_queues = [
+    # 1 slot per pod. Two concurrent forecast-hour steps in a single
+    # pod stack HRRR grid + native duct grid + scored band map into
+    # ~5-6 GiB RSS, hitting the 6 GiB limit and OOM-killing the pod
+    # mid-chain. With 3 pods we still get 3-way parallel fan-out
+    # cluster-wide, which is enough to finish the f00–f18 chain
+    # comfortably inside the hourly interval. PropagationPruneWorker
+    # shares this queue at a lower priority; if a prune job blocks
+    # the grid chain briefly, the chain's unique window protects
+    # the next hourly fire.
+    propagation: 1,
+    commercial: 1,
+    solar: 1
+  ]
+
+  shared_queues = [
+    # 1 slot per pod (3 cluster-wide). Any higher than this and the
+    # ASOS backfill hammers IEM hard enough to get HTTP 429s, which
+    # fill the retryable queue and thrash pod CPU without making
+    # forward progress.
+    weather: 1,
+    gefs: 1,
+    hrrr: 2,
+    terrain: 3,
+    iemre: 3,
+    # Historical backfill for pre-2014 contacts (pre-HRRR archive).
+    # NARR is fetched anonymously from NCEI with no quota, no job
+    # queue. Replaced the ERA5/CDS pipeline which never produced a
+    # single row in prod. era5_cds_jobs table is kept for inspection
+    # but its workers/client are gone. See
+    # docs/plans/2026-04-15-merra2-historical-backfill.md.
+    narr: 6,
+    rtma: 2,
+    backfill_enqueue: 1,
+    admin: 1,
+    nexrad: 2,
+    radar: 2,
+    ionosphere: 1,
+    space_weather: 1,
+    mechanism: 4,
+    exports: 1,
+    contact_import: 4
+  ]
+
+  oban_queues =
+    case prop_role do
+      "backfill" -> shared_queues
+      _ -> hot_only_queues ++ shared_queues
+    end
+
+  cron_plugin =
+    {Oban.Plugins.Cron,
+     crontab: [
+       {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker},
+       {"*/5 * * * *", Microwaveprop.Commercial.PollWorker},
+       # Hourly at :05. With scores written as binary files (not
+       # Postgres) and HRRR grid profile persistence removed, a
+       # full f00-f18 chain runs in ~45-60 min, so hourly fits.
+       # If one chain slips past 60 min the :propagation queue's
+       # concurrency-of-2 lets the next chain interleave; files
+       # are keyed by (band, valid_time) so last-writer-wins gives
+       # newer analysis data naturally.
+       {"5 * * * *", Microwaveprop.Workers.PropagationGridWorker},
+       # GEFS runs publish ~3-4h after the 00/06/12/18Z cycle; seed the
+       # Day 2-7 outlook 5h after each run to stay safely past NOMADS'
+       # publication lag. Seeder enqueues f024-f168 at 6-hour cadence
+       # (25 jobs per run, 100/day) into the :gefs queue.
+       {"30 5,11,17,23 * * *", Microwaveprop.Workers.GefsFetchWorker},
+       # MRMS refreshes the PrecipRate cache that AsosAdjustmentWorker
+       # overlays onto the score grid. Both must run together — an
+       # ASOS nudge without a fresh MRMS frame reverts to HRRR-only
+       # rain. Requires wgrib2 built with PNG decoder support (DRT
+       # 5.41), which the Dockerfile now enables via NCEPLIBS-g2c.
+       {"*/2 * * * *", Microwaveprop.Workers.MrmsFetchWorker},
+       # AsosAdjustmentWorker disabled: depends on hrrr_profiles rows
+       # that PropagationGridWorker no longer persists. The 10-min
+       # ASOS nudge is a nice-to-have refinement we're not paying the
+       # full grid-side JSONB write for.
+       # {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker},
+       {"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
+       # The :narr type is virtual: it targets contacts with hrrr_status =
+       # :unavailable (pre-2014, missing from the HRRR archive) and dispatches
+       # NarrFetchWorker against NCEI. See narr_jobs_for_contact/1.
+       {"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker,
+        args: %{"types" => ["hrrr", "weather", "terrain", "iemre", "narr", "radar", "mechanism"]}},
+       # Hourly safety net for pos1/pos2/distance_km. Normally every contact
+       # gets positions at insert time via Radio.resolve_grids_and_insert/1,
+       # but direct DB writes (manual fixes, bulk imports) can bypass that.
+       {"0 * * * *", Microwaveprop.Workers.ContactPositionBackfillWorker, args: %{"limit" => 500}},
+       # UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch
+       {"30 1 * * *", CanadianSoundingFetchWorker},
+       {"30 13 * * *", CanadianSoundingFetchWorker},
+       # GIRO DIDBase publishes foF2/foEs/hmF2 at ~7.5 min cadence; poll
+       # every 10 min per station (Millstone Hill, Alpena) for the 144
+       # MHz sporadic-E and HF MUF scoring inputs.
+       {"*/10 * * * *", Microwaveprop.Workers.IonosphereFetchWorker},
+       # NOAA SWPC: Kp / GOES X-ray at 1-min cadence, F10.7 hourly.
+       # 5-min poll balances freshness vs request volume.
+       {"*/5 * * * *", Microwaveprop.Workers.SpaceWeatherFetchWorker}
+     ]}
+
+  base_plugins = [
+    {Oban.Plugins.Pruner, max_age: 3600 * 24},
+    # DynamicLifeline uses producer records (heartbeats from each
+    # live node) to rescue orphans, not a timer. When a pod dies
+    # mid-deploy its Producer row disappears and any in-flight job
+    # with that producer in `attempted_by` gets moved back to
+    # `available` on the next rescue cycle — within ~30s instead of
+    # waiting for a timer-based `rescue_after` to expire. Critical
+    # for PropagationGridWorker chain steps to recover from rolling
+    # deploys before the next hourly cron fire.
+    {Oban.Pro.Plugins.DynamicLifeline, rescue_interval: to_timeout(second: 30)}
+  ]
+
+  # Backfill pods skip the cron plugin entirely — no risk of a
+  # backfill node becoming cron leader and enqueueing work targeted
+  # at queues it doesn't even run (propagation, commercial). Hot pods
+  # own scheduling; backfill pods just drain the queues.
+  oban_plugins =
+    case prop_role do
+      "backfill" -> base_plugins
+      _ -> base_plugins ++ [cron_plugin]
+    end
+
   config :libcluster,
     topologies: [
       k8s: [
@@ -156,114 +292,11 @@ if config_env() == :prod do
     ],
     secret_key_base: secret_key_base
 
-  # Production Oban: live scoring, polling, and on-demand QSO enrichment.
-  # Runs on the Oban Pro Smart engine so we can use global_limit / rate_limit
-  # to protect rate-limited upstreams (Copernicus CDS in particular).
   config :microwaveprop, Oban,
     engine: Oban.Pro.Engines.Smart,
-    # Per-pod concurrency (×3 pods = effective cluster total)
+    queues: oban_queues
-    queues: [
-      # 1 slot per pod. Two concurrent forecast-hour steps in a single
-      # pod stack HRRR grid + native duct grid + scored band map into
-      # ~5-6 GiB RSS, hitting the 6 GiB limit and OOM-killing the pod
-      # mid-chain. With 3 pods we still get 3-way parallel fan-out
-      # cluster-wide, which is enough to finish the f00–f18 chain
-      # comfortably inside the hourly interval. PropagationPruneWorker
-      # shares this queue at a lower priority; if a prune job blocks
-      # the grid chain briefly, the chain's unique window protects
-      # the next hourly fire.
-      propagation: 1,
-      commercial: 1,
-      solar: 1,
-      # 1 slot per pod (3 cluster-wide). Any higher than this and the
-      # ASOS backfill hammers IEM hard enough to get HTTP 429s, which
-      # fill the retryable queue and thrash pod CPU without making
-      # forward progress.
-      weather: 1,
-      gefs: 1,
-      hrrr: 2,
-      terrain: 3,
-      iemre: 3,
-      # Historical backfill for pre-2014 contacts (pre-HRRR archive).
-      # NARR is fetched anonymously from NCEI with no quota, no job
-      # queue. Replaced the ERA5/CDS pipeline which never produced a
-      # single row in prod. era5_cds_jobs table is kept for inspection
-      # but its workers/client are gone. See
-      # docs/plans/2026-04-15-merra2-historical-backfill.md.
-      narr: 6,
-      rtma: 2,
-      backfill_enqueue: 1,
-      admin: 1,
-      nexrad: 2,
-      radar: 2,
-      ionosphere: 1,
-      space_weather: 1,
-      mechanism: 4,
-      exports: 1,
-      contact_import: 4
-    ],
-    plugins: [
-      {Oban.Plugins.Pruner, max_age: 3600 * 24},
-      # DynamicLifeline uses producer records (heartbeats from each
-      # live node) to rescue orphans, not a timer. When a pod dies
-      # mid-deploy its Producer row disappears and any in-flight job
-      # with that producer in `attempted_by` gets moved back to
-      # `available` on the next rescue cycle — within ~30s instead of
-      # waiting for a timer-based `rescue_after` to expire. Critical
-      # for PropagationGridWorker chain steps to recover from rolling
-      # deploys before the next hourly cron fire.
-      {Oban.Pro.Plugins.DynamicLifeline, rescue_interval: to_timeout(second: 30)},
-      {Oban.Plugins.Cron,
-       crontab: [
-         {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker},
-         {"*/5 * * * *", Microwaveprop.Commercial.PollWorker},
-         # Hourly at :05. With scores written as binary files (not
-         # Postgres) and HRRR grid profile persistence removed, a
-         # full f00-f18 chain runs in ~45-60 min, so hourly fits.
-         # If one chain slips past 60 min the :propagation queue's
-         # concurrency-of-2 lets the next chain interleave; files
-         # are keyed by (band, valid_time) so last-writer-wins gives
-         # newer analysis data naturally.
-         {"5 * * * *", Microwaveprop.Workers.PropagationGridWorker},
-         # GEFS runs publish ~3-4h after the 00/06/12/18Z cycle; seed the
-         # Day 2-7 outlook 5h after each run to stay safely past NOMADS'
-         # publication lag. Seeder enqueues f024-f168 at 6-hour cadence
-         # (25 jobs per run, 100/day) into the :gefs queue.
-         {"30 5,11,17,23 * * *", Microwaveprop.Workers.GefsFetchWorker},
-         # MRMS refreshes the PrecipRate cache that AsosAdjustmentWorker
-         # overlays onto the score grid. Both must run together — an
-         # ASOS nudge without a fresh MRMS frame reverts to HRRR-only
-         # rain. Requires wgrib2 built with PNG decoder support (DRT
-         # 5.41), which the Dockerfile now enables via NCEPLIBS-g2c.
-         {"*/2 * * * *", Microwaveprop.Workers.MrmsFetchWorker},
-         # AsosAdjustmentWorker disabled: depends on hrrr_profiles rows
-         # that PropagationGridWorker no longer persists. The 10-min
-         # ASOS nudge is a nice-to-have refinement we're not paying the
-         # full grid-side JSONB write for.
-         # {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker},
-         {"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
-         # The :narr type is virtual: it targets contacts with hrrr_status =
-         # :unavailable (pre-2014, missing from the HRRR archive) and dispatches
-         # NarrFetchWorker against NCEI. See narr_jobs_for_contact/1.
-         {"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker,
-          args: %{"types" => ["hrrr", "weather", "terrain", "iemre", "narr", "radar", "mechanism"]}},
-         # Hourly safety net for pos1/pos2/distance_km. Normally every contact
-         # gets positions at insert time via Radio.resolve_grids_and_insert/1,
-         # but direct DB writes (manual fixes, bulk imports) can bypass that.
-         {"0 * * * *", Microwaveprop.Workers.ContactPositionBackfillWorker, args: %{"limit" => 500}},
-         # UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch
-         {"30 1 * * *", CanadianSoundingFetchWorker},
-         {"30 13 * * *", CanadianSoundingFetchWorker},
-         # GIRO DIDBase publishes foF2/foEs/hmF2 at ~7.5 min cadence; poll
-         # every 10 min per station (Millstone Hill, Alpena) for the 144
-         # MHz sporadic-E and HF MUF scoring inputs.
-         {"*/10 * * * *", Microwaveprop.Workers.IonosphereFetchWorker},
-         # NOAA SWPC: Kp / GOES X-ray at 1-min cadence, F10.7 hourly.
-         # 5-min poll balances freshness vs request volume.
-         {"*/5 * * * *", Microwaveprop.Workers.SpaceWeatherFetchWorker}
-       ]}
-    ]
 
+  config :microwaveprop, Oban, plugins: oban_plugins
   config :microwaveprop, :email_from, {"NTMS Propagation", "prop@w5isp.com"}
   config :microwaveprop, hrrr_base_url: System.get_env("HRRR_BASE_URL", "http://skippy.w5isp.com:8080")
   config :microwaveprop, srtm_tiles_dir: "/data/srtm"

k8s/deployment-backfill.yaml

@@ -0,0 +1,113 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prop-backfill
+  namespace: prop
+spec:
+  # Start with 1. Scale up as additional prop-backfill=true nodes come online
+  # (podAntiAffinity keys on physical-host, same as prop, so replicas stay
+  # spread across distinct physical hosts).
+  replicas: 1
+  minReadySeconds: 5
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+  selector:
+    matchLabels:
+      app: prop
+      tier: backfill
+  template:
+    metadata:
+      labels:
+        # app=prop matches the libcluster selector in runtime.exs, so these
+        # pods join the same Erlang cluster as the hot-path prop pods and
+        # share Oban's queue ownership / leader election.
+        app: prop
+        tier: backfill
+    spec:
+      nodeSelector:
+        prop-backfill: "true"
+      tolerations:
+        # Matches the taint recommended for prop-backfill=true nodes. Remove
+        # this block if you haven't tainted the nodes.
+        - key: workload
+          operator: Equal
+          value: backfill
+          effect: NoSchedule
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchLabels:
+                  app: prop
+              topologyKey: topology.kubernetes.io/physical-host
+      serviceAccountName: prop
+      imagePullSecrets:
+        - name: forgejo-registry
+      securityContext:
+        runAsUser: 65534
+        runAsNonRoot: true
+        fsGroup: 65534
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: prop-backfill
+          image: git.mcintire.me/graham/prop:main-1776620433-6f63f3b # {"$imagepolicy": "flux-system:prop"}
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            # No PHX_SERVER — endpoint stays inert. No PORT either.
+            - name: PROP_ROLE
+              value: "backfill"
+            - name: PHX_HOST
+              value: "prop.w5isp.com"
+            - name: HRRR_BASE_URL
+              value: "http://skippy.w5isp.com:8080"
+          envFrom:
+            - secretRef:
+                name: prop-secrets
+          # No HTTP probes (no web server). Use exec probe against the
+          # running BEAM to confirm the VM is responsive.
+          startupProbe:
+            exec:
+              command: ["/bin/sh", "-c", "/app/bin/microwaveprop rpc ':ok'"]
+            initialDelaySeconds: 10
+            periodSeconds: 5
+            failureThreshold: 24
+            timeoutSeconds: 5
+          livenessProbe:
+            exec:
+              command: ["/bin/sh", "-c", "/app/bin/microwaveprop rpc ':ok'"]
+            periodSeconds: 30
+            failureThreshold: 3
+            timeoutSeconds: 5
+          resources:
+            # 8GB T620 box — leave ~3GB for kubelet/containerd/OS.
+            requests:
+              cpu: 100m
+              memory: 512Mi
+            limits:
+              cpu: "2"
+              memory: 4Gi
+          securityContext:
+            allowPrivilegeEscalation: false
+            runAsNonRoot: true
+            runAsUser: 65534
+            capabilities:
+              drop:
+                - ALL
+            seccompProfile:
+              type: RuntimeDefault
+          volumeMounts:
+            - name: data
+              mountPath: /data
+      volumes:
+        - name: data
+          nfs:
+            server: 10.0.15.103
+            path: /data

k8s/kustomization.yaml

@@ -7,5 +7,6 @@ resources:
   - namespace.yaml
   - rbac.yaml
   - deployment.yaml
+  - deployment-backfill.yaml
   - service.yaml
   - metrics-service.yaml