From 1b2424a1c16055aa26013c4a2b22d469975fdb44 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Sun, 19 Apr 2026 14:14:28 -0500 Subject: [PATCH] feat(infra): add prop-backfill deployment for T620 nodes PROP_ROLE=backfill drops the hot-path propagation/commercial/solar queues and the Cron plugin. Paired with a dedicated Deployment (nodeSelector: prop-backfill=true, 4Gi memory, no web) this lets 8GB T620 workers drain the hrrr/weather/terrain/iemre/narr backfill queues without competing with the hourly grid chain. --- config/runtime.exs | 243 ++++++++++++++++++++--------------- k8s/deployment-backfill.yaml | 113 ++++++++++++++++ k8s/kustomization.yaml | 1 + 3 files changed, 252 insertions(+), 105 deletions(-) create mode 100644 k8s/deployment-backfill.yaml diff --git a/config/runtime.exs b/config/runtime.exs index 99e2b4fe..9954ae9e 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -108,6 +108,142 @@ if config_env() == :prod do # handshake fails with an "unexpected_message" TLS alert. email_server = System.get_env("EMAIL_SERVER") + # Production Oban: live scoring, polling, and on-demand QSO enrichment. + # Runs on the Oban Pro Smart engine so we can use global_limit / rate_limit + # to protect rate-limited upstreams (Copernicus CDS in particular). + # + # PROP_ROLE controls which workloads this pod picks up: + # * "hot" (default) — full hot-path: web, crons, all queues. + # * "backfill" — no web server (unset PHX_SERVER), no crons, and only + # backfill-style queues. Intended for smaller nodes dedicated to + # catching up historical enrichment without competing with the + # hourly propagation grid chain. + prop_role = System.get_env("PROP_ROLE", "hot") + + hot_only_queues = [ + # 1 slot per pod. Two concurrent forecast-hour steps in a single + # pod stack HRRR grid + native duct grid + scored band map into + # ~5-6 GiB RSS, hitting the 6 GiB limit and OOM-killing the pod + # mid-chain. With 3 pods we still get 3-way parallel fan-out + # cluster-wide, which is enough to finish the f00–f18 chain + # comfortably inside the hourly interval. PropagationPruneWorker + # shares this queue at a lower priority; if a prune job blocks + # the grid chain briefly, the chain's unique window protects + # the next hourly fire. + propagation: 1, + commercial: 1, + solar: 1 + ] + + shared_queues = [ + # 1 slot per pod (3 cluster-wide). Any higher than this and the + # ASOS backfill hammers IEM hard enough to get HTTP 429s, which + # fill the retryable queue and thrash pod CPU without making + # forward progress. + weather: 1, + gefs: 1, + hrrr: 2, + terrain: 3, + iemre: 3, + # Historical backfill for pre-2014 contacts (pre-HRRR archive). + # NARR is fetched anonymously from NCEI with no quota, no job + # queue. Replaced the ERA5/CDS pipeline which never produced a + # single row in prod. era5_cds_jobs table is kept for inspection + # but its workers/client are gone. See + # docs/plans/2026-04-15-merra2-historical-backfill.md. + narr: 6, + rtma: 2, + backfill_enqueue: 1, + admin: 1, + nexrad: 2, + radar: 2, + ionosphere: 1, + space_weather: 1, + mechanism: 4, + exports: 1, + contact_import: 4 + ] + + oban_queues = + case prop_role do + "backfill" -> shared_queues + _ -> hot_only_queues ++ shared_queues + end + + cron_plugin = + {Oban.Plugins.Cron, + crontab: [ + {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker}, + {"*/5 * * * *", Microwaveprop.Commercial.PollWorker}, + # Hourly at :05. With scores written as binary files (not + # Postgres) and HRRR grid profile persistence removed, a + # full f00-f18 chain runs in ~45-60 min, so hourly fits. + # If one chain slips past 60 min the :propagation queue's + # concurrency-of-2 lets the next chain interleave; files + # are keyed by (band, valid_time) so last-writer-wins gives + # newer analysis data naturally. + {"5 * * * *", Microwaveprop.Workers.PropagationGridWorker}, + # GEFS runs publish ~3-4h after the 00/06/12/18Z cycle; seed the + # Day 2-7 outlook 5h after each run to stay safely past NOMADS' + # publication lag. Seeder enqueues f024-f168 at 6-hour cadence + # (25 jobs per run, 100/day) into the :gefs queue. + {"30 5,11,17,23 * * *", Microwaveprop.Workers.GefsFetchWorker}, + # MRMS refreshes the PrecipRate cache that AsosAdjustmentWorker + # overlays onto the score grid. Both must run together — an + # ASOS nudge without a fresh MRMS frame reverts to HRRR-only + # rain. Requires wgrib2 built with PNG decoder support (DRT + # 5.41), which the Dockerfile now enables via NCEPLIBS-g2c. + {"*/2 * * * *", Microwaveprop.Workers.MrmsFetchWorker}, + # AsosAdjustmentWorker disabled: depends on hrrr_profiles rows + # that PropagationGridWorker no longer persists. The 10-min + # ASOS nudge is a nice-to-have refinement we're not paying the + # full grid-side JSONB write for. + # {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker}, + {"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker}, + # The :narr type is virtual: it targets contacts with hrrr_status = + # :unavailable (pre-2014, missing from the HRRR archive) and dispatches + # NarrFetchWorker against NCEI. See narr_jobs_for_contact/1. + {"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker, + args: %{"types" => ["hrrr", "weather", "terrain", "iemre", "narr", "radar", "mechanism"]}}, + # Hourly safety net for pos1/pos2/distance_km. Normally every contact + # gets positions at insert time via Radio.resolve_grids_and_insert/1, + # but direct DB writes (manual fixes, bulk imports) can bypass that. + {"0 * * * *", Microwaveprop.Workers.ContactPositionBackfillWorker, args: %{"limit" => 500}}, + # UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch + {"30 1 * * *", CanadianSoundingFetchWorker}, + {"30 13 * * *", CanadianSoundingFetchWorker}, + # GIRO DIDBase publishes foF2/foEs/hmF2 at ~7.5 min cadence; poll + # every 10 min per station (Millstone Hill, Alpena) for the 144 + # MHz sporadic-E and HF MUF scoring inputs. + {"*/10 * * * *", Microwaveprop.Workers.IonosphereFetchWorker}, + # NOAA SWPC: Kp / GOES X-ray at 1-min cadence, F10.7 hourly. + # 5-min poll balances freshness vs request volume. + {"*/5 * * * *", Microwaveprop.Workers.SpaceWeatherFetchWorker} + ]} + + base_plugins = [ + {Oban.Plugins.Pruner, max_age: 3600 * 24}, + # DynamicLifeline uses producer records (heartbeats from each + # live node) to rescue orphans, not a timer. When a pod dies + # mid-deploy its Producer row disappears and any in-flight job + # with that producer in `attempted_by` gets moved back to + # `available` on the next rescue cycle — within ~30s instead of + # waiting for a timer-based `rescue_after` to expire. Critical + # for PropagationGridWorker chain steps to recover from rolling + # deploys before the next hourly cron fire. + {Oban.Pro.Plugins.DynamicLifeline, rescue_interval: to_timeout(second: 30)} + ] + + # Backfill pods skip the cron plugin entirely — no risk of a + # backfill node becoming cron leader and enqueueing work targeted + # at queues it doesn't even run (propagation, commercial). Hot pods + # own scheduling; backfill pods just drain the queues. + oban_plugins = + case prop_role do + "backfill" -> base_plugins + _ -> base_plugins ++ [cron_plugin] + end + config :libcluster, topologies: [ k8s: [ @@ -156,114 +292,11 @@ if config_env() == :prod do ], secret_key_base: secret_key_base - # Production Oban: live scoring, polling, and on-demand QSO enrichment. - # Runs on the Oban Pro Smart engine so we can use global_limit / rate_limit - # to protect rate-limited upstreams (Copernicus CDS in particular). config :microwaveprop, Oban, engine: Oban.Pro.Engines.Smart, - # Per-pod concurrency (×3 pods = effective cluster total) - queues: [ - # 1 slot per pod. Two concurrent forecast-hour steps in a single - # pod stack HRRR grid + native duct grid + scored band map into - # ~5-6 GiB RSS, hitting the 6 GiB limit and OOM-killing the pod - # mid-chain. With 3 pods we still get 3-way parallel fan-out - # cluster-wide, which is enough to finish the f00–f18 chain - # comfortably inside the hourly interval. PropagationPruneWorker - # shares this queue at a lower priority; if a prune job blocks - # the grid chain briefly, the chain's unique window protects - # the next hourly fire. - propagation: 1, - commercial: 1, - solar: 1, - # 1 slot per pod (3 cluster-wide). Any higher than this and the - # ASOS backfill hammers IEM hard enough to get HTTP 429s, which - # fill the retryable queue and thrash pod CPU without making - # forward progress. - weather: 1, - gefs: 1, - hrrr: 2, - terrain: 3, - iemre: 3, - # Historical backfill for pre-2014 contacts (pre-HRRR archive). - # NARR is fetched anonymously from NCEI with no quota, no job - # queue. Replaced the ERA5/CDS pipeline which never produced a - # single row in prod. era5_cds_jobs table is kept for inspection - # but its workers/client are gone. See - # docs/plans/2026-04-15-merra2-historical-backfill.md. - narr: 6, - rtma: 2, - backfill_enqueue: 1, - admin: 1, - nexrad: 2, - radar: 2, - ionosphere: 1, - space_weather: 1, - mechanism: 4, - exports: 1, - contact_import: 4 - ], - plugins: [ - {Oban.Plugins.Pruner, max_age: 3600 * 24}, - # DynamicLifeline uses producer records (heartbeats from each - # live node) to rescue orphans, not a timer. When a pod dies - # mid-deploy its Producer row disappears and any in-flight job - # with that producer in `attempted_by` gets moved back to - # `available` on the next rescue cycle — within ~30s instead of - # waiting for a timer-based `rescue_after` to expire. Critical - # for PropagationGridWorker chain steps to recover from rolling - # deploys before the next hourly cron fire. - {Oban.Pro.Plugins.DynamicLifeline, rescue_interval: to_timeout(second: 30)}, - {Oban.Plugins.Cron, - crontab: [ - {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker}, - {"*/5 * * * *", Microwaveprop.Commercial.PollWorker}, - # Hourly at :05. With scores written as binary files (not - # Postgres) and HRRR grid profile persistence removed, a - # full f00-f18 chain runs in ~45-60 min, so hourly fits. - # If one chain slips past 60 min the :propagation queue's - # concurrency-of-2 lets the next chain interleave; files - # are keyed by (band, valid_time) so last-writer-wins gives - # newer analysis data naturally. - {"5 * * * *", Microwaveprop.Workers.PropagationGridWorker}, - # GEFS runs publish ~3-4h after the 00/06/12/18Z cycle; seed the - # Day 2-7 outlook 5h after each run to stay safely past NOMADS' - # publication lag. Seeder enqueues f024-f168 at 6-hour cadence - # (25 jobs per run, 100/day) into the :gefs queue. - {"30 5,11,17,23 * * *", Microwaveprop.Workers.GefsFetchWorker}, - # MRMS refreshes the PrecipRate cache that AsosAdjustmentWorker - # overlays onto the score grid. Both must run together — an - # ASOS nudge without a fresh MRMS frame reverts to HRRR-only - # rain. Requires wgrib2 built with PNG decoder support (DRT - # 5.41), which the Dockerfile now enables via NCEPLIBS-g2c. - {"*/2 * * * *", Microwaveprop.Workers.MrmsFetchWorker}, - # AsosAdjustmentWorker disabled: depends on hrrr_profiles rows - # that PropagationGridWorker no longer persists. The 10-min - # ASOS nudge is a nice-to-have refinement we're not paying the - # full grid-side JSONB write for. - # {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker}, - {"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker}, - # The :narr type is virtual: it targets contacts with hrrr_status = - # :unavailable (pre-2014, missing from the HRRR archive) and dispatches - # NarrFetchWorker against NCEI. See narr_jobs_for_contact/1. - {"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker, - args: %{"types" => ["hrrr", "weather", "terrain", "iemre", "narr", "radar", "mechanism"]}}, - # Hourly safety net for pos1/pos2/distance_km. Normally every contact - # gets positions at insert time via Radio.resolve_grids_and_insert/1, - # but direct DB writes (manual fixes, bulk imports) can bypass that. - {"0 * * * *", Microwaveprop.Workers.ContactPositionBackfillWorker, args: %{"limit" => 500}}, - # UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch - {"30 1 * * *", CanadianSoundingFetchWorker}, - {"30 13 * * *", CanadianSoundingFetchWorker}, - # GIRO DIDBase publishes foF2/foEs/hmF2 at ~7.5 min cadence; poll - # every 10 min per station (Millstone Hill, Alpena) for the 144 - # MHz sporadic-E and HF MUF scoring inputs. - {"*/10 * * * *", Microwaveprop.Workers.IonosphereFetchWorker}, - # NOAA SWPC: Kp / GOES X-ray at 1-min cadence, F10.7 hourly. - # 5-min poll balances freshness vs request volume. - {"*/5 * * * *", Microwaveprop.Workers.SpaceWeatherFetchWorker} - ]} - ] + queues: oban_queues + config :microwaveprop, Oban, plugins: oban_plugins config :microwaveprop, :email_from, {"NTMS Propagation", "prop@w5isp.com"} config :microwaveprop, hrrr_base_url: System.get_env("HRRR_BASE_URL", "http://skippy.w5isp.com:8080") config :microwaveprop, srtm_tiles_dir: "/data/srtm" diff --git a/k8s/deployment-backfill.yaml b/k8s/deployment-backfill.yaml new file mode 100644 index 00000000..cf6416a7 --- /dev/null +++ b/k8s/deployment-backfill.yaml @@ -0,0 +1,113 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prop-backfill + namespace: prop +spec: + # Start with 1. Scale up as additional prop-backfill=true nodes come online + # (podAntiAffinity keys on physical-host, same as prop, so replicas stay + # spread across distinct physical hosts). + replicas: 1 + minReadySeconds: 5 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + selector: + matchLabels: + app: prop + tier: backfill + template: + metadata: + labels: + # app=prop matches the libcluster selector in runtime.exs, so these + # pods join the same Erlang cluster as the hot-path prop pods and + # share Oban's queue ownership / leader election. + app: prop + tier: backfill + spec: + nodeSelector: + prop-backfill: "true" + tolerations: + # Matches the taint recommended for prop-backfill=true nodes. Remove + # this block if you haven't tainted the nodes. + - key: workload + operator: Equal + value: backfill + effect: NoSchedule + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: prop + topologyKey: topology.kubernetes.io/physical-host + serviceAccountName: prop + imagePullSecrets: + - name: forgejo-registry + securityContext: + runAsUser: 65534 + runAsNonRoot: true + fsGroup: 65534 + seccompProfile: + type: RuntimeDefault + containers: + - name: prop-backfill + image: git.mcintire.me/graham/prop:main-1776620433-6f63f3b # {"$imagepolicy": "flux-system:prop"} + imagePullPolicy: IfNotPresent + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + # No PHX_SERVER — endpoint stays inert. No PORT either. + - name: PROP_ROLE + value: "backfill" + - name: PHX_HOST + value: "prop.w5isp.com" + - name: HRRR_BASE_URL + value: "http://skippy.w5isp.com:8080" + envFrom: + - secretRef: + name: prop-secrets + # No HTTP probes (no web server). Use exec probe against the + # running BEAM to confirm the VM is responsive. + startupProbe: + exec: + command: ["/bin/sh", "-c", "/app/bin/microwaveprop rpc ':ok'"] + initialDelaySeconds: 10 + periodSeconds: 5 + failureThreshold: 24 + timeoutSeconds: 5 + livenessProbe: + exec: + command: ["/bin/sh", "-c", "/app/bin/microwaveprop rpc ':ok'"] + periodSeconds: 30 + failureThreshold: 3 + timeoutSeconds: 5 + resources: + # 8GB T620 box — leave ~3GB for kubelet/containerd/OS. + requests: + cpu: 100m + memory: 512Mi + limits: + cpu: "2" + memory: 4Gi + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 65534 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + nfs: + server: 10.0.15.103 + path: /data diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml index e7621acb..875d6b43 100644 --- a/k8s/kustomization.yaml +++ b/k8s/kustomization.yaml @@ -7,5 +7,6 @@ resources: - namespace.yaml - rbac.yaml - deployment.yaml + - deployment-backfill.yaml - service.yaml - metrics-service.yaml