graham/prop
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(infra): add prop-backfill deployment for T620 nodes

Browse source 
PROP_ROLE=backfill drops the hot-path propagation/commercial/solar
queues and the Cron plugin. Paired with a dedicated Deployment
(nodeSelector: prop-backfill=true, 4Gi memory, no web) this lets 8GB
T620 workers drain the hrrr/weather/terrain/iemre/narr backfill
queues without competing with the hourly grid chain.
This commit is contained in:
Graham McIntire 2026-04-19 14:14:28 -05:00
parent 23e4e3fef2
commit 1b2424a1c1
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
3 changed files with 252 additions and 105 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

243
config/runtime.exs
View file

@ -108,6 +108,142 @@ if config_env() == :prod do
# handshake fails with an "unexpected_message" TLS alert.
email_server = System.get_env("EMAIL_SERVER")
# Production Oban: live scoring, polling, and on-demand QSO enrichment.
# Runs on the Oban Pro Smart engine so we can use global_limit / rate_limit
# to protect rate-limited upstreams (Copernicus CDS in particular).
#
# PROP_ROLE controls which workloads this pod picks up:
# * "hot" (default) — full hot-path: web, crons, all queues.
# * "backfill" — no web server (unset PHX_SERVER), no crons, and only
# backfill-style queues. Intended for smaller nodes dedicated to
# catching up historical enrichment without competing with the
# hourly propagation grid chain.
prop_role = System.get_env("PROP_ROLE", "hot")
hot_only_queues = [
# 1 slot per pod. Two concurrent forecast-hour steps in a single
# pod stack HRRR grid + native duct grid + scored band map into
# ~5-6 GiB RSS, hitting the 6 GiB limit and OOM-killing the pod
# mid-chain. With 3 pods we still get 3-way parallel fan-out
# cluster-wide, which is enough to finish the f00–f18 chain
# comfortably inside the hourly interval. PropagationPruneWorker
# shares this queue at a lower priority; if a prune job blocks
# the grid chain briefly, the chain's unique window protects
# the next hourly fire.
propagation: 1,
commercial: 1,
solar: 1
]
shared_queues = [
# 1 slot per pod (3 cluster-wide). Any higher than this and the
# ASOS backfill hammers IEM hard enough to get HTTP 429s, which
# fill the retryable queue and thrash pod CPU without making
# forward progress.
weather: 1,
gefs: 1,
hrrr: 2,
terrain: 3,
iemre: 3,
# Historical backfill for pre-2014 contacts (pre-HRRR archive).
# NARR is fetched anonymously from NCEI with no quota, no job
# queue. Replaced the ERA5/CDS pipeline which never produced a
# single row in prod. era5_cds_jobs table is kept for inspection
# but its workers/client are gone. See
# docs/plans/2026-04-15-merra2-historical-backfill.md.
narr: 6,
rtma: 2,
backfill_enqueue: 1,
admin: 1,
nexrad: 2,
radar: 2,
ionosphere: 1,
space_weather: 1,
mechanism: 4,
exports: 1,
contact_import: 4
]
oban_queues =
case prop_role do
"backfill" -> shared_queues
_ -> hot_only_queues ++ shared_queues
end
cron_plugin =
{Oban.Plugins.Cron,
crontab: [
{"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker},
{"*/5 * * * *", Microwaveprop.Commercial.PollWorker},
# Hourly at :05. With scores written as binary files (not
# Postgres) and HRRR grid profile persistence removed, a
# full f00-f18 chain runs in ~45-60 min, so hourly fits.
# If one chain slips past 60 min the :propagation queue's
# concurrency-of-2 lets the next chain interleave; files
# are keyed by (band, valid_time) so last-writer-wins gives
# newer analysis data naturally.
{"5 * * * *", Microwaveprop.Workers.PropagationGridWorker},
# GEFS runs publish ~3-4h after the 00/06/12/18Z cycle; seed the
# Day 2-7 outlook 5h after each run to stay safely past NOMADS'
# publication lag. Seeder enqueues f024-f168 at 6-hour cadence
# (25 jobs per run, 100/day) into the :gefs queue.
{"30 5,11,17,23 * * *", Microwaveprop.Workers.GefsFetchWorker},
# MRMS refreshes the PrecipRate cache that AsosAdjustmentWorker
# overlays onto the score grid. Both must run together — an
# ASOS nudge without a fresh MRMS frame reverts to HRRR-only
# rain. Requires wgrib2 built with PNG decoder support (DRT
# 5.41), which the Dockerfile now enables via NCEPLIBS-g2c.
{"*/2 * * * *", Microwaveprop.Workers.MrmsFetchWorker},
# AsosAdjustmentWorker disabled: depends on hrrr_profiles rows
# that PropagationGridWorker no longer persists. The 10-min
# ASOS nudge is a nice-to-have refinement we're not paying the
# full grid-side JSONB write for.
# {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker},
{"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
# The :narr type is virtual: it targets contacts with hrrr_status =
# :unavailable (pre-2014, missing from the HRRR archive) and dispatches
# NarrFetchWorker against NCEI. See narr_jobs_for_contact/1.
{"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker,
args: %{"types" => ["hrrr", "weather", "terrain", "iemre", "narr", "radar", "mechanism"]}},
# Hourly safety net for pos1/pos2/distance_km. Normally every contact
# gets positions at insert time via Radio.resolve_grids_and_insert/1,
# but direct DB writes (manual fixes, bulk imports) can bypass that.
{"0 * * * *", Microwaveprop.Workers.ContactPositionBackfillWorker, args: %{"limit" => 500}},
# UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch
{"30 1 * * *", CanadianSoundingFetchWorker},
{"30 13 * * *", CanadianSoundingFetchWorker},
# GIRO DIDBase publishes foF2/foEs/hmF2 at ~7.5 min cadence; poll
# every 10 min per station (Millstone Hill, Alpena) for the 144
# MHz sporadic-E and HF MUF scoring inputs.
{"*/10 * * * *", Microwaveprop.Workers.IonosphereFetchWorker},
# NOAA SWPC: Kp / GOES X-ray at 1-min cadence, F10.7 hourly.
# 5-min poll balances freshness vs request volume.
{"*/5 * * * *", Microwaveprop.Workers.SpaceWeatherFetchWorker}
]}
base_plugins = [
{Oban.Plugins.Pruner, max_age: 3600 * 24},
# DynamicLifeline uses producer records (heartbeats from each
# live node) to rescue orphans, not a timer. When a pod dies
# mid-deploy its Producer row disappears and any in-flight job
# with that producer in `attempted_by` gets moved back to
# `available` on the next rescue cycle — within ~30s instead of
# waiting for a timer-based `rescue_after` to expire. Critical
# for PropagationGridWorker chain steps to recover from rolling
# deploys before the next hourly cron fire.
{Oban.Pro.Plugins.DynamicLifeline, rescue_interval: to_timeout(second: 30)}
]
# Backfill pods skip the cron plugin entirely — no risk of a
# backfill node becoming cron leader and enqueueing work targeted
# at queues it doesn't even run (propagation, commercial). Hot pods
# own scheduling; backfill pods just drain the queues.
oban_plugins =
case prop_role do
"backfill" -> base_plugins
_ -> base_plugins ++ [cron_plugin]
end
config :libcluster,
topologies: [
k8s: [
@ -156,114 +292,11 @@ if config_env() == :prod do
],
secret_key_base: secret_key_base
# Production Oban: live scoring, polling, and on-demand QSO enrichment.
# Runs on the Oban Pro Smart engine so we can use global_limit / rate_limit
# to protect rate-limited upstreams (Copernicus CDS in particular).
config :microwaveprop, Oban,
engine: Oban.Pro.Engines.Smart,
# Per-pod concurrency (×3 pods = effective cluster total)
queues: [
# 1 slot per pod. Two concurrent forecast-hour steps in a single
# pod stack HRRR grid + native duct grid + scored band map into
# ~5-6 GiB RSS, hitting the 6 GiB limit and OOM-killing the pod
# mid-chain. With 3 pods we still get 3-way parallel fan-out
# cluster-wide, which is enough to finish the f00–f18 chain
# comfortably inside the hourly interval. PropagationPruneWorker
# shares this queue at a lower priority; if a prune job blocks
# the grid chain briefly, the chain's unique window protects
# the next hourly fire.
propagation: 1,
commercial: 1,
solar: 1,
# 1 slot per pod (3 cluster-wide). Any higher than this and the
# ASOS backfill hammers IEM hard enough to get HTTP 429s, which
# fill the retryable queue and thrash pod CPU without making
# forward progress.
weather: 1,
gefs: 1,
hrrr: 2,
terrain: 3,
iemre: 3,
# Historical backfill for pre-2014 contacts (pre-HRRR archive).
# NARR is fetched anonymously from NCEI with no quota, no job
# queue. Replaced the ERA5/CDS pipeline which never produced a
# single row in prod. era5_cds_jobs table is kept for inspection
# but its workers/client are gone. See
# docs/plans/2026-04-15-merra2-historical-backfill.md.
narr: 6,
rtma: 2,
backfill_enqueue: 1,
admin: 1,
nexrad: 2,
radar: 2,
ionosphere: 1,
space_weather: 1,
mechanism: 4,
exports: 1,
contact_import: 4
],
plugins: [
{Oban.Plugins.Pruner, max_age: 3600 * 24},
# DynamicLifeline uses producer records (heartbeats from each
# live node) to rescue orphans, not a timer. When a pod dies
# mid-deploy its Producer row disappears and any in-flight job
# with that producer in `attempted_by` gets moved back to
# `available` on the next rescue cycle — within ~30s instead of
# waiting for a timer-based `rescue_after` to expire. Critical
# for PropagationGridWorker chain steps to recover from rolling
# deploys before the next hourly cron fire.
{Oban.Pro.Plugins.DynamicLifeline, rescue_interval: to_timeout(second: 30)},
{Oban.Plugins.Cron,
crontab: [
{"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker},
{"*/5 * * * *", Microwaveprop.Commercial.PollWorker},
# Hourly at :05. With scores written as binary files (not
# Postgres) and HRRR grid profile persistence removed, a
# full f00-f18 chain runs in ~45-60 min, so hourly fits.
# If one chain slips past 60 min the :propagation queue's
# concurrency-of-2 lets the next chain interleave; files
# are keyed by (band, valid_time) so last-writer-wins gives
# newer analysis data naturally.
{"5 * * * *", Microwaveprop.Workers.PropagationGridWorker},
# GEFS runs publish ~3-4h after the 00/06/12/18Z cycle; seed the
# Day 2-7 outlook 5h after each run to stay safely past NOMADS'
# publication lag. Seeder enqueues f024-f168 at 6-hour cadence
# (25 jobs per run, 100/day) into the :gefs queue.
{"30 5,11,17,23 * * *", Microwaveprop.Workers.GefsFetchWorker},
# MRMS refreshes the PrecipRate cache that AsosAdjustmentWorker
# overlays onto the score grid. Both must run together — an
# ASOS nudge without a fresh MRMS frame reverts to HRRR-only
# rain. Requires wgrib2 built with PNG decoder support (DRT
# 5.41), which the Dockerfile now enables via NCEPLIBS-g2c.
{"*/2 * * * *", Microwaveprop.Workers.MrmsFetchWorker},
# AsosAdjustmentWorker disabled: depends on hrrr_profiles rows
# that PropagationGridWorker no longer persists. The 10-min
# ASOS nudge is a nice-to-have refinement we're not paying the
# full grid-side JSONB write for.
# {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker},
{"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
# The :narr type is virtual: it targets contacts with hrrr_status =
# :unavailable (pre-2014, missing from the HRRR archive) and dispatches
# NarrFetchWorker against NCEI. See narr_jobs_for_contact/1.
{"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker,
args: %{"types" => ["hrrr", "weather", "terrain", "iemre", "narr", "radar", "mechanism"]}},
# Hourly safety net for pos1/pos2/distance_km. Normally every contact
# gets positions at insert time via Radio.resolve_grids_and_insert/1,
# but direct DB writes (manual fixes, bulk imports) can bypass that.
{"0 * * * *", Microwaveprop.Workers.ContactPositionBackfillWorker, args: %{"limit" => 500}},
# UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch
{"30 1 * * *", CanadianSoundingFetchWorker},
{"30 13 * * *", CanadianSoundingFetchWorker},
# GIRO DIDBase publishes foF2/foEs/hmF2 at ~7.5 min cadence; poll
# every 10 min per station (Millstone Hill, Alpena) for the 144
# MHz sporadic-E and HF MUF scoring inputs.
{"*/10 * * * *", Microwaveprop.Workers.IonosphereFetchWorker},
# NOAA SWPC: Kp / GOES X-ray at 1-min cadence, F10.7 hourly.
# 5-min poll balances freshness vs request volume.
{"*/5 * * * *", Microwaveprop.Workers.SpaceWeatherFetchWorker}
]}
]
queues: oban_queues
config :microwaveprop, Oban, plugins: oban_plugins
config :microwaveprop, :email_from, {"NTMS Propagation", "prop@w5isp.com"}
config :microwaveprop, hrrr_base_url: System.get_env("HRRR_BASE_URL", "http://skippy.w5isp.com:8080")
config :microwaveprop, srtm_tiles_dir: "/data/srtm"

113
k8s/deployment-backfill.yaml Normal file
View file

@ -0,0 +1,113 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: prop-backfill
namespace: prop
spec:
# Start with 1. Scale up as additional prop-backfill=true nodes come online
# (podAntiAffinity keys on physical-host, same as prop, so replicas stay
# spread across distinct physical hosts).
replicas: 1
minReadySeconds: 5
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 0
maxUnavailable: 1
selector:
matchLabels:
app: prop
tier: backfill
template:
metadata:
labels:
# app=prop matches the libcluster selector in runtime.exs, so these
# pods join the same Erlang cluster as the hot-path prop pods and
# share Oban's queue ownership / leader election.
app: prop
tier: backfill
spec:
nodeSelector:
prop-backfill: "true"
tolerations:
# Matches the taint recommended for prop-backfill=true nodes. Remove
# this block if you haven't tainted the nodes.
- key: workload
operator: Equal
value: backfill
effect: NoSchedule
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
app: prop
topologyKey: topology.kubernetes.io/physical-host
serviceAccountName: prop
imagePullSecrets:
- name: forgejo-registry
securityContext:
runAsUser: 65534
runAsNonRoot: true
fsGroup: 65534
seccompProfile:
type: RuntimeDefault
containers:
- name: prop-backfill
image: git.mcintire.me/graham/prop:main-1776620433-6f63f3b # {"$imagepolicy": "flux-system:prop"}
imagePullPolicy: IfNotPresent
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
# No PHX_SERVER — endpoint stays inert. No PORT either.
- name: PROP_ROLE
value: "backfill"
- name: PHX_HOST
value: "prop.w5isp.com"
- name: HRRR_BASE_URL
value: "http://skippy.w5isp.com:8080"
envFrom:
- secretRef:
name: prop-secrets
# No HTTP probes (no web server). Use exec probe against the
# running BEAM to confirm the VM is responsive.
startupProbe:
exec:
command: ["/bin/sh", "-c", "/app/bin/microwaveprop rpc ':ok'"]
initialDelaySeconds: 10
periodSeconds: 5
failureThreshold: 24
timeoutSeconds: 5
livenessProbe:
exec:
command: ["/bin/sh", "-c", "/app/bin/microwaveprop rpc ':ok'"]
periodSeconds: 30
failureThreshold: 3
timeoutSeconds: 5
resources:
# 8GB T620 box — leave ~3GB for kubelet/containerd/OS.
requests:
cpu: 100m
memory: 512Mi
limits:
cpu: "2"
memory: 4Gi
securityContext:
allowPrivilegeEscalation: false
runAsNonRoot: true
runAsUser: 65534
capabilities:
drop:
- ALL
seccompProfile:
type: RuntimeDefault
volumeMounts:
- name: data
mountPath: /data
volumes:
- name: data
nfs:
server: 10.0.15.103
path: /data

1
k8s/kustomization.yaml
View file

@ -7,5 +7,6 @@ resources:
- namespace.yaml
- rbac.yaml
- deployment.yaml
- deployment-backfill.yaml
- service.yaml
- metrics-service.yaml