Graham McIntire
024db2a5b4
Per-pod IEM rate limiter was 700ms (~1.4 req/sec); four pods put the fleet at ~5.7 req/sec, above the observed 429 threshold. Retry storm (Req exponential backoff up to 32s) piled concurrent sleeps on top of live workers and tripped Bandit's acceptor + Oban's leader heartbeat. - Raise default interval to 1500ms (~2.7 req/sec fleet). - Hot pod /live timeout 3s → 5s (acceptor stalls under retry sleeps). - prop-backfill exec probe period 30s → 120s, timeout 15s → 30s; the release CLI round-trips through distributed Erlang and 15s wasn't enough under load, restart-looping every ~10 min.
121 lines
4 KiB
YAML
121 lines
4 KiB
YAML
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: prop-backfill
|
|
namespace: prop
|
|
spec:
|
|
# Start with 1. Scale up as additional prop-backfill=true nodes come online
|
|
# (podAntiAffinity keys on physical-host, same as prop, so replicas stay
|
|
# spread across distinct physical hosts).
|
|
replicas: 1
|
|
minReadySeconds: 5
|
|
strategy:
|
|
type: RollingUpdate
|
|
rollingUpdate:
|
|
maxSurge: 0
|
|
maxUnavailable: 1
|
|
selector:
|
|
matchLabels:
|
|
app: prop
|
|
tier: backfill
|
|
template:
|
|
metadata:
|
|
labels:
|
|
# app=prop matches the libcluster selector in runtime.exs, so these
|
|
# pods join the same Erlang cluster as the hot-path prop pods and
|
|
# share Oban's queue ownership / leader election.
|
|
app: prop
|
|
tier: backfill
|
|
spec:
|
|
nodeSelector:
|
|
prop-backfill: "true"
|
|
tolerations:
|
|
# Matches the taint recommended for prop-backfill=true nodes. Remove
|
|
# this block if you haven't tainted the nodes.
|
|
- key: workload
|
|
operator: Equal
|
|
value: backfill
|
|
effect: NoSchedule
|
|
affinity:
|
|
podAntiAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
- labelSelector:
|
|
matchLabels:
|
|
app: prop
|
|
topologyKey: topology.kubernetes.io/physical-host
|
|
serviceAccountName: prop
|
|
imagePullSecrets:
|
|
- name: forgejo-registry
|
|
securityContext:
|
|
runAsUser: 65534
|
|
runAsNonRoot: true
|
|
fsGroup: 65534
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
containers:
|
|
- name: prop-backfill
|
|
image: git.mcintire.me/graham/prop:main-1776884183-04146af # {"$imagepolicy": "flux-system:prop"}
|
|
imagePullPolicy: IfNotPresent
|
|
env:
|
|
- name: POD_IP
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: status.podIP
|
|
# No PHX_SERVER — endpoint stays inert. No PORT either.
|
|
- name: PROP_ROLE
|
|
value: "backfill"
|
|
- name: PHX_HOST
|
|
value: "prop.w5isp.com"
|
|
- name: HRRR_BASE_URL
|
|
value: "http://skippy.w5isp.com:8080"
|
|
# OTLP disabled — see deployment.yaml for rationale.
|
|
envFrom:
|
|
- secretRef:
|
|
name: prop-secrets
|
|
# No HTTP probes (no web server). Use exec probe against the
|
|
# running BEAM to confirm the VM is responsive. `rpc ':ok'`
|
|
# has to spawn a release CLI and round-trip through distributed
|
|
# Erlang; during an IEM 429 retry storm the scheduler is
|
|
# responsive but slow. 15 s still tripped failureThreshold
|
|
# every ~10 min, so the liveness probe now runs every 2 min
|
|
# with a 30 s ceiling — the BEAM itself would have crashed
|
|
# long before that if it were genuinely wedged, and rescue
|
|
# relies on the BEAM's own crash-loop handling anyway.
|
|
startupProbe:
|
|
exec:
|
|
command: ["/bin/sh", "-c", "/app/bin/microwaveprop rpc ':ok'"]
|
|
initialDelaySeconds: 10
|
|
periodSeconds: 10
|
|
failureThreshold: 18
|
|
timeoutSeconds: 30
|
|
livenessProbe:
|
|
exec:
|
|
command: ["/bin/sh", "-c", "/app/bin/microwaveprop rpc ':ok'"]
|
|
periodSeconds: 120
|
|
failureThreshold: 3
|
|
timeoutSeconds: 30
|
|
resources:
|
|
# 8GB T620 box — leave ~3GB for kubelet/containerd/OS.
|
|
requests:
|
|
cpu: 100m
|
|
memory: 512Mi
|
|
limits:
|
|
cpu: "2"
|
|
memory: 4Gi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
runAsNonRoot: true
|
|
runAsUser: 65534
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
volumeMounts:
|
|
- name: data
|
|
mountPath: /data
|
|
volumes:
|
|
- name: data
|
|
nfs:
|
|
server: 10.0.15.103
|
|
path: /data
|