prop/k8s/hpa.yaml

# Horizontal pod autoscalers for the three load-responsive Deployments.
# prop-backfill is deliberately excluded — it's a single-replica catch-up
# worker that's bottlenecked on IEM rate-limit waits rather than CPU, so
# scaling by CPU would just add idle replicas parked on 429 backoffs.
#
# CPU targets use AverageValue (absolute cores) rather than Utilization
# (% of request) so the scale-up thresholds don't depend on the request
# number. Request fields in the Deployments can be tuned independently
# for scheduler bin-packing without accidentally changing when HPA
# decides to add a replica.
#
# The corresponding Deployments omit their spec.replicas so ArgoCD's
# periodic reconcile doesn't fight HPA's live scaling.
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: prop
  namespace: prop
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: prop
  # Floor at 2 pods so a single-pod restart never takes the site down,
  # surge up to 4 under load (LiveView burst + hourly propagation
  # chain). Combined with deployment.yaml's `maxSurge: 1, maxUnavailable: 0`
  # rolling-update strategy, deploys still get an extra pod up before
  # tearing one down.
  minReplicas: 2
  maxReplicas: 4
  metrics:
    - type: Resource
      resource:
        name: cpu
        target:
          # Average of 800m across pods triggers scale-up. The hourly
          # :05 propagation chain + ScoreCache warming on one pod can
          # peak near 2 cores — averaging across 2–3 replicas keeps the
          # threshold meaningful without flapping on every cron tick.
          type: AverageValue
          averageValue: 800m
  behavior:
    scaleUp:
      # React quickly to sudden surges (LiveView + propagation chain
      # can coincide). Add up to 2 pods or 100% of current replica
      # count per 30 s window, whichever is larger.
      stabilizationWindowSeconds: 30
      policies:
        - type: Pods
          value: 2
          periodSeconds: 30
        - type: Percent
          value: 100
          periodSeconds: 30
      selectPolicy: Max
    scaleDown:
      # Slower scale-down so the hourly chain doesn't immediately
      # trigger a churn of terminations five minutes later.
      stabilizationWindowSeconds: 600
      policies:
        - type: Pods
          value: 1
          periodSeconds: 120
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: prop-grid-rs
  namespace: prop
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: prop-grid-rs
  minReplicas: 1
  maxReplicas: 4
  metrics:
    - type: Resource
      resource:
        name: cpu
        target:
          type: AverageValue
          averageValue: 400m
  behavior:
    scaleUp:
      # Genuine surges (a stuck queue, multiple cycles draining together)
      # still bring up replicas within ~30s. Most hourly chain steps
      # finish in 2-3 min on a single pod, so transient single-pod
      # spikes mostly resolve before HPA notices.
      stabilizationWindowSeconds: 30
      policies:
        - type: Pods
          value: 1
          periodSeconds: 30
    scaleDown:
      # The previous 10-minute stabilization + 1-pod-per-2-min drain meant
      # an hourly chain spike that briefly scaled to 4 pods sat at 4 pods
      # for ~16 min after the spike cleared. Bring it down to 2 min
      # stabilization + 100%-per-60s drain so the steady state is 1 pod
      # within ~3 min of any spike ending.
      stabilizationWindowSeconds: 120
      policies:
        - type: Percent
          value: 100
          periodSeconds: 60
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: hrrr-point-rs
  namespace: prop
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: hrrr-point-rs
  minReplicas: 1
  maxReplicas: 4
  metrics:
    - type: Resource
      resource:
        name: cpu
        target:
          type: AverageValue
          averageValue: 400m
  behavior:
    scaleUp:
      # Genuine surges (a stuck queue, multiple cycles draining together)
      # still bring up replicas within ~30s. Most hourly chain steps
      # finish in 2-3 min on a single pod, so transient single-pod
      # spikes mostly resolve before HPA notices.
      stabilizationWindowSeconds: 30
      policies:
        - type: Pods
          value: 1
          periodSeconds: 30
    scaleDown:
      # The previous 10-minute stabilization + 1-pod-per-2-min drain meant
      # an hourly chain spike that briefly scaled to 4 pods sat at 4 pods
      # for ~16 min after the spike cleared. Bring it down to 2 min
      # stabilization + 100%-per-60s drain so the steady state is 1 pod
      # within ~3 min of any spike ending.
      stabilizationWindowSeconds: 120
      policies:
        - type: Percent
          value: 100
          periodSeconds: 60