Graham McIntire
022e6928bc
Replaces the serial claim→process loop with a JoinSet of N worker tasks, each running the same claim→fetch→decode→score→complete flow against grid_tasks. FOR UPDATE SKIP LOCKED already prevents contention between workers so no additional coordination is needed. talos5 is 4c/4t (i5-7260U); PROP_GRID_RS_PARALLELISM=4 turns the full f01..f18 chain from ~18 min serial into roughly ~5 min. Per-task peak memory after the streamed-bands refactor is ~250 Mi, so 4 concurrent is ~1 Gi — well under the 2 Gi container limit. Postgres pool sized to parallelism + 2 so NOTIFY and an opportunistic claim retry share the pool without blocking a worker transaction.
109 lines
3.9 KiB
YAML
109 lines
3.9 KiB
YAML
apiVersion: apps/v1
|
||
kind: Deployment
|
||
metadata:
|
||
name: prop-grid-rs
|
||
namespace: prop
|
||
# Phase 2 cutover: Rust writes directly to /data/scores. Elixir only
|
||
# runs the f00 analysis-hour step (which carries enrichment Rust
|
||
# doesn't own yet — native-level duct merge, NEXRAD composite,
|
||
# commercial-link degradation, ProfilesFile write). f01..f18 land
|
||
# here from the Rust worker.
|
||
spec:
|
||
replicas: 1
|
||
minReadySeconds: 5
|
||
strategy:
|
||
type: RollingUpdate
|
||
rollingUpdate:
|
||
maxSurge: 0
|
||
maxUnavailable: 1
|
||
selector:
|
||
matchLabels:
|
||
app: prop-grid-rs
|
||
template:
|
||
metadata:
|
||
labels:
|
||
app: prop-grid-rs
|
||
tier: grid-rs
|
||
spec:
|
||
# Pin to talos5 — the 32 GB NUC with NVMe. The worker peaks at ~500 Mi
|
||
# steady-state but benefits from local NVMe for the NFS scores-dir
|
||
# writes (faster fsync) and from never sharing the node with a hot
|
||
# Elixir pod. Label talos5 with `prop-grid-rs=primary` and taint it
|
||
# `workload=grid-rs:NoSchedule` to enforce this.
|
||
nodeSelector:
|
||
prop-grid-rs: "primary"
|
||
tolerations:
|
||
- key: workload
|
||
operator: Equal
|
||
value: grid-rs
|
||
effect: NoSchedule
|
||
imagePullSecrets:
|
||
- name: forgejo-registry
|
||
securityContext:
|
||
runAsUser: 65534
|
||
runAsNonRoot: true
|
||
fsGroup: 65534
|
||
seccompProfile:
|
||
type: RuntimeDefault
|
||
containers:
|
||
- name: prop-grid-rs
|
||
# Image built by a new CI pipeline from rust/prop_grid_rs/.
|
||
# Multi-arch (amd64 + arm64) — reuses the repo's existing
|
||
# buildx wiring.
|
||
image: git.mcintire.me/graham/prop-grid-rs:main-1776635096-6d91461 # {"$imagepolicy": "flux-system:prop-grid-rs"}
|
||
imagePullPolicy: IfNotPresent
|
||
env:
|
||
# Elixir secrets bundle carries DATABASE_URL already. Rust
|
||
# reads it directly (sqlx connects with the same URL).
|
||
- name: HRRR_BASE_URL
|
||
value: "http://skippy.w5isp.com:8080"
|
||
- name: PROP_SCORES_DIR
|
||
value: "/data/scores"
|
||
- name: RUST_LOG
|
||
value: "info"
|
||
# talos5 is 4c/4t (i5-7260U). Claiming 4 forecast hours in
|
||
# parallel drops full-chain wall time from ~18 min serial to
|
||
# ~5 min. Each concurrent task peaks around 250 Mi after the
|
||
# streamed-bands refactor; 4×250 ≈ 1 Gi is comfortably under
|
||
# the 2 Gi memory limit.
|
||
- name: PROP_GRID_RS_PARALLELISM
|
||
value: "4"
|
||
# Pool size = parallelism + 2 (NOTIFY + retry slack). Kept in
|
||
# sync with PROP_GRID_RS_PARALLELISM when either changes.
|
||
- name: PROP_GRID_RS_PG_CONNS
|
||
value: "6"
|
||
envFrom:
|
||
- secretRef:
|
||
name: prop-secrets
|
||
# No HTTP surface. Liveness is "the process is still running";
|
||
# let the kubelet restart us if we crash. Claiming work from
|
||
# grid_tasks is the implicit readiness signal — an idle pod is
|
||
# still healthy (the queue is simply empty).
|
||
resources:
|
||
requests:
|
||
cpu: 100m
|
||
memory: 512Mi
|
||
limits:
|
||
cpu: "4"
|
||
# OOM at 1 Gi during active chain work (92k points × 23
|
||
# bands ≈ 2.1M scores, plus decoded GRIB2 buffers, plus
|
||
# wgrib2 subprocess memory). 2 Gi leaves headroom; we're
|
||
# still an order of magnitude below the 6 Gi Elixir budget.
|
||
memory: 2Gi
|
||
securityContext:
|
||
allowPrivilegeEscalation: false
|
||
runAsNonRoot: true
|
||
runAsUser: 65534
|
||
capabilities:
|
||
drop:
|
||
- ALL
|
||
seccompProfile:
|
||
type: RuntimeDefault
|
||
volumeMounts:
|
||
- name: data
|
||
mountPath: /data
|
||
volumes:
|
||
- name: data
|
||
nfs:
|
||
server: 10.0.15.103
|
||
path: /data
|