Graham McIntire
4fefb81c24
Phase 3 Stream C Rust side. Completes the HrrrFetchWorker port. Pipeline: - db::claim_next_hrrr_task — FOR UPDATE SKIP LOCKED on hrrr_fetch_tasks, newest valid_time first. Accepts the points JSONB directly. - hrrr_points::process_batch — fetch surface + pressure GRIB2 once per task (tokio::try_join), decode via the existing wgrib2 plumbing, then for each requested point pull the cell and UPSERT INTO hrrr_profiles (conflict on lat/lon/valid_time). - db::complete_hrrr_task / fail_hrrr_task — status transitions; Elixir backfill re-enqueues failed rows on next /30-min scan. Shipping pieces: - new bin src/bin/hrrr_point_worker.rs - new module src/hrrr_points.rs (process_batch, upsert_profile) - new Cargo [[bin]] entry; Dockerfile builds both binaries in one stage and ships them in the runtime image so a single CI pipeline covers the whole cluster - k8s/deployment-hrrr-point-rs.yaml (1 replica, 1 Gi limit, anti-affinity against prop-grid-rs so chain + point work don't fight for wgrib2 slots). Uses the same image; command: override picks the right binary. - kustomization.yaml: include the new deployment so flux applies it - deployment-grid-rs.yaml: bump readiness initialDelaySeconds 3→15 + failureThreshold 3→6 so a slow DB connect during startup can't race the first probe 119 Rust tests green.
84 lines
2.8 KiB
YAML
84 lines
2.8 KiB
YAML
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: hrrr-point-rs
|
|
namespace: prop
|
|
# Phase 3 Stream C: per-QSO HRRR fetches move off the BEAM pods.
|
|
# Elixir writes to hrrr_fetch_tasks (one row per valid_time with a
|
|
# points JSONB array); this deployment drains via FOR UPDATE SKIP
|
|
# LOCKED, fetches each HRRR cycle once, and upserts hrrr_profiles.
|
|
spec:
|
|
replicas: 1
|
|
strategy:
|
|
type: RollingUpdate
|
|
rollingUpdate:
|
|
maxSurge: 0
|
|
maxUnavailable: 1
|
|
selector:
|
|
matchLabels:
|
|
app: hrrr-point-rs
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: hrrr-point-rs
|
|
tier: hrrr-points
|
|
spec:
|
|
affinity:
|
|
# Avoid co-locating with prop-grid-rs pods — they compete for
|
|
# the same wgrib2 subprocess slots and NFS fsyncs. Grid chain
|
|
# is latency-sensitive (users refreshing /map); per-QSO point
|
|
# work is throughput-sensitive but latency-tolerant.
|
|
podAntiAffinity:
|
|
preferredDuringSchedulingIgnoredDuringExecution:
|
|
- weight: 80
|
|
podAffinityTerm:
|
|
labelSelector:
|
|
matchLabels:
|
|
app: prop-grid-rs
|
|
topologyKey: kubernetes.io/hostname
|
|
imagePullSecrets:
|
|
- name: forgejo-registry
|
|
securityContext:
|
|
runAsUser: 65534
|
|
runAsNonRoot: true
|
|
fsGroup: 65534
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
containers:
|
|
- name: hrrr-point-rs
|
|
# Shares the prop-grid-rs image so one CI pipeline produces
|
|
# both binaries. k8s `command:` overrides the default
|
|
# ENTRYPOINT to launch the per-QSO point worker.
|
|
image: git.mcintire.me/graham/prop-grid-rs:main-1776635096-6d91461 # {"$imagepolicy": "flux-system:prop-grid-rs"}
|
|
imagePullPolicy: IfNotPresent
|
|
command: ["/usr/bin/tini", "--", "/usr/local/bin/hrrr-point-worker"]
|
|
env:
|
|
- name: HRRR_BASE_URL
|
|
value: "http://skippy.w5isp.com:8080"
|
|
- name: RUST_LOG
|
|
value: "info"
|
|
- name: PROP_GRID_RS_PG_CONNS
|
|
value: "4"
|
|
envFrom:
|
|
- secretRef:
|
|
name: prop-secrets
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 256Mi
|
|
limits:
|
|
# Per-batch peak is a single CONUS surface + pressure
|
|
# decode (~40 MB blob, ~200 MB wgrib2 working set plus
|
|
# the merged CellValues map). 1 Gi gives headroom with
|
|
# room for occasional retries.
|
|
cpu: "2"
|
|
memory: 1Gi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
runAsNonRoot: true
|
|
runAsUser: 65534
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
seccompProfile:
|
|
type: RuntimeDefault
|