prop/lib/microwaveprop/application.ex
Graham McIntire 9edaee9344
feat(weather): GridCache backend pluggable, Valkey for cross-pod cache
Each cached HRRR valid_time is ~32 MiB compressed × cap of 8 = up to
256 MiB per pod. With 4 hot pods + 1 backfill that's ~1.25 GiB of
cluster RAM holding identical content. Replacing the per-pod ETS
replicas with a single shared Valkey copy collapses that to ~256 MiB
total (off-pod) and makes the BEAM heap on each replica meaningfully
smaller — directly addresses the headroom the 2026-05-03 OOM cascade
ate into.

* New `Microwaveprop.Valkey`: single Redix connection (named conn,
  sync_connect: false, exit_on_disconnection: false). `start_link/0`
  returns `:ignore` when VALKEY_URL is unset so dev/test boot without
  Valkey. Helpers wrap GET/MGET/SET-with-TTL/ZADD/ZREVRANGE/SCAN with
  `:erlang.term_to_binary` round-tripping under the safe atom flag.

* `Microwaveprop.Weather.GridCache` now branches on
  `Valkey.configured?/0`. Public API is identical so callers (Weather,
  GridCachePruneWorker, MapLive) don't change. ETS path is preserved
  unchanged for dev/test.

* Storage layout when on Valkey:
    prop:wg:vts                   ZSET of valid_time iso8601 strings
    prop:wg:<vt>:<lat>:<lon>      one chunk per 5°×5° spatial bucket
  fetch_bounds MGETs only the chunks intersecting the viewport, so a
  regional view is 1-4 round-trips instead of pulling the full grid.

* Per-key TTL = 8 h (matches the ETS valid_time cap). Eviction is
  automatic; prune_keep_latest is a no-op on Valkey, prune_older_than
  uses ZRANGEBYSCORE + DEL.

* All Valkey errors fall back to disk (ScalarFile.read_bounds), logged
  as warnings — page works through Valkey outages, no user-visible
  break beyond a one-shot latency bump.

ScoreCache, Microwaveprop.Cache, NexradCache, telemetry/Postgres
protocol/libcluster registry tables remain ETS — they're sub-ms
hot-path lookups where a network hop would be felt.

VALKEY_URL must be present in prop-secrets for production rollout
(applied out-of-band, secret.yaml is git-ignored).
2026-05-03 17:00:51 -05:00

167 lines
6.2 KiB
Elixir

defmodule Microwaveprop.Application do
# See https://hexdocs.pm/elixir/Application.html
# for more information on OTP Applications
@moduledoc false
use Application
require Logger
@build_timestamp DateTime.utc_now()
@impl true
def start(_type, _args) do
topologies = Application.get_env(:libcluster, :topologies, [])
children = [
MicrowavepropWeb.Telemetry,
Microwaveprop.PromEx,
Microwaveprop.Repo,
# AprsRepo is optional — only starts when configured with a :url
# (dev/test from config blocks, prod from APRS_DATABASE_URL). Missing
# config logs a warning and skips startup so the app still boots.
aprs_repo_child_spec(),
{Cluster.Supervisor, [topologies, [name: Microwaveprop.ClusterSupervisor]]},
{Phoenix.PubSub, name: Microwaveprop.PubSub},
# Partitioned Task.Supervisor — callers route heavy `async_stream`
# work through `{:via, PartitionSupervisor, {Microwaveprop.TaskSupervisor,
# self()}}` so concurrent workers (recalibrator, HRRR/GEFS range
# downloads, GEFS scoring) do not serialize on a single supervisor PID.
{PartitionSupervisor,
child_spec: Task.Supervisor, name: Microwaveprop.TaskSupervisor, partitions: System.schedulers_online()},
# Valkey connection — must start before GridCache so the cache
# backend selector sees an alive Redix process. Returns :ignore
# when VALKEY_URL is unset (dev/test), and GridCache falls back
# to its ETS implementation.
Microwaveprop.Valkey,
Microwaveprop.Cache,
Microwaveprop.Buildings.Index,
Microwaveprop.Propagation.ScoreCache,
Microwaveprop.Weather.GridCache,
{Microwaveprop.Weather.IemRateLimiter,
interval_ms: Application.get_env(:microwaveprop, :iem_rate_limiter_interval_ms, 700)},
Microwaveprop.Weather.NexradCache,
Microwaveprop.Qrz.Client,
{Oban, Application.fetch_env!(:microwaveprop, Oban)},
Microwaveprop.RepoListener,
# Start to serve requests, typically the last entry
MicrowavepropWeb.Endpoint,
Microwaveprop.Propagation.FreshnessMonitor
]
# See https://hexdocs.pm/elixir/Supervisor.html
# for other strategies and supported options
opts = [strategy: :one_for_one, name: Microwaveprop.Supervisor]
result = children |> Enum.reject(&is_nil/1) |> Supervisor.start_link(opts)
# Attach the Oban exception telemetry tap so per-worker recovery
# callbacks and structured logs fire on job failure. Must run after
# the Oban supervisor starts (above in `children`).
:ok = Microwaveprop.ObanErrorReporter.attach()
# Run migrations after Repo is started
_ = Microwaveprop.Release.migrate()
# Warm GridCache from the latest persisted ProfilesFile so /weather
# has data immediately after a pod restart. Runs after the GridCache
# GenServer is up (it's in `children` above).
#
# Must run in a short-lived process, not inline in the application
# master: when executed here the loaded rows live on the app master's
# heap and never GC (it's idle after boot), pinning ~800 MiB for the
# lifetime of the pod.
{:ok, _pid} = Task.start(fn -> Microwaveprop.Weather.warm_grid_cache_from_latest_profile() end)
# One-shot backfill: enqueue a QRZ-lookup job for every user with a
# missing home QTH. Idempotent — the worker no-ops if home_grid is
# already set, so re-running on every boot is safe (and useful: any
# users registered while the worker module was unavailable get
# picked up the next time the pod restarts).
{:ok, _pid} =
Task.start(fn ->
try do
Microwaveprop.Accounts.backfill_missing_home_qth()
rescue
e ->
Logger.error("Application: backfill_missing_home_qth crashed: #{Exception.format(:error, e, __STACKTRACE__)}")
end
end)
# Load ML model in dev/test only (Nx/Axon not compiled for prod)
if Application.get_env(:microwaveprop, :load_ml_model, false) do
Microwaveprop.Propagation.load_ml_model()
end
result
end
@impl true
def config_change(changed, _new, removed) do
MicrowavepropWeb.Endpoint.config_change(changed, removed)
:ok
end
# Returns Microwaveprop.AprsRepo child spec if configured (has a :url
# or :database key), otherwise logs a warning and returns nil so the
# supervisor children list can filter it out. APRS calibration is an
# optional integration with the sister aprs.me database — we never
# crash boot when it's unavailable.
defp aprs_repo_child_spec do
config = Application.get_env(:microwaveprop, Microwaveprop.AprsRepo, [])
if Keyword.has_key?(config, :url) or Keyword.has_key?(config, :database) do
Microwaveprop.AprsRepo
else
Logger.warning("Microwaveprop.AprsRepo not configured (no :url or :database) — APRS calibration disabled.")
nil
end
end
@doc """
Returns the deployment (image build) timestamp.
Sources, in priority order:
1. The `/app/BUILD_TIMESTAMP` file baked into the Docker image at build
time (the CI pipeline passes `--build-arg BUILD_TIMESTAMP=...` in the
final stage, so this value is fresh on every image and is not affected
by earlier-stage layer caching). The path is overridable via
`config :microwaveprop, :build_timestamp_file` for tests.
2. The `DEPLOY_TIMESTAMP` environment variable, for environments that set
it imperatively (e.g. `kubectl set env`).
3. The compile-time fallback captured when this module was compiled.
All accepted values are ISO 8601, e.g. `2026-04-17T18:30:00Z`.
"""
@spec build_timestamp() :: DateTime.t()
def build_timestamp do
with :error <- from_file(),
:error <- from_env() do
@build_timestamp
end
end
defp from_file do
path = Application.get_env(:microwaveprop, :build_timestamp_file, "/app/BUILD_TIMESTAMP")
case File.read(path) do
{:ok, contents} -> parse_iso8601(contents)
{:error, _} -> :error
end
end
defp from_env do
case System.get_env("DEPLOY_TIMESTAMP") do
nil -> :error
string -> parse_iso8601(string)
end
end
defp parse_iso8601(string) do
case string |> String.trim() |> DateTime.from_iso8601() do
{:ok, dt, _offset} -> dt
{:error, _} -> :error
end
end
end