prop/lib/microwaveprop/application.ex
Graham McIntire 4f82bd691e
perf(prop): cap ScoreCache LRU + drop eager warmers to fix hot-pod OOMs
ScoreCache held 437 entries × ~1.94 MiB each (~850 MiB per pod) of
{band_mhz, valid_time} grids — data already on NFS as compact .prop
files. NotifyListener and ScoreCacheReconciler both eagerly materialised
every band into ETS on each Rust completion / 60s sweep, so 5 hot
replicas wasted ~4.2 GiB of redundant cache and OOMed at 6 GiB.

Bound the cache to 32 entries with eviction by oldest valid_time, drop
the eager warm loops, and let LiveView callers lazy-fill via the
existing ScoresFile read path. ScoreCacheReconciler had no remaining
purpose and is deleted; runbook + prom_ex counters updated to match.

Steady-state cache footprint ~60 MiB per pod instead of ~850 MiB.
2026-04-24 18:49:36 -05:00

123 lines
4.4 KiB
Elixir

defmodule Microwaveprop.Application do
# See https://hexdocs.pm/elixir/Application.html
# for more information on OTP Applications
@moduledoc false
use Application
@build_timestamp DateTime.utc_now()
@impl true
def start(_type, _args) do
topologies = Application.get_env(:libcluster, :topologies, [])
children = [
MicrowavepropWeb.Telemetry,
Microwaveprop.PromEx,
Microwaveprop.Repo,
{Cluster.Supervisor, [topologies, [name: Microwaveprop.ClusterSupervisor]]},
{Phoenix.PubSub, name: Microwaveprop.PubSub},
# Partitioned Task.Supervisor — callers route heavy `async_stream`
# work through `{:via, PartitionSupervisor, {Microwaveprop.TaskSupervisor,
# self()}}` so concurrent workers (recalibrator, HRRR/GEFS range
# downloads, GEFS scoring) do not serialize on a single supervisor PID.
{PartitionSupervisor,
child_spec: Task.Supervisor, name: Microwaveprop.TaskSupervisor, partitions: System.schedulers_online()},
Microwaveprop.Cache,
Microwaveprop.Propagation.ScoreCache,
Microwaveprop.Weather.GridCache,
{Microwaveprop.Weather.IemRateLimiter,
interval_ms: Application.get_env(:microwaveprop, :iem_rate_limiter_interval_ms, 700)},
Microwaveprop.Weather.NexradCache,
Microwaveprop.Qrz.Client,
{Oban, Application.fetch_env!(:microwaveprop, Oban)},
Microwaveprop.RepoListener,
# Start to serve requests, typically the last entry
MicrowavepropWeb.Endpoint,
Microwaveprop.Propagation.FreshnessMonitor
]
# See https://hexdocs.pm/elixir/Supervisor.html
# for other strategies and supported options
opts = [strategy: :one_for_one, name: Microwaveprop.Supervisor]
result = Supervisor.start_link(children, opts)
# Attach the Oban exception telemetry tap so per-worker recovery
# callbacks and structured logs fire on job failure. Must run after
# the Oban supervisor starts (above in `children`).
:ok = Microwaveprop.ObanErrorReporter.attach()
# Run migrations after Repo is started
_ = Microwaveprop.Release.migrate()
# Warm GridCache from the latest persisted ProfilesFile so /weather
# has data immediately after a pod restart. Runs after the GridCache
# GenServer is up (it's in `children` above).
#
# Must run in a short-lived process, not inline in the application
# master: when executed here the loaded rows live on the app master's
# heap and never GC (it's idle after boot), pinning ~800 MiB for the
# lifetime of the pod.
{:ok, _pid} = Task.start(fn -> Microwaveprop.Weather.warm_grid_cache_from_latest_profile() end)
# Load ML model in dev/test only (Nx/Axon not compiled for prod)
if Application.get_env(:microwaveprop, :load_ml_model, false) do
Microwaveprop.Propagation.load_ml_model()
end
result
end
@impl true
def config_change(changed, _new, removed) do
MicrowavepropWeb.Endpoint.config_change(changed, removed)
:ok
end
@doc """
Returns the deployment (image build) timestamp.
Sources, in priority order:
1. The `/app/BUILD_TIMESTAMP` file baked into the Docker image at build
time (the CI pipeline passes `--build-arg BUILD_TIMESTAMP=...` in the
final stage, so this value is fresh on every image and is not affected
by earlier-stage layer caching). The path is overridable via
`config :microwaveprop, :build_timestamp_file` for tests.
2. The `DEPLOY_TIMESTAMP` environment variable, for environments that set
it imperatively (e.g. `kubectl set env`).
3. The compile-time fallback captured when this module was compiled.
All accepted values are ISO 8601, e.g. `2026-04-17T18:30:00Z`.
"""
@spec build_timestamp() :: DateTime.t()
def build_timestamp do
with :error <- from_file(),
:error <- from_env() do
@build_timestamp
end
end
defp from_file do
path = Application.get_env(:microwaveprop, :build_timestamp_file, "/app/BUILD_TIMESTAMP")
case File.read(path) do
{:ok, contents} -> parse_iso8601(contents)
{:error, _} -> :error
end
end
defp from_env do
case System.get_env("DEPLOY_TIMESTAMP") do
nil -> :error
string -> parse_iso8601(string)
end
end
defp parse_iso8601(string) do
case string |> String.trim() |> DateTime.from_iso8601() do
{:ok, dt, _offset} -> dt
{:error, _} -> :error
end
end
end