prop/mix.exs
Graham McIntire 9edaee9344
feat(weather): GridCache backend pluggable, Valkey for cross-pod cache
Each cached HRRR valid_time is ~32 MiB compressed × cap of 8 = up to
256 MiB per pod. With 4 hot pods + 1 backfill that's ~1.25 GiB of
cluster RAM holding identical content. Replacing the per-pod ETS
replicas with a single shared Valkey copy collapses that to ~256 MiB
total (off-pod) and makes the BEAM heap on each replica meaningfully
smaller — directly addresses the headroom the 2026-05-03 OOM cascade
ate into.

* New `Microwaveprop.Valkey`: single Redix connection (named conn,
  sync_connect: false, exit_on_disconnection: false). `start_link/0`
  returns `:ignore` when VALKEY_URL is unset so dev/test boot without
  Valkey. Helpers wrap GET/MGET/SET-with-TTL/ZADD/ZREVRANGE/SCAN with
  `:erlang.term_to_binary` round-tripping under the safe atom flag.

* `Microwaveprop.Weather.GridCache` now branches on
  `Valkey.configured?/0`. Public API is identical so callers (Weather,
  GridCachePruneWorker, MapLive) don't change. ETS path is preserved
  unchanged for dev/test.

* Storage layout when on Valkey:
    prop:wg:vts                   ZSET of valid_time iso8601 strings
    prop:wg:<vt>:<lat>:<lon>      one chunk per 5°×5° spatial bucket
  fetch_bounds MGETs only the chunks intersecting the viewport, so a
  regional view is 1-4 round-trips instead of pulling the full grid.

* Per-key TTL = 8 h (matches the ETS valid_time cap). Eviction is
  automatic; prune_keep_latest is a no-op on Valkey, prune_older_than
  uses ZRANGEBYSCORE + DEL.

* All Valkey errors fall back to disk (ScalarFile.read_bounds), logged
  as warnings — page works through Valkey outages, no user-visible
  break beyond a one-shot latency bump.

ScoreCache, Microwaveprop.Cache, NexradCache, telemetry/Postgres
protocol/libcluster registry tables remain ETS — they're sub-ms
hot-path lookups where a network hop would be felt.

VALKEY_URL must be present in prop-secrets for production rollout
(applied out-of-band, secret.yaml is git-ignored).
2026-05-03 17:00:51 -05:00

222 lines
7.9 KiB
Elixir

defmodule Microwaveprop.MixProject do
use Mix.Project
def project do
[
app: :microwaveprop,
version: "0.1.0",
elixir: "~> 1.15",
elixirc_paths: elixirc_paths(Mix.env()),
start_permanent: Mix.env() == :prod,
aliases: aliases(),
deps: deps(),
compilers: [:phoenix_live_view] ++ Mix.compilers(),
listeners: [Phoenix.CodeReloader],
dialyzer: [
plt_add_apps: [:mix, :ex_unit],
plt_file: {:no_warn, "priv/plts/project.plt"},
flags: [
:error_handling,
:unknown,
:unmatched_returns,
:extra_return,
:missing_return
]
]
]
end
# Configuration for the OTP application.
#
# Type `mix help compile.app` for more information.
def application do
[
mod: {Microwaveprop.Application, []},
extra_applications: [:logger, :runtime_tools, :os_mon, :xmerl]
]
end
def cli do
[
preferred_envs: [precommit: :test]
]
end
# Specifies which paths to compile per environment.
defp elixirc_paths(:test), do: ["lib", "lib_ml", "test/support"]
defp elixirc_paths(:dev), do: ["lib", "lib_ml"]
defp elixirc_paths(_), do: ["lib"]
# Specifies your project dependencies.
#
# Type `mix help deps` for examples and options.
defp deps do
[
{:bcrypt_elixir, "~> 3.0"},
{:phoenix, "~> 1.8.3"},
{:phoenix_ecto, "~> 4.5"},
{:ecto_sql, "~> 3.13"},
{:postgrex, ">= 0.0.0"},
{:phoenix_html, "~> 4.1"},
{:phoenix_live_reload, "~> 1.2", only: :dev},
{:phoenix_live_view, "~> 1.1.0"},
{:lazy_html, ">= 0.1.0", only: :test},
{:stream_data, "~> 1.0", only: :test},
{:live_stash, "~> 0.2"},
{:phoenix_live_dashboard, "~> 0.8.3"},
{:esbuild, "~> 0.10", runtime: Mix.env() == :dev},
{:tailwind, "~> 0.3", runtime: Mix.env() == :dev},
{:heroicons,
github: "tailwindlabs/heroicons", tag: "v2.2.0", sparse: "optimized", app: false, compile: false, depth: 1},
{:swoosh, "~> 1.16"},
{:gen_smtp, "~> 1.2"},
{:req, "~> 0.5"},
{:telemetry_metrics, "~> 1.0"},
{:telemetry_poller, "~> 1.0"},
{:prom_ex, "~> 1.11"},
{:jason, "~> 1.2"},
# Structured JSON log formatter plugged into Elixir 1.15+ :logger.
# OTP 28 / Elixir 1.19 ships `Logger.Formatter.new/1` but no built-in
# JSON encoder, so the hex package fills the gap. Prod flips to
# LOG_FORMAT=json; dev/test stay on the default text formatter.
{:logger_json, "~> 7.0"},
{:msgpax, "~> 2.4"},
{:bandit, "~> 1.5"},
{:styler, "~> 1.11", only: [:dev, :test], runtime: false},
{:oban, "~> 2.21"},
# All three Oban Pro packages stay vendored so production deploys
# don't have to authenticate against the licensed oban.pro hex
# repo on every image build. Bumping a version is a fetch-and-
# extract step (`mix hex.package fetch oban_pro <ver> --repo=oban
# --unpack`) into vendor/, then commit.
{:oban_met, "~> 1.0", path: "vendor/oban_met", override: true},
{:oban_pro, "~> 1.7", path: "vendor/oban_pro"},
# Note: override: true on oban_web is required because live_table
# transitively pulls `oban_web ~> 2.11` from hexpm; without the
# override mix tries to resolve from hex and fails. towerops-web
# doesn't have live_table so it doesn't need the override.
{:oban_web, "~> 2.12", path: "vendor/oban_web", override: true},
{:credo, "~> 1.7", only: [:dev, :test], runtime: false},
{:nx, "~> 0.9", only: [:dev, :test]},
{:axon, "~> 0.7", only: [:dev, :test]},
{:exla, "~> 0.9", only: [:dev, :test]},
{:polaris, "~> 0.1", only: [:dev, :test]},
{:tidewave, "~> 0.5", only: :dev},
{:libcluster, "~> 3.4"},
# Cross-pod cache backend for Microwaveprop.Weather.GridCache.
# Talks to the Valkey/Redis instance at VALKEY_URL; falls back to
# the legacy ETS implementation when unset (dev/test).
{:redix, "~> 1.5"},
{:ecto_psql_extras, "~> 0.8"},
{:dialyxir, "~> 1.4", only: [:dev, :test], runtime: false},
{:live_table, "~> 0.4.1"},
# Transitive dep of live_table (via LiveTable.TableComponent's
# generated code pulls in SutraUI.DropdownMenu). Pinned here
# because live_table doesn't list it in its own deps.
{:sutra_ui, "~> 0.3.0"},
# Required as a compile-time build dependency for live_table; its mix
# tasks (lib/mix/tasks/live_table.gen.live.ex, live_table.install.ex)
# unconditionally `use Igniter.Mix.Task` so the dep has to exist in
# every env live_table is compiled in, not just dev/test. runtime: false
# keeps it out of the release bundle.
{:igniter, "~> 0.7", runtime: false}
]
end
# Aliases are shortcuts or tasks specific to the current project.
# For example, to install project dependencies and perform other setup tasks, run:
#
# $ mix setup
#
# See the documentation for `Mix` for more info on aliases.
defp aliases do
[
setup: ["deps.get", "ecto.setup", "assets.setup", "assets.build"],
"deps.get": ["deps.get", &apply_dep_patches/1],
"ecto.setup": ["ecto.create", "ecto.migrate", "run priv/repo/seeds.exs"],
"ecto.reset": ["ecto.drop", "ecto.setup"],
test: ["ecto.create --quiet", "ecto.migrate --quiet", "test"],
"assets.setup": ["tailwind.install --if-missing", "esbuild.install --if-missing"],
"assets.build": [
"compile",
"tailwind microwaveprop",
"esbuild microwaveprop",
&copy_leaflet_images/1
],
"assets.deploy": [
"tailwind microwaveprop --minify",
"esbuild microwaveprop --minify",
&copy_leaflet_images/1,
"phx.digest"
],
precommit: ["compile --warnings-as-errors", "deps.unlock --unused", "format", "test"]
]
end
# Mirrors `assets/vendor/leaflet/images/` into `priv/static/assets/css/images/`
# so Leaflet's CSS (which references `images/layers.png` etc. relative to the
# bundled stylesheet) can resolve those URLs in production. `priv/static/`
# is gitignored, so this needs to run on every build/deploy.
defp copy_leaflet_images(_args) do
src = "assets/vendor/leaflet/images"
dest = "priv/static/assets/css/images"
if File.dir?(src) do
File.mkdir_p!(dest)
File.cp_r!(src, dest)
end
:ok
end
# Applies every patch file in `priv/dep_patches/*.patch` to its corresponding
# dependency under `deps/`. Inlined here (instead of a `Mix.Task` module
# under `lib/mix/tasks/`) so it can run during `mix deps.get` itself —
# `lib/` isn't compiled yet at that point, so a proper task would not be
# resolvable. Idempotent: a sentinel file per patch prevents re-application.
@patches_dir "priv/dep_patches"
defp apply_dep_patches(_args) do
case File.ls(@patches_dir) do
{:ok, entries} ->
entries
|> Enum.filter(&String.ends_with?(&1, ".patch"))
|> Enum.sort()
|> Enum.each(&apply_single_patch/1)
{:error, :enoent} ->
:ok
end
end
defp apply_single_patch(filename) do
dep_name =
filename
|> Path.basename(".patch")
|> String.split("-")
|> Enum.drop(-1)
|> Enum.join("-")
dep_dir = Path.join("deps", dep_name)
sentinel = Path.join(dep_dir, ".#{filename}.applied")
cond do
not File.dir?(dep_dir) ->
Mix.shell().info("deps.patch: skipping #{filename} — deps/#{dep_name} not present")
File.exists?(sentinel) ->
Mix.shell().info("deps.patch: #{filename} already applied")
true ->
patch_path = Path.absname(Path.join(@patches_dir, filename))
{output, exit_code} = System.cmd("patch", ["-p1", "-i", patch_path], cd: dep_dir)
if exit_code == 0 do
Mix.shell().info("deps.patch: applied #{filename}")
File.touch!(sentinel)
else
Mix.raise("deps.patch: failed to apply #{filename}:\n#{output}")
end
end
end
end