feat(workers): add GridCachePruneWorker
The Microwaveprop.Weather.GridCache ETS table grew unbounded — each hourly PropagationGridWorker run plus peer broadcast_put added ~92k points per forecast hour, so long-lived pods were strong OOM candidates. GridCache already exposed prune_older_than/1 but nothing called it. Adds an Oban worker on the :propagation queue (lower priority than the grid chain) that prunes entries older than 3 hours. Scheduled hourly at :15 in dev, prod, and the base crontab.
This commit is contained in:
parent
7ab426c01d
commit
8e73583926
5 changed files with 71 additions and 0 deletions
|
|
@ -98,6 +98,7 @@ config :microwaveprop, Oban,
|
||||||
{"*/5 * * * *", Microwaveprop.Commercial.PollWorker},
|
{"*/5 * * * *", Microwaveprop.Commercial.PollWorker},
|
||||||
{"5 * * * *", Microwaveprop.Workers.PropagationGridWorker},
|
{"5 * * * *", Microwaveprop.Workers.PropagationGridWorker},
|
||||||
{"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
|
{"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
|
||||||
|
{"15 * * * *", Microwaveprop.Workers.GridCachePruneWorker},
|
||||||
# UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch
|
# UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch
|
||||||
{"30 1 * * *", CanadianSoundingFetchWorker},
|
{"30 1 * * *", CanadianSoundingFetchWorker},
|
||||||
{"30 13 * * *", CanadianSoundingFetchWorker},
|
{"30 13 * * *", CanadianSoundingFetchWorker},
|
||||||
|
|
|
||||||
|
|
@ -97,6 +97,7 @@ config :microwaveprop, Oban,
|
||||||
crontab: [
|
crontab: [
|
||||||
{"5 * * * *", Microwaveprop.Workers.PropagationGridWorker},
|
{"5 * * * *", Microwaveprop.Workers.PropagationGridWorker},
|
||||||
{"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
|
{"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
|
||||||
|
{"15 * * * *", Microwaveprop.Workers.GridCachePruneWorker},
|
||||||
{"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker}
|
{"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker}
|
||||||
]}
|
]}
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -205,6 +205,7 @@ if config_env() == :prod do
|
||||||
# (25 jobs per run, 100/day) into the :gefs queue.
|
# (25 jobs per run, 100/day) into the :gefs queue.
|
||||||
{"30 5,11,17,23 * * *", Microwaveprop.Workers.GefsFetchWorker},
|
{"30 5,11,17,23 * * *", Microwaveprop.Workers.GefsFetchWorker},
|
||||||
{"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
|
{"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
|
||||||
|
{"15 * * * *", Microwaveprop.Workers.GridCachePruneWorker},
|
||||||
# The :narr type is virtual: it targets contacts with hrrr_status =
|
# The :narr type is virtual: it targets contacts with hrrr_status =
|
||||||
# :unavailable (pre-2014, missing from the HRRR archive) and dispatches
|
# :unavailable (pre-2014, missing from the HRRR archive) and dispatches
|
||||||
# NarrFetchWorker against NCEI. See narr_jobs_for_contact/1.
|
# NarrFetchWorker against NCEI. See narr_jobs_for_contact/1.
|
||||||
|
|
|
||||||
28
lib/microwaveprop/workers/grid_cache_prune_worker.ex
Normal file
28
lib/microwaveprop/workers/grid_cache_prune_worker.ex
Normal file
|
|
@ -0,0 +1,28 @@
|
||||||
|
defmodule Microwaveprop.Workers.GridCachePruneWorker do
|
||||||
|
@moduledoc """
|
||||||
|
Prunes stale rows from the node-local `Microwaveprop.Weather.GridCache` ETS
|
||||||
|
table. Without this cron the cache grew unbounded: every hourly
|
||||||
|
`PropagationGridWorker` run plus peer `broadcast_put` added an entry per
|
||||||
|
forecast hour (~92k points * 19 hours), so long-lived pods eventually OOM'd.
|
||||||
|
|
||||||
|
Runs on the shared `:propagation` queue alongside `PropagationPruneWorker`
|
||||||
|
at a lower priority so the hourly grid chain never waits behind it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
use Oban.Worker,
|
||||||
|
queue: :propagation,
|
||||||
|
priority: 5,
|
||||||
|
max_attempts: 3,
|
||||||
|
unique: [period: 300, states: [:available, :scheduled, :executing, :retryable]]
|
||||||
|
|
||||||
|
alias Microwaveprop.Weather.GridCache
|
||||||
|
|
||||||
|
@cutoff_hours 3
|
||||||
|
|
||||||
|
@impl Oban.Worker
|
||||||
|
def perform(%Oban.Job{}) do
|
||||||
|
cutoff = DateTime.add(DateTime.utc_now(), -@cutoff_hours, :hour)
|
||||||
|
_removed = GridCache.prune_older_than(cutoff)
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
end
|
||||||
40
test/microwaveprop/workers/grid_cache_prune_worker_test.exs
Normal file
40
test/microwaveprop/workers/grid_cache_prune_worker_test.exs
Normal file
|
|
@ -0,0 +1,40 @@
|
||||||
|
defmodule Microwaveprop.Workers.GridCachePruneWorkerTest do
|
||||||
|
use ExUnit.Case, async: false
|
||||||
|
|
||||||
|
alias Microwaveprop.Weather.GridCache
|
||||||
|
alias Microwaveprop.Workers.GridCachePruneWorker
|
||||||
|
|
||||||
|
setup do
|
||||||
|
GridCache.clear()
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "perform/1" do
|
||||||
|
test "removes cache entries with valid_time older than 3 hours" do
|
||||||
|
now = DateTime.utc_now()
|
||||||
|
old = now |> DateTime.add(-3, :hour) |> DateTime.add(-1, :second) |> DateTime.truncate(:second)
|
||||||
|
fresh = now |> DateTime.add(-30, :minute) |> DateTime.truncate(:second)
|
||||||
|
future = now |> DateTime.add(6, :hour) |> DateTime.truncate(:second)
|
||||||
|
|
||||||
|
GridCache.put(old, [%{lat: 32.0, lon: -97.0, temperature: 10.0}])
|
||||||
|
GridCache.put(fresh, [%{lat: 32.0, lon: -97.0, temperature: 20.0}])
|
||||||
|
GridCache.put(future, [%{lat: 32.0, lon: -97.0, temperature: 30.0}])
|
||||||
|
|
||||||
|
assert :ok = GridCachePruneWorker.perform(%Oban.Job{})
|
||||||
|
|
||||||
|
assert GridCache.fetch(old) == :miss
|
||||||
|
assert {:ok, [%{temperature: 20.0}]} = GridCache.fetch(fresh)
|
||||||
|
assert {:ok, [%{temperature: 30.0}]} = GridCache.fetch(future)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "is a no-op when nothing is stale" do
|
||||||
|
fresh =
|
||||||
|
DateTime.utc_now() |> DateTime.add(-30, :minute) |> DateTime.truncate(:second)
|
||||||
|
|
||||||
|
GridCache.put(fresh, [%{lat: 32.0, lon: -97.0, temperature: 42.0}])
|
||||||
|
|
||||||
|
assert :ok = GridCachePruneWorker.perform(%Oban.Job{})
|
||||||
|
assert {:ok, [%{temperature: 42.0}]} = GridCache.fetch(fresh)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
Loading…
Add table
Reference in a new issue