diff --git a/config/config.exs b/config/config.exs index 09bee50..bfdec90 100644 --- a/config/config.exs +++ b/config/config.exs @@ -31,12 +31,7 @@ config :ammoprices, AmmopricesWeb.Endpoint, config :ammoprices, Oban, repo: Ammoprices.Repo, queues: [scraping: 2], - plugins: [ - {Oban.Plugins.Cron, - crontab: [ - {"0 */4 * * *", Ammoprices.Scraping.ScrapeJob} - ]} - ] + plugins: [Oban.Plugins.Lifeline] config :ammoprices, ecto_repos: [Ammoprices.Repo], diff --git a/config/test.exs b/config/test.exs index 72ab1c7..761c9c2 100644 --- a/config/test.exs +++ b/config/test.exs @@ -29,8 +29,9 @@ config :ammoprices, Oban, testing: :manual # Stub HTTP requests in tests config :ammoprices, :req_options, plug: {Req.Test, Ammoprices.Scraping.HttpClient} -# No delay between scrape requests in tests +# No delays in tests config :ammoprices, :scrape_delay_ms, {0, 0} +config :ammoprices, :scrape_interval_seconds, {0, 0} # Print only warnings and errors during test config :logger, level: :warning diff --git a/lib/ammoprices/application.ex b/lib/ammoprices/application.ex index 19ea89c..b767bc0 100644 --- a/lib/ammoprices/application.ex +++ b/lib/ammoprices/application.ex @@ -19,7 +19,12 @@ defmodule Ammoprices.Application do # See https://hexdocs.pm/elixir/Supervisor.html # for other strategies and supported options opts = [strategy: :one_for_one, name: Ammoprices.Supervisor] - Supervisor.start_link(children, opts) + result = Supervisor.start_link(children, opts) + + # Seed the self-scheduling scrape job chain (unique constraint prevents duplicates) + Ammoprices.Scraping.ScrapeJob.schedule_next() + + result end # Tell Phoenix to update the endpoint configuration diff --git a/lib/ammoprices/scraping/scrape_job.ex b/lib/ammoprices/scraping/scrape_job.ex index 88b965a..5528f98 100644 --- a/lib/ammoprices/scraping/scrape_job.ex +++ b/lib/ammoprices/scraping/scrape_job.ex @@ -1,6 +1,9 @@ defmodule Ammoprices.Scraping.ScrapeJob do @moduledoc false - use Oban.Worker, queue: :scraping, max_attempts: 3 + use Oban.Worker, + queue: :scraping, + max_attempts: 3, + unique: [period: :infinity, states: [:available, :scheduled, :executing, :retryable]] alias Ammoprices.Catalog alias Ammoprices.Scraping.Runner @@ -26,12 +29,24 @@ defmodule Ammoprices.Scraping.ScrapeJob do Phoenix.PubSub.broadcast(Ammoprices.PubSub, "prices:updated", {:prices_updated, %{}}) + schedule_next() + :ok end - # Random delay between requests: 5-15 minutes in prod, 0 in test + @doc false + def schedule_next do + {min, max} = Application.get_env(:ammoprices, :scrape_interval_seconds, {25_200, 43_200}) + delay_seconds = Enum.random(min..max) + + %{} + |> new(scheduled_at: DateTime.add(DateTime.utc_now(), delay_seconds, :second)) + |> Oban.insert() + end + + # Random delay between requests: 10-20 minutes in prod, 0 in test defp scrape_delay do - {min, max} = Application.get_env(:ammoprices, :scrape_delay_ms, {300_000, 900_000}) + {min, max} = Application.get_env(:ammoprices, :scrape_delay_ms, {600_000, 1_200_000}) delay = Enum.random(min..max) if delay > 0 do