Switch to self-scheduling scrapes with longer intervals

Replace cron-based scheduling (every 4h) with self-scheduling jobs
that randomly pick 7-12 hours between runs. Also increase the
inter-request delay from 5-15 minutes to 10-20 minutes.

config/config.exs

@@ -31,12 +31,7 @@ config :ammoprices, AmmopricesWeb.Endpoint,
 config :ammoprices, Oban,
   repo: Ammoprices.Repo,
   queues: [scraping: 2],
-  plugins: [
-    {Oban.Plugins.Cron,
-     crontab: [
-       {"0 */4 * * *", Ammoprices.Scraping.ScrapeJob}
-     ]}
-  ]
+  plugins: [Oban.Plugins.Lifeline]
 
 config :ammoprices,
   ecto_repos: [Ammoprices.Repo],

config/test.exs

@@ -29,8 +29,9 @@ config :ammoprices, Oban, testing: :manual
 # Stub HTTP requests in tests
 config :ammoprices, :req_options, plug: {Req.Test, Ammoprices.Scraping.HttpClient}
 
-# No delay between scrape requests in tests
+# No delays in tests
 config :ammoprices, :scrape_delay_ms, {0, 0}
+config :ammoprices, :scrape_interval_seconds, {0, 0}
 
 # Print only warnings and errors during test
 config :logger, level: :warning

lib/ammoprices/application.ex

@@ -19,7 +19,12 @@ defmodule Ammoprices.Application do
     # See https://hexdocs.pm/elixir/Supervisor.html
     # for other strategies and supported options
     opts = [strategy: :one_for_one, name: Ammoprices.Supervisor]
-    Supervisor.start_link(children, opts)
+    result = Supervisor.start_link(children, opts)
+
+    # Seed the self-scheduling scrape job chain (unique constraint prevents duplicates)
+    Ammoprices.Scraping.ScrapeJob.schedule_next()
+
+    result
   end
 
   # Tell Phoenix to update the endpoint configuration

lib/ammoprices/scraping/scrape_job.ex

@@ -1,6 +1,9 @@
 defmodule Ammoprices.Scraping.ScrapeJob do
   @moduledoc false
-  use Oban.Worker, queue: :scraping, max_attempts: 3
+  use Oban.Worker,
+    queue: :scraping,
+    max_attempts: 3,
+    unique: [period: :infinity, states: [:available, :scheduled, :executing, :retryable]]
 
   alias Ammoprices.Catalog
   alias Ammoprices.Scraping.Runner
@@ -26,12 +29,24 @@ defmodule Ammoprices.Scraping.ScrapeJob do
 
     Phoenix.PubSub.broadcast(Ammoprices.PubSub, "prices:updated", {:prices_updated, %{}})
 
+    schedule_next()
+
     :ok
   end
 
-  # Random delay between requests: 5-15 minutes in prod, 0 in test
+  @doc false
+  def schedule_next do
+    {min, max} = Application.get_env(:ammoprices, :scrape_interval_seconds, {25_200, 43_200})
+    delay_seconds = Enum.random(min..max)
+
+    %{}
+    |> new(scheduled_at: DateTime.add(DateTime.utc_now(), delay_seconds, :second))
+    |> Oban.insert()
+  end
+
+  # Random delay between requests: 10-20 minutes in prod, 0 in test
   defp scrape_delay do
-    {min, max} = Application.get_env(:ammoprices, :scrape_delay_ms, {300_000, 900_000})
+    {min, max} = Application.get_env(:ammoprices, :scrape_delay_ms, {600_000, 1_200_000})
     delay = Enum.random(min..max)
 
     if delay > 0 do