Graham McIntire
77ff082ebb
Replace cron-based scheduling (every 4h) with self-scheduling jobs that randomly pick 7-12 hours between runs. Also increase the inter-request delay from 5-15 minutes to 10-20 minutes.
56 lines
1.5 KiB
Elixir
56 lines
1.5 KiB
Elixir
defmodule Ammoprices.Scraping.ScrapeJob do
|
|
@moduledoc false
|
|
use Oban.Worker,
|
|
queue: :scraping,
|
|
max_attempts: 3,
|
|
unique: [period: :infinity, states: [:available, :scheduled, :executing, :retryable]]
|
|
|
|
alias Ammoprices.Catalog
|
|
alias Ammoprices.Scraping.Runner
|
|
|
|
@scrapers [
|
|
Ammoprices.Scraping.Retailers.LuckyGunner,
|
|
Ammoprices.Scraping.Retailers.SgAmmo,
|
|
Ammoprices.Scraping.Retailers.TargetSportsUsa,
|
|
Ammoprices.Scraping.Retailers.TrueShotAmmo,
|
|
Ammoprices.Scraping.Retailers.PalmettoStateArmory,
|
|
Ammoprices.Scraping.Retailers.BulkAmmo,
|
|
Ammoprices.Scraping.Retailers.Natchez
|
|
]
|
|
|
|
@impl Oban.Worker
|
|
def perform(_job) do
|
|
calibers = Catalog.list_calibers()
|
|
|
|
for scraper <- @scrapers, caliber <- calibers do
|
|
scrape_delay()
|
|
Runner.run(scraper, caliber)
|
|
end
|
|
|
|
Phoenix.PubSub.broadcast(Ammoprices.PubSub, "prices:updated", {:prices_updated, %{}})
|
|
|
|
schedule_next()
|
|
|
|
:ok
|
|
end
|
|
|
|
@doc false
|
|
def schedule_next do
|
|
{min, max} = Application.get_env(:ammoprices, :scrape_interval_seconds, {25_200, 43_200})
|
|
delay_seconds = Enum.random(min..max)
|
|
|
|
%{}
|
|
|> new(scheduled_at: DateTime.add(DateTime.utc_now(), delay_seconds, :second))
|
|
|> Oban.insert()
|
|
end
|
|
|
|
# Random delay between requests: 10-20 minutes in prod, 0 in test
|
|
defp scrape_delay do
|
|
{min, max} = Application.get_env(:ammoprices, :scrape_delay_ms, {600_000, 1_200_000})
|
|
delay = Enum.random(min..max)
|
|
|
|
if delay > 0 do
|
|
Process.sleep(delay)
|
|
end
|
|
end
|
|
end
|