defmodule Ammoprices.Scraping.ScrapeJob do @moduledoc false use Oban.Worker, queue: :scraping, max_attempts: 3 alias Ammoprices.Catalog alias Ammoprices.Scraping.Runner require Logger @scrapers [ Ammoprices.Scraping.Retailers.LuckyGunner, Ammoprices.Scraping.Retailers.SgAmmo, Ammoprices.Scraping.Retailers.TargetSportsUsa, Ammoprices.Scraping.Retailers.TrueShotAmmo, Ammoprices.Scraping.Retailers.PalmettoStateArmory, Ammoprices.Scraping.Retailers.BulkAmmo, Ammoprices.Scraping.Retailers.Natchez ] # Products not seen for this many days are hard-deleted after each scrape cycle. @stale_product_days 30 @impl Oban.Worker def perform(_job) do calibers = Catalog.list_calibers() caliber_ids = Enum.map(calibers, & &1.id) for scraper <- @scrapers, caliber <- calibers do scrape_delay() safe_run(scraper, caliber) end prune_stale_products() Phoenix.PubSub.broadcast(Ammoprices.PubSub, "prices:updated", {:prices_updated, caliber_ids}) :ok after # Always reseed the chain, even if something above crashed. Without this, # a single unhandled error kills the chain until the app is restarted. schedule_next() end defp safe_run(scraper, caliber) do Runner.run(scraper, caliber) rescue error -> Logger.error( "Scrape crashed for #{inspect(scraper)} / #{caliber.slug}: " <> Exception.format(:error, error, __STACKTRACE__) ) {:error, :crashed} end defp prune_stale_products do cutoff = DateTime.add(DateTime.utc_now(), -@stale_product_days, :day) Catalog.delete_stale_products(cutoff) end @doc false def schedule_next do {min, max} = Application.get_env(:ammoprices, :scrape_interval_seconds, {25_200, 43_200}) delay_seconds = Enum.random(min..max) %{} |> new(scheduled_at: DateTime.add(DateTime.utc_now(), delay_seconds, :second)) |> Oban.insert() end # Random delay between requests: 10-20 minutes in prod, 0 in test defp scrape_delay do {min, max} = Application.get_env(:ammoprices, :scrape_delay_ms, {600_000, 1_200_000}) delay = Enum.random(min..max) if delay > 0 do Process.sleep(delay) end end end