Graham McIntire
16a28ac01c
- Add subsonic boolean field to products with composite filter indexes - Add TextDetector utility for detecting subsonic/casing from product titles - Wire TextDetector into SgAmmo (casing + subsonic) and LuckyGunner (subsonic + fallback casing) - Extend latest_prices_for_caliber with casing, grain_weight, and subsonic filter options - Add distinct_grain_weights_for_caliber and distinct_casings_for_caliber queries - Add filter UI with toggleable casing, grain weight, and subsonic buttons - Add grain weight column and subsonic badge to product table - Refresh filter options on PubSub price updates for real-time UI - Implement Target Sports USA scraper with 18 caliber mappings - Display all prices in dollar format ($0.38 instead of 38¢)
37 lines
919 B
Elixir
37 lines
919 B
Elixir
defmodule Ammoprices.Scraping.ScrapeJob do
|
|
@moduledoc false
|
|
use Oban.Worker, queue: :scraping, max_attempts: 3
|
|
|
|
alias Ammoprices.Catalog
|
|
alias Ammoprices.Scraping.Runner
|
|
|
|
@scrapers [
|
|
Ammoprices.Scraping.Retailers.LuckyGunner,
|
|
Ammoprices.Scraping.Retailers.SgAmmo,
|
|
Ammoprices.Scraping.Retailers.TargetSportsUsa
|
|
]
|
|
|
|
@impl Oban.Worker
|
|
def perform(_job) do
|
|
calibers = Catalog.list_calibers()
|
|
|
|
for scraper <- @scrapers, caliber <- calibers do
|
|
scrape_delay()
|
|
Runner.run(scraper, caliber)
|
|
end
|
|
|
|
Phoenix.PubSub.broadcast(Ammoprices.PubSub, "prices:updated", {:prices_updated, %{}})
|
|
|
|
:ok
|
|
end
|
|
|
|
# Random delay between requests: 5-15 seconds in prod, 0 in test
|
|
defp scrape_delay do
|
|
{min, max} = Application.get_env(:ammoprices, :scrape_delay_ms, {5_000, 15_000})
|
|
delay = Enum.random(min..max)
|
|
|
|
if delay > 0 do
|
|
Process.sleep(delay)
|
|
end
|
|
end
|
|
end
|