Graham McIntire
872c94bea1
- Fix Oban self-scheduling chain silently dying (remove unique constraint so after block can schedule next job)
- Fix Natchez and TrueShotAmmo extract_price defaulting to 0 for missing price data
- Fix caliber_matcher redundant String.downcase inside Enum.find (pre-downcase outside loop)
- Fix runner.ex double iteration (Enum.map + Enum.count merged into single Enum.reduce)
- Fix price_stats_for_caliber firing 3 queries instead of 2 (merge all_time + thirty_day via FILTER)
- Fix PSA duplicate Floki.find on same selector (merge extract_title + extract_url)
- Fix PubSub broadcast carrying no caliber IDs (now includes IDs; clients skip irrelevant reloads)
- Fix BulkAmmo ^ anchor skipping non-leading round counts
- Fix TargetSportsUSA fragile exact text match 'Add To Cart' (use case-insensitive contains)
- Fix SgAmmo dead destructure {_, _, _} = row
- Fix text_detector brass/nickel detection order
- Fix Natchez GraphQL string interpolation (add escape_gql_string)
- Fix chart data triple Enum.map merged into single reduce
- Change Oban scraping queue workers from 2 to 1 (only one needed with Process.sleep)
82 lines
2.2 KiB
Elixir
82 lines
2.2 KiB
Elixir
defmodule Ammoprices.Scraping.ScrapeJob do
|
|
@moduledoc false
|
|
use Oban.Worker,
|
|
queue: :scraping,
|
|
max_attempts: 3
|
|
|
|
alias Ammoprices.Catalog
|
|
alias Ammoprices.Scraping.Runner
|
|
|
|
require Logger
|
|
|
|
@scrapers [
|
|
Ammoprices.Scraping.Retailers.LuckyGunner,
|
|
Ammoprices.Scraping.Retailers.SgAmmo,
|
|
Ammoprices.Scraping.Retailers.TargetSportsUsa,
|
|
Ammoprices.Scraping.Retailers.TrueShotAmmo,
|
|
Ammoprices.Scraping.Retailers.PalmettoStateArmory,
|
|
Ammoprices.Scraping.Retailers.BulkAmmo,
|
|
Ammoprices.Scraping.Retailers.Natchez
|
|
]
|
|
|
|
# Products not seen for this many days are hard-deleted after each scrape cycle.
|
|
@stale_product_days 30
|
|
|
|
@impl Oban.Worker
|
|
def perform(_job) do
|
|
calibers = Catalog.list_calibers()
|
|
caliber_ids = Enum.map(calibers, & &1.id)
|
|
|
|
for scraper <- @scrapers, caliber <- calibers do
|
|
scrape_delay()
|
|
safe_run(scraper, caliber)
|
|
end
|
|
|
|
prune_stale_products()
|
|
|
|
Phoenix.PubSub.broadcast(Ammoprices.PubSub, "prices:updated", {:prices_updated, caliber_ids})
|
|
|
|
:ok
|
|
after
|
|
# Always reseed the chain, even if something above crashed. Without this,
|
|
# a single unhandled error kills the chain until the app is restarted.
|
|
schedule_next()
|
|
end
|
|
|
|
defp safe_run(scraper, caliber) do
|
|
Runner.run(scraper, caliber)
|
|
rescue
|
|
error ->
|
|
Logger.error(
|
|
"Scrape crashed for #{inspect(scraper)} / #{caliber.slug}: " <>
|
|
Exception.format(:error, error, __STACKTRACE__)
|
|
)
|
|
|
|
{:error, :crashed}
|
|
end
|
|
|
|
defp prune_stale_products do
|
|
cutoff = DateTime.add(DateTime.utc_now(), -@stale_product_days, :day)
|
|
Catalog.delete_stale_products(cutoff)
|
|
end
|
|
|
|
@doc false
|
|
def schedule_next do
|
|
{min, max} = Application.get_env(:ammoprices, :scrape_interval_seconds, {25_200, 43_200})
|
|
delay_seconds = Enum.random(min..max)
|
|
|
|
%{}
|
|
|> new(scheduled_at: DateTime.add(DateTime.utc_now(), delay_seconds, :second))
|
|
|> Oban.insert()
|
|
end
|
|
|
|
# Random delay between requests: 10-20 minutes in prod, 0 in test
|
|
defp scrape_delay do
|
|
{min, max} = Application.get_env(:ammoprices, :scrape_delay_ms, {600_000, 1_200_000})
|
|
delay = Enum.random(min..max)
|
|
|
|
if delay > 0 do
|
|
Process.sleep(delay)
|
|
end
|
|
end
|
|
end
|