Graham McIntire
16a28ac01c
- Add subsonic boolean field to products with composite filter indexes - Add TextDetector utility for detecting subsonic/casing from product titles - Wire TextDetector into SgAmmo (casing + subsonic) and LuckyGunner (subsonic + fallback casing) - Extend latest_prices_for_caliber with casing, grain_weight, and subsonic filter options - Add distinct_grain_weights_for_caliber and distinct_casings_for_caliber queries - Add filter UI with toggleable casing, grain weight, and subsonic buttons - Add grain weight column and subsonic badge to product table - Refresh filter options on PubSub price updates for real-time UI - Implement Target Sports USA scraper with 18 caliber mappings - Display all prices in dollar format ($0.38 instead of 38¢)
72 lines
2.2 KiB
Elixir
72 lines
2.2 KiB
Elixir
defmodule Ammoprices.Scraping.Runner do
|
|
@moduledoc false
|
|
|
|
alias Ammoprices.Catalog
|
|
alias Ammoprices.Prices
|
|
alias Ammoprices.Scraping.HttpClient
|
|
|
|
def run(scraper, caliber) do
|
|
retailer = Catalog.get_retailer_by_slug!(scraper.retailer_slug())
|
|
|
|
case scraper.category_url(caliber) do
|
|
nil ->
|
|
{:ok, %{products_count: 0, snapshots_count: 0}}
|
|
|
|
path ->
|
|
url = retailer.base_url <> path
|
|
do_scrape(scraper, retailer, caliber, url)
|
|
end
|
|
end
|
|
|
|
defp do_scrape(scraper, retailer, caliber, url) do
|
|
case HttpClient.get(url) do
|
|
{:ok, %{status: 200, body: body}} ->
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
parsed = scraper.parse_products(body)
|
|
|
|
results =
|
|
Enum.map(parsed, fn product_data ->
|
|
product_attrs = %{
|
|
title: product_data.title,
|
|
url: product_data.url,
|
|
brand: product_data.brand,
|
|
grain_weight: product_data.grain_weight,
|
|
round_count: product_data.round_count,
|
|
casing: product_data.casing,
|
|
condition: "new",
|
|
in_stock: product_data.in_stock,
|
|
subsonic: Map.get(product_data, :subsonic, false),
|
|
last_seen_at: now
|
|
}
|
|
|
|
case Catalog.upsert_product(retailer.id, caliber.id, product_attrs) do
|
|
{:ok, product} ->
|
|
snapshot_attrs = %{
|
|
price_cents: product_data.price_cents,
|
|
price_per_round_cents: product_data.price_per_round_cents,
|
|
in_stock: product_data.in_stock,
|
|
recorded_at: now
|
|
}
|
|
|
|
{:ok, _snapshot} = Prices.create_snapshot(product.id, snapshot_attrs)
|
|
:ok
|
|
|
|
{:error, _changeset} ->
|
|
:error
|
|
end
|
|
end)
|
|
|
|
successful = Enum.count(results, &(&1 == :ok))
|
|
|
|
Catalog.update_retailer(retailer, %{last_scraped_at: now})
|
|
|
|
{:ok, %{products_count: successful, snapshots_count: successful}}
|
|
|
|
{:ok, %{status: status}} ->
|
|
{:error, "HTTP #{status}"}
|
|
|
|
{:error, reason} ->
|
|
{:error, reason}
|
|
end
|
|
end
|
|
end
|