Graham McIntire
872c94bea1
- Fix Oban self-scheduling chain silently dying (remove unique constraint so after block can schedule next job)
- Fix Natchez and TrueShotAmmo extract_price defaulting to 0 for missing price data
- Fix caliber_matcher redundant String.downcase inside Enum.find (pre-downcase outside loop)
- Fix runner.ex double iteration (Enum.map + Enum.count merged into single Enum.reduce)
- Fix price_stats_for_caliber firing 3 queries instead of 2 (merge all_time + thirty_day via FILTER)
- Fix PSA duplicate Floki.find on same selector (merge extract_title + extract_url)
- Fix PubSub broadcast carrying no caliber IDs (now includes IDs; clients skip irrelevant reloads)
- Fix BulkAmmo ^ anchor skipping non-leading round counts
- Fix TargetSportsUSA fragile exact text match 'Add To Cart' (use case-insensitive contains)
- Fix SgAmmo dead destructure {_, _, _} = row
- Fix text_detector brass/nickel detection order
- Fix Natchez GraphQL string interpolation (add escape_gql_string)
- Fix chart data triple Enum.map merged into single reduce
- Change Oban scraping queue workers from 2 to 1 (only one needed with Process.sleep)
107 lines
3 KiB
Elixir
107 lines
3 KiB
Elixir
defmodule Ammoprices.Scraping.Runner do
|
|
@moduledoc false
|
|
|
|
alias Ammoprices.Catalog
|
|
alias Ammoprices.Prices
|
|
alias Ammoprices.Scraping.HttpClient
|
|
|
|
require Logger
|
|
|
|
def run(scraper, caliber) do
|
|
retailer = Catalog.get_retailer_by_slug!(scraper.retailer_slug())
|
|
|
|
if function_exported?(scraper, :fetch, 2) do
|
|
do_fetch(scraper, retailer, caliber)
|
|
else
|
|
case scraper.category_url(caliber) do
|
|
nil ->
|
|
{:ok, %{products_count: 0, snapshots_count: 0}}
|
|
|
|
path ->
|
|
url = retailer.base_url <> path
|
|
do_scrape(scraper, retailer, caliber, url)
|
|
end
|
|
end
|
|
end
|
|
|
|
defp do_fetch(scraper, retailer, caliber) do
|
|
case scraper.fetch(retailer, caliber) do
|
|
{:ok, parsed} ->
|
|
process_results(parsed, retailer, caliber)
|
|
|
|
{:error, reason} ->
|
|
{:error, reason}
|
|
end
|
|
end
|
|
|
|
defp do_scrape(scraper, retailer, caliber, url) do
|
|
case HttpClient.get(url) do
|
|
{:ok, %{status: 200, body: body}} ->
|
|
parsed = scraper.parse_products(body)
|
|
process_results(parsed, retailer, caliber)
|
|
|
|
{:ok, %{status: status}} ->
|
|
{:error, "HTTP #{status}"}
|
|
|
|
{:error, reason} ->
|
|
{:error, reason}
|
|
end
|
|
end
|
|
|
|
defp process_results(parsed, retailer, caliber) do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
{_results, successful} =
|
|
Enum.reduce(parsed, {[], 0}, fn product_data, {acc, count} ->
|
|
case process_product(product_data, retailer, caliber, now) do
|
|
:ok -> {[:ok | acc], count + 1}
|
|
:error -> {[:error | acc], count}
|
|
end
|
|
end)
|
|
|
|
if successful > 0 do
|
|
Catalog.mark_unseen_products_out_of_stock(retailer.id, caliber.id, now)
|
|
end
|
|
|
|
Catalog.update_retailer(retailer, %{last_scraped_at: now})
|
|
|
|
{:ok, %{products_count: successful, snapshots_count: successful}}
|
|
end
|
|
|
|
defp process_product(product_data, retailer, caliber, now) do
|
|
product_attrs = %{
|
|
title: product_data.title,
|
|
url: product_data.url,
|
|
brand: product_data.brand,
|
|
grain_weight: product_data.grain_weight,
|
|
round_count: product_data.round_count,
|
|
casing: product_data.casing,
|
|
condition: "new",
|
|
in_stock: product_data.in_stock,
|
|
subsonic: Map.get(product_data, :subsonic, false),
|
|
last_seen_at: now
|
|
}
|
|
|
|
with {:ok, product} <- Catalog.upsert_product(retailer.id, caliber.id, product_attrs),
|
|
{:ok, _snapshot} <- create_snapshot(product, product_data, now) do
|
|
:ok
|
|
else
|
|
{:error, %Ecto.Changeset{} = changeset} ->
|
|
Logger.warning(
|
|
"Scrape insert failed for #{retailer.slug} #{product_data.url}: " <>
|
|
inspect(changeset.errors)
|
|
)
|
|
|
|
:error
|
|
end
|
|
end
|
|
|
|
defp create_snapshot(product, product_data, now) do
|
|
Prices.create_snapshot(product.id, %{
|
|
price_cents: product_data.price_cents,
|
|
price_per_round_cents: product_data.price_per_round_cents,
|
|
in_stock: product_data.in_stock,
|
|
recorded_at: now
|
|
})
|
|
end
|
|
end
|