Add True Shot Ammo (Shopify JSON), Palmetto State Armory (Magento HTML), Bulk Ammo (Magento HTML), and Natchez Shooters Supply (GraphQL API). Extends scraper infrastructure with HttpClient.post_json/3 and an optional fetch/2 callback on the Scraper behaviour for scrapers that need custom HTTP flows (e.g. GraphQL POST requests).
155 lines
4 KiB
Elixir
155 lines
4 KiB
Elixir
defmodule Ammoprices.Scraping.Retailers.BulkAmmo do
|
|
@moduledoc false
|
|
@behaviour Ammoprices.Scraping.Scraper
|
|
|
|
alias Ammoprices.Scraping.TextDetector
|
|
|
|
@slug_to_path %{
|
|
"9mm-luger" => "/handgun/bulk-9mm-ammo",
|
|
"45-acp" => "/handgun/bulk-45-acp-ammo",
|
|
"380-acp" => "/handgun/bulk-380-acp-ammo",
|
|
"40-sw" => "/handgun/bulk-40-sandw-ammo",
|
|
"38-special" => "/handgun/bulk-38-special-ammo",
|
|
"357-magnum" => "/handgun/bulk-357-mag-ammo",
|
|
"10mm-auto" => "/handgun/bulk-10mm-ammo",
|
|
"556-223" => "/rifle/bulk-223-remington-ammo",
|
|
"308-win" => "/rifle/bulk-308-ammo",
|
|
"762x39" => "/rifle/bulk-7.62x39-ammo",
|
|
"30-06" => "/rifle/bulk-30-06-ammo",
|
|
"300-blackout" => "/rifle/bulk-300-blackout-ammo",
|
|
"65-creedmoor" => "/rifle/bulk-6.5-creedmoor-ammo",
|
|
"22-lr" => "/rimfire/bulk-22-lr-ammo",
|
|
"12-gauge" => "/shotgun/bulk-12-gauge-ammo",
|
|
"20-gauge" => "/shotgun/bulk-20-gauge-ammo"
|
|
}
|
|
|
|
@impl true
|
|
def retailer_slug, do: "bulk-ammo"
|
|
|
|
@impl true
|
|
def category_url(caliber) do
|
|
Map.get(@slug_to_path, caliber.slug)
|
|
end
|
|
|
|
@impl true
|
|
def parse_products(html) do
|
|
html
|
|
|> Floki.parse_document!()
|
|
|> Floki.find(".products-grid li.item")
|
|
|> Enum.map(&parse_item/1)
|
|
|> Enum.reject(&is_nil/1)
|
|
end
|
|
|
|
defp parse_item(item) do
|
|
with {:ok, title} <- extract_title(item),
|
|
{:ok, url} <- extract_url(item),
|
|
{:ok, price_cents} <- extract_price(item) do
|
|
round_count = extract_round_count(title)
|
|
ppr = if round_count && round_count > 0, do: round(price_cents / round_count)
|
|
|
|
%{
|
|
title: title,
|
|
url: url,
|
|
price_cents: price_cents,
|
|
price_per_round_cents: ppr,
|
|
brand: extract_brand(title),
|
|
grain_weight: extract_grain_weight(title),
|
|
round_count: round_count,
|
|
casing: TextDetector.detect_casing(title),
|
|
subsonic: TextDetector.detect_subsonic(title),
|
|
in_stock: extract_in_stock(item)
|
|
}
|
|
else
|
|
_ -> nil
|
|
end
|
|
end
|
|
|
|
defp extract_title(item) do
|
|
case Floki.find(item, "a.product-name") do
|
|
[{_, _, _} = node] -> {:ok, node |> Floki.text() |> String.trim()}
|
|
_ -> :error
|
|
end
|
|
end
|
|
|
|
defp extract_url(item) do
|
|
case Floki.find(item, "a.product-name") do
|
|
[{_, attrs, _}] ->
|
|
case List.keyfind(attrs, "href", 0) do
|
|
{"href", href} -> {:ok, href}
|
|
_ -> :error
|
|
end
|
|
|
|
_ ->
|
|
:error
|
|
end
|
|
end
|
|
|
|
defp extract_price(item) do
|
|
# Prefer special-price (sale price), fall back to regular-price
|
|
price_text =
|
|
case Floki.find(item, ".special-price .price") do
|
|
[{_, _, _} = node] -> node |> Floki.text() |> String.trim()
|
|
_ -> extract_regular_price_text(item)
|
|
end
|
|
|
|
case price_text do
|
|
nil ->
|
|
:error
|
|
|
|
text ->
|
|
case Regex.run(~r/\$([0-9,]+(?:\.\d{2})?)/, text) do
|
|
[_, amount] ->
|
|
cents = parse_dollar_amount(amount)
|
|
{:ok, cents}
|
|
|
|
_ ->
|
|
:error
|
|
end
|
|
end
|
|
end
|
|
|
|
defp extract_regular_price_text(item) do
|
|
case Floki.find(item, ".regular-price .price") do
|
|
[{_, _, _} = node] -> node |> Floki.text() |> String.trim()
|
|
_ -> nil
|
|
end
|
|
end
|
|
|
|
defp parse_dollar_amount(amount) do
|
|
clean = String.replace(amount, ",", "")
|
|
|
|
if String.contains?(clean, ".") do
|
|
clean |> String.to_float() |> Kernel.*(100) |> round()
|
|
else
|
|
String.to_integer(clean) * 100
|
|
end
|
|
end
|
|
|
|
defp extract_round_count(title) do
|
|
case Regex.run(~r/^(\d[\d,]*)\s+[Rr]ounds?/i, title) do
|
|
[_, count] -> count |> String.replace(",", "") |> String.to_integer()
|
|
_ -> nil
|
|
end
|
|
end
|
|
|
|
defp extract_brand(title) do
|
|
case Regex.run(~r/by\s+(.+?)(?:\s*-|$)/, title) do
|
|
[_, brand] -> String.trim(brand)
|
|
_ -> nil
|
|
end
|
|
end
|
|
|
|
defp extract_grain_weight(title) do
|
|
case Regex.run(~r/(\d+)\s*gr/i, title) do
|
|
[_, weight] -> String.to_integer(weight)
|
|
_ -> nil
|
|
end
|
|
end
|
|
|
|
defp extract_in_stock(item) do
|
|
case Floki.find(item, "span.in-stock") do
|
|
[_ | _] -> true
|
|
_ -> false
|
|
end
|
|
end
|
|
end
|