Add 4 new retailer scrapers for broader price coverage
Add True Shot Ammo (Shopify JSON), Palmetto State Armory (Magento HTML), Bulk Ammo (Magento HTML), and Natchez Shooters Supply (GraphQL API). Extends scraper infrastructure with HttpClient.post_json/3 and an optional fetch/2 callback on the Scraper behaviour for scrapers that need custom HTTP flows (e.g. GraphQL POST requests).
This commit is contained in:
parent
0583bd14cf
commit
9eb85faebd
18 changed files with 1544 additions and 47 deletions
|
|
@ -24,4 +24,11 @@ defmodule Ammoprices.Scraping.HttpClient do
|
|||
Application.get_env(:ammoprices, :req_options, [])
|
||||
)
|
||||
end
|
||||
|
||||
def post_json(url, body, extra_headers \\ []) do
|
||||
Req.post(
|
||||
[base_url: url, headers: @default_headers ++ extra_headers, json: body, retry: :transient, max_retries: 3],
|
||||
Application.get_env(:ammoprices, :req_options, [])
|
||||
)
|
||||
end
|
||||
end
|
||||
|
|
|
|||
155
lib/ammoprices/scraping/retailers/bulk_ammo.ex
Normal file
155
lib/ammoprices/scraping/retailers/bulk_ammo.ex
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
defmodule Ammoprices.Scraping.Retailers.BulkAmmo do
|
||||
@moduledoc false
|
||||
@behaviour Ammoprices.Scraping.Scraper
|
||||
|
||||
alias Ammoprices.Scraping.TextDetector
|
||||
|
||||
@slug_to_path %{
|
||||
"9mm-luger" => "/handgun/bulk-9mm-ammo",
|
||||
"45-acp" => "/handgun/bulk-45-acp-ammo",
|
||||
"380-acp" => "/handgun/bulk-380-acp-ammo",
|
||||
"40-sw" => "/handgun/bulk-40-sandw-ammo",
|
||||
"38-special" => "/handgun/bulk-38-special-ammo",
|
||||
"357-magnum" => "/handgun/bulk-357-mag-ammo",
|
||||
"10mm-auto" => "/handgun/bulk-10mm-ammo",
|
||||
"556-223" => "/rifle/bulk-223-remington-ammo",
|
||||
"308-win" => "/rifle/bulk-308-ammo",
|
||||
"762x39" => "/rifle/bulk-7.62x39-ammo",
|
||||
"30-06" => "/rifle/bulk-30-06-ammo",
|
||||
"300-blackout" => "/rifle/bulk-300-blackout-ammo",
|
||||
"65-creedmoor" => "/rifle/bulk-6.5-creedmoor-ammo",
|
||||
"22-lr" => "/rimfire/bulk-22-lr-ammo",
|
||||
"12-gauge" => "/shotgun/bulk-12-gauge-ammo",
|
||||
"20-gauge" => "/shotgun/bulk-20-gauge-ammo"
|
||||
}
|
||||
|
||||
@impl true
|
||||
def retailer_slug, do: "bulk-ammo"
|
||||
|
||||
@impl true
|
||||
def category_url(caliber) do
|
||||
Map.get(@slug_to_path, caliber.slug)
|
||||
end
|
||||
|
||||
@impl true
|
||||
def parse_products(html) do
|
||||
html
|
||||
|> Floki.parse_document!()
|
||||
|> Floki.find(".products-grid li.item")
|
||||
|> Enum.map(&parse_item/1)
|
||||
|> Enum.reject(&is_nil/1)
|
||||
end
|
||||
|
||||
defp parse_item(item) do
|
||||
with {:ok, title} <- extract_title(item),
|
||||
{:ok, url} <- extract_url(item),
|
||||
{:ok, price_cents} <- extract_price(item) do
|
||||
round_count = extract_round_count(title)
|
||||
ppr = if round_count && round_count > 0, do: round(price_cents / round_count)
|
||||
|
||||
%{
|
||||
title: title,
|
||||
url: url,
|
||||
price_cents: price_cents,
|
||||
price_per_round_cents: ppr,
|
||||
brand: extract_brand(title),
|
||||
grain_weight: extract_grain_weight(title),
|
||||
round_count: round_count,
|
||||
casing: TextDetector.detect_casing(title),
|
||||
subsonic: TextDetector.detect_subsonic(title),
|
||||
in_stock: extract_in_stock(item)
|
||||
}
|
||||
else
|
||||
_ -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp extract_title(item) do
|
||||
case Floki.find(item, "a.product-name") do
|
||||
[{_, _, _} = node] -> {:ok, node |> Floki.text() |> String.trim()}
|
||||
_ -> :error
|
||||
end
|
||||
end
|
||||
|
||||
defp extract_url(item) do
|
||||
case Floki.find(item, "a.product-name") do
|
||||
[{_, attrs, _}] ->
|
||||
case List.keyfind(attrs, "href", 0) do
|
||||
{"href", href} -> {:ok, href}
|
||||
_ -> :error
|
||||
end
|
||||
|
||||
_ ->
|
||||
:error
|
||||
end
|
||||
end
|
||||
|
||||
defp extract_price(item) do
|
||||
# Prefer special-price (sale price), fall back to regular-price
|
||||
price_text =
|
||||
case Floki.find(item, ".special-price .price") do
|
||||
[{_, _, _} = node] -> node |> Floki.text() |> String.trim()
|
||||
_ -> extract_regular_price_text(item)
|
||||
end
|
||||
|
||||
case price_text do
|
||||
nil ->
|
||||
:error
|
||||
|
||||
text ->
|
||||
case Regex.run(~r/\$([0-9,]+(?:\.\d{2})?)/, text) do
|
||||
[_, amount] ->
|
||||
cents = parse_dollar_amount(amount)
|
||||
{:ok, cents}
|
||||
|
||||
_ ->
|
||||
:error
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
defp extract_regular_price_text(item) do
|
||||
case Floki.find(item, ".regular-price .price") do
|
||||
[{_, _, _} = node] -> node |> Floki.text() |> String.trim()
|
||||
_ -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_dollar_amount(amount) do
|
||||
clean = String.replace(amount, ",", "")
|
||||
|
||||
if String.contains?(clean, ".") do
|
||||
clean |> String.to_float() |> Kernel.*(100) |> round()
|
||||
else
|
||||
String.to_integer(clean) * 100
|
||||
end
|
||||
end
|
||||
|
||||
defp extract_round_count(title) do
|
||||
case Regex.run(~r/^(\d[\d,]*)\s+[Rr]ounds?/i, title) do
|
||||
[_, count] -> count |> String.replace(",", "") |> String.to_integer()
|
||||
_ -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp extract_brand(title) do
|
||||
case Regex.run(~r/by\s+(.+?)(?:\s*-|$)/, title) do
|
||||
[_, brand] -> String.trim(brand)
|
||||
_ -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp extract_grain_weight(title) do
|
||||
case Regex.run(~r/(\d+)\s*gr/i, title) do
|
||||
[_, weight] -> String.to_integer(weight)
|
||||
_ -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp extract_in_stock(item) do
|
||||
case Floki.find(item, "span.in-stock") do
|
||||
[_ | _] -> true
|
||||
_ -> false
|
||||
end
|
||||
end
|
||||
end
|
||||
132
lib/ammoprices/scraping/retailers/natchez.ex
Normal file
132
lib/ammoprices/scraping/retailers/natchez.ex
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
defmodule Ammoprices.Scraping.Retailers.Natchez do
|
||||
@moduledoc false
|
||||
@behaviour Ammoprices.Scraping.Scraper
|
||||
|
||||
alias Ammoprices.Scraping.HttpClient
|
||||
alias Ammoprices.Scraping.TextDetector
|
||||
|
||||
@slug_to_caliber %{
|
||||
"9mm-luger" => "9mm Luger",
|
||||
"45-acp" => "45 Auto",
|
||||
"380-acp" => "380 Auto",
|
||||
"40-sw" => "40 S&W",
|
||||
"38-special" => "38 Spl",
|
||||
"357-magnum" => "357 Mag",
|
||||
"10mm-auto" => "10mm Auto",
|
||||
"556-223" => "223 Rem",
|
||||
"308-win" => "308 Win",
|
||||
"762x39" => "7.62x39mm",
|
||||
"30-06" => "30-06 Sprg",
|
||||
"300-blackout" => "300 Blk",
|
||||
"65-creedmoor" => "6.5 Creedmoor",
|
||||
"22-lr" => "22 LR",
|
||||
"22-wmr" => "22 WMR",
|
||||
"17-hmr" => "17 HMR",
|
||||
"12-gauge" => "12 Gauge",
|
||||
"16-gauge" => "16 Gauge",
|
||||
"20-gauge" => "20 Gauge"
|
||||
}
|
||||
|
||||
@impl true
|
||||
def retailer_slug, do: "natchez"
|
||||
|
||||
@impl true
|
||||
def category_url(_caliber), do: nil
|
||||
|
||||
@impl true
|
||||
def parse_products(_), do: []
|
||||
|
||||
@impl true
|
||||
def fetch(retailer, caliber) do
|
||||
case build_query(caliber.slug) do
|
||||
nil ->
|
||||
{:ok, []}
|
||||
|
||||
query ->
|
||||
url = retailer.base_url <> "/graphql"
|
||||
|
||||
case HttpClient.post_json(url, %{query: query}, [{"store", "default"}]) do
|
||||
{:ok, %{status: 200, body: body}} -> {:ok, parse_graphql_response(body)}
|
||||
{:ok, %{status: s}} -> {:error, "HTTP #{s}"}
|
||||
{:error, reason} -> {:error, reason}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def build_query(slug) do
|
||||
case Map.get(@slug_to_caliber, slug) do
|
||||
nil ->
|
||||
nil
|
||||
|
||||
caliber_value ->
|
||||
"""
|
||||
{
|
||||
products(
|
||||
search: "#{caliber_value}"
|
||||
filter: { caliber: { eq: "#{caliber_value}" } }
|
||||
pageSize: 48
|
||||
currentPage: 1
|
||||
) {
|
||||
items {
|
||||
name sku url_key stock_status brand grain rounds case_material
|
||||
price_range { minimum_price { final_price { value } } }
|
||||
}
|
||||
total_count
|
||||
page_info { total_pages }
|
||||
}
|
||||
}
|
||||
"""
|
||||
end
|
||||
end
|
||||
|
||||
def parse_graphql_response(%{"data" => %{"products" => %{"items" => items}}}) do
|
||||
items
|
||||
|> Enum.map(&parse_item/1)
|
||||
|> Enum.reject(&is_nil/1)
|
||||
end
|
||||
|
||||
def parse_graphql_response(_), do: []
|
||||
|
||||
defp parse_item(item) do
|
||||
price_cents = extract_price(item)
|
||||
round_count = parse_integer(item["rounds"])
|
||||
ppr = if round_count && round_count > 0, do: round(price_cents / round_count)
|
||||
name = item["name"]
|
||||
|
||||
%{
|
||||
title: name,
|
||||
url: "/#{item["url_key"]}.html",
|
||||
brand: item["brand"],
|
||||
price_cents: price_cents,
|
||||
price_per_round_cents: ppr,
|
||||
grain_weight: parse_integer(item["grain"]),
|
||||
round_count: round_count,
|
||||
casing: parse_casing(item["case_material"]),
|
||||
subsonic: TextDetector.detect_subsonic(name),
|
||||
in_stock: item["stock_status"] == "IN_STOCK"
|
||||
}
|
||||
end
|
||||
|
||||
defp extract_price(%{"price_range" => %{"minimum_price" => %{"final_price" => %{"value" => value}}}}) do
|
||||
value
|
||||
|> String.to_float()
|
||||
|> Kernel.*(100)
|
||||
|> round()
|
||||
end
|
||||
|
||||
defp extract_price(_), do: 0
|
||||
|
||||
defp parse_integer(nil), do: nil
|
||||
|
||||
defp parse_integer(str) when is_binary(str) do
|
||||
case Integer.parse(str) do
|
||||
{n, _} -> n
|
||||
:error -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_integer(_), do: nil
|
||||
|
||||
defp parse_casing(nil), do: nil
|
||||
defp parse_casing(material), do: String.downcase(material)
|
||||
end
|
||||
146
lib/ammoprices/scraping/retailers/palmetto_state_armory.ex
Normal file
146
lib/ammoprices/scraping/retailers/palmetto_state_armory.ex
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
defmodule Ammoprices.Scraping.Retailers.PalmettoStateArmory do
|
||||
@moduledoc false
|
||||
@behaviour Ammoprices.Scraping.Scraper
|
||||
|
||||
alias Ammoprices.Scraping.TextDetector
|
||||
|
||||
@slug_to_path %{
|
||||
"9mm-luger" => "/9mm-ammo.html",
|
||||
"45-acp" => "/45-acp-ammo.html",
|
||||
"380-acp" => "/380-acp-ammo.html",
|
||||
"40-sw" => "/40-s-w-ammo.html",
|
||||
"38-special" => "/38-special-ammo.html",
|
||||
"357-magnum" => "/357-magnum-ammo.html",
|
||||
"10mm-auto" => "/10mm-auto-ammo.html",
|
||||
"556-223" => "/223-remington-ammo.html",
|
||||
"308-win" => "/308-winchester-ammo.html",
|
||||
"762x39" => "/7-62x39mm-ammo.html",
|
||||
"30-06" => "/30-06-springfield-ammo.html",
|
||||
"300-blackout" => "/300-aac-blackout-ammo.html",
|
||||
"65-creedmoor" => "/6-5-creedmoor-ammo.html",
|
||||
"22-lr" => "/22-lr-ammo.html",
|
||||
"22-wmr" => "/22-wmr-ammo.html",
|
||||
"17-hmr" => "/17-hmr-ammo.html",
|
||||
"12-gauge" => "/12-gauge-ammo.html",
|
||||
"16-gauge" => "/16-gauge-ammo.html",
|
||||
"20-gauge" => "/20-gauge-ammo.html"
|
||||
}
|
||||
|
||||
@impl true
|
||||
def retailer_slug, do: "palmetto-state-armory"
|
||||
|
||||
@impl true
|
||||
def category_url(caliber) do
|
||||
Map.get(@slug_to_path, caliber.slug)
|
||||
end
|
||||
|
||||
@impl true
|
||||
def parse_products(html) do
|
||||
html
|
||||
|> Floki.parse_document!()
|
||||
|> Floki.find("li.product-item")
|
||||
|> Enum.map(&parse_item/1)
|
||||
|> Enum.reject(&is_nil/1)
|
||||
end
|
||||
|
||||
defp parse_item(item) do
|
||||
with {:ok, title} <- extract_title(item),
|
||||
{:ok, url} <- extract_url(item),
|
||||
{:ok, price_cents} <- extract_price(item),
|
||||
{:ok, ppr_cents} <- extract_unit_price(item) do
|
||||
%{
|
||||
title: title,
|
||||
url: url,
|
||||
price_cents: price_cents,
|
||||
price_per_round_cents: ppr_cents,
|
||||
brand: extract_brand(title),
|
||||
grain_weight: extract_grain_weight(title),
|
||||
round_count: extract_round_count(title),
|
||||
casing: TextDetector.detect_casing(title),
|
||||
subsonic: TextDetector.detect_subsonic(title),
|
||||
in_stock: true
|
||||
}
|
||||
else
|
||||
_ -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp extract_title(item) do
|
||||
case Floki.find(item, ".product-item-link") do
|
||||
[{_, _, _} = node] -> {:ok, node |> Floki.text() |> String.trim()}
|
||||
_ -> :error
|
||||
end
|
||||
end
|
||||
|
||||
defp extract_url(item) do
|
||||
case Floki.find(item, ".product-item-link") do
|
||||
[{_, attrs, _}] ->
|
||||
case List.keyfind(attrs, "href", 0) do
|
||||
{"href", href} -> {:ok, href}
|
||||
_ -> :error
|
||||
end
|
||||
|
||||
_ ->
|
||||
:error
|
||||
end
|
||||
end
|
||||
|
||||
defp extract_price(item) do
|
||||
case Floki.find(item, "[data-price-type=\"finalPrice\"]") do
|
||||
[{_, attrs, _} | _] ->
|
||||
case List.keyfind(attrs, "data-price-amount", 0) do
|
||||
{"data-price-amount", amount} ->
|
||||
{:ok, amount |> String.to_float() |> Kernel.*(100) |> round()}
|
||||
|
||||
_ ->
|
||||
:error
|
||||
end
|
||||
|
||||
_ ->
|
||||
:error
|
||||
end
|
||||
end
|
||||
|
||||
defp extract_unit_price(item) do
|
||||
case Floki.find(item, "[data-price-type=\"unit_price\"]") do
|
||||
[{_, attrs, _} | _] ->
|
||||
case List.keyfind(attrs, "data-price-amount", 0) do
|
||||
{"data-price-amount", amount} ->
|
||||
{:ok, amount |> String.to_float() |> Kernel.*(100) |> round()}
|
||||
|
||||
_ ->
|
||||
:error
|
||||
end
|
||||
|
||||
_ ->
|
||||
:error
|
||||
end
|
||||
end
|
||||
|
||||
defp extract_brand(title) do
|
||||
case String.split(title, " ", parts: 2) do
|
||||
[brand | _] -> brand
|
||||
_ -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp extract_grain_weight(title) do
|
||||
case Regex.run(~r/(\d+)\s*(?:gr(?:ain)?)/i, title) do
|
||||
[_, weight] -> String.to_integer(weight)
|
||||
_ -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp extract_round_count(title) do
|
||||
cond do
|
||||
match = Regex.run(~r/(\d[\d,]*)\s*(?:rds?|rounds?)/i, title) ->
|
||||
match |> Enum.at(1) |> String.replace(",", "") |> String.to_integer()
|
||||
|
||||
match = Regex.run(~r/(\d+)\s*rd\/box/i, title) ->
|
||||
match |> Enum.at(1) |> String.to_integer()
|
||||
|
||||
true ->
|
||||
nil
|
||||
end
|
||||
end
|
||||
end
|
||||
97
lib/ammoprices/scraping/retailers/true_shot_ammo.ex
Normal file
97
lib/ammoprices/scraping/retailers/true_shot_ammo.ex
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
defmodule Ammoprices.Scraping.Retailers.TrueShotAmmo do
|
||||
@moduledoc false
|
||||
@behaviour Ammoprices.Scraping.Scraper
|
||||
|
||||
alias Ammoprices.Scraping.TextDetector
|
||||
|
||||
@slug_to_collection %{
|
||||
"9mm-luger" => "ammunition-pistol-ammo-9mm",
|
||||
"45-acp" => "ammunition-pistol-ammo-45-acp",
|
||||
"380-acp" => "ammunition-pistol-ammo-380-acp",
|
||||
"40-sw" => "ammunition-pistol-ammo-40-s-w",
|
||||
"38-special" => "ammunition-pistol-ammo-38-special",
|
||||
"357-magnum" => "ammunition-pistol-ammo-357-magnum",
|
||||
"10mm-auto" => "ammunition-pistol-ammo-10mm",
|
||||
"556-223" => "ammunition-rifle-ammo-223-5-56",
|
||||
"308-win" => "ammunition-rifle-ammo-308-7-62x51",
|
||||
"762x39" => "ammunition-rifle-ammo-7-62x39",
|
||||
"300-blackout" => "ammunition-rifle-ammo-300-blackout",
|
||||
"65-creedmoor" => "ammunition-rifle-ammo-6-5-creedmoor",
|
||||
"22-lr" => "ammunition-rimfire-ammo-22-lr",
|
||||
"12-gauge" => "ammunition-shotgun-ammo-12-gauge",
|
||||
"20-gauge" => "ammunition-shotgun-ammo-20-gauge"
|
||||
}
|
||||
|
||||
@impl true
|
||||
def retailer_slug, do: "true-shot-ammo"
|
||||
|
||||
@impl true
|
||||
def category_url(caliber) do
|
||||
case Map.get(@slug_to_collection, caliber.slug) do
|
||||
nil -> nil
|
||||
collection -> "/collections/#{collection}/products.json"
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def parse_products(%{"products" => products}) do
|
||||
products
|
||||
|> Enum.map(&parse_product/1)
|
||||
|> Enum.reject(&is_nil/1)
|
||||
end
|
||||
|
||||
def parse_products(_), do: []
|
||||
|
||||
defp parse_product(product) do
|
||||
title = product["title"]
|
||||
first_variant = List.first(product["variants"])
|
||||
|
||||
case first_variant do
|
||||
nil ->
|
||||
nil
|
||||
|
||||
variant ->
|
||||
price_cents = parse_price(variant["price"])
|
||||
round_count = parse_round_count(variant["title"])
|
||||
ppr = if round_count && round_count > 0, do: round(price_cents / round_count)
|
||||
|
||||
%{
|
||||
title: title,
|
||||
url: "/products/#{product["handle"]}",
|
||||
brand: product["vendor"],
|
||||
price_cents: price_cents,
|
||||
price_per_round_cents: ppr,
|
||||
grain_weight: extract_grain_weight(title),
|
||||
round_count: round_count,
|
||||
casing: TextDetector.detect_casing(title),
|
||||
subsonic: TextDetector.detect_subsonic(title),
|
||||
in_stock: variant["available"] == true
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_price(price_str) when is_binary(price_str) do
|
||||
price_str
|
||||
|> String.to_float()
|
||||
|> Kernel.*(100)
|
||||
|> round()
|
||||
end
|
||||
|
||||
defp parse_price(_), do: 0
|
||||
|
||||
defp parse_round_count(title) when is_binary(title) do
|
||||
case Integer.parse(title) do
|
||||
{count, _} -> count
|
||||
:error -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_round_count(_), do: nil
|
||||
|
||||
defp extract_grain_weight(title) do
|
||||
case Regex.run(~r/(\d+)\s*[Gg]rain/i, title) do
|
||||
[_, weight] -> String.to_integer(weight)
|
||||
_ -> nil
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -8,59 +8,35 @@ defmodule Ammoprices.Scraping.Runner do
|
|||
def run(scraper, caliber) do
|
||||
retailer = Catalog.get_retailer_by_slug!(scraper.retailer_slug())
|
||||
|
||||
case scraper.category_url(caliber) do
|
||||
nil ->
|
||||
{:ok, %{products_count: 0, snapshots_count: 0}}
|
||||
if function_exported?(scraper, :fetch, 2) do
|
||||
do_fetch(scraper, retailer, caliber)
|
||||
else
|
||||
case scraper.category_url(caliber) do
|
||||
nil ->
|
||||
{:ok, %{products_count: 0, snapshots_count: 0}}
|
||||
|
||||
path ->
|
||||
url = retailer.base_url <> path
|
||||
do_scrape(scraper, retailer, caliber, url)
|
||||
path ->
|
||||
url = retailer.base_url <> path
|
||||
do_scrape(scraper, retailer, caliber, url)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
defp do_fetch(scraper, retailer, caliber) do
|
||||
case scraper.fetch(retailer, caliber) do
|
||||
{:ok, parsed} ->
|
||||
process_results(parsed, retailer, caliber)
|
||||
|
||||
{:error, reason} ->
|
||||
{:error, reason}
|
||||
end
|
||||
end
|
||||
|
||||
defp do_scrape(scraper, retailer, caliber, url) do
|
||||
case HttpClient.get(url) do
|
||||
{:ok, %{status: 200, body: body}} ->
|
||||
now = DateTime.truncate(DateTime.utc_now(), :second)
|
||||
parsed = scraper.parse_products(body)
|
||||
|
||||
results =
|
||||
Enum.map(parsed, fn product_data ->
|
||||
product_attrs = %{
|
||||
title: product_data.title,
|
||||
url: product_data.url,
|
||||
brand: product_data.brand,
|
||||
grain_weight: product_data.grain_weight,
|
||||
round_count: product_data.round_count,
|
||||
casing: product_data.casing,
|
||||
condition: "new",
|
||||
in_stock: product_data.in_stock,
|
||||
subsonic: Map.get(product_data, :subsonic, false),
|
||||
last_seen_at: now
|
||||
}
|
||||
|
||||
case Catalog.upsert_product(retailer.id, caliber.id, product_attrs) do
|
||||
{:ok, product} ->
|
||||
snapshot_attrs = %{
|
||||
price_cents: product_data.price_cents,
|
||||
price_per_round_cents: product_data.price_per_round_cents,
|
||||
in_stock: product_data.in_stock,
|
||||
recorded_at: now
|
||||
}
|
||||
|
||||
{:ok, _snapshot} = Prices.create_snapshot(product.id, snapshot_attrs)
|
||||
:ok
|
||||
|
||||
{:error, _changeset} ->
|
||||
:error
|
||||
end
|
||||
end)
|
||||
|
||||
successful = Enum.count(results, &(&1 == :ok))
|
||||
|
||||
Catalog.update_retailer(retailer, %{last_scraped_at: now})
|
||||
|
||||
{:ok, %{products_count: successful, snapshots_count: successful}}
|
||||
process_results(parsed, retailer, caliber)
|
||||
|
||||
{:ok, %{status: status}} ->
|
||||
{:error, "HTTP #{status}"}
|
||||
|
|
@ -69,4 +45,46 @@ defmodule Ammoprices.Scraping.Runner do
|
|||
{:error, reason}
|
||||
end
|
||||
end
|
||||
|
||||
defp process_results(parsed, retailer, caliber) do
|
||||
now = DateTime.truncate(DateTime.utc_now(), :second)
|
||||
|
||||
results =
|
||||
Enum.map(parsed, fn product_data ->
|
||||
product_attrs = %{
|
||||
title: product_data.title,
|
||||
url: product_data.url,
|
||||
brand: product_data.brand,
|
||||
grain_weight: product_data.grain_weight,
|
||||
round_count: product_data.round_count,
|
||||
casing: product_data.casing,
|
||||
condition: "new",
|
||||
in_stock: product_data.in_stock,
|
||||
subsonic: Map.get(product_data, :subsonic, false),
|
||||
last_seen_at: now
|
||||
}
|
||||
|
||||
case Catalog.upsert_product(retailer.id, caliber.id, product_attrs) do
|
||||
{:ok, product} ->
|
||||
snapshot_attrs = %{
|
||||
price_cents: product_data.price_cents,
|
||||
price_per_round_cents: product_data.price_per_round_cents,
|
||||
in_stock: product_data.in_stock,
|
||||
recorded_at: now
|
||||
}
|
||||
|
||||
{:ok, _snapshot} = Prices.create_snapshot(product.id, snapshot_attrs)
|
||||
:ok
|
||||
|
||||
{:error, _changeset} ->
|
||||
:error
|
||||
end
|
||||
end)
|
||||
|
||||
successful = Enum.count(results, &(&1 == :ok))
|
||||
|
||||
Catalog.update_retailer(retailer, %{last_scraped_at: now})
|
||||
|
||||
{:ok, %{products_count: successful, snapshots_count: successful}}
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -8,7 +8,11 @@ defmodule Ammoprices.Scraping.ScrapeJob do
|
|||
@scrapers [
|
||||
Ammoprices.Scraping.Retailers.LuckyGunner,
|
||||
Ammoprices.Scraping.Retailers.SgAmmo,
|
||||
Ammoprices.Scraping.Retailers.TargetSportsUsa
|
||||
Ammoprices.Scraping.Retailers.TargetSportsUsa,
|
||||
Ammoprices.Scraping.Retailers.TrueShotAmmo,
|
||||
Ammoprices.Scraping.Retailers.PalmettoStateArmory,
|
||||
Ammoprices.Scraping.Retailers.BulkAmmo,
|
||||
Ammoprices.Scraping.Retailers.Natchez
|
||||
]
|
||||
|
||||
@impl Oban.Worker
|
||||
|
|
|
|||
|
|
@ -3,5 +3,8 @@ defmodule Ammoprices.Scraping.Scraper do
|
|||
|
||||
@callback retailer_slug() :: String.t()
|
||||
@callback category_url(caliber :: map()) :: String.t() | nil
|
||||
@callback parse_products(html :: String.t()) :: [map()]
|
||||
@callback parse_products(html :: term()) :: [map()]
|
||||
@callback fetch(retailer :: map(), caliber :: map()) :: {:ok, [map()]} | {:error, term()}
|
||||
|
||||
@optional_callbacks [fetch: 2]
|
||||
end
|
||||
|
|
|
|||
36
priv/repo/migrations/20260312144740_add_new_retailers.exs
Normal file
36
priv/repo/migrations/20260312144740_add_new_retailers.exs
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
defmodule Ammoprices.Repo.Migrations.AddNewRetailers do
|
||||
use Ecto.Migration
|
||||
|
||||
def up do
|
||||
now = DateTime.utc_now() |> DateTime.truncate(:second) |> DateTime.to_naive()
|
||||
|
||||
retailers = [
|
||||
%{name: "True Shot Ammo", slug: "true-shot-ammo", base_url: "https://trueshotammo.com"},
|
||||
%{
|
||||
name: "Palmetto State Armory",
|
||||
slug: "palmetto-state-armory",
|
||||
base_url: "https://palmettostatearmory.com"
|
||||
},
|
||||
%{name: "Bulk Ammo", slug: "bulk-ammo", base_url: "https://www.bulkammo.com"},
|
||||
%{
|
||||
name: "Natchez Shooters Supply",
|
||||
slug: "natchez",
|
||||
base_url: "https://www.natchezss.com"
|
||||
}
|
||||
]
|
||||
|
||||
for r <- retailers do
|
||||
execute("""
|
||||
INSERT INTO retailers (id, name, slug, base_url, inserted_at, updated_at)
|
||||
VALUES (gen_random_uuid(), '#{r.name}', '#{r.slug}', '#{r.base_url}', '#{now}', '#{now}')
|
||||
ON CONFLICT (slug) DO NOTHING
|
||||
""")
|
||||
end
|
||||
end
|
||||
|
||||
def down do
|
||||
execute(
|
||||
"DELETE FROM retailers WHERE slug IN ('true-shot-ammo', 'palmetto-state-armory', 'bulk-ammo', 'natchez')"
|
||||
)
|
||||
end
|
||||
end
|
||||
115
test/ammoprices/scraping/retailers/bulk_ammo_test.exs
Normal file
115
test/ammoprices/scraping/retailers/bulk_ammo_test.exs
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
defmodule Ammoprices.Scraping.Retailers.BulkAmmoTest do
|
||||
use ExUnit.Case, async: true
|
||||
|
||||
alias Ammoprices.Scraping.Retailers.BulkAmmo
|
||||
|
||||
@fixture_path "test/fixtures/bulk_ammo/9mm.html"
|
||||
|
||||
describe "retailer_slug/0" do
|
||||
test "returns bulk-ammo" do
|
||||
assert BulkAmmo.retailer_slug() == "bulk-ammo"
|
||||
end
|
||||
end
|
||||
|
||||
describe "category_url/1" do
|
||||
test "maps 9mm-luger caliber to correct URL" do
|
||||
caliber = %{slug: "9mm-luger"}
|
||||
assert BulkAmmo.category_url(caliber) == "/handgun/bulk-9mm-ammo"
|
||||
end
|
||||
|
||||
test "maps 556-223 caliber to correct URL" do
|
||||
caliber = %{slug: "556-223"}
|
||||
assert BulkAmmo.category_url(caliber) == "/rifle/bulk-223-remington-ammo"
|
||||
end
|
||||
|
||||
test "returns nil for unmapped caliber" do
|
||||
caliber = %{slug: "unknown-caliber"}
|
||||
assert BulkAmmo.category_url(caliber) == nil
|
||||
end
|
||||
end
|
||||
|
||||
describe "parse_products/1" do
|
||||
setup do
|
||||
html = File.read!(@fixture_path)
|
||||
%{html: html}
|
||||
end
|
||||
|
||||
test "extracts all products from HTML", %{html: html} do
|
||||
products = BulkAmmo.parse_products(html)
|
||||
assert length(products) == 4
|
||||
end
|
||||
|
||||
test "extracts product title", %{html: html} do
|
||||
[first | _] = BulkAmmo.parse_products(html)
|
||||
assert first.title == "1500 Rounds of 9mm Ammo by Sterling Steel - 115gr FMJ"
|
||||
end
|
||||
|
||||
test "extracts product URL", %{html: html} do
|
||||
[first | _] = BulkAmmo.parse_products(html)
|
||||
assert first.url == "https://www.bulkammo.com/1500-rounds-of-9mm-ammo-by-sterling-115gr-fmj"
|
||||
end
|
||||
|
||||
test "extracts sale price when present", %{html: html} do
|
||||
[first | _] = BulkAmmo.parse_products(html)
|
||||
# $285 sale price
|
||||
assert first.price_cents == 28_500
|
||||
end
|
||||
|
||||
test "extracts regular price when no sale", %{html: html} do
|
||||
products = BulkAmmo.parse_products(html)
|
||||
second = Enum.at(products, 1)
|
||||
# $239.99 regular price
|
||||
assert second.price_cents == 23_999
|
||||
end
|
||||
|
||||
test "extracts round count from title", %{html: html} do
|
||||
[first | _] = BulkAmmo.parse_products(html)
|
||||
assert first.round_count == 1500
|
||||
end
|
||||
|
||||
test "calculates price per round", %{html: html} do
|
||||
[first | _] = BulkAmmo.parse_products(html)
|
||||
# 28500 / 1500 = 19
|
||||
assert first.price_per_round_cents == 19
|
||||
end
|
||||
|
||||
test "extracts brand from title", %{html: html} do
|
||||
products = BulkAmmo.parse_products(html)
|
||||
assert hd(products).brand == "Sterling Steel"
|
||||
assert Enum.at(products, 1).brand == "Remington"
|
||||
assert Enum.at(products, 2).brand == "Federal Brass"
|
||||
end
|
||||
|
||||
test "extracts grain weight from title", %{html: html} do
|
||||
[first | _] = BulkAmmo.parse_products(html)
|
||||
assert first.grain_weight == 115
|
||||
end
|
||||
|
||||
test "detects in-stock status", %{html: html} do
|
||||
products = BulkAmmo.parse_products(html)
|
||||
assert hd(products).in_stock == true
|
||||
assert Enum.at(products, 1).in_stock == true
|
||||
# Fourth product is out of stock
|
||||
assert Enum.at(products, 3).in_stock == false
|
||||
end
|
||||
|
||||
test "detects casing from title", %{html: html} do
|
||||
products = BulkAmmo.parse_products(html)
|
||||
# First product: "Sterling Steel" in title
|
||||
assert hd(products).casing == "steel"
|
||||
# Third product: "Federal Brass" in title
|
||||
assert Enum.at(products, 2).casing == "brass"
|
||||
end
|
||||
|
||||
test "detects subsonic from title", %{html: html} do
|
||||
products = BulkAmmo.parse_products(html)
|
||||
# Fourth product: "Subsonic" in title
|
||||
assert Enum.at(products, 3).subsonic == true
|
||||
end
|
||||
|
||||
test "non-subsonic products return false", %{html: html} do
|
||||
[first | _] = BulkAmmo.parse_products(html)
|
||||
assert first.subsonic == false
|
||||
end
|
||||
end
|
||||
end
|
||||
110
test/ammoprices/scraping/retailers/natchez_test.exs
Normal file
110
test/ammoprices/scraping/retailers/natchez_test.exs
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
defmodule Ammoprices.Scraping.Retailers.NatchezTest do
|
||||
use ExUnit.Case, async: true
|
||||
|
||||
alias Ammoprices.Scraping.Retailers.Natchez
|
||||
|
||||
@fixture_path "test/fixtures/natchez/9mm.json"
|
||||
|
||||
describe "retailer_slug/0" do
|
||||
test "returns natchez" do
|
||||
assert Natchez.retailer_slug() == "natchez"
|
||||
end
|
||||
end
|
||||
|
||||
describe "category_url/1" do
|
||||
test "returns nil for all calibers since fetch/2 is used" do
|
||||
caliber = %{slug: "9mm-luger"}
|
||||
assert Natchez.category_url(caliber) == nil
|
||||
end
|
||||
end
|
||||
|
||||
describe "build_query/1" do
|
||||
test "builds GraphQL query for mapped caliber" do
|
||||
query = Natchez.build_query("9mm-luger")
|
||||
assert query =~ "9mm Luger"
|
||||
assert query =~ "caliber"
|
||||
assert query =~ "pageSize"
|
||||
end
|
||||
|
||||
test "returns nil for unmapped caliber" do
|
||||
assert Natchez.build_query("unknown-caliber") == nil
|
||||
end
|
||||
end
|
||||
|
||||
describe "parse_graphql_response/1" do
|
||||
setup do
|
||||
body = @fixture_path |> File.read!() |> Jason.decode!()
|
||||
%{body: body}
|
||||
end
|
||||
|
||||
test "extracts all products from response", %{body: body} do
|
||||
products = Natchez.parse_graphql_response(body)
|
||||
assert length(products) == 4
|
||||
end
|
||||
|
||||
test "extracts product title from name", %{body: body} do
|
||||
[first | _] = Natchez.parse_graphql_response(body)
|
||||
|
||||
assert first.title ==
|
||||
"CCI Blazer Brass Handgun Ammunition 9mm Luger 115 gr FMJ 1145 fps 1000/ct Loose Bulk Pack"
|
||||
end
|
||||
|
||||
test "extracts product URL from url_key", %{body: body} do
|
||||
[first | _] = Natchez.parse_graphql_response(body)
|
||||
assert first.url == "/cci-9mm-115gr-fmj-1000-rds-bulk-pack-cc5000bk1000.html"
|
||||
end
|
||||
|
||||
test "extracts brand directly", %{body: body} do
|
||||
[first | _] = Natchez.parse_graphql_response(body)
|
||||
assert first.brand == "CCI"
|
||||
end
|
||||
|
||||
test "extracts price in cents", %{body: body} do
|
||||
[first | _] = Natchez.parse_graphql_response(body)
|
||||
assert first.price_cents == 24_999
|
||||
end
|
||||
|
||||
test "extracts grain weight from grain field", %{body: body} do
|
||||
[first | _] = Natchez.parse_graphql_response(body)
|
||||
# "115 gr" → 115
|
||||
assert first.grain_weight == 115
|
||||
end
|
||||
|
||||
test "extracts round count from rounds field", %{body: body} do
|
||||
[first | _] = Natchez.parse_graphql_response(body)
|
||||
assert first.round_count == 1000
|
||||
end
|
||||
|
||||
test "calculates price per round", %{body: body} do
|
||||
[first | _] = Natchez.parse_graphql_response(body)
|
||||
# 24999 / 1000 = 24.999 → 25
|
||||
assert first.price_per_round_cents == 25
|
||||
end
|
||||
|
||||
test "extracts casing from case_material", %{body: body} do
|
||||
products = Natchez.parse_graphql_response(body)
|
||||
assert hd(products).casing == "brass"
|
||||
# Third product: Steel case
|
||||
assert Enum.at(products, 2).casing == "steel"
|
||||
end
|
||||
|
||||
test "detects in_stock from stock_status", %{body: body} do
|
||||
products = Natchez.parse_graphql_response(body)
|
||||
# First: OUT_OF_STOCK
|
||||
assert hd(products).in_stock == false
|
||||
# Second: IN_STOCK
|
||||
assert Enum.at(products, 1).in_stock == true
|
||||
end
|
||||
|
||||
test "detects subsonic from name", %{body: body} do
|
||||
products = Natchez.parse_graphql_response(body)
|
||||
# Fourth product has "Subsonic" in name
|
||||
assert Enum.at(products, 3).subsonic == true
|
||||
end
|
||||
|
||||
test "non-subsonic products return false", %{body: body} do
|
||||
[first | _] = Natchez.parse_graphql_response(body)
|
||||
assert first.subsonic == false
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
defmodule Ammoprices.Scraping.Retailers.PalmettoStateArmoryTest do
|
||||
use ExUnit.Case, async: true
|
||||
|
||||
alias Ammoprices.Scraping.Retailers.PalmettoStateArmory
|
||||
|
||||
@fixture_path "test/fixtures/palmetto_state_armory/9mm.html"
|
||||
|
||||
describe "retailer_slug/0" do
|
||||
test "returns palmetto-state-armory" do
|
||||
assert PalmettoStateArmory.retailer_slug() == "palmetto-state-armory"
|
||||
end
|
||||
end
|
||||
|
||||
describe "category_url/1" do
|
||||
test "maps 9mm-luger caliber to correct URL" do
|
||||
caliber = %{slug: "9mm-luger"}
|
||||
assert PalmettoStateArmory.category_url(caliber) == "/9mm-ammo.html"
|
||||
end
|
||||
|
||||
test "maps 556-223 caliber to correct URL" do
|
||||
caliber = %{slug: "556-223"}
|
||||
assert PalmettoStateArmory.category_url(caliber) == "/223-remington-ammo.html"
|
||||
end
|
||||
|
||||
test "returns nil for unmapped caliber" do
|
||||
caliber = %{slug: "unknown-caliber"}
|
||||
assert PalmettoStateArmory.category_url(caliber) == nil
|
||||
end
|
||||
end
|
||||
|
||||
describe "parse_products/1" do
|
||||
setup do
|
||||
html = File.read!(@fixture_path)
|
||||
%{html: html}
|
||||
end
|
||||
|
||||
test "extracts all products from HTML", %{html: html} do
|
||||
products = PalmettoStateArmory.parse_products(html)
|
||||
assert length(products) == 4
|
||||
end
|
||||
|
||||
test "extracts product title", %{html: html} do
|
||||
[first | _] = PalmettoStateArmory.parse_products(html)
|
||||
assert first.title == "TROY 9MM 115GR FMJ Ammo, 50rds - YTR9M115FMJ"
|
||||
end
|
||||
|
||||
test "extracts product URL", %{html: html} do
|
||||
[first | _] = PalmettoStateArmory.parse_products(html)
|
||||
|
||||
assert first.url ==
|
||||
"https://palmettostatearmory.com/troy-9mm-115gr-fmj-ammo-50rds-ytr9m115fmj.html"
|
||||
end
|
||||
|
||||
test "extracts price in cents from data-price-amount", %{html: html} do
|
||||
[first | _] = PalmettoStateArmory.parse_products(html)
|
||||
assert first.price_cents == 1199
|
||||
end
|
||||
|
||||
test "extracts price per round in cents from unit_price", %{html: html} do
|
||||
[first | _] = PalmettoStateArmory.parse_products(html)
|
||||
# 0.2398 * 100 = 23.98 → 24 cents
|
||||
assert first.price_per_round_cents == 24
|
||||
end
|
||||
|
||||
test "extracts grain weight from title", %{html: html} do
|
||||
[first | _] = PalmettoStateArmory.parse_products(html)
|
||||
assert first.grain_weight == 115
|
||||
end
|
||||
|
||||
test "extracts round count from title", %{html: html} do
|
||||
[first | _] = PalmettoStateArmory.parse_products(html)
|
||||
assert first.round_count == 50
|
||||
end
|
||||
|
||||
test "marks all listed products as in stock", %{html: html} do
|
||||
products = PalmettoStateArmory.parse_products(html)
|
||||
assert Enum.all?(products, & &1.in_stock)
|
||||
end
|
||||
|
||||
test "detects casing from title", %{html: html} do
|
||||
products = PalmettoStateArmory.parse_products(html)
|
||||
# Third product: "Steel Case" in title
|
||||
assert Enum.at(products, 2).casing == "steel"
|
||||
# Second product: "Brass" in title
|
||||
assert Enum.at(products, 1).casing == "brass"
|
||||
end
|
||||
|
||||
test "detects subsonic from title", %{html: html} do
|
||||
products = PalmettoStateArmory.parse_products(html)
|
||||
# Fourth product: "Subsonic" in title
|
||||
assert Enum.at(products, 3).subsonic == true
|
||||
end
|
||||
|
||||
test "non-subsonic products return false", %{html: html} do
|
||||
[first | _] = PalmettoStateArmory.parse_products(html)
|
||||
assert first.subsonic == false
|
||||
end
|
||||
|
||||
test "extracts brand from beginning of title", %{html: html} do
|
||||
products = PalmettoStateArmory.parse_products(html)
|
||||
assert hd(products).brand == "TROY"
|
||||
assert Enum.at(products, 1).brand == "CCI"
|
||||
assert Enum.at(products, 2).brand == "Tula"
|
||||
end
|
||||
end
|
||||
end
|
||||
105
test/ammoprices/scraping/retailers/true_shot_ammo_test.exs
Normal file
105
test/ammoprices/scraping/retailers/true_shot_ammo_test.exs
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
defmodule Ammoprices.Scraping.Retailers.TrueShotAmmoTest do
|
||||
use ExUnit.Case, async: true
|
||||
|
||||
alias Ammoprices.Scraping.Retailers.TrueShotAmmo
|
||||
|
||||
@fixture_path "test/fixtures/true_shot_ammo/9mm.json"
|
||||
|
||||
describe "retailer_slug/0" do
|
||||
test "returns true-shot-ammo" do
|
||||
assert TrueShotAmmo.retailer_slug() == "true-shot-ammo"
|
||||
end
|
||||
end
|
||||
|
||||
describe "category_url/1" do
|
||||
test "maps 9mm-luger caliber to correct URL" do
|
||||
caliber = %{slug: "9mm-luger"}
|
||||
assert TrueShotAmmo.category_url(caliber) == "/collections/ammunition-pistol-ammo-9mm/products.json"
|
||||
end
|
||||
|
||||
test "maps 556-223 caliber to correct URL" do
|
||||
caliber = %{slug: "556-223"}
|
||||
assert TrueShotAmmo.category_url(caliber) == "/collections/ammunition-rifle-ammo-223-5-56/products.json"
|
||||
end
|
||||
|
||||
test "returns nil for unmapped caliber" do
|
||||
caliber = %{slug: "unknown-caliber"}
|
||||
assert TrueShotAmmo.category_url(caliber) == nil
|
||||
end
|
||||
end
|
||||
|
||||
describe "parse_products/1" do
|
||||
setup do
|
||||
json = @fixture_path |> File.read!() |> Jason.decode!()
|
||||
%{json: json}
|
||||
end
|
||||
|
||||
test "extracts all products from JSON", %{json: json} do
|
||||
products = TrueShotAmmo.parse_products(json)
|
||||
assert length(products) == 4
|
||||
end
|
||||
|
||||
test "extracts product title", %{json: json} do
|
||||
[first | _] = TrueShotAmmo.parse_products(json)
|
||||
assert first.title == "CCI - Blazer - 9mm - 115 Grain - FMJ"
|
||||
end
|
||||
|
||||
test "extracts product URL from handle", %{json: json} do
|
||||
[first | _] = TrueShotAmmo.parse_products(json)
|
||||
assert first.url == "/products/cci-blazer-9mm-115-grain-fmj-2"
|
||||
end
|
||||
|
||||
test "extracts brand from vendor", %{json: json} do
|
||||
[first | _] = TrueShotAmmo.parse_products(json)
|
||||
assert first.brand == "CCI"
|
||||
end
|
||||
|
||||
test "extracts price in cents from first variant", %{json: json} do
|
||||
[first | _] = TrueShotAmmo.parse_products(json)
|
||||
assert first.price_cents == 1299
|
||||
end
|
||||
|
||||
test "extracts in_stock from first variant availability", %{json: json} do
|
||||
products = TrueShotAmmo.parse_products(json)
|
||||
# First product: all variants unavailable
|
||||
assert hd(products).in_stock == false
|
||||
# Second product: first variant available
|
||||
assert Enum.at(products, 1).in_stock == true
|
||||
end
|
||||
|
||||
test "extracts grain weight from title", %{json: json} do
|
||||
[first | _] = TrueShotAmmo.parse_products(json)
|
||||
assert first.grain_weight == 115
|
||||
end
|
||||
|
||||
test "extracts round count from first variant title", %{json: json} do
|
||||
[first | _] = TrueShotAmmo.parse_products(json)
|
||||
assert first.round_count == 50
|
||||
end
|
||||
|
||||
test "calculates price per round", %{json: json} do
|
||||
[first | _] = TrueShotAmmo.parse_products(json)
|
||||
# 1299 cents / 50 rounds = 25.98 → 26 cents
|
||||
assert first.price_per_round_cents == 26
|
||||
end
|
||||
|
||||
test "detects casing from title", %{json: json} do
|
||||
products = TrueShotAmmo.parse_products(json)
|
||||
# Second product has "Steel Case" in title
|
||||
assert Enum.at(products, 1).casing == "steel"
|
||||
# Third product has "Brass Case" in title
|
||||
assert Enum.at(products, 2).casing == "brass"
|
||||
end
|
||||
|
||||
test "detects subsonic from title", %{json: json} do
|
||||
products = TrueShotAmmo.parse_products(json)
|
||||
# Fourth product has "Subsonic" in title
|
||||
assert Enum.at(products, 3).subsonic == true
|
||||
end
|
||||
|
||||
test "non-subsonic products return false", %{json: json} do
|
||||
[first | _] = TrueShotAmmo.parse_products(json)
|
||||
assert first.subsonic == false
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -18,6 +18,30 @@ defmodule Ammoprices.Scraping.ScrapeJobTest do
|
|||
base_url: "https://www.targetsportsusa.com"
|
||||
})
|
||||
|
||||
retailer_fixture(%{
|
||||
name: "True Shot Ammo",
|
||||
slug: "true-shot-ammo",
|
||||
base_url: "https://trueshotammo.com"
|
||||
})
|
||||
|
||||
retailer_fixture(%{
|
||||
name: "Palmetto State Armory",
|
||||
slug: "palmetto-state-armory",
|
||||
base_url: "https://palmettostatearmory.com"
|
||||
})
|
||||
|
||||
retailer_fixture(%{
|
||||
name: "Bulk Ammo",
|
||||
slug: "bulk-ammo",
|
||||
base_url: "https://www.bulkammo.com"
|
||||
})
|
||||
|
||||
retailer_fixture(%{
|
||||
name: "Natchez Shooters Supply",
|
||||
slug: "natchez",
|
||||
base_url: "https://www.natchezss.com"
|
||||
})
|
||||
|
||||
caliber_fixture(%{name: "9mm Luger", slug: "9mm-luger", category: "handgun", aliases: ["9mm", "9x19"]})
|
||||
|
||||
Req.Test.stub(Ammoprices.Scraping.HttpClient, fn conn ->
|
||||
|
|
|
|||
97
test/fixtures/bulk_ammo/9mm.html
vendored
Normal file
97
test/fixtures/bulk_ammo/9mm.html
vendored
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
<div class="category-products">
|
||||
<ul class="products-grid">
|
||||
<li class="item first">
|
||||
<div>
|
||||
<a href="https://www.bulkammo.com/1500-rounds-of-9mm-ammo-by-sterling-115gr-fmj" title="1500 Rounds of 9mm Ammo by Sterling Steel - 115gr FMJ" class="product-image">
|
||||
<img src="https://cdn.bulkammo.com/picture-gallery/small_image/sterling.jpg" alt="1500 Rounds of 9mm Ammo by Sterling Steel - 115gr FMJ">
|
||||
</a>
|
||||
<a class="product-name" href="https://www.bulkammo.com/1500-rounds-of-9mm-ammo-by-sterling-115gr-fmj" title="1500 Rounds of 9mm Ammo by Sterling Steel - 115gr FMJ">1500 Rounds of 9mm Ammo by Sterling Steel - 115gr FMJ</a>
|
||||
</div>
|
||||
<div>
|
||||
<div class="price-box">
|
||||
<p class="old-price">
|
||||
<span class="price-label">Regular Price:</span>
|
||||
<span class="price" id="old-price-sterling">$304</span>
|
||||
</p>
|
||||
<p class="special-price">
|
||||
<span class="price-label">On Sale:</span>
|
||||
<span class="price" id="product-price-sterling">$285</span>
|
||||
</p>
|
||||
</div>
|
||||
<div class="actions">
|
||||
<p class="availability">
|
||||
<span class="in-stock">
|
||||
<span class="stock-qty">101 </span>
|
||||
Ready to Ship
|
||||
</span>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</li>
|
||||
<li class="item">
|
||||
<div>
|
||||
<a href="https://www.bulkammo.com/1000-rounds-of-9mm-ammo-by-remington-115gr-fmj" title="1000 Rounds of 9mm Ammo by Remington - 115gr FMJ" class="product-image">
|
||||
<img src="https://cdn.bulkammo.com/picture-gallery/small_image/remington.jpg" alt="1000 Rounds of 9mm Ammo by Remington - 115gr FMJ">
|
||||
</a>
|
||||
<a class="product-name" href="https://www.bulkammo.com/1000-rounds-of-9mm-ammo-by-remington-115gr-fmj" title="1000 Rounds of 9mm Ammo by Remington - 115gr FMJ">1000 Rounds of 9mm Ammo by Remington - 115gr FMJ</a>
|
||||
</div>
|
||||
<div>
|
||||
<div class="price-box">
|
||||
<span class="regular-price" id="product-price-remington">
|
||||
<span class="price">$239.99</span>
|
||||
</span>
|
||||
</div>
|
||||
<div class="actions">
|
||||
<p class="availability">
|
||||
<span class="in-stock">
|
||||
<span class="stock-qty">50 </span>
|
||||
Ready to Ship
|
||||
</span>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</li>
|
||||
<li class="item">
|
||||
<div>
|
||||
<a href="https://www.bulkammo.com/500-rounds-of-9mm-ammo-by-federal-brass-124gr-fmj" title="500 Rounds of 9mm Ammo by Federal Brass - 124gr FMJ" class="product-image">
|
||||
<img src="https://cdn.bulkammo.com/picture-gallery/small_image/federal.jpg" alt="500 Rounds of 9mm Ammo by Federal Brass - 124gr FMJ">
|
||||
</a>
|
||||
<a class="product-name" href="https://www.bulkammo.com/500-rounds-of-9mm-ammo-by-federal-brass-124gr-fmj" title="500 Rounds of 9mm Ammo by Federal Brass - 124gr FMJ">500 Rounds of 9mm Ammo by Federal Brass - 124gr FMJ</a>
|
||||
</div>
|
||||
<div>
|
||||
<div class="price-box">
|
||||
<span class="regular-price" id="product-price-federal">
|
||||
<span class="price">$149.50</span>
|
||||
</span>
|
||||
</div>
|
||||
<div class="actions">
|
||||
<p class="availability">
|
||||
<span class="in-stock">
|
||||
Ready to Ship
|
||||
</span>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</li>
|
||||
<li class="item last">
|
||||
<div>
|
||||
<a href="https://www.bulkammo.com/50-rounds-of-9mm-subsonic-ammo-by-fiocchi-147gr-fmj" title="50 Rounds of 9mm Subsonic Ammo by Fiocchi - 147gr FMJ" class="product-image">
|
||||
<img src="https://cdn.bulkammo.com/picture-gallery/small_image/fiocchi.jpg" alt="50 Rounds of 9mm Subsonic Ammo by Fiocchi - 147gr FMJ">
|
||||
</a>
|
||||
<a class="product-name" href="https://www.bulkammo.com/50-rounds-of-9mm-subsonic-ammo-by-fiocchi-147gr-fmj" title="50 Rounds of 9mm Subsonic Ammo by Fiocchi - 147gr FMJ">50 Rounds of 9mm Subsonic Ammo by Fiocchi - 147gr FMJ</a>
|
||||
</div>
|
||||
<div>
|
||||
<div class="price-box">
|
||||
<span class="regular-price" id="product-price-fiocchi">
|
||||
<span class="price">$18.99</span>
|
||||
</span>
|
||||
</div>
|
||||
<div class="actions">
|
||||
<p class="availability">
|
||||
<span class="out-of-stock">Out of Stock</span>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
80
test/fixtures/natchez/9mm.json
vendored
Normal file
80
test/fixtures/natchez/9mm.json
vendored
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
{
|
||||
"data": {
|
||||
"products": {
|
||||
"items": [
|
||||
{
|
||||
"name": "CCI Blazer Brass Handgun Ammunition 9mm Luger 115 gr FMJ 1145 fps 1000/ct Loose Bulk Pack",
|
||||
"sku": "CC5000BK1000",
|
||||
"url_key": "cci-9mm-115gr-fmj-1000-rds-bulk-pack-cc5000bk1000",
|
||||
"stock_status": "OUT_OF_STOCK",
|
||||
"brand": "CCI",
|
||||
"grain": "115 gr",
|
||||
"rounds": "1000",
|
||||
"case_material": "Brass",
|
||||
"price_range": {
|
||||
"minimum_price": {
|
||||
"final_price": {
|
||||
"value": "249.99"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "CCI Blazer Brass Handgun Ammunition 9mm Luger 124 gr. FMJ 1090 fps 1000/ct (20-50/ct)",
|
||||
"sku": "CC5201C",
|
||||
"url_key": "cci-blazer-brass-handgun-ammunition-9mm-luger-124-gr-fmj-1090-fps-1000ct-20-50ct",
|
||||
"stock_status": "IN_STOCK",
|
||||
"brand": "CCI",
|
||||
"grain": "124 gr",
|
||||
"rounds": "1000",
|
||||
"case_material": "Brass",
|
||||
"price_range": {
|
||||
"minimum_price": {
|
||||
"final_price": {
|
||||
"value": "259.99"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Wolf Performance 9mm Luger 115 gr FMJ Steel Case 50/ct",
|
||||
"sku": "WO917",
|
||||
"url_key": "wolf-performance-9mm-luger-115gr-fmj-steel-case-50ct",
|
||||
"stock_status": "IN_STOCK",
|
||||
"brand": "Wolf",
|
||||
"grain": "115 gr",
|
||||
"rounds": "50",
|
||||
"case_material": "Steel",
|
||||
"price_range": {
|
||||
"minimum_price": {
|
||||
"final_price": {
|
||||
"value": "11.99"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Fiocchi Subsonic Handgun Ammunition 9mm Luger 147 gr FMJ 950 fps 50/ct",
|
||||
"sku": "FI9APE",
|
||||
"url_key": "fiocchi-subsonic-handgun-ammunition-9mm-luger-147-gr-fmj-950-fps-50ct",
|
||||
"stock_status": "IN_STOCK",
|
||||
"brand": "Fiocchi",
|
||||
"grain": "147 gr",
|
||||
"rounds": "50",
|
||||
"case_material": "Brass",
|
||||
"price_range": {
|
||||
"minimum_price": {
|
||||
"final_price": {
|
||||
"value": "16.99"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"total_count": 48,
|
||||
"page_info": {
|
||||
"total_pages": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
160
test/fixtures/palmetto_state_armory/9mm.html
vendored
Normal file
160
test/fixtures/palmetto_state_armory/9mm.html
vendored
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
<div class="products-grid">
|
||||
<ol class="products list items product-items">
|
||||
<li class="item product product-item" id="product-159202">
|
||||
<div class="product-item-info" data-container="product-list">
|
||||
<a href="https://palmettostatearmory.com/troy-9mm-115gr-fmj-ammo-50rds-ytr9m115fmj.html" class="product photo product-item-photo" tabindex="-1">
|
||||
<span class="product-image-container">
|
||||
<img class="product-image-photo" src="https://palmettostatearmory.com/media/catalog/product/troy.png" alt="TROY 9MM 115GR FMJ Ammo, 50rds">
|
||||
</span>
|
||||
</a>
|
||||
<div class="product details product-item-details">
|
||||
<div class="inner-details">
|
||||
<h2 class="product name product-item-name">
|
||||
<a class="product-item-link" href="https://palmettostatearmory.com/troy-9mm-115gr-fmj-ammo-50rds-ytr9m115fmj.html">
|
||||
TROY 9MM 115GR FMJ Ammo, 50rds - YTR9M115FMJ
|
||||
</a>
|
||||
</h2>
|
||||
<div class="details-bottom">
|
||||
<div class="price-box price-final_price" data-role="priceBox" data-product-id="159202">
|
||||
<span class="special-price">
|
||||
<span class="price-container price-final_price tax weee">
|
||||
<span id="product-price-159202" data-price-amount="11.99" data-price-type="finalPrice" class="price-wrapper final-price"><span class="price">$11.99</span></span>
|
||||
</span>
|
||||
</span>
|
||||
<span class="price-container price-final_price tax weee">
|
||||
<span id="unit-price-159202" data-price-amount="0.2398" data-price-type="unit_price" class="price-wrapper unit-price"><span class="price">$0.24</span></span>
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="product-item-inner">
|
||||
<div class="product actions product-item-actions">
|
||||
<div class="actions-primary">
|
||||
<form data-role="tocart-form" action="https://palmettostatearmory.com/checkout/cart/add/" method="post">
|
||||
<button type="submit" title="Add to Cart" class="action tocart primary"><span>Add to Cart</span></button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</li>
|
||||
<li class="item product product-item" id="product-1293">
|
||||
<div class="product-item-info" data-container="product-list">
|
||||
<a href="https://palmettostatearmory.com/blazer-brass-115gr-9mm.html" class="product photo product-item-photo" tabindex="-1">
|
||||
<span class="product-image-container">
|
||||
<img class="product-image-photo" src="https://palmettostatearmory.com/media/catalog/product/blazer.jpg" alt="CCI Blazer Brass 9mm 115gr FMJ">
|
||||
</span>
|
||||
</a>
|
||||
<div class="product details product-item-details">
|
||||
<div class="inner-details">
|
||||
<h2 class="product name product-item-name">
|
||||
<a class="product-item-link" href="https://palmettostatearmory.com/blazer-brass-115gr-9mm.html">
|
||||
CCI Blazer Brass 9mm 115 Grain FMJ 50 rd/box - 5200
|
||||
</a>
|
||||
</h2>
|
||||
<div class="details-bottom">
|
||||
<div class="price-box price-final_price" data-role="priceBox" data-product-id="1293">
|
||||
<span class="normal-price">
|
||||
<span class="price-container price-final_price tax weee">
|
||||
<span id="product-price-1293" data-price-amount="14.99" data-price-type="finalPrice" class="price-wrapper"><span class="price">$14.99</span></span>
|
||||
</span>
|
||||
</span>
|
||||
<span class="price-container price-final_price tax weee">
|
||||
<span id="unit-price-1293" data-price-amount="0.2998" data-price-type="unit_price" class="price-wrapper unit-price"><span class="price">$0.30</span></span>
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="product-item-inner">
|
||||
<div class="product actions product-item-actions">
|
||||
<div class="actions-primary">
|
||||
<form data-role="tocart-form" action="https://palmettostatearmory.com/checkout/cart/add/" method="post">
|
||||
<button type="submit" title="Add to Cart" class="action tocart primary"><span>Add to Cart</span></button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</li>
|
||||
<li class="item product product-item" id="product-44821">
|
||||
<div class="product-item-info" data-container="product-list">
|
||||
<a href="https://palmettostatearmory.com/tula-9mm-steel-case-115gr-fmj.html" class="product photo product-item-photo" tabindex="-1">
|
||||
<span class="product-image-container">
|
||||
<img class="product-image-photo" src="https://palmettostatearmory.com/media/catalog/product/tula.jpg" alt="Tula 9mm Steel Case">
|
||||
</span>
|
||||
</a>
|
||||
<div class="product details product-item-details">
|
||||
<div class="inner-details">
|
||||
<h2 class="product name product-item-name">
|
||||
<a class="product-item-link" href="https://palmettostatearmory.com/tula-9mm-steel-case-115gr-fmj.html">
|
||||
Tula 9mm 115gr FMJ Steel Case Ammunition 50rds
|
||||
</a>
|
||||
</h2>
|
||||
<div class="details-bottom">
|
||||
<div class="price-box price-final_price" data-role="priceBox" data-product-id="44821">
|
||||
<span class="normal-price">
|
||||
<span class="price-container price-final_price tax weee">
|
||||
<span id="product-price-44821" data-price-amount="9.99" data-price-type="finalPrice" class="price-wrapper"><span class="price">$9.99</span></span>
|
||||
</span>
|
||||
</span>
|
||||
<span class="price-container price-final_price tax weee">
|
||||
<span id="unit-price-44821" data-price-amount="0.1998" data-price-type="unit_price" class="price-wrapper unit-price"><span class="price">$0.20</span></span>
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="product-item-inner">
|
||||
<div class="product actions product-item-actions">
|
||||
<div class="actions-primary">
|
||||
<form data-role="tocart-form" action="https://palmettostatearmory.com/checkout/cart/add/" method="post">
|
||||
<button type="submit" title="Add to Cart" class="action tocart primary"><span>Add to Cart</span></button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</li>
|
||||
<li class="item product product-item" id="product-78932">
|
||||
<div class="product-item-info" data-container="product-list">
|
||||
<a href="https://palmettostatearmory.com/sellier-bellot-9mm-subsonic-150gr-fmj.html" class="product photo product-item-photo" tabindex="-1">
|
||||
<span class="product-image-container">
|
||||
<img class="product-image-photo" src="https://palmettostatearmory.com/media/catalog/product/sb.jpg" alt="Sellier & Bellot 9mm Subsonic">
|
||||
</span>
|
||||
</a>
|
||||
<div class="product details product-item-details">
|
||||
<div class="inner-details">
|
||||
<h2 class="product name product-item-name">
|
||||
<a class="product-item-link" href="https://palmettostatearmory.com/sellier-bellot-9mm-subsonic-150gr-fmj.html">
|
||||
Sellier & Bellot 9mm 150 Grain Subsonic FMJ 50rds
|
||||
</a>
|
||||
</h2>
|
||||
<div class="details-bottom">
|
||||
<div class="price-box price-final_price" data-role="priceBox" data-product-id="78932">
|
||||
<span class="normal-price">
|
||||
<span class="price-container price-final_price tax weee">
|
||||
<span id="product-price-78932" data-price-amount="18.99" data-price-type="finalPrice" class="price-wrapper"><span class="price">$18.99</span></span>
|
||||
</span>
|
||||
</span>
|
||||
<span class="price-container price-final_price tax weee">
|
||||
<span id="unit-price-78932" data-price-amount="0.3798" data-price-type="unit_price" class="price-wrapper unit-price"><span class="price">$0.38</span></span>
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="product-item-inner">
|
||||
<div class="product actions product-item-actions">
|
||||
<div class="actions-primary">
|
||||
<form data-role="tocart-form" action="https://palmettostatearmory.com/checkout/cart/add/" method="post">
|
||||
<button type="submit" title="Add to Cart" class="action tocart primary"><span>Add to Cart</span></button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</li>
|
||||
</ol>
|
||||
</div>
|
||||
102
test/fixtures/true_shot_ammo/9mm.json
vendored
Normal file
102
test/fixtures/true_shot_ammo/9mm.json
vendored
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
{
|
||||
"products": [
|
||||
{
|
||||
"id": 7518232510569,
|
||||
"title": "CCI - Blazer - 9mm - 115 Grain - FMJ",
|
||||
"handle": "cci-blazer-9mm-115-grain-fmj-2",
|
||||
"vendor": "CCI",
|
||||
"variants": [
|
||||
{
|
||||
"id": 42242779414633,
|
||||
"title": "50",
|
||||
"price": "12.99",
|
||||
"available": false
|
||||
},
|
||||
{
|
||||
"id": 42242779447401,
|
||||
"title": "500",
|
||||
"price": "125.95",
|
||||
"available": false
|
||||
},
|
||||
{
|
||||
"id": 42242779480169,
|
||||
"title": "1000",
|
||||
"price": "239.99",
|
||||
"available": false
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7518249451625,
|
||||
"title": "Magtech - 9mm - 115 Grain - FMJ - Steel Case",
|
||||
"handle": "magtech-steel-case-9mm-115-grain-fmj",
|
||||
"vendor": "Magtech",
|
||||
"variants": [
|
||||
{
|
||||
"id": 42242813329513,
|
||||
"title": "50",
|
||||
"price": "11.99",
|
||||
"available": true
|
||||
},
|
||||
{
|
||||
"id": 42242813296745,
|
||||
"title": "500",
|
||||
"price": "111.51",
|
||||
"available": true
|
||||
},
|
||||
{
|
||||
"id": 42242813263977,
|
||||
"title": "1000",
|
||||
"price": "214.99",
|
||||
"available": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7518237491305,
|
||||
"title": "Maxxtech - 9mm - 124 Grain - FMJ - Brass Case",
|
||||
"handle": "maxxtech-9mm-124-grain-fmj",
|
||||
"vendor": "Maxxtech",
|
||||
"variants": [
|
||||
{
|
||||
"id": 42242790916201,
|
||||
"title": "50",
|
||||
"price": "14.99",
|
||||
"available": true
|
||||
},
|
||||
{
|
||||
"id": 42242790948969,
|
||||
"title": "500",
|
||||
"price": "129.99",
|
||||
"available": true
|
||||
},
|
||||
{
|
||||
"id": 42242790981737,
|
||||
"title": "1000",
|
||||
"price": "249.99",
|
||||
"available": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7518237556841,
|
||||
"title": "Fiocchi - 9mm - 147 Grain - Subsonic FMJ",
|
||||
"handle": "fiocchi-9mm-147-grain-subsonic-fmj",
|
||||
"vendor": "Fiocchi",
|
||||
"variants": [
|
||||
{
|
||||
"id": 42242791112809,
|
||||
"title": "50",
|
||||
"price": "16.99",
|
||||
"available": true
|
||||
},
|
||||
{
|
||||
"id": 42242791080041,
|
||||
"title": "500",
|
||||
"price": "159.99",
|
||||
"available": true
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue