diff --git a/lib/ammoprices/scraping/http_client.ex b/lib/ammoprices/scraping/http_client.ex index 61a6e65..5d2a8b6 100644 --- a/lib/ammoprices/scraping/http_client.ex +++ b/lib/ammoprices/scraping/http_client.ex @@ -24,4 +24,11 @@ defmodule Ammoprices.Scraping.HttpClient do Application.get_env(:ammoprices, :req_options, []) ) end + + def post_json(url, body, extra_headers \\ []) do + Req.post( + [base_url: url, headers: @default_headers ++ extra_headers, json: body, retry: :transient, max_retries: 3], + Application.get_env(:ammoprices, :req_options, []) + ) + end end diff --git a/lib/ammoprices/scraping/retailers/bulk_ammo.ex b/lib/ammoprices/scraping/retailers/bulk_ammo.ex new file mode 100644 index 0000000..4e47370 --- /dev/null +++ b/lib/ammoprices/scraping/retailers/bulk_ammo.ex @@ -0,0 +1,155 @@ +defmodule Ammoprices.Scraping.Retailers.BulkAmmo do + @moduledoc false + @behaviour Ammoprices.Scraping.Scraper + + alias Ammoprices.Scraping.TextDetector + + @slug_to_path %{ + "9mm-luger" => "/handgun/bulk-9mm-ammo", + "45-acp" => "/handgun/bulk-45-acp-ammo", + "380-acp" => "/handgun/bulk-380-acp-ammo", + "40-sw" => "/handgun/bulk-40-sandw-ammo", + "38-special" => "/handgun/bulk-38-special-ammo", + "357-magnum" => "/handgun/bulk-357-mag-ammo", + "10mm-auto" => "/handgun/bulk-10mm-ammo", + "556-223" => "/rifle/bulk-223-remington-ammo", + "308-win" => "/rifle/bulk-308-ammo", + "762x39" => "/rifle/bulk-7.62x39-ammo", + "30-06" => "/rifle/bulk-30-06-ammo", + "300-blackout" => "/rifle/bulk-300-blackout-ammo", + "65-creedmoor" => "/rifle/bulk-6.5-creedmoor-ammo", + "22-lr" => "/rimfire/bulk-22-lr-ammo", + "12-gauge" => "/shotgun/bulk-12-gauge-ammo", + "20-gauge" => "/shotgun/bulk-20-gauge-ammo" + } + + @impl true + def retailer_slug, do: "bulk-ammo" + + @impl true + def category_url(caliber) do + Map.get(@slug_to_path, caliber.slug) + end + + @impl true + def parse_products(html) do + html + |> Floki.parse_document!() + |> Floki.find(".products-grid li.item") + |> Enum.map(&parse_item/1) + |> Enum.reject(&is_nil/1) + end + + defp parse_item(item) do + with {:ok, title} <- extract_title(item), + {:ok, url} <- extract_url(item), + {:ok, price_cents} <- extract_price(item) do + round_count = extract_round_count(title) + ppr = if round_count && round_count > 0, do: round(price_cents / round_count) + + %{ + title: title, + url: url, + price_cents: price_cents, + price_per_round_cents: ppr, + brand: extract_brand(title), + grain_weight: extract_grain_weight(title), + round_count: round_count, + casing: TextDetector.detect_casing(title), + subsonic: TextDetector.detect_subsonic(title), + in_stock: extract_in_stock(item) + } + else + _ -> nil + end + end + + defp extract_title(item) do + case Floki.find(item, "a.product-name") do + [{_, _, _} = node] -> {:ok, node |> Floki.text() |> String.trim()} + _ -> :error + end + end + + defp extract_url(item) do + case Floki.find(item, "a.product-name") do + [{_, attrs, _}] -> + case List.keyfind(attrs, "href", 0) do + {"href", href} -> {:ok, href} + _ -> :error + end + + _ -> + :error + end + end + + defp extract_price(item) do + # Prefer special-price (sale price), fall back to regular-price + price_text = + case Floki.find(item, ".special-price .price") do + [{_, _, _} = node] -> node |> Floki.text() |> String.trim() + _ -> extract_regular_price_text(item) + end + + case price_text do + nil -> + :error + + text -> + case Regex.run(~r/\$([0-9,]+(?:\.\d{2})?)/, text) do + [_, amount] -> + cents = parse_dollar_amount(amount) + {:ok, cents} + + _ -> + :error + end + end + end + + defp extract_regular_price_text(item) do + case Floki.find(item, ".regular-price .price") do + [{_, _, _} = node] -> node |> Floki.text() |> String.trim() + _ -> nil + end + end + + defp parse_dollar_amount(amount) do + clean = String.replace(amount, ",", "") + + if String.contains?(clean, ".") do + clean |> String.to_float() |> Kernel.*(100) |> round() + else + String.to_integer(clean) * 100 + end + end + + defp extract_round_count(title) do + case Regex.run(~r/^(\d[\d,]*)\s+[Rr]ounds?/i, title) do + [_, count] -> count |> String.replace(",", "") |> String.to_integer() + _ -> nil + end + end + + defp extract_brand(title) do + case Regex.run(~r/by\s+(.+?)(?:\s*-|$)/, title) do + [_, brand] -> String.trim(brand) + _ -> nil + end + end + + defp extract_grain_weight(title) do + case Regex.run(~r/(\d+)\s*gr/i, title) do + [_, weight] -> String.to_integer(weight) + _ -> nil + end + end + + defp extract_in_stock(item) do + case Floki.find(item, "span.in-stock") do + [_ | _] -> true + _ -> false + end + end +end diff --git a/lib/ammoprices/scraping/retailers/natchez.ex b/lib/ammoprices/scraping/retailers/natchez.ex new file mode 100644 index 0000000..a47d2d4 --- /dev/null +++ b/lib/ammoprices/scraping/retailers/natchez.ex @@ -0,0 +1,132 @@ +defmodule Ammoprices.Scraping.Retailers.Natchez do + @moduledoc false + @behaviour Ammoprices.Scraping.Scraper + + alias Ammoprices.Scraping.HttpClient + alias Ammoprices.Scraping.TextDetector + + @slug_to_caliber %{ + "9mm-luger" => "9mm Luger", + "45-acp" => "45 Auto", + "380-acp" => "380 Auto", + "40-sw" => "40 S&W", + "38-special" => "38 Spl", + "357-magnum" => "357 Mag", + "10mm-auto" => "10mm Auto", + "556-223" => "223 Rem", + "308-win" => "308 Win", + "762x39" => "7.62x39mm", + "30-06" => "30-06 Sprg", + "300-blackout" => "300 Blk", + "65-creedmoor" => "6.5 Creedmoor", + "22-lr" => "22 LR", + "22-wmr" => "22 WMR", + "17-hmr" => "17 HMR", + "12-gauge" => "12 Gauge", + "16-gauge" => "16 Gauge", + "20-gauge" => "20 Gauge" + } + + @impl true + def retailer_slug, do: "natchez" + + @impl true + def category_url(_caliber), do: nil + + @impl true + def parse_products(_), do: [] + + @impl true + def fetch(retailer, caliber) do + case build_query(caliber.slug) do + nil -> + {:ok, []} + + query -> + url = retailer.base_url <> "/graphql" + + case HttpClient.post_json(url, %{query: query}, [{"store", "default"}]) do + {:ok, %{status: 200, body: body}} -> {:ok, parse_graphql_response(body)} + {:ok, %{status: s}} -> {:error, "HTTP #{s}"} + {:error, reason} -> {:error, reason} + end + end + end + + def build_query(slug) do + case Map.get(@slug_to_caliber, slug) do + nil -> + nil + + caliber_value -> + """ + { + products( + search: "#{caliber_value}" + filter: { caliber: { eq: "#{caliber_value}" } } + pageSize: 48 + currentPage: 1 + ) { + items { + name sku url_key stock_status brand grain rounds case_material + price_range { minimum_price { final_price { value } } } + } + total_count + page_info { total_pages } + } + } + """ + end + end + + def parse_graphql_response(%{"data" => %{"products" => %{"items" => items}}}) do + items + |> Enum.map(&parse_item/1) + |> Enum.reject(&is_nil/1) + end + + def parse_graphql_response(_), do: [] + + defp parse_item(item) do + price_cents = extract_price(item) + round_count = parse_integer(item["rounds"]) + ppr = if round_count && round_count > 0, do: round(price_cents / round_count) + name = item["name"] + + %{ + title: name, + url: "/#{item["url_key"]}.html", + brand: item["brand"], + price_cents: price_cents, + price_per_round_cents: ppr, + grain_weight: parse_integer(item["grain"]), + round_count: round_count, + casing: parse_casing(item["case_material"]), + subsonic: TextDetector.detect_subsonic(name), + in_stock: item["stock_status"] == "IN_STOCK" + } + end + + defp extract_price(%{"price_range" => %{"minimum_price" => %{"final_price" => %{"value" => value}}}}) do + value + |> String.to_float() + |> Kernel.*(100) + |> round() + end + + defp extract_price(_), do: 0 + + defp parse_integer(nil), do: nil + + defp parse_integer(str) when is_binary(str) do + case Integer.parse(str) do + {n, _} -> n + :error -> nil + end + end + + defp parse_integer(_), do: nil + + defp parse_casing(nil), do: nil + defp parse_casing(material), do: String.downcase(material) +end diff --git a/lib/ammoprices/scraping/retailers/palmetto_state_armory.ex b/lib/ammoprices/scraping/retailers/palmetto_state_armory.ex new file mode 100644 index 0000000..2351bbb --- /dev/null +++ b/lib/ammoprices/scraping/retailers/palmetto_state_armory.ex @@ -0,0 +1,146 @@ +defmodule Ammoprices.Scraping.Retailers.PalmettoStateArmory do + @moduledoc false + @behaviour Ammoprices.Scraping.Scraper + + alias Ammoprices.Scraping.TextDetector + + @slug_to_path %{ + "9mm-luger" => "/9mm-ammo.html", + "45-acp" => "/45-acp-ammo.html", + "380-acp" => "/380-acp-ammo.html", + "40-sw" => "/40-s-w-ammo.html", + "38-special" => "/38-special-ammo.html", + "357-magnum" => "/357-magnum-ammo.html", + "10mm-auto" => "/10mm-auto-ammo.html", + "556-223" => "/223-remington-ammo.html", + "308-win" => "/308-winchester-ammo.html", + "762x39" => "/7-62x39mm-ammo.html", + "30-06" => "/30-06-springfield-ammo.html", + "300-blackout" => "/300-aac-blackout-ammo.html", + "65-creedmoor" => "/6-5-creedmoor-ammo.html", + "22-lr" => "/22-lr-ammo.html", + "22-wmr" => "/22-wmr-ammo.html", + "17-hmr" => "/17-hmr-ammo.html", + "12-gauge" => "/12-gauge-ammo.html", + "16-gauge" => "/16-gauge-ammo.html", + "20-gauge" => "/20-gauge-ammo.html" + } + + @impl true + def retailer_slug, do: "palmetto-state-armory" + + @impl true + def category_url(caliber) do + Map.get(@slug_to_path, caliber.slug) + end + + @impl true + def parse_products(html) do + html + |> Floki.parse_document!() + |> Floki.find("li.product-item") + |> Enum.map(&parse_item/1) + |> Enum.reject(&is_nil/1) + end + + defp parse_item(item) do + with {:ok, title} <- extract_title(item), + {:ok, url} <- extract_url(item), + {:ok, price_cents} <- extract_price(item), + {:ok, ppr_cents} <- extract_unit_price(item) do + %{ + title: title, + url: url, + price_cents: price_cents, + price_per_round_cents: ppr_cents, + brand: extract_brand(title), + grain_weight: extract_grain_weight(title), + round_count: extract_round_count(title), + casing: TextDetector.detect_casing(title), + subsonic: TextDetector.detect_subsonic(title), + in_stock: true + } + else + _ -> nil + end + end + + defp extract_title(item) do + case Floki.find(item, ".product-item-link") do + [{_, _, _} = node] -> {:ok, node |> Floki.text() |> String.trim()} + _ -> :error + end + end + + defp extract_url(item) do + case Floki.find(item, ".product-item-link") do + [{_, attrs, _}] -> + case List.keyfind(attrs, "href", 0) do + {"href", href} -> {:ok, href} + _ -> :error + end + + _ -> + :error + end + end + + defp extract_price(item) do + case Floki.find(item, "[data-price-type=\"finalPrice\"]") do + [{_, attrs, _} | _] -> + case List.keyfind(attrs, "data-price-amount", 0) do + {"data-price-amount", amount} -> + {:ok, amount |> String.to_float() |> Kernel.*(100) |> round()} + + _ -> + :error + end + + _ -> + :error + end + end + + defp extract_unit_price(item) do + case Floki.find(item, "[data-price-type=\"unit_price\"]") do + [{_, attrs, _} | _] -> + case List.keyfind(attrs, "data-price-amount", 0) do + {"data-price-amount", amount} -> + {:ok, amount |> String.to_float() |> Kernel.*(100) |> round()} + + _ -> + :error + end + + _ -> + :error + end + end + + defp extract_brand(title) do + case String.split(title, " ", parts: 2) do + [brand | _] -> brand + _ -> nil + end + end + + defp extract_grain_weight(title) do + case Regex.run(~r/(\d+)\s*(?:gr(?:ain)?)/i, title) do + [_, weight] -> String.to_integer(weight) + _ -> nil + end + end + + defp extract_round_count(title) do + cond do + match = Regex.run(~r/(\d[\d,]*)\s*(?:rds?|rounds?)/i, title) -> + match |> Enum.at(1) |> String.replace(",", "") |> String.to_integer() + + match = Regex.run(~r/(\d+)\s*rd\/box/i, title) -> + match |> Enum.at(1) |> String.to_integer() + + true -> + nil + end + end +end diff --git a/lib/ammoprices/scraping/retailers/true_shot_ammo.ex b/lib/ammoprices/scraping/retailers/true_shot_ammo.ex new file mode 100644 index 0000000..4881edb --- /dev/null +++ b/lib/ammoprices/scraping/retailers/true_shot_ammo.ex @@ -0,0 +1,97 @@ +defmodule Ammoprices.Scraping.Retailers.TrueShotAmmo do + @moduledoc false + @behaviour Ammoprices.Scraping.Scraper + + alias Ammoprices.Scraping.TextDetector + + @slug_to_collection %{ + "9mm-luger" => "ammunition-pistol-ammo-9mm", + "45-acp" => "ammunition-pistol-ammo-45-acp", + "380-acp" => "ammunition-pistol-ammo-380-acp", + "40-sw" => "ammunition-pistol-ammo-40-s-w", + "38-special" => "ammunition-pistol-ammo-38-special", + "357-magnum" => "ammunition-pistol-ammo-357-magnum", + "10mm-auto" => "ammunition-pistol-ammo-10mm", + "556-223" => "ammunition-rifle-ammo-223-5-56", + "308-win" => "ammunition-rifle-ammo-308-7-62x51", + "762x39" => "ammunition-rifle-ammo-7-62x39", + "300-blackout" => "ammunition-rifle-ammo-300-blackout", + "65-creedmoor" => "ammunition-rifle-ammo-6-5-creedmoor", + "22-lr" => "ammunition-rimfire-ammo-22-lr", + "12-gauge" => "ammunition-shotgun-ammo-12-gauge", + "20-gauge" => "ammunition-shotgun-ammo-20-gauge" + } + + @impl true + def retailer_slug, do: "true-shot-ammo" + + @impl true + def category_url(caliber) do + case Map.get(@slug_to_collection, caliber.slug) do + nil -> nil + collection -> "/collections/#{collection}/products.json" + end + end + + @impl true + def parse_products(%{"products" => products}) do + products + |> Enum.map(&parse_product/1) + |> Enum.reject(&is_nil/1) + end + + def parse_products(_), do: [] + + defp parse_product(product) do + title = product["title"] + first_variant = List.first(product["variants"]) + + case first_variant do + nil -> + nil + + variant -> + price_cents = parse_price(variant["price"]) + round_count = parse_round_count(variant["title"]) + ppr = if round_count && round_count > 0, do: round(price_cents / round_count) + + %{ + title: title, + url: "/products/#{product["handle"]}", + brand: product["vendor"], + price_cents: price_cents, + price_per_round_cents: ppr, + grain_weight: extract_grain_weight(title), + round_count: round_count, + casing: TextDetector.detect_casing(title), + subsonic: TextDetector.detect_subsonic(title), + in_stock: variant["available"] == true + } + end + end + + defp parse_price(price_str) when is_binary(price_str) do + price_str + |> String.to_float() + |> Kernel.*(100) + |> round() + end + + defp parse_price(_), do: 0 + + defp parse_round_count(title) when is_binary(title) do + case Integer.parse(title) do + {count, _} -> count + :error -> nil + end + end + + defp parse_round_count(_), do: nil + + defp extract_grain_weight(title) do + case Regex.run(~r/(\d+)\s*[Gg]rain/i, title) do + [_, weight] -> String.to_integer(weight) + _ -> nil + end + end +end diff --git a/lib/ammoprices/scraping/runner.ex b/lib/ammoprices/scraping/runner.ex index 7ab03da..b613b5c 100644 --- a/lib/ammoprices/scraping/runner.ex +++ b/lib/ammoprices/scraping/runner.ex @@ -8,59 +8,35 @@ defmodule Ammoprices.Scraping.Runner do def run(scraper, caliber) do retailer = Catalog.get_retailer_by_slug!(scraper.retailer_slug()) - case scraper.category_url(caliber) do - nil -> - {:ok, %{products_count: 0, snapshots_count: 0}} + if function_exported?(scraper, :fetch, 2) do + do_fetch(scraper, retailer, caliber) + else + case scraper.category_url(caliber) do + nil -> + {:ok, %{products_count: 0, snapshots_count: 0}} - path -> - url = retailer.base_url <> path - do_scrape(scraper, retailer, caliber, url) + path -> + url = retailer.base_url <> path + do_scrape(scraper, retailer, caliber, url) + end + end + end + + defp do_fetch(scraper, retailer, caliber) do + case scraper.fetch(retailer, caliber) do + {:ok, parsed} -> + process_results(parsed, retailer, caliber) + + {:error, reason} -> + {:error, reason} end end defp do_scrape(scraper, retailer, caliber, url) do case HttpClient.get(url) do {:ok, %{status: 200, body: body}} -> - now = DateTime.truncate(DateTime.utc_now(), :second) parsed = scraper.parse_products(body) - - results = - Enum.map(parsed, fn product_data -> - product_attrs = %{ - title: product_data.title, - url: product_data.url, - brand: product_data.brand, - grain_weight: product_data.grain_weight, - round_count: product_data.round_count, - casing: product_data.casing, - condition: "new", - in_stock: product_data.in_stock, - subsonic: Map.get(product_data, :subsonic, false), - last_seen_at: now - } - - case Catalog.upsert_product(retailer.id, caliber.id, product_attrs) do - {:ok, product} -> - snapshot_attrs = %{ - price_cents: product_data.price_cents, - price_per_round_cents: product_data.price_per_round_cents, - in_stock: product_data.in_stock, - recorded_at: now - } - - {:ok, _snapshot} = Prices.create_snapshot(product.id, snapshot_attrs) - :ok - - {:error, _changeset} -> - :error - end - end) - - successful = Enum.count(results, &(&1 == :ok)) - - Catalog.update_retailer(retailer, %{last_scraped_at: now}) - - {:ok, %{products_count: successful, snapshots_count: successful}} + process_results(parsed, retailer, caliber) {:ok, %{status: status}} -> {:error, "HTTP #{status}"} @@ -69,4 +45,46 @@ defmodule Ammoprices.Scraping.Runner do {:error, reason} end end + + defp process_results(parsed, retailer, caliber) do + now = DateTime.truncate(DateTime.utc_now(), :second) + + results = + Enum.map(parsed, fn product_data -> + product_attrs = %{ + title: product_data.title, + url: product_data.url, + brand: product_data.brand, + grain_weight: product_data.grain_weight, + round_count: product_data.round_count, + casing: product_data.casing, + condition: "new", + in_stock: product_data.in_stock, + subsonic: Map.get(product_data, :subsonic, false), + last_seen_at: now + } + + case Catalog.upsert_product(retailer.id, caliber.id, product_attrs) do + {:ok, product} -> + snapshot_attrs = %{ + price_cents: product_data.price_cents, + price_per_round_cents: product_data.price_per_round_cents, + in_stock: product_data.in_stock, + recorded_at: now + } + + {:ok, _snapshot} = Prices.create_snapshot(product.id, snapshot_attrs) + :ok + + {:error, _changeset} -> + :error + end + end) + + successful = Enum.count(results, &(&1 == :ok)) + + Catalog.update_retailer(retailer, %{last_scraped_at: now}) + + {:ok, %{products_count: successful, snapshots_count: successful}} + end end diff --git a/lib/ammoprices/scraping/scrape_job.ex b/lib/ammoprices/scraping/scrape_job.ex index 39d5a32..ed1cafc 100644 --- a/lib/ammoprices/scraping/scrape_job.ex +++ b/lib/ammoprices/scraping/scrape_job.ex @@ -8,7 +8,11 @@ defmodule Ammoprices.Scraping.ScrapeJob do @scrapers [ Ammoprices.Scraping.Retailers.LuckyGunner, Ammoprices.Scraping.Retailers.SgAmmo, - Ammoprices.Scraping.Retailers.TargetSportsUsa + Ammoprices.Scraping.Retailers.TargetSportsUsa, + Ammoprices.Scraping.Retailers.TrueShotAmmo, + Ammoprices.Scraping.Retailers.PalmettoStateArmory, + Ammoprices.Scraping.Retailers.BulkAmmo, + Ammoprices.Scraping.Retailers.Natchez ] @impl Oban.Worker diff --git a/lib/ammoprices/scraping/scraper.ex b/lib/ammoprices/scraping/scraper.ex index 1dd0ed9..5a4648e 100644 --- a/lib/ammoprices/scraping/scraper.ex +++ b/lib/ammoprices/scraping/scraper.ex @@ -3,5 +3,8 @@ defmodule Ammoprices.Scraping.Scraper do @callback retailer_slug() :: String.t() @callback category_url(caliber :: map()) :: String.t() | nil - @callback parse_products(html :: String.t()) :: [map()] + @callback parse_products(html :: term()) :: [map()] + @callback fetch(retailer :: map(), caliber :: map()) :: {:ok, [map()]} | {:error, term()} + + @optional_callbacks [fetch: 2] end diff --git a/priv/repo/migrations/20260312144740_add_new_retailers.exs b/priv/repo/migrations/20260312144740_add_new_retailers.exs new file mode 100644 index 0000000..837fc9b --- /dev/null +++ b/priv/repo/migrations/20260312144740_add_new_retailers.exs @@ -0,0 +1,36 @@ +defmodule Ammoprices.Repo.Migrations.AddNewRetailers do + use Ecto.Migration + + def up do + now = DateTime.utc_now() |> DateTime.truncate(:second) |> DateTime.to_naive() + + retailers = [ + %{name: "True Shot Ammo", slug: "true-shot-ammo", base_url: "https://trueshotammo.com"}, + %{ + name: "Palmetto State Armory", + slug: "palmetto-state-armory", + base_url: "https://palmettostatearmory.com" + }, + %{name: "Bulk Ammo", slug: "bulk-ammo", base_url: "https://www.bulkammo.com"}, + %{ + name: "Natchez Shooters Supply", + slug: "natchez", + base_url: "https://www.natchezss.com" + } + ] + + for r <- retailers do + execute(""" + INSERT INTO retailers (id, name, slug, base_url, inserted_at, updated_at) + VALUES (gen_random_uuid(), '#{r.name}', '#{r.slug}', '#{r.base_url}', '#{now}', '#{now}') + ON CONFLICT (slug) DO NOTHING + """) + end + end + + def down do + execute( + "DELETE FROM retailers WHERE slug IN ('true-shot-ammo', 'palmetto-state-armory', 'bulk-ammo', 'natchez')" + ) + end +end diff --git a/test/ammoprices/scraping/retailers/bulk_ammo_test.exs b/test/ammoprices/scraping/retailers/bulk_ammo_test.exs new file mode 100644 index 0000000..e6b45b8 --- /dev/null +++ b/test/ammoprices/scraping/retailers/bulk_ammo_test.exs @@ -0,0 +1,115 @@ +defmodule Ammoprices.Scraping.Retailers.BulkAmmoTest do + use ExUnit.Case, async: true + + alias Ammoprices.Scraping.Retailers.BulkAmmo + + @fixture_path "test/fixtures/bulk_ammo/9mm.html" + + describe "retailer_slug/0" do + test "returns bulk-ammo" do + assert BulkAmmo.retailer_slug() == "bulk-ammo" + end + end + + describe "category_url/1" do + test "maps 9mm-luger caliber to correct URL" do + caliber = %{slug: "9mm-luger"} + assert BulkAmmo.category_url(caliber) == "/handgun/bulk-9mm-ammo" + end + + test "maps 556-223 caliber to correct URL" do + caliber = %{slug: "556-223"} + assert BulkAmmo.category_url(caliber) == "/rifle/bulk-223-remington-ammo" + end + + test "returns nil for unmapped caliber" do + caliber = %{slug: "unknown-caliber"} + assert BulkAmmo.category_url(caliber) == nil + end + end + + describe "parse_products/1" do + setup do + html = File.read!(@fixture_path) + %{html: html} + end + + test "extracts all products from HTML", %{html: html} do + products = BulkAmmo.parse_products(html) + assert length(products) == 4 + end + + test "extracts product title", %{html: html} do + [first | _] = BulkAmmo.parse_products(html) + assert first.title == "1500 Rounds of 9mm Ammo by Sterling Steel - 115gr FMJ" + end + + test "extracts product URL", %{html: html} do + [first | _] = BulkAmmo.parse_products(html) + assert first.url == "https://www.bulkammo.com/1500-rounds-of-9mm-ammo-by-sterling-115gr-fmj" + end + + test "extracts sale price when present", %{html: html} do + [first | _] = BulkAmmo.parse_products(html) + # $285 sale price + assert first.price_cents == 28_500 + end + + test "extracts regular price when no sale", %{html: html} do + products = BulkAmmo.parse_products(html) + second = Enum.at(products, 1) + # $239.99 regular price + assert second.price_cents == 23_999 + end + + test "extracts round count from title", %{html: html} do + [first | _] = BulkAmmo.parse_products(html) + assert first.round_count == 1500 + end + + test "calculates price per round", %{html: html} do + [first | _] = BulkAmmo.parse_products(html) + # 28500 / 1500 = 19 + assert first.price_per_round_cents == 19 + end + + test "extracts brand from title", %{html: html} do + products = BulkAmmo.parse_products(html) + assert hd(products).brand == "Sterling Steel" + assert Enum.at(products, 1).brand == "Remington" + assert Enum.at(products, 2).brand == "Federal Brass" + end + + test "extracts grain weight from title", %{html: html} do + [first | _] = BulkAmmo.parse_products(html) + assert first.grain_weight == 115 + end + + test "detects in-stock status", %{html: html} do + products = BulkAmmo.parse_products(html) + assert hd(products).in_stock == true + assert Enum.at(products, 1).in_stock == true + # Fourth product is out of stock + assert Enum.at(products, 3).in_stock == false + end + + test "detects casing from title", %{html: html} do + products = BulkAmmo.parse_products(html) + # First product: "Sterling Steel" in title + assert hd(products).casing == "steel" + # Third product: "Federal Brass" in title + assert Enum.at(products, 2).casing == "brass" + end + + test "detects subsonic from title", %{html: html} do + products = BulkAmmo.parse_products(html) + # Fourth product: "Subsonic" in title + assert Enum.at(products, 3).subsonic == true + end + + test "non-subsonic products return false", %{html: html} do + [first | _] = BulkAmmo.parse_products(html) + assert first.subsonic == false + end + end +end diff --git a/test/ammoprices/scraping/retailers/natchez_test.exs b/test/ammoprices/scraping/retailers/natchez_test.exs new file mode 100644 index 0000000..21b302a --- /dev/null +++ b/test/ammoprices/scraping/retailers/natchez_test.exs @@ -0,0 +1,110 @@ +defmodule Ammoprices.Scraping.Retailers.NatchezTest do + use ExUnit.Case, async: true + + alias Ammoprices.Scraping.Retailers.Natchez + + @fixture_path "test/fixtures/natchez/9mm.json" + + describe "retailer_slug/0" do + test "returns natchez" do + assert Natchez.retailer_slug() == "natchez" + end + end + + describe "category_url/1" do + test "returns nil for all calibers since fetch/2 is used" do + caliber = %{slug: "9mm-luger"} + assert Natchez.category_url(caliber) == nil + end + end + + describe "build_query/1" do + test "builds GraphQL query for mapped caliber" do + query = Natchez.build_query("9mm-luger") + assert query =~ "9mm Luger" + assert query =~ "caliber" + assert query =~ "pageSize" + end + + test "returns nil for unmapped caliber" do + assert Natchez.build_query("unknown-caliber") == nil + end + end + + describe "parse_graphql_response/1" do + setup do + body = @fixture_path |> File.read!() |> Jason.decode!() + %{body: body} + end + + test "extracts all products from response", %{body: body} do + products = Natchez.parse_graphql_response(body) + assert length(products) == 4 + end + + test "extracts product title from name", %{body: body} do + [first | _] = Natchez.parse_graphql_response(body) + + assert first.title == + "CCI Blazer Brass Handgun Ammunition 9mm Luger 115 gr FMJ 1145 fps 1000/ct Loose Bulk Pack" + end + + test "extracts product URL from url_key", %{body: body} do + [first | _] = Natchez.parse_graphql_response(body) + assert first.url == "/cci-9mm-115gr-fmj-1000-rds-bulk-pack-cc5000bk1000.html" + end + + test "extracts brand directly", %{body: body} do + [first | _] = Natchez.parse_graphql_response(body) + assert first.brand == "CCI" + end + + test "extracts price in cents", %{body: body} do + [first | _] = Natchez.parse_graphql_response(body) + assert first.price_cents == 24_999 + end + + test "extracts grain weight from grain field", %{body: body} do + [first | _] = Natchez.parse_graphql_response(body) + # "115 gr" → 115 + assert first.grain_weight == 115 + end + + test "extracts round count from rounds field", %{body: body} do + [first | _] = Natchez.parse_graphql_response(body) + assert first.round_count == 1000 + end + + test "calculates price per round", %{body: body} do + [first | _] = Natchez.parse_graphql_response(body) + # 24999 / 1000 = 24.999 → 25 + assert first.price_per_round_cents == 25 + end + + test "extracts casing from case_material", %{body: body} do + products = Natchez.parse_graphql_response(body) + assert hd(products).casing == "brass" + # Third product: Steel case + assert Enum.at(products, 2).casing == "steel" + end + + test "detects in_stock from stock_status", %{body: body} do + products = Natchez.parse_graphql_response(body) + # First: OUT_OF_STOCK + assert hd(products).in_stock == false + # Second: IN_STOCK + assert Enum.at(products, 1).in_stock == true + end + + test "detects subsonic from name", %{body: body} do + products = Natchez.parse_graphql_response(body) + # Fourth product has "Subsonic" in name + assert Enum.at(products, 3).subsonic == true + end + + test "non-subsonic products return false", %{body: body} do + [first | _] = Natchez.parse_graphql_response(body) + assert first.subsonic == false + end + end +end diff --git a/test/ammoprices/scraping/retailers/palmetto_state_armory_test.exs b/test/ammoprices/scraping/retailers/palmetto_state_armory_test.exs new file mode 100644 index 0000000..0045d52 --- /dev/null +++ b/test/ammoprices/scraping/retailers/palmetto_state_armory_test.exs @@ -0,0 +1,106 @@ +defmodule Ammoprices.Scraping.Retailers.PalmettoStateArmoryTest do + use ExUnit.Case, async: true + + alias Ammoprices.Scraping.Retailers.PalmettoStateArmory + + @fixture_path "test/fixtures/palmetto_state_armory/9mm.html" + + describe "retailer_slug/0" do + test "returns palmetto-state-armory" do + assert PalmettoStateArmory.retailer_slug() == "palmetto-state-armory" + end + end + + describe "category_url/1" do + test "maps 9mm-luger caliber to correct URL" do + caliber = %{slug: "9mm-luger"} + assert PalmettoStateArmory.category_url(caliber) == "/9mm-ammo.html" + end + + test "maps 556-223 caliber to correct URL" do + caliber = %{slug: "556-223"} + assert PalmettoStateArmory.category_url(caliber) == "/223-remington-ammo.html" + end + + test "returns nil for unmapped caliber" do + caliber = %{slug: "unknown-caliber"} + assert PalmettoStateArmory.category_url(caliber) == nil + end + end + + describe "parse_products/1" do + setup do + html = File.read!(@fixture_path) + %{html: html} + end + + test "extracts all products from HTML", %{html: html} do + products = PalmettoStateArmory.parse_products(html) + assert length(products) == 4 + end + + test "extracts product title", %{html: html} do + [first | _] = PalmettoStateArmory.parse_products(html) + assert first.title == "TROY 9MM 115GR FMJ Ammo, 50rds - YTR9M115FMJ" + end + + test "extracts product URL", %{html: html} do + [first | _] = PalmettoStateArmory.parse_products(html) + + assert first.url == + "https://palmettostatearmory.com/troy-9mm-115gr-fmj-ammo-50rds-ytr9m115fmj.html" + end + + test "extracts price in cents from data-price-amount", %{html: html} do + [first | _] = PalmettoStateArmory.parse_products(html) + assert first.price_cents == 1199 + end + + test "extracts price per round in cents from unit_price", %{html: html} do + [first | _] = PalmettoStateArmory.parse_products(html) + # 0.2398 * 100 = 23.98 → 24 cents + assert first.price_per_round_cents == 24 + end + + test "extracts grain weight from title", %{html: html} do + [first | _] = PalmettoStateArmory.parse_products(html) + assert first.grain_weight == 115 + end + + test "extracts round count from title", %{html: html} do + [first | _] = PalmettoStateArmory.parse_products(html) + assert first.round_count == 50 + end + + test "marks all listed products as in stock", %{html: html} do + products = PalmettoStateArmory.parse_products(html) + assert Enum.all?(products, & &1.in_stock) + end + + test "detects casing from title", %{html: html} do + products = PalmettoStateArmory.parse_products(html) + # Third product: "Steel Case" in title + assert Enum.at(products, 2).casing == "steel" + # Second product: "Brass" in title + assert Enum.at(products, 1).casing == "brass" + end + + test "detects subsonic from title", %{html: html} do + products = PalmettoStateArmory.parse_products(html) + # Fourth product: "Subsonic" in title + assert Enum.at(products, 3).subsonic == true + end + + test "non-subsonic products return false", %{html: html} do + [first | _] = PalmettoStateArmory.parse_products(html) + assert first.subsonic == false + end + + test "extracts brand from beginning of title", %{html: html} do + products = PalmettoStateArmory.parse_products(html) + assert hd(products).brand == "TROY" + assert Enum.at(products, 1).brand == "CCI" + assert Enum.at(products, 2).brand == "Tula" + end + end +end diff --git a/test/ammoprices/scraping/retailers/true_shot_ammo_test.exs b/test/ammoprices/scraping/retailers/true_shot_ammo_test.exs new file mode 100644 index 0000000..20ea6b3 --- /dev/null +++ b/test/ammoprices/scraping/retailers/true_shot_ammo_test.exs @@ -0,0 +1,105 @@ +defmodule Ammoprices.Scraping.Retailers.TrueShotAmmoTest do + use ExUnit.Case, async: true + + alias Ammoprices.Scraping.Retailers.TrueShotAmmo + + @fixture_path "test/fixtures/true_shot_ammo/9mm.json" + + describe "retailer_slug/0" do + test "returns true-shot-ammo" do + assert TrueShotAmmo.retailer_slug() == "true-shot-ammo" + end + end + + describe "category_url/1" do + test "maps 9mm-luger caliber to correct URL" do + caliber = %{slug: "9mm-luger"} + assert TrueShotAmmo.category_url(caliber) == "/collections/ammunition-pistol-ammo-9mm/products.json" + end + + test "maps 556-223 caliber to correct URL" do + caliber = %{slug: "556-223"} + assert TrueShotAmmo.category_url(caliber) == "/collections/ammunition-rifle-ammo-223-5-56/products.json" + end + + test "returns nil for unmapped caliber" do + caliber = %{slug: "unknown-caliber"} + assert TrueShotAmmo.category_url(caliber) == nil + end + end + + describe "parse_products/1" do + setup do + json = @fixture_path |> File.read!() |> Jason.decode!() + %{json: json} + end + + test "extracts all products from JSON", %{json: json} do + products = TrueShotAmmo.parse_products(json) + assert length(products) == 4 + end + + test "extracts product title", %{json: json} do + [first | _] = TrueShotAmmo.parse_products(json) + assert first.title == "CCI - Blazer - 9mm - 115 Grain - FMJ" + end + + test "extracts product URL from handle", %{json: json} do + [first | _] = TrueShotAmmo.parse_products(json) + assert first.url == "/products/cci-blazer-9mm-115-grain-fmj-2" + end + + test "extracts brand from vendor", %{json: json} do + [first | _] = TrueShotAmmo.parse_products(json) + assert first.brand == "CCI" + end + + test "extracts price in cents from first variant", %{json: json} do + [first | _] = TrueShotAmmo.parse_products(json) + assert first.price_cents == 1299 + end + + test "extracts in_stock from first variant availability", %{json: json} do + products = TrueShotAmmo.parse_products(json) + # First product: all variants unavailable + assert hd(products).in_stock == false + # Second product: first variant available + assert Enum.at(products, 1).in_stock == true + end + + test "extracts grain weight from title", %{json: json} do + [first | _] = TrueShotAmmo.parse_products(json) + assert first.grain_weight == 115 + end + + test "extracts round count from first variant title", %{json: json} do + [first | _] = TrueShotAmmo.parse_products(json) + assert first.round_count == 50 + end + + test "calculates price per round", %{json: json} do + [first | _] = TrueShotAmmo.parse_products(json) + # 1299 cents / 50 rounds = 25.98 → 26 cents + assert first.price_per_round_cents == 26 + end + + test "detects casing from title", %{json: json} do + products = TrueShotAmmo.parse_products(json) + # Second product has "Steel Case" in title + assert Enum.at(products, 1).casing == "steel" + # Third product has "Brass Case" in title + assert Enum.at(products, 2).casing == "brass" + end + + test "detects subsonic from title", %{json: json} do + products = TrueShotAmmo.parse_products(json) + # Fourth product has "Subsonic" in title + assert Enum.at(products, 3).subsonic == true + end + + test "non-subsonic products return false", %{json: json} do + [first | _] = TrueShotAmmo.parse_products(json) + assert first.subsonic == false + end + end +end diff --git a/test/ammoprices/scraping/scrape_job_test.exs b/test/ammoprices/scraping/scrape_job_test.exs index 1a93eb0..d5b9889 100644 --- a/test/ammoprices/scraping/scrape_job_test.exs +++ b/test/ammoprices/scraping/scrape_job_test.exs @@ -18,6 +18,30 @@ defmodule Ammoprices.Scraping.ScrapeJobTest do base_url: "https://www.targetsportsusa.com" }) + retailer_fixture(%{ + name: "True Shot Ammo", + slug: "true-shot-ammo", + base_url: "https://trueshotammo.com" + }) + + retailer_fixture(%{ + name: "Palmetto State Armory", + slug: "palmetto-state-armory", + base_url: "https://palmettostatearmory.com" + }) + + retailer_fixture(%{ + name: "Bulk Ammo", + slug: "bulk-ammo", + base_url: "https://www.bulkammo.com" + }) + + retailer_fixture(%{ + name: "Natchez Shooters Supply", + slug: "natchez", + base_url: "https://www.natchezss.com" + }) + caliber_fixture(%{name: "9mm Luger", slug: "9mm-luger", category: "handgun", aliases: ["9mm", "9x19"]}) Req.Test.stub(Ammoprices.Scraping.HttpClient, fn conn -> diff --git a/test/fixtures/bulk_ammo/9mm.html b/test/fixtures/bulk_ammo/9mm.html new file mode 100644 index 0000000..19d4b5b --- /dev/null +++ b/test/fixtures/bulk_ammo/9mm.html @@ -0,0 +1,97 @@ +
+ Regular Price: + $304 +
++ On Sale: + $285 +
++ + 101 + Ready to Ship + +
++ + 50 + Ready to Ship + +
++ + Ready to Ship + +
++ Out of Stock +
+
+
+
+
+
+
+
+
+
+
+
+
+