Graham McIntire
ae7e117b23
When a product no longer appears in a successful scrape for its retailer+caliber, mark it out of stock. After each scrape cycle, hard delete any product whose last_seen_at is older than 30 days (snapshots cascade via FK). A zero-result scrape leaves existing listings alone so a parse failure or transient site glitch can't nuke the catalog.
112 lines
3.5 KiB
Elixir
112 lines
3.5 KiB
Elixir
defmodule Ammoprices.Scraping.RunnerTest do
|
|
use Ammoprices.DataCase, async: true
|
|
|
|
import Ammoprices.Fixtures
|
|
|
|
alias Ammoprices.Catalog
|
|
alias Ammoprices.Catalog.Product
|
|
alias Ammoprices.Scraping.HttpClient
|
|
alias Ammoprices.Scraping.Retailers.LuckyGunner
|
|
alias Ammoprices.Scraping.Runner
|
|
|
|
@fixture_html File.read!("test/fixtures/lucky_gunner/9mm.html")
|
|
|
|
setup do
|
|
retailer = retailer_fixture(%{name: "Lucky Gunner", slug: "lucky-gunner", base_url: "https://www.luckygunner.com"})
|
|
caliber = caliber_fixture(%{name: "9mm Luger", slug: "9mm-luger", category: "handgun", aliases: ["9mm", "9x19"]})
|
|
|
|
Req.Test.stub(HttpClient, fn conn ->
|
|
Req.Test.html(conn, @fixture_html)
|
|
end)
|
|
|
|
%{retailer: retailer, caliber: caliber}
|
|
end
|
|
|
|
describe "run/2" do
|
|
test "creates products and snapshots from scrape", %{caliber: caliber} do
|
|
scraper = LuckyGunner
|
|
|
|
assert {:ok, stats} = Runner.run(scraper, caliber)
|
|
assert stats.products_count == 4
|
|
assert stats.snapshots_count == 4
|
|
|
|
products = Catalog.list_products_for_caliber(caliber.id)
|
|
assert length(products) == 4
|
|
end
|
|
|
|
test "updates retailer last_scraped_at", %{caliber: caliber} do
|
|
scraper = LuckyGunner
|
|
|
|
{:ok, _stats} = Runner.run(scraper, caliber)
|
|
|
|
retailer = Catalog.get_retailer_by_slug!("lucky-gunner")
|
|
assert retailer.last_scraped_at
|
|
end
|
|
|
|
test "handles scraper returning nil category_url", %{caliber: _caliber} do
|
|
caliber = caliber_fixture(%{name: "Unknown", slug: "unknown", category: "handgun"})
|
|
scraper = LuckyGunner
|
|
|
|
assert {:ok, stats} = Runner.run(scraper, caliber)
|
|
assert stats.products_count == 0
|
|
end
|
|
|
|
test "marks products not seen in current scrape as out of stock",
|
|
%{retailer: retailer, caliber: caliber} do
|
|
# Pre-existing product that won't appear in the fixture scrape
|
|
old = DateTime.utc_now() |> DateTime.add(-3600, :second) |> DateTime.truncate(:second)
|
|
|
|
stale =
|
|
product_fixture(%{
|
|
retailer: retailer,
|
|
caliber: caliber,
|
|
url: "/products/no-longer-listed",
|
|
last_seen_at: old,
|
|
in_stock: true
|
|
})
|
|
|
|
assert {:ok, _stats} = Runner.run(LuckyGunner, caliber)
|
|
|
|
assert Ammoprices.Repo.get!(Product, stale.id).in_stock == false
|
|
end
|
|
|
|
test "does not prune products from other retailers", %{caliber: caliber} do
|
|
other_retailer = retailer_fixture(%{slug: "other", base_url: "https://other.com"})
|
|
old = DateTime.utc_now() |> DateTime.add(-3600, :second) |> DateTime.truncate(:second)
|
|
|
|
other =
|
|
product_fixture(%{
|
|
retailer: other_retailer,
|
|
caliber: caliber,
|
|
url: "/products/other-retailer",
|
|
last_seen_at: old,
|
|
in_stock: true
|
|
})
|
|
|
|
assert {:ok, _stats} = Runner.run(LuckyGunner, caliber)
|
|
|
|
assert Ammoprices.Repo.get!(Product, other.id).in_stock == true
|
|
end
|
|
|
|
test "does not prune when scrape returns zero products", %{retailer: retailer, caliber: caliber} do
|
|
Req.Test.stub(HttpClient, fn conn ->
|
|
Req.Test.html(conn, "<html><body>no products here</body></html>")
|
|
end)
|
|
|
|
old = DateTime.utc_now() |> DateTime.add(-3600, :second) |> DateTime.truncate(:second)
|
|
|
|
stale =
|
|
product_fixture(%{
|
|
retailer: retailer,
|
|
caliber: caliber,
|
|
url: "/products/still-here",
|
|
last_seen_at: old,
|
|
in_stock: true
|
|
})
|
|
|
|
assert {:ok, _stats} = Runner.run(LuckyGunner, caliber)
|
|
|
|
assert Ammoprices.Repo.get!(Product, stale.id).in_stock == true
|
|
end
|
|
end
|
|
end
|