Graham McIntire
ae7e117b23
When a product no longer appears in a successful scrape for its retailer+caliber, mark it out of stock. After each scrape cycle, hard delete any product whose last_seen_at is older than 30 days (snapshots cascade via FK). A zero-result scrape leaves existing listings alone so a parse failure or transient site glitch can't nuke the catalog.
79 lines
2.2 KiB
Elixir
79 lines
2.2 KiB
Elixir
defmodule Ammoprices.Scraping.ScrapeJobTest do
|
|
use Ammoprices.DataCase
|
|
use Oban.Testing, repo: Ammoprices.Repo
|
|
|
|
import Ammoprices.Fixtures
|
|
|
|
alias Ammoprices.Scraping.ScrapeJob
|
|
|
|
@fixture_html File.read!("test/fixtures/lucky_gunner/9mm.html")
|
|
|
|
setup do
|
|
retailer_fixture(%{name: "Lucky Gunner", slug: "lucky-gunner", base_url: "https://www.luckygunner.com"})
|
|
retailer_fixture(%{name: "SGAmmo", slug: "sgammo", base_url: "https://www.sgammo.com"})
|
|
|
|
retailer_fixture(%{
|
|
name: "Target Sports USA",
|
|
slug: "target-sports-usa",
|
|
base_url: "https://www.targetsportsusa.com"
|
|
})
|
|
|
|
retailer_fixture(%{
|
|
name: "True Shot Ammo",
|
|
slug: "true-shot-ammo",
|
|
base_url: "https://trueshotammo.com"
|
|
})
|
|
|
|
retailer_fixture(%{
|
|
name: "Palmetto State Armory",
|
|
slug: "palmetto-state-armory",
|
|
base_url: "https://palmettostatearmory.com"
|
|
})
|
|
|
|
retailer_fixture(%{
|
|
name: "Bulk Ammo",
|
|
slug: "bulk-ammo",
|
|
base_url: "https://www.bulkammo.com"
|
|
})
|
|
|
|
retailer_fixture(%{
|
|
name: "Natchez Shooters Supply",
|
|
slug: "natchez",
|
|
base_url: "https://www.natchezss.com"
|
|
})
|
|
|
|
caliber_fixture(%{name: "9mm Luger", slug: "9mm-luger", category: "handgun", aliases: ["9mm", "9x19"]})
|
|
|
|
Req.Test.stub(Ammoprices.Scraping.HttpClient, fn conn ->
|
|
Req.Test.html(conn, @fixture_html)
|
|
end)
|
|
|
|
:ok
|
|
end
|
|
|
|
describe "perform/1" do
|
|
test "executes scrape for all enabled scrapers and calibers" do
|
|
assert :ok = perform_job(ScrapeJob, %{})
|
|
end
|
|
|
|
test "hard-deletes products not seen in 30+ days after the scrape cycle" do
|
|
retailer = Ammoprices.Catalog.get_retailer_by_slug!("lucky-gunner")
|
|
caliber = Ammoprices.Catalog.get_caliber_by_slug!("9mm-luger")
|
|
|
|
long_ago = DateTime.utc_now() |> DateTime.add(-31, :day) |> DateTime.truncate(:second)
|
|
|
|
stale =
|
|
product_fixture(%{
|
|
retailer: retailer,
|
|
caliber: caliber,
|
|
url: "/products/ancient",
|
|
last_seen_at: long_ago,
|
|
in_stock: false
|
|
})
|
|
|
|
assert :ok = perform_job(ScrapeJob, %{})
|
|
|
|
refute Ammoprices.Repo.get(Ammoprices.Catalog.Product, stale.id)
|
|
end
|
|
end
|
|
end
|