diff --git a/lib/ammoprices/catalog.ex b/lib/ammoprices/catalog.ex index 3d2002d..4105d55 100644 --- a/lib/ammoprices/catalog.ex +++ b/lib/ammoprices/catalog.ex @@ -84,6 +84,34 @@ defmodule Ammoprices.Catalog do ) end + @doc """ + Marks products belonging to `retailer_id`/`caliber_id` as out of stock when + their `last_seen_at` is strictly before `cutoff`. Returns `{count, nil}`. + + Used by the scrape runner to flag listings that disappeared from the + retailer's current category response. + """ + def mark_unseen_products_out_of_stock(retailer_id, caliber_id, %DateTime{} = cutoff) do + now = DateTime.truncate(DateTime.utc_now(), :second) + + Product + |> where([p], p.retailer_id == ^retailer_id) + |> where([p], p.caliber_id == ^caliber_id) + |> where([p], p.last_seen_at < ^cutoff) + |> where([p], p.in_stock == true) + |> Repo.update_all(set: [in_stock: false, updated_at: now]) + end + + @doc """ + Hard-deletes products whose `last_seen_at` is older than `cutoff`. Related + price snapshots cascade via the foreign key. Returns `{count, nil}`. + """ + def delete_stale_products(%DateTime{} = cutoff) do + Product + |> where([p], p.last_seen_at < ^cutoff) + |> Repo.delete_all() + end + def list_products_for_caliber(caliber_id, opts \\ []) do limit = Keyword.get(opts, :limit, 100) diff --git a/lib/ammoprices/scraping/runner.ex b/lib/ammoprices/scraping/runner.ex index b613b5c..3b4b477 100644 --- a/lib/ammoprices/scraping/runner.ex +++ b/lib/ammoprices/scraping/runner.ex @@ -83,6 +83,10 @@ defmodule Ammoprices.Scraping.Runner do successful = Enum.count(results, &(&1 == :ok)) + if successful > 0 do + Catalog.mark_unseen_products_out_of_stock(retailer.id, caliber.id, now) + end + Catalog.update_retailer(retailer, %{last_scraped_at: now}) {:ok, %{products_count: successful, snapshots_count: successful}} diff --git a/lib/ammoprices/scraping/scrape_job.ex b/lib/ammoprices/scraping/scrape_job.ex index 5528f98..d39b353 100644 --- a/lib/ammoprices/scraping/scrape_job.ex +++ b/lib/ammoprices/scraping/scrape_job.ex @@ -18,6 +18,9 @@ defmodule Ammoprices.Scraping.ScrapeJob do Ammoprices.Scraping.Retailers.Natchez ] + # Products not seen for this many days are hard-deleted after each scrape cycle. + @stale_product_days 30 + @impl Oban.Worker def perform(_job) do calibers = Catalog.list_calibers() @@ -27,6 +30,8 @@ defmodule Ammoprices.Scraping.ScrapeJob do Runner.run(scraper, caliber) end + prune_stale_products() + Phoenix.PubSub.broadcast(Ammoprices.PubSub, "prices:updated", {:prices_updated, %{}}) schedule_next() @@ -34,6 +39,11 @@ defmodule Ammoprices.Scraping.ScrapeJob do :ok end + defp prune_stale_products do + cutoff = DateTime.add(DateTime.utc_now(), -@stale_product_days, :day) + Catalog.delete_stale_products(cutoff) + end + @doc false def schedule_next do {min, max} = Application.get_env(:ammoprices, :scrape_interval_seconds, {25_200, 43_200}) diff --git a/test/ammoprices/catalog_test.exs b/test/ammoprices/catalog_test.exs index 226c5ae..7798952 100644 --- a/test/ammoprices/catalog_test.exs +++ b/test/ammoprices/catalog_test.exs @@ -4,6 +4,86 @@ defmodule Ammoprices.CatalogTest do import Ammoprices.Fixtures alias Ammoprices.Catalog + alias Ammoprices.Catalog.Product + alias Ammoprices.Repo + + describe "mark_unseen_products_out_of_stock/3" do + test "marks products in scope with last_seen_at < cutoff as out of stock" do + retailer = retailer_fixture() + caliber = caliber_fixture() + cutoff = ~U[2026-04-22 12:00:00Z] + old = ~U[2026-04-22 11:00:00Z] + new = ~U[2026-04-22 12:30:00Z] + + stale = product_fixture(%{retailer: retailer, caliber: caliber, last_seen_at: old, in_stock: true}) + fresh = product_fixture(%{retailer: retailer, caliber: caliber, last_seen_at: new, in_stock: true}) + + assert {count, _} = Catalog.mark_unseen_products_out_of_stock(retailer.id, caliber.id, cutoff) + assert count == 1 + + assert Repo.get!(Product, stale.id).in_stock == false + assert Repo.get!(Product, fresh.id).in_stock == true + end + + test "does not affect products from other retailers" do + retailer = retailer_fixture() + other_retailer = retailer_fixture(%{slug: "other"}) + caliber = caliber_fixture() + cutoff = ~U[2026-04-22 12:00:00Z] + old = ~U[2026-04-22 11:00:00Z] + + other_stale = + product_fixture(%{retailer: other_retailer, caliber: caliber, last_seen_at: old, in_stock: true}) + + Catalog.mark_unseen_products_out_of_stock(retailer.id, caliber.id, cutoff) + + assert Repo.get!(Product, other_stale.id).in_stock == true + end + + test "does not affect products from other calibers" do + retailer = retailer_fixture() + caliber = caliber_fixture() + other_caliber = caliber_fixture(%{slug: "other"}) + cutoff = ~U[2026-04-22 12:00:00Z] + old = ~U[2026-04-22 11:00:00Z] + + other_stale = + product_fixture(%{retailer: retailer, caliber: other_caliber, last_seen_at: old, in_stock: true}) + + Catalog.mark_unseen_products_out_of_stock(retailer.id, caliber.id, cutoff) + + assert Repo.get!(Product, other_stale.id).in_stock == true + end + end + + describe "delete_stale_products/1" do + test "deletes products whose last_seen_at is older than cutoff" do + cutoff = ~U[2026-04-22 12:00:00Z] + old = ~U[2026-03-22 00:00:00Z] + new = ~U[2026-04-22 12:30:00Z] + + stale = product_fixture(%{last_seen_at: old}) + fresh = product_fixture(%{last_seen_at: new}) + + assert {count, _} = Catalog.delete_stale_products(cutoff) + assert count == 1 + + assert_raise Ecto.NoResultsError, fn -> Repo.get!(Product, stale.id) end + assert Repo.get!(Product, fresh.id) + end + + test "cascades to delete related price snapshots" do + old = ~U[2026-03-22 00:00:00Z] + cutoff = ~U[2026-04-22 12:00:00Z] + + stale = product_fixture(%{last_seen_at: old}) + snapshot = snapshot_fixture(%{product: stale}) + + Catalog.delete_stale_products(cutoff) + + refute Repo.get(Ammoprices.Prices.PriceSnapshot, snapshot.id) + end + end describe "retailers" do test "list_retailers/0 returns all retailers" do diff --git a/test/ammoprices/scraping/runner_test.exs b/test/ammoprices/scraping/runner_test.exs index dcaec8e..5dc27aa 100644 --- a/test/ammoprices/scraping/runner_test.exs +++ b/test/ammoprices/scraping/runner_test.exs @@ -4,6 +4,8 @@ defmodule Ammoprices.Scraping.RunnerTest do import Ammoprices.Fixtures alias Ammoprices.Catalog + alias Ammoprices.Catalog.Product + alias Ammoprices.Scraping.HttpClient alias Ammoprices.Scraping.Retailers.LuckyGunner alias Ammoprices.Scraping.Runner @@ -13,7 +15,7 @@ defmodule Ammoprices.Scraping.RunnerTest do retailer = retailer_fixture(%{name: "Lucky Gunner", slug: "lucky-gunner", base_url: "https://www.luckygunner.com"}) caliber = caliber_fixture(%{name: "9mm Luger", slug: "9mm-luger", category: "handgun", aliases: ["9mm", "9x19"]}) - Req.Test.stub(Ammoprices.Scraping.HttpClient, fn conn -> + Req.Test.stub(HttpClient, fn conn -> Req.Test.html(conn, @fixture_html) end) @@ -48,5 +50,63 @@ defmodule Ammoprices.Scraping.RunnerTest do assert {:ok, stats} = Runner.run(scraper, caliber) assert stats.products_count == 0 end + + test "marks products not seen in current scrape as out of stock", + %{retailer: retailer, caliber: caliber} do + # Pre-existing product that won't appear in the fixture scrape + old = DateTime.utc_now() |> DateTime.add(-3600, :second) |> DateTime.truncate(:second) + + stale = + product_fixture(%{ + retailer: retailer, + caliber: caliber, + url: "/products/no-longer-listed", + last_seen_at: old, + in_stock: true + }) + + assert {:ok, _stats} = Runner.run(LuckyGunner, caliber) + + assert Ammoprices.Repo.get!(Product, stale.id).in_stock == false + end + + test "does not prune products from other retailers", %{caliber: caliber} do + other_retailer = retailer_fixture(%{slug: "other", base_url: "https://other.com"}) + old = DateTime.utc_now() |> DateTime.add(-3600, :second) |> DateTime.truncate(:second) + + other = + product_fixture(%{ + retailer: other_retailer, + caliber: caliber, + url: "/products/other-retailer", + last_seen_at: old, + in_stock: true + }) + + assert {:ok, _stats} = Runner.run(LuckyGunner, caliber) + + assert Ammoprices.Repo.get!(Product, other.id).in_stock == true + end + + test "does not prune when scrape returns zero products", %{retailer: retailer, caliber: caliber} do + Req.Test.stub(HttpClient, fn conn -> + Req.Test.html(conn, "no products here") + end) + + old = DateTime.utc_now() |> DateTime.add(-3600, :second) |> DateTime.truncate(:second) + + stale = + product_fixture(%{ + retailer: retailer, + caliber: caliber, + url: "/products/still-here", + last_seen_at: old, + in_stock: true + }) + + assert {:ok, _stats} = Runner.run(LuckyGunner, caliber) + + assert Ammoprices.Repo.get!(Product, stale.id).in_stock == true + end end end diff --git a/test/ammoprices/scraping/scrape_job_test.exs b/test/ammoprices/scraping/scrape_job_test.exs index d5b9889..23d059b 100644 --- a/test/ammoprices/scraping/scrape_job_test.exs +++ b/test/ammoprices/scraping/scrape_job_test.exs @@ -55,5 +55,25 @@ defmodule Ammoprices.Scraping.ScrapeJobTest do test "executes scrape for all enabled scrapers and calibers" do assert :ok = perform_job(ScrapeJob, %{}) end + + test "hard-deletes products not seen in 30+ days after the scrape cycle" do + retailer = Ammoprices.Catalog.get_retailer_by_slug!("lucky-gunner") + caliber = Ammoprices.Catalog.get_caliber_by_slug!("9mm-luger") + + long_ago = DateTime.utc_now() |> DateTime.add(-31, :day) |> DateTime.truncate(:second) + + stale = + product_fixture(%{ + retailer: retailer, + caliber: caliber, + url: "/products/ancient", + last_seen_at: long_ago, + in_stock: false + }) + + assert :ok = perform_job(ScrapeJob, %{}) + + refute Ammoprices.Repo.get(Ammoprices.Catalog.Product, stale.id) + end end end