Prune listings that disappear from retailer scrapes

When a product no longer appears in a successful scrape for its
retailer+caliber, mark it out of stock. After each scrape cycle, hard
delete any product whose last_seen_at is older than 30 days (snapshots
cascade via FK). A zero-result scrape leaves existing listings alone so
a parse failure or transient site glitch can't nuke the catalog.

lib/ammoprices/catalog.ex

@@ -84,6 +84,34 @@ defmodule Ammoprices.Catalog do
     )
   end
 
+  @doc """
+  Marks products belonging to `retailer_id`/`caliber_id` as out of stock when
+  their `last_seen_at` is strictly before `cutoff`. Returns `{count, nil}`.
+
+  Used by the scrape runner to flag listings that disappeared from the
+  retailer's current category response.
+  """
+  def mark_unseen_products_out_of_stock(retailer_id, caliber_id, %DateTime{} = cutoff) do
+    now = DateTime.truncate(DateTime.utc_now(), :second)
+
+    Product
+    |> where([p], p.retailer_id == ^retailer_id)
+    |> where([p], p.caliber_id == ^caliber_id)
+    |> where([p], p.last_seen_at < ^cutoff)
+    |> where([p], p.in_stock == true)
+    |> Repo.update_all(set: [in_stock: false, updated_at: now])
+  end
+
+  @doc """
+  Hard-deletes products whose `last_seen_at` is older than `cutoff`. Related
+  price snapshots cascade via the foreign key. Returns `{count, nil}`.
+  """
+  def delete_stale_products(%DateTime{} = cutoff) do
+    Product
+    |> where([p], p.last_seen_at < ^cutoff)
+    |> Repo.delete_all()
+  end
+
   def list_products_for_caliber(caliber_id, opts \\ []) do
     limit = Keyword.get(opts, :limit, 100)
 

lib/ammoprices/scraping/runner.ex

@@ -83,6 +83,10 @@ defmodule Ammoprices.Scraping.Runner do
 
     successful = Enum.count(results, &(&1 == :ok))
 
+    if successful > 0 do
+      Catalog.mark_unseen_products_out_of_stock(retailer.id, caliber.id, now)
+    end
+
     Catalog.update_retailer(retailer, %{last_scraped_at: now})
 
     {:ok, %{products_count: successful, snapshots_count: successful}}

lib/ammoprices/scraping/scrape_job.ex

@@ -18,6 +18,9 @@ defmodule Ammoprices.Scraping.ScrapeJob do
     Ammoprices.Scraping.Retailers.Natchez
   ]
 
+  # Products not seen for this many days are hard-deleted after each scrape cycle.
+  @stale_product_days 30
+
   @impl Oban.Worker
   def perform(_job) do
     calibers = Catalog.list_calibers()
@@ -27,6 +30,8 @@ defmodule Ammoprices.Scraping.ScrapeJob do
       Runner.run(scraper, caliber)
     end
 
+    prune_stale_products()
+
     Phoenix.PubSub.broadcast(Ammoprices.PubSub, "prices:updated", {:prices_updated, %{}})
 
     schedule_next()
@@ -34,6 +39,11 @@ defmodule Ammoprices.Scraping.ScrapeJob do
     :ok
   end
 
+  defp prune_stale_products do
+    cutoff = DateTime.add(DateTime.utc_now(), -@stale_product_days, :day)
+    Catalog.delete_stale_products(cutoff)
+  end
+
   @doc false
   def schedule_next do
     {min, max} = Application.get_env(:ammoprices, :scrape_interval_seconds, {25_200, 43_200})

test/ammoprices/catalog_test.exs

@@ -4,6 +4,86 @@ defmodule Ammoprices.CatalogTest do
   import Ammoprices.Fixtures
 
   alias Ammoprices.Catalog
+  alias Ammoprices.Catalog.Product
+  alias Ammoprices.Repo
+
+  describe "mark_unseen_products_out_of_stock/3" do
+    test "marks products in scope with last_seen_at < cutoff as out of stock" do
+      retailer = retailer_fixture()
+      caliber = caliber_fixture()
+      cutoff = ~U[2026-04-22 12:00:00Z]
+      old = ~U[2026-04-22 11:00:00Z]
+      new = ~U[2026-04-22 12:30:00Z]
+
+      stale = product_fixture(%{retailer: retailer, caliber: caliber, last_seen_at: old, in_stock: true})
+      fresh = product_fixture(%{retailer: retailer, caliber: caliber, last_seen_at: new, in_stock: true})
+
+      assert {count, _} = Catalog.mark_unseen_products_out_of_stock(retailer.id, caliber.id, cutoff)
+      assert count == 1
+
+      assert Repo.get!(Product, stale.id).in_stock == false
+      assert Repo.get!(Product, fresh.id).in_stock == true
+    end
+
+    test "does not affect products from other retailers" do
+      retailer = retailer_fixture()
+      other_retailer = retailer_fixture(%{slug: "other"})
+      caliber = caliber_fixture()
+      cutoff = ~U[2026-04-22 12:00:00Z]
+      old = ~U[2026-04-22 11:00:00Z]
+
+      other_stale =
+        product_fixture(%{retailer: other_retailer, caliber: caliber, last_seen_at: old, in_stock: true})
+
+      Catalog.mark_unseen_products_out_of_stock(retailer.id, caliber.id, cutoff)
+
+      assert Repo.get!(Product, other_stale.id).in_stock == true
+    end
+
+    test "does not affect products from other calibers" do
+      retailer = retailer_fixture()
+      caliber = caliber_fixture()
+      other_caliber = caliber_fixture(%{slug: "other"})
+      cutoff = ~U[2026-04-22 12:00:00Z]
+      old = ~U[2026-04-22 11:00:00Z]
+
+      other_stale =
+        product_fixture(%{retailer: retailer, caliber: other_caliber, last_seen_at: old, in_stock: true})
+
+      Catalog.mark_unseen_products_out_of_stock(retailer.id, caliber.id, cutoff)
+
+      assert Repo.get!(Product, other_stale.id).in_stock == true
+    end
+  end
+
+  describe "delete_stale_products/1" do
+    test "deletes products whose last_seen_at is older than cutoff" do
+      cutoff = ~U[2026-04-22 12:00:00Z]
+      old = ~U[2026-03-22 00:00:00Z]
+      new = ~U[2026-04-22 12:30:00Z]
+
+      stale = product_fixture(%{last_seen_at: old})
+      fresh = product_fixture(%{last_seen_at: new})
+
+      assert {count, _} = Catalog.delete_stale_products(cutoff)
+      assert count == 1
+
+      assert_raise Ecto.NoResultsError, fn -> Repo.get!(Product, stale.id) end
+      assert Repo.get!(Product, fresh.id)
+    end
+
+    test "cascades to delete related price snapshots" do
+      old = ~U[2026-03-22 00:00:00Z]
+      cutoff = ~U[2026-04-22 12:00:00Z]
+
+      stale = product_fixture(%{last_seen_at: old})
+      snapshot = snapshot_fixture(%{product: stale})
+
+      Catalog.delete_stale_products(cutoff)
+
+      refute Repo.get(Ammoprices.Prices.PriceSnapshot, snapshot.id)
+    end
+  end
 
   describe "retailers" do
     test "list_retailers/0 returns all retailers" do

test/ammoprices/scraping/runner_test.exs

@@ -4,6 +4,8 @@ defmodule Ammoprices.Scraping.RunnerTest do
   import Ammoprices.Fixtures
 
   alias Ammoprices.Catalog
+  alias Ammoprices.Catalog.Product
+  alias Ammoprices.Scraping.HttpClient
   alias Ammoprices.Scraping.Retailers.LuckyGunner
   alias Ammoprices.Scraping.Runner
 
@@ -13,7 +15,7 @@ defmodule Ammoprices.Scraping.RunnerTest do
     retailer = retailer_fixture(%{name: "Lucky Gunner", slug: "lucky-gunner", base_url: "https://www.luckygunner.com"})
     caliber = caliber_fixture(%{name: "9mm Luger", slug: "9mm-luger", category: "handgun", aliases: ["9mm", "9x19"]})
 
-    Req.Test.stub(Ammoprices.Scraping.HttpClient, fn conn ->
+    Req.Test.stub(HttpClient, fn conn ->
       Req.Test.html(conn, @fixture_html)
     end)
 
@@ -48,5 +50,63 @@ defmodule Ammoprices.Scraping.RunnerTest do
       assert {:ok, stats} = Runner.run(scraper, caliber)
       assert stats.products_count == 0
     end
+
+    test "marks products not seen in current scrape as out of stock",
+         %{retailer: retailer, caliber: caliber} do
+      # Pre-existing product that won't appear in the fixture scrape
+      old = DateTime.utc_now() |> DateTime.add(-3600, :second) |> DateTime.truncate(:second)
+
+      stale =
+        product_fixture(%{
+          retailer: retailer,
+          caliber: caliber,
+          url: "/products/no-longer-listed",
+          last_seen_at: old,
+          in_stock: true
+        })
+
+      assert {:ok, _stats} = Runner.run(LuckyGunner, caliber)
+
+      assert Ammoprices.Repo.get!(Product, stale.id).in_stock == false
+    end
+
+    test "does not prune products from other retailers", %{caliber: caliber} do
+      other_retailer = retailer_fixture(%{slug: "other", base_url: "https://other.com"})
+      old = DateTime.utc_now() |> DateTime.add(-3600, :second) |> DateTime.truncate(:second)
+
+      other =
+        product_fixture(%{
+          retailer: other_retailer,
+          caliber: caliber,
+          url: "/products/other-retailer",
+          last_seen_at: old,
+          in_stock: true
+        })
+
+      assert {:ok, _stats} = Runner.run(LuckyGunner, caliber)
+
+      assert Ammoprices.Repo.get!(Product, other.id).in_stock == true
+    end
+
+    test "does not prune when scrape returns zero products", %{retailer: retailer, caliber: caliber} do
+      Req.Test.stub(HttpClient, fn conn ->
+        Req.Test.html(conn, "<html><body>no products here</body></html>")
+      end)
+
+      old = DateTime.utc_now() |> DateTime.add(-3600, :second) |> DateTime.truncate(:second)
+
+      stale =
+        product_fixture(%{
+          retailer: retailer,
+          caliber: caliber,
+          url: "/products/still-here",
+          last_seen_at: old,
+          in_stock: true
+        })
+
+      assert {:ok, _stats} = Runner.run(LuckyGunner, caliber)
+
+      assert Ammoprices.Repo.get!(Product, stale.id).in_stock == true
+    end
   end
 end

test/ammoprices/scraping/scrape_job_test.exs

@@ -55,5 +55,25 @@ defmodule Ammoprices.Scraping.ScrapeJobTest do
     test "executes scrape for all enabled scrapers and calibers" do
       assert :ok = perform_job(ScrapeJob, %{})
     end
+
+    test "hard-deletes products not seen in 30+ days after the scrape cycle" do
+      retailer = Ammoprices.Catalog.get_retailer_by_slug!("lucky-gunner")
+      caliber = Ammoprices.Catalog.get_caliber_by_slug!("9mm-luger")
+
+      long_ago = DateTime.utc_now() |> DateTime.add(-31, :day) |> DateTime.truncate(:second)
+
+      stale =
+        product_fixture(%{
+          retailer: retailer,
+          caliber: caliber,
+          url: "/products/ancient",
+          last_seen_at: long_ago,
+          in_stock: false
+        })
+
+      assert :ok = perform_job(ScrapeJob, %{})
+
+      refute Ammoprices.Repo.get(Ammoprices.Catalog.Product, stale.id)
+    end
   end
 end