defmodule Ammoprices.Scraping.Runner do @moduledoc false alias Ammoprices.Catalog alias Ammoprices.Prices alias Ammoprices.Scraping.HttpClient require Logger def run(scraper, caliber) do retailer = Catalog.get_retailer_by_slug!(scraper.retailer_slug()) if function_exported?(scraper, :fetch, 2) do do_fetch(scraper, retailer, caliber) else case scraper.category_url(caliber) do nil -> {:ok, %{products_count: 0, snapshots_count: 0}} path -> url = retailer.base_url <> path do_scrape(scraper, retailer, caliber, url) end end end defp do_fetch(scraper, retailer, caliber) do case scraper.fetch(retailer, caliber) do {:ok, parsed} -> process_results(parsed, retailer, caliber) {:error, reason} -> {:error, reason} end end defp do_scrape(scraper, retailer, caliber, url) do case HttpClient.get(url) do {:ok, %{status: 200, body: body}} -> parsed = scraper.parse_products(body) process_results(parsed, retailer, caliber) {:ok, %{status: status}} -> {:error, "HTTP #{status}"} {:error, reason} -> {:error, reason} end end defp process_results(parsed, retailer, caliber) do now = DateTime.truncate(DateTime.utc_now(), :second) {_results, successful} = Enum.reduce(parsed, {[], 0}, fn product_data, {acc, count} -> case process_product(product_data, retailer, caliber, now) do :ok -> {[:ok | acc], count + 1} :error -> {[:error | acc], count} end end) if successful > 0 do Catalog.mark_unseen_products_out_of_stock(retailer.id, caliber.id, now) end Catalog.update_retailer(retailer, %{last_scraped_at: now}) {:ok, %{products_count: successful, snapshots_count: successful}} end defp process_product(product_data, retailer, caliber, now) do product_attrs = %{ title: product_data.title, url: product_data.url, brand: product_data.brand, grain_weight: product_data.grain_weight, round_count: product_data.round_count, casing: product_data.casing, condition: "new", in_stock: product_data.in_stock, subsonic: Map.get(product_data, :subsonic, false), last_seen_at: now } with {:ok, product} <- Catalog.upsert_product(retailer.id, caliber.id, product_attrs), {:ok, _snapshot} <- create_snapshot(product, product_data, now) do :ok else {:error, %Ecto.Changeset{} = changeset} -> Logger.warning( "Scrape insert failed for #{retailer.slug} #{product_data.url}: " <> inspect(changeset.errors) ) :error end end defp create_snapshot(product, product_data, now) do Prices.create_snapshot(product.id, %{ price_cents: product_data.price_cents, price_per_round_cents: product_data.price_per_round_cents, in_stock: product_data.in_stock, recorded_at: now }) end end