Graham McIntire
bbe5bde145
Phoenix 1.8 app that scrapes ammunition retailers (Lucky Gunner, SGAmmo) for price data and displays historical price trends. - Data model: retailers, calibers, products, price snapshots - Scraper infrastructure with Req, Floki, realistic browser headers - Oban-scheduled scrape jobs (every 4h with randomized delays) - LiveView pages: homepage with category cards, caliber detail with price table, Chart.js price history, and price stats banner - 18 seeded calibers across handgun/rifle/rimfire/shotgun categories - 77 tests
27 lines
983 B
Elixir
27 lines
983 B
Elixir
defmodule Ammoprices.Scraping.HttpClient do
|
|
@moduledoc false
|
|
|
|
@default_headers [
|
|
{"accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8"},
|
|
{"accept-language", "en-US,en;q=0.9"},
|
|
{"cache-control", "no-cache"},
|
|
{"pragma", "no-cache"},
|
|
{"sec-ch-ua", ~s("Chromium";v="128", "Not;A=Brand";v="24", "Google Chrome";v="128")},
|
|
{"sec-ch-ua-mobile", "?0"},
|
|
{"sec-ch-ua-platform", "\"macOS\""},
|
|
{"sec-fetch-dest", "document"},
|
|
{"sec-fetch-mode", "navigate"},
|
|
{"sec-fetch-site", "none"},
|
|
{"sec-fetch-user", "?1"},
|
|
{"upgrade-insecure-requests", "1"},
|
|
{"user-agent",
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"}
|
|
]
|
|
|
|
def get(url) do
|
|
Req.get(
|
|
[base_url: url, headers: @default_headers, retry: :transient, max_retries: 3],
|
|
Application.get_env(:ammoprices, :req_options, [])
|
|
)
|
|
end
|
|
end
|