Graham McIntire
9eb85faebd
Add True Shot Ammo (Shopify JSON), Palmetto State Armory (Magento HTML), Bulk Ammo (Magento HTML), and Natchez Shooters Supply (GraphQL API). Extends scraper infrastructure with HttpClient.post_json/3 and an optional fetch/2 callback on the Scraper behaviour for scrapers that need custom HTTP flows (e.g. GraphQL POST requests).
34 lines
1.2 KiB
Elixir
34 lines
1.2 KiB
Elixir
defmodule Ammoprices.Scraping.HttpClient do
|
|
@moduledoc false
|
|
|
|
@default_headers [
|
|
{"accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8"},
|
|
{"accept-language", "en-US,en;q=0.9"},
|
|
{"cache-control", "no-cache"},
|
|
{"pragma", "no-cache"},
|
|
{"sec-ch-ua", ~s("Chromium";v="128", "Not;A=Brand";v="24", "Google Chrome";v="128")},
|
|
{"sec-ch-ua-mobile", "?0"},
|
|
{"sec-ch-ua-platform", "\"macOS\""},
|
|
{"sec-fetch-dest", "document"},
|
|
{"sec-fetch-mode", "navigate"},
|
|
{"sec-fetch-site", "none"},
|
|
{"sec-fetch-user", "?1"},
|
|
{"upgrade-insecure-requests", "1"},
|
|
{"user-agent",
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"}
|
|
]
|
|
|
|
def get(url) do
|
|
Req.get(
|
|
[base_url: url, headers: @default_headers, retry: :transient, max_retries: 3],
|
|
Application.get_env(:ammoprices, :req_options, [])
|
|
)
|
|
end
|
|
|
|
def post_json(url, body, extra_headers \\ []) do
|
|
Req.post(
|
|
[base_url: url, headers: @default_headers ++ extra_headers, json: body, retry: :transient, max_retries: 3],
|
|
Application.get_env(:ammoprices, :req_options, [])
|
|
)
|
|
end
|
|
end
|