update robots.txt

This commit is contained in:
Graham McIntire 2026-06-04 16:29:53 -05:00
parent 2e0529ee75
commit f2c8f7020c
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59

View file

@ -3,7 +3,9 @@
# To ban all spiders from the entire site uncomment the next two lines:
User-agent: *
Disallow: /
Disallow: /info
Disallow: /packets
Disallow: /api/v1
User-agent: AhrefsBot
Disallow: /