update robots.txt
This commit is contained in:
parent
2e0529ee75
commit
f2c8f7020c
1 changed files with 3 additions and 1 deletions
|
|
@ -3,7 +3,9 @@
|
|||
# To ban all spiders from the entire site uncomment the next two lines:
|
||||
|
||||
User-agent: *
|
||||
Disallow: /
|
||||
Disallow: /info
|
||||
Disallow: /packets
|
||||
Disallow: /api/v1
|
||||
|
||||
User-agent: AhrefsBot
|
||||
Disallow: /
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue