diff --git a/priv/static/robots.txt b/priv/static/robots.txt index 777e3b1..453cf89 100644 --- a/priv/static/robots.txt +++ b/priv/static/robots.txt @@ -3,7 +3,9 @@ # To ban all spiders from the entire site uncomment the next two lines: User-agent: * -Disallow: / +Disallow: /info +Disallow: /packets +Disallow: /api/v1 User-agent: AhrefsBot Disallow: /