User-agent: *
Disallow: /search
Disallow: /.preview/
Disallow: /.context-results
Disallow: /.applenews/

#AI crawlers

User-agent: bytespider
User-agent: mistralai
User-agent: cohere
User-agent: ai2bot
User-agent: youbot
User-agent: omgili
User-agent: diffbot
User-agent: kangaroo
User-agent: img2dataset
User-agent: amazonbot
User-agent: amazon-qbusiness

User-Agent: meta-externalagent
User-Agent: meta-webindexer
Disallow: /

#Allow AmazonAdBot
User-agent: AmazonAdBot
Disallow:

Sitemap: https://www.twice.com/sitemap_index.xml