User-agent: MJ12bot
Crawl-delay: 5

User-agent: SemrushBot
Crawl-delay: 1000

User-agent: DotBot
Crawl-delay: 2000

User-agent: AhrefsBot
Crawl-delay: 500

User-agent: ia_archiver 
Crawl-delay: 500

User-agent: rogerbot
Crawl-delay: 500

User-agent: OnCrawl
Crawl-delay: 500

User-agent: bingbot 
Crawl-delay: 1000


User-agent: *
Allow: /

Sitemap: http://agregation.net/sitemap_mag.txt
Sitemap: http://agregation.net/sitemap.txt