# robots.txt
User-agent: *
Disallow: order-consumerinfo.html
Disallow: order-freecreditprofile.html
Disallow: order-credit.html
Disallow: order-transunion-credit-report.html
Disallow: order-experian-credit-report.html

User-agent: *
Crawl-delay: 20 

User-agent: scooter
Disallow: /

User-agent: grub-client
Disallow: /

User-agent: grub
Disallow: /