User-agent: *
Allow: /
User-agent: *
Crawl-delay: 1000
Request-rate: 1/1000