User-agent: Amazonbot # Amazon's user agent Disallow: /do-not-crawl/ # disallow this directory