User-agent: * # No searches Disallow: /codesearch/raw/ Sitemap: http://searchcode.com/sitemap.xml User-agent: Slurp Crawl-delay: 0.5 User-agent: MSNBot Crawl-delay: 0.5 User-agent: Googlebot Crawl-delay: 0.5