User-agent: * Crawl-Delay: 10 Disallow: /redirect.php Disallow: /account.php Disallow: /checkout_shipping.php Disallow: /create_account.php Disallow: /cookie_usage.php Disallow: /login.php Disallow: /password_forgotten.php Disallow: /popup_image.php Disallow: /shopping_cart.php Disallow: /advanced_search_result.php Disallow: /tell_a_friend.php User-agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.2.1; aggregator:Spinn3r (Spinn3r 3.1); http://spinn3r.com/robot) Gecko/20021130 Disallow: / User-agent: dotbot Disallow: / user-agent: AhrefsBot disallow: /