# http://www.searchengineworld.com/robots/robots_tutorial.htm # Robots.txt file to disallow indexing by search engine robots. # # The bars all robots from the following directories: ser-agent: * Disallow: /awstats/ Disallow: /cgi-bin/ Disallow: /@mail/ Disallow: /cp/ Disallow: /css/ Disallow: /favorites/ Disallow: /fmtemplates/ Disallow: /gb/ Disallow: /holiday/ Disallow: /images/ Disallow: /js/ Disallow: /modlogan/ Disallow: /online/ Disallow: /people/ Disallow: /photos/ Disallow: /search/ Disallow: /tree/ Disallow: /upload/ Disallow: /web/ Disallow: /webalizer/ # # This bans all robots from getting at the following files: User-agent: * Disallow: boldchat.php Disallow: bookmark.htm Disallow: checkmail.php Disallow: checkmail.php.bak Disallow: formmail.php Disallow: livechat.swf Disallow: nothanks.htm Disallow: people.htm Disallow: scripts.txt Disallow: thanks.htm Disallow: web.htm Disallow: wmcontact.htm # # This bans Roverdog from all files on the server: # User-agent: Roverdog # Disallow: / # # This bans googlebot from getting at the cheese.htm file: # User-agent: googlebot # Disallow: cheese.htm # # This keeps all robots out (enabled). # User-agent: * # Disallow: /