# $Id: robots.txt,v 1.9 2007/06/27 22:37:44 goba Exp $ # # robots.txt # # This file is to prevent the crawling and indexing of certain parts # of your site by web crawlers and spiders run by sites like Yahoo! # and Google. By telling these "robots" where not to go on your site, # you save bandwidth and server resources. # # This file will be ignored unless it is at the root of your host: # Used: http://example.com/robots.txt # Ignored: http://example.com/site/robots.txt # # For more information about the robots.txt standard, see: # http://www.robotstxt.org/wc/robots.html # # For syntax checking, see: # http://www.sxw.org.uk/computing/robots/check.html #User-agent: ia_archiver #Disallow: / User-agent: * Crawl-delay: 5 Disallow: /fastgallery/ Disallow: /gallery/v/slideshow/ Disallow: /v/slideshow/ Disallow: /gallery/v/slideshow Disallow: /gallery/v/slideshow.html Disallow: /gallery/v/slideshowapplet.html #Disallow: /gallery/v/gallery/ Disallow: /v/slideshow #Disallow: /gallery/map/ Disallow: /embed/ Disallow: /gallery/sitemap/ Disallow: /gallery/sitemap Disallow: /gallery/random?g2_albumId=7 Disallow: /gallery/random? Disallow: /gallery/random #Disallow: /gallery/updates?g2_albumId=7 #Disallow: /gallery/updates? #Disallow: /gallery/updates #Disallow: /gallery/updates/ #Disallow: /gallery/popular?g2_albumId=7 #Disallow: /gallery/popular? #Disallow: /gallery/popular #Disallow: /gallery/popular/ #Disallow: /gallery/main.php? #Disallow: /gallery/begin/ #Disallow: /gallery/key/ #Disallow: /gallery/srss/ #Sitemap: http://www.hardcoredreamer.com/sitemapindex.xml