# # this is the robots.txt file for http://www.tonypolito.com/ # # the pound symbol in first position indicates a comment # statement to be ignored in file reading and processing. # # Place the robots.txt file in the root directory of your # website to control what robots can and cannot index from # your website. # # To disallow robot activity, a two line statement is included # in the robots.txt file. The first line is titled User-Agent # and the # second line is titled Disallow. # # The two line pair can be repeated as needed to specify # different specifications for different robots. # # This example disallows all robots from any access # # User-agent: * # Disallow: # # This example disallows webcrawler from any access # # User-agent: webcrawler # Disallow: # # This example disallows lycra from the all subdirectories # INCLUDING the root directory, hence it effectively # denies all acccess # # User-agent: lycra # Disallow: / # # This example disallows all robots from the tmp and logs # subdirectories only. # # User-agent: * # Disallow: /tmp # Disallow: /logs # # This example disallows all robots from the the two files # foo.html and bar.html in the root directory only # # User-Agent: * # Disallow: /foo.html # Disallow: /bar.html # # for a list of cureently active robot names see # http://www.robotstxt.org/wc/active.html # # Remember that the robots CAN do as they please. The file # serves as a No Tresspassing Sign ... not a padlock. # User-Agent: * Disallow: /mybookmarks.htm # User-agent: AboutUsBot Disallow: /