# ***************************************************************** #
# Web spired (Aspirateurs de sites): keep it/them out of everything.
# ***************************************************************** #
User-agent: teleport
Disallow: /
User-agent: wget
Disallow: /
User-agent: webzip
Disallow: /
User-agent: webcopier
Disallow: /
User-agent: offline
Disallow: /
User-agent: webbandit
Disallow: /
User-agent: webstripper
Disallow: /
User-agent: sitesnagger
Disallow: /
User-agent: website
Disallow: /
User-agent: teleportpro
Disallow: /
User-agent: webstripper
Disallow: /
# *********************************************** #
# Mail Collectors: keep it/them out of everything.
# *********************************************** #
User-agent: crescent
Disallow: /
User-agent: emailsiphon
Disallow: /
User-agent: extractorpro
Disallow: /
User-agent: emailcollector
Disallow: /
User-agent: emailwolf
Disallow: /
User-agent: nicerspro
Disallow: /
# ************************************************** #
# Other specific one: keep it/them out of everything.
# ************************************************** #
User-agent: scooter	 	# Robot d'Astalavista
Disallow: /
# ************************ #
# Other "good" webspiders:
# ************************ #
User-agent: * 			# match all other bots.
Crawl-delay: 10			# number of seconds to wait between successive requests to the same server, by crawlers.
Visit-time: 0000-1200		# only visit between 00:00 and 12:00 UTC (GMT)
# Now keep them out of:
Disallow: /*?vc=
Disallow: /picasso

#and so, allow all the rest:
Disallow: