# Robots.txt file

# Disallow this extremely silly version of Direct Hit Grabber
User-Agent: Direct Hit Grabber
User-Agent: Mozilla/4.06 [en] (Direct Hit Grabber)
Disallow: /

# Disallow this extremely silly version of Inktomi's Web robot
User-Agent: Slurp
Disallow: /

# Disallow this extremely silly version of Scooter
User-Agent: Scooter/2.0
Disallow: /

# Let all other crawlers do their stuff unhindered
User-Agent: *