User-agent: *
Disallow: /agents
Disallow: /intranet
Disallow: /fpdb
Disallow: /telnet
Disallow: /images
Disallow: /_private
Disallow: /_scripts
Disallow: /1999
Disallow: /2000
Disallow: /_2001
Disallow: /cgi-bin
Disallow: /forms
Disallow: /fpdb
Disallow: /01ask.css
Disallow: /0701.ask.css
Disallow: /101ask.css
Disallow: /ask.css
Disallow: /askorder.css
Disallow: /browser_sensing.js
Disallow: global.asa
Disallow: /pre-employment/pre-index.htm
# exclude help system from robots
Disallow: /manual/
Disallow: /manual-2.0/
Disallow: /manual-1.3/
Disallow: /addon-modules/
Disallow: /doc/
Disallow: /images/
# the next line is a spam bot trap, for grepping the logs. you should _really_ change this to something else...
#Disallow: /all_our_e-mail_addresses
# same idea here...
#Disallow: /admin/
# but allow htdig to index our doc-tree
#User-agent: htdig
#Disallow:
# disallow stress test
user-agent: stress-agent
Disallow: /