# Pete, Feb 2005 - added blocking of various bots
# Paul, Jan 2009 - fixes as suggested by validate tool.
# Pete, May 2009 - removed googlebot as it is murdeing us on bandwidth.
#
# Useful URL to validate this:
# http://tool.motoricerca.info/robots-checker.phtml

# Notes   pgb:
#
#         Each record consists of multiple lines
#
#         blank lines separate records
#
#         a record consts of one or more user-agent lines, followed
#         by one or more dissallow lines - no blank lines within a record.
#
#         best to have the 'User-agent: *' block at the end

# Allow ok search engines
#User-agent: Googlebot
User-agent: ia_archiver
User-agent: ia_archiver-web.archive.org
# Disallow all twiki cgis except view (nb only when used with no query string)
Disallow: /twiki/bin/rdiff
Disallow: /twiki/bin/attach
Disallow: /twiki/bin/changes
Disallow: /twiki/bin/configure
Disallow: /twiki/bin/edit
Disallow: /twiki/bin/login
Disallow: /twiki/bin/logon
Disallow: /twiki/bin/logos
Disallow: /twiki/bin/manage
Disallow: /twiki/bin/oops
Disallow: /twiki/bin/passwd
Disallow: /twiki/bin/preview
Disallow: /twiki/bin/rdiff
Disallow: /twiki/bin/rdiffauth
Disallow: /twiki/bin/register
Disallow: /twiki/bin/rename
Disallow: /twiki/bin/resetpasswd
Disallow: /twiki/bin/rest
Disallow: /twiki/bin/save
Disallow: /twiki/bin/search
Disallow: /twiki/bin/setlib.cfg
Disallow: /twiki/bin/statistics
Disallow: /twiki/bin/twiki
Disallow: /twiki/bin/upload
Disallow: /twiki/bin/viewauth
Disallow: /twiki/bin/viewfile
# Fake path created using mod_redirect to identify use of /twiki/bin/view with a
# query string - reject those.
Disallow: /twiki/query/
# We only want Main to be searched so exclude other webs
Disallow: /twiki/bin/view/TWiki/
Disallow: /twiki/bin/view/Sandbox/
Disallow: /twiki/bin/view/Trash/

# Disallow everything else
User-agent: *
Disallow: /