User-agent: iecrawler
Disallow: /

User-agent: msiecrawler
Disallow: /

User-agent: httrack
Disallow: /

User-agent: rufusbot
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Slurp
# Custom extension: (see http://help.yahoo.com/help/us/ysearch/slurp/slurp-03.html)
Crawl-delay: 60
# Repeating User-agent: * here (spec is not clear if needed)
# This section needs to be in sync with section below!
Disallow: /twiki/pub/
Disallow: /cgi-bin/twiki/bin/attach
Disallow: /cgi-bin/twiki/bin/changes
Disallow: /cgi-bin/twiki/bin/edit
Disallow: /cgi-bin/twiki/bin/geturl
Disallow: /cgi-bin/twiki/bin/installpasswd
Disallow: /cgi-bin/twiki/bin/mailnotify
Disallow: /cgi-bin/twiki/bin/manage
Disallow: /cgi-bin/twiki/bin/oops
Disallow: /cgi-bin/twiki/bin/passwd
Disallow: /cgi-bin/twiki/bin/preview
Disallow: /cgi-bin/twiki/bin/rdiff
Disallow: /cgi-bin/twiki/bin/register
Disallow: /cgi-bin/twiki/bin/rename
Disallow: /cgi-bin/twiki/bin/save
Disallow: /cgi-bin/twiki/bin/search
Disallow: /cgi-bin/twiki/bin/statistics
Disallow: /cgi-bin/twiki/bin/upload
Disallow: /cgi-bin/twiki/bin/viewauth
Disallow: /cgi-bin/twiki/bin/viewfile

Disallow: /cgi-bin/twiki/bin/survey
Disallow: /cgi-bin/twiki/bin/twikicvs

Disallow: /cgi-bin/twiki/bin/*?skin=print*
Disallow: /cgi-bin/twiki/bin/*?rev=*
Disallow: /cgi-bin/twiki/bin/*?raw=*
Disallow: /cgi-bin/twiki/bin/*?foo=*
Disallow: /cgi-bin/twiki/bin/*?sortcol=*

Disallow: /cgi-bin/twiki/bin/awstats/*
Disallow: /cgi-bin/twiki/bin/*Trash*
Disallow: /cgi-bin/twiki/bin/*Sandbox*

User-agent: *
Disallow: /twiki/pub/
Disallow: /cgi-bin/twiki/bin/attach
Disallow: /cgi-bin/twiki/bin/changes
Disallow: /cgi-bin/twiki/bin/edit
Disallow: /cgi-bin/twiki/bin/geturl
Disallow: /cgi-bin/twiki/bin/installpasswd
Disallow: /cgi-bin/twiki/bin/mailnotify
Disallow: /cgi-bin/twiki/bin/manage
Disallow: /cgi-bin/twiki/bin/oops
Disallow: /cgi-bin/twiki/bin/passwd
Disallow: /cgi-bin/twiki/bin/preview
Disallow: /cgi-bin/twiki/bin/rdiff
Disallow: /cgi-bin/twiki/bin/register
Disallow: /cgi-bin/twiki/bin/rename
Disallow: /cgi-bin/twiki/bin/save
Disallow: /cgi-bin/twiki/bin/search
Disallow: /cgi-bin/twiki/bin/statistics
Disallow: /cgi-bin/twiki/bin/upload
Disallow: /cgi-bin/twiki/bin/viewauth
Disallow: /cgi-bin/twiki/bin/viewfile

Disallow: /cgi-bin/twiki/bin/survey
Disallow: /cgi-bin/twiki/bin/twikicvs

Disallow: /cgi-bin/twiki/bin/*?skin=print*
Disallow: /cgi-bin/twiki/bin/*?rev=*
Disallow: /cgi-bin/twiki/bin/*?raw=*
Disallow: /cgi-bin/twiki/bin/*?foo=*
Disallow: /cgi-bin/twiki/bin/*?sortcol=*

Disallow: /cgi-bin/twiki/bin/awstats/*
Disallow: /cgi-bin/twiki/bin/*Trash*
Disallow: /cgi-bin/twiki/bin/*Sandbox*