###############################
#
User-agent: athenusbot
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: psbot
Disallow: /

User-agent: FindLinks
Disallow: /

User-agent: Gaisbot
Disallow: /

User-agent: NextGenSearchBot
Disallow: /

User-agent: Gigabot
Disallow: /

User-agent: BecomeBot
Disallow: /

User-agent: OutfoxBot
Disallow: /

User-agent: e-SocietyRobot
Disallow: /

User-agent: FAST
Disallow: /

User-agent: ZyBorg
Disallow: /

User-agent: grub-client
Disallow: /

User-agent: fast
Disallow: /

User-agent: NaverBot
Disallow: /

User-agent: baiduspider
Disallow: /

User-agent: LNSpiderguy
Disallow: /

User-agent: QuepasaCreep
Disallow: /

User-agent: VoilaBot
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: dotbot
Disallow: / 

#
User-agent: *
#
# list folders robots are not allowed to index 
#
Disallow: /php/
Disallow: /amazon_functions/
Disallow: /amazon_icons/
Disallow: /dw_test/
Disallow: /logs/
Disallow: /mail_list/
Disallow: /news_images/
Disallow: /subscribers/
Disallow: /testing/
Disallow: /news/amazon/
Disallow: /printfriendly/
Disallow: /buy/item/
Disallow: /brightsurf/
Disallow: /news/science/tag/
Disallow: /news/email/
Disallow: /news/science/tag/
#
# list specific files robots are not allowed to index
#
Disallow: NEWS_TEST.html
Disallow: /news/inc/nav.cat.php?CatDiv=CatDiv1
Disallow: /news/inc/nav.cat.php?CatDiv=CatDiv2
Disallow: /news/inc/nav.cat.php?CatDiv=CatDiv3
Disallow: /news/inc/nav.cat.php?CatDiv=CatDiv4
Disallow: /news/inc/nav.cat.php?CatDiv=CatDiv5
Disallow: /news/inc/nav.cat.php?CatDiv=CatDiv6
Disallow: /news/inc/nav.cat.php?CatDiv=CatDiv7





# 
#
###############################