# robots.txt for http://modernsage.com/
# Welcome bots 
# This is an robots.txt file that can be used to indicate to WWW robots which parts of a web site should not be indexed.
# Following two lines indicates that all robotes should stay away from the /cgi-bin/ folder.
 
User-agent: *
Disallow: /cgi-bin/

# Note to google: We have just added this file in case it's absence was causing a problem with our 404 page being served #instead.
# Following two lines indicates that all robotes should stay away from the /_private/ folder.

User-agent: *
Disallow: /_private/

# Following two lines indicates that all robotes should stay away from the /beta/ folder.

User-agent: *
Disallow: /beta/

# Uncomment following two lines by removing "#" at the beginning to 
# indicate that all robots should stay away from the /admin/ folder
#User-agent: *
#Disallow: /admin/

# Following two lines indicate that robots named "WWW_Wanderer_Name"
# should not index files in the /private/ folder.

User-agent: WWW_Wanderer_Name
Disallow: /private/

# Following two lines indicates that bot called "TurnitinBot" should stay away from all the URL's of the site.

User-agent: TurnitinBot
Disallow: / 

#Not Welcome Bad Bots should stay away from all the URL's of the site.

User-agent: ProPowerBot/2.14
Disallow: /

User-agent: Wget
Disallow: /

User-agent: WebZip
Disallow: /

User-agent: NetAnts
Disallow: /

User-agent: Black Hole
Disallow: /

User-agent: httplib
Disallow: /

User-agent: LinkextractorPro
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: CherryPicker
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: libWeb/clsHTTP
Disallow: /

User-agent: DittoSpyder
Disallow: /

User-agent: Mata Hari
Disallow: /

User-agent: WebEnhancer
Disallow: /

User-agent: LinkScan/8.1a Unix
Disallow: /

User-agent: turingos
Disallow: /

User-agent: moget/2.1
Disallow: /

User-agent: asterias
Disallow: /