# Whitelisted user-agents are allowed
User-agent: googlebot
User-agent: slurp
User-agent: msnbot
User-agent: teoma
User-Agent: Charlotte
User-agent: ia_archiver
user-agent: SearchImprove-crawler
user-agent: SiteCheck-sitecrawl
Disallow: /cgi-bin
Disallow: /email-addr
# Disallow all others
User-agent: *
Disallow: /