# www.robotstxt.org/
# www.google.com/support/webmasters/bin/answer.py?hl=en&answer=156449


# DISALLOW ALL 
User-agent: *
Disallow: /
Crawl-delay: 600


# ALLOW ONLY IMPORTANT
User-agent: Googlebot
# Directories
Disallow: /admin-login
Disallow: /admin/
Disallow: /assets/vendor
Disallow: /uploads
Disallow: /realisations
Disallow: /bundles
Disallow: /downloads
# Files
Disallow: /app/logs/prod.log
Crawl-delay: 20


User-agent: Googlebot-Image
# Directories
Disallow: /admin-login
Disallow: /admin/
Disallow: /assets/vendor
Disallow: /uploads
Disallow: /realisations
Disallow: /bundles
Disallow: /downloads
# Files
Disallow: /app/logs/prod.log
Crawl-delay: 20

User-agent: Mediapartners-Google
# Directories
Disallow: /admin-login
Disallow: /admin/
Disallow: /assets/vendor
Disallow: /uploads
Disallow: /realisations
Disallow: /bundles
Disallow: /downloads
# Files
Disallow: /app/logs/prod.log
Crawl-delay: 20


User-agent: msnbot
# Directories
Disallow: /admin-login
Disallow: /admin/
Disallow: /assets/vendor
Disallow: /uploads
Disallow: /realisations
Disallow: /bundles
Disallow: /downloads
# Files
Disallow: /app/logs/prod.log
Crawl-delay: 20


User-agent: msnbot-media
# Directories
Disallow: /admin-login
Disallow: /admin/
Disallow: /assets/vendor
Disallow: /download
Disallow: /uploads
Disallow: /realisations
Disallow: /bundles
Disallow: /downloads
# Files
Disallow: /app/logs/prod.log
Crawl-delay: 20


User-agent: Slurp
# Directories
Disallow: /admin-login
Disallow: /admin/
Disallow: /assets/vendor
Disallow: /uploads
Disallow: /realisations
Disallow: /bundles
Disallow: /downloads
# Files
Disallow: /app/logs/prod.log
Crawl-delay: 20


User-agent: Yahoo-Blogs
# Directories
Disallow: /admin-login
Disallow: /admin/
Disallow: /assets/vendor
Disallow: /uploads
Disallow: /realisations
Disallow: /bundles
Disallow: /downloads
# Files
Disallow: /app/logs/prod.log
Crawl-delay: 20

User-agent: Yahoo-MMCrawler
# Directories
Disallow: /admin-login
Disallow: /admin/
Disallow: /assets/vendor
Disallow: /uploads
Disallow: /realisations
Disallow: /bundles
Disallow: /downloads
# Files
Disallow: /app/logs/prod.log
Crawl-delay: 20

User-agent: Bingbot
# Directories
Disallow: /admin-login
Disallow: /admin/
Disallow: /assets/vendor
Disallow: /uploads
Disallow: /realisations
Disallow: /bundles
Disallow: /downloads
# Files
Disallow: /app/logs/prod.log
Crawl-delay: 20


User-agent: AddSearchBot
# Directories
Disallow: /admin-login
Disallow: /admin/
Disallow: /assets/vendor
Disallow: /uploads
Disallow: /realisations
Disallow: /bundles
Disallow: /downloads
# Files
Disallow: /app/logs/prod.log
Crawl-delay: 20


User-agent: rogerbot
# Directories
Disallow: /admin-login
Disallow: /admin/
Disallow: /assets/vendor
Disallow: /uploads
Disallow: /realisations
Disallow: /bundles
Disallow: /downloads
# Files
Disallow: /app/logs/prod.log