User-agent: Seekport Crawler
Disallow: /

User-agent: seekbot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: Vagabondo
Disallow: /

User-agent: Bilbo
Disallow: /

User-agent: Linguee
Disallow: /

User-agent: Fasterfox
Disallow: /

User-agent: otbot
Disallow: /

User-agent: OOZBOT
Disallow: /

User-agent: twiceler
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Baiduspider-video
Disallow: /

User-agent: Baiduspider-image
Disallow: /

#User-agent: Yandex
#Disallow: /

User-agent: YandexAntivirus
Disallow: /

User-agent: sistrix
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: CazoodleBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: dotbot/1.0
Disallow: /

User-agent: Gigabot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-Agent: ia_archiver
Disallow: /

User-agent: TwengaBot-2.0
Disallow: /

User-agent: WooRank
Disallow: /

User-agent: spbot
Disallow: /

User-agent: Qwantify
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: seoscanners.net
Disallow: /

User-agent: DnyzBot
Disallow: /

User-agent: BUbiNG
Disallow: /

User-agent: Cliqzbot
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: Mail.RU_Bot
Disallow: /

User-agent: Mail.Ru
Disallow: /

User-agent: dataprovider
Disallow: /

User-agent: AlphaSeoBot
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: oBot
Disallow: /

User-agent: SEOkicks-Robot
Disallow: /

User-agent: ExaBot
Disallow: /

User-agent: ltx71 - (http://ltx71.com/)
Disallow: /

User-agent: VelenPublicWebCrawler
Disallow: /

User-agent: WebDataStats/1.0
Disallow: /

User-agent: Sogou Spider
Disallow: /

User-agent: Barkrowler
Disallow:/

User-agent: MegaIndex.ru/2.0
Disallow: /

User-agent: SafeDNSBot
Disallow: /

User-agent: CheckMarkNetwork/1.0 (+http://www.checkmarknetwork.com/spider.html)
Disallow: /

User-agent: DomainStatsBot
Disallow: /

User-agent: Httrack
Disallow: /

User-agent: Uptimebot
Disallow: /

User-agent: coccocbot-web
Disallow: /

User-agent: Cliqzbot
Disallow: /

User-agent: elefent
Disallow: /

User-agent: pricebot
Disallow: /

User-agent: serpstatbot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-Agent: adscanner
Disallow: /

User-agent: PetalBot
Disallow: /

User-Agent: msnbot
Crawl-Delay: 20

User-Agent: Slurp
Crawl-delay: 20

User-Agent: Yahoo! Slurp
Crawl-delay: 20

User-agent: AhrefsBot
Crawl-Delay: 2

# Crawlers Setup
User-agent: *

# Allowable Index
# Mind that Allow is not an official standard
Allow: /*?p=
#Allow: /index.php/blog/
#Allow: /catalogsearch/result/
Allow: /media/catalog/
Disallow: /pdf/

# Directories
Disallow: /404/
Disallow: /app/
Disallow: /cgi-bin/
Disallow: /downloader/
Disallow: /errors/
Disallow: /includes/
#Disallow: /js/
Disallow: /lib/
Disallow: /magento/

# Media
Disallow: /media/import/
Disallow: /media/pdf/
Disallow: /media/sales/
Disallow: /media/tmp/
Disallow: /media/wysiwyg/
Disallow: /media/xmlconnect/
Disallow: /pkginfo/
Disallow: /report/
Disallow: /scripts/
Disallow: /shell/
Disallow: /stats/
Disallow: /var/
Disallow: /spree/
Disallow: /tmp/
Disallow: /catalog/seo_sitemap/

# Paths (clean URLs)
Disallow: /index.php
Disallow: /index.php/
#Disallow: /home
Disallow: /catalog/product_compare/
Disallow: /catalog/category/view/
Disallow: /catalog/product/view/
Disallow: /catalog/product/gallery/
Disallow: /catalogsearch/
Disallow: /checkout/
Disallow: /control/
#Disallow: /contacts/
Disallow: /customer/
Disallow: /customer
Disallow: /customize/
Disallow: /newsletter/
Disallow: /poll/
Disallow: /review/
Disallow: /sendfriend/
Disallow: /tag/
Disallow: /wishlist/
Disallow: /onestepcheckout
Disallow: /admin
Disallow: /links
#Disallow: /op-aanvraag
#Disallow: /ideal
#Disallow: /privacy-policy
#Disallow: /algemene-voorwaarden
#Disallow: /klantenservice
#Disallow: /over-ons

# Files
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /STATUS.txt
Disallow: /get.php

# Paths (no clean URLs)
#Disallow: /*.js$
#Disallow: /*.css$
Disallow: /*.php$
Disallow: /*?SID=
Disallow: /rss*
Disallow: /*PHPSESSID
Disallow: /*?
Disallow: /ajaxsearch/
Disallow: /*?q=
Disallow: /*?order=
Disallow: /*?dir=
Disallow: /*?limit=
Disallow: /*?p=
Disallow: /*?price=
Disallow: /*?index=
Disallow: /*?gclid=
Disallow: /*?cat=
Disallow: /social/popup
Disallow: /autocomplete
Disallow: /omnikassa
Disallow: /mpm
Disallow: /yandex_*
Disallow: /cdn-cgi/