# Google Image Crawler Setup
User-agent: Googlebot-Image
# Crawlers Setup
User-agent: *
Disallow: /*?
# Directories
User-agent: *
Disallow: /404/
Disallow: /app/
Disallow: /cgi-bin/
Disallow: /downloader/
Disallow: /includes/
Disallow: /js/
Disallow: /lib/
Disallow: /magento/
Disallow: /pkginfo/
Disallow: /report/
Disallow: /skin/
Disallow: /stats/
Disallow: /var/
Disallow: /blog/

# Paths (clean URLs)
User-agent: *
Disallow: /index.php/
Disallow: /rest_test.php/
Disallow: /product_test_woimage.php/
Disallow: /reindexing.php/
Disallow: /product_test.php/
Disallow: /orders.php/
Disallow: /import_special_feed.php/
Disallow: /import_media_feed.php/
Disallow: /getMailchimpResponse.php/
Disallow: /applysinglerule.php/
Disallow: /applyrule.php/
Disallow: /cron.php/
Disallow: /general_org.php/
Disallow: /testemail.php/
Disallow: /applysinglerule.php/
Disallow: /catalog/product_compare/
Disallow: /catalog/category/view/
Disallow: /catalog/product/view/
Disallow: /catalogsearch/
Disallow: /checkout/
Disallow: /control/
Disallow: /contacts/
Disallow: /customer/
Disallow: /customize/
Disallow: /newsletter/
Disallow: /poll/
Disallow: /review/
Disallow: /sendfriend/
Disallow: /tag/
Disallow: /wishlist/

# Files
User-agent: *
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /STATUS.txt
Disallow: /testemail.php
Disallow: /sk.php
Disallow: /rest_test.php
Disallow: /reindex.php
Disallow: /product_test_woimage.php
Disallow: /product_rest_example_backup.php
Disallow: /orders.php
Disallow: /reindexing.php
Disallow: /adminer.php
Disallow: /magmi-importer/
Disallow: /braintree-payments-1.0.0/
Disallow: /FME_quickrfq/


# Paths (no clean URLs)
User-agent: *
Disallow: /*.js$
Disallow: /*.css$
Disallow: /*.php$
Disallow: /*?p=*&
Disallow: /*?SID=
# robots.txt
User-agent: Baiduspider
Disallow: /
User-agent: SemrushBot
Disallow: /


User-agent: ahrefs
Disallow: /
User-agent: anderspinkbot
Disallow: /
User-agent: baidu
Disallow: /
User-agent: bash_cve
Disallow: /
User-agent: bidswitch
Disallow: /
User-agent: blexbot
Disallow: /
User-agent: bomborabot
Disallow: /
User-agent: changedetection
Disallow: /
User-agent: coccoc
Disallow: /
User-agent: cognitiveseo
Disallow: /
User-agent: companybook
Disallow: /
User-agent: contentkingapp
Disallow: /
User-agent: cron-job.org
Disallow: /
User-agent: deusu
Disallow: /
User-agent: domaincrawler
Disallow: /
User-agent: dotbot
Disallow: /
User-agent: epicbot
Disallow: /
User-agent: exabot
Disallow: /
User-agent: eyeotabot
Disallow: /
User-agent: growit
Disallow: /
User-agent: houzzbot
Disallow: /
User-agent: linkdex
Disallow: /
User-agent: linkpad
Disallow: /
User-agent: ltx71
Disallow: /
User-agent: mappy
Disallow: /
User-agent: mauibot
Disallow: /
User-agent: mediatoolki
Disallow: /
User-agent: tbot
Disallow: /
User-agent: megaindex
Disallow: /
User-agent: mj12
Disallow: /
User-agent: mtbot
Disallow: /
User-agent: musobot
Disallow: /
User-agent: ru_bot
Disallow: /
User-agent: safedns
Disallow: /
User-agent: scrapy
Disallow: /
User-agent: semrush
Disallow: /
User-agent: seokicks
Disallow: /
User-agent: seznambot
Disallow: /
User-agent: shoppimon
Disallow: /
User-agent: smtbot
Disallow: /
User-agent: sogou
Disallow: /
User-agent: spbot
Disallow: /
User-agent: tineye
Disallow: /
User-agent: tweetmeme
Disallow: /
User-agent: vebot
Disallow: /
User-agent: veoozbot
Disallow: /
User-agent: yandex
Disallow: /
User-agent: zoombot
Disallow: /