User-agent: *
Crawl-delay: 10

# Custom storeview pages to block
# brands pages
Disallow: /merken

# Block blog query strings, but not page=
# disallow all querystrings
Disallow: /blog?*
# allow specific values
Allow: /blog?page=*

# Directories
Disallow: /app/
Disallow: /bin/
Disallow: /dev/
Disallow: /lib/
Disallow: /phpserver/
Disallow: /pkginfo/
Disallow: /pub/
Disallow: /report/
Disallow: /setup/
Disallow: /update/
Disallow: /var/
Disallow: /vendor/
Disallow: /admin/
Disallow: /directory/

# Paths (clean URLs)
Disallow: /index.php/
Disallow: /catalog/
Disallow: /catalog/product_compare/
Disallow: /catalog/category/view/
Disallow: /catalog/product/view/
Disallow: /catalogsearch/
Disallow: /checkout/
Disallow: /control/
Disallow: /contacts/
Disallow: /customer/
Disallow: /customer/account/
Disallow: /customer/account/login/
Disallow: /customize/
Disallow: /newsletter/
Disallow: /onestepcheckout/
Disallow: /review/
Disallow: /sendfriend/
Disallow: /tag/
Disallow: /wishlist/

# Do not index the general technical directories and files on a server
Disallow: /cgi-bin/

# (Technical) Magento Files
Disallow: /api.php
Disallow: /composer.json
Disallow: /composer.lock
Disallow: /CONTRIBUTING.md
Disallow: /CONTRIBUTOR_LICENSE_AGREEMENT.html
Disallow: /COPYING.txt
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /get.php
Disallow: /Gruntfile.js
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /nginx.conf.sample
Disallow: /package.json
Disallow: /php.ini.sample
Disallow: /RELEASE_NOTES.txt

# Paths (no clean URLs)
# Disallow: /*.js$
# Disallow: /*.css$
Disallow: /*.php$

# Do not index session ID
Disallow: /*SID=

# Block all parameters
Disallow: /*?

# Disable sorting (Avoid duplicate content)
Disallow: /*?*product_list_mode=
Disallow: /*?*product_list_order=
Disallow: /*?*product_list_limit=
Disallow: /*?*product_list_dir=

# Block paging
# Disallow: /*?p=*&

# Block specific parameters
Disallow: */filter/

# Disallow: /*?bio=
# Disallow: /*&bio=
# Disallow: /*?brand=
# Disallow: /*&brand=
# Disallow: /*?cat=
# Disallow: /*&cat=
# Disallow: /*?color=
# Disallow: /*&color=
# Disallow: /*?price=
# Disallow: /*&price=
# Disallow: /*?producttype=
# Disallow: /*&producttype=
# Disallow: /*?vegan=
# Disallow: /*&vegan=
# Disallow: /*?size=
# Disallow: /*&size=
# Disallow: /*?material=
# Disallow: /*&material=
# Disallow: /*?dir*
# Disallow: /*&dir*
# Disallow: /*?dir=desc
# Disallow: /*&dir=desc
# Disallow: /*?dir=asc
# Disallow: /*&dir=asc
# Disallow: /*?limit=
# Disallow: /*&limit=
# Disallow: /*?mode*
# Disallow: /*&mode*

# CVS, SVN directory and dump files
Disallow: /*.CVS
Disallow: /*.git
Disallow: /*.Idea$
Disallow: /*.Sql$
Disallow: /*.Svn$
Disallow: /*.Tgz$
Disallow: /*.Zip$

# Image Crawler Setup - give Google instructions to crawl your images
User-agent: Googlebot-Image
Disallow:

User-agent: SemrushBot
Disallow: /
Sitemap: https://www.dutch-headshop.eu/sitemap_dhseu.xml