robots.txt

Same filename in other branches
  1. 9 core/tests/Drupal/Tests/Composer/Plugin/Scaffold/fixtures/drupal-assets-fixture/assets/robots.txt
  2. 9 core/assets/scaffold/files/robots.txt
  3. 9 robots.txt
  4. 8.9.x core/tests/Drupal/Tests/Composer/Plugin/Scaffold/fixtures/drupal-assets-fixture/assets/robots.txt
  5. 8.9.x core/assets/scaffold/files/robots.txt
  6. 8.9.x robots.txt
  7. 10 core/tests/Drupal/Tests/Composer/Plugin/Scaffold/fixtures/drupal-assets-fixture/assets/robots.txt
  8. 10 core/assets/scaffold/files/robots.txt
  9. 10 robots.txt
  10. 11.x core/tests/Drupal/Tests/Composer/Plugin/Scaffold/fixtures/drupal-assets-fixture/assets/robots.txt
  11. 11.x core/assets/scaffold/files/robots.txt
  12. 11.x robots.txt
#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: *
Crawl-delay: 10
# CSS, JS, Images
Allow: /misc/*.css$
Allow: /misc/*.css?
Allow: /misc/*.js$
Allow: /misc/*.js?
Allow: /misc/*.gif
Allow: /misc/*.jpg
Allow: /misc/*.jpeg
Allow: /misc/*.png
Allow: /modules/*.css$
Allow: /modules/*.css?
Allow: /modules/*.js$
Allow: /modules/*.js?
Allow: /modules/*.gif
Allow: /modules/*.jpg
Allow: /modules/*.jpeg
Allow: /modules/*.png
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /themes/*.css$
Allow: /themes/*.css?
Allow: /themes/*.js$
Allow: /themes/*.js?
Allow: /themes/*.gif
Allow: /themes/*.jpg
Allow: /themes/*.jpeg
Allow: /themes/*.png
# Directories
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /themes/
# Files
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /INSTALL.sqlite.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=filter/tips/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/
Disallow: /?q=user/logout/

File

./robots.txt

View source
  1. #
  2. # robots.txt
  3. #
  4. # This file is to prevent the crawling and indexing of certain parts
  5. # of your site by web crawlers and spiders run by sites like Yahoo!
  6. # and Google. By telling these "robots" where not to go on your site,
  7. # you save bandwidth and server resources.
  8. #
  9. # This file will be ignored unless it is at the root of your host:
  10. # Used: http://example.com/robots.txt
  11. # Ignored: http://example.com/site/robots.txt
  12. #
  13. # For more information about the robots.txt standard, see:
  14. # http://www.robotstxt.org/robotstxt.html
  15. User-agent: *
  16. Crawl-delay: 10
  17. # CSS, JS, Images
  18. Allow: /misc/*.css$
  19. Allow: /misc/*.css?
  20. Allow: /misc/*.js$
  21. Allow: /misc/*.js?
  22. Allow: /misc/*.gif
  23. Allow: /misc/*.jpg
  24. Allow: /misc/*.jpeg
  25. Allow: /misc/*.png
  26. Allow: /modules/*.css$
  27. Allow: /modules/*.css?
  28. Allow: /modules/*.js$
  29. Allow: /modules/*.js?
  30. Allow: /modules/*.gif
  31. Allow: /modules/*.jpg
  32. Allow: /modules/*.jpeg
  33. Allow: /modules/*.png
  34. Allow: /profiles/*.css$
  35. Allow: /profiles/*.css?
  36. Allow: /profiles/*.js$
  37. Allow: /profiles/*.js?
  38. Allow: /profiles/*.gif
  39. Allow: /profiles/*.jpg
  40. Allow: /profiles/*.jpeg
  41. Allow: /profiles/*.png
  42. Allow: /themes/*.css$
  43. Allow: /themes/*.css?
  44. Allow: /themes/*.js$
  45. Allow: /themes/*.js?
  46. Allow: /themes/*.gif
  47. Allow: /themes/*.jpg
  48. Allow: /themes/*.jpeg
  49. Allow: /themes/*.png
  50. # Directories
  51. Disallow: /includes/
  52. Disallow: /misc/
  53. Disallow: /modules/
  54. Disallow: /profiles/
  55. Disallow: /scripts/
  56. Disallow: /themes/
  57. # Files
  58. Disallow: /CHANGELOG.txt
  59. Disallow: /cron.php
  60. Disallow: /INSTALL.mysql.txt
  61. Disallow: /INSTALL.pgsql.txt
  62. Disallow: /INSTALL.sqlite.txt
  63. Disallow: /install.php
  64. Disallow: /INSTALL.txt
  65. Disallow: /LICENSE.txt
  66. Disallow: /MAINTAINERS.txt
  67. Disallow: /update.php
  68. Disallow: /UPGRADE.txt
  69. Disallow: /xmlrpc.php
  70. # Paths (clean URLs)
  71. Disallow: /admin/
  72. Disallow: /comment/reply/
  73. Disallow: /filter/tips/
  74. Disallow: /node/add/
  75. Disallow: /search/
  76. Disallow: /user/register/
  77. Disallow: /user/password/
  78. Disallow: /user/login/
  79. Disallow: /user/logout/
  80. # Paths (no clean URLs)
  81. Disallow: /?q=admin/
  82. Disallow: /?q=comment/reply/
  83. Disallow: /?q=filter/tips/
  84. Disallow: /?q=node/add/
  85. Disallow: /?q=search/
  86. Disallow: /?q=user/password/
  87. Disallow: /?q=user/register/
  88. Disallow: /?q=user/login/
  89. Disallow: /?q=user/logout/

Buggy or inaccurate documentation? Please file an issue. Need support? Need help programming? Connect with the Drupal community.